Skip to content

fix(lancedb): drop undeclared pandas runtime dependency (clean-env import was broken) #22620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wguo7 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(lancedb): drop undeclared pandas runtime dependency (clean-env import was broken) #22620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 20 additions & 17 deletions ...vector_stores/llama-index-vector-stores-lancedb/llama_index/vector_stores/lancedb/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import os
import logging
from typing import Any, List, Optional, Union
from typing import Any, Dict, List, Optional, Union
import warnings

import lancedb.rerankers
Expand Down Expand Up @@ -30,7 +30,6 @@
metadata_dict_to_node,
node_to_metadata_dict,
)
from pandas import DataFrame

import lancedb
import lancedb.remote.table # type: ignore
Expand Down Expand Up @@ -80,13 +79,17 @@ def _to_lance_filter(
return " AND ".join(filters)


def _to_llama_similarities(results: DataFrame) -> List[float]:
keys = results.keys()
def _to_llama_similarities(results: List[Dict[str, Any]]) -> List[float]:
if not results:
return []
keys = results[0].keys()
normalized_similarities: np.ndarray
if "score" in keys:
normalized_similarities = np.exp(results["score"] - np.max(results["score"]))
scores = np.array([item["score"] for item in results])
normalized_similarities = np.exp(scores - np.max(scores))
elif "_distance" in keys:
normalized_similarities = np.exp(-results["_distance"])
distances = np.array([item["_distance"] for item in results])
normalized_similarities = np.exp(-distances)
else:
normalized_similarities = np.linspace(1, 0, len(results))
return normalized_similarities.tolist()
Expand Down Expand Up @@ -457,28 +460,28 @@ def get_nodes(
if node_ids is not None:
where = f'id in ("' + '","'.join(node_ids) + '")'

results = self.table.search().where(where).to_pandas()
results = self.table.search().where(where).to_list()

nodes = []

for _, item in results.iterrows():
for item in results:
try:
node = metadata_dict_to_node(item.metadata)
node = metadata_dict_to_node(item["metadata"])
node.embedding = list(item[self.vector_column_name])
except Exception:
# deprecated legacy logic for backward compatibility
_logger.debug(
"Failed to parse Node metadata, fallback to legacy logic."
)
if item.metadata:
if item["metadata"]:
metadata, node_info, _relation = legacy_metadata_dict_to_node(
item.metadata, text_key=self.text_key
)
else:
metadata, node_info = {}, {}
node = TextNode(
text=item[self.text_key] or "",
id_=item.id,
id_=item["id"],
metadata=metadata,
start_char_idx=node_info.get("start"),
end_char_idx=node_info.get("end"),
Expand Down Expand Up @@ -557,31 +560,31 @@ def query(
if self.refine_factor is not None:
lance_query.refine_factor(self.refine_factor)

results = lance_query.to_pandas()
results = lance_query.to_list()

if len(results) == 0:
raise Warning("query results are empty..")

nodes = []

for _, item in results.iterrows():
for item in results:
try:
node = metadata_dict_to_node(item.metadata)
node = metadata_dict_to_node(item["metadata"])
node.embedding = list(item[self.vector_column_name])
except Exception:
# deprecated legacy logic for backward compatibility
_logger.debug(
"Failed to parse Node metadata, fallback to legacy logic."
)
if item.metadata:
if item["metadata"]:
metadata, node_info, _relation = legacy_metadata_dict_to_node(
item.metadata, text_key=self.text_key
)
else:
metadata, node_info = {}, {}
node = TextNode(
text=item[self.text_key] or "",
id_=item.id,
id_=item["id"],
metadata=metadata,
start_char_idx=node_info.get("start"),
end_char_idx=node_info.get("end"),
Expand All @@ -597,5 +600,5 @@ def query(
return VectorStoreQueryResult(
nodes=nodes,
similarities=_to_llama_similarities(results),
ids=results["id"].tolist(),
ids=[item["id"] for item in results],
)
2 changes: 1 addition & 1 deletion llama-index-integrations/vector_stores/llama-index-vector-stores-lancedb/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ dev = [

[project]
name = "llama-index-vector-stores-lancedb"
version = "0.6.0"
version = "0.6.1"
description = "llama-index vector_stores lancedb integration"
authors = [{name = "Your Name", email = "you@example.com"}]
requires-python = ">=3.10,<4.0"
Expand Down
Loading