diff --git a/docs/src/distributed-indexing.md b/docs/src/distributed-indexing.md index ba54e9bd..6354539c 100755 --- a/docs/src/distributed-indexing.md +++ b/docs/src/distributed-indexing.md @@ -270,10 +270,7 @@ dataset = lance.dataset("path/to/dataset") # Build distributed index updated_dataset = lr.create_scalar_index( - uri=dataset.uri, - column="text", - index_type="INVERTED", - num_workers=4 + uri=dataset.uri, column="text", index_type="INVERTED", num_workers=4 ) # Verify index creation @@ -282,8 +279,7 @@ print(f"Index list: {indices}") # Use index for search results = updated_dataset.scanner( - full_text_query="search term", - columns=["id", "text"] + full_text_query="search term", columns=["id", "text"] ).to_table() print(f"Search results: {results}") ``` @@ -304,7 +300,9 @@ updated_dataset = lr.create_scalar_index( # Example queries updated_dataset.scanner(filter="id = 100", columns=["id", "text"]).to_table() -updated_dataset.scanner(filter="id >= 200 AND id < 800", columns=["id", "text"]).to_table() +updated_dataset.scanner( + filter="id >= 200 AND id < 800", columns=["id", "text"] +).to_table() ``` ### Vector Index (IVF_PQ / IVF_RQ / IVF_SQ / IVF_FLAT) @@ -321,7 +319,7 @@ updated_dataset = lr.create_index( num_partitions=256, num_sub_vectors=16, sample_rate=64, - metric="l2" + metric="l2", ) # Build a distributed IVF_SQ index @@ -399,11 +397,11 @@ print(plan) ```python updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - num_workers=4, - ray_remote_args={"num_cpus": 2, "resources": {"custom_resource": 1}} + uri="path/to/dataset", + column="text", + index_type="INVERTED", + num_workers=4, + ray_remote_args={"num_cpus": 2, "resources": {"custom_resource": 1}}, ) ``` @@ -444,21 +442,21 @@ The current global Pool integration is limited to `vector_search()`. The same p ```python # Create index with custom name updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", + num_workers=4, ) # Try to create another index with the same name (will replace by default) updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", # Same name as before - replace=True, # Explicitly allow replacement (default behavior) - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", # Same name as before + replace=True, # Explicitly allow replacement (default behavior) + num_workers=4, ) # Prevent index replacement @@ -466,12 +464,12 @@ import lance_ray as lr try: updated_dataset = lr.create_scalar_index( - uri="path/to/dataset", - column="text", - index_type="INVERTED", - name="my_text_index", # Same name as existing index - replace=False, # Prevent replacement - num_workers=4 + uri="path/to/dataset", + column="text", + index_type="INVERTED", + name="my_text_index", # Same name as existing index + replace=False, # Prevent replacement + num_workers=4, ) except ValueError as e: print(f"Index creation failed: {e}") diff --git a/docs/src/examples.md b/docs/src/examples.md index 39164a35..47f2331f 100644 --- a/docs/src/examples.md +++ b/docs/src/examples.md @@ -37,9 +37,7 @@ print(f"Filtered count: {filtered_ds.count()}") # Read with column selection and filtering ds_filtered = read_lance( - "sample_dataset.lance", - columns=["user_id", "name", "score"], - filter="score > 75.0" + "sample_dataset.lance", columns=["user_id", "name", "score"], filter="score > 75.0" ) print(f"Schema: {ds_filtered.schema()}") ``` @@ -51,16 +49,14 @@ print(f"Schema: {ds_filtered.schema()}") from lance_ray import add_columns import pyarrow as pa + def add_computed_column(batch: pa.RecordBatch) -> pa.RecordBatch: df = batch.to_pandas() - df['computed'] = df['value'] * 2 + df['id'] + df["computed"] = df["value"] * 2 + df["id"] return pa.RecordBatch.from_pandas(df[["computed"]]) -add_columns( - uri="sample_dataset.lance", - transform=add_computed_column, - concurrency=4 -) + +add_columns(uri="sample_dataset.lance", transform=add_computed_column, concurrency=4) ``` ## Using Namespace @@ -110,7 +106,7 @@ from lance_ray import read_lance, write_lance # Initialize Ray ray.init() -# Connect to AWS Glue catalog +# Connect to AWS Glue catalog # using the default account and region in the current AWS environment namespace = ln.connect("glue", {}) @@ -119,10 +115,10 @@ data = ray.data.range(1000).map(lambda row: {"id": row["id"], "value": row["id"] # Write to Lance format using metadata catalog write_lance( - data, - uri="s3://my-bucket/my-table", - namespace=namespace, - table_id=["default", "my_table"] + data, + uri="s3://my-bucket/my-table", + namespace=namespace, + table_id=["default", "my_table"], ) # Read Lance dataset back using metadata catalog diff --git a/lance_ray/datasource.py b/lance_ray/datasource.py index 436aad89..4c293b63 100644 --- a/lance_ray/datasource.py +++ b/lance_ray/datasource.py @@ -201,18 +201,7 @@ def get_read_tasks(self, parallelism: int, **kwargs) -> list[ReadTask]: ) read_task = ReadTask( - lambda fids=fragment_ids, - uri=dataset_uri, - version=dataset_version, - storage_options=dataset_storage_options, - manifest=serialized_manifest, - ns_impl=namespace_impl, - ns_props=namespace_properties, - tbl_id=table_id, - base_params=base_store_params, - scanner_options=self._scanner_options, - retry_params=self._retry_params, - with_metadata=self._with_metadata: ( + lambda fids=fragment_ids, uri=dataset_uri, version=dataset_version, storage_options=dataset_storage_options, manifest=serialized_manifest, ns_impl=namespace_impl, ns_props=namespace_properties, tbl_id=table_id, base_params=base_store_params, scanner_options=self._scanner_options, retry_params=self._retry_params, with_metadata=self._with_metadata: ( _read_fragments_with_retry( fids, uri, diff --git a/pyproject.toml b/pyproject.toml index f11ec3b2..9e42e9fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ dependencies = [ "ray[data]>=2.41.0", - "pylance>=9.0.0", + "pylance>=10.0.0b5", "lance-namespace", "packaging", "pyarrow>=17.0.0", diff --git a/uv.lock b/uv.lock index 97f9fac6..8625f408 100644 --- a/uv.lock +++ b/uv.lock @@ -387,7 +387,7 @@ requires-dist = [ { name = "more-itertools", marker = "python_full_version < '3.12'", specifier = ">=2.6.0" }, { name = "packaging" }, { name = "pyarrow", specifier = ">=17.0.0" }, - { name = "pylance", specifier = ">=9.0.0" }, + { name = "pylance", specifier = ">=10.0.0b5" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, @@ -1024,8 +1024,8 @@ wheels = [ [[package]] name = "pylance" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } +version = "10.0.0b5" +source = { registry = "https://pypi.fury.io/lance-format" } dependencies = [ { name = "lance-namespace" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -1033,12 +1033,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" }, - { url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" }, - { url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" }, - { url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" }, - { url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" }, + { url = "https://pypi.fury.io/lance-format/-/ver_sbbYE/pylance-10.0.0b5-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a60c172ba0daeced51658caa76dff6ba525309b613328b8411a35d700ed50d77" }, + { url = "https://pypi.fury.io/lance-format/-/ver_edwAr/pylance-10.0.0b5-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88b0fe5e4a331a7417f861d705e7e47b13ed28004d860c2aa90ca5c440b413b8" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1lVxD3/pylance-10.0.0b5-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6da6c4056254699ac0ae5f05f33a2cd172b5e01b6845b0f58026f319d117d3" }, + { url = "https://pypi.fury.io/lance-format/-/ver_jonEs/pylance-10.0.0b5-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:90fc3ec0e61cdb55e9db311a0a9ab18f7be40fe2baab31d671852249fcbbf5d6" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1Tl244/pylance-10.0.0b5-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:37fb9d05a9c4825b814959cdb01d2cf3c8e6f0e913a45207f32deb6624a373f2" }, + { url = "https://pypi.fury.io/lance-format/-/ver_MeqKq/pylance-10.0.0b5-cp310-abi3-win_amd64.whl", hash = "sha256:d02eed33d7b8972b13db70849522db303bdd527e5ded8b2dc064700496554b71" }, ] [[package]]