diff --git a/docs/migration_guides/v0-43_to_v0-44.md b/docs/migration_guides/v0-43_to_v0-44.md index 08ec4601..f6181075 100644 --- a/docs/migration_guides/v0-43_to_v0-44.md +++ b/docs/migration_guides/v0-43_to_v0-44.md @@ -485,8 +485,12 @@ error. ```python # Before -- silently produced a plain `bytes` array on v0.44 and v0.45 to_ome_zarr( - "image.ome.zarr", multiscales, version="0.5", - filters=[...], serializer=..., compressors=[...], + "image.ome.zarr", + multiscales, + version="0.5", + filters=[...], + serializer=..., + compressors=[...], ) # After @@ -500,13 +504,15 @@ for dataset, level in zip(multiscales.metadata.datasets, multiscales.images): "image.ome.zarr", name=dataset.path, shape=skeleton.shape, - chunks=skeleton.chunks, # keep the layout ngff-zarr chose ... - shards=skeleton.shards, # ... including its sharding + chunks=skeleton.chunks, # keep the layout ngff-zarr chose ... + shards=skeleton.shards, # ... including its sharding dtype=skeleton.dtype, - filters=[...], serializer=..., compressors=[...], + filters=[...], + serializer=..., + compressors=[...], fill_value=0, dimension_names=list(level.dims), - overwrite=True, # replace the skeleton, codec chain and all + overwrite=True, # replace the skeleton, codec chain and all ) # One dask chunk per stored write unit: every chunk (or shard) has a # single writer and needs no lock, and no level is held in memory whole. diff --git a/docs/python.md b/docs/python.md index b2f10cc4..f31d0ca6 100644 --- a/docs/python.md +++ b/docs/python.md @@ -443,6 +443,17 @@ the fill stays safe across processes and cluster ranks. The handle from writer can fill the arrays as well; `to_ome_zarr` is not involved. Chunks that are never written read back as the fill value. +`open_array` also opens an array in a remote store (http(s), S3, GCS, Azure), +through the same engine `from_ome_zarr` reads one with, and takes the same +`storage_options`. A remote handle is read-only -- filling a store region by +region needs a local directory -- but it reports the same `shape`, `dtype` and +`chunks`, and serves the same windowed reads: + +```python +level0 = nz.open_array("s3://bucket/volume.ome.zarr", "scale0/image") +window = level0[0:1, 0:64, :, :] +``` + ### Append coarser levels to an existing store Once the finest level is on disk, `start_level` derives the next levels from diff --git a/py/examples/itk_elastix_transform_resample_s3.ipynb b/py/examples/itk_elastix_transform_resample_s3.ipynb index 830a9e9f..0bb800cd 100644 --- a/py/examples/itk_elastix_transform_resample_s3.ipynb +++ b/py/examples/itk_elastix_transform_resample_s3.ipynb @@ -46,7 +46,14 @@ "cell_type": "code", "execution_count": 1, "id": "acae54e37e7d407bbb7b55eff062a284", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:24:57.496022Z", + "iopub.status.busy": "2026-09-01T19:24:57.495950Z", + "iopub.status.idle": "2026-09-01T19:24:57.921963Z", + "shell.execute_reply": "2026-09-01T19:24:57.921613Z" + } + }, "outputs": [], "source": [ "import logging\n", @@ -104,7 +111,14 @@ "cell_type": "code", "execution_count": 2, "id": "8dd0d8092fe74a7c96281538738b07e2", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:24:57.923699Z", + "iopub.status.busy": "2026-09-01T19:24:57.923494Z", + "iopub.status.idle": "2026-09-01T19:25:10.418404Z", + "shell.execute_reply": "2026-09-01T19:25:10.417048Z" + } + }, "outputs": [ { "name": "stdout", @@ -131,7 +145,9 @@ "\n", "dask.config.set(scheduler=\"threads\", num_workers=16)\n", "\n", - "storage_options = {\"anon\": True}\n", + "# The AIND bucket is public and lives in us-west-2. The region is explicit\n", + "# because the S3 backend does not discover it from the URI.\n", + "storage_options = {\"anon\": True, \"region\": \"us-west-2\"}\n", "fixed_multiscales = from_ome_zarr(FIXED_URI, storage_options=storage_options)\n", "moving_multiscales = from_ome_zarr(MOVING_URI, storage_options=storage_options)\n", "\n", @@ -194,13 +210,20 @@ "cell_type": "code", "execution_count": 3, "id": "8edb47106e1a46a883d545849b8ab81b", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:10.420792Z", + "iopub.status.busy": "2026-09-01T19:25:10.420700Z", + "iopub.status.idle": "2026-09-01T19:25:22.843132Z", + "shell.execute_reply": "2026-09-01T19:25:22.842680Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Registration at level 3: 0.9 s\n" + "Registration at level 3: 1.0 s\n" ] } ], @@ -237,7 +260,14 @@ "cell_type": "code", "execution_count": 4, "id": "10185d26023b46108eb7d9f57d49d2b3", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:22.844931Z", + "iopub.status.busy": "2026-09-01T19:25:22.844839Z", + "iopub.status.idle": "2026-09-01T19:25:23.221234Z", + "shell.execute_reply": "2026-09-01T19:25:23.220677Z" + } + }, "outputs": [ { "data": { @@ -306,7 +336,14 @@ "cell_type": "code", "execution_count": 5, "id": "7623eae2785240b9bd12b16a66d81610", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:23.222863Z", + "iopub.status.busy": "2026-09-01T19:25:23.222764Z", + "iopub.status.idle": "2026-09-01T19:25:30.367860Z", + "shell.execute_reply": "2026-09-01T19:25:30.367397Z" + } + }, "outputs": [ { "name": "stdout", @@ -319,7 +356,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Wrote registered_822174_on_773889_level2.ome.zarr in 15 s (1.27 Mvoxel/s end to end)\n" + "Wrote registered_822174_on_773889_level2.ome.zarr in 7 s (2.77 Mvoxel/s end to end)\n" ] } ], @@ -347,7 +384,14 @@ "cell_type": "code", "execution_count": 6, "id": "7cdc8c89c7104fffa095e18ddfef8986", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:30.368913Z", + "iopub.status.busy": "2026-09-01T19:25:30.368839Z", + "iopub.status.idle": "2026-09-01T19:25:40.140009Z", + "shell.execute_reply": "2026-09-01T19:25:40.138742Z" + } + }, "outputs": [ { "data": { @@ -408,7 +452,14 @@ "cell_type": "code", "execution_count": 7, "id": "938c804e27f84196a10c8828c723f798", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:40.143725Z", + "iopub.status.busy": "2026-09-01T19:25:40.143636Z", + "iopub.status.idle": "2026-09-01T19:25:51.700051Z", + "shell.execute_reply": "2026-09-01T19:25:51.699427Z" + } + }, "outputs": [ { "name": "stdout", @@ -507,7 +558,14 @@ "cell_type": "code", "execution_count": 8, "id": "504fb2a444614c0babb325280ed9130a", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:51.701128Z", + "iopub.status.busy": "2026-09-01T19:25:51.701049Z", + "iopub.status.idle": "2026-09-01T19:25:51.877916Z", + "shell.execute_reply": "2026-09-01T19:25:51.877262Z" + } + }, "outputs": [ { "data": { @@ -542,7 +600,14 @@ "cell_type": "code", "execution_count": 9, "id": "59bbdb311c014d738909a11f9e486628", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:51.879675Z", + "iopub.status.busy": "2026-09-01T19:25:51.879586Z", + "iopub.status.idle": "2026-09-01T19:25:51.993036Z", + "shell.execute_reply": "2026-09-01T19:25:51.992556Z" + } + }, "outputs": [ { "data": { @@ -576,7 +641,14 @@ "cell_type": "code", "execution_count": 10, "id": "b43b363d81ae4b689946ece5c682cd59", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:51.994330Z", + "iopub.status.busy": "2026-09-01T19:25:51.994228Z", + "iopub.status.idle": "2026-09-01T19:25:52.258503Z", + "shell.execute_reply": "2026-09-01T19:25:52.257221Z" + } + }, "outputs": [ { "data": { @@ -641,15 +713,22 @@ "cell_type": "code", "execution_count": 11, "id": "c3933fab20d04ec698c2621248eb3be0", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:25:52.260535Z", + "iopub.status.busy": "2026-09-01T19:25:52.260446Z", + "iopub.status.idle": "2026-09-01T19:26:01.007678Z", + "shell.execute_reply": "2026-09-01T19:26:01.007245Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "level block in flight voxels (M) blocks build (s) seconds Mvoxel/s peak RSS (GB)\n", - " 3 222 8 2.4 1 0.0 15.4 0.2 1.57\n", - " 2 256 8 18.9 4 0.01 24.1 0.8 1.73\n" + " 3 222 8 2.4 1 0.0 3.7 0.6 1.81\n", + " 2 256 8 18.9 4 0.01 5.0 3.8 1.92\n" ] } ], @@ -741,19 +820,26 @@ "cell_type": "code", "execution_count": 12, "id": "06873177ff2d10fdd2e9ac23bd3ab778", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-01T19:26:01.008656Z", + "iopub.status.busy": "2026-09-01T19:26:01.008556Z", + "iopub.status.idle": "2026-09-01T19:26:25.656477Z", + "shell.execute_reply": "2026-09-01T19:26:25.656018Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "level block in flight voxels (M) blocks build (s) seconds Mvoxel/s peak RSS (GB)\n", - " 2 128 8 18.9 12 0.02 21.8 0.9 1.74\n", - " 2 128 16 18.9 12 0.02 13.7 1.4 1.72\n", - " 2 192 16 18.9 6 0.01 14.3 1.3 1.86\n", - " 2 192 24 18.9 6 0.01 21.9 0.9 1.86\n", - " 2 256 8 18.9 4 0.01 15.3 1.2 1.85\n", - " 2 256 16 18.9 4 0.01 15.1 1.3 1.84\n" + " 2 128 8 18.9 12 0.02 3.2 5.9 1.96\n", + " 2 128 16 18.9 12 0.02 3.9 4.9 1.96\n", + " 2 192 16 18.9 6 0.01 4.3 4.3 1.99\n", + " 2 192 24 18.9 6 0.01 3.3 5.7 2.05\n", + " 2 256 8 18.9 4 0.01 5.7 3.3 2.05\n", + " 2 256 16 18.9 4 0.01 4.1 4.6 2.02\n" ] } ], @@ -864,7 +950,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.13" + "version": "3.11.16" } }, "nbformat": 4, diff --git a/py/ngff_zarr/_remote_reader.py b/py/ngff_zarr/_remote_reader.py index 114aea5e..730cbb54 100644 --- a/py/ngff_zarr/_remote_reader.py +++ b/py/ngff_zarr/_remote_reader.py @@ -26,8 +26,11 @@ import asyncio import importlib.util +import os import posixpath import threading +import urllib.error +import urllib.request import warnings from urllib.parse import urlsplit @@ -182,12 +185,63 @@ def _translate_storage_options( return translated, passthrough +#: Regions already resolved, keyed by bucket. A bucket does not move. +_BUCKET_REGIONS: dict[str, str] = {} + +_REGION_TIMEOUT = 10.0 + + +def _resolve_bucket_region(bucket: str) -> str | None: + """The region AWS reports for *bucket*, or ``None`` if it does not say. + + obstore sends the request to ``us-east-1`` when no region is configured, + and S3 answers for a bucket held elsewhere with a 301 that carries no + ``Location``, so the read fails instead of being redirected. AWS names + the region in a header on the bucket itself, and answers unsigned. + """ + if bucket in _BUCKET_REGIONS: + return _BUCKET_REGIONS[bucket] + request = urllib.request.Request( + f"https://{bucket}.s3.amazonaws.com/", method="HEAD" + ) + try: + with urllib.request.urlopen(request, timeout=_REGION_TIMEOUT) as response: + region = response.headers.get("x-amz-bucket-region") + except urllib.error.HTTPError as error: + # A bucket that exists still names its region when it refuses the read. + region = error.headers.get("x-amz-bucket-region") + except OSError: + return None + if region: + _BUCKET_REGIONS[bucket] = region + return region + + +def _fill_bucket_region(url: str, translated: dict, passthrough: dict) -> None: + """Add the bucket's region to *translated* when nothing else supplies one.""" + if "region" in translated or "region" in passthrough: + return + if "endpoint" in translated or "endpoint" in passthrough: + # An endpoint names an S3-compatible service, not AWS. + return + if os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION"): + return + bucket = urlsplit(url).netloc + if not bucket: + return + region = _resolve_bucket_region(bucket) + if region: + translated["region"] = region + + def _build_obstore(url: str, storage_options: dict | None): """Construct the obstore store for *url*, translating *storage_options*.""" from obstore.store import from_url scheme = urlsplit(url).scheme.lower() translated, passthrough = _translate_storage_options(scheme, storage_options) + if scheme in ("s3", "s3a"): + _fill_bucket_region(url, translated, passthrough) def _construct(options: dict): if scheme in ("http", "https"): @@ -293,7 +347,13 @@ def _remote_array_to_dask(arr) -> dask.array.Array: class RemoteZarrArray: - """Read-only handle for an array node within a remote store.""" + """Read-only handle for an array node within a remote store. + + The read half of :class:`~ngff_zarr._zarrista_utils.LocalZarrArray`: the + node's ``shape``, ``dtype``, ``chunks`` and ``attrs``, numpy style ``[]`` + reads, and :meth:`to_dask` for lazy pixel access. Writes are refused, a + remote store being read-only here. + """ def __init__(self, store: RemoteZarrStore, component: str, arr): self._store = store @@ -302,6 +362,30 @@ def __init__(self, store: RemoteZarrStore, component: str, arr): self.shape = tuple(int(s) for s in arr.shape) self.ndim = arr.ndim self.attrs = AttrsDict(dict(arr.attrs)) + self._adapter = None + + def _array(self): + """The synchronous adapter over the async node, made once.""" + if self._adapter is None: + self._adapter = _async_adapter(self._arr) + return self._adapter + + @property + def dtype(self) -> np.dtype: + return self._array().dtype + + @property + def chunks(self) -> tuple[int, ...]: + """The stored chunk grid, the inner chunk shape of a sharded array.""" + if self._arr.is_sharded: + return tuple(int(c) for c in self._arr.subchunk_shape) + return tuple(int(c) for c in self._arr.chunk_shape([0] * self._arr.ndim)) + + def __getitem__(self, selection) -> np.ndarray: + return self._array()[selection] + + def __setitem__(self, selection, value) -> None: + raise TypeError("Remote zarr stores are read-only") def to_dask(self) -> dask.array.Array: return _remote_array_to_dask(self._arr) diff --git a/py/ngff_zarr/_zarrista_utils.py b/py/ngff_zarr/_zarrista_utils.py index 688bc85a..f09cb9be 100644 --- a/py/ngff_zarr/_zarrista_utils.py +++ b/py/ngff_zarr/_zarrista_utils.py @@ -1070,14 +1070,50 @@ def open_local_node(store, node_path=None, *, zarr_format: int): return LocalZarrGroup(root, path, zarr_format, doc) -def open_array(store, path: str | None = None) -> LocalZarrArray: - """Open the array at ``path`` within the local directory store ``store``. +def _remote_handle(store, storage_options: dict | None): + """A handle on ``store``'s remote backend, or ``None`` when it has none. + + The reader's own rule (``from_ome_zarr``): a URL string, or a handle + already built around one. A zarr-python ``FsspecStore`` wrapping a URL is + neither -- its ``str()`` is a repr rather than the URL obstore would be + given -- and the reader does not route one here either, so it keeps the + local path it has today. + """ + from ._remote_reader import RemoteZarrStore, remote_read_available + from .from_ngff_zarr import REMOTE_URL_SCHEMES, _remote_backend_import_error + + if isinstance(store, RemoteZarrStore): + return store + if isinstance(store, str) and store.startswith(REMOTE_URL_SCHEMES): + if not remote_read_available(): + raise _remote_backend_import_error(store, None) + return RemoteZarrStore(store, storage_options=storage_options) + return None + + +def open_array(store, path: str | None = None, storage_options: dict | None = None): + """Open the array at ``path`` within ``store``. The handle reads and writes regions through zarrista and needs no other Zarr library: ``to_ome_zarr(..., metadata_only=True)`` creates the arrays of a store, and this opens one of them for a producer to fill. The node is looked up as zarr format 3 first, then format 2. + + A URL (http(s), S3, GCS, Azure) is opened through the same async engine + ``from_ome_zarr`` reads it with, and comes back read-only: writing a region + needs a local directory store. ``storage_options`` is passed to that + engine, fsspec-style names included. """ + remote = _remote_handle(store, storage_options) + if remote is not None: + from ._remote_reader import RemoteZarrArray, open_remote_node + + node = open_remote_node(remote, path) + if node is None: + raise FileNotFoundError(f"No array at '{path}' in '{remote}'.") + if not isinstance(node, RemoteZarrArray): + raise ValueError(f"'{path}' in '{remote}' is a group, not an array.") + return node for zarr_format in (3, 2): node = open_local_node(store, path, zarr_format=zarr_format) if node is None: diff --git a/py/test/test_remote_zarrista.py b/py/test/test_remote_zarrista.py index 23c0c9fc..e7b1bf7d 100644 --- a/py/test/test_remote_zarrista.py +++ b/py/test/test_remote_zarrista.py @@ -257,3 +257,257 @@ def test_remote_hcs_plate_read(tmp_path): assert isinstance(image.images[0].data, dask.array.Array) # Pixel data resolves through the shared obstore client. np.asarray(image.images[0].data) + + +def test_open_array_reads_regions_from_a_remote_store(tmp_path): + """``open_array`` serves a remote node the way it serves a local one. + + The handle is what a consumer reads regions through, and it was local-only: + a caller holding a URL had to fall back to another Zarr library for the one + thing this package could otherwise do end to end. + """ + from ngff_zarr import from_ome_zarr, open_array + + data = _write_ramp_store(tmp_path / "image.zarr", "0.5") + + open_group_patch, from_zarr_patch = _forbid_zarr_python() + with _serve(tmp_path) as base_url, open_group_patch, from_zarr_patch: + store = f"{base_url}/image.zarr" + path = from_ome_zarr(store).metadata.datasets[0].path + array = open_array(store, path) + + assert array.shape == data.shape + assert array.dtype == data.dtype + assert array.chunks == (8, 8) + # A window, and the whole array: the same values the store holds. + assert np.array_equal(np.asarray(array[4:12, 2:6]), data[4:12, 2:6]) + assert np.array_equal(np.asarray(array[...]), data) + + +def test_a_remote_array_refuses_a_region_write(tmp_path): + """Read-only, and said so: the write path needs a local directory store, + and a silent no-op would leave a producer believing its region landed.""" + from ngff_zarr import from_ome_zarr, open_array + + _write_ramp_store(tmp_path / "image.zarr", "0.5") + with _serve(tmp_path) as base_url: + store = f"{base_url}/image.zarr" + array = open_array(store, from_ome_zarr(store).metadata.datasets[0].path) + with pytest.raises(TypeError, match="read-only"): + array[0:2, 0:2] = np.zeros((2, 2), dtype="uint16") + + +def test_open_array_on_a_remote_group_is_refused(tmp_path): + from ngff_zarr import open_array + + _write_ramp_store(tmp_path / "image.zarr", "0.5") + with _serve(tmp_path) as base_url: + with pytest.raises(ValueError, match="is a group, not an array"): + open_array(f"{base_url}/image.zarr") + + +def test_a_store_object_is_not_opened_as_a_url(): + """A wrapper around a URL is remote, but it is not a URL. + + ``_is_remote_store`` calls a zarr-python ``FsspecStore`` remote by its + filesystem protocol, while its ``str()`` is a repr rather than the URL + obstore would be handed. ``from_ome_zarr`` does not route one through the + remote engine either, so neither does this: it keeps the path it has today. + """ + from ngff_zarr._zarrista_utils import _remote_handle + from ngff_zarr.from_ngff_zarr import _is_remote_store + + class _FsspecStoreLike: + class fs: + protocol = "s3" + + path = "bucket/image.zarr" + + store = _FsspecStoreLike() + assert _is_remote_store(store) + assert _remote_handle(store, None) is None + + +def test_a_url_without_the_remote_extra_names_the_extra(monkeypatch): + """The install hint ``from_ome_zarr`` gives, given here too: reaching + obstore's own constructor instead raises a bare ModuleNotFoundError.""" + import ngff_zarr._remote_reader as remote_reader + from ngff_zarr._zarrista_utils import _remote_handle + + monkeypatch.setattr(remote_reader, "remote_read_available", lambda: False) + with pytest.raises(ImportError, match=r"ngff-zarr\[remote\]"): + _remote_handle("https://example.org/image.zarr", None) + + +def test_open_array_reads_a_real_s3_store(): + """A windowed read from S3 itself, not from the localhost stand-in. + + The harness above serves HTTP; the S3 path differs in the store obstore + builds and in what signature and region settings reach. The bucket is + public, so no credentials are involved: opt in with + ``NGFF_ZARR_NETWORK_TESTS=1``. + """ + import obstore.exceptions as obstore_exceptions + from ngff_zarr import open_array + + if not os.environ.get("NGFF_ZARR_NETWORK_TESTS"): + pytest.skip("live remote read is opt-in; set NGFF_ZARR_NETWORK_TESTS=1 to run") + + url = ( + "s3://aind-open-data/exaSPIM_773889_2026-04-10_15-04-57_processed" + "_2026-07-08_23-41-18/fusion2halves/SPIM.ome.zarr/3" + ) + # obstore reaches us-east-1 without a region, and S3 answers for a bucket + # held elsewhere with a redirect it does not follow. + options = {"anon": True, "region": "us-west-2"} + # The bucket going away, changing its policy or the network failing is not + # a regression here; a decoding or indexing error is one, and raises. + unavailable = ( + OSError, + obstore_exceptions.GenericError, + obstore_exceptions.NotFoundError, + obstore_exceptions.PermissionDeniedError, + obstore_exceptions.UnauthenticatedError, + ) + try: + array = open_array(url, storage_options=options) + window = np.asarray(array[:, :, :4, :8, :8]) + except unavailable as exc: + pytest.skip(f"remote store unavailable: {exc!r}") + + assert array.dtype == np.dtype("uint16") + assert array.chunks == (1, 1, 256, 256, 256) + assert window.shape == (1, 1, 4, 8, 8) + assert window.any() + + +@pytest.fixture +def forget_bucket_regions(): + from ngff_zarr._remote_reader import _BUCKET_REGIONS + + _BUCKET_REGIONS.clear() + yield + _BUCKET_REGIONS.clear() + + +def _record_head_requests(monkeypatch, region="us-west-2"): + """Answer the region probe without leaving the machine; log what it asked.""" + import urllib.request + + asked = [] + + class _Response: + def __init__(self): + self.headers = {"x-amz-bucket-region": region} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def _urlopen(request, timeout=None): + asked.append(request.full_url) + return _Response() + + monkeypatch.setattr(urllib.request, "urlopen", _urlopen) + return asked + + +def test_a_bucket_region_is_resolved_when_nothing_supplies_one( + monkeypatch, forget_bucket_regions +): + """obstore reaches us-east-1 without a region, and S3 answers for a bucket + held elsewhere with a redirect carrying no Location, which it does not + follow. AWS names the region on the bucket, unsigned, so ask it.""" + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + asked = _record_head_requests(monkeypatch) + + handle = RemoteZarrStore("s3://bucket/data.zarr", storage_options={"anon": True}) + + assert dict(handle._store.config)["region"] == "us-west-2" + assert asked == ["https://bucket.s3.amazonaws.com/"] + + +def test_a_resolved_region_is_asked_for_once(monkeypatch, forget_bucket_regions): + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + asked = _record_head_requests(monkeypatch) + + RemoteZarrStore("s3://bucket/first.zarr", storage_options={"anon": True}) + RemoteZarrStore("s3://bucket/second.zarr", storage_options={"anon": True}) + + assert len(asked) == 1 + + +@pytest.mark.parametrize( + ("storage_options", "environment"), + [ + ({"anon": True, "client_kwargs": {"region_name": "eu-west-1"}}, {}), + ({"anon": True, "region": "eu-west-1"}, {}), + ({"anon": True, "endpoint_url": "https://minio.example.com"}, {}), + ({"anon": True}, {"AWS_REGION": "eu-west-1"}), + ({"anon": True}, {"AWS_DEFAULT_REGION": "eu-west-1"}), + ], + ids=["client_kwargs", "region", "endpoint", "AWS_REGION", "AWS_DEFAULT_REGION"], +) +def test_the_region_is_not_asked_for_when_something_supplies_one( + monkeypatch, forget_bucket_regions, storage_options, environment +): + """A region already given, or an endpoint naming a service that is not AWS, + leaves nothing to resolve.""" + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + asked = _record_head_requests(monkeypatch) + + RemoteZarrStore("s3://bucket/data.zarr", storage_options=storage_options) + + assert asked == [] + + +def test_a_bucket_that_names_no_region_is_left_alone( + monkeypatch, forget_bucket_regions +): + """Nothing to add, and the store is built as before: obstore's own error is + what the caller sees, rather than one invented here.""" + import urllib.request + + monkeypatch.delenv("AWS_REGION", raising=False) + monkeypatch.delenv("AWS_DEFAULT_REGION", raising=False) + + def _urlopen(request, timeout=None): + raise OSError("no route to host") + + monkeypatch.setattr(urllib.request, "urlopen", _urlopen) + + handle = RemoteZarrStore("s3://bucket/data.zarr", storage_options={"anon": True}) + + assert "region" not in dict(handle._store.config) + + +def test_a_real_s3_store_reads_without_a_region(): + """The failure this resolves, end to end: a public bucket in us-west-2, + read with the options a caller would pass. Opt in with + ``NGFF_ZARR_NETWORK_TESTS=1``.""" + from ngff_zarr import from_ome_zarr + + if not os.environ.get("NGFF_ZARR_NETWORK_TESTS"): + pytest.skip("live remote read is opt-in; set NGFF_ZARR_NETWORK_TESTS=1 to run") + + url = ( + "s3://aind-open-data/exaSPIM_773889_2026-04-10_15-04-57_processed" + "_2026-07-08_23-41-18/fusion2halves/SPIM.ome.zarr" + ) + try: + multiscales = from_ome_zarr(url, storage_options={"anon": True}) + except OSError as exc: + pytest.skip(f"remote store unavailable: {exc!r}") + except Exception as exc: + if type(exc).__module__.startswith(("zarrista", "obstore")): + pytest.skip(f"remote store unavailable: {exc!r}") + raise + + assert multiscales.images[0].data.shape[-1] > 0