From 813c19d8d938b2a4d9bcea7d5639039bc4c47cdd Mon Sep 17 00:00:00 2001 From: Art Pelling Date: Fri, 7 Aug 2026 15:33:10 +0200 Subject: [PATCH] first shot --- docs/contributor-guide/adding_dataset.rst | 5 +++ scripts/update_sofacoustics_hashes.py | 51 ----------------------- src/irdl/base.py | 50 ++++++++++++++++++---- src/irdl/downloader.py | 14 +++++++ src/irdl/registry/direct_sofa_hashes.json | 1 + tests/test_base.py | 44 +++++++++++++++++++ uv.lock | 2 +- 7 files changed, 107 insertions(+), 60 deletions(-) delete mode 100644 scripts/update_sofacoustics_hashes.py create mode 100644 src/irdl/registry/direct_sofa_hashes.json diff --git a/docs/contributor-guide/adding_dataset.rst b/docs/contributor-guide/adding_dataset.rst index 9d31160..ca5043b 100644 --- a/docs/contributor-guide/adding_dataset.rst +++ b/docs/contributor-guide/adding_dataset.rst @@ -20,6 +20,11 @@ If the provider data is already SOFA-native, keep inheriting from :class:`~irdl.base.BaseDataset`. The default ``_ingest()`` promotes SOFA files directly without rewriting them. +If the DOI provider is not SOFA-native but every Dataset selection has a direct SOFA +source, override ``direct_sofa_url(source_filename)``. Add each URL and SHA-256 digest to +``src/irdl/registry/direct_sofa_hashes.json``. This path serves non-raw retrieval only; +``output_format="raw"`` continues to download the DOI provider artifact. + Choose a module ---------------- diff --git a/scripts/update_sofacoustics_hashes.py b/scripts/update_sofacoustics_hashes.py deleted file mode 100644 index 5a4943f..0000000 --- a/scripts/update_sofacoustics_hashes.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Regenerate the packaged SOFAcoustics hash registry for supported files.""" - -from __future__ import annotations - -import json -from concurrent.futures import ThreadPoolExecutor, as_completed -from hashlib import sha256 -from pathlib import Path -from urllib.request import Request, urlopen - -PROVIDER_ROOT = "https://sofacoustics.org/data/database" -OUTPUT_PATH = Path("src/irdl/registry/sofacoustics_hashes.json") -SUPPORTED_FILES = { - "hutubs": [f"pp{subject}_HRIRs_{kind}.sofa" for subject in range(1, 97) for kind in ("measured", "simulated")] -} - - -def _hash_static_file(provider: str, filename: str) -> tuple[str, str]: - path_key = f"{provider}/{filename}" - request = Request( # noqa: S310 - f"{PROVIDER_ROOT}/{path_key}", - headers={"User-Agent": "irdl-hash-generator"}, - ) - digest = sha256() - with urlopen(request, timeout=120) as response: # noqa: S310 - while chunk := response.read(1024 * 1024): - digest.update(chunk) - return path_key, f"sha256:{digest.hexdigest()}" - - -def main() -> None: - """Regenerate the flat provider-relative-path hash registry.""" - jobs = [(provider, filename) for provider, files in SUPPORTED_FILES.items() for filename in files] - registry: dict[str, str] = {} - - with ThreadPoolExecutor(max_workers=6) as executor: - futures = { - executor.submit(_hash_static_file, provider, filename): (provider, filename) for provider, filename in jobs - } - for future in as_completed(futures): - path_key, digest = future.result() - registry[path_key] = digest - print(f"hashed {path_key}") # noqa: T201 - - OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) - OUTPUT_PATH.write_text(json.dumps(dict(sorted(registry.items())), indent=2) + "\n", encoding="utf-8") - print(f"wrote {OUTPUT_PATH}") # noqa: T201 - - -if __name__ == "__main__": - main() diff --git a/src/irdl/base.py b/src/irdl/base.py index 9c2f72c..0b80a70 100644 --- a/src/irdl/base.py +++ b/src/irdl/base.py @@ -23,6 +23,7 @@ from inspect import isabstract from pathlib import Path from types import ModuleType +from urllib.parse import urlparse import h5py as h5 import netCDF4 @@ -31,8 +32,9 @@ import sofar as sf from irdl.cache import IRDL_CACHE_DIR +from irdl.downloader import _fetch, _pooch_from_static_registry from irdl.logging import logger -from irdl.utils import _link_or_copy, _preserve_permissions +from irdl.utils import _link_or_copy, _preserve_permissions, load_hash_registry _SOFA_FIR_E_DIMS = 4 DEFAULT_CHUNK_SIZE = 256 @@ -61,6 +63,8 @@ class BaseDataset(ABC): Validate dataset-specific parameters (including output_format). _source_filename(**dataset_kwargs) -> str Construct the raw input filename with extension. + direct_sofa_url(source_filename) -> str | None + Optionally locate a direct SOFA source for non-raw retrieval. _download(**dataset_kwargs) -> Path Download and return Path to raw file. _process(provider_artifact: Path, ingest_path: Path, **_dataset_kwargs) -> Path: @@ -167,21 +171,26 @@ def _get( output_path = self._output_path(output_dir, source_filename, output_format) ingest_path = cache_dir / "ingest" / source_filename - # Special handling for raw output format + # Raw always uses the Dataset DOI. if output_format == "raw": provider_artifact = self.download(provider_dir, **dataset_kwargs) if export_dir is None: return provider_artifact return self._export_raw(provider_artifact, export_dir) - # Early exit if output file already exists (not applicable for raw format, handled above) + sofa_path = self._output_path(cache_dir / "output", source_filename, "sofa") if output_path is not None and output_path.exists(): logger.info(f"Output file already exists at {output_path}, skipping download and conversion.") + if output_format == "sofa": + self._verify_sofa_convention(output_path) return output_path - sofa_path = self._output_path(cache_dir / "output", source_filename, "sofa") + direct_sofa_url = self.direct_sofa_url(source_filename) if sofa_path.exists(): logger.info(f"Cache hit: {sofa_path}.") + elif direct_sofa_url is not None: + provider_artifact = self.download(provider_dir, direct_sofa_url=direct_sofa_url) + _link_or_copy(provider_artifact, sofa_path) else: if ingest_path.exists(): logger.info(f"Ingestible file already exists at {ingest_path}, skipping download and processing.") @@ -192,8 +201,8 @@ def _get( logger.debug(f"Ingesting {ingest_artifact} to SOFA file {sofa_path}") with logger.spin(f"Writing SOFA {sofa_path.name}..."): self._ingest(ingest_artifact, sofa_path, **dataset_kwargs) - with logger.spin(f"Verifying SOFA conventions for {sofa_path.name}..."): - self._verify_sofa_convention(sofa_path) + with logger.spin(f"Verifying SOFA conventions for {sofa_path.name}..."): + self._verify_sofa_convention(sofa_path) return self._to_output(output_format, sofa_path, output_path) @@ -259,8 +268,31 @@ def _source_filename(self, **dataset_kwargs) -> str: "FABIAN_HRIR_measured_HATO_0.sofa"). """ - def download(self, provider_dir: Path, **dataset_kwargs) -> Path: - """Download raw files and return Path to the primary artifact. + def direct_sofa_url(self, _source_filename: str) -> str | None: + """Return a direct SOFA URL for non-raw retrieval, if available.""" + return None + + def _download_direct_sofa(self, provider_dir: Path, url: str) -> Path: + """Download one hash-verified direct SOFA artifact.""" + filename = Path(urlparse(url).path).name + if Path(filename).suffix.lower() != ".sofa": + msg = f"Direct SOFA URL must name a .sofa file: {url!r}" + raise ValueError(msg) + try: + known_hash = load_hash_registry("direct_sofa")[url] + except KeyError as error: + msg = f"Missing direct SOFA hash registry entry for {url!r}" + raise ValueError(msg) from error + pup = _pooch_from_static_registry( + path=provider_dir, + registry={filename: known_hash}, + urls={filename: url}, + ) + _fetch(pup, filename) + return provider_dir / filename + + def download(self, provider_dir: Path, *, direct_sofa_url: str | None = None, **dataset_kwargs) -> Path: + """Download raw or direct-SOFA files and return the primary artifact. This method wraps _download to enforce provider_dir existence for all subclasses. @@ -279,6 +311,8 @@ def download(self, provider_dir: Path, **dataset_kwargs) -> Path: Path to the downloaded artifact on disk (file or directory). """ provider_dir.mkdir(exist_ok=True, parents=True) + if direct_sofa_url is not None: + return self._download_direct_sofa(provider_dir, direct_sofa_url) return self._download(provider_dir, **dataset_kwargs) @abstractmethod diff --git a/src/irdl/downloader.py b/src/irdl/downloader.py index de0583c..de0f76d 100644 --- a/src/irdl/downloader.py +++ b/src/irdl/downloader.py @@ -1,5 +1,8 @@ """Implements download and post-processing based on pooch.""" +from collections.abc import Mapping +from pathlib import Path + import pooch as po from irdl.cache import IRDL_CACHE_DIR @@ -28,6 +31,17 @@ def _fetch(pup: po.Pooch, fname: str) -> str: return pup.fetch(fname, progressbar=RichProgressBar(fname, preset_total=preset_total)) +def _pooch_from_static_registry( + path: str | Path, + registry: Mapping[str, str], + urls: Mapping[str, str], +) -> po.Pooch: + """Create a Pooch instance for hash-verified direct downloads.""" + pup = po.create(path=path, base_url="", registry=dict(registry), urls=dict(urls), retry_if_failed=2) + pup.file_sizes = {} + return pup + + def _pooch_from_doi(doi: str, path: str = IRDL_CACHE_DIR) -> po.Pooch: """Create a Pooch instance from a DOI. diff --git a/src/irdl/registry/direct_sofa_hashes.json b/src/irdl/registry/direct_sofa_hashes.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/irdl/registry/direct_sofa_hashes.json @@ -0,0 +1 @@ +{} diff --git a/tests/test_base.py b/tests/test_base.py index aa8f8fa..b572cc0 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest +import sofar as sf from irdl.base import BaseDataset from irdl.ista import IstaBaseDataset @@ -88,6 +89,49 @@ def _process(self, _provider_artifact: Path, ingest_path: Path, **dataset_kwargs assert ingest_path.exists() +class TestDirectSofaProvider: + """Tests for optional direct-SOFA retrieval.""" + + def test_non_raw_uses_direct_sofa_without_ingest(self, monkeypatch, sofa_object, tmp_path): + """Direct SOFA bypasses the DOI download and ingest stage.""" + + class DirectSofaDataset(BaseDataset): + name = "direct" + doi = "10.0000/direct" + + def _validate_params(self, **_dataset_kwargs): + pass + + def _source_filename(self, **_dataset_kwargs): + return "canonical.h5" + + def _download(self, _provider_dir: Path, **_dataset_kwargs): + msg = "DOI download must only serve raw requests" + raise AssertionError(msg) + + def direct_sofa_url(self, source_filename: str): + assert source_filename == "canonical.h5" + return "https://example.invalid/alternate.sofa" + + dataset = DirectSofaDataset() + + def download_direct(provider_dir: Path, url: str) -> Path: + assert url == "https://example.invalid/alternate.sofa" + provider_dir.mkdir(parents=True, exist_ok=True) + path = provider_dir / "alternate.sofa" + sf.write_sofa(path, sofa_object) + return path + + monkeypatch.setattr(dataset, "_download_direct_sofa", download_direct) + + result = dataset._get(cache_dir=tmp_path, export_dir=None, output_format="sofa") + + assert result == tmp_path / "DIRECT" / "output" / "canonical.sofa" + assert result.exists() + assert (tmp_path / "DIRECT" / "provider" / "alternate.sofa").exists() + assert not (tmp_path / "DIRECT" / "ingest").exists() + + class TestIstaBaseDatasetAbstract: """Tests for IstaBaseDataset abstract class behavior.""" diff --git a/uv.lock b/uv.lock index 7ebfd7b..e845f16 100644 --- a/uv.lock +++ b/uv.lock @@ -643,7 +643,7 @@ wheels = [ [[package]] name = "irdl" -version = "1.0.0b6" +version = "1.0.0b5" source = { editable = "." } dependencies = [ { name = "h5py" },