Add optional download path for SOFA-formatted datasets

docs/contributor-guide/adding_dataset.rst

@@ -20,6 +20,11 @@ If the provider data is already SOFA-native, keep inheriting from
 :class:`~irdl.base.BaseDataset`. The default ``_ingest()`` promotes SOFA files directly
 without rewriting them.
 
+If the DOI provider is not SOFA-native but every Dataset selection has a direct SOFA
+source, override ``direct_sofa_url(source_filename)``. Add each URL and SHA-256 digest to
+``src/irdl/registry/direct_sofa_hashes.json``. This path serves non-raw retrieval only;
+``output_format="raw"`` continues to download the DOI provider artifact.
+
 Choose a module
 ----------------
 

src/irdl/base.py

@@ -23,6 +23,7 @@
 from inspect import isabstract
 from pathlib import Path
 from types import ModuleType
+from urllib.parse import urlparse
 
 import h5py as h5
 import netCDF4
@@ -31,8 +32,9 @@
 import sofar as sf
 
 from irdl.cache import IRDL_CACHE_DIR
+from irdl.downloader import _fetch, _pooch_from_static_registry
 from irdl.logging import logger
-from irdl.utils import _link_or_copy, _preserve_permissions
+from irdl.utils import _link_or_copy, _preserve_permissions, load_hash_registry
 
 _SOFA_FIR_E_DIMS = 4
 DEFAULT_CHUNK_SIZE = 256
@@ -61,6 +63,8 @@ class BaseDataset(ABC):
         Validate dataset-specific parameters (including output_format).
     _source_filename(**dataset_kwargs) -> str
         Construct the raw input filename with extension.
+    direct_sofa_url(source_filename) -> str | None
+        Optionally locate a direct SOFA source for non-raw retrieval.
     _download(**dataset_kwargs) -> Path
         Download and return Path to raw file.
     _process(provider_artifact: Path, ingest_path: Path, **_dataset_kwargs) -> Path:
@@ -167,21 +171,26 @@ def _get(
         output_path = self._output_path(output_dir, source_filename, output_format)
         ingest_path = cache_dir / "ingest" / source_filename
 
-        # Special handling for raw output format
+        # Raw always uses the Dataset DOI.
         if output_format == "raw":
             provider_artifact = self.download(provider_dir, **dataset_kwargs)
             if export_dir is None:
                 return provider_artifact
             return self._export_raw(provider_artifact, export_dir)
 
-        # Early exit if output file already exists (not applicable for raw format, handled above)
+        sofa_path = self._output_path(cache_dir / "output", source_filename, "sofa")
         if output_path is not None and output_path.exists():
             logger.info(f"Output file already exists at {output_path}, skipping download and conversion.")
+            if output_format == "sofa":
+                self._verify_sofa_convention(output_path)
             return output_path
 
-        sofa_path = self._output_path(cache_dir / "output", source_filename, "sofa")
+        direct_sofa_url = self.direct_sofa_url(source_filename)
         if sofa_path.exists():
             logger.info(f"Cache hit: {sofa_path}.")
+        elif direct_sofa_url is not None:
+            provider_artifact = self.download(provider_dir, direct_sofa_url=direct_sofa_url)
+            _link_or_copy(provider_artifact, sofa_path)
         else:
             if ingest_path.exists():
                 logger.info(f"Ingestible file already exists at {ingest_path}, skipping download and processing.")
@@ -192,8 +201,8 @@ def _get(
             logger.debug(f"Ingesting {ingest_artifact} to SOFA file {sofa_path}")
             with logger.spin(f"Writing SOFA {sofa_path.name}..."):
                 self._ingest(ingest_artifact, sofa_path, **dataset_kwargs)
-            with logger.spin(f"Verifying SOFA conventions for {sofa_path.name}..."):
-                self._verify_sofa_convention(sofa_path)
+        with logger.spin(f"Verifying SOFA conventions for {sofa_path.name}..."):
+            self._verify_sofa_convention(sofa_path)
 
         return self._to_output(output_format, sofa_path, output_path)
 
@@ -259,8 +268,31 @@ def _source_filename(self, **dataset_kwargs) -> str:
             "FABIAN_HRIR_measured_HATO_0.sofa").
         """
 
-    def download(self, provider_dir: Path, **dataset_kwargs) -> Path:
-        """Download raw files and return Path to the primary artifact.
+    def direct_sofa_url(self, _source_filename: str) -> str | None:
+        """Return a direct SOFA URL for non-raw retrieval, if available."""
+        return None
+
+    def _download_direct_sofa(self, provider_dir: Path, url: str) -> Path:
+        """Download one hash-verified direct SOFA artifact."""
+        filename = Path(urlparse(url).path).name
+        if Path(filename).suffix.lower() != ".sofa":
+            msg = f"Direct SOFA URL must name a .sofa file: {url!r}"
+            raise ValueError(msg)
+        try:
+            known_hash = load_hash_registry("direct_sofa")[url]
+        except KeyError as error:
+            msg = f"Missing direct SOFA hash registry entry for {url!r}"
+            raise ValueError(msg) from error
+        pup = _pooch_from_static_registry(
+            path=provider_dir,
+            registry={filename: known_hash},
+            urls={filename: url},
+        )
+        _fetch(pup, filename)
+        return provider_dir / filename
+
+    def download(self, provider_dir: Path, *, direct_sofa_url: str | None = None, **dataset_kwargs) -> Path:
+        """Download raw or direct-SOFA files and return the primary artifact.
 
         This method wraps _download to enforce provider_dir existence for all subclasses.
 
@@ -279,6 +311,8 @@ def download(self, provider_dir: Path, **dataset_kwargs) -> Path:
             Path to the downloaded artifact on disk (file or directory).
         """
         provider_dir.mkdir(exist_ok=True, parents=True)
+        if direct_sofa_url is not None:
+            return self._download_direct_sofa(provider_dir, direct_sofa_url)
         return self._download(provider_dir, **dataset_kwargs)
 
     @abstractmethod

src/irdl/downloader.py

@@ -1,5 +1,8 @@
 """Implements download and post-processing based on pooch."""
 
+from collections.abc import Mapping
+from pathlib import Path
+
 import pooch as po
 
 from irdl.cache import IRDL_CACHE_DIR
@@ -28,6 +31,17 @@ def _fetch(pup: po.Pooch, fname: str) -> str:
     return pup.fetch(fname, progressbar=RichProgressBar(fname, preset_total=preset_total))
 
 
+def _pooch_from_static_registry(
+    path: str | Path,
+    registry: Mapping[str, str],
+    urls: Mapping[str, str],
+) -> po.Pooch:
+    """Create a Pooch instance for hash-verified direct downloads."""
+    pup = po.create(path=path, base_url="", registry=dict(registry), urls=dict(urls), retry_if_failed=2)
+    pup.file_sizes = {}
+    return pup
+
+
 def _pooch_from_doi(doi: str, path: str = IRDL_CACHE_DIR) -> po.Pooch:
     """Create a Pooch instance from a DOI.
 

src/irdl/registry/direct_sofa_hashes.json

@@ -0,0 +1 @@
+{}

tests/test_base.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import pytest
+import sofar as sf
 
 from irdl.base import BaseDataset
 from irdl.ista import IstaBaseDataset
@@ -88,6 +89,49 @@ def _process(self, _provider_artifact: Path, ingest_path: Path, **dataset_kwargs
         assert ingest_path.exists()
 
 
+class TestDirectSofaProvider:
+    """Tests for optional direct-SOFA retrieval."""
+
+    def test_non_raw_uses_direct_sofa_without_ingest(self, monkeypatch, sofa_object, tmp_path):
+        """Direct SOFA bypasses the DOI download and ingest stage."""
+
+        class DirectSofaDataset(BaseDataset):
+            name = "direct"
+            doi = "10.0000/direct"
+
+            def _validate_params(self, **_dataset_kwargs):
+                pass
+
+            def _source_filename(self, **_dataset_kwargs):
+                return "canonical.h5"
+
+            def _download(self, _provider_dir: Path, **_dataset_kwargs):
+                msg = "DOI download must only serve raw requests"
+                raise AssertionError(msg)
+
+            def direct_sofa_url(self, source_filename: str):
+                assert source_filename == "canonical.h5"
+                return "https://example.invalid/alternate.sofa"
+
+        dataset = DirectSofaDataset()
+
+        def download_direct(provider_dir: Path, url: str) -> Path:
+            assert url == "https://example.invalid/alternate.sofa"
+            provider_dir.mkdir(parents=True, exist_ok=True)
+            path = provider_dir / "alternate.sofa"
+            sf.write_sofa(path, sofa_object)
+            return path
+
+        monkeypatch.setattr(dataset, "_download_direct_sofa", download_direct)
+
+        result = dataset._get(cache_dir=tmp_path, export_dir=None, output_format="sofa")
+
+        assert result == tmp_path / "DIRECT" / "output" / "canonical.sofa"
+        assert result.exists()
+        assert (tmp_path / "DIRECT" / "provider" / "alternate.sofa").exists()
+        assert not (tmp_path / "DIRECT" / "ingest").exists()
+
+
 class TestIstaBaseDatasetAbstract:
     """Tests for IstaBaseDataset abstract class behavior."""