diff --git a/docs/source/concepts/inputs/from_source.rst b/docs/source/concepts/inputs/from_source.rst index 91385ced6..96f496cde 100644 --- a/docs/source/concepts/inputs/from_source.rst +++ b/docs/source/concepts/inputs/from_source.rst @@ -77,6 +77,8 @@ from_source - deprecated, use :ref:`data-sources-wekeo-cds` instead * - :ref:`data-sources-zarr` - load data from a `Zarr `_ store + * - :ref:`data-sources-zenodo` + - retrieve data from a `Zenodo `_ record ---------------------------------- @@ -1282,6 +1284,23 @@ zarr :param str path: path or URL to the Zarr store +.. _data-sources-zenodo: + +zenodo +-------- + +.. py:function:: from_source("zenodo", identifier, only=None, **kwargs) + :noindex: + + `Zenodo `_ is an open repository for research data and related information. + The ``zenodo`` source provides access to files attached to a Zenodo record via the Zenodo API. + + :param identifier: a record ID, URL or DOI. + :type identifier: int, str + :param only: the files to select from the record. Accepts a glob pattern that is matched against the file names in the record or an explicit list of file names to select. By default, all files in the record are selected. + :type only: str, sequence of str, None + :param dict **kwargs: other keyword arguments passed to the :ref:`url ` source. + .. _MARS catalog: https://apps.ecmwf.int/archive-catalogue/ .. _MARS user documentation: https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation diff --git a/pyproject.toml b/pyproject.toml index 221e95334..610e08090 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "pandas", "pdbufr>=0.11", "pyyaml", + "requests", "tqdm>=4.63", "xarray>=0.19" ] diff --git a/src/earthkit/data/sources/__init__.py b/src/earthkit/data/sources/__init__.py index 6d3a02e72..b90014fad 100644 --- a/src/earthkit/data/sources/__init__.py +++ b/src/earthkit/data/sources/__init__.py @@ -445,6 +445,15 @@ def from_source( ) -> "Data": ... +@overload +def from_source( + name: Literal["zenodo"], + identifier: str | int, + only: str | list[str], + **kwargs, +) -> "Data": ... + + def from_source(name: str, *args, lazily=False, **kwargs) -> "Data": if lazily: return from_source_lazily(name, *args, **kwargs) diff --git a/src/earthkit/data/sources/zenodo.py b/src/earthkit/data/sources/zenodo.py new file mode 100644 index 000000000..c8e8752e6 --- /dev/null +++ b/src/earthkit/data/sources/zenodo.py @@ -0,0 +1,130 @@ +# (C) Copyright 2026- ECMWF and individual contributors. + +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +import fnmatch +import logging +import re + +import requests + +from earthkit.data.core.config import CONFIG +from earthkit.data.sources import Source +from earthkit.data.sources.multi_url import MultiUrl + +LOG = logging.getLogger(__name__) + +_DOI_PATTERN = re.compile( + r"^(?:doi:\s*|(?:https?:\/\/)?(?:dx\.)?doi\.org\/)?10\.5281/zenodo\.(\d+)\/?$", + flags=re.IGNORECASE, +) +_URL_PATTERN = re.compile(r"^(?:https?:\/\/)?zenodo\.org\/records?\/(\d+)\/?(?:\?.*)?$", flags=re.IGNORECASE) + + +def _get_record_files(record_id): + timeout = CONFIG.get("url-download-timeout") + + api_url = f"https://zenodo.org/api/records/{record_id}" + LOG.debug(f"Fetching file list for record {record_id} from {api_url}") + try: + r = requests.get(api_url, timeout=timeout) + r.raise_for_status() + except requests.ConnectionError as e: + raise RuntimeError("could not connect to zenodo.org") from e + except requests.Timeout as e: + raise RuntimeError(f"request to zenodo.org timed out after {timeout}s.") from e + except requests.HTTPError as e: + raise RuntimeError(f"Zenodo API returned HTTP {r.status_code}") from e + + try: + data = r.json() + except ValueError as e: + raise RuntimeError("failed to parse Zenodo API response") from e + + if not isinstance(data, dict) or "files" not in data: + raise RuntimeError(f"unexpected Zenodo API response for record {record_id}") + if not data["files"]: + raise RuntimeError(f"Record {record_id} has no accessible files. The record may be restricted or embargoed.") + + try: + # URLs from API response, works for record and concept IDs + file_urls = {f["key"]: f["links"]["self"] for f in data["files"]} + except (KeyError, TypeError) as e: + raise RuntimeError(f"unexpected file entry in the Zenodo API response for record {record_id}") from e + + LOG.debug(f"Record {record_id} contains {len(file_urls)} file(s): {list(file_urls)!r}") + return file_urls + + +class Zenodo(Source): + """Source for downloading files from Zenodo records. + + Parameters + ---------- + identifier : int | str + Record ID, Zenodo URL or DOI. A DOI may also be given as a doi.org URL. + only : str | Sequence[str] | None, optional + File selection with a glob string or an explicit list of file names. + By default, all files are selected. + **kwargs + Additional keyword arguments forwarded to the URL source. + """ + + def __init__(self, identifier, only=None, **kwargs): + super().__init__() + self._kwargs = kwargs + + if isinstance(identifier, str): + identifier = identifier.strip() + + # A Zenodo DOI is 10.5281/zenodo., so no lookup via doi.org is needed. + # For a concept DOI this is the concept record's ID, which the API redirects to the + # latest version, and the file URLs then refer to that version. + if isinstance(identifier, str) and (match := _DOI_PATTERN.match(identifier)): + self.record_id = int(match.group(1)) + elif isinstance(identifier, int): + self.record_id = identifier + elif isinstance(identifier, str) and (match := _URL_PATTERN.match(identifier)): + self.record_id = int(match.group(1)) + elif isinstance(identifier, str) and identifier.isnumeric(): + self.record_id = int(identifier) + else: + raise ValueError(f"unable to determine record ID from identifier: {identifier!r}") + + LOG.info(f"Zenodo record ID: {self.record_id}") + + # Fetch file metadata from the Zenodo API + record_files = _get_record_files(self.record_id) + + # No filenames specified -> select all + if only is None: + self._file_urls = record_files + # Match filenames with provided pattern + elif isinstance(only, str): + matched = fnmatch.filter(record_files.keys(), only) + if not matched: + raise ValueError(f"no files in record {self.record_id} match the pattern: {only!r}") + self._file_urls = {name: record_files[name] for name in matched} + # Select filenames based on provided list + else: + only = list(dict.fromkeys(only)) # deduplicate while preserving order + if not only: + raise ValueError(f"no files selected from record {self.record_id}") + self._file_urls = {name: record_files[name] for name in only if name in record_files} + if len(self._file_urls) != len(only): + missing = ", ".join(repr(name) for name in only if name not in record_files) + raise ValueError(f"file(s) not found in record {self.record_id}: " + missing) + + selected = ", ".join(self._file_urls.keys()) + LOG.info(f"Selected {len(self._file_urls)} file(s) from record {self.record_id}: {selected}") + + def mutate(self): + urls = list(self._file_urls.values()) + return MultiUrl(urls, **self._kwargs) + + +source = Zenodo diff --git a/tests/sources/test_zenodo.py b/tests/sources/test_zenodo.py new file mode 100644 index 000000000..a50541674 --- /dev/null +++ b/tests/sources/test_zenodo.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +# (C) Copyright 2026 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + +import pytest +import requests + +from earthkit.data import from_source +from earthkit.data.sources import _from_source_internal, get_source +from earthkit.data.sources import zenodo as zenodo_module + +RECORD_ID = 123 +CONCEPT_ID = 678 +FILES = ["b.grib", "a.grib", "c.nc"] + + +def DOI(id): + return f"10.5281/zenodo.{id}" # as per https://support.zenodo.org/help/en-gb/18-general/216-what-is-a-doi + + +def download_url(name, record_id=RECORD_ID): + """The download URL as the Zenodo API reports it in files[*].links.self.""" + return f"https://zenodo.org/api/records/{record_id}/files/{name}/content" + + +def assert_selected(zenodo, files=set(FILES), record_id=RECORD_ID): + # Files ignores order of elements while list enforces it + expected = type(files)(download_url(file, record_id) for file in files) + assert type(files)(zenodo.urls) == expected + + +class MockResponse: + """Stand-in for requests.Response.""" + + def __init__(self, url=None, status_code=200, payload=None): + self.url = url + self.status_code = status_code + self._payload = payload + + def raise_for_status(self): + if self.status_code >= 400: + raise requests.HTTPError(f"HTTP {self.status_code} for {self.url}", response=self) + + def json(self): + if self._payload is None: + raise ValueError("invalid JSON") + return self._payload + + +class MockRequests: + """Stand-in for the requests module, pretending to be the Zenodo API.""" + + ConnectionError = requests.ConnectionError + Timeout = requests.Timeout + HTTPError = requests.HTTPError + + def __init__(self): + self.records = {} + self.response = None + # Observed behaviour + self.urls = None + self.kwargs = None + + def register_record(self, record_id, files, resolves_to=None): + self.records[int(record_id)] = (list(files), int(resolves_to or record_id)) + + def respond_with(self, response): + self.response = response + + def get(self, url, **kwargs): + assert url.startswith("https://zenodo.org/api/records/"), f"unexpected request to {url}" + if self.response is None: + record_id = int(url.rsplit("/", 1)[-1]) + if record_id not in self.records: + return MockResponse(url, status_code=404) + # Minimal valid response from the Zenodo API + names, resolved_id = self.records[record_id] + files = [{"key": name, "links": {"self": download_url(name, resolved_id)}} for name in names] + return MockResponse(url, payload={"id": resolved_id, "files": files}) + if isinstance(self.response, Exception): + raise self.response + return self.response + + +class TestZenodoSourceOffline: + """Offline test for the Zenodo source. + + Test up to the point where the Zenodo source mutates into a MultiURL source + with a fake Zenodo API. + """ + + @pytest.fixture + def zenodo(self, monkeypatch): + api = MockRequests() + api.register_record(RECORD_ID, FILES) + api.register_record(CONCEPT_ID, FILES, resolves_to=RECORD_ID) + + def capture(urls, **kwargs): + api.urls = list(urls) + api.kwargs = kwargs + return _from_source_internal("empty") + + # Patch the names bound in the zenodo module only + monkeypatch.setattr(zenodo_module, "requests", api) + monkeypatch.setattr(zenodo_module, "MultiUrl", capture) + return api + + def test_zenodo_source_is_registered(self): + assert get_source._lookup("zenodo") is not None + + @pytest.mark.parametrize("identifier", [RECORD_ID, CONCEPT_ID]) + def test_valid_identifier_int(self, zenodo, identifier): + from_source("zenodo", identifier) + assert_selected(zenodo) + + @pytest.mark.parametrize("identifier", [RECORD_ID, CONCEPT_ID]) + @pytest.mark.parametrize( + "url", + [ + "{id}", + " {id}", + "https://zenodo.org/record/{id}", + "https://zenodo.org/records/{id}", + "https://ZENODO.ORG/records/{id}", + "https://zenodo.org/records/{id}/", + "https://zenodo.org/records/{id}?download=1", + " https://zenodo.org/records/{id}?download=1", + "https://zenodo.org/records/{id}?download=1 ", + "http://zenodo.org/record/{id}", + "http://zenodo.org/records/{id}", + "http://zenodo.org/records/{id}/", + "zenodo.org/record/{id}", + "zenodo.org/records/{id}", + "zenodo.org/records/{id}/", + "10.5281/zenodo.{id}", + "doi:10.5281/zenodo.{id}", + "https://doi.org/10.5281/zenodo.{id}", + "https://doi.org/10.5281/zenodo.{id}/", + "https://DOI.ORG/10.5281/zenodo.{id}", + "http://doi.org/10.5281/zenodo.{id}", + "http://doi.org/10.5281/zenodo.{id}/", + "doi.org/10.5281/zenodo.{id}", + "doi.org/10.5281/zenodo.{id}/", + "https://dx.doi.org/10.5281/zenodo.{id}", + "https://dx.doi.org/10.5281/zenodo.{id}/", + "dx.doi.org/10.5281/zenodo.{id}", + ], + ) + @pytest.mark.parametrize( + "only,expected", + [ + (None, set(FILES)), + ("c.nc", {"c.nc"}), + ("*.grib", {"a.grib", "b.grib"}), + (FILES, FILES), + (FILES[::-1], FILES[::-1]), # maintains order + (["a.grib", "a.grib", "b.grib"], ["a.grib", "b.grib"]), # ignores duplicates + ], + ) + def test_valid_identifier_str(self, zenodo, identifier, url, only, expected): + from_source("zenodo", url.format(id=identifier), only=only) + assert_selected(zenodo, files=expected) + + def test_without_kwargs(self, zenodo): + from_source("zenodo", RECORD_ID) + assert zenodo.kwargs == {} + + def test_kwargs_forwarded(self, zenodo): + from_source("zenodo", RECORD_ID, only="a.grib", foo="bar", bar=False) + assert zenodo.kwargs == {"foo": "bar", "bar": False} + + # ValueErrors for input validation problems and invalid file selection + + @pytest.mark.parametrize( + "identifier", + [ + None, + "", + " ", + "not-a-record", + "10.1234/foo.567", + "10.5281/foobar.12345", + "https://example.com/records/12345", + "https://zenodo.org/communities/abc", + "https://zenodo.org/records/abc", + "zenodo.org/records/12345/files/a.grib", + # Near misses of the accepted doi.org URL forms + "https://doi.org/10.1234/foo.567", + "https://doi.org/10.5281/foobar.12345", + "https://doi.org/", + "https://doi.org", + "https://example.com/10.5281/zenodo.12345", + # The host must be matched, not merely found at the end of another one + "https://mydoi.org/10.5281/zenodo.12345", + # The doi: prefix and the URL form are alternatives, not combinable + "doi:https://doi.org/10.5281/zenodo.12345", + ], + ) + def test_invalid_identifier(self, zenodo, identifier): + with pytest.raises(ValueError): + from_source("zenodo", identifier) + + @pytest.mark.parametrize( + "only", + [ + "", + "*.zip", + "d.grib", + "grib", + "a.gri", + ["d.grib"], + ["a.grib", "d.grib"], + ["A.GRIB"], # case sensitive + ["*.grib"], # list entry does not trigger pattern matching + [], + ], + ) + def test_invalid_only(self, zenodo, only): + with pytest.raises(ValueError): + from_source("zenodo", RECORD_ID, only=only) + + # RuntimeErrors raised for problems with Zenodo API and response + + def test_zenodo_unknown_record(self, zenodo): + with pytest.raises(RuntimeError): + from_source("zenodo", 999) + + @pytest.mark.parametrize( + "response", + [ + requests.ConnectionError(), + requests.ReadTimeout(), + MockResponse(status_code=404), + MockResponse(status_code=503), + MockResponse(payload=None), # invalid JSON + MockResponse(payload=5), # not dict-typed + # No files in records + MockResponse(payload={}), + MockResponse(payload={"files": []}), + MockResponse(payload={"files": None}), + MockResponse(payload={"metadata": {}}), + # Malformed file entry + MockResponse(payload={"files": [{"key": "a.grib"}]}), + MockResponse(payload={"files": [{"key": "a.grib", "links": {}}]}), + MockResponse(payload={"files": [{"links": {"self": "https://example.com/a.grib"}}]}), + MockResponse(payload={"files": ["a.grib"]}), + ], + ) + def test_api_failure_runtime_errors(self, zenodo, response): + zenodo.respond_with(response) + with pytest.raises(RuntimeError): + from_source("zenodo", RECORD_ID) + + +if __name__ == "__main__": + from earthkit.data.utils.testing import main + + main(__file__)