Support new ACCESS-ESM1.6 raw output filenames

src/access_moppy/file_discovery.py

@@ -10,10 +10,9 @@
 1. Per-variable ``file_pattern`` in the mapping entry — explicit override,
    useful for edge-cases or legacy folder layouts.
 2. Component-level ``frequency_patterns`` from the ``model_info.file_discovery``
-   block in the model mapping JSON, with ``{model_var}`` substituted by every
-   entry in ``model_variables`` (one file per model variable, as used by the
-   ocean component).  Atmosphere and sea-ice components pack all variables into
-   a single file per frequency, so no substitution is needed there.
+    block in the model mapping JSON, with ``{model_var}`` substituted by every
+    entry in ``model_variables`` for one-variable-per-file patterns.  Alternative
+    legacy packed-file patterns can be listed alongside them.
 3. :class:`FileDiscoveryError` is raised when neither source provides a pattern.
 
 Year-based filtering
@@ -176,6 +175,10 @@ def _extract_year_from_path(path: Path) -> int | None:
     * ``ocean-2d-surface_temp-1monthly-mean-ym_0001_01.nc`` → ``1``  (spinup, ``_YYYY_MM``)
     """
     name = path.stem  # strip .nc extension
+    # ACCESS component convention: hyphen-delimited YYYY or YYYY-MM datestamp.
+    m = re.search(r"\.(\d{4})(?:-\d{2})?$", name)
+    if m:
+        return int(m.group(1))
     # Unified pattern: _YYYYMM-YYYYMM at end of stem (start of time range)
     m = re.search(r"_(\d{6})-\d{6}$", name)
     if m:
@@ -208,6 +211,11 @@ def _extract_year_month_from_path(
     for filenames that encode only a year (e.g. annual ocean files).
     """
     name = path.stem
+    # ACCESS component convention: hyphen-delimited YYYY-MM or YYYY datestamp.
+    m = re.search(r"\.(\d{4})(?:-(\d{2}))?$", name)
+    if m:
+        month = int(m.group(2)) if m.group(2) is not None else None
+        return int(m.group(1)), month
     # Unified range pattern _YYYYMM-YYYYMM → start
     m = re.search(r"_(\d{6})-\d{6}$", name)
     if m:
@@ -277,28 +285,43 @@ def _build_patterns(
     # "1mon-mean-y_*" and the newer "1monthly-mean-ym_*" ocean layouts.
     globs = [file_glob] if isinstance(file_glob, str) else list(file_glob)
 
-    subdir = comp_cfg.get("subdir", "")
+    configured_subdirs = comp_cfg.get("subdir", "")
+    subdirs = (
+        [configured_subdirs]
+        if isinstance(configured_subdirs, str)
+        else list(configured_subdirs)
+    )
     output_dir_pattern = file_discovery_cfg.get(
         "output_dir_pattern", "output[0-9][0-9][0-9]"
     )
 
+    unsubstitutable_globs: list[str] = []
     patterns: list[str] = []
-    for file_glob in globs:
-        if "{model_var}" in file_glob:
-            # Per-variable files (e.g. ocean): one pattern per model variable
-            model_variables = var_entry.get("model_variables") or []
-            if not model_variables:
-                raise FileDiscoveryError(
-                    f"Pattern '{file_glob}' requires {{model_var}} substitution but "
-                    "the mapping entry has no 'model_variables'."
+    for subdir in subdirs:
+        for file_glob in globs:
+            if "{model_var}" in file_glob:
+                # Per-variable files: one pattern per model variable.  Some
+                # mixed legacy + per-variable configs also include packed-file
+                # fallbacks that do not need model variables; keep those usable
+                # for mapping entries that intentionally omit model_variables.
+                model_variables = var_entry.get("model_variables") or []
+                if not model_variables:
+                    unsubstitutable_globs.append(file_glob)
+                    continue
+                patterns.extend(
+                    f"{output_dir_pattern}/{subdir}/{file_glob.replace('{model_var}', mv)}"
+                    for mv in model_variables
                 )
-            patterns.extend(
-                f"{output_dir_pattern}/{subdir}/{file_glob.replace('{model_var}', mv)}"
-                for mv in model_variables
-            )
-        else:
-            # Single file per frequency (atmosphere, sea-ice): all vars packed in
-            patterns.append(f"{output_dir_pattern}/{subdir}/{file_glob}")
+            else:
+                # Legacy single-file streams pack all variables by frequency.
+                patterns.append(f"{output_dir_pattern}/{subdir}/{file_glob}")
+    if not patterns and unsubstitutable_globs:
+        raise FileDiscoveryError(
+            "All patterns for frequency "
+            f"'{freq}' under component '{component}' require {{model_var}} "
+            "substitution but the mapping entry has no 'model_variables'. "
+            f"Patterns: {unsubstitutable_globs}"
+        )
     return patterns
 
 

src/access_moppy/mappings/ACCESS-ESM1-6_mappings.json

@@ -20,65 +20,69 @@
             "output_dir_pattern": "output[0-9][0-9][0-9]",
             "components": {
                 "aerosol": {
-                    "subdir": "atmosphere/netCDF",
+                    "subdir": ["atmosphere/netCDF", "atmosphere"],
                     "frequency_patterns": {
-                        "mon": "*.pa-*_mon.nc",
-                        "day": "*.pe-*_dai.nc",
-                        "3hr": "*.pi-*_3hr.nc",
-                        "6hr": "*.pj-*_6hr.nc"
+                        "mon": ["*.pa-*_mon.nc", "access-esm1p6.um7p3.*.{model_var}.1mon.*.*.nc"],
+                        "day": ["*.pe-*_dai.nc", "access-esm1p6.um7p3.*.{model_var}.1day.*.*.nc"],
+                        "3hr": ["*.pi-*_3hr.nc", "access-esm1p6.um7p3.*.{model_var}.3hr.*.*.nc"],
+                        "6hr": ["*.pj-*_6hr.nc", "access-esm1p6.um7p3.*.{model_var}.6hr.*.*.nc"]
                     }
                 },
                 "atmosphere": {
-                    "subdir": "atmosphere/netCDF",
+                    "subdir": ["atmosphere/netCDF", "atmosphere"],
                     "frequency_patterns": {
-                        "mon": "*.pa-*_mon.nc",
-                        "day": "*.pe-*_dai.nc",
-                        "3hr": "*.pi-*_3hr.nc",
-                        "6hr": "*.pj-*_6hr.nc",
-                        "1hr": "*.pc-*.nc",
-                        "subhr": "*.pc-*.nc",
+                        "mon": ["*.pa-*_mon.nc", "access-esm1p6.um7p3.*.{model_var}.1mon.*.*.nc"],
+                        "day": ["*.pe-*_dai.nc", "access-esm1p6.um7p3.*.{model_var}.1day.*.*.nc"],
+                        "3hr": ["*.pi-*_3hr.nc", "access-esm1p6.um7p3.*.{model_var}.3hr.*.*.nc"],
+                        "6hr": ["*.pj-*_6hr.nc", "access-esm1p6.um7p3.*.{model_var}.6hr.*.*.nc"],
+                        "1hr": ["*.pc-*.nc", "access-esm1p6.um7p3.*.{model_var}.1hr.*.*.nc"],
+                        "subhr": ["*.pc-*.nc", "access-esm1p6.um7p3.*.{model_var}.*min.*.*.nc"],
                         "fx": "*.pa-*_mon.nc"
                     }
                 },
                 "land": {
-                    "subdir": "atmosphere/netCDF",
+                    "subdir": ["atmosphere/netCDF", "atmosphere"],
                     "frequency_patterns": {
-                        "mon": "*.pa-*_mon.nc",
-                        "day": "*.pe-*_dai.nc",
+                        "mon": ["*.pa-*_mon.nc", "access-esm1p6.um7p3.*.{model_var}.1mon.*.*.nc"],
+                        "day": ["*.pe-*_dai.nc", "access-esm1p6.um7p3.*.{model_var}.1day.*.*.nc"],
                         "fx": "*.pa-*_mon.nc"
                     }
                 },
                 "landIce": {
-                    "subdir": "atmosphere/netCDF",
+                    "subdir": ["atmosphere/netCDF", "atmosphere"],
                     "frequency_patterns": {
-                        "mon": "*.pa-*_mon.nc"
+                        "mon": ["*.pa-*_mon.nc", "access-esm1p6.um7p3.*.{model_var}.1mon.*.*.nc"]
                     }
                 },
                 "sea_ice": {
                     "subdir": "ice",
                     "frequency_patterns": {
-                        "mon": "iceh-1monthly-mean_*.nc",
-                        "day": "iceh-1daily-mean_*.nc"
+                        "mon": ["iceh-1monthly-mean_*.nc", "access-esm1p6.cice5.*.{model_var}.1mon.*.*.nc"],
+                        "day": ["iceh-1daily-mean_*.nc", "access-esm1p6.cice5.*.{model_var}.1day.*.*.nc"]
                     }
                 },
                 "ocean": {
                     "subdir": "ocean",
                     "frequency_patterns": {
                         "mon": [
                             "ocean-*-{model_var}-1mon-mean-y_*.nc",
-                            "ocean-*-{model_var}-1monthly-*-ym_*.nc"
+                            "ocean-*-{model_var}-1monthly-*-ym_*.nc",
+                            "access-esm1p6.mom5.*.{model_var}.1mon.*.*.nc"
                         ],
                         "day": [
                             "ocean-*-{model_var}-1day-mean-y_*.nc",
-                            "ocean-*-{model_var}-1daily-*-ym_*.nc"
+                            "ocean-*-{model_var}-1daily-*-ym_*.nc",
+                            "access-esm1p6.mom5.*.{model_var}.1day.*.*.nc"
                         ],
                         "yr": [
                             "ocean-*-{model_var}-1yr-mean-y_*.nc",
-                            "ocean-*-{model_var}-1yearly-*-ym_*.nc"
+                            "ocean-*-{model_var}-1yearly-*-ym_*.nc",
+                            "access-esm1p6.mom5.*.{model_var}.1yr.*.*.nc"
                         ],
                         "fx": [
                             "ocean-*-{model_var}-fx.nc",
-                            "ocean-*-{model_var}.nc"
+                            "ocean-*-{model_var}.nc",
+                            "access-esm1p6.mom5.static.nc"
                         ]
                     }
                 },

tests/unit/test_file_discovery.py

@@ -47,6 +47,9 @@ class TestExtractYearFromPath:
             ("tos_mean_ocean_1mon_185001-185012.nc", 1850),
             ("wt_mean_ocean_1yr_234501-234512.nc", 2345),
             ("tas_mean_atm_1mon_200101-200112.nc", 2001),
+            ("access-esm1p6.um7p3.2d.fld_s03i261.1mon.mean.1850.nc", 1850),
+            ("access-esm1p6.cice5.2d.aice.1mon.mean.1850-02.nc", 1850),
+            ("access-esm1p6.mom5.2d.surface_temp.1mon.mean.1850.nc", 1850),
         ],
     )
     def test_known_patterns(self, filename, expected):
@@ -134,28 +137,27 @@ def setup_method(self):
         self.fd_cfg = mappings["model_info"]["file_discovery"]
 
     def test_atmosphere_monthly_no_model_var(self):
-        # Atmosphere: all variables packed in one file; no {model_var}
+        # Legacy atmosphere output is packed; the new convention is per-variable.
         var_entry = {"model_variables": ["fld_s30i297"]}
         patterns = _build_patterns(var_entry, "atmosphere", "mon", self.fd_cfg)
-        assert len(patterns) == 1
-        assert "{model_var}" not in patterns[0]
-        assert "*.pa-*_mon.nc" in patterns[0]
+        assert len(patterns) == 4
+        assert all("{model_var}" not in pattern for pattern in patterns)
+        assert any("*.pa-*_mon.nc" in pattern for pattern in patterns)
+        assert any(".fld_s30i297.1mon." in pattern for pattern in patterns)
 
     def test_ocean_monthly_one_model_var(self):
-        # The ocean "mon" frequency lists two naming conventions (legacy
-        # "1mon-mean-y_" and newer "1monthly-mean-ym_"), so one model variable
-        # yields one pattern per convention.
+        # One model variable yields one pattern for each supported convention.
         var_entry = {"model_variables": ["surface_temp"]}
         patterns = _build_patterns(var_entry, "ocean", "mon", self.fd_cfg)
-        assert len(patterns) == 2
+        assert len(patterns) == 3
         assert all("surface_temp" in p for p in patterns)
         assert all("{model_var}" not in p for p in patterns)
 
     def test_ocean_multi_model_vars_produces_multiple_patterns(self):
-        # Two model variables × two naming conventions per frequency.
+        # Two model variables times three naming conventions per frequency.
         var_entry = {"model_variables": ["ty_trans_rho", "ty_trans_rho_gm"]}
         patterns = _build_patterns(var_entry, "ocean", "mon", self.fd_cfg)
-        assert len(patterns) == 4
+        assert len(patterns) == 6
         assert all("{model_var}" not in p for p in patterns)
         assert any("ty_trans_rho_gm" in p for p in patterns)
 
@@ -177,14 +179,43 @@ def test_unknown_freq_raises(self):
 
     def test_ocean_missing_model_variables_raises(self):
         var_entry = {"model_variables": []}
-        with pytest.raises(FileDiscoveryError, match="no 'model_variables'"):
+        with pytest.raises(FileDiscoveryError, match="require \\{model_var\\}"):
             _build_patterns(var_entry, "ocean", "mon", self.fd_cfg)
 
+    def test_mixed_globs_skip_model_var_patterns_when_mapping_has_no_model_vars(self):
+        fd_cfg = {
+            "output_dir_pattern": "output[0-9][0-9][0-9]",
+            "components": {
+                "atmosphere": {
+                    "subdir": "atmosphere",
+                    "frequency_patterns": {
+                        "mon": [
+                            "*.pa-*_mon.nc",
+                            "access-esm1p6.um7p3.*.{model_var}.1mon.*.*.nc",
+                        ],
+                    },
+                },
+            },
+        }
+
+        patterns = _build_patterns({}, "atmosphere", "mon", fd_cfg)
+
+        assert patterns == ["output[0-9][0-9][0-9]/atmosphere/*.pa-*_mon.nc"]
+
+    def test_subhourly_uses_minute_frequency_token_for_new_access_layout(self):
+        patterns = _build_patterns(
+            {"model_variables": ["fld_s03i236"]}, "atmosphere", "subhr", self.fd_cfg
+        )
+
+        assert any(".fld_s03i236.*min." in pattern for pattern in patterns)
+        assert not any(".fld_s03i236.subhr." in pattern for pattern in patterns)
+
     def test_sea_ice_monthly_no_model_var(self):
         var_entry = {"model_variables": ["aice"]}
         patterns = _build_patterns(var_entry, "sea_ice", "mon", self.fd_cfg)
-        assert len(patterns) == 1
-        assert "iceh-1monthly-mean" in patterns[0]
+        assert len(patterns) == 2
+        assert any("iceh-1monthly-mean" in pattern for pattern in patterns)
+        assert any(".aice.1mon." in pattern for pattern in patterns)
 
 
 # ---------------------------------------------------------------------------
@@ -226,6 +257,29 @@ def test_atmosphere_monthly(self, tmp_path):
         # Daily file must NOT appear in a monthly query
         assert "aiihca.pe-185001_dai.nc" not in names
 
+    def test_atmosphere_monthly_legacy_and_per_variable(self, tmp_path):
+        archive = self._make_archive(
+            tmp_path,
+            [
+                ("output000/atmosphere/netCDF", "aiihca.pa-185001_mon.nc"),
+                (
+                    "output001/atmosphere",
+                    "access-esm1p6.um7p3.2d.fld_s03i261.1mon.mean.1851.nc",
+                ),
+                (
+                    "output001/atmosphere",
+                    "access-esm1p6.um7p3.2d.fld_s03i236.1mon.mean.1851.nc",
+                ),
+            ],
+        )
+
+        result = discover_files(archive, "Amon.tas", model_id="ACCESS-ESM1-6")
+
+        assert [path.name for path in result] == [
+            "aiihca.pa-185001_mon.nc",
+            "access-esm1p6.um7p3.2d.fld_s03i236.1mon.mean.1851.nc",
+        ]
+
     def test_ocean_monthly_per_variable(self, tmp_path):
         archive = self._make_archive(
             tmp_path,
@@ -245,6 +299,29 @@ def test_ocean_monthly_per_variable(self, tmp_path):
         # Different model variable should NOT be included
         assert "ocean-2d-eta_t-1mon-mean-y_1850.nc" not in names
 
+    def test_ocean_monthly_legacy_and_access_esm_layout(self, tmp_path):
+        archive = self._make_archive(
+            tmp_path,
+            [
+                ("output000/ocean", "ocean-2d-surface_temp-1mon-mean-y_1850.nc"),
+                (
+                    "output001/ocean",
+                    "access-esm1p6.mom5.2d.surface_temp.1mon.mean.1851.nc",
+                ),
+                (
+                    "output001/ocean",
+                    "access-esm1p6.mom5.2d.eta_t.1mon.mean.1851.nc",
+                ),
+            ],
+        )
+
+        result = discover_files(archive, "Omon.tos", model_id="ACCESS-ESM1-6")
+
+        assert [path.name for path in result] == [
+            "ocean-2d-surface_temp-1mon-mean-y_1850.nc",
+            "access-esm1p6.mom5.2d.surface_temp.1mon.mean.1851.nc",
+        ]
+
     def test_ocean_monthly_newer_naming_convention(self, tmp_path):
         # Newer experiments name ocean output "1monthly-mean-ym_YYYY_MM"
         # instead of the legacy "1mon-mean-y_YYYY".  Discovery must handle both
@@ -306,6 +383,7 @@ def test_ocean_fx_both_conventions_no_overmatch(self, tmp_path):
             [
                 ("output000/ocean", "ocean-2d-area_t.nc"),  # newer fx
                 ("output000/ocean", "ocean-2d-area_t-fx.nc"),  # legacy fx
+                ("output000/ocean", "access-esm1p6.mom5.static.nc"),
                 # time-varying — must NOT be treated as fx
                 ("output000/ocean", "ocean-2d-area_t-1monthly-mean-ym_0001_01.nc"),
             ],
@@ -320,6 +398,7 @@ def test_ocean_fx_both_conventions_no_overmatch(self, tmp_path):
             names.update(p.name for p in archive.glob(pat))
         assert "ocean-2d-area_t.nc" in names
         assert "ocean-2d-area_t-fx.nc" in names
+        assert "access-esm1p6.mom5.static.nc" in names
         assert "ocean-2d-area_t-1monthly-mean-ym_0001_01.nc" not in names
 
     def test_year_extraction_newer_convention(self, tmp_path):
@@ -786,6 +865,9 @@ class TestExtractYearMonthFromPath:
             ("wt_mean_ocean_1yr_234507-234512.nc", (2345, 7)),
             # Newer _YYYY_MM convention (spinup)
             ("ocean-2d-surface_temp-1monthly-mean-ym_0001_01.nc", (1, 1)),
+            # ACCESS component convention: hyphen-delimited datestamp
+            ("access-esm1p6.um7p3.2d.fld_s03i261.1mon.mean.1850.nc", (1850, None)),
+            ("access-esm1p6.cice5.2d.aice.1mon.mean.1850-02.nc", (1850, 2)),
         ],
     )
     def test_known_patterns(self, filename, expected):