diff --git a/deployment/aws/lambda_handler.py b/deployment/aws/lambda_handler.py index 9892b5c00..4bb98992e 100644 --- a/deployment/aws/lambda_handler.py +++ b/deployment/aws/lambda_handler.py @@ -81,7 +81,11 @@ block for the manifest, sourced from the ShardMap metadata by the orchestrator (matching the local dispatcher). Absent on flat runs. "times_us": [int, ...] (raster only, issue #264) -- the catalog-derived - time coordinate, int64 microseconds since the epoch; the orchestrator + time coordinate, in whatever encoding "config" declares: int64 + microseconds since the Unix epoch by default, or uint64 mortie toc + words under output.time_encoding: toc (spec §8, issue #443). Plain + ints either way; the worker re-derives the dtype from the config, so + the key name is historical and NOT a width claim. The orchestrator owns the global timestep index and threads it here so the template write needs no S3 access from the dispatcher. "run_manifest": dict (optional, issue #327) -- {"run_id", "shards" @@ -838,10 +842,13 @@ def _handle_setup(event: Dict[str, Any]) -> Dict[str, Any]: import numpy as np from zagg.processing.raster import emit_raster_template + from zagg.time_axis import time_axis_dtype, time_encoding store = open_store(event["store_path"], **_output_store_kwargs(event)) grid = from_config(config) - times_us = np.asarray(event["times_us"], dtype=np.int64) + # The wire carries plain ints; the cast is the config's declared + # time encoding (spec §8) — a toc word does not fit int64. + times_us = np.asarray(event["times_us"], dtype=time_axis_dtype(time_encoding(config))) if times_us.size == 0: # A zero-timestep template is degenerate: the arrays get a # 0-length time axis no worker can slab-write into. diff --git a/docs/hive_layout.md b/docs/hive_layout.md index cec5f2bff..9e2b9ac70 100644 --- a/docs/hive_layout.md +++ b/docs/hive_layout.md @@ -905,7 +905,10 @@ cap. The honest options today are: Raster (pull-NN) pipelines write the same tree with **windowed `(time, cells)` leaves**: one vanilla zarr v3 leaf per **(shard, window)** unit at `shard_leaf_path(root, shard, window=label)`, each carrying leaf-local `time` -(int64 microseconds, CF attrs) and `morton` (packed u64 words) as the sole cell +(int64 microseconds with CF attrs by default, or `uint64` mortie toc words +carrying the spec §8 `temporal` declaration and no CF attrs under +`output.time_encoding: toc` — which the shipped Sentinel-2 config sets, issue +#443) and `morton` (packed u64 words) as the sole cell coordinate — `cell_ids` (NESTED) rides only the `emit_cell_ids` transition hatch (issue #304) — plus one `(T_leaf, cells_per_shard)` array per configured band, chunked diff --git a/docs/specification.md b/docs/specification.md index fe1641b92..933d249bc 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -2,8 +2,8 @@ This page is the **normative record** of zagg's byte-level store conventions: the ragged vlen-bytes layout, the t-digest payload bytes, the packed -composition word, the pyramid/overview declarations, and the O11 content-hash -recipe. It exists so an independent reader +composition word, the pyramid/overview declarations, the O11 content-hash +recipe, and the temporal declaration on a time coordinate. It exists so an independent reader ([moczarr](https://github.com/espg/moczarr)) can decode a zagg store from this page and the committed conformance fixtures alone — no zagg import, no reverse-engineering of `grids/base.py` @@ -55,7 +55,7 @@ text marked *informative* explains or motivates and binds nothing. explicit succession clause; readers add revisions, they never drop them. - The committed conformance fixtures (§7) are part of the contract: a reader implementation that reproduces the fixtures' expected decoded values and - content hashes conforms to §1–§3 and §5. zagg's own test suite asserts the + content hashes conforms to §1–§3, §5 and §8. zagg's own test suite asserts the same expectations (`tests/test_spec_conformance.py`), so the spec, the fixtures, and the shipping reader cannot drift apart silently. @@ -68,6 +68,7 @@ Contents: 5. [O11 content hashes](#5-o11-content-hashes) 6. [`zagg-ragged/2` — the typed `vlen-ndarray` revision](#6-zagg-ragged2) 7. [Conformance fixtures](#7-conformance-fixtures) +8. [`zagg-toc/1` — the temporal declaration](#8-zagg-toc1) --- @@ -1397,8 +1398,8 @@ stores may deliberately stay `/1` for vanilla-zarr openability). **Status: contract.** The committed stores under [`tests/data/spec/`](https://github.com/englacial/zagg/tree/main/tests/data/spec) are part of this specification: a reader implementation that reproduces -their expected decoded values and content hashes conforms to §1–§3 and §5. -They are generated by +their expected decoded values and content hashes conforms to §1–§3, §5 and +§8. They are generated by [`tools/generate_spec_fixtures.py`](https://github.com/englacial/zagg/blob/main/tools/generate_spec_fixtures.py) through zagg's **production write path** (manifest, sharded leaf template, dense + ragged writes, coverage sidecar, commit stamp), so writer↔spec @@ -1406,10 +1407,11 @@ drift fails zagg's own suite (`tests/test_spec_conformance.py`) on whichever side moved. moczarr vendors the same fixtures for its parity gates (espg/moczarr#19/#20). -Four tiny single-shard hive stores plus one manifest-only declaration, all +Five tiny single-shard hive stores plus one manifest-only declaration, all on the same deliberately small geometry — shard order 4, inner-chunk order 5, cell order 6 (16 cells, K = 4 inner chunks of 4 cells), sharded (the -hive default): +hive default; `raster_toc/` is the one exception — a `(time, cells)` +product is never sharded, §8/#247): - **`minimal/`** — one *unlocated* digest field (`h_tdigest`) plus `count`. The smallest thing that is a conforming store. @@ -1461,6 +1463,20 @@ hive default): arrives with the sweep-side fixtures of [#384](https://github.com/englacial/zagg/issues/384). +- **`raster_toc/`** — the §8 temporal declaration surface: one raster + `(time, cells)` hive leaf whose `time` coordinate is `uint64` toc words + carrying `temporal: {"spec": "zagg-toc/1", "shape": "coordinate", …}` and no CF + `units`/`calendar` attrs, beside `morton` and two band arrays. Its axis + mixes both word variants deliberately — one single-item timestep encoded + as an exact **timestamp** word and two multi-item acquisition groups + encoded as **range** words — so a reader that implements only one variant + fails a §7 fixture. `raster_toc.expected.json` records the words as + decimal strings (JSON numbers cannot carry `uint64` faithfully) together + with the `(start, end)` nanoseconds a conforming decode yields and the + real acquisition spans they must contain, which is the §8 conservative + containment claim pinned on committed bytes. The other four fixtures, + which carry no `temporal` key anywhere, are the absent-key ⇒ legacy pin. + `minimal/` and `kitchen_sink/` pin the layout edge cases a reader must handle (`column/`'s leaf is `minimal/`'s, so it pins them again): inner chunk ordinal 2 is **empty** (absent from the shard index — the §1.5 sentinel, and @@ -1506,9 +1522,9 @@ itself on both sides, which is also the only mechanism that catches a future zagg↔moczarr divergence (neither side's fixture can: espg/moczarr#23). **Conformance criteria for an external reader**: decode every ragged array -per §1–§2 and the composition array per §3, reproducing the expected -decoded values exactly (byte-exact float32/uint64 — no tolerance), and -reproduce `content_hashes` per §5. zagg's own suite additionally decodes +per §1–§2, the composition array per §3, and the declared time coordinate +per §8, reproducing the expected decoded values exactly (byte-exact +float32/uint64 — no tolerance), and reproduce `content_hashes` per §5. zagg's own suite additionally decodes the shard objects with **spec-text-only** decoders (struct + zstd, no zagg read path) to prove the byte recipes in §1.4/§1.5 are sufficient on their own. @@ -1521,3 +1537,175 @@ principle as §5). In the committed D20 sidecar `cells_with_data` are pinned: `timestamp`, `zagg_version`, `run_id` and the run counters are **informative provenance**, they churn on every regeneration, and conformance never asserts them. + +--- + +## 8. `zagg-toc/1` + +**Status: contract** ([issue #443](https://github.com/englacial/zagg/issues/443) +— the first shape of the temporal series, +[#410](https://github.com/englacial/zagg/issues/410)). + +An array whose elements are **packed words** rather than self-describing +scalars carries a **word-typed coordinate declaration**: a spec-owned attrs +block, keyed by its domain, holding `{spec, shape, grammar}` — the +convention revision, *where the words sit* relative to the store's cells, +and *which word grammar* the values follow. The key is spec-owned: the +writer stamps it, never author-transcribed (§1.2's reserved-key discipline, +extended to coordinate arrays). The `shape` vocabulary is domain-neutral and +defined once, here: + +- **`"coordinate"`** — the declaring array **is** the coordinate variable of + a dimension (CF/xarray sense): one word per index along that dimension, + row-aligned with every array sharing it. +- **`"per-cell"`** — one word per cell of the store's cell grid, aligned + with the `morton` coordinate. +- **`"per-centroid"`** — one word per centroid inside a cell's ragged + payload, aligned element-for-element with the §1/§2 digest it accompanies. + +This section instantiates that pattern for the **temporal** domain: attrs key +`temporal`, `spec: "zagg-toc/1"`, `grammar: "mortie-toc/1"`. + +**An absent `temporal` key MUST be read as the legacy encoding** — signed +`int64` microseconds since `1970-01-01T00:00:00` UTC, self-described by the +CF `units`/`calendar` attrs the writer stamps beside it. Every store written +before this revision is conformant verbatim, no byte rewritten, and a reader +MUST NOT refuse a store for lacking the declaration. + +```json +"temporal": { + "spec": "zagg-toc/1", + "shape": "coordinate", + "grammar": "mortie-toc/1" +} +``` + +Three keys, and deliberately no more — +[ruled on #410](https://github.com/englacial/zagg/issues/410#issuecomment-5310533396): +the declaration is `{shape, grammar revision}` under the `spec` marker, with +**no per-store epoch or quantization guards**. + +- **`spec`** — the convention revision. Readers MUST strict-check it: an + unknown or future revision raises, never half-parses under a guessed + layout. +- **`shape`** — the vocabulary value above. **`"coordinate"`** is the only + one this revision defines for `temporal`: the declaring array **is** the + time coordinate of a `(time, cells)` product — one word per timestep, + row-aligned with the leading axis of every `(time, cells)` array in the + same group. A reader MUST refuse a `shape` it does not implement. +- **`grammar`** — the word grammar the values follow, named as a **grammar + revision** in the `{name}/{major}` style this document uses throughout + (`zagg-ragged/1`, `morton-hive/2`). It is a fixed token of this revision — + not a documentation URL, and not a stamp of the writer's installed mortie + — so a store's bytes move neither when a dependency floor moves nor when + the documentation that describes the grammar moves. The pointer to that + documentation is prose (below), which is where it can be updated freely. A + reader MUST refuse a `grammar` it does not implement, and SHOULD record it + as what it decoded against. + +**Forward instantiations** (informative, both +[#410](https://github.com/englacial/zagg/issues/410)). The temporal +companions — one word per cell, one word per centroid — arrive as the +`"per-cell"` and `"per-centroid"` shapes under *this* `spec` and *this* +grammar. The **located** (spatial) companion family declares under the same +pattern with `grammar: "mortie-morton/1"`, landing with the #410 kernel PR. +Neither re-declares the store's **primary** morton axis: that surface stays +declared by the `morton-hive/{1,2}` manifest grammar (`morton_hive.json`, +`docs/hive_layout.md`) and the store's [DGGS-convention](https://github.com/zarr-conventions/dggs) +`dggs` attrs (`docs/morton_arrow.md`). This pattern is for word-typed +coordinate and companion arrays *beyond* that primary surface, never a +second, competing declaration of it. + +**The epoch, the timescale, and the range variant's rounding quanta are +properties of the cited grammar and are deliberately NOT echoed here.** A +store cannot re-base them — a differing origin would be a different grammar, +which is precisely what `grammar` (and `spec` above it) already discriminates +— so a per-store copy could only ever restate a constant, while entering the +committed conformance bytes and the §5 content hash and forcing a fixture +regeneration for any upstream constant that changed. + +A writer MAY add further **informative** keys to the block (source-time +lineage and the like). They are non-normative: nothing in §8 is decoded from +them, and a reader MUST ignore keys it does not recognize rather than refuse +the store. + +### The word grammar is mortie's + +A `zagg-toc/1` value is a mortie **toc word** (temporal order coverage): one +`uint64` that is a tagged union of an exact nanosecond **timestamp** and an +outward-rounded, conservative **range**, on a continuous, leap-free, +GPS-aligned timescale with a fixed `1850-01-01T00:00:00` epoch. The bit +layout, the flag position, the unsigned sort order, and the semilattice +merge law are normative in mortie's +[`mortie.toc` reference](https://espg.github.io/mortie/0.9.6/api/toc/) and +its decision ledger +([espg/mortie#175](https://github.com/espg/mortie/issues/175)), and are +**not restated here**. What follows is zagg's half of the contract. + +The stored token is the grammar revision `mortie-toc/1`; the **URL above is +the documentation pointer**, and it lives in this prose precisely so it can +be re-pointed without moving a store byte. It is **release-pinned +deliberately**: mortie's documentation is `mike`-versioned and the +unversioned `/api/toc/` path was never published, so only a versioned URL +resolves; the pin is `0.9.6`, the earliest release carrying the reference, +and the page's normative words are unchanged through the current dependency +floor. + +**It is also not yet the same class of citation as §2.2's**, and §8 does not +claim it is. §2.2 defers the morton word's layout to mortie's +`docs/specification.md` §1/§4 — sections that document's §10 *"Frozen for +1.x"* enumerates as immutable within the major version. Mortie's frozen +specification contains **no toc section**: the grammar above is normative as +a module reference pinned by mortie's own golden fixtures, which is a weaker +guarantee. [espg/mortie#193](https://github.com/espg/mortie/issues/193) +tracks adding the frozen section; when it lands, this pointer swaps to it, +the stored `grammar` token does not move, and nothing else in §8 changes. + +### 8.1 `shape: "coordinate"` + +**Contract.** + +- The declaring array's element type is `uint64`, one word per timestep. +- A timestep whose real acquisition is a single instant MUST be encoded as a + **timestamp** word, exact to the nanosecond; a timestep covering a real + interval MUST be encoded as a **range** word whose envelope conservatively + contains that interval. A conforming writer therefore never widens an + instant into a range, and never narrows a real interval into an instant. +- **Row order is the acquisition-group order**: timesteps are ordered by the + group's **earliest member observation time** — the order the time axis has + always had, the order the `(time, cells)` slabs were indexed by, and + identical under both encodings, so a row assignment never drifts with the + encoding. +- **Stored word order is not that key, and a reader MUST NOT assume the + stored words ascend.** Unsigned word order is order by the *encoded* start, + which is the conservative envelope start — and an envelope may begin before + its group's earliest member observation time, by an amount nothing in this + section bounds. Where it does, the word leads the row key and the stored + axis is materially out of order: `np.sort(words)` yields ascending + *envelope-start* order, which is **not guaranteed to equal row order**. In + particular a reader MUST NOT bisect the stored axis to resolve a time + window; use the overlap predicate below, which is correct regardless of + stored order. +- Decoding to wall time yields **`(start, end)`** per timestep: for a + timestamp both bounds are its exact instant; for a range `end` is the + envelope's **exclusive** upper bound. A reader that must present one + instant per timestep SHOULD present `start`, and MUST NOT present a + midpoint as if it were the observation time — the midpoint of a + conservative envelope is not an observation. +- Temporal window selection is the grammar's overlap predicate applied to + the stored words directly: it over-reports by at most one quantum at a + window edge and **never under-reports**, so a selection is a conservative + superset of the timesteps whose real acquisition intersects the window. + +**Composition.** Time axes compose only between **matching declarations** — +`zagg-toc/1` with `zagg-toc/1`, legacy with legacy (an absent key is legacy +for this rule too). A concatenation across encodings would produce an axis +whose values mean two different things, so a reader or writer joining two +stores' time axes MUST refuse a mismatch. This is a *join* rule only: +reading either store on its own is always legal. + +**What this revision does not cover** (informative). Only +`shape: "coordinate"` has a contract here; the `"per-cell"` and +`"per-centroid"` temporal companions are the forward instantiations noted +above and carry none of §8.1's clauses yet. Nothing in §8 constrains the +`(time, cells)` band arrays themselves, which are unchanged. diff --git a/src/zagg/config.py b/src/zagg/config.py index b50442ed1..01ea7d05f 100644 --- a/src/zagg/config.py +++ b/src/zagg/config.py @@ -263,6 +263,13 @@ def validate_config(config: PipelineConfig) -> None: # validated before the kind branch, like credentials_provider above. _validate_worker(config) + # The time coordinate's encoding (spec §8, issue #443) is checked on every + # pipeline kind, like the two blocks above: the value is only meaningful + # where a time coordinate exists, so declaring it elsewhere is a typo + # worth catching at submission rather than a knob that silently does + # nothing. + _validate_time_encoding(config) + ptype = get_pipeline_type(config) if ptype != "spatial": _validate_temporal_config(config) @@ -761,6 +768,24 @@ def _validate_temporal_config(config: PipelineConfig) -> None: _validate_collection_options(config) +def _validate_time_encoding(config: PipelineConfig) -> None: + """Validate ``output.time_encoding`` (spec §8, issue #443). + + The knob declares how the ``(time, cells)`` time coordinate is encoded — + legacy int64 microseconds (absent/default) or mortie toc words. It is + raster-only: the spatial and temporal pipelines write no time coordinate, + so a declaration there would be inert. + """ + from zagg.time_axis import DEFAULT_TIME_ENCODING, time_encoding + + encoding = time_encoding(config) + if encoding != DEFAULT_TIME_ENCODING and (config.data_source or {}).get("reader") != "raster": + raise ValueError( + f"output.time_encoding: {encoding!r} applies to raster (time, cells) " + f"products only — this pipeline writes no time coordinate (spec §8)" + ) + + def _validate_raster_config(config: PipelineConfig) -> None: """Validate a raster (pull-NN) pipeline config (issue #218). diff --git a/src/zagg/configs/sentinel2_l2a.yaml b/src/zagg/configs/sentinel2_l2a.yaml index 6424d1572..7a371c22e 100644 --- a/src/zagg/configs/sentinel2_l2a.yaml +++ b/src/zagg/configs/sentinel2_l2a.yaml @@ -5,7 +5,7 @@ # cell-ID joins are direct) takes the source pixel nearest its center -- # dense by construction, exact uint16 DNs. Reflectance = DN * 1e-4 - 0.1, # recorded as CF scale_factor/add_offset attrs and never applied to the -# stored data. Timesteps are datatakes (time_key), not item datetimes: +# stored data. Timesteps are datatakes (time_key, toc-encoded), not item datetimes: # adjacent MGRS tiles of one datatake are a single (time, cells) slab, with # the 9,840 m tile overlap resolved by nearest-tile-center ownership. # @@ -32,6 +32,12 @@ data_source: anonymous: true output: + # Time axis as mortie toc words (spec §8, issue #443): a datatake is a + # ~seconds-long acquisition whose adjacent MGRS tiles are sensed seconds + # apart, so the honest coordinate value is a conservative RANGE -- which + # int64 microseconds cannot state and a toc word can. Output-defining: a + # store born on toc is a different product from a legacy-axis one. + time_encoding: toc grid: type: healpix parent_order: 11 diff --git a/src/zagg/processing/raster.py b/src/zagg/processing/raster.py index d9111746f..fcc6cd2c9 100644 --- a/src/zagg/processing/raster.py +++ b/src/zagg/processing/raster.py @@ -31,6 +31,14 @@ import numpy as np +from zagg.time_axis import ( + DEFAULT_TIME_ENCODING, + encode_time_axis, + time_axis_attrs, + time_axis_dtype, + time_encoding, +) + logger = logging.getLogger(__name__) # TIFF SampleFormat x BitsPerSample -> numpy dtype, for sizing the fill return @@ -504,26 +512,50 @@ def _us_iso(us: int) -> str: ) -def raster_time_index(granules) -> tuple[dict, np.ndarray]: +def raster_time_index( + granules, *, encoding: str = DEFAULT_TIME_ENCODING +) -> tuple[dict, np.ndarray]: """Global timestep index from ShardMap granule lists. A timestep is an *acquisition group* — entries sharing a ``time_key`` (e.g. the Sentinel-2 datatake id; adjacent MGRS tiles of one datatake are items seconds apart) — falling back to the entry datetime when no key was - configured. The group's coordinate value is its earliest datetime. + configured. + + Under the default ``microseconds`` encoding the group's coordinate value + is its earliest datetime, as it always has been. Under ``toc`` (spec §8, + issue #443) it is the group's acquisition ENVELOPE — the earliest + ``time_start`` (or item datetime) to the latest ``time_end`` (or item + datetime) across its members — encoded as one toc word: an exact + timestamp where the envelope is a single instant, a conservative range + where it is not. A datatake's tiles are sensed seconds apart, which is + the range this encoding exists to state honestly. + + Row ORDER is the group's earliest item datetime either way, so a leaf's + row assignment cannot drift with the encoding. That is *not* the same as + ascending stored words under ``toc``: the word encodes the ENVELOPE + start, which a declared ``time_start`` can push before an earlier row's + key, so the returned words may descend (§8.1 — a reader must use the + overlap predicate, never a bisect). Parameters ---------- granules : list of list of dict - ``ShardMap.granules`` (raster entries carry ``assets`` + ``datetime``). + ``ShardMap.granules`` (raster entries carry ``assets`` + ``datetime``, + optionally ``time_start``/``time_end`` from the STAC item's + ``start_datetime``/``end_datetime``). + encoding : str + ``output.time_encoding`` (:func:`zagg.time_axis.time_encoding`). Returns ------- - (time_index, times_us) - ``{group_key: t_idx}`` and the int64 microseconds-since-epoch time - coordinate, both in ascending time order. + (time_index, times) + ``{group_key: t_idx}`` and the encoded time coordinate (int64 + microseconds, or uint64 toc words under ``toc``), both in row order + — ascending group earliest-item datetime. """ earliest: dict = {} + span: dict = {} for shard_entries in granules: for e in shard_entries: if not e.get("assets"): @@ -532,12 +564,22 @@ def raster_time_index(granules) -> tuple[dict, np.ndarray]: raise ValueError(f"raster granule entry {e.get('id')!r} carries no datetime") key = e.get("time_key") or e["datetime"] us = _iso_us(e["datetime"]) - if key not in earliest or us < earliest[key]: - earliest[key] = us + lo = _iso_us(e["time_start"]) if e.get("time_start") else us + hi = _iso_us(e["time_end"]) if e.get("time_end") else us + if key not in earliest: + earliest[key], span[key] = us, (lo, hi) + else: + earliest[key] = min(earliest[key], us) + span[key] = (min(span[key][0], lo), max(span[key][1], hi)) ordered = sorted(earliest, key=lambda k: (earliest[k], k)) time_index = {k: i for i, k in enumerate(ordered)} - times_us = np.array([earliest[k] for k in ordered], dtype=np.int64) - return time_index, times_us + if encoding == DEFAULT_TIME_ENCODING: + times = np.array([earliest[k] for k in ordered], dtype=np.int64) + else: + times = encode_time_axis( + [span[k][0] for k in ordered], [span[k][1] for k in ordered], encoding=encoding + ) + return time_index, times def _chord2(lons, lats, lon0: float, lat0: float) -> np.ndarray: @@ -832,8 +874,6 @@ def _combine_by_ownership(sampled, lonlat, bands): # resize-then-write-slab pattern and are single-writer (the runner owns the # resize, as it owns template emission). -_TIME_ATTRS = {"units": "microseconds since 1970-01-01T00:00:00", "calendar": "proleptic_gregorian"} - def _raster_array_spec(shape, chunks, dims, dtype, fill, attrs=None): """ArraySpec shared by the flat template and the hive leaf spec.""" @@ -880,12 +920,23 @@ def _raster_members(grid, config, n_time: int, n_cells: int) -> dict: review: one default cell coordinate everywhere). The legacy NESTED ``cell_ids`` array rides only the same ``emit_cell_ids`` transition hatch as the spatial path — never a separate schedule. + + ``time``'s element type and attrs come from the config's declared + encoding (spec §8): int64 microseconds with CF ``units``/``calendar`` + by default, uint64 toc words with the ``temporal`` declaration and no CF + pair under ``time_encoding: toc``. """ from zagg.config import get_raster_bands + encoding = time_encoding(config) members = { "time": _raster_array_spec( - (n_time,), (max(n_time, 1),), ("time",), "int64", 0, dict(_TIME_ATTRS) + (n_time,), + (max(n_time, 1),), + ("time",), + time_axis_dtype(encoding), + 0, + time_axis_attrs(encoding), ), "morton": _raster_array_spec((n_cells,), (grid.cells_per_chunk,), ("cells",), "uint64", 0), } @@ -931,7 +982,7 @@ def emit_raster_template(store, grid, config, times_us: np.ndarray, *, overwrite from zarr import open_array from zarr.errors import ArrayNotFoundError, ContainsGroupError - times_us = np.asarray(times_us, dtype=np.int64) + times_us = np.asarray(times_us, dtype=time_axis_dtype(time_encoding(config))) spec = raster_group_spec(grid, config, int(len(times_us))) time_path = f"{grid.group_path}/time" with zarr_config.set({"async.concurrency": 128}): @@ -1014,7 +1065,7 @@ def emit_raster_leaf_template( with zarr_config.set({"async.concurrency": 128}): spec.to_zarr(store, "", overwrite=overwrite) arr = open_array(store, path=f"{grid.group_path}/time", zarr_format=3, consolidated=False) - times = np.asarray(times_us, dtype=np.int64) + times = np.asarray(times_us, dtype=time_axis_dtype(time_encoding(config))) arr[:] = times arr = open_array(store, path=f"{grid.group_path}/morton", zarr_format=3, consolidated=False) arr[:] = children @@ -1313,7 +1364,7 @@ def process_and_write_raster_hive( # The leaf's own time axis, from the dispatched subset. Every group key in # the subset is in this index by construction, so the worker never trips # the foreign-manifest guard. - time_index, times_us = raster_time_index([granules]) + time_index, times_us = raster_time_index([granules], encoding=time_encoding(config)) box: dict = {} write_s = 0.0 # O11 incremental hashing (issue #342 phase 5). The raster leaf never diff --git a/src/zagg/readers/__init__.py b/src/zagg/readers/__init__.py index db6597ad5..206aca152 100644 --- a/src/zagg/readers/__init__.py +++ b/src/zagg/readers/__init__.py @@ -19,9 +19,16 @@ read_tensors, ) +# The raster ``(time, cells)`` family's time coordinate (spec §8, issue +# #443). Re-exported rather than reimplemented: the encode and decode halves +# of one wire contract belong in one module (``zagg.time_axis``), and the +# writer imports the same constants. +from zagg.time_axis import decode_time_axis, read_time_axis, time_axis_overlaps + __all__ = [ "cell_index", "chunk_z_range", + "decode_time_axis", "has_exact_occupancy", "rank_to_rowcol", "rasterize_cell", @@ -29,5 +36,7 @@ "read_locations", "read_raw_values", "read_tensors", + "read_time_axis", "rowcol_to_rank", + "time_axis_overlaps", ] diff --git a/src/zagg/runner.py b/src/zagg/runner.py index 775ff7015..0d9932aa5 100644 --- a/src/zagg/runner.py +++ b/src/zagg/runner.py @@ -799,6 +799,7 @@ def run( write_raster_slab, ) from zagg.telemetry import build_record, raster_granule_ids, write_sidecar + from zagg.time_axis import time_encoding catalog_path = catalog or config.catalog if not catalog_path: @@ -823,7 +824,9 @@ def run( grid = from_config(config) _check_signature(grid, catalog_data) - time_index, times_us = raster_time_index(catalog_data["granules"]) + time_index, times_us = raster_time_index( + catalog_data["granules"], encoding=time_encoding(config) + ) if not time_index: raise ValueError("catalog carries no raster granule entries (no assets/datetime)") @@ -5099,8 +5102,11 @@ def _invoke_lambda_raster_setup( deliberately invoke-only and must never PUT to the store. Synchronous because the template is load-bearing before fan-out (workers write slabs into its arrays) — the flat point-path lifecycle, not the #252 async - hybrid. ``times_us`` is the catalog-derived global time coordinate - (int64 μs since the epoch), JSON-safe as a plain int list. + hybrid. ``times_us`` is the catalog-derived global time coordinate in the + config's declared encoding — int64 μs since the Unix epoch by default, or + uint64 mortie toc words under ``output.time_encoding: toc`` (spec §8) — + JSON-safe as a plain int list either way; the worker re-derives the dtype + from ``config``, so the parameter name is historical, not a width claim. A deployed function that predates the raster setup branch would fall through to the point-path ``grid.emit_template`` — wrong template, right @@ -5115,8 +5121,9 @@ def _invoke_lambda_raster_setup( "overwrite": overwrite, "config": config_dict, # times_us rides inline in this sync RequestResponse payload (6 MB - # cap, not the 256 KB async/Event cap); int64 μs stamps serialize at - # ~17 B each → ~350K-timestep headroom, orders above any real catalog + # cap, not the 256 KB async/Event cap); the widest stamp is a 20-digit + # uint64 toc word at ~21 B → ~285K-timestep headroom (int64 μs stamps + # are ~17 B / ~350K), both orders above any real catalog # (the pinned NEON benchmark catalog is 85 items). No chunking # fallback — a pathological series would surface as a boto ClientError. "times_us": [int(t) for t in times_us], diff --git a/src/zagg/semantics.py b/src/zagg/semantics.py index b8b4287b9..97c7d8ee5 100644 --- a/src/zagg/semantics.py +++ b/src/zagg/semantics.py @@ -24,7 +24,11 @@ - the **pipeline type** (``spatial`` | ``temporal`` | ``event``; absent normalizes to ``spatial`` — espg-ruled on the PR #316 review: a temporal engine over the same aggregation block is a different product; D19's - ratified list omitted it only because the temporal path wasn't in frame). + ratified list omitted it only because the temporal path wasn't in frame); +- the raster **time-coordinate encoding** (``output.time_encoding``, spec §8 / + issue #443), keyed only when non-default: toc words and legacy microseconds + are different stored values meaning different things, exactly as + ``weights: "flux"`` is. Excluded as packaging: all orders (``parent_order``/``child_order``/ ``chunk_inner``), ``sharded``, store layout/path, ``emit_cell_ids`` (the @@ -55,6 +59,7 @@ import json from zagg.config import PipelineConfig, get_pipeline_type +from zagg.time_axis import DEFAULT_TIME_ENCODING #: ``data_source`` keys that are read machinery, not output semantics (D19). #: Changing any of these must never change the ``semantic_hash``. @@ -258,6 +263,14 @@ def semantic_core(config: PipelineConfig) -> dict: # every existing config hashes stably. "pipeline": {"type": get_pipeline_type(config)}, } + # The time coordinate's encoding (spec §8, issue #443) is output-defining + # for the same reason `weights: "flux"` is: the stored axis MEANS + # something else. Keyed only when non-default, so every config written + # before §8 — and the explicit `microseconds` spelling of the absent-key + # default — hashes byte-identically to today. + encoding = (config.output or {}).get("time_encoding") + if encoding not in (None, DEFAULT_TIME_ENCODING): + core["time_encoding"] = encoding return _prune_nulls(core) diff --git a/src/zagg/time_axis.py b/src/zagg/time_axis.py new file mode 100644 index 000000000..402c4b4f3 --- /dev/null +++ b/src/zagg/time_axis.py @@ -0,0 +1,246 @@ +"""The time coordinate's encoding contract (spec §8, issues #443/#410). + +A raster ``(time, cells)`` product's ``time`` array carries one value per +acquisition group. Two encodings are defined: + +- ``"microseconds"`` (the default, and every store written before §8) — + signed ``int64`` microseconds since the Unix epoch, self-described by the + CF ``units``/``calendar`` attrs in :data:`LEGACY_TIME_ATTRS`; +- ``"toc"`` — mortie **toc words** (``uint64``), a tagged union of an exact + nanosecond timestamp and an outward-rounded conservative range, declared + by the spec-owned ``temporal`` attrs block this module stamps and reads. + +The point of the second one is honesty: a Sentinel-2 datatake is a +~seconds-long acquisition whose adjacent MGRS tiles are sensed seconds +apart, which ``datetime64`` cannot represent and a toc range can. Word +semantics — bit layout, sort order, merge law — are mortie's +(``mortie.toc``), never restated here; this module owns zagg's half: the +declaration grammar, the acquisition-group mapping, and the decode. +""" + +from __future__ import annotations + +import numpy as np + +#: The spec §8 attrs key on a time-carrying array. Spec-owned: writers stamp +#: it from the config's declared encoding, config authors never write it. +TEMPORAL_ATTR = "temporal" +#: The §8 convention revision, strict-checked on read. +TOC_SPEC = "zagg-toc/1" +#: The only §8 ``shape`` this revision defines: the declaring array IS the +#: CF/xarray coordinate variable of the time dimension. Per-cell / +#: per-centroid companions (#410) land as further values under the same +#: marker, from the same domain-neutral shape vocabulary. +TOC_SHAPE_COORDINATE = "coordinate" +#: The §8 word-grammar citation — a grammar REVISION token in the ecosystem's +#: {name}/{major} style (``zagg-ragged/1``, ``morton-hive/2``), never a +#: documentation URL or a stamp of the writer's installed mortie: store bytes +#: must not move when a floor moves or the documentation moves. §8's prose +#: carries the documentation pointer (the mike-versioned API page today, +#: mortie's frozen spec section once espg/mortie#193 lands). +TOC_GRAMMAR = "mortie-toc/1" +#: The cited grammar's time origin. A grammar property, not a declared key +#: (#410 ruled out per-store epoch/quantization guards); kept here only to +#: refuse a time the words cannot represent. +TOC_EPOCH = "1850-01-01T00:00:00" +#: The legacy encoding's CF attrs, stamped only when ``temporal`` is absent. +LEGACY_TIME_ATTRS = { + "units": "microseconds since 1970-01-01T00:00:00", + "calendar": "proleptic_gregorian", +} +#: Config values for ``output.time_encoding``. +TIME_ENCODINGS = ("microseconds", "toc") +#: The absent-key default: what an undeclared axis means, both in config and +#: in stored attrs. +DEFAULT_TIME_ENCODING = "microseconds" + +__all__ = [ + "DEFAULT_TIME_ENCODING", + "LEGACY_TIME_ATTRS", + "TEMPORAL_ATTR", + "TIME_ENCODINGS", + "TOC_EPOCH", + "TOC_GRAMMAR", + "TOC_SHAPE_COORDINATE", + "TOC_SPEC", + "decode_time_axis", + "encode_time_axis", + "read_time_axis", + "temporal_declaration", + "time_axis_attrs", + "time_axis_dtype", + "time_axis_overlaps", + "time_encoding", +] + + +def time_encoding(config) -> str: + """The config's declared ``output.time_encoding`` (§8), validated. + + Absent is :data:`DEFAULT_TIME_ENCODING` — the legacy encoding, so every + existing config resolves exactly as before. + """ + value = (getattr(config, "output", None) or {}).get("time_encoding") + if value is None: + return DEFAULT_TIME_ENCODING + if value not in TIME_ENCODINGS: + raise ValueError( + f"output.time_encoding must be one of {TIME_ENCODINGS} (got {value!r}) — spec §8" + ) + return str(value) + + +def time_axis_dtype(encoding: str) -> str: + """The stored element type for a declared encoding.""" + return "uint64" if encoding == "toc" else "int64" + + +def time_axis_attrs(encoding: str) -> dict: + """The attrs a writer stamps on the time array for ``encoding``. + + The legacy encoding keeps its CF pair and declares nothing (§8: an absent + ``temporal`` key IS the legacy encoding); ``toc`` carries the declaration + and no CF attrs, because ``units``/``calendar`` would describe the words + wrongly and a CF-decoding client would silently produce garbage dates. + """ + if encoding != "toc": + return dict(LEGACY_TIME_ATTRS) + return { + TEMPORAL_ATTR: { + "spec": TOC_SPEC, + "shape": TOC_SHAPE_COORDINATE, + "grammar": TOC_GRAMMAR, + } + } + + +def temporal_declaration(attrs) -> dict | None: + """The §8 ``temporal`` block from an array's attrs, strict-checked. + + Returns ``None`` for the legacy encoding (absent key) — never raises for + a store written before §8, which is the schema-evolution rule. Raises on + a declaration this revision cannot decode: an unknown ``spec``, an + unimplemented ``shape``, or an uncited word ``grammar``. Unrecognized + keys are informative by §8 and ignored, never a refusal. + """ + block = dict(attrs or {}).get(TEMPORAL_ATTR) + if block is None: + return None + if not isinstance(block, dict): + raise ValueError(f"{TEMPORAL_ATTR!r} attrs must be a mapping (got {block!r}) — spec §8") + spec = block.get("spec") + if spec != TOC_SPEC: + raise ValueError( + f"unknown temporal declaration spec {spec!r} (spec §8 defines {TOC_SPEC!r}); " + f"refusing to guess a future revision's time encoding" + ) + shape = block.get("shape") + if shape != TOC_SHAPE_COORDINATE: + raise ValueError( + f"temporal declaration shape {shape!r} is not implemented " + f"(spec §8 defines {TOC_SHAPE_COORDINATE!r} for a time coordinate)" + ) + grammar = block.get("grammar") + if grammar != TOC_GRAMMAR: + raise ValueError( + f"temporal declaration cites word grammar {grammar!r}, not {TOC_GRAMMAR!r} " + f"(spec §8); refusing to decode words under a grammar this reader does not " + f"implement" + ) + return block + + +def _internal_ns(when: np.ndarray) -> np.ndarray: + """UTC ``datetime64`` -> mortie's internal ns on the continuous scale.""" + import mortie + + if when.size and when.min() < np.datetime64(TOC_EPOCH, "us"): + raise ValueError( + f"time {when.min()} precedes the toc epoch {TOC_EPOCH} — the word " + f"grammar cannot represent it (spec §8)" + ) + return np.asarray(mortie.from_datetime64(when), dtype="uint64") + + +def encode_time_axis(starts_us, ends_us, *, encoding: str) -> np.ndarray: + """Encode acquisition envelopes as the stored time-axis values. + + ``starts_us``/``ends_us`` are the group's earliest and latest member + times in microseconds since the Unix epoch (equal for a single-instant + acquisition). Under the legacy encoding only ``starts_us`` is stored; + under ``toc`` a degenerate envelope becomes an exact **timestamp** word + and a real interval an outward-rounded **range** word, so the writer + neither widens an instant nor narrows an interval (§8.1). + """ + starts = np.asarray(starts_us, dtype="int64") + if encoding != "toc": + return starts + import mortie + + ends = np.asarray(ends_us, dtype="int64") + if starts.shape != ends.shape: + raise ValueError(f"time-axis starts {starts.shape} and ends {ends.shape} differ in shape") + if starts.size and (ends < starts).any(): + raise ValueError("time-axis envelope ends before it starts") + s_ns = _internal_ns(starts.astype("datetime64[us]")) + e_ns = _internal_ns(ends.astype("datetime64[us]")) + words = np.asarray(mortie.span2toc(s_ns, e_ns), dtype="uint64") + instant = s_ns == e_ns + if instant.any(): + words[instant] = np.asarray(mortie.time2toc(s_ns[instant]), dtype="uint64") + return words + + +def decode_time_axis(values, attrs) -> tuple[np.ndarray, np.ndarray]: + """Decode a stored time axis to ``(start, end)`` ``datetime64[ns]``. + + Both bounds are the exact instant for a legacy value or a toc timestamp + word; for a toc range word ``end`` is the envelope's **exclusive** upper + bound (§8.1). The pair — rather than one instant — is the return shape + because the envelope's midpoint is not an observation, and because the + reader surface is numpy tuples by convention (D13). + """ + if temporal_declaration(attrs) is None: + when = np.asarray(values, dtype="int64").astype("datetime64[us]").astype("datetime64[ns]") + return when, when + import mortie + + words = np.asarray(values, dtype="uint64") + start_ns, end_ns = mortie.toc2time(words) + return mortie.to_datetime64(start_ns), mortie.to_datetime64(end_ns) + + +def time_axis_overlaps(values, attrs, start, end) -> np.ndarray: + """Boolean mask of timesteps intersecting the half-open window. + + ``start``/``end`` are anything ``numpy.datetime64`` accepts (ISO strings, + ``datetime64``, ``datetime``). A toc axis is tested by the grammar's own + predicate on the stored words — no decode, and the answer is a + conservative superset (over-reports by at most one quantum at an edge, + never under-reports, §8.1). A legacy axis compares its instants directly. + """ + q_start, q_end = np.datetime64(start, "us"), np.datetime64(end, "us") + if q_end < q_start: + raise ValueError(f"time window is inverted ({q_start} .. {q_end})") + if temporal_declaration(attrs) is None: + when = np.asarray(values, dtype="int64").astype("datetime64[us]") + return (when >= q_start) & (when < q_end) + import mortie + + words = np.asarray(values, dtype="uint64") + bounds = _internal_ns(np.array([q_start, q_end], dtype="datetime64[us]")) + return np.asarray(mortie.toc_overlaps(words, int(bounds[0]), int(bounds[1])), dtype=bool) + + +def read_time_axis(store, group_path: str = "") -> tuple[np.ndarray, np.ndarray]: + """Open a product's ``time`` coordinate and decode it (§8). + + Returns the same ``(start, end)`` ``datetime64[ns]`` pair as + :func:`decode_time_axis`, resolving the encoding from the array's own + attrs — so a caller reads legacy and toc stores through one code path. + """ + from zarr import open_array + + path = f"{group_path}/time" if group_path else "time" + arr = open_array(store, path=path, zarr_format=3, consolidated=False) + return decode_time_axis(arr[:], dict(arr.attrs)) diff --git a/tests/data/spec/raster_toc.expected.json b/tests/data/spec/raster_toc.expected.json new file mode 100644 index 000000000..6c39423f1 --- /dev/null +++ b/tests/data/spec/raster_toc.expected.json @@ -0,0 +1,195 @@ +{ + "shard": "11213", + "leaf": "1/1/2/1/3/11213.zarr", + "group": "6", + "shard_order": 4, + "chunk_order": 5, + "cell_order": 6, + "cells_per_chunk": 4, + "cells_per_shard": 16, + "time_encoding": "toc", + "time_attrs": { + "temporal": { + "spec": "zagg-toc/1", + "shape": "coordinate", + "grammar": "mortie-toc/1" + } + }, + "time_words": [ + "11073651233525252499", + "11074169631782972753", + "11074688036670682112" + ], + "time_bounds_ns": [ + [ + "1749999998118054912", + "1750000011002956800" + ], + [ + "1750259197246884864", + "1750259212279270400" + ], + [ + "1750518400000000000", + "1750518400000000000" + ] + ], + "acquisitions": [ + { + "key": "dt-1", + "start": "2025-06-15T15:06:40", + "end": "2025-06-15T15:06:47" + }, + { + "key": "dt-2", + "start": "2025-06-18T15:06:38", + "end": "2025-06-18T15:06:49" + }, + { + "key": "dt-3", + "start": "2025-06-21T15:06:40", + "end": "2025-06-21T15:06:40" + } + ], + "morton": [ + "1233986297899515910", + "1234267772876226566", + "1234549247852937222", + "1234830722829647878", + "1235112197806358534", + "1235393672783069190", + "1235675147759779846", + "1235956622736490502", + "1236238097713201158", + "1236519572689911814", + "1236801047666622470", + "1237082522643333126", + "1237363997620043782", + "1237645472596754438", + "1237926947573465094", + "1238208422550175750" + ], + "bands": { + "red": [ + [ + 0, + 1001, + 1002, + 1003, + 1004, + 0, + 1006, + 1007, + 1008, + 1009, + 0, + 1011, + 1012, + 1013, + 1014, + 0 + ], + [ + 1037, + 0, + 1039, + 1040, + 1041, + 1042, + 0, + 1044, + 1045, + 1046, + 1047, + 0, + 1049, + 1050, + 1051, + 1052 + ], + [ + 1074, + 1075, + 0, + 1077, + 1078, + 1079, + 1080, + 0, + 1082, + 1083, + 1084, + 1085, + 0, + 1087, + 1088, + 1089 + ] + ], + "scl": [ + [ + 0, + 5, + 6, + 4, + 5, + 0, + 4, + 5, + 6, + 4, + 0, + 6, + 4, + 5, + 6, + 0 + ], + [ + 4, + 0, + 6, + 4, + 5, + 6, + 0, + 5, + 6, + 4, + 5, + 0, + 4, + 5, + 6, + 4 + ], + [ + 4, + 5, + 0, + 4, + 5, + 6, + 4, + 0, + 6, + 4, + 5, + 6, + 0, + 5, + 6, + 4 + ] + ] + }, + "content_hashes": { + "arrays": { + "6/morton": "e77df9192c988fcc15a35afefc3595732946b7a09f865eb87d637b2626a88b9b", + "6/red": "f9000f894c996f280e1a00801159b247b4dee761320c8ae078e285565e7ac605", + "6/scl": "7595bb826b7cabdc36c7a3fd2fa1279b2ea691760b8722984269565a1b6ad208", + "6/time": "551f7be5718086c2ca1d379e1d1581368d21e4ca1433ce3ae4ae397e11b08f7f" + }, + "combined": "42263e046ecf4d71de8460063b38e6f15522e245cdc2182ce3a0acaf35db7e4e" + } +} diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/0 new file mode 100644 index 000000000..fcba0d5e7 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/1 new file mode 100644 index 000000000..5b3d68d81 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/2 new file mode 100644 index 000000000..aa23e5420 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/3 new file mode 100644 index 000000000..ff2abf77f Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/c/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/zarr.json new file mode 100644 index 000000000..52753f985 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/morton/zarr.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 16 + ], + "data_type": "uint64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 4 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 3, + "checksum": false + } + } + ], + "attributes": {}, + "dimension_names": [ + "cells" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/0 new file mode 100644 index 000000000..b26f57f2e Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/1 new file mode 100644 index 000000000..a373e99e8 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/2 new file mode 100644 index 000000000..d31011bd0 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/3 new file mode 100644 index 000000000..430fe29ab Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/0/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/0 new file mode 100644 index 000000000..5a1664aa0 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/1 new file mode 100644 index 000000000..d8f7f45da Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/2 new file mode 100644 index 000000000..fe83d438b Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/3 new file mode 100644 index 000000000..c592d970b Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/1/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/0 new file mode 100644 index 000000000..b1e2a7fc5 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/1 new file mode 100644 index 000000000..6d76c3e16 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/2 new file mode 100644 index 000000000..3fa122bcc Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/3 new file mode 100644 index 000000000..e1a3166f4 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/c/2/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/zarr.json new file mode 100644 index 000000000..f60b00915 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/red/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 3, + 16 + ], + "data_type": "uint16", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 4 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 3, + "checksum": false + } + } + ], + "attributes": { + "scale_factor": 0.0001, + "add_offset": -0.1 + }, + "dimension_names": [ + "time", + "cells" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/0 new file mode 100644 index 000000000..e630b0cdd Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/1 new file mode 100644 index 000000000..27a24351c Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/2 new file mode 100644 index 000000000..2032e596b Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/3 new file mode 100644 index 000000000..0887ca07b Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/0/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/0 new file mode 100644 index 000000000..14db4a37a Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/1 new file mode 100644 index 000000000..1982d5e70 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/2 new file mode 100644 index 000000000..c87c8e6ff Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/3 new file mode 100644 index 000000000..2633dab85 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/1/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/0 new file mode 100644 index 000000000..bd9d7271f Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/1 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/1 new file mode 100644 index 000000000..9e2ee2d1f Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/1 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/2 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/2 new file mode 100644 index 000000000..b4212f2e0 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/2 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/3 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/3 new file mode 100644 index 000000000..e630b0cdd Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/c/2/3 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/zarr.json new file mode 100644 index 000000000..eec6b4728 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/scl/zarr.json @@ -0,0 +1,43 @@ +{ + "shape": [ + 3, + 16 + ], + "data_type": "uint8", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1, + 4 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes" + }, + { + "name": "zstd", + "configuration": { + "level": 3, + "checksum": false + } + } + ], + "attributes": {}, + "dimension_names": [ + "time", + "cells" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/c/0 b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/c/0 new file mode 100644 index 000000000..2f0390e19 Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/c/0 differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/zarr.json new file mode 100644 index 000000000..a136787e7 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/time/zarr.json @@ -0,0 +1,49 @@ +{ + "shape": [ + 3 + ], + "data_type": "uint64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 3 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 3, + "checksum": false + } + } + ], + "attributes": { + "temporal": { + "spec": "zagg-toc/1", + "shape": "coordinate", + "grammar": "mortie-toc/1" + } + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/zarr.json new file mode 100644 index 000000000..c87640057 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/6/zarr.json @@ -0,0 +1,34 @@ +{ + "attributes": { + "zarr_conventions": [ + { + "schema_url": "https://raw.githubusercontent.com/zarr-conventions/dggs/refs/tags/v1/schema.json", + "spec_url": "https://github.com/zarr-conventions/dggs/blob/v1/README.md", + "uuid": "7b255807-140c-42ca-97f6-7a1cfecdbc38", + "name": "dggs", + "description": "Discrete Global Grid Systems convention for zarr" + }, + { + "schema_url": "https://github.com/espg/mortie/blob/main/docs/specification.md#dggs-attrs", + "spec_url": "https://github.com/espg/mortie/blob/main/docs/specification.md", + "uuid": "3e22156d-ea9e-4e01-95fe-e3809a4b41e7", + "name": "morton-dggs", + "description": "Packed-u64 morton (HEALPix) DGGS convention" + } + ], + "dggs": { + "name": "morton", + "refinement_level": 6, + "spatial_dimension": "cells", + "ellipsoid": { + "name": "WGS84", + "semimajor_axis": 6378137.0, + "inverse_flattening": 298.257223563 + }, + "compression": "none", + "coordinate": "morton" + } + }, + "zarr_format": 3, + "node_type": "group" +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/coverage.moc b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/coverage.moc new file mode 100644 index 000000000..e58bcdcbe Binary files /dev/null and b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/coverage.moc differ diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/zarr.json b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/zarr.json new file mode 100644 index 000000000..6ccbbb3af --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/11213.zarr/zarr.json @@ -0,0 +1,28 @@ +{ + "attributes": { + "morton_hive_commit": { + "spec": "morton-hive/1", + "complete": true, + "cells_with_data": 16, + "granule_count": 4, + "written_at": "2026-08-17T01:39:28+00:00", + "coverage": { + "spec": "morton-moc/1", + "box": [ + "11213", + null, + null, + null + ], + "cell_order": 6, + "source": "worker", + "encoding": "bitmap", + "sidecar": "coverage.moc", + "nbytes": 11, + "raw_nbytes": 2 + } + } + }, + "zarr_format": 3, + "node_type": "group" +} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/1/1/2/1/3/granules.json b/tests/data/spec/raster_toc/1/1/2/1/3/granules.json new file mode 100644 index 000000000..029544613 --- /dev/null +++ b/tests/data/spec/raster_toc/1/1/2/1/3/granules.json @@ -0,0 +1 @@ +{"spec": "zagg-granule-ids/1", "granules_sha256": "cb9641dd3c03a8121dbcc4eb174117524de11abf52c859944ae3612dad0ecfc4", "granule_ids": ["dt-1-a", "dt-1-b", "dt-2-a", "dt-3-a"]} \ No newline at end of file diff --git a/tests/data/spec/raster_toc/morton_hive.json b/tests/data/spec/raster_toc/morton_hive.json new file mode 100644 index 000000000..3cb3c8f28 --- /dev/null +++ b/tests/data/spec/raster_toc/morton_hive.json @@ -0,0 +1,32 @@ +{ + "spec": "morton-hive/1", + "dataset": { + "short_name": "SPEC_FIXTURE_RASTER", + "version": "1" + }, + "semantic_hash": "af17d15801d112a1d6378896e9bf22ceffa4404f7d3b70731aa7ef3920dca3d6", + "cell_order": 6, + "shard_order": 4, + "split_schedule": [ + 1, + 1, + 1, + 1 + ], + "path_grouping": 1, + "pyramid": { + "spec": "zagg-pyramid/1", + "overview": { + "spacing": 2, + "orders": [ + 2, + 0 + ], + "all_time": false, + "fold_source": "cascade", + "exact_levels": 1, + "fields": {} + } + }, + "generated_at": "2026-08-17T01:39:28+00:00" +} \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index 094d8f57e..7f1439da0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2710,3 +2710,45 @@ def test_shipped_atl06_warns_and_stays_valid(self, caplog): cfg = default_config("atl06") # h_min/h_max: min/max, float32, NaN default validate_config(cfg) # must NOT raise assert "h_min" in caplog.text and "h_max" in caplog.text + + +class TestTimeEncoding: + """``output.time_encoding`` — the spec §8 declaration (issue #443).""" + + def _raster_cfg(self, encoding=None): + output = {"grid": {"type": "healpix", "parent_order": 10, "child_order": 16}} + if encoding is not None: + output["time_encoding"] = encoding + return load_config_from_dict( + { + "data_source": { + "reader": "raster", + "bands": {"red": {"asset": "red", "dtype": "uint16"}}, + }, + "output": output, + } + ) + + def test_absent_and_both_values_validate(self): + for encoding in (None, "microseconds", "toc"): + validate_config(self._raster_cfg(encoding)) + + def test_unknown_value_rejected(self): + with pytest.raises(ValueError, match="output.time_encoding must be one of"): + validate_config(self._raster_cfg("datetime64")) + + def test_toc_rejected_on_a_non_raster_pipeline(self): + cfg = default_config("atl06") + cfg.output["time_encoding"] = "toc" + with pytest.raises(ValueError, match="applies to raster"): + validate_config(cfg) + + def test_default_still_validates_on_a_non_raster_pipeline(self): + cfg = default_config("atl06") + cfg.output["time_encoding"] = "microseconds" + validate_config(cfg) + + def test_packaged_sentinel2_config_declares_toc(self): + cfg = default_config("sentinel2_l2a") + validate_config(cfg) + assert cfg.output["time_encoding"] == "toc" diff --git a/tests/test_raster_pipeline.py b/tests/test_raster_pipeline.py index 49779918c..ab92e3681 100644 --- a/tests/test_raster_pipeline.py +++ b/tests/test_raster_pipeline.py @@ -21,6 +21,7 @@ _shard_cell_range, _shard_workers, _write_buffer, + emit_raster_leaf_template, emit_raster_template, new_stage_stats, process_raster_shard, @@ -31,13 +32,20 @@ write_raster_coords, write_raster_slab, ) +from zagg.time_axis import decode_time_axis, read_time_axis, time_axis_attrs T0 = "2026-07-13T16:02:20+00:00" T0B = "2026-07-13T16:02:24+00:00" # same datatake, adjacent tile: seconds later T1 = "2026-07-18T16:02:20+00:00" -def _raster_config(bands=None, nodata=0, grid=None): +def _raster_config(bands=None, nodata=0, grid=None, time_encoding=None): + output = { + "grid": grid or {"type": "healpix", "parent_order": 10, "child_order": 16}, + "store": "memory://", + } + if time_encoding: + output["time_encoding"] = time_encoding return load_config_from_dict( { "data_source": { @@ -55,18 +63,19 @@ def _raster_config(bands=None, nodata=0, grid=None): }, "nodata": nodata, }, - "output": { - "grid": grid or {"type": "healpix", "parent_order": 10, "child_order": 16}, - "store": "memory://", - }, + "output": output, } ) -def _entry(gid, assets, dt, time_key=None): +def _entry(gid, assets, dt, time_key=None, time_start=None, time_end=None): e = {"id": gid, "s3": None, "https": None, "assets": assets, "datetime": dt} if time_key: e["time_key"] = time_key + if time_start: + e["time_start"] = time_start + if time_end: + e["time_end"] = time_end return e @@ -324,6 +333,133 @@ def test_missing_datetime_raises(self): raster_time_index([[{"id": "bad", "assets": {"red": "x"}}]]) +class TestTocTimeIndex: + """The §8 toc encoding of the acquisition-group axis (issue #443).""" + + def _granules(self): + return [ + [ + _entry("a", {"red": "x"}, T0, time_key="dt-1"), + _entry("b", {"red": "y"}, T0B, time_key="dt-1"), + _entry("c", {"red": "z"}, T1, time_key="dt-2"), + ] + ] + + def test_group_span_becomes_a_range_word(self): + index, words = raster_time_index(self._granules(), encoding="toc") + # Row assignment is the encoding-independent part: same order, same + # index, so a leaf's slab rows cannot drift with the encoding. + assert index == raster_time_index(self._granules())[0] == {"dt-1": 0, "dt-2": 1} + assert words.dtype == np.uint64 and words.shape == (2,) + lo, hi = decode_time_axis(words, time_axis_attrs("toc")) + # dt-1 spans T0..T0B (4 s apart) — a conservative range containing both. + assert lo[0] <= np.datetime64(T0[:-6], "ns") and hi[0] > np.datetime64(T0B[:-6], "ns") + # dt-2 is a single item: an exact instant, not widened into a range. + assert lo[1] == hi[1] == np.datetime64(T1[:-6], "ns") + + def test_stac_start_end_datetime_widens_the_envelope(self): + granules = [ + [ + _entry( + "a", + {"red": "x"}, + T0, + time_key="dt-1", + time_start="2026-07-13T16:02:18+00:00", + time_end="2026-07-13T16:02:29+00:00", + ) + ] + ] + _index, words = raster_time_index(granules, encoding="toc") + lo, hi = decode_time_axis(words, time_axis_attrs("toc")) + assert lo[0] <= np.datetime64("2026-07-13T16:02:18", "ns") + assert hi[0] > np.datetime64("2026-07-13T16:02:29", "ns") + + def test_legacy_encoding_ignores_the_span(self): + # A pre-§8 axis is the earliest ITEM datetime, spans or not — the + # legacy values must not move when a catalog gains start/end columns. + granules = [ + [ + _entry( + "a", + {"red": "x"}, + T0, + time_key="dt-1", + time_start="2026-07-13T16:02:18+00:00", + time_end="2026-07-13T16:02:29+00:00", + ) + ] + ] + _index, times = raster_time_index(granules) + assert times.dtype == np.int64 and times[0] == np.int64(1_783_958_540_000_000) + + def test_a_leading_span_puts_the_stored_words_out_of_row_order(self): + # The §8.1 divergence, on the production function: row order keys on + # the group's earliest ITEM datetime, but the word encodes the + # ENVELOPE start. A later row whose declared start_datetime precedes + # an earlier row's item datetime therefore encodes a SMALLER word -- + # so the stored axis is materially unsorted and MUST NOT be bisected, + # while the row assignment stays identical to the legacy encoding. + granules = [ + [ + _entry("a", {"red": "x"}, T0, time_key="dt-1"), + _entry( + "b", + {"red": "y"}, + "2026-07-13T16:02:30+00:00", + time_key="dt-2", + time_start="2026-07-13T16:00:59+00:00", + time_end="2026-07-13T16:02:40+00:00", + ), + ] + ] + index, words = raster_time_index(granules, encoding="toc") + # Row assignment is stable across encodings — the load-bearing claim. + assert index == raster_time_index(granules)[0] == {"dt-1": 0, "dt-2": 1} + # ...but the stored words descend, by the span's ~81 s lead. + assert words[1] < words[0] + assert not np.array_equal(np.sort(words), words) + lo, _hi = decode_time_axis(words, time_axis_attrs("toc")) + assert lo[1] <= np.datetime64("2026-07-13T16:00:59", "ns") < lo[0] + + def test_empty_toc_axis(self): + index, words = raster_time_index([[]], encoding="toc") + assert index == {} and words.dtype == np.uint64 and words.size == 0 + + def test_template_declares_and_round_trips(self, tmp_path): + cfg, grid, _shard = _healpix_setup(tmp_path, time_encoding="toc") + _index, words = raster_time_index(self._granules(), encoding="toc") + store = MemoryStore() + emit_raster_template(store, grid, cfg, words) + tarr = open_array(store, path=f"{grid.group_path}/time", zarr_format=3, consolidated=False) + assert tarr.dtype == np.uint64 + assert "units" not in tarr.attrs and "calendar" not in tarr.attrs + assert dict(tarr.attrs)["temporal"]["spec"] == "zagg-toc/1" + np.testing.assert_array_equal(tarr[:], words) + # And the read path decodes the store without being told the encoding. + lo, hi = read_time_axis(store, grid.group_path) + assert lo.shape == (2,) and (hi >= lo).all() + + def test_leaf_template_declares_toc(self, tmp_path): + cfg, grid, shard = _healpix_setup(tmp_path, time_encoding="toc") + _index, words = raster_time_index(self._granules(), encoding="toc") + store = MemoryStore() + emit_raster_leaf_template(store, grid, cfg, shard, words) + tarr = open_array(store, path=f"{grid.group_path}/time", zarr_format=3, consolidated=False) + assert tarr.dtype == np.uint64 + np.testing.assert_array_equal(tarr[:], words) + + def test_legacy_store_still_reads(self, tmp_path): + # Schema evolution: a store written before §8 decodes through the same + # reader, no declaration and no refusal. + cfg, grid, _shard = _healpix_setup(tmp_path) + store = MemoryStore() + emit_raster_template(store, grid, cfg, np.array([1_000_000, 2_000_000], dtype=np.int64)) + lo, hi = read_time_axis(store, grid.group_path) + np.testing.assert_array_equal(lo, hi) + assert lo[0] == np.datetime64("1970-01-01T00:00:01", "ns") + + class _FakeGrid: """Minimal grid for the sampling-concurrency test: only ``children`` is touched on the single-item-per-group path (no ownership combine).""" @@ -629,7 +765,7 @@ def _sink(t_idx, slab): np.testing.assert_array_equal(streamed[t]["red"], golden[t]["red"]) -def _healpix_setup(tmp_path): +def _healpix_setup(tmp_path, time_encoding=None): """Order-10 shard over the synthetic raster; order-16 cells (~97 m).""" from mortie import clip2order, geo2mort @@ -640,6 +776,7 @@ def _healpix_setup(tmp_path): cfg = _raster_config( bands={"red": {"asset": "red", "dtype": "uint16", "scale": 0.0001, "offset": -0.1}}, grid={"type": "healpix", "parent_order": 10, "child_order": 16}, + time_encoding=time_encoding, ) grid = HealpixGrid(10, 16, config=cfg) return cfg, grid, shard diff --git a/tests/test_readers.py b/tests/test_readers.py index c6b6e8618..1b196a352 100644 --- a/tests/test_readers.py +++ b/tests/test_readers.py @@ -1295,3 +1295,92 @@ def test_non_consolidated_is_navigable_and_reads_parity(self): assert set(locs_cons) == set(locs_plain) for key in locs_plain: np.testing.assert_array_equal(locs_plain[key], locs_cons[key]) + + +class TestTimeAxisReader: + """The §8 time-coordinate read surface (issue #443). + + One code path for both encodings: the reader resolves the declaration + from the array's own attrs, so a caller never has to know which era a + store was written in. + """ + + def _store(self, encoding): + from zagg.config import load_config_from_dict + from zagg.processing.raster import emit_raster_template, raster_time_index + + cfg = load_config_from_dict( + { + "data_source": { + "reader": "raster", + "bands": {"red": {"asset": "red", "dtype": "uint16"}}, + }, + "output": { + "grid": {"type": "healpix", "parent_order": 4, "child_order": 6}, + "time_encoding": encoding, + }, + } + ) + grid = HealpixGrid(4, 6, config=cfg) + granules = [ + [ + { + "id": "a", + "assets": {"red": "x"}, + "datetime": "2025-06-15T15:06:40+00:00", + "time_key": "dt-1", + }, + { + "id": "b", + "assets": {"red": "y"}, + "datetime": "2025-06-15T15:06:47+00:00", + "time_key": "dt-1", + }, + { + "id": "c", + "assets": {"red": "z"}, + "datetime": "2025-06-16T15:06:40+00:00", + "time_key": "dt-2", + }, + ] + ] + _index, times = raster_time_index(granules, encoding=encoding) + store = MemoryStore() + emit_raster_template(store, grid, cfg, times) + return store, grid, times + + def test_toc_axis_decodes_to_conservative_bounds(self): + from zagg.readers import read_time_axis + + store, grid, _times = self._store("toc") + lo, hi = read_time_axis(store, grid.group_path) + assert lo.dtype == np.dtype("datetime64[ns]") and lo.shape == (2,) + # The datatake's two tiles are 7 s apart: a range that contains both. + assert lo[0] <= np.datetime64("2025-06-15T15:06:40", "ns") + assert hi[0] > np.datetime64("2025-06-15T15:06:47", "ns") + # The single-item group stays an exact instant. + assert lo[1] == hi[1] == np.datetime64("2025-06-16T15:06:40", "ns") + + def test_legacy_axis_reads_through_the_same_call(self): + from zagg.readers import read_time_axis + + store, grid, _times = self._store("microseconds") + lo, hi = read_time_axis(store, grid.group_path) + np.testing.assert_array_equal(lo, hi) + assert lo[0] == np.datetime64("2025-06-15T15:06:40", "ns") + + def test_window_selection_runs_on_the_stored_words(self): + import zarr as _zarr + + from zagg.readers import time_axis_overlaps + + store, grid, times = self._store("toc") + arr = _zarr.open_array( + store, path=f"{grid.group_path}/time", zarr_format=3, consolidated=False + ) + mask = time_axis_overlaps( + arr[:], dict(arr.attrs), "2025-06-16T00:00:00", "2025-06-17T00:00:00" + ) + np.testing.assert_array_equal(mask, [False, True]) + # No decode happened: the predicate consumed the raw uint64 words. + assert arr.dtype == np.uint64 and times.dtype == np.uint64 diff --git a/tests/test_semantics.py b/tests/test_semantics.py index 0a6b72811..3d225b0bf 100644 --- a/tests/test_semantics.py +++ b/tests/test_semantics.py @@ -282,3 +282,32 @@ def test_overview_delta_is_packaging(self): assert semantic_hash(_digest_cfg(overview_delta=512)) == semantic_hash(_digest_cfg()) core = semantic_core(_digest_cfg(overview_delta=512)) assert "overview_delta" not in core["aggregation"]["variables"]["d"] + + +class TestTimeEncodingHashing: + """Issue #443: the §8 time-coordinate encoding is output-defining.""" + + def _raster_cfg(self, encoding=None) -> PipelineConfig: + output = {"grid": {"type": "healpix", "parent_order": 10, "child_order": 16}} + if encoding: + output["time_encoding"] = encoding + return PipelineConfig( + data_source={"reader": "raster", "bands": {"red": {"asset": "red", "dtype": "uint16"}}}, + output=output, + ) + + def test_default_encoding_hashes_as_absent(self): + # The §8 absent-key default: an explicit `microseconds` is the same + # stored axis, so it must be the same product identity — which is + # also what keeps every pre-#443 config's hash byte-identical. + assert semantic_hash(self._raster_cfg("microseconds")) == semantic_hash(self._raster_cfg()) + assert "time_encoding" not in semantic_core(self._raster_cfg("microseconds")) + + def test_toc_is_semantic(self): + toc = self._raster_cfg("toc") + assert semantic_hash(toc) != semantic_hash(self._raster_cfg()) + assert semantic_core(toc)["time_encoding"] == "toc" + + def test_packaged_s2_config_declares_toc(self): + cfg = default_config("sentinel2_l2a") + assert semantic_core(cfg)["time_encoding"] == "toc" diff --git a/tests/test_spec_conformance.py b/tests/test_spec_conformance.py index 4c31d09d3..f6c00ad28 100644 --- a/tests/test_spec_conformance.py +++ b/tests/test_spec_conformance.py @@ -73,6 +73,10 @@ # The §2.0 flux fixture (issue #424) — asserted by TestFluxDeclaration # (flux is not in FIXTURES: the leaf-shaped suite hardcodes h_tdigest). "flux": "910a9b12d34b4d072f454b2dd1cc232a6a9a92b536a8d26f5395154a1c02bc32", + # The §8 temporal fixture (issue #443) — asserted by + # TestTemporalDeclaration (raster_toc is not in FIXTURES: it carries no + # ragged array at all, so nothing the leaf-shaped suite asserts applies). + "raster_toc": "42263e046ecf4d71de8460063b38e6f15522e245cdc2182ce3a0acaf35db7e4e", } #: The same pin over the §4.6 COLUMN artifact of the ``column/`` fixture (not #: its leaf, which is ``minimal``'s): the only committed store whose §5 key @@ -92,6 +96,11 @@ "6/composition", ): "04886a9dfb60c8a53de48202dc3d5ac694698864825426f084be961aa678acd5", ("flux", "6/rx_flux"): "3ba141cb29b1771dee4c4c3f2aadec42b7de69e075d4334d6d7187301dc59eb8", + # The §8 uint64 toc-word time coordinate (issue #443). + ( + "raster_toc", + "6/time", + ): "551f7be5718086c2ca1d379e1d1581368d21e4ca1433ce3ae4ae397e11b08f7f", } @@ -962,3 +971,133 @@ def test_manifest_declares_the_v2_schedule(self): assert block["spec"] == "zagg-pyramid/2" assert block["overviews"][0] == {"node": exp["shard_order"], "cells": [5]} assert exp["declared"] == {"overviews": 5} + + +class TestTemporalDeclaration: + """§8 — the temporal declaration, pinned on the committed `raster_toc/`. + + The absent-declaration ⇒ legacy side needs no new fixture: the four + digest fixtures carry no ``temporal`` key anywhere, which is the pin. + """ + + def _time_meta(self): + exp = _expected("raster_toc") + meta = json.loads( + (_leaf_dir("raster_toc", exp) / exp["group"] / "time" / "zarr.json").read_text() + ) + return exp, meta + + def test_declaration_grammar_and_dtype(self): + exp, meta = self._time_meta() + attrs = meta["attributes"] + assert attrs == exp["time_attrs"] + # §8: exactly the #410-ruled {spec, shape, grammar revision} -- the + # committed bytes carry NO per-store epoch, timescale, or quantum + # keys, and the grammar is a {name}/{major} revision token, never a + # documentation URL or a release stamp. + assert attrs["temporal"] == { + "spec": "zagg-toc/1", + "shape": "coordinate", + "grammar": "mortie-toc/1", + } + # §8.1: uint64 words, one per timestep, no CF pair to mislead a + # units/calendar-decoding client. + assert meta["data_type"] == "uint64" + assert set(attrs) == {"temporal"} + assert meta["shape"] == [len(exp["time_words"])] + + def test_stored_words_match_the_golden(self): + exp = _expected("raster_toc") + store = _leaf_store("raster_toc", exp) + words = zarr.open_array( + store, path=f"{exp['group']}/time", zarr_format=3, consolidated=False + )[:] + assert words.dtype == np.uint64 + np.testing.assert_array_equal(words, np.array(exp["time_words"], dtype=np.uint64)) + # These particular words ascend, but that is INCIDENTAL to this + # fixture: its three groups are days apart, so no envelope start can + # lead the row before it. §8.1 does NOT promise ascending stored + # words -- the span-lead counterexample is pinned by + # test_raster_pipeline.py::TestTocTimeIndex:: + # test_a_leading_span_puts_the_stored_words_out_of_row_order. + np.testing.assert_array_equal(np.sort(words), words) + + def test_decode_matches_the_golden_bounds(self): + from zagg.readers import read_time_axis + + exp = _expected("raster_toc") + lo, hi = read_time_axis(_leaf_store("raster_toc", exp), exp["group"]) + want = np.array(exp["time_bounds_ns"], dtype="int64") + np.testing.assert_array_equal(lo.astype("int64"), want[:, 0]) + np.testing.assert_array_equal(hi.astype("int64"), want[:, 1]) + + def test_bounds_conservatively_contain_the_real_acquisitions(self): + # The §8.1 honesty claim, on committed bytes: a range never narrows + # its acquisition, and a single-instant acquisition never widens. + exp = _expected("raster_toc") + bounds = np.array(exp["time_bounds_ns"], dtype="int64") + for (start_ns, end_ns), acq in zip(bounds, exp["acquisitions"], strict=True): + real_lo = np.datetime64(acq["start"], "ns").astype("int64") + real_hi = np.datetime64(acq["end"], "ns").astype("int64") + assert start_ns <= real_lo + if real_lo == real_hi: + assert start_ns == end_ns == real_lo # exact timestamp word + else: + assert end_ns > real_hi # exclusive envelope end + + def test_both_word_variants_are_committed(self): + from mortie import toc_is_range + + exp = _expected("raster_toc") + words = np.array(exp["time_words"], dtype=np.uint64) + kinds = np.asarray(toc_is_range(words), dtype=bool) + assert kinds.any() and not kinds.all() + + def test_window_predicate_runs_on_the_stored_words(self): + from zagg.readers import time_axis_overlaps + + exp = _expected("raster_toc") + store = _leaf_store("raster_toc", exp) + arr = zarr.open_array(store, path=f"{exp['group']}/time", zarr_format=3, consolidated=False) + mask = time_axis_overlaps( + arr[:], dict(arr.attrs), "2025-06-18T00:00:00", "2025-06-19T00:00:00" + ) + np.testing.assert_array_equal(mask, [False, True, False]) + + def test_bands_and_morton_decode(self): + exp = _expected("raster_toc") + store = _leaf_store("raster_toc", exp) + for name, want in exp["bands"].items(): + got = zarr.open_array( + store, path=f"{exp['group']}/{name}", zarr_format=3, consolidated=False + )[:] + assert got.shape == (len(exp["time_words"]), exp["cells_per_shard"]) + np.testing.assert_array_equal(got, np.array(want, dtype=got.dtype)) + morton = zarr.open_array( + store, path=f"{exp['group']}/morton", zarr_format=3, consolidated=False + )[:] + np.testing.assert_array_equal(morton, np.array(exp["morton"], dtype=np.uint64)) + + def test_absent_declaration_on_every_pre_section_8_fixture(self): + from zagg.time_axis import temporal_declaration + + for name in (*FIXTURES, "flux"): + exp = _expected(name) + attrs = _array_meta(name, exp, "morton")["attributes"] + assert temporal_declaration(attrs) is None + + def test_leaf_is_stamped_and_manifest_marked(self): + exp = _expected("raster_toc") + attrs = json.loads((_leaf_dir("raster_toc", exp) / "zarr.json").read_text())["attributes"] + assert attrs["morton_hive_commit"]["complete"] is True + manifest = json.loads((SPEC_DATA / "raster_toc" / "morton_hive.json").read_text()) + assert manifest["spec"] == "morton-hive/1" + + def test_frozen_digests_pin_the_recipe(self): + exp = _expected("raster_toc") + got = TestContentHashes._hash_leaf(_leaf_dir("raster_toc", exp)) + assert got == exp["content_hashes"]["arrays"] + assert exp["content_hashes"]["combined"] == FROZEN_COMBINED["raster_toc"] + assert got["6/time"] == FROZEN_ARRAYS[("raster_toc", "6/time")] + combined = hashlib.sha256("\n".join(sorted(got.values())).encode()).hexdigest() + assert combined == exp["content_hashes"]["combined"] diff --git a/tests/test_time_axis.py b/tests/test_time_axis.py new file mode 100644 index 000000000..6a9c368bf --- /dev/null +++ b/tests/test_time_axis.py @@ -0,0 +1,200 @@ +"""Tests for the spec §8 temporal declaration (issue #443). + +The encode/decode contract on its own: the declaration grammar and its +strict-checks, envelope -> word -> conservative range round-trips inside the +grammar's quantization, the ordering property the stored axis relies on, the +window predicate, and the legacy (absent-declaration) path that every +pre-§8 store still reads through. +""" + +import numpy as np +import pytest +from mortie.toc import Q_END_NS, Q_START_NS + +from zagg.time_axis import ( + LEGACY_TIME_ATTRS, + TOC_EPOCH, + TOC_GRAMMAR, + TOC_SPEC, + decode_time_axis, + encode_time_axis, + temporal_declaration, + time_axis_attrs, + time_axis_dtype, + time_axis_overlaps, + time_encoding, +) + +TOC_ATTRS = time_axis_attrs("toc") +#: 2025-06-15T15:06:40Z, a comfortable distance from any grid boundary. +BASE_US = 1_750_000_000_000_000 + + +def _cfg(value): + class _C: + output = {} if value is None else {"time_encoding": value} + + return _C() + + +class TestDeclaration: + def test_default_encoding_is_legacy(self): + assert time_encoding(_cfg(None)) == "microseconds" + assert time_axis_dtype("microseconds") == "int64" + assert time_axis_attrs("microseconds") == LEGACY_TIME_ATTRS + + def test_toc_stamps_the_declaration_and_no_cf_attrs(self): + # units/calendar would describe the words wrongly, so a CF-decoding + # client must find nothing to decode rather than plausible garbage. + assert set(TOC_ATTRS) == {"temporal"} + block = TOC_ATTRS["temporal"] + # Exactly the #410-ruled shape: {spec, shape, grammar revision}. NO + # per-store epoch/timescale/quantum guards -- those are properties of + # the cited grammar, and echoing them would only put a restated + # constant into the fixture bytes and the §5 hash. + assert block == {"spec": TOC_SPEC, "shape": "coordinate", "grammar": TOC_GRAMMAR} + assert time_axis_dtype("toc") == "uint64" + + def test_the_cited_grammar_is_the_one_this_reader_decodes_with(self): + # The citation is a fixed {name}/{major} revision token -- never a + # documentation URL or a release stamp (store bytes must not move when + # docs or a floor move) -- but it must not drift from the constants + # the decode actually runs on. + assert TOC_GRAMMAR == "mortie-toc/1" + assert TOC_EPOCH == "1850-01-01T00:00:00" + assert (int(Q_START_NS), int(Q_END_NS)) == (2**31, 2**32) + + def test_absent_key_is_legacy_never_a_refusal(self): + for attrs in (None, {}, {"units": "microseconds since 1970-01-01T00:00:00"}): + assert temporal_declaration(attrs) is None + + def test_unknown_spec_refused(self): + block = {**TOC_ATTRS["temporal"], "spec": "zagg-toc/2"} + with pytest.raises(ValueError, match="unknown temporal declaration spec"): + temporal_declaration({"temporal": block}) + + def test_unimplemented_shape_refused(self): + block = {**TOC_ATTRS["temporal"], "shape": "per-centroid"} + with pytest.raises(ValueError, match="shape 'per-centroid' is not implemented"): + temporal_declaration({"temporal": block}) + + def test_uncited_grammar_refused(self): + block = {**TOC_ATTRS["temporal"], "grammar": "mortie-toc/2"} + with pytest.raises(ValueError, match="cites word grammar"): + temporal_declaration({"temporal": block}) + + def test_informative_keys_are_ignored_not_refused(self): + # §8: unrecognized keys are non-normative provenance, so a reader + # passes them through rather than refusing the store. + block = {**TOC_ATTRS["temporal"], "source_time_field": "start_datetime"} + assert temporal_declaration({"temporal": block}) == block + + def test_non_mapping_declaration_refused(self): + with pytest.raises(ValueError, match="must be a mapping"): + temporal_declaration({"temporal": "zagg-toc/1"}) + + def test_bad_config_value_refused(self): + with pytest.raises(ValueError, match="output.time_encoding must be one of"): + time_encoding(_cfg("datetime64")) + + +class TestRoundTrip: + def test_span_decodes_to_a_containing_range_within_quantization(self): + starts = np.array([BASE_US, BASE_US + 86_400_000_000]) + ends = np.array([BASE_US + 7_500_000, BASE_US + 86_400_000_000 + 12_000_000]) + words = encode_time_axis(starts, ends, encoding="toc") + assert words.dtype == np.uint64 + lo, hi = decode_time_axis(words, TOC_ATTRS) + real_lo = starts.astype("datetime64[us]").astype("datetime64[ns]") + real_hi = ends.astype("datetime64[us]").astype("datetime64[ns]") + # Conservative: the decoded envelope contains the real interval... + assert (lo <= real_lo).all() and (hi > real_hi).all() + # ...and is loose by at most one quantum at each end. + assert ((real_lo - lo) < np.timedelta64(int(Q_START_NS), "ns")).all() + assert ((hi - real_hi) <= np.timedelta64(int(Q_END_NS), "ns")).all() + + def test_degenerate_envelope_is_an_exact_timestamp(self): + # A single-item acquisition stays exact to the nanosecond: the writer + # never widens an instant into a range (§8.1). + words = encode_time_axis([BASE_US], [BASE_US], encoding="toc") + lo, hi = decode_time_axis(words, TOC_ATTRS) + exact = np.datetime64(BASE_US, "us").astype("datetime64[ns]") + assert lo[0] == exact and hi[0] == exact + + def test_mixed_variants_sort_by_envelope_start(self): + # Unsigned word order IS order by encoded envelope start (§8.1), so + # envelopes that themselves ascend encode to ascending words, mixing + # timestamp and range variants. + day = 86_400_000_000 + starts = np.array([BASE_US, BASE_US + day, BASE_US + 2 * day]) + ends = np.array([BASE_US + 9_000_000, BASE_US + day, BASE_US + 2 * day + 3_000_000]) + words = encode_time_axis(starts, ends, encoding="toc") + np.testing.assert_array_equal(np.sort(words), words) + + def test_a_leading_envelope_start_breaks_stored_word_order(self): + # The §8.1 divergence, at the encode layer: row order is the group's + # earliest MEMBER time, but the word encodes the ENVELOPE start. A + # later row whose envelope begins before an earlier row's encodes a + # smaller word, so the stored axis is materially unsorted -- which is + # why §8.1 forbids bisecting it. + starts = np.array([BASE_US, BASE_US - 63_000_000]) + ends = np.array([BASE_US, BASE_US + 10_000_000]) + words = encode_time_axis(starts, ends, encoding="toc") + assert words[1] < words[0] + assert not np.array_equal(np.sort(words), words) + + def test_empty_axis(self): + words = encode_time_axis([], [], encoding="toc") + assert words.dtype == np.uint64 and words.size == 0 + lo, hi = decode_time_axis(words, TOC_ATTRS) + assert lo.size == 0 and hi.size == 0 + + def test_inverted_envelope_refused(self): + with pytest.raises(ValueError, match="ends before it starts"): + encode_time_axis([BASE_US], [BASE_US - 1], encoding="toc") + + def test_pre_epoch_time_refused(self): + with pytest.raises(ValueError, match="precedes the toc epoch"): + encode_time_axis([-4_000_000_000_000_000], [0], encoding="toc") + + def test_legacy_encoding_stores_the_start_only(self): + values = encode_time_axis([BASE_US], [BASE_US + 7_500_000], encoding="microseconds") + assert values.dtype == np.int64 and values[0] == BASE_US + lo, hi = decode_time_axis(values, {}) + exact = np.datetime64(BASE_US, "us").astype("datetime64[ns]") + assert lo[0] == exact and hi[0] == exact + + +class TestWindowSelection: + def _axis(self): + day = 86_400_000_000 + starts = np.array([BASE_US, BASE_US + day, BASE_US + 2 * day]) + ends = np.array([BASE_US + 7_500_000, BASE_US + day, BASE_US + 2 * day + 3_000_000]) + return starts, ends + + def test_toc_window_selects_by_overlap(self): + starts, ends = self._axis() + words = encode_time_axis(starts, ends, encoding="toc") + mask = time_axis_overlaps(words, TOC_ATTRS, "2025-06-16T00:00:00", "2025-06-17T00:00:00") + np.testing.assert_array_equal(mask, [False, True, False]) + + def test_toc_window_never_under_reports(self): + # A window that clips the middle of a range still selects it. + starts, ends = self._axis() + words = encode_time_axis(starts, ends, encoding="toc") + mask = time_axis_overlaps(words, TOC_ATTRS, "2025-06-15T15:06:44", "2025-06-15T15:06:45") + assert bool(mask[0]) and not mask[1:].any() + + def test_legacy_window_matches_the_toc_selection(self): + starts, ends = self._axis() + legacy = encode_time_axis(starts, ends, encoding="microseconds") + words = encode_time_axis(starts, ends, encoding="toc") + window = ("2025-06-16T00:00:00", "2025-06-17T00:00:00") + np.testing.assert_array_equal( + time_axis_overlaps(legacy, {}, *window), + time_axis_overlaps(words, TOC_ATTRS, *window), + ) + + def test_inverted_window_refused(self): + with pytest.raises(ValueError, match="window is inverted"): + time_axis_overlaps(np.array([0], dtype="int64"), {}, "2025-01-02", "2025-01-01") diff --git a/tools/generate_spec_fixtures.py b/tools/generate_spec_fixtures.py index e03a299d1..e228d276a 100644 --- a/tools/generate_spec_fixtures.py +++ b/tools/generate_spec_fixtures.py @@ -1,13 +1,14 @@ """Generate the committed spec-conformance fixtures with zagg's REAL writers. The `docs/specification.md` §7 fixtures (issue #340), under -``tests/data/spec/``: two tiny hive stores, each one shard leaf written by -the production write path — ``hive.ensure_manifest`` + +``tests/data/spec/``: tiny hive stores, each one shard leaf written by the +production write path — ``hive.ensure_manifest`` + ``hive.process_and_write_hive`` (leaf template, sharded dense + ragged -writes, coverage sidecar, commit stamp) — plus one MANIFEST-ONLY pyramid -declaration written by the production declaration paths (issue #382, no -leaf beneath it). Each carries a committed ``*.expected.json`` recording the -decoded values and the §5 O11 content hashes. +writes, coverage sidecar, commit stamp), or its raster twin +``processing.raster.process_and_write_raster_hive`` — plus one MANIFEST-ONLY +pyramid declaration written by the production declaration paths (issue #382, +no leaf beneath it). Each carries a committed ``*.expected.json`` recording +the decoded values and the §5 O11 content hashes. ``tests/test_spec_conformance.py`` asserts the fixtures against the shipping readers AND against spec-text-only decoders, so the spec, the fixtures, and the reader cannot drift apart @@ -58,6 +59,18 @@ it on purpose — the block is a template-time artifact, decodable from ``morton_hive.json`` alone; the ``/2`` artifacts a fleet writes are the ``column/`` fixture's job (issue #383 — sweep-side levels are issue #384). +- ``raster_toc/`` — the §8 temporal-declaration surface (issue #443): a + RASTER ``(time, cells)`` hive leaf (two bands + ``morton`` + ``time``, the + one unsharded fixture — raster never shards) whose ``time`` coordinate is + ``uint64`` toc words carrying the ``temporal`` attrs block and no CF + ``units``/``calendar``. Its three timesteps commit BOTH word variants: two + multi-member acquisition groups become conservative RANGE words (one from + member instants seconds apart, one from the STAC ``start_datetime``/ + ``end_datetime`` pair) and a single-member group stays an exact TIMESTAMP. + Written through ``processing.raster.process_and_write_raster_hive`` with + only the COG *sampling* faked — a committed fixture must regenerate with + no network and no GDAL. The other four fixtures, which carry no + ``temporal`` key anywhere, are the absent-key ⇒ legacy pin. STALE BY DESIGN: ``minimal/`` and ``kitchen_sink/`` were committed before issue #382 and their ``morton_hive.json`` still carries the pre-#382 @@ -670,6 +683,205 @@ def build_pyramid(out: Path) -> None: print(f"{out.name}: manifest-only, {len(levels)} level entries, /1 actuals preserved") +#: The ``raster_toc/`` fixture's acquisition groups (§8, issue #443). Three +#: timesteps that between them exercise both toc word variants: two +#: multi-member datatakes (a RANGE word — one derived from member instants +#: seconds apart, one from the STAC ``start_datetime``/``end_datetime`` pair) +#: and one single-member acquisition (an exact TIMESTAMP word). +RASTER_GRANULES = [ + { + "id": "dt-1-a", + "assets": {"red": "s3://fixture/dt1a_red.tif", "scl": "s3://fixture/dt1a_scl.tif"}, + "datetime": "2025-06-15T15:06:40+00:00", + "time_key": "dt-1", + }, + { + "id": "dt-1-b", + "assets": {"red": "s3://fixture/dt1b_red.tif", "scl": "s3://fixture/dt1b_scl.tif"}, + "datetime": "2025-06-15T15:06:47+00:00", + "time_key": "dt-1", + }, + { + "id": "dt-2-a", + "assets": {"red": "s3://fixture/dt2a_red.tif", "scl": "s3://fixture/dt2a_scl.tif"}, + "datetime": "2025-06-18T15:06:40+00:00", + "time_key": "dt-2", + "time_start": "2025-06-18T15:06:38+00:00", + "time_end": "2025-06-18T15:06:49+00:00", + }, + { + "id": "dt-3-a", + "assets": {"red": "s3://fixture/dt3a_red.tif", "scl": "s3://fixture/dt3a_scl.tif"}, + "datetime": "2025-06-21T15:06:40+00:00", + "time_key": "dt-3", + }, +] +#: The raster fixture's bands: the shipped Sentinel-2 pair, trimmed to two. +RASTER_BANDS = { + "red": {"asset": "red", "dtype": "uint16", "fill_value": 0, "scale": 0.0001, "offset": -0.1}, + "scl": {"asset": "scl", "dtype": "uint8", "fill_value": 0}, +} + + +def _raster_acquisitions() -> list[dict]: + """The real acquisition envelope per group, from ``RASTER_GRANULES``. + + The same grouping the encoder applies (``time_key``, ``time_start``/ + ``time_end`` falling back to ``datetime``), in the same row order the + axis has (ascending group earliest-item ``datetime``), with the ``+00:00`` + suffix trimmed so the values compare as naive ISO instants. + """ + span: dict[str, tuple[str, str, str]] = {} + for g in RASTER_GRANULES: + key = g["time_key"] + lo, hi = g.get("time_start", g["datetime"]), g.get("time_end", g["datetime"]) + if key in span: + was_lo, was_hi, was_dt = span[key] + span[key] = (min(was_lo, lo), max(was_hi, hi), min(was_dt, g["datetime"])) + else: + span[key] = (lo, hi, g["datetime"]) + return [ + {"key": key, "start": lo[:-6], "end": hi[:-6]} + for key, (lo, hi, _dt) in sorted(span.items(), key=lambda kv: (kv[1][2], kv[0])) + ] + + +def _raster_slab(t_idx: int, n_cells: int): + """One deterministic ``(cells,)`` slab per band, with fill holes. + + Cells whose ordinal is congruent to the timestep mod 5 stay at the band + fill — pull-NN leaves a cell outside the source footprint untouched, so a + reader must not assume every band row is populated. + """ + ordinals = np.arange(n_cells) + valid = (ordinals % 5) != (t_idx % 5) + red = np.where(valid, 1000 + 37 * t_idx + ordinals, 0).astype(np.uint16) + scl = np.where(valid, 4 + (ordinals % 3), 0).astype(np.uint8) + return {"red": red, "scl": scl}, valid + + +def build_raster_toc(out: Path) -> None: + """The §8 ``raster_toc/`` fixture: a toc-declared ``(time, cells)`` leaf. + + Written through the production raster hive seam + (``processing.raster.process_and_write_raster_hive`` — leaf template, + per-timestep slab streaming, coverage sidecar, commit stamp, O11 + hashing), with only the COG **sampling** faked: the fixture pins the + stored bytes and the time-axis declaration, not the pull-NN arithmetic + (``tests/test_raster.py`` owns that), and a committed fixture must never + need network or GDAL to regenerate. + """ + import zarr + + from zagg import hive + from zagg.config import load_config_from_dict + from zagg.grids import from_config + from zagg.grids.morton import morton_word + from zagg.processing import raster as raster_mod + from zagg.time_axis import decode_time_axis, time_axis_attrs + + cfg = load_config_from_dict( + { + "data_source": {"reader": "raster", "bands": RASTER_BANDS, "nodata": 0}, + "output": { + "grid": { + "type": "healpix", + "parent_order": 4, + "child_order": 6, + "chunk_inner": 5, + # Raster never shards (§8/#247): K = 4 inner chunks of 4 + # cells, one object per (timestep, chunk). + "sharded": False, + }, + "store_layout": "hive", + "time_encoding": "toc", + }, + } + ) + grid = from_config(cfg) + shard = morton_word(SHARD_KEY) + n_cells = grid.cells_per_shard + + def _fake_process_raster_shard( + _grid, shard_key, granules, _config, time_index, *, on_slab=None, occupied_out=None, **_kw + ): + cells = np.asarray(_grid.children(int(shard_key)), dtype=np.uint64) + occupied = np.zeros(n_cells, dtype=bool) + for t_idx in sorted(time_index.values()): + slab, valid = _raster_slab(t_idx, n_cells) + occupied |= valid + on_slab(t_idx, slab) + if occupied_out is not None: + occupied_out.append(cells[occupied]) + return {}, { + "shard_key": shard_key, + "granule_count": len(granules), + "skipped": 0, + "timesteps": len(time_index), + "raster_bytes_read": 0, + "raster_px_decoded": 0, + "raster_px_sampled": 0, + } + + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + root = str(out) + hive.ensure_manifest( + root, + hive.build_manifest(grid, dataset={"short_name": "SPEC_FIXTURE_RASTER", "version": "1"}), + ) + original = raster_mod.process_raster_shard + raster_mod.process_raster_shard = _fake_process_raster_shard + try: + meta = raster_mod.process_and_write_raster_hive( + shard, RASTER_GRANULES, grid, root, cfg, store_kwargs={} + ) + finally: + raster_mod.process_raster_shard = original + assert meta.get("leaf_written"), meta + + leaf_rel = hive.shard_leaf_path("", shard).lstrip("/") + store = zarr.storage.LocalStore(str(out / leaf_rel)) + group = zarr.open_group(store, path=grid.group_path, mode="r", zarr_format=3) + words = np.asarray(group["time"][:], dtype=np.uint64) + attrs = dict(group["time"].attrs) + assert attrs == time_axis_attrs("toc"), attrs + lo, hi = decode_time_axis(words, attrs) + expected = { + "shard": SHARD_KEY, + "leaf": leaf_rel, + "group": grid.group_path, + "shard_order": 4, + "chunk_order": 5, + "cell_order": 6, + "cells_per_chunk": grid.cells_per_chunk, + "cells_per_shard": n_cells, + "time_encoding": "toc", + # The ``time`` array's attrs verbatim: the §8 declaration and + # nothing else (no CF units/calendar under this encoding). + "time_attrs": attrs, + # uint64 words as decimal strings — JSON numbers cannot carry them. + "time_words": [str(int(w)) for w in words], + # What a conforming decode yields: ns since the Unix epoch, ``end`` + # exclusive for a range and equal to ``start`` for a timestamp. + "time_bounds_ns": [ + [str(int(bound.astype("int64"))) for bound in pair] for pair in zip(lo, hi, strict=True) + ], + # The REAL acquisition envelopes the stored words must contain — the + # §8 conservative-containment claim, pinned on committed bytes. + # DERIVED from RASTER_GRANULES, never transcribed: a hand-typed copy + # would keep passing after an input edit moved the words, which is + # the one drift this block exists to catch. + "acquisitions": _raster_acquisitions(), + "morton": [str(int(w)) for w in group["morton"][:]], + "bands": {name: group[name][:].tolist() for name in RASTER_BANDS}, + "content_hashes": _o11_hashes(str(out / leaf_rel)), + } + (out.parent / f"{out.name}.expected.json").write_text(json.dumps(expected, indent=1) + "\n") + print(f"{out.name}: leaf {leaf_rel}, {len(words)} timesteps, {n_cells} cells") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -692,6 +904,7 @@ def main() -> None: "column": lambda: build(args.out / "column", kitchen_sink=False, pyramid={"overviews": 5}), "pyramid": lambda: build_pyramid(args.out / "pyramid"), "flux": lambda: build(args.out / "flux", kitchen_sink=False, flux=True), + "raster_toc": lambda: build_raster_toc(args.out / "raster_toc"), } unknown = set(args.only or ()) - set(builders) if unknown: