diff --git a/CHANGELOG.md b/CHANGELOG.md index 467a77f1..52ef2c4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- **BREAKING: strict input validation family-wide — previously-accepted-and-mangled + word/offset arrays now raise** (issue #194, ruled 2026-08-24; lands ahead of + the 1.0 release). The toc module's validators (`_as_offsets` / `_as_u64`) + are hoisted to a shared home and applied at every polymorphic choke point, + retiring the batch family's silent `np.asarray(..., dtype=...)` coercions: + a **float-typed** word or offset array raises `ValueError` instead of + truncating (`2.9` no longer becomes a group boundary at 2 — the issue #185 + panic-arc class), a **negative** word raises instead of wrapping into a + different — possibly valid — packed word, and a **uint64 offset ≥ 2⁶³** + raises instead of wrapping negative through the int64 cast (the PR #192 + class). Every refusal names the parameter and the first offending value. + Affected entry points, by module: + + | module | entry points now strict | + |---|---| + | `_moc` | `compress_moc`, `moc_to_order`, `moc_or`, `moc_and`, `moc_intersects`, `moc_minus`, `moc_xor`, `moc_min`, `moc_not`, `common_ancestor`, `split_base_cells` (both arms where polymorphic) | + | `batch` | `polygons_to_morton_mocs` and every ragged kernel behind the `offsets=` forms | + | `geometry` | `from_wkb(offsets=)`; the `to_geometry` / `to_wkb` / `to_wkt` word intake — validated at the shared seam, so **both** `dissolve` arms refuse alike (the default `dissolve=True` route through `dissolve` included) | + | `orders` | `generate_morton_children`, `clip2order`, `orders_of`, `is_point`, `infer_order_from_morton`, `validate_morton`; `orders_of_uniq` (phase-5 fold — its guard was *value*-based, refusing only non-integral floats, so `orders_of_uniq([16.0])` decoded to order 1 while `unique2parent([16.0])` refused the same column; oversized values are now named as `uniq must fit in int64` rather than as an out-of-range UNIQ) | + | `convert` | `mort2norm`, `mort2geo`, `mort2bbox`, `mort2polygon` (and `mort2healpix` through them); the UNIQ/normed intakes `unique2parent`, `uniq2geo`, and the `normed`/`parent` operands of `norm2mort` **and `norm2uniq`** (phase 5, espg ruling 2026-08-24 — `unique2parent([16.5, 20.9])` used to truncate to UNIQ 16/20 and answer `[0, 1]`; UNIQ ids stay int64-domain, so negatives keep their own `Not a valid UNIQ` refusal downstream, while the **unsigned** `normed`/`parent` operands refuse them by name up front — `normed must be non-negative, got -3` where `norm2mort` used to wrap to `18446744073709551613` and panic inside the kernel (the #185 arc again, stderr dump included), and where `norm2uniq(-3, 0, 4)` used to answer UNIQ `1021`, a real order-3 cell in base 11, with no error anywhere downstream) | + | `buffer` | `morton_buffer`, `morton_buffer_meters` | + | `moc_object` | `Moc` / `moc` word sources and set-operation operands (float arrays remain *geometry* there, by the documented polymorphism) | + + Two deliberate edges: **zero-size input of any dtype passes as a typed + empty** (an untyped `[]` is not numeric, it is empty — the ruling `Toc` + already applied to its source, now uniform; this also *loosens* the toc + functions, which previously refused `[]`), and **`split_children` keeps + accepting the signed `int64` bit-view of packed words** (the trie branches + on the decimal characteristic, whose first column *is* the sign) while + refusing floats like everything else. Valid inputs are unaffected — + byte-identity is pinned against pre-change goldens + (`mortie/tests/data/strict_validation_goldens.json`, captured at + `4900a7e`). + - **BREAKING: one polymorphic function per operation — the plural batch names are removed** (issue #187, ruled 2026-08-19). Every scalar/batch pair now has **one** public entry point: the input shape (or the keyword-only `offsets=`) diff --git a/mortie/_moc.py b/mortie/_moc.py index c2394e61..19fbd09c 100644 --- a/mortie/_moc.py +++ b/mortie/_moc.py @@ -28,6 +28,11 @@ cost — and the array-first consumers keep calling them on plain ndarrays. :class:`~mortie.moc_object.Moc` is the **object layer** over them, and every one of its methods is a single delegation to a function on this page. + +Input validation is strict family-wide (issue #194): float-typed word or offset +arrays are refused rather than truncated, negative words and past-int64 offsets +are refused rather than wrapped, and the refusal names the argument and the +offending value. """ import warnings @@ -35,6 +40,7 @@ import numpy as np from . import _rustie +from ._validate import _as_u64 from .batch import ( _common_ancestors, _mocs_and, @@ -64,7 +70,7 @@ def compress_moc(morton): numpy.ndarray Sorted, compacted morton indices (``uint64``). """ - morton = np.asarray(morton, dtype=np.uint64).ravel() + morton = _as_u64(morton, "morton").ravel() return np.asarray(_rustie.rust_moc_normalize(morton)) @@ -136,6 +142,8 @@ def moc_to_order(morton, order, max_cells=_FLAT_COVER_WARN_THRESHOLD, *, If ``order`` is outside 0-29, or the estimated densified count exceeds ``max_cells``. In the ragged form, also for offsets that are non-monotone, out of bounds, or do not exactly cover ``morton``. + Float-typed or negative ``morton`` and float or past-int64 + ``offsets`` are refused by name (issue #194), never silently cast. See Also -------- @@ -143,8 +151,11 @@ def moc_to_order(morton, order, max_cells=_FLAT_COVER_WARN_THRESHOLD, *, mortie.batch._mocs_to_orders : the ragged batch kernel this delegates to. """ if offsets is not None: - return _mocs_to_orders(morton, offsets, order, max_cells) - morton = np.asarray(morton, dtype=np.uint64).ravel() + # Name the caller-facing parameter before delegating -- the kernel's + # own pass stays as the backstop and sees uint64 (no rescan). + return _mocs_to_orders(_as_u64(morton, "morton"), offsets, order, + max_cells) + morton = _as_u64(morton, "morton").ravel() if not 0 <= order <= 29: raise ValueError(f"Order must be between 0 and 29, got {order}") if max_cells is not None: @@ -184,8 +195,8 @@ def moc_or(a, b): moc_minus : difference ``a \ b``. compress_moc : ``moc_or(a, b) == compress_moc(concatenate([a, b]))``. """ - a = np.asarray(a, dtype=np.uint64).ravel() - b = np.asarray(b, dtype=np.uint64).ravel() + a = _as_u64(a, "a").ravel() + b = _as_u64(b, "b").ravel() return np.asarray(_rustie.rust_moc_or(a, b)) @@ -225,9 +236,11 @@ def moc_and(a, b, *, offsets=None): mortie.batch._mocs_and : the 1 x N broadcast kernel this delegates to. """ if offsets is not None: - return _mocs_and(a, b, offsets) - a = np.asarray(a, dtype=np.uint64).ravel() - b = np.asarray(b, dtype=np.uint64).ravel() + # Name the caller-facing parameter before delegating -- the kernel's + # own pass stays as the backstop and sees uint64 (no rescan). + return _mocs_and(_as_u64(a, "a"), _as_u64(b, "b"), offsets) + a = _as_u64(a, "a").ravel() + b = _as_u64(b, "b").ravel() return np.asarray(_rustie.rust_moc_and(a, b)) @@ -273,9 +286,11 @@ def moc_intersects(a, b, *, offsets=None): to. """ if offsets is not None: - return _mocs_intersect(a, b, offsets) - a = np.asarray(a, dtype=np.uint64).ravel() - b = np.asarray(b, dtype=np.uint64).ravel() + # Name the caller-facing parameter before delegating -- the kernel's + # own pass stays as the backstop and sees uint64 (no rescan). + return _mocs_intersect(_as_u64(a, "a"), _as_u64(b, "b"), offsets) + a = _as_u64(a, "a").ravel() + b = _as_u64(b, "b").ravel() return bool(_rustie.rust_moc_intersects(a, b)) @@ -302,8 +317,8 @@ def moc_minus(a, b): moc_or : union of two covers. moc_and : intersection of two covers. """ - a = np.asarray(a, dtype=np.uint64).ravel() - b = np.asarray(b, dtype=np.uint64).ravel() + a = _as_u64(a, "a").ravel() + b = _as_u64(b, "b").ravel() return np.asarray(_rustie.rust_moc_minus(a, b)) @@ -334,8 +349,8 @@ def moc_xor(a, b): moc_and : intersection of two covers. moc_minus : difference ``a \ b`` (the directional half of ``xor``). """ - a = np.asarray(a, dtype=np.uint64).ravel() - b = np.asarray(b, dtype=np.uint64).ravel() + a = _as_u64(a, "a").ravel() + b = _as_u64(b, "b").ravel() return np.asarray(_rustie.rust_moc_xor(a, b)) @@ -403,11 +418,11 @@ def moc_not(cover, domain=None): >>> enumerated = mortie.from_geometry(aoi, moc=True) # doctest: +SKIP >>> gaps = mortie.moc_not(enumerated, domain=shard) # doctest: +SKIP """ - cover = np.asarray(cover, dtype=np.uint64).ravel() + cover = _as_u64(cover, "cover").ravel() if domain is None: domain = _whole_sphere() else: - domain = np.asarray(domain, dtype=np.uint64).ravel() + domain = _as_u64(domain, "domain").ravel() if domain.size == 0: # The complement within an empty domain is empty for any cover; the @@ -471,7 +486,8 @@ def common_ancestor(morton, *, offsets=None): (non-existent) whole-sphere root. In the ragged form the message names the lowest-index offending group *within its kind* (layout errors are screened in their own pass, ahead of the per-group content check), and - bad offsets raise here too. + bad offsets raise here too. Float-typed or negative ``morton`` and + float or past-int64 ``offsets`` are refused by name (issue #194). See Also -------- @@ -491,8 +507,10 @@ def common_ancestor(morton, *, offsets=None): True """ if offsets is not None: - return _common_ancestors(morton, offsets) - morton = np.asarray(morton, dtype=np.uint64).ravel() + # Name the caller-facing parameter before delegating -- the kernel's + # own pass stays as the backstop and sees uint64 (no rescan). + return _common_ancestors(_as_u64(morton, "morton"), offsets) + morton = _as_u64(morton, "morton").ravel() return np.uint64(_rustie.rust_moc_min(morton)) @@ -551,7 +569,7 @@ def split_base_cells(words, sort=False): >>> sorted(int(np.uint64(k) >> np.uint64(60)) - 1 for k in groups) [2, 5] """ - words = np.asarray(words, dtype=np.uint64).ravel() + words = _as_u64(words, "words").ravel() if words.size == 0: return {} diff --git a/mortie/_toc.py b/mortie/_toc.py index 3ab67c75..97264dfd 100644 --- a/mortie/_toc.py +++ b/mortie/_toc.py @@ -60,6 +60,9 @@ from . import _rustie +# The family's shared strict validators (hoisted from this module, issue #194). +from ._validate import _as_offsets, _as_u64 + Q_START_NS = 1 << 31 """Start quantum: 2^31 ns (~2.15 s); a range's start code floors to this.""" @@ -71,40 +74,6 @@ (~4 s short of year 2142); the end code must fit its 31-bit field.""" -def _as_u64(values, name): - """Validate non-negative integer input and return it as uint64.""" - arr = np.atleast_1d(np.asarray(values)) - if arr.dtype.kind not in "iu": - raise ValueError( - f"{name} must be integer-typed, got dtype {arr.dtype}") - if arr.dtype.kind == "i" and arr.size and np.any(arr < 0): - raise ValueError(f"{name} must be non-negative") - return arr.astype(np.uint64) - - -def _as_offsets(offsets): - """Validate arrow list offsets and return them as contiguous int64. - - Integer-typed by the same rule :func:`_as_u64` applies to words: a float - offset array would otherwise cast silently, truncating ``2.9`` to a group - boundary at 2 rather than saying so. The same standard rules out the - ``uint64`` values the cast cannot represent -- at or above ``2**63`` they - would wrap negative, and the Rust validator would then describe the - wrapped copy rather than the offset that was passed. Monotonicity and - bounds stay the Rust validator's job -- it names the offending group. - """ - arr = np.atleast_1d(np.asarray(offsets)) - if arr.dtype.kind not in "iu": - raise ValueError( - f"offsets must be integer-typed, got dtype {arr.dtype}") - if arr.dtype.kind == "u" and arr.size: - too_big = arr > np.iinfo(np.int64).max - if too_big.any(): - raise ValueError( - f"offsets must fit in int64, got {int(arr[too_big][0])}") - return np.ascontiguousarray(arr.astype(np.int64).ravel()) - - def _as_scalar_ns(value, name): """Validate a scalar ns argument and return it as a plain int.""" value = operator.index(value) diff --git a/mortie/_validate.py b/mortie/_validate.py new file mode 100644 index 00000000..a5867549 --- /dev/null +++ b/mortie/_validate.py @@ -0,0 +1,169 @@ +"""Shared strict input validators for word and offset arrays (issue #194). + +The toc module's validation discipline, hoisted to one home so the whole +family answers bad input the same way: refuse float-typed words and offsets +instead of truncating them, range-check before any narrowing cast instead of +wrapping, and name the parameter and the offending value. The strict form +has caught two real bug classes -- the issue #185 uncatchable-panic arc and +PR #192's silent uint64 wrap -- so with issue #187's consolidation giving +each operation one polymorphic entry point, that posture is applied at every +choke point rather than kept as a toc-only stance. + +Zero-size input is the one deliberate acceptance: an untyped empty container +(``[]``, ``()``, ``np.array([])``) is float64 by numpy's default, but it is +not numeric, it is empty (the ruling :class:`~mortie.toc_object.Toc` already +applied to its source argument) -- so it passes through as a typed empty +array rather than being refused for a dtype it never chose. +""" + +import numpy as np + + +def _as_u64(values, name): + """Validate non-negative integer input and return it as uint64. + + Float input is refused rather than truncated, and negative input is + refused rather than wrapped -- packed words are unsigned, and a negative + here is almost always a signed *reinterpretation* of a word whose top + bit is set (base cells 7-11; spec section 1) or a legacy signed id, both + of which would wrap into a different, possibly valid, word. + + Parameters + ---------- + values : array_like + Integer-typed values (any shape); zero-size input of any dtype is + accepted as empty. + name : str + Parameter name to blame in refusal messages. + + Returns + ------- + numpy.ndarray + The values as ``uint64``, at least 1-D; no copy when the input is + already ``uint64``. + + Raises + ------ + ValueError + If ``values`` is not integer-typed, or any value is negative -- + naming ``name`` and the first offending value. + """ + arr = np.atleast_1d(np.asarray(values)) + if arr.size == 0: + return arr.astype(np.uint64) + if arr.dtype.kind not in "iu": + raise ValueError( + f"{name} must be integer-typed, got dtype {arr.dtype}") + if arr.dtype.kind == "i": + flat = arr.ravel() + neg = flat[flat < 0] + if neg.size: + raise ValueError( + f"{name} must be non-negative, got {int(neg[0])}") + return arr.astype(np.uint64, copy=False) + + +def _as_i64(values, name): + """Validate int64-representable integer input and return it as int64. + + The signed counterpart of :func:`_as_u64`, for encodings whose working + dtype is ``int64`` (UNIQ ids, arrow offsets): float input is refused + rather than truncated, and a value the cast cannot represent -- a + ``uint64`` at or above ``2**63``, or a Python int outside int64 -- is + refused naming the value rather than wrapped or left to numpy's own + error. Negative values pass: they are representable, and the caller's + domain check owns their refusal (and its message). + + Parameters + ---------- + values : array_like + Integer-typed values (any shape); zero-size input of any dtype is + accepted as empty. + name : str + Parameter name to blame in refusal messages. + + Returns + ------- + numpy.ndarray + The values as ``int64``, at least 1-D; no copy when the input is + already ``int64``. + + Raises + ------ + ValueError + If ``values`` is not integer-typed, or a value does not fit in + ``int64`` -- naming ``name`` and the first offending value. Every + refusal is this family's own message: no numpy cast error or warning + (strings, ``None``, ``NaN``) surfaces in its place. + """ + arr = np.atleast_1d(np.asarray(values)) + if arr.size == 0: + return arr.astype(np.int64) + if arr.dtype.kind not in "iu": + # A Python int past int64 lands as float64 (or, further out, object) + # in the untyped asarray above, so an oversized *integer* would + # otherwise be blamed on its promoted dtype. Look for one directly + # rather than probe-casting (issue #194 review). + # + # Only an *untyped* container can hide an int inside a float64 + # promotion. An input that already carries a numpy dtype (ndarray, + # numpy scalar, pandas Series) holds numpy floats, and + # `asarray(..., dtype=object)` on it yields Python floats -- so the + # probe cannot fire, and would only materialize an object list the + # size of the column in front of a refusal it cannot change (0.33 s + # and ~96 MB on a 5M-element float64 UNIQ column, the likeliest way + # to reach this validator). Gate it on the two cases that can carry + # an oversized int: an object array, or an untyped container. + if arr.dtype.kind == "O" or getattr(values, "dtype", None) is None: + flat = np.atleast_1d(np.asarray(values, dtype=object)).ravel() + bad = next((v for v in flat.tolist() if isinstance(v, int) + and not -2**63 <= v < 2**63), None) + if bad is not None: + raise ValueError(f"{name} must fit in int64, got {bad}") + raise ValueError( + f"{name} must be integer-typed, got dtype {arr.dtype}") + if arr.dtype.kind == "u": + too_big = arr > np.iinfo(np.int64).max + if too_big.any(): + raise ValueError( + f"{name} must fit in int64, got {int(arr[too_big][0])}") + return arr.astype(np.int64, copy=False) + + +def _as_offsets(offsets): + """Validate arrow list offsets and return them as contiguous int64. + + Integer-typed by the same rule :func:`_as_u64` applies to words: a float + offset array would otherwise cast silently, truncating ``2.9`` to a group + boundary at 2 rather than saying so. The same standard rules out the + ``uint64`` values the cast cannot represent -- at or above ``2**63`` they + would wrap negative, and the Rust validator would then describe the + wrapped copy rather than the offset that was passed. Monotonicity and + bounds stay the Rust validator's job -- it names the offending group. + + Parameters + ---------- + offsets : array_like + Integer-typed arrow list offsets; zero-size input of any dtype is + accepted and left for the kernel's own emptiness refusal. + + Returns + ------- + numpy.ndarray + The offsets as a contiguous 1-D ``int64`` array. Not a private + copy: like :func:`_as_u64` and :func:`_as_i64` this is a no-copy + validator, so an input that is already contiguous ``int64`` comes + back as a **view aliasing the caller's array** (a read-only input + therefore yields a read-only result). No consumer mutates or + retains it -- the ragged kernels read it under a released GIL and + take it read-only -- so the buffer is validated, not owned. + + Raises + ------ + ValueError + If ``offsets`` is not integer-typed, or a value is at or above + ``2**63`` -- naming the first offending value. Every refusal is + this family's own message: no numpy cast error or warning (strings, + ``None``, ``NaN``) is allowed to surface in its place. + """ + return np.ascontiguousarray(_as_i64(offsets, "offsets").ravel()) diff --git a/mortie/arrow.py b/mortie/arrow.py index 85466076..ce95cbab 100644 --- a/mortie/arrow.py +++ b/mortie/arrow.py @@ -20,6 +20,8 @@ import numpy as np +from ._validate import _as_u64 + # The extension-type / array helpers are provided via module-level # ``__getattr__`` (built lazily so a numpy-only install can import this module), # so they are intentionally not named in ``__all__`` here. @@ -228,10 +230,14 @@ def from_morton_index(array): ------ ImportError If pyarrow is not installed. + ValueError + If *array* is not integer-typed, or holds a negative value -- a raw + ``array_like`` carries whatever dtype the caller gave it, so this + intake is strict like the rest of the family (issue #194). """ pa = _require_pyarrow() ext_type = _build_type() - data = np.asarray(getattr(array, "_data", array), dtype=np.uint64) + data = _as_u64(getattr(array, "_data", array), "array") # The empty sentinel (all-zero word, prefix 0) is the missing value on the # pandas side; mirror it as an Arrow null so isna() round-trips both ways. from .morton_index import MortonIndexArray @@ -693,12 +699,19 @@ def export_c_array(words): ``morton_index`` extension column (``ARROW:extension:name`` on the schema), with the all-zero empty sentinel mapped to an Arrow null via a real validity bitmap. + + Raises + ------ + ValueError + If *words* is not integer-typed, or holds a negative value -- a raw + ``array_like`` carries whatever dtype the caller gave it, so this + intake is strict like the rest of the family (issue #194). It is the + sharper case: unvalidated words cross an FFI boundary from here. """ from . import _rustie - data = np.ascontiguousarray( - np.asarray(getattr(words, "_data", words), dtype=np.uint64) - ) + data = np.ascontiguousarray(_as_u64(getattr(words, "_data", words), + "words")) return _rustie.rust_mi_export_c_array(data) diff --git a/mortie/batch.py b/mortie/batch.py index dd1a728a..238c47ac 100644 --- a/mortie/batch.py +++ b/mortie/batch.py @@ -32,6 +32,7 @@ import numpy as np from . import _rustie +from ._validate import _as_offsets, _as_u64 from .coverage import _FLAT_COVER_WARN_THRESHOLD from .geometry import _wkb_bytes @@ -142,6 +143,8 @@ def polygons_to_morton_mocs(lats, lons, offsets, order=18, tolerance=None, NaN/infinite coordinate. Also for offsets that do not exactly cover the vertex arrays (``offsets[0] != 0``, or ``offsets[-1]`` short of or past ``len(lats)`` — the message names which endpoint failed), + float-typed or past-int64 ``offsets`` (refused by name rather than + silently cast; issue #194), ``order`` outside 1-29, mismatched ``lats``/``lons`` lengths, or both ``tolerance`` and ``max_cells`` given. @@ -169,7 +172,7 @@ def polygons_to_morton_mocs(lats, lons, offsets, order=18, tolerance=None, raise ValueError("pass at most one of tolerance / max_cells") lats = np.ascontiguousarray(np.asarray(lats, dtype=np.float64).ravel()) lons = np.ascontiguousarray(np.asarray(lons, dtype=np.float64).ravel()) - offsets = np.ascontiguousarray(np.asarray(offsets, dtype=np.int64).ravel()) + offsets = _as_offsets(offsets) tol_rad = None if tolerance is None else np.radians(float(tolerance)) values, out_offsets = _rustie.rust_polygons_coverage_mocs( lats, lons, offsets, order, tol_rad, max_cells, normalize, latitude @@ -465,8 +468,8 @@ def _mocs_to_orders(values, offsets, order, max_cells=_FLAT_COVER_WARN_THRESHOLD >>> flat, flat_off = mortie.moc_to_order(mocs, 6, offsets=off) >>> first = flat[flat_off[0]:flat_off[1]] # flat cover of the first triangle """ - values = np.ascontiguousarray(np.asarray(values, dtype=np.uint64).ravel()) - offsets = np.ascontiguousarray(np.asarray(offsets, dtype=np.int64).ravel()) + values = np.ascontiguousarray(_as_u64(values, "values").ravel()) + offsets = _as_offsets(offsets) if max_cells is not None: max_cells = int(max_cells) if max_cells < 0: @@ -571,9 +574,9 @@ def _mocs_and(a, values, offsets): >>> [int(off[i + 1] - off[i]) for i in range(2)] # item 0 overlaps, 1 not [1, 0] """ - a = np.ascontiguousarray(np.asarray(a, dtype=np.uint64).ravel()) - values = np.ascontiguousarray(np.asarray(values, dtype=np.uint64).ravel()) - offsets = np.ascontiguousarray(np.asarray(offsets, dtype=np.int64).ravel()) + a = np.ascontiguousarray(_as_u64(a, "a").ravel()) + values = np.ascontiguousarray(_as_u64(values, "values").ravel()) + offsets = _as_offsets(offsets) out_values, out_offsets = _rustie.rust_mocs_and(a, values, offsets) return np.asarray(out_values), np.asarray(out_offsets) @@ -647,9 +650,9 @@ def _mocs_intersect(a, values, offsets): >>> mortie.moc_intersects(aoi, items, offsets=[0, 1, 2]).tolist() [True, False] """ - a = np.ascontiguousarray(np.asarray(a, dtype=np.uint64).ravel()) - values = np.ascontiguousarray(np.asarray(values, dtype=np.uint64).ravel()) - offsets = np.ascontiguousarray(np.asarray(offsets, dtype=np.int64).ravel()) + a = np.ascontiguousarray(_as_u64(a, "a").ravel()) + values = np.ascontiguousarray(_as_u64(values, "values").ravel()) + offsets = _as_offsets(offsets) return np.asarray(_rustie.rust_mocs_intersect(a, values, offsets)) @@ -757,8 +760,8 @@ def _common_ancestors(values, offsets): ... ] True """ - values = np.ascontiguousarray(np.asarray(values, dtype=np.uint64).ravel()) - offsets = np.ascontiguousarray(np.asarray(offsets, dtype=np.int64).ravel()) + values = np.ascontiguousarray(_as_u64(values, "values").ravel()) + offsets = _as_offsets(offsets) return np.asarray(_rustie.rust_common_ancestors(values, offsets)) @@ -887,7 +890,7 @@ def _children_of(words, order, max_cells=None): ... ValueError: generate_morton_children would generate 2097152 cells ... max_cells=1048576... """ - words = np.ascontiguousarray(np.asarray(words, dtype=np.uint64).ravel()) + words = np.ascontiguousarray(_as_u64(words, "words").ravel()) # Range-checked here so an out-of-range order is a catchable ValueError # rather than the binding's u8 OverflowError (the issue #108 posture). if not 0 <= order <= 29: diff --git a/mortie/buffer.py b/mortie/buffer.py index 5b4516b8..c71ce04b 100644 --- a/mortie/buffer.py +++ b/mortie/buffer.py @@ -15,6 +15,7 @@ import numpy as np from . import _rustie +from ._validate import _as_u64 from .orders import infer_order_from_morton @@ -45,7 +46,7 @@ def morton_buffer(morton_indices, k=1): ValueError If indices have mixed orders or k is out of range. """ - morton_indices = np.asarray(morton_indices, dtype=np.uint64) + morton_indices = _as_u64(morton_indices, "morton_indices") return _rustie.rust_morton_buffer(np.ascontiguousarray(morton_indices), k) @@ -103,7 +104,7 @@ def morton_buffer_meters(morton_indices, width_m): >>> border = mortie.morton_buffer_meters(cells, width_m=5000.0) >>> expanded = np.union1d(cells, border) """ - morton_indices = np.asarray(morton_indices, dtype=np.uint64) + morton_indices = _as_u64(morton_indices, "morton_indices") if morton_indices.size == 0: raise ValueError("morton_indices must be non-empty") if not (width_m > 0): diff --git a/mortie/convert.py b/mortie/convert.py index 097590be..989cdff9 100644 --- a/mortie/convert.py +++ b/mortie/convert.py @@ -19,6 +19,7 @@ from . import _healpix as hp from . import _rustie +from ._validate import _as_i64, _as_u64 from .orders import ( MAX_ORDER, _rust_mort2nested, @@ -164,9 +165,11 @@ def unique2parent(unique): ------ ValueError If a value is not a valid UNIQ cell number for orders 0-``MAX_ORDER``. + A float-typed ``unique`` or a value past int64 is refused by name + (issue #194, phase 5), never silently cast. """ is_scalar = np.ndim(unique) == 0 - u = np.atleast_1d(np.asarray(unique, dtype=np.int64)) + u = _as_i64(unique, "unique") # int64, not the public uint8: the shifts below would otherwise run in # uint8 and wrap (the same trap order2res documents for `orders_of`). orders = orders_of_uniq(u).astype(np.int64) @@ -212,12 +215,19 @@ def norm2mort(normed, parent, order): Packed morton word(s) — a ``uint64`` scalar when both ``normed`` and ``parent`` are scalars, a 1-D array (of the broadcast length, length 1 included) whenever either is an array. + + Raises + ------ + ValueError + If ``normed`` or ``parent`` is float-typed or negative — refused by + name (issue #194, phase 5) rather than silently cast into a + different, possibly valid, word. """ # Rank of the *inputs*, read before coercion: it is what selects the form, # so a length-1 array stays an array (issue #187). is_scalar = np.ndim(normed) == 0 and np.ndim(parent) == 0 - normed = np.atleast_1d(np.asarray(normed, dtype=np.int64)) - parent = np.atleast_1d(np.asarray(parent, dtype=np.int64)) + normed = _as_u64(normed, "normed") + parent = _as_u64(parent, "parent") # nested = parent * nside^2 + normed; pack via the kernel bridge. nested = (parent.astype(np.uint64) << np.uint64(2 * order)) | normed.astype( np.uint64 @@ -477,7 +487,7 @@ def mort2norm(morton): # norm2mort follows -- the pair is documented as exact inverses, and a # squeeze on one side alone made the round trip lose its shape. is_scalar = np.ndim(morton) == 0 - morton = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + morton = _as_u64(morton, "morton") # Empty input: nothing to decode. Return empty int64 arrays (matching the # array-path dtype) and order 0. @@ -545,8 +555,18 @@ def norm2uniq(normed, parent, order=MAX_ORDER): ------ ValueError If an order lies outside 0-``MAX_ORDER``, or an order array's length - does not match the input. + does not match the input, or ``normed`` / ``parent`` is float-typed + or negative -- refused by name (issue #194, phase 5) rather than + silently cast into a different, possibly valid, UNIQ id. """ + # Validated, not rebound: :func:`norm2mort` takes the same unsigned intake + # on the same two operands, and this is the documented producer of the ids + # its two consumers now refuse floats for. The arithmetic below keeps the + # caller's own dtypes and scalar/array form, so valid input answers exactly + # as before -- `norm2uniq(-3, 0, 4)` used to answer 1021, a real order-3 + # cell in base 11, with no error at any point downstream. + _as_u64(normed, "normed") + _as_u64(parent, "parent") bcast = np.broadcast(np.asarray(normed), np.asarray(parent)) order = _encoder_orders(order, bcast.size) if isinstance(order, np.ndarray) and len(bcast.shape) > 1: @@ -622,11 +642,13 @@ def uniq2geo(uniq, *, latitude="authalic"): ------ ValueError If a value is not a valid UNIQ cell number for orders 0-``MAX_ORDER``, - or *latitude* is not a valid convention. + or *latitude* is not a valid convention. A float-typed ``uniq`` or a + value past int64 is refused by name (issue #194, phase 5), never + silently cast. """ _check_latitude(latitude) is_scalar = np.ndim(uniq) == 0 - u = np.atleast_1d(np.asarray(uniq, dtype=np.int64)) + u = _as_i64(uniq, "uniq") # int64, not the public uint8 -- see the note in unique2parent. orders = orders_of_uniq(u).astype(np.int64) @@ -685,7 +707,7 @@ def mort2geo(morton, *, latitude="authalic"): # Group-by-order dispatch for mixed-order input (issue #116). if not input_is_scalar: - words = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + words = _as_u64(morton, "morton") orders = orders_of(words) unique_orders = np.unique(orders) if unique_orders.size > 1: @@ -749,7 +771,7 @@ def mort2bbox(morton, *, latitude="authalic"): morton = np.atleast_1d(morton) is_scalar = len(morton) == 1 - words = np.asarray(morton, dtype=np.uint64) + words = _as_u64(morton, "morton") # Group-by-order dispatch for mixed-order input (issue #116). orders = orders_of(words) unique_orders = np.unique(orders) @@ -950,7 +972,7 @@ def mort2polygon(morton, step=1, *, latitude="authalic"): morton = np.atleast_1d(morton) is_scalar = len(morton) == 1 - words = np.asarray(morton, dtype=np.uint64) + words = _as_u64(morton, "morton") # Group-by-order dispatch for mixed-order input (issue #116). orders = orders_of(words) unique_orders = np.unique(orders) diff --git a/mortie/geometry.py b/mortie/geometry.py index f9a53a8e..7ca3bd6f 100644 --- a/mortie/geometry.py +++ b/mortie/geometry.py @@ -22,6 +22,7 @@ import numpy as np +from ._validate import _as_offsets, _as_u64 from .codec import ( _geometry_from_wkt, _geometry_to_wkb, @@ -386,9 +387,9 @@ def _wkb_column_views(data, offsets): blob ``i`` spanning ``data[offsets[i]:offsets[i + 1]]``. offsets : array_like ``int64`` arrow list offsets. Must exactly cover ``data`` — - ``offsets[0] == 0`` and ``offsets[-1] == len(data)``. Coerced with - ``np.asarray(..., dtype=np.int64)`` as the rest of the batch family - is, so float offsets truncate toward zero. + ``offsets[0] == 0`` and ``offsets[-1] == len(data)``. Validated + strictly, as everywhere in the family (issue #194): float-typed or + past-int64 offsets are refused by name, never silently cast. Returns ------- @@ -423,12 +424,7 @@ def _wkb_column_views(data, offsets): f"{view.itemsize}-byte items (format {view.format!r})" ) view = view.cast("B") # shape normalization only; itemsize is already 1 - try: - off = np.asarray(offsets, dtype=np.int64).ravel() - except OverflowError: - raise ValueError( - "offsets must fit in int64 (arrow list offsets)" - ) from None + off = _as_offsets(offsets) if off.size == 0: raise ValueError("offsets must have at least one element") if off[0] != 0: @@ -701,7 +697,7 @@ def _per_cell_polygons(mod, morton, step, latitude): from .convert import mort2polygon from .orders import _rust_mort2nested - morton = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + morton = _as_u64(morton, "morton") if morton.size == 0: return [] @@ -754,7 +750,9 @@ def to_geometry(morton, dissolve=True, step=1, *, latitude="authalic"): into no exterior (pass ``dissolve=False``). ValueError If *latitude* is not one of the two conventions — checked before the - empty-cover early return, so the contract does not depend on input. + empty-cover early return, so the contract does not depend on input; + or if *morton* is not integer-typed or holds a negative value + (issue #194), refused for both ``dissolve`` arms alike. Notes ----- @@ -775,6 +773,10 @@ def to_geometry(morton, dissolve=True, step=1, *, latitude="authalic"): # Up front: an empty cover short-circuits below either branch, and would # otherwise return silently on an invalid convention (issue #186). _check_latitude(latitude) + # One validation seam for both arms -- and so for to_wkb/to_wkt, which + # route here. The dissolved arm's own coercion (dissolve.py) would + # otherwise truncate floats and wrap negatives on the *default* spelling. + morton = _as_u64(morton, "morton") if dissolve: return mod.MultiPolygon(_dissolved_polygons(mod, morton, step, latitude)) return mod.MultiPolygon(_per_cell_polygons(mod, morton, step, latitude)) @@ -809,6 +811,9 @@ def to_wkb(morton, dissolve=True, step=1, srid=None, *, latitude="authalic"): NotImplementedError As :func:`to_geometry` — a non-shapely backend, or a dissolved hole that nests into no exterior. + ValueError + As :func:`to_geometry` — a bad *latitude*, or a *morton* that is not + integer-typed or holds a negative value (issue #194). See Also -------- @@ -847,6 +852,9 @@ def to_wkt(morton, dissolve=True, step=1, srid=None, *, latitude="authalic"): NotImplementedError As :func:`to_geometry` — a non-shapely backend, or a dissolved hole that nests into no exterior. + ValueError + As :func:`to_geometry` — a bad *latitude*, or a *morton* that is not + integer-typed or holds a negative value (issue #194). See Also -------- diff --git a/mortie/moc_object.py b/mortie/moc_object.py index bc79dab9..75e55f4d 100644 --- a/mortie/moc_object.py +++ b/mortie/moc_object.py @@ -51,6 +51,7 @@ moc_to_order, moc_xor, ) +from ._validate import _as_u64 from .coverage import _FLAT_COVER_WARN_THRESHOLD, _morton_coverage_moc from .orders import orders_of, res2display @@ -92,7 +93,7 @@ def _words(operand): protocol = getattr(operand, "__morton_moc__", None) if protocol is not None: operand = protocol() - return np.asarray(operand, dtype=np.uint64).ravel() + return _as_u64(operand, "operand").ravel() def _nest_depth(node): @@ -330,7 +331,7 @@ def _source_words(source, tolerance, max_cells, latitude): if protocol is not None: _reject_coverage_knobs(tolerance, max_cells, latitude, "already a cover (__morton_moc__)") - return np.asarray(protocol(), dtype=np.uint64).ravel() + return _as_u64(protocol(), "source").ravel() if isinstance(source, dict): groups = _geojson_ring_groups(source) else: @@ -340,7 +341,7 @@ def _source_words(source, tolerance, max_cells, latitude): if words is not None and words.ndim == 1 and words.dtype.kind in "ui": _reject_coverage_knobs(tolerance, max_cells, latitude, "already morton words") - return words.astype(np.uint64, copy=False).ravel() + return _as_u64(words, "source").ravel() groups = [_source_rings(source)] covers = [] for rings in groups: @@ -388,9 +389,10 @@ class Moc: descent — nested rings carve holes — while separate geometries of a ``FeatureCollection`` are **unioned**, since overlapping and nested features are legal there and must add area rather than cancel it. - Words are taken as given: any integer array is cast to ``uint64``, so a - negative or downcast value wraps rather than being rejected here and - fails later inside the kernel. + Words are validated strictly (issue #194): a negative value raises + ``ValueError`` naming it, rather than wrapping into a different -- + possibly valid -- word. Float arrays are geometry here by the rules + above, never words. tolerance : float, optional Stop refining a boundary cell once its angular radius (in degrees) drops to this value; the coverage kernel's angular stop criterion, as diff --git a/mortie/orders.py b/mortie/orders.py index e7cd4a1a..b3316787 100644 --- a/mortie/orders.py +++ b/mortie/orders.py @@ -22,6 +22,7 @@ import numpy as np from . import _rustie +from ._validate import _as_i64, _as_u64 from .batch import _children_of # One row of the res2display resolution ladder (issue #68): the display pair @@ -181,33 +182,18 @@ def orders_of_uniq(uniq): Raises ------ ValueError - If any value lies outside the UNIQ range for orders 0-``MAX_ORDER``. + If ``uniq`` is not integer-typed, or a value does not fit in + ``int64`` -- refused by name (issue #194) -- or if a value lies + outside the UNIQ range for orders 0-``MAX_ORDER``. """ - # Cast defensively: `asarray(..., dtype=int64)` raises OverflowError for a - # value above int64 and silently *truncates* a float, both of which would - # bypass the ValueError this function documents. Normalize them here so the - # contract holds for every input, not just int64-representable ones. - arr = np.atleast_1d(np.asarray(uniq)) - if arr.dtype.kind == "f" and not np.all(np.equal(np.mod(arr, 1), 0)): - raise ValueError( - f"Not a valid UNIQ cell number for orders 0-{MAX_ORDER}: " - f"{arr.ravel()[0]!r} is not an integer") - if arr.dtype.kind == "u": - # uint64 -> int64 *wraps* silently rather than raising, so an oversized - # value would reach the range check as a meaningless negative and be - # reported as such. Every wrap lands negative so nothing mis-decodes as - # valid, but the message would name a number the caller never passed. - over = arr > np.iinfo(np.int64).max - if np.any(over): - raise ValueError( - f"Not a valid UNIQ cell number for orders 0-{MAX_ORDER}: " - f"{int(arr[over].ravel()[0])} is out of the int64 range") - try: - u = np.atleast_1d(np.asarray(arr, dtype=np.int64)) - except (OverflowError, ValueError, TypeError) as exc: - raise ValueError( - f"Not a valid UNIQ cell number for orders 0-{MAX_ORDER}: " - f"{uniq!r} is out of the int64 range") from exc + # Strict intake, shared with the two callers below (`unique2parent` and + # `uniq2geo` validate the same column before handing it here): float is + # refused by dtype rather than by value, so the one UNIQ entry point that + # still decoded an *integral* float -- `orders_of_uniq([16.0]) -> 1` while + # `unique2parent([16.0])` refused -- now answers like the rest of the + # family (issue #194 review). Oversized values keep being named rather + # than wrapping through the int64 cast or leaking numpy's OverflowError. + u = _as_i64(uniq, "uniq") # bounds[k] = 4**(k+1) is the first UNIQ value of order k; the trailing # entry closes order MAX_ORDER's range (4**31 still fits int64). bounds = np.int64(4) ** np.arange(1, MAX_ORDER + 3, dtype=np.int64) @@ -253,7 +239,7 @@ def orders_of(morton): ``uint8`` order per element, 0-29 (scalar in -> length-1 ndarray, matching :func:`geo2mort`). """ - m = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + m = _as_u64(morton, "morton") suffix = (m & np.uint64(0x3F)).astype(np.uint8) # 0..=27: order == suffix. 28..=47: order-28 on the 5-block parent slots, # order 29 otherwise. 48..=63: order-29 point. @@ -286,7 +272,7 @@ def is_point(morton): ``bool`` per element, True for point words (scalar in -> length-1 ndarray, matching :func:`geo2mort`). """ - m = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + m = _as_u64(morton, "morton") return (m & np.uint64(0x3F)) >= np.uint64(48) @@ -318,7 +304,7 @@ def infer_order_from_morton(morton): ValueError If the words are at mixed orders. """ - m = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + m = _as_u64(morton, "morton") _, depths = _rust_mort2nested(np.ascontiguousarray(m)) distinct = np.unique(depths) if distinct.size > 1: @@ -363,7 +349,9 @@ def validate_morton(morton, order=None): Raises ------ ValueError - If a word does not decode -- the kernel's own refusal, which names no + If ``morton`` is float-typed or negative -- refused by name rather + than silently cast (issue #194). Or if a word does not decode -- + the kernel's own refusal, which names no index and **takes precedence** over the order check, since the decode runs first and over the whole array. Or, past a clean decode, if any word's order disagrees with ``order`` -- that refusal names the @@ -399,7 +387,7 @@ def validate_morton(morton, order=None): # array, so it is indexed like one in the message (issue #187, the same # rule norm2mort / mort2norm follow for their return form). is_scalar = np.ndim(morton) == 0 - m = np.atleast_1d(np.asarray(morton, dtype=np.uint64)) + m = _as_u64(morton, "morton") # The kernel raises ValueError on the empty sentinel / an invalid prefix. _, depths = _rust_mort2nested(np.ascontiguousarray(m)) if order is not None: @@ -444,7 +432,7 @@ def clip2order(clip_order, midx): ndarray Coarsened packed words, one per input word. """ - midx = np.ascontiguousarray(np.asarray(midx, dtype=np.uint64).ravel()) + midx = np.ascontiguousarray(_as_u64(midx, "midx").ravel()) return _rustie.rust_mi_coarsen(midx, int(clip_order)) @@ -486,7 +474,8 @@ def generate_morton_children(parent_morton, target_order, *, max_cells=None): ValueError If ``target_order`` is coarser than the parent word's own order, or (array form) the parents do not share one order or the result would - exceed ``max_cells``. + exceed ``max_cells``. A float-typed or negative ``parent_morton`` + is refused by name (issue #194), never silently cast. See Also -------- @@ -507,8 +496,10 @@ def generate_morton_children(parent_morton, target_order, *, max_cells=None): ) if np.ndim(parent_morton) > 0: try: - return _children_of(parent_morton, target_order, - max_cells=max_cells) + # Name the caller-facing parameter before delegating -- the + # kernel's own pass stays as the backstop and sees uint64. + return _children_of(_as_u64(parent_morton, "parent_morton"), + target_order, max_cells=max_cells) except ValueError as exc: # The kernel's refusals predate the plural name's retirement # (issue #187); re-raise naming the surviving entry point. Same @@ -517,7 +508,7 @@ def generate_morton_children(parent_morton, target_order, *, max_cells=None): str(exc).replace("children_of", "generate_morton_children") ) from None # Decode the parent to its (nested, depth) via the packed kernel. - parent_morton = np.uint64(parent_morton) + parent_morton = _as_u64(parent_morton, "parent_morton")[0] nested, depths = _rust_mort2nested( np.ascontiguousarray(np.atleast_1d(parent_morton)) ) diff --git a/mortie/prefix_trie.py b/mortie/prefix_trie.py index 0cc7191b..c089bec6 100644 --- a/mortie/prefix_trie.py +++ b/mortie/prefix_trie.py @@ -301,6 +301,9 @@ def split_children(morton_array, max_depth=4): ---------- morton_array : array-like of int Morton indices (packed ``uint64`` words; base cells 7-11 set bit 63). + An ``int64`` bit-view of packed words is accepted -- the sign it + shows *is* the southern flag the characteristic branches on -- but a + float-typed array is refused (issue #194). max_depth : int or None Maximum branching depth. ``None`` means full recursion. Default is 4. @@ -313,11 +316,23 @@ def split_children(morton_array, max_depth=4): Raises ------ ValueError - If *morton_array* is empty or not 1-D. + If *morton_array* is float-typed (issue #194), or is empty, a scalar, + or not 1-D. """ - morton_array = np.ascontiguousarray(np.asarray(morton_array, dtype=np.uint64)) - if morton_array.ndim != 1 or len(morton_array) == 0: + # Not atleast_1d: a 0-D scalar must keep its rank so the 1-D check below + # still refuses it, as it did before this seam was validated. + arr = np.asarray(morton_array) + if arr.size and arr.dtype.kind not in "iu": + # Strict on floats like the rest of the family (issue #194), but the + # signed int64 bit-view of packed words stays accepted here: the trie + # branches on the decimal characteristic, whose first column *is* the + # sign (bit 63, the southern base cells), and the golden fixtures pin + # that form. + raise ValueError( + f"morton_array must be integer-typed, got dtype {arr.dtype}") + if arr.ndim != 1 or arr.size == 0: raise ValueError("morton_array must be a non-empty 1-D integer array") + morton_array = np.ascontiguousarray(arr.astype(np.uint64, copy=False)) flat_nodes, permutation = _rust_split_children( morton_array, max_depth=max_depth diff --git a/mortie/tests/data/strict_validation_goldens.json b/mortie/tests/data/strict_validation_goldens.json new file mode 100644 index 00000000..ca3ee03a --- /dev/null +++ b/mortie/tests/data/strict_validation_goldens.json @@ -0,0 +1,1108 @@ +{ + "words": [ + 1202461100507922436, + 10407818738853216260, + 13848568854164275204 + ], + "words_b": [ + 1202461100507922436, + 11560740243460063236 + ], + "compress_moc": [ + 1202461100507922436, + 10407818738853216260, + 13848568854164275204 + ], + "moc_to_order": [ + 1202461100507922439, + 1202531469252100103, + 1202601837996277767, + 1202672206740455431, + 1202742575484633095, + 1202812944228810759, + 1202883312972988423, + 1202953681717166087, + 1203024050461343751, + 1203094419205521415, + 1203164787949699079, + 1203235156693876743, + 1203305525438054407, + 1203375894182232071, + 1203446262926409735, + 1203516631670587399, + 1203587000414765063, + 1203657369158942727, + 1203727737903120391, + 1203798106647298055, + 1203868475391475719, + 1203938844135653383, + 1204009212879831047, + 1204079581624008711, + 1204149950368186375, + 1204220319112364039, + 1204290687856541703, + 1204361056600719367, + 1204431425344897031, + 1204501794089074695, + 1204572162833252359, + 1204642531577430023, + 1204712900321607687, + 1204783269065785351, + 1204853637809963015, + 1204924006554140679, + 1204994375298318343, + 1205064744042496007, + 1205135112786673671, + 1205205481530851335, + 1205275850275028999, + 1205346219019206663, + 1205416587763384327, + 1205486956507561991, + 1205557325251739655, + 1205627693995917319, + 1205698062740094983, + 1205768431484272647, + 1205838800228450311, + 1205909168972627975, + 1205979537716805639, + 1206049906460983303, + 1206120275205160967, + 1206190643949338631, + 1206261012693516295, + 1206331381437693959, + 1206401750181871623, + 1206472118926049287, + 1206542487670226951, + 1206612856414404615, + 1206683225158582279, + 1206753593902759943, + 1206823962646937607, + 1206894331391115271, + 10407818738853216263, + 10407889107597393927, + 10407959476341571591, + 10408029845085749255, + 10408100213829926919, + 10408170582574104583, + 10408240951318282247, + 10408311320062459911, + 10408381688806637575, + 10408452057550815239, + 10408522426294992903, + 10408592795039170567, + 10408663163783348231, + 10408733532527525895, + 10408803901271703559, + 10408874270015881223, + 10408944638760058887, + 10409015007504236551, + 10409085376248414215, + 10409155744992591879, + 10409226113736769543, + 10409296482480947207, + 10409366851225124871, + 10409437219969302535, + 10409507588713480199, + 10409577957457657863, + 10409648326201835527, + 10409718694946013191, + 10409789063690190855, + 10409859432434368519, + 10409929801178546183, + 10410000169922723847, + 10410070538666901511, + 10410140907411079175, + 10410211276155256839, + 10410281644899434503, + 10410352013643612167, + 10410422382387789831, + 10410492751131967495, + 10410563119876145159, + 10410633488620322823, + 10410703857364500487, + 10410774226108678151, + 10410844594852855815, + 10410914963597033479, + 10410985332341211143, + 10411055701085388807, + 10411126069829566471, + 10411196438573744135, + 10411266807317921799, + 10411337176062099463, + 10411407544806277127, + 10411477913550454791, + 10411548282294632455, + 10411618651038810119, + 10411689019782987783, + 10411759388527165447, + 10411829757271343111, + 10411900126015520775, + 10411970494759698439, + 10412040863503876103, + 10412111232248053767, + 10412181600992231431, + 10412251969736409095, + 13848568854164275207, + 13848639222908452871, + 13848709591652630535, + 13848779960396808199, + 13848850329140985863, + 13848920697885163527, + 13848991066629341191, + 13849061435373518855, + 13849131804117696519, + 13849202172861874183, + 13849272541606051847, + 13849342910350229511, + 13849413279094407175, + 13849483647838584839, + 13849554016582762503, + 13849624385326940167, + 13849694754071117831, + 13849765122815295495, + 13849835491559473159, + 13849905860303650823, + 13849976229047828487, + 13850046597792006151, + 13850116966536183815, + 13850187335280361479, + 13850257704024539143, + 13850328072768716807, + 13850398441512894471, + 13850468810257072135, + 13850539179001249799, + 13850609547745427463, + 13850679916489605127, + 13850750285233782791, + 13850820653977960455, + 13850891022722138119, + 13850961391466315783, + 13851031760210493447, + 13851102128954671111, + 13851172497698848775, + 13851242866443026439, + 13851313235187204103, + 13851383603931381767, + 13851453972675559431, + 13851524341419737095, + 13851594710163914759, + 13851665078908092423, + 13851735447652270087, + 13851805816396447751, + 13851876185140625415, + 13851946553884803079, + 13852016922628980743, + 13852087291373158407, + 13852157660117336071, + 13852228028861513735, + 13852298397605691399, + 13852368766349869063, + 13852439135094046727, + 13852509503838224391, + 13852579872582402055, + 13852650241326579719, + 13852720610070757383, + 13852790978814935047, + 13852861347559112711, + 13852931716303290375, + 13853002085047468039 + ], + "moc_to_order_ragged": [ + [ + 1202461100507922439, + 1202531469252100103, + 1202601837996277767, + 1202672206740455431, + 1202742575484633095, + 1202812944228810759, + 1202883312972988423, + 1202953681717166087, + 1203024050461343751, + 1203094419205521415, + 1203164787949699079, + 1203235156693876743, + 1203305525438054407, + 1203375894182232071, + 1203446262926409735, + 1203516631670587399, + 1203587000414765063, + 1203657369158942727, + 1203727737903120391, + 1203798106647298055, + 1203868475391475719, + 1203938844135653383, + 1204009212879831047, + 1204079581624008711, + 1204149950368186375, + 1204220319112364039, + 1204290687856541703, + 1204361056600719367, + 1204431425344897031, + 1204501794089074695, + 1204572162833252359, + 1204642531577430023, + 1204712900321607687, + 1204783269065785351, + 1204853637809963015, + 1204924006554140679, + 1204994375298318343, + 1205064744042496007, + 1205135112786673671, + 1205205481530851335, + 1205275850275028999, + 1205346219019206663, + 1205416587763384327, + 1205486956507561991, + 1205557325251739655, + 1205627693995917319, + 1205698062740094983, + 1205768431484272647, + 1205838800228450311, + 1205909168972627975, + 1205979537716805639, + 1206049906460983303, + 1206120275205160967, + 1206190643949338631, + 1206261012693516295, + 1206331381437693959, + 1206401750181871623, + 1206472118926049287, + 1206542487670226951, + 1206612856414404615, + 1206683225158582279, + 1206753593902759943, + 1206823962646937607, + 1206894331391115271, + 10407818738853216263, + 10407889107597393927, + 10407959476341571591, + 10408029845085749255, + 10408100213829926919, + 10408170582574104583, + 10408240951318282247, + 10408311320062459911, + 10408381688806637575, + 10408452057550815239, + 10408522426294992903, + 10408592795039170567, + 10408663163783348231, + 10408733532527525895, + 10408803901271703559, + 10408874270015881223, + 10408944638760058887, + 10409015007504236551, + 10409085376248414215, + 10409155744992591879, + 10409226113736769543, + 10409296482480947207, + 10409366851225124871, + 10409437219969302535, + 10409507588713480199, + 10409577957457657863, + 10409648326201835527, + 10409718694946013191, + 10409789063690190855, + 10409859432434368519, + 10409929801178546183, + 10410000169922723847, + 10410070538666901511, + 10410140907411079175, + 10410211276155256839, + 10410281644899434503, + 10410352013643612167, + 10410422382387789831, + 10410492751131967495, + 10410563119876145159, + 10410633488620322823, + 10410703857364500487, + 10410774226108678151, + 10410844594852855815, + 10410914963597033479, + 10410985332341211143, + 10411055701085388807, + 10411126069829566471, + 10411196438573744135, + 10411266807317921799, + 10411337176062099463, + 10411407544806277127, + 10411477913550454791, + 10411548282294632455, + 10411618651038810119, + 10411689019782987783, + 10411759388527165447, + 10411829757271343111, + 10411900126015520775, + 10411970494759698439, + 10412040863503876103, + 10412111232248053767, + 10412181600992231431, + 10412251969736409095, + 13848568854164275207, + 13848639222908452871, + 13848709591652630535, + 13848779960396808199, + 13848850329140985863, + 13848920697885163527, + 13848991066629341191, + 13849061435373518855, + 13849131804117696519, + 13849202172861874183, + 13849272541606051847, + 13849342910350229511, + 13849413279094407175, + 13849483647838584839, + 13849554016582762503, + 13849624385326940167, + 13849694754071117831, + 13849765122815295495, + 13849835491559473159, + 13849905860303650823, + 13849976229047828487, + 13850046597792006151, + 13850116966536183815, + 13850187335280361479, + 13850257704024539143, + 13850328072768716807, + 13850398441512894471, + 13850468810257072135, + 13850539179001249799, + 13850609547745427463, + 13850679916489605127, + 13850750285233782791, + 13850820653977960455, + 13850891022722138119, + 13850961391466315783, + 13851031760210493447, + 13851102128954671111, + 13851172497698848775, + 13851242866443026439, + 13851313235187204103, + 13851383603931381767, + 13851453972675559431, + 13851524341419737095, + 13851594710163914759, + 13851665078908092423, + 13851735447652270087, + 13851805816396447751, + 13851876185140625415, + 13851946553884803079, + 13852016922628980743, + 13852087291373158407, + 13852157660117336071, + 13852228028861513735, + 13852298397605691399, + 13852368766349869063, + 13852439135094046727, + 13852509503838224391, + 13852579872582402055, + 13852650241326579719, + 13852720610070757383, + 13852790978814935047, + 13852861347559112711, + 13852931716303290375, + 13853002085047468039, + 1202461100507922439, + 1202531469252100103, + 1202601837996277767, + 1202672206740455431, + 1202742575484633095, + 1202812944228810759, + 1202883312972988423, + 1202953681717166087, + 1203024050461343751, + 1203094419205521415, + 1203164787949699079, + 1203235156693876743, + 1203305525438054407, + 1203375894182232071, + 1203446262926409735, + 1203516631670587399, + 1203587000414765063, + 1203657369158942727, + 1203727737903120391, + 1203798106647298055, + 1203868475391475719, + 1203938844135653383, + 1204009212879831047, + 1204079581624008711, + 1204149950368186375, + 1204220319112364039, + 1204290687856541703, + 1204361056600719367, + 1204431425344897031, + 1204501794089074695, + 1204572162833252359, + 1204642531577430023, + 1204712900321607687, + 1204783269065785351, + 1204853637809963015, + 1204924006554140679, + 1204994375298318343, + 1205064744042496007, + 1205135112786673671, + 1205205481530851335, + 1205275850275028999, + 1205346219019206663, + 1205416587763384327, + 1205486956507561991, + 1205557325251739655, + 1205627693995917319, + 1205698062740094983, + 1205768431484272647, + 1205838800228450311, + 1205909168972627975, + 1205979537716805639, + 1206049906460983303, + 1206120275205160967, + 1206190643949338631, + 1206261012693516295, + 1206331381437693959, + 1206401750181871623, + 1206472118926049287, + 1206542487670226951, + 1206612856414404615, + 1206683225158582279, + 1206753593902759943, + 1206823962646937607, + 1206894331391115271, + 11560740243460063239, + 11560810612204240903, + 11560880980948418567, + 11560951349692596231, + 11561021718436773895, + 11561092087180951559, + 11561162455925129223, + 11561232824669306887, + 11561303193413484551, + 11561373562157662215, + 11561443930901839879, + 11561514299646017543, + 11561584668390195207, + 11561655037134372871, + 11561725405878550535, + 11561795774622728199, + 11561866143366905863, + 11561936512111083527, + 11562006880855261191, + 11562077249599438855, + 11562147618343616519, + 11562217987087794183, + 11562288355831971847, + 11562358724576149511, + 11562429093320327175, + 11562499462064504839, + 11562569830808682503, + 11562640199552860167, + 11562710568297037831, + 11562780937041215495, + 11562851305785393159, + 11562921674529570823, + 11562992043273748487, + 11563062412017926151, + 11563132780762103815, + 11563203149506281479, + 11563273518250459143, + 11563343886994636807, + 11563414255738814471, + 11563484624482992135, + 11563554993227169799, + 11563625361971347463, + 11563695730715525127, + 11563766099459702791, + 11563836468203880455, + 11563906836948058119, + 11563977205692235783, + 11564047574436413447, + 11564117943180591111, + 11564188311924768775, + 11564258680668946439, + 11564329049413124103, + 11564399418157301767, + 11564469786901479431, + 11564540155645657095, + 11564610524389834759, + 11564680893134012423, + 11564751261878190087, + 11564821630622367751, + 11564891999366545415, + 11564962368110723079, + 11565032736854900743, + 11565103105599078407, + 11565173474343256071 + ], + [ + 0, + 192, + 320 + ] + ], + "moc_or": [ + 1202461100507922436, + 10407818738853216260, + 11560740243460063236, + 13848568854164275204 + ], + "moc_and": [ + 1202461100507922436 + ], + "moc_and_ragged": [ + [ + 1202461100507922436, + 10407818738853216260, + 13848568854164275204, + 1202461100507922436 + ], + [ + 0, + 3, + 4 + ] + ], + "moc_intersects": true, + "moc_intersects_ragged": [ + true, + true + ], + "moc_minus": [ + 10407818738853216260, + 13848568854164275204 + ], + "moc_xor": [ + 10407818738853216260, + 11560740243460063236, + 13848568854164275204 + ], + "moc_min": [ + 1202461100507922436 + ], + "moc_not": [ + 1152921504606846979, + 1170935903116328963, + 1188950301625810948, + 1193453901253181444, + 1197957500880551940, + 1206964700135292931, + 1224979098644774914, + 1297036692682702850, + 1369094286720630786, + 1441151880758558721, + 1729382256910270465, + 2017612633061982209, + 2305843009213693952, + 3458764513820540928, + 4611686018427387904, + 5764607523034234880, + 6917529027641081856, + 8070450532247928832, + 9223372036854775808, + 10376293541461622787, + 10394307939971104772, + 10398811539598475268, + 10403315139225845764, + 10412322338480586755, + 10430336736990068739, + 10448351135499550722, + 10520408729537478658, + 10592466323575406594, + 10664523917613334529, + 10952754293765046273, + 11240984669916758017, + 11529215046068469760, + 12682136550675316736, + 13835058055282163716, + 13839561654909534212, + 13844065254536904708, + 13853072453791645699, + 13871086852301127683, + 13889101250810609667, + 13907115649320091650, + 13979173243358019586, + 14051230837395947522, + 14123288431433875457, + 14411518807585587201, + 14699749183737298945 + ], + "moc_not_domain": [ + 1204712900321607685, + 1205838800228450309, + 10407818738853216260, + 13848568854164275204 + ], + "common_ancestor": [ + 1202461100507922436 + ], + "common_ancestor_ragged": [ + 1202461100507922436, + 11560740243460063236 + ], + "split_base_cells_values": [ + [ + 1202461100507922436 + ], + [ + 10407818738853216260 + ], + [ + 13848568854164275204 + ], + [ + 11560740243460063236 + ] + ], + "polygons_to_morton_mocs": [ + [ + 6052556424209235974, + 6237204008931426310, + 6239737283721822213, + 6242833508465639430, + 6243114983442350085, + 6244240883349192709, + 6245366783256035333, + 6246492683162877957, + 6248744482976563205, + 6249870382883405830, + 6250433332836827142, + 6437051242395992070, + 6629298651489370117, + 6630424551396212741, + 6631831926279766022, + 6632676351209897989, + 6633802251116740613, + 6634928151023583238, + 6635491100977004550, + 6636054050930425861, + 6637179950837268486, + 6637742900790689798, + 6639713225627664390, + 6642809450371481605, + 6643935350278324230, + 6644498300231745542, + 6645342725161877510, + 6646187150092009478 + ], + [ + 0, + 28 + ] + ], + "generate_morton_children_scalar": [ + 1202461100507922438, + 1202742575484633094, + 1203024050461343750, + 1203305525438054406, + 1203587000414765062, + 1203868475391475718, + 1204149950368186374, + 1204431425344897030, + 1204712900321607686, + 1204994375298318342, + 1205275850275028998, + 1205557325251739654, + 1205838800228450310, + 1206120275205160966, + 1206401750181871622, + 1206683225158582278 + ], + "generate_morton_children_array": [ + 1202461100507922438, + 1202742575484633094, + 1203024050461343750, + 1203305525438054406, + 1203587000414765062, + 1203868475391475718, + 1204149950368186374, + 1204431425344897030, + 1204712900321607686, + 1204994375298318342, + 1205275850275028998, + 1205557325251739654, + 1205838800228450310, + 1206120275205160966, + 1206401750181871622, + 1206683225158582278, + 10407818738853216262, + 10408100213829926918, + 10408381688806637574, + 10408663163783348230, + 10408944638760058886, + 10409226113736769542, + 10409507588713480198, + 10409789063690190854, + 10410070538666901510, + 10410352013643612166, + 10410633488620322822, + 10410914963597033478, + 10411196438573744134, + 10411477913550454790, + 10411759388527165446, + 10412040863503876102 + ], + "clip2order": [ + 1202461100507922436, + 1202461100507922436, + 1202461100507922436, + 1202461100507922436 + ], + "orders_of": [ + 4, + 4, + 4, + 4, + 4 + ], + "is_point": [ + false, + false, + false, + false, + false + ], + "infer_order_from_morton": 5, + "validate_morton": true, + "mort2norm_normed": [ + 44, + 45, + 46, + 47 + ], + "mort2norm_parent": [ + 0, + 0, + 0, + 0 + ], + "mort2norm_order": [ + 5 + ], + "mort2geo": [ + [ + 10.854294464723, + 12.07710662623, + 12.07710662623, + 13.305385094652 + ], + [ + 39.375, + 40.78125, + 37.96875, + 39.375 + ] + ], + "morton_buffer": [ + 1192328001346338821, + 1195705701066866693, + 1196831600973709317, + 1199083400787394565, + 1201335200601079813, + 1209216499948978181, + 1215971899390033925, + 1218223699203719173, + 1298162592589545477, + 1301540292310073349, + 1302666192216915973, + 1315051091192184837 + ], + "morton_buffer_meters": [ + 1192328001346338821, + 1195705701066866693, + 1196831600973709317, + 1199083400787394565, + 1201335200601079813, + 1209216499948978181, + 1215971899390033925, + 1218223699203719173, + 1298162592589545477, + 1301540292310073349, + 1302666192216915973, + 1315051091192184837 + ], + "split_children_roots": [ + "-31124", + "-41124", + "-61114", + "11134" + ], + "moc_object_and": [ + 1202461100507922436 + ], + "mort2bbox": [ + [ + 37.96875, + 9.636338620241, + 40.78125, + 12.07710662623 + ], + [ + 39.375, + 10.854294464723, + 42.1875, + 13.305385094652 + ], + [ + 36.5625, + 10.854294464723, + 39.375, + 13.305385094652 + ], + [ + 37.96875, + 12.07710662623, + 40.78125, + 14.539764060139 + ] + ], + "mort2polygon": [ + [ + [ + 12.07710662623, + 39.375 + ], + [ + 10.854294464723, + 37.96875 + ], + [ + 9.636338620241, + 39.375 + ], + [ + 10.854294464723, + 40.78125 + ], + [ + 12.07710662623, + 39.375 + ] + ], + [ + [ + 13.305385094652, + 40.78125 + ], + [ + 12.07710662623, + 39.375 + ], + [ + 10.854294464723, + 40.78125 + ], + [ + 12.07710662623, + 42.1875 + ], + [ + 13.305385094652, + 40.78125 + ] + ], + [ + [ + 13.305385094652, + 37.96875 + ], + [ + 12.07710662623, + 36.5625 + ], + [ + 10.854294464723, + 37.96875 + ], + [ + 12.07710662623, + 39.375 + ], + [ + 13.305385094652, + 37.96875 + ] + ], + [ + [ + 14.539764060139, + 39.375 + ], + [ + 13.305385094652, + 37.96875 + ], + [ + 12.07710662623, + 39.375 + ], + [ + 13.305385094652, + 40.78125 + ], + [ + 14.539764060139, + 39.375 + ] + ] + ], + "norm2mort": [ + 1202461100507922436, + 10407818738853216260, + 13848568854164275204 + ], + "norm2uniq": [ + 1035, + 3079, + 3843, + 4140, + 4143 + ], + "unique2parent": [ + 0, + 8, + 11, + 0, + 0 + ], + "uniq2geo": [ + [ + 12.07710662623, + -75.403416070074, + -81.258436102474, + 10.854294464723, + 13.305385094652 + ], + [ + 39.375, + 63.0, + 315.0, + 39.375, + 39.375 + ] + ], + "from_wkb_ragged": [ + [ + 6052556424209235974, + 6237204008931426310, + 6239737283721822213, + 6242833508465639430, + 6243114983442350085, + 6244240883349192709, + 6245366783256035333, + 6246492683162877957, + 6248744482976563205, + 6249870382883405830, + 6250433332836827142, + 6437051242395992070, + 6629298651489370117, + 6630424551396212741, + 6631831926279766022, + 6632676351209897989, + 6633802251116740613, + 6634928151023583238, + 6635491100977004550, + 6636054050930425861, + 6637179950837268486, + 6637742900790689798, + 6639713225627664390, + 6642809450371481605, + 6643935350278324230, + 6644498300231745542, + 6645342725161877510, + 6646187150092009478, + 1343480063839961094, + 1343761538816671750, + 1344043013793382406, + 1344605963746803718, + 1344887438723514374, + 1345168913700225030, + 1345450388676935685, + 1346576288583778310, + 1347139238537199622, + 1348828088397463558, + 1349391038350884870, + 1729382256910270468, + 1733885856537640966, + 1734448806491062278, + 1736137656351326214, + 1736700606304747526, + 1738389456165011462, + 1738670931141722118, + 1739233881095143430, + 1739515356071854085, + 1741767155885539334, + 1742048630862249990, + 1742611580815671302, + 1742893055792381958, + 1743456005745803270, + 1745144855606067206, + 1745707805559488518, + 1769914653556604934, + 6725281618547703814, + 6725563093524414470, + 6725844568501125126, + 6727251943384678406, + 6727533418361389062, + 6728096368314810374 + ], + [ + 0, + 28, + 62 + ] + ], + "time2toc": [ + 2000001530494976, + 4000000913506304, + 6000000296517632 + ], + "span2toc": [ + 1999998766721148 + ], + "toc2time": [ + [ + 1000000000000000, + 2000000000000000, + 3000000000000000 + ], + [ + 1000000000000000, + 2000000000000000, + 3000000000000000 + ] + ], + "toc_reduce": [ + 1999998766721148 + ], + "toc_reduce_ragged": [ + [ + 2000001530494976 + ], + [ + 3999997532743804 + ] + ], + "toc_normalize": [ + 2000001530494976, + 4000000913506304, + 6000000296517632 + ], + "toc_and": [ + 2000001530494976, + 4000000913506304, + 6000000296517632 + ], + "toc_merge": [ + 1999998766488318 + ], + "from_gps_ns": [ + 4103790400000000000, + 4104790400000000000, + 4105790400000000000 + ], + "to_gps_ns": [ + 97209600000000000, + 197209600000000000 + ], + "to_datetime64": [ + "1850-01-12T13:46:40.000000000", + "1850-01-24T03:33:20.000000000", + "1850-02-04T17:20:00.000000000" + ] +} diff --git a/mortie/tests/generate_strict_goldens.py b/mortie/tests/generate_strict_goldens.py new file mode 100644 index 00000000..bf201a1f --- /dev/null +++ b/mortie/tests/generate_strict_goldens.py @@ -0,0 +1,227 @@ +"""Generate valid-path goldens for the strict-validation sweep (issue #194). + +Captured at ``4900a7e`` -- the commit *before* the shared strict validators +were adopted family-wide -- so ``test_strict_validation.py`` can pin that the +touched entry points answer byte-identically for valid input before and after +the posture change. Regenerating on a later commit only re-asserts the +current answers; the committed JSON is the pre-change record. + +**What the capture covers, and what it does not.** Every word/offset seam +this PR validated that answers with plain numbers, across every phase: +the ``_moc`` operators (both arms), ``batch``'s ragged forms, ``orders``, +``convert`` (``mort2norm`` / ``mort2geo`` / ``mort2bbox`` / ``mort2polygon``, +and the phase-5 UNIQ/normed intakes ``norm2mort`` / ``norm2uniq`` / +``unique2parent`` / ``uniq2geo``), +``buffer``, ``prefix_trie``, ``Moc``, ``geometry.from_wkb(offsets=)``, and the +whole **toc** family -- whose ``_as_u64`` / ``_as_offsets`` moved house in +phase 1, so its valid answers are pinned here rather than left to prose. + +Two deliberate omissions. ``to_geometry`` / ``to_wkb`` / ``to_wkt`` need the +shapely backend, which is a test extra rather than a runtime dependency; this +generator stays numpy-only so the golden test never turns on an optional +install, and those three are pinned instead by ``test_geometry.py``'s own +behavior suite plus the refusal rows in ``test_strict_validation.py``. And +``time2toc([])`` is *not* captured: the untyped-empty acceptance deliberately +changed its answer from a refusal to an empty cover (see the CHANGELOG and +the PR's Questions for review), so a golden there would pin the one valid +path this PR does not claim is unchanged. + +Run from the repo root:: + + python mortie/tests/generate_strict_goldens.py + +Writes ``mortie/tests/data/strict_validation_goldens.json``. +""" + +import json +import pathlib + +import numpy as np + +import mortie + +OUT = pathlib.Path(__file__).parent / "data" / "strict_validation_goldens.json" + + +def _ints(arr): + """Flatten an array to a JSON-serializable list of Python ints. + + Parameters + ---------- + arr : array_like + Integer array of any shape. + + Returns + ------- + list of int + ``arr`` raveled, as plain ints. + """ + return [int(x) for x in np.asarray(arr).ravel()] + + +def capture(): + """Capture one answer per touched entry point (see the module docstring). + + Numpy-only by design -- no optional backend -- so the two shapely-gated + emit surfaces are covered elsewhere. + + Returns + ------- + dict + Golden entry name -> JSON-serializable answer. + """ + # Mixed-order words across northern and southern base cells: base cells + # 7..11 set bit 63 (spec section 1, "Unsigned storage"), so the set pins + # that words >= 2**63 stay byte-identical through the validators. + parents4 = np.asarray(mortie.norm2mort([11, 7, 3], [0, 8, 11], 4)) + kids5 = np.asarray( + mortie.norm2mort([11 * 4 + s for s in range(4)], [0] * 4, 5)) + kids5_south = np.asarray( + mortie.norm2mort([7 * 4 + s for s in range(4)], [9] * 4, 5)) + cover_a = np.asarray(mortie.compress_moc(np.concatenate([parents4, kids5]))) + cover_b = np.asarray(mortie.compress_moc(np.concatenate([kids5, kids5_south]))) + ragged = np.concatenate([cover_a, cover_b]) + ragged_off = [0, cover_a.size, cover_a.size + cover_b.size] + groups = np.concatenate([kids5, kids5_south]) + groups_off = [0, 4, 8] + + tri_lats = [0.0, 0.0, 8.0] + tri_lons = [0.0, 8.0, 0.0] + + # A two-blob WKB column for the from_wkb(offsets=) seam. Hard-coded hex + # rather than built with shapely, so this stays a numpy-only capture. + wkb_a = bytes.fromhex( + "010300000001000000040000000000000000000000000000000000000000000000" + "00002040000000000000000000000000000000000000000000002040000000000000" + "0000000000000000000000000000000000") + wkb_b = bytes.fromhex( + "0103000000010000000400000000000000000034400000000000003440000000000" + "0003C40000000000000344000000000000034400000000000003C40000000000000" + "34400000000000003440") + wkb_buf = np.frombuffer(wkb_a + wkb_b, dtype=np.uint8) + wkb_off = [0, len(wkb_a), len(wkb_a) + len(wkb_b)] + + # Toc words: phase 1 moved _as_u64/_as_offsets out from under _toc.py, so + # the toc family's valid answers belong in the pre-change record too. + t_ns = np.asarray([10**15, 2 * 10**15, 3 * 10**15], dtype=np.uint64) + toc_words = np.asarray(mortie.time2toc(t_ns)) + toc_off = [0, 1, 3] + poly_vals, poly_off = mortie.polygons_to_morton_mocs( + tri_lats, tri_lons, [0, 3], order=6) + + # UNIQ ids for the phase-5 intakes: the same three order-4 cells as + # `parents4` plus two order-5 children, so `uniq2geo`'s group-by-order + # dispatch runs over a genuinely mixed-resolution column. + uniq4 = np.asarray(mortie.norm2uniq( + np.asarray([11, 7, 3]), np.asarray([0, 8, 11]), 4)) + uniq5 = np.asarray(mortie.norm2uniq( + np.asarray([11 * 4, 11 * 4 + 3]), np.asarray([0, 0]), 5)) + uniq_mixed = np.concatenate([uniq4, uniq5]) + + and_vals, and_off = mortie.moc_and(cover_a, ragged, offsets=ragged_off) + to7_vals, to7_off = mortie.moc_to_order(ragged, 7, offsets=ragged_off) + anc_ragged = mortie.common_ancestor(groups, offsets=groups_off) + children = mortie.generate_morton_children(parents4[:2], 6) + + g = { + "words": _ints(cover_a), + "words_b": _ints(cover_b), + "compress_moc": _ints(cover_a), + "moc_to_order": _ints(mortie.moc_to_order(cover_a, 7)), + "moc_to_order_ragged": [_ints(to7_vals), _ints(to7_off)], + "moc_or": _ints(mortie.moc_or(cover_a, cover_b)), + "moc_and": _ints(mortie.moc_and(cover_a, cover_b)), + "moc_and_ragged": [_ints(and_vals), _ints(and_off)], + "moc_intersects": bool(mortie.moc_intersects(cover_a, cover_b)), + "moc_intersects_ragged": [ + bool(x) for x in + mortie.moc_intersects(cover_a, ragged, offsets=ragged_off)], + "moc_minus": _ints(mortie.moc_minus(cover_a, cover_b)), + "moc_xor": _ints(mortie.moc_xor(cover_a, cover_b)), + "moc_min": _ints(np.atleast_1d(mortie.moc_min(kids5))), + "moc_not": _ints(mortie.moc_not(cover_a)), + "moc_not_domain": _ints(mortie.moc_not(kids5[:2], domain=cover_a)), + "common_ancestor": _ints(np.atleast_1d(mortie.common_ancestor(kids5))), + "common_ancestor_ragged": _ints(anc_ragged), + "split_base_cells_values": [ + _ints(part) for part in mortie.split_base_cells(ragged)], + "polygons_to_morton_mocs": [_ints(poly_vals), _ints(poly_off)], + "generate_morton_children_scalar": _ints( + mortie.generate_morton_children(int(parents4[0]), 6)), + "generate_morton_children_array": _ints(children), + "clip2order": _ints(mortie.clip2order(4, kids5)), + "orders_of": _ints(mortie.orders_of(ragged)), + "is_point": [bool(x) for x in np.atleast_1d(mortie.is_point(ragged))], + "infer_order_from_morton": int(mortie.infer_order_from_morton(kids5)), + "validate_morton": bool(mortie.validate_morton(ragged)), + "mort2norm_normed": _ints(mortie.mort2norm(kids5)[0]), + "mort2norm_parent": _ints(mortie.mort2norm(kids5)[1]), + "mort2norm_order": _ints(np.atleast_1d(mortie.mort2norm(kids5)[2])), + "mort2geo": [ + [round(float(v), 12) for v in axis.ravel()] + for axis in mortie.mort2geo(kids5)], + "morton_buffer": _ints(mortie.morton_buffer(kids5, k=1)), + "morton_buffer_meters": _ints( + mortie.morton_buffer_meters(kids5, width_m=50000.0)), + "split_children_roots": sorted( + c.characteristic + for c in mortie.split_children(ragged, max_depth=2)), + "moc_object_and": _ints( + (mortie.Moc(cover_a) & mortie.Moc(cover_b)).words), + # -- phase-3 convert surfaces the first capture missed -------------- + "mort2bbox": [ + [round(float(box[k]), 12) + for k in ("west", "south", "east", "north")] + for box in np.asarray(mortie.mort2bbox(kids5)).ravel()], + "mort2polygon": [ + [[round(float(v), 12) for v in vertex] for vertex in ring] + for ring in mortie.mort2polygon(kids5)], + # -- the phase-5 UNIQ/normed intakes -------------------------------- + "norm2mort": _ints(parents4), + "norm2uniq": _ints(uniq_mixed), + "unique2parent": _ints(mortie.unique2parent(uniq_mixed)), + "uniq2geo": [ + [round(float(v), 12) for v in axis.ravel()] + for axis in mortie.uniq2geo(uniq_mixed)], + # -- the phase-2 offsets seam in geometry.from_wkb ------------------ + "from_wkb_ragged": [ + _ints(part) for part in + mortie.from_wkb(wkb_buf, order=6, offsets=wkb_off)], + # -- the toc family, whose validators moved house in phase 1 -------- + "time2toc": _ints(toc_words), + "span2toc": _ints(np.atleast_1d( + mortie.span2toc(int(t_ns[0]), int(t_ns[-1])))), + "toc2time": [_ints(axis) for axis in mortie.toc2time(toc_words)], + "toc_reduce": _ints(np.atleast_1d(mortie.toc_reduce(toc_words))), + "toc_reduce_ragged": [ + _ints(part) for part in + mortie.toc_reduce(toc_words, offsets=toc_off)], + "toc_normalize": _ints(mortie.toc_normalize(toc_words)), + "toc_and": _ints(mortie.toc_and(toc_words, toc_words)), + "toc_merge": _ints(np.atleast_1d( + mortie.toc_merge(toc_words[0], toc_words[1]))), + "from_gps_ns": _ints(mortie.from_gps_ns(t_ns)), + "to_gps_ns": _ints(mortie.to_gps_ns( + np.asarray([4.2 * 10**18, 4.3 * 10**18], dtype=np.uint64))), + "to_datetime64": [ + str(v) for v in np.atleast_1d(mortie.to_datetime64(t_ns))], + } + return g + + +def main(): + """Write the captured answers to ``OUT``. + + Returns + ------- + None + Writes ``OUT`` as a side effect. + """ + g = capture() + OUT.parent.mkdir(parents=True, exist_ok=True) + OUT.write_text(json.dumps(g, indent=1) + "\n") + print(f"wrote {OUT} ({len(g)} entries)") + + +if __name__ == "__main__": + main() diff --git a/mortie/tests/test_convert.py b/mortie/tests/test_convert.py index ce7df1a9..08488f65 100644 --- a/mortie/tests/test_convert.py +++ b/mortie/tests/test_convert.py @@ -254,16 +254,21 @@ def test_orders_of_uniq_mirrors_orders_of_contract(self): assert orders_mod.orders_of(np.uint64(0)).dtype == got.dtype def test_uniq_orders_raises_valueerror_not_overflow(self): - """Out-of-int64 and non-integer input raise the documented ValueError. + """Out-of-int64 and float input raise this family's named ValueError. `asarray(..., dtype=int64)` raises OverflowError above int64 and silently truncates a float, both of which bypassed the ValueError this - function -- and uniq2geo / unique2parent through it -- promises. + function -- and uniq2geo / unique2parent through it -- promises. The + guard is now the shared `_as_i64`, so the refusal is dtype-based like + its two callers' (issue #194 review): an *integral* float such as + `16.0` used to decode here while `unique2parent([16.0])` refused it. """ - with pytest.raises(ValueError, match="int64 range"): + with pytest.raises(ValueError, match="uniq must fit in int64"): orders_mod.orders_of_uniq(2**63) - with pytest.raises(ValueError, match="not an integer"): + with pytest.raises(ValueError, match="uniq must be integer-typed"): orders_mod.orders_of_uniq(1.5) + with pytest.raises(ValueError, match="uniq must be integer-typed"): + orders_mod.orders_of_uniq(np.asarray([16.0])) def test_norm2uniq_array_order_uint64_no_float_promotion(self): """uint64 input must not promote to float64 on the array-order path. diff --git a/mortie/tests/test_mort_inverse.py b/mortie/tests/test_mort_inverse.py index 4218da35..d24f89f0 100644 --- a/mortie/tests/test_mort_inverse.py +++ b/mortie/tests/test_mort_inverse.py @@ -99,11 +99,16 @@ def test_mort2polygon(self): def test_array_input(self): """Test that array inputs work correctly""" + # dtype pinned (issue #194): the base-8 word tops int64, so numpy + # promoted this mixed Python-int list to float64 -- and the old + # silent cast then zeroed every word's suffix bits, decoding three + # order-6 cells as order-0 base cells without any test noticing. + # The strict validators now refuse the float array outright. mortons = np.array([ int(convert.norm2mort(2120, 2, 6)), int(convert.norm2mort(2120, 8, 6)), int(convert.norm2mort(1402, 3, 6)), - ]) + ], dtype=np.uint64) # Test mort2geo with array lats, lons = convert.mort2geo(mortons) diff --git a/mortie/tests/test_strict_validation.py b/mortie/tests/test_strict_validation.py new file mode 100644 index 00000000..1be3429f --- /dev/null +++ b/mortie/tests/test_strict_validation.py @@ -0,0 +1,578 @@ +"""Family-wide strict input validation (issue #194). + +One posture everywhere, per the ruling on issue #194: float-typed word and +offset arrays are refused rather than truncated, out-of-range values are +refused before any narrowing cast rather than wrapped, and the refusal names +the parameter and the offending value. The two historical bug classes stay +pinned by name: the issue #185 uncatchable-panic arc (a bad value crossing +into Rust unchecked) and PR #192's silent uint64 wrap. + +Valid paths are pinned byte-identical against +``data/strict_validation_goldens.json``, captured at ``4900a7e`` -- the +commit *before* the validators were adopted -- by +``generate_strict_goldens.py``, whose module docstring enumerates exactly +which seams the capture covers and why two are left to other suites. +""" + +import json +import pathlib +import warnings + +import numpy as np +import pytest + +import mortie +from mortie._validate import _as_i64, _as_offsets, _as_u64 + +GOLDENS = json.loads( + (pathlib.Path(__file__).parent / "data" / + "strict_validation_goldens.json").read_text()) + + +def _u64(key): + """Load a golden entry as a uint64 array. + + Parameters + ---------- + key : str + Golden entry name. + + Returns + ------- + numpy.ndarray + The pinned words as ``uint64``. + """ + return np.asarray(GOLDENS[key], dtype=np.uint64) + + +WORDS = _u64("words") +WORDS_B = _u64("words_b") + + +class TestValidators: + """The shared validators themselves (hoisted from the toc module).""" + + def test_u64_refuses_floats(self): + with pytest.raises(ValueError, match="w must be integer-typed"): + _as_u64(np.asarray([1.5]), "w") + # Integral-valued floats are still float-typed: refused, not trusted. + with pytest.raises(ValueError, match="w must be integer-typed"): + _as_u64(np.asarray([2.0]), "w") + + def test_u64_refuses_negative_naming_value(self): + with pytest.raises(ValueError, match=r"w must be non-negative, got -7"): + _as_u64(np.asarray([3, -7, -2], dtype=np.int64), "w") + + def test_u64_passes_top_bit_words(self): + # Base cells 7-11 set bit 63 (spec section 1): large uint64 words are + # valid and must survive unchanged. + big = np.asarray([2**63 + 5], dtype=np.uint64) + assert _as_u64(big, "w")[0] == np.uint64(2**63 + 5) + + def test_u64_accepts_untyped_empty(self): + # The Toc-source ruling: an untyped empty container is not numeric, + # it is empty. + out = _as_u64([], "w") + assert out.size == 0 and out.dtype == np.uint64 + + def test_offsets_refuse_floats(self): + with pytest.raises(ValueError, match="offsets must be integer-typed"): + _as_offsets(np.asarray([0.0, 2.9])) + + def test_offsets_refuse_uint64_wrap_naming_value(self): + # The PR #192 wrap class: >= 2**63 would wrap negative through the + # int64 cast and the kernel would then describe the wrapped copy. + bad = np.asarray([0, 2**63 + 5], dtype=np.uint64) + with pytest.raises( + ValueError, + match=r"offsets must fit in int64, got 9223372036854775813"): + _as_offsets(bad) + + def test_offsets_valid_passthrough(self): + out = _as_offsets([0, 2, 4]) + assert out.dtype == np.int64 and out.tolist() == [0, 2, 4] + + @pytest.mark.parametrize("bad,dtype", [ + (["a"], "