diff --git a/agents/DEVELOPING.md b/agents/DEVELOPING.md index 2d9be93a..1fe2367a 100644 --- a/agents/DEVELOPING.md +++ b/agents/DEVELOPING.md @@ -174,9 +174,10 @@ Shared fixtures (in `fixtures.py`): ## Other directories - `benchmarks/` — `benchmark_{digest,integration,storage}.py`, `run_benchmarks.py`, `compare_results.py` (diff two `results.csv` runs), `utils.py`, `profile_digest_types.py` (per-type cProfile harness), `results.csv`. -- `docs/` — Sphinx sources, grouped by topic: root holds `index`, `installation`, `parallel_execution`; `usage/` holds `tldr`, `helpers`, `lazy_call`, `query` (`tldr` first, PR #745); `digests/` holds `digests_as_args`, `digest_equivalence`, `entry_points` (added by PR #752 as the dedicated fleche-ase / third-party plug-in page); `storage/` holds `configuration`, `destructuring` (figures + prose on destructuring/dedup/`remaining_depth`), `cache_stack`, `security`; `dev/` holds `call_lifecycle` (the two-phase `prepare`/`commit`/`abandon` save protocol and the miss/hit sequences behind a decorated call), `custom_digests`, `extending_destructurer`, `function_profile`, `ssh_cache`, `sql_test_backends`, `storage_hierarchy` (the former grab-bag `dev/developer.rst` was flattened into those standalone pages, keeping its `.. _…:` anchor labels). `docs/figures/` holds `gen_diagrams.py` plus the three destructuring SVGs it generates — the script builds the depicted layouts in real `ValueMemory` stores so the digest labels in the figures match what `notebooks/Destructuring.ipynb` computes — and the two hand-written Graphviz sources `storage_hierarchy.dot` / `storage_mro.dot` with their rendered SVGs (`dot -Tsvg -o `; these replaced the old `devnotes/storage-hierarchy.{dot,md,svg}`, which was stale from PR #245 — it predated `OperationContext`, the thread-safety mixins, and destructuring becoming default on every value storage), plus `gen_sequence.py`, which emits `storage_sequence.svg` (a destructured save/load through the MRO, showing the per-entry lock scopes) and `cache_sequence.svg` (cache miss/hit around the two-phase save) — its panels are transcribed from instrumented traces, not from reading the code. Every figure SVG is checked in rather than rendered at build time because `.readthedocs.yaml` declares no `apt_packages`, so `dot` cannot be assumed on the builder. The HTML theme is `shibuya`, skinned solarized-light/dark with the green accent via CSS custom properties in `docs/_static/custom.css` (which also keeps the "on this page" rail in-flow down to 720px-wide viewports and sets an 18px base font); the figure palette in `gen_diagrams.py` is the matching solarized-light set, so retheming the docs means updating both and rerunning the generator. `docs/notebooks/` is **symlinks** into `../../notebooks/` (all nine non-`Destructuring` entries — `Caches`/`TransferWorkflow` added by PR #839, `FiveMinuteTour` by PR #841 — plus `Destructuring` since the destructuring-docs work); the `rendernb.yml` workflow re-executes `notebooks/*.ipynb` in place when a PR carries the `rendernb` label. +- `docs/` — Sphinx sources, grouped by topic: root holds `index`, `installation`, `parallel_execution`; `usage/` holds `tldr`, `purity`, `helpers`, `file_semantics`, `lazy_call`, `query` (`tldr` first, PR #745; `purity` carries the general "fleche caches pure functions" contract — arguments keyed as passed, mutation and other side effects not replayed — that `file_semantics` states for paths); `digests/` holds `digests_as_args`, `digest_equivalence`, `entry_points` (added by PR #752 as the dedicated fleche-ase / third-party plug-in page); `storage/` holds `configuration`, `destructuring` (figures + prose on destructuring/dedup/`remaining_depth`), `cache_stack`, `security`; `dev/` holds `call_lifecycle` (the two-phase `prepare`/`commit`/`abandon` save protocol and the miss/hit sequences behind a decorated call), `custom_digests`, `extending_destructurer`, `function_profile`, `path_storage`, `ssh_cache`, `sql_test_backends`, `storage_hierarchy` (the former grab-bag `dev/developer.rst` was flattened into those standalone pages, keeping its `.. _…:` anchor labels); `recipes/` holds `files_and_paths`. `docs/figures/` holds `gen_diagrams.py` plus the three destructuring SVGs it generates — the script builds the depicted layouts in real `ValueMemory` stores so the digest labels in the figures match what `notebooks/Destructuring.ipynb` computes — and the two hand-written Graphviz sources `storage_hierarchy.dot` / `storage_mro.dot` with their rendered SVGs (`dot -Tsvg -o `; these replaced the old `devnotes/storage-hierarchy.{dot,md,svg}`, which was stale from PR #245 — it predated `OperationContext`, the thread-safety mixins, and destructuring becoming default on every value storage), plus `gen_sequence.py`, which emits `storage_sequence.svg` (a destructured save/load through the MRO, showing the per-entry lock scopes) and `cache_sequence.svg` (cache miss/hit around the two-phase save) — its panels are transcribed from instrumented traces, not from reading the code. Every figure SVG is checked in rather than rendered at build time because `.readthedocs.yaml` declares no `apt_packages`, so `dot` cannot be assumed on the builder. The HTML theme is `shibuya`, skinned solarized-light/dark with the green accent via CSS custom properties in `docs/_static/custom.css` (which also keeps the "on this page" rail in-flow down to 720px-wide viewports and sets an 18px base font); the figure palette in `gen_diagrams.py` is the matching solarized-light set, so retheming the docs means updating both and rerunning the generator. `docs/notebooks/` is **symlinks** into `../../notebooks/` — all twelve entries, and all twelve are in the toctree (`Caches`/`TransferWorkflow` added by PR #839, `FiveMinuteTour` by PR #841, `Destructuring` by PR #844, `Files`/`PathsInContainers` by PR #797); the `rendernb.yml` workflow re-executes `notebooks/*.ipynb` in place when a PR carries the `rendernb` label. -- `notebooks/` — usage examples (`FiveMinuteTour`, `GettingStarted`, `Caches`, `CacheStack`, `Destructuring`, `StorageBackends`, `SecureStorage`, `ConcurrentExecution`, `ExtraMethods`, `TransferWorkflow`); all are executed by `tests/integration/test_notebooks.py` since PR #839, which flipped the parametrisation from a hand-maintained list to `sorted(notebooks/*.ipynb)` so a notebook added later cannot silently rot out of coverage. `Destructuring.ipynb` embeds the `docs/figures/` figures as **2x PNG** markdown-cell attachments (no display code; PNG because JupyterLab refuses to render `image/svg+xml` attachments) and rebuilds every depicted storage layout in `ValueMemory` so readers can play; its digest outputs match the figure labels by content-addressing. Running `python docs/figures/gen_diagrams.py` regenerates the SVGs **and** refreshes the notebook's embedded PNGs (`refresh_notebook_attachments`, rasterizing via `rsvg-convert` or `cairosvg`); each SVG carries a generated-by comment pointing back at the script. + +- `notebooks/` — usage examples (`FiveMinuteTour`, `GettingStarted`, `Caches`, `CacheStack`, `Destructuring`, `StorageBackends`, `SecureStorage`, `ConcurrentExecution`, `ExtraMethods`, `Files`, `PathsInContainers`, `TransferWorkflow`); all twelve are executed by `tests/integration/test_notebooks.py`, which since PR #839 parametrises over `sorted(notebooks/*.ipynb)` rather than a hand-maintained list, so a notebook added later cannot silently rot out of coverage. Committed **with outputs** — `rendernb.yml` re-executes them in place on a PR labelled `rendernb`, so an edited notebook should be re-run before it lands. `Destructuring.ipynb` embeds the `docs/figures/` figures as **2x PNG** markdown-cell attachments (no display code; PNG because JupyterLab refuses to render `image/svg+xml` attachments) and rebuilds every depicted storage layout in `ValueMemory` so readers can play; its digest outputs match the figure labels by content-addressing. Running `python docs/figures/gen_diagrams.py` regenerates the SVGs **and** refreshes the notebook's embedded PNGs (`refresh_notebook_attachments`, rasterizing via `rsvg-convert` or `cairosvg`); each SVG carries a generated-by comment pointing back at the script. - `.github/workflows/` — CI: `tests.yml` (PR sweep across 3.11–3.14 + `sql-backends` job that boots Postgres 16 + MariaDB 11 service containers and sets `FLECHE_TEST_{POSTGRES,MYSQL}_URL`), `ty.yml`, `test-minimum-deps.yml` (PR #645, installs every direct dep at its declared floor via `uv pip install --resolution lowest-direct -e ".[tests]"` and runs the suite — `uv pip install` rather than `uv sync` because the universal lock floats deps up through cross-extra constraints and would mask the real minimums), `benchmarks.yml`/`benchmarks-main.yml`/`updatebenchmarks.yml`, `perf-triage.yml` (PR #628 — Haiku reads the PR diff/description and adds the `benchmark` label when the change touches a hot path — that label is the existing `benchmarks.yml` trigger), `rendernb.yml` (re-executes `notebooks/*.ipynb` on PRs labelled `rendernb`), `release-please.yml`, `pypi-publish.yml` (trusted-publisher upload triggered by `release: published`), `claude.yaml` + `claude_ci_details.yaml` (the latter exposes CI status as a tool the in-PR Claude can call). Releases use **release-please** (`release-please-config.json`, `.release-please-manifest.json`) — release PRs are opened automatically from conventional-commit history on `main`. ## Design themes / open scope (issue tracker) @@ -193,7 +194,7 @@ Cheat sheet of what's been considered. Issue numbers are the entry points — fe **PR #804** (opened 2026-07-31) drops the `SizeLimitedCache(max_size=10)` config from `benchmarks/benchmark_integration.py`: uniform-random eviction under the 60 call records that config accumulates across the three benchmarked functions meant most "hit"-phase samples were silently re-measuring misses — hit-phase numbers for that config in older `results.csv` runs are unreliable; the surviving `max_size=100` config never evicts. -- **Distributed / remote caching** (#552; #551 **landed**). Still open: #552 — a `TieredValues` + `GlobusValues` cold tier for HPC value blobs that `Cache.query()` never touches, only `Cache.load_value()` on a hot miss. +- **Distributed / remote caching** (#552; #551 **landed**). `SshCache` refuses `Path` values outright (`RemotePathUnsupported`, a `SaveError` subclass so a path argument degrades to a locally-computed digest-only reference and a path result becomes `Rejected`) — a pickled `Path` is only its string, so the server was resolving it against its own filesystem and filing records under keys no client recomputes. **#829** tracks making them actually work: run the path→blob reduction client-side (the blobs' `__digest__` already matches the `Path` digest arm) plus one new unmended `load_value` verb. Still open: #552 — a `TieredValues` + `GlobusValues` cold tier for HPC value blobs that `Cache.query()` never touches, only `Cache.load_value()` on a hot miss. - **Config redesign** (#568). Next-generation TOML/YAML schema with top-level `value`/`call`/`stash`/`metadata` namespaces and cross-file named references. Supersedes the just-landed merged-discovery model (PR #553) once the schema firms up; expect a `hash_version`-style migration story. @@ -304,6 +305,7 @@ Cheat sheet of what's been considered. Issue numbers are the entry points — fe - Cache-level race fixes on top of #569: #217 → PR #629, #451 → PR #631, #452 → PR #630, #485 → PR #627. Regression-pinned in `tests/regression/test_issue_{217,451,452,485}.py`; concurrency stress tests share `run_workers()` from `tests/fixtures.py` (PR #633). - Digest dispatch grew a `builtins`-type arm and a `not isinstance(value, type)` guard on the dataclass/attrs arms so `digest(int)` / `digest(SomeDataclass)` (the class object) stop raising — PR #651, closes #469. +- `digest()` grew a `subprocess.CompletedProcess` arm (args + returncode + stdout + stderr) so functions wrapping command-line tools hash without a user hook — `run()`'s result is neither a dataclass nor iterable, so it was `Indigestible` before. Purely additive (those values raised previously), so no `hash_version` bump. Pinned in `tests/unit/digest/test_digest.py::test_completedprocess_*`; `notebooks/Files.ipynb` used to register this by hand via `add_hook` and no longer does. - `digest()` grew explicit `pd.DataFrame` / `pd.Series` / `pd.Index` arms so pandas inputs hash by content (columns/name + dtype + index + `hash_pandas_object`), not by the column names yielded by `iter()` — PR #675. Silent change with no `hash_version` bump (previous pandas cache entries are now unreachable). Pinned in `tests/unit/digest/test_digest.py::test_pandas_{dataframe,series,index}_hashes_by_content`. diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst new file mode 100644 index 00000000..4aa16a45 --- /dev/null +++ b/docs/dev/path_storage.rst @@ -0,0 +1,227 @@ +Path Storage Internals +====================== + +How ``fleche`` stores :class:`~pathlib.Path` values — single files and whole +directory trees — by content. For the practical version, see +:doc:`/recipes/files_and_paths`. + +The model, in one line +---------------------- + +``fleche`` follows git's split: **content is content-addressed; names live in +trees.** Concretely — + +* a **file** is identified by ``(basename, content)`` — its name matters, its + bytes deduplicate; +* a **directory** is identified by its **tree alone** — child names are part of + it, but the directory's own root name is not (a reloaded directory is named by + its digest); +* plain **bytes** are anonymous file content — return them when you don't want a + name to enter the cache key. + +:class:`~fleche.storage.paths.PathValueMixin` owns the traversal. In the default +value storages it sits between ``DestructuringMixin`` and ``ValueMixin`` in the +method-resolution order, so a bare ``Path`` (or one nested inside a list, dict, +or dataclass) is intercepted on save and materialized again on load. + +How a file is stored +-------------------- + +A file is split into two pieces so that content deduplicates while the name is +still part of the key: + +* the bytes are saved as **plain** ``bytes`` under their content digest — shared + by every file (and every ``bytes`` value) with the same content; +* a small :class:`~fleche.storage.paths.FileBlob` record pairs the basename with + a *reference* to that content blob, and is keyed + ``digest(("FileBlob", name, content_digest))``. + +On load the content is materialized at ``/`` and returned as an +ordinary path, so ``.name`` / ``.suffix`` / ``.stem`` are faithful and a consumer +needs no special-casing. Because the bytes live in a shared, content-keyed blob +and only the tiny ``FileBlob`` differs per name, **renaming a file never +duplicates its body** — a rename adds one record and reuses the blob. + +How a directory is stored +------------------------- + +A directory is a :class:`~fleche.storage.paths.DirectoryBlob`: a +``{name: content_ref}`` mapping keyed by ``digest(("DirectoryBlob", contents))``. +A file child is referenced by its content bytes; a subdirectory child by its own +``DirectoryBlob``. The directory's **own** name never enters this — two trees +with identical contents under different root names hash identically. On load the +tree is rebuilt under ``/`` (hashed root, faithful children). + +The ``digest(path) == values.save(path)`` invariant +--------------------------------------------------- + +This is the load-bearing property. ``fleche`` *looks up* a cached call by +``digest(arguments)`` but *stores* it under ``values.save(arguments)``; the two +must agree, or a call that takes or returns a path would never hit. The ``Path`` +arm of :func:`~fleche.digest.digest` therefore mirrors storage exactly — a file +as ``digest(("FileBlob", name, digest(bytes)))``, a directory as the content-only +tree — with the ``"FileBlob"`` / ``"DirectoryBlob"`` salts matching +:class:`~fleche.storage.paths.FileBlob` and +:class:`~fleche.storage.paths.DirectoryBlob`'s ``__digest__``. + +Why ``remaining_depth`` cannot reach a path +------------------------------------------- + +``PathValueMixin`` only ever sees a value that +:class:`~fleche.storage.destructuring.DestructuringMixin` decided to **write +out** — an inlined value is carried inside its parent's ``Digested`` wrapper and +pickled with it, never handed down the MRO. So "a nested path is always stored +by content" holds only if a path can never be inlined, and that is arranged +rather than hoped for: a ``Path`` matches no destructurer, so ``_intern_rec`` +leaves its depth at ``float("inf")``, and ``inf < remaining_depth`` is false for +every setting. That is the general opaque-value rule +(:doc:`/storage/destructuring`) doing the work here, not a path-specific +mechanism. + +That is the enforcement mechanism, not a side effect of one. It also +propagates, since a parent's depth is ``1 + max(child_depths)``: every container +between the root and a path inherits ``inf`` and is written out as its own +entry too. Sibling subtrees are untouched and inline exactly as they would +without the path. + +``[[[1, 2], [3, path]], [4, 5]]`` at ``remaining_depth=10`` — a setting that +collapses the same structure without the path into a *single* entry: + +.. code-block:: text + + bytes b'xyz' + FileBlob FileBlob('data.txt', ) + DigestedIterable [3, ] <- innermost, own entry + DigestedIterable [[1, 2], ] <- own entry; [1, 2] inlined + DigestedIterable [, [4, 5]] <- root; [4, 5] inlined + +So one path costs *its own nesting depth* in extra entries, not the size of the +structure around it, and the scalars beside it inline under the usual rules +(``[1, 3, 4, path]`` stores as ``DigestedIterable([1, 3, 4, ])``, not as +four separate slots). Tuning ``remaining_depth`` for a path-heavy workload +therefore changes how the *non-path* parts are packed and nothing else — the +content addressing of the paths is not on the table. + +Blobs must declare their references +----------------------------------- + +Storing a path by content splits it in two: the ``bytes`` live under their own +content digest, and the :class:`~fleche.storage.paths.FileBlob` / +:class:`~fleche.storage.paths.DirectoryBlob` record holds a *reference* to +them. That makes the blob a node in the value store's reference graph, and +anything walking that graph — :meth:`~fleche.caches.Cache.gc`, +:meth:`~fleche.storage.destructuring.DestructuringMixin.count_reuses` — has to +be told so. A blob that reports no children looks like a leaf, its content +looks unreferenced, and ``gc`` reclaims the bytes out from under it; the entry +survives as a record pointing at nothing, and the next load raises +``KeyError``, which the wrapper reports as an ordinary cache miss. Silent data +loss, triggered by routine maintenance. + +:meth:`PathValueMixin._raw_sub_digests` therefore declares them, and each +mixin that wraps values in a record of its own does the same for its own +wrappers, delegating anything it does not recognise down the MRO — so a +storage's reachable set is the union over its layers rather than whichever +layer happens to answer first. + +The walk reads through :meth:`~fleche.storage.base.ValueStorage.load_raw` +rather than ``load``, for two reasons. Correctness: ``load`` mends, and +mending resolves child references away — a materialized ``Path`` no longer +knows which blob it came from. Cost: mending a stored path means copying its +whole tree into a temp directory, so a graph walk through ``load`` would +rebuild every stored file on disk purely to ask what it points at. + +Deduplication +------------- + +Everything bottoms out in content-keyed ``bytes`` blobs, so an identical body is +stored once — across names, across directories, and across plain ``bytes`` +values. Only the small ``FileBlob`` / ``DirectoryBlob`` records (references, not +bytes) differ. + +Salting (the tuple idiom) +------------------------- + +``FileBlob`` and ``DirectoryBlob`` salt their digests with the class name via the +tuple idiom from :doc:`custom_digests`. This keeps a ``DirectoryBlob`` from +colliding with a plain ``dict`` carrying the same ``{name: digest}`` mapping, and +a ``FileBlob`` record from colliding with an unrelated value of the same shape. +Note that file *content* needs no such marker: it is plain ``bytes``, and the +"this is a file" information lives in the ``FileBlob`` record (top level) or the +parent ``DirectoryBlob`` entry (inside a tree), never in the content blob itself. + +Choosing content-only +--------------------- + +There is no separate "anonymous path" type: if you want a file's content without +its name in the key, return the ``bytes``. There is deliberately no way to make +a *directory's* root name significant — directories are trees, and their root +name is treated as incidental (typically a temp dir). If a root name carries +meaning, name a *child* meaningfully instead, or wrap the tree's identity in your +own value. + +Why paths stop at the SSH boundary +---------------------------------- + +:class:`~fleche.remote.SshCache` is the one cache where "store this path" +cannot mean what it means everywhere else, because the two sides do not share +a filesystem. Values cross the wire by cloudpickle, and a pickled ``Path`` is +only its string — so a path handed to the remote is a *name it resolves +itself*. Three things follow, and each of them is silent: + +* ``save_value`` on the remote runs :class:`PathValueMixin` against the + **server's** disk. If a file happens to sit at that name it is stored, under + the digest of *those* bytes; ``digest(path) == values.save(path)`` — the + invariant the seal in :meth:`~fleche.caches.BaseCache.prepare` rests on — + is broken, and the record lands under a key no client ever recomputes. +* if nothing sits there, the server raises ``Indigestible`` from the middle of + an RPC. +* ``load_value`` materializes into a temp directory on the **server** and can + only ship the name back. The :class:`TempPath` guard does not survive the + hop either — ``PurePath.__reduce__`` reconstructs from ``parts`` alone, so + the ``_temp_root`` attribute is dropped and the class-level ``_live_roots`` + registry is per-process — so the server frees the tree as soon as its own + reference dies, and the client is left with a dangling name for a filesystem + it cannot see. + +So :class:`~fleche.remote.SshCache` refuses instead, via +:func:`~fleche.storage.paths.find_path` and +:class:`~fleche.remote.RemotePathUnsupported`. ``find_path`` mirrors +:func:`~fleche.digest.digest`, **not** destructuring: what makes a path +dangerous here is that it decides the key, and ``digest`` reads files inside +namedtuples, sets, and arbitrary iterables that a destructuring save stores +verbatim. Mirroring the narrower walk would let ``Bundle(path, 0.5)`` cross. That exception subclasses +:class:`~fleche.storage.SaveError` so the existing two-phase-save degradations +carry it: a path argument becomes a digest-only reference **whose digest was +computed locally**, which keeps the seal intact and lookups correct, and a path +result becomes :class:`~fleche.caches.Rejected`, so the call runs uncached +rather than wrongly cached. + +The degradation is deliberately per *argument*, not per call. ``prepare`` is +the one RPC that carries argument values rather than digests, so a call +carrying a path cannot be shipped whole; it stashes the arguments one at a +time through :class:`~fleche.remote._RemoteValues` instead, and +:meth:`~fleche.call.Call.stash`'s existing per-argument ``SaveError`` fallback +does the rest — only the path arguments are digested-and-not-stored, and their +siblings are stored and remain loadable off the record. Falling back to the +blanket digest-only admission instead would cost every other argument for the +sake of one path, which is neither what the guard promises nor necessary. See :ref:`file-remote-caches` for the user-facing +version. + +Making paths genuinely work over SSH is a separate feature, tracked in +`issue #829 `_. The shape is +already visible above: :meth:`PathValueMixin.save` reduces a path to ``bytes`` +plus a :class:`FileBlob` / :class:`DirectoryBlob`, all of which ship fine, and +those blobs' ``__digest__`` is *defined* to match the ``Path`` arm — so running +the reduction **client-side** keeps the seal intact by construction. The one +new piece is on the load side: an unmended ``load_value`` that hands back the +blob instead of materializing it on the server, so the client materializes into +its own temp directory and owns the :class:`TempPath` lifetime. That is a +change to the RPC surface, not to this module. + +See also +-------- + +* :doc:`/usage/file_semantics` — the user-facing contract this implements. +* :doc:`/recipes/files_and_paths` — copy-paste recipes. +* :doc:`/notebooks/Files` — a runnable walkthrough. +* :doc:`custom_digests` — the tuple-digest idiom these blobs use. diff --git a/docs/index.rst b/docs/index.rst index a8bb145d..010b7ece 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,10 +33,18 @@ Welcome to the **Fleche** library documentation. :hidden: usage/tldr + usage/purity usage/helpers + usage/file_semantics usage/lazy_call usage/query +.. toctree:: + :maxdepth: 2 + :caption: Recipes + + recipes/files_and_paths + .. toctree:: :maxdepth: 2 :caption: Digests @@ -72,6 +80,7 @@ Welcome to the **Fleche** library documentation. dev/custom_digests dev/extending_destructurer dev/function_profile + dev/path_storage dev/ssh_cache dev/sql_test_backends dev/storage_hierarchy @@ -91,6 +100,8 @@ Welcome to the **Fleche** library documentation. notebooks/CacheStack notebooks/TransferWorkflow notebooks/ConcurrentExecution + notebooks/Files + notebooks/PathsInContainers .. toctree:: :maxdepth: 2 diff --git a/docs/notebooks/Files.ipynb b/docs/notebooks/Files.ipynb new file mode 120000 index 00000000..542734c5 --- /dev/null +++ b/docs/notebooks/Files.ipynb @@ -0,0 +1 @@ +../../notebooks/Files.ipynb \ No newline at end of file diff --git a/docs/notebooks/PathsInContainers.ipynb b/docs/notebooks/PathsInContainers.ipynb new file mode 120000 index 00000000..6382d357 --- /dev/null +++ b/docs/notebooks/PathsInContainers.ipynb @@ -0,0 +1 @@ +../../notebooks/PathsInContainers.ipynb \ No newline at end of file diff --git a/docs/recipes/files_and_paths.rst b/docs/recipes/files_and_paths.rst new file mode 100644 index 00000000..0c97b0f2 --- /dev/null +++ b/docs/recipes/files_and_paths.rst @@ -0,0 +1,98 @@ +Caching Functions that Work with Files +====================================== + +Short, copy-paste recipes for caching functions that produce or consume files +and directories. For the precise contract (identity, materialization, +lifetime, nesting), see :doc:`/usage/file_semantics`; for *why* any of this +works, see :doc:`/dev/path_storage`; for a runnable walkthrough, see the +:doc:`/notebooks/Files` notebook. + +.. note:: + + A cache hit returns a **fresh temporary copy**, not the original location. + The copy lives as long as you hold a ``Path`` object pointing into it — + copy it out (``shutil.copy``) if you need it at a stable place. Details: + :doc:`/usage/file_semantics`. + +Return a file from a cached function +------------------------------------ + +Just return the :class:`~pathlib.Path`. ``fleche`` stores the file's *contents* +(not the path string), keyed on its ``(name, content)`` — so the cache is +portable across machines, and a cache hit comes back as a path with the **same +name and extension**. + +.. code-block:: python + + from pathlib import Path + from fleche import fleche + + @fleche + def render(text) -> Path: + out = Path("report.pdf") + out.write_text(text) + return out + +Downstream code needs nothing special — a cache hit is an ordinary ``Path``: + +.. code-block:: python + + @fleche + def count_pages(doc: Path) -> int: + assert doc.suffix == ".pdf" # still true on a cache hit + return ... + + count_pages(render("hello")) + +Don't care about the name? Return ``bytes`` +------------------------------------------- + +If the filename is irrelevant and you only care about the content, return the +``bytes`` instead of a ``Path``. Content-only values deduplicate maximally — +the same bytes under different would-be names share one cache entry. + +.. code-block:: python + + @fleche + def serialize(obj) -> bytes: + return pickle.dumps(obj) + +Return a directory +------------------ + +Return the directory ``Path``; the whole tree round-trips, and its children keep +their names. + +.. code-block:: python + + @fleche + def build(src) -> Path: + out = Path("build") + out.mkdir() + (out / "result.bin").write_bytes(compile(src)) + return out + +.. note:: + + A directory is identified by its *tree*, not its root name — so a reloaded + directory's own ``.name`` is a hash (its children's names are faithful). + Don't rely on the top-level directory name surviving a cache hit; do rely on + everything inside it. + +Sharing files with a remote (SSH) cache +--------------------------------------- + +Return ``bytes``. A :class:`~pathlib.Path` cannot cross a +:class:`~fleche.remote.SshCache` — only the path string would travel, and the +remote would resolve it against its own filesystem — so fleche refuses it and +runs the call uncached. Content ships and deduplicates normally: + +.. code-block:: python + + @fleche + def render(text) -> bytes: # not `-> Path` + return _render_to_pdf(text) + +If you want path semantics *and* a remote cache, put a local layer in front of +it: saves land in the local layer, so paths never reach the wire. See +:ref:`file-remote-caches` for the full contract. diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst new file mode 100644 index 00000000..0694fe30 --- /dev/null +++ b/docs/usage/file_semantics.rst @@ -0,0 +1,290 @@ +File and Path Semantics +======================= + +What happens, precisely, when a :class:`~pathlib.Path` crosses the cache +boundary — as an argument, as a return value, or nested inside one. This page +is the contract: everything here is intended behavior you may rely on. For +copy-paste recipes see :doc:`/recipes/files_and_paths`; for the storage design +see :doc:`/dev/path_storage`. + +The model in three sentences +---------------------------- + +A **file** is identified by its ``(basename, content)`` — where it lives never +matters, what it is called and what is in it always do. A **directory** is +identified by its tree alone — child names and contents, recursively — while its +own root name is ignored. A cache hit does not give you back the original +location: it **materializes a fresh copy** under a temporary directory and hands +you the path to that. + +Identity: what makes two calls "the same call" +---------------------------------------------- + +Arguments are turned into a lookup key by digesting them; for paths the digest +follows the model above. + +* ``f(Path("a/data.csv"))`` and ``f(Path("b/data.csv"))`` are the **same call** + if both files have the same bytes — location is not part of the key. +* ``f(Path("data.csv"))`` and ``f(Path("data.tsv"))`` are **different calls** + even with identical bytes — the basename is part of a file's key. +* Editing a file's content changes the key: the next call is a miss and + recomputes. +* For a **directory** argument, renaming or moving the directory itself does + *not* change the key, but renaming or moving anything **inside** it does. +* ``bytes`` arguments are content-keyed too: two calls passing equal bytes are + the same call, regardless of where the bytes came from. +* Two distinct arguments whose digests agree are interchangeable: a hit can be + served for a path argument your process never saw before, as long as name and + bytes match. + +The same identity governs return values: results are stored by content, so two +functions returning identical files share one stored body (see +:ref:`file-dedup`). + +Argument mutation +~~~~~~~~~~~~~~~~~ + +Arguments are keyed **as passed** — captured before the body runs — and a +mutation the body performs on one is neither recorded nor replayed. That is +a general rule (:ref:`argument-mutation`); its most common instance here is a +function that writes an output file *into* a directory it received. The call +is recorded under the directory's pre-call tree, so honest repeat calls hit, +but the written file does not reappear on a hit — it exists only on cold +calls. + +Returning what you wrote to is the normal shape, not a workaround for this: a +function that writes into a directory and hands that directory back is recorded +with it in its final, post-mutation state, and that is the intended way to +produce files. What is not replayed is a write to something you never return — +so a path that is pure *input* should be treated as read-only, with new files +written somewhere you do return (``tempfile.mkdtemp``). + +Only *content* changes count as mutation: permissions are not part of +identity (see :ref:`fidelity-limits`), so a ``chmod`` on a received path is +harmless. + +Paths with no readable content +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A path fleche cannot read has no content and therefore **no digest**. That +covers one that does not exist, one that is neither a file nor a directory, and +one whose content the process cannot actually read — no permission, an I/O +error, a network mount that went away. + +None of these raise. fleche logs a warning (``"No hash for argument: ..."``) +and **runs the function uncached**: every call executes, nothing is stored or +looked up. An unreadable file is a caching problem, not a reason to fail a call +the function itself might well handle — and if the function does need to read +it, it will raise on its own terms, inside the body, where the traceback means +something. + +If you meant "an output location the function should write to", pass the +location as a ``str`` or annotate the parameter :class:`~fleche.Ignored` +(``dest: Ignored[Path]``) so it stays out of the key. + +Cache hits materialize copies +----------------------------- + +On a hit, fleche rebuilds the file (or tree) from stored content in a **fresh +temporary directory** and returns that path. Consequences you should design +for: + +* **The location changes.** The cold (computing) call returns the function's + return value untouched — same objects, same locations, same aliasing; every + subsequent hit returns a path under a new temporary directory. Never + compare returned paths by location or store ``str(path)`` as a stable + identifier. +* **The type changes.** Hits return a ``Path`` subclass that manages the + temporary tree's lifetime. The subclass is an implementation detail — do not + import it or dispatch on its name; rely only on ``isinstance(p, Path)``, and + do not assume ``type(p) is PosixPath``. +* **File names are faithful.** A file materializes as ``/``: + ``.name``, ``.suffix``, and ``.stem`` are the same cold and warm. +* **Directory root names are not.** A directory materializes as + ``/``: its own ``.name`` is a hex digest on a hit. The digest + name is deterministic — hits for the same content always carry the same root + name (under differing temporary parents) — but treat it as opaque. + Everything *inside* — child names, nesting, contents — is faithful. +* **Every hit is a fresh copy.** Each hit materializes under its own private + temporary directory, so two hits for the same call return two different + locations with equal content. For large files this means disk traffic per + hit; hold on to the result rather than re-calling in a loop. +* **Siblings do not come along.** Only the returned (or passed) path is + captured. A hit's ``p.parent`` contains nothing but ``p`` itself — code + like ``p.parent / "meta.json"`` works on the cold call and fails on hits. + Return the sibling too, or return the whole directory. +* **The original is not needed.** Once stored, hits are served from cache + content; the file the cold call returned can be deleted, moved, or edited + without affecting later hits. + +.. _fidelity-limits: + +Fidelity limits +~~~~~~~~~~~~~~~ + +What is stored — and thus what a hit restores — is **names and bytes, nothing +else**: + +* **Permissions are not preserved.** A materialized file has default + permissions; in particular the executable bit is lost, so a returned script + is not runnable on a hit without a fresh ``chmod``. (Permissions are also + not part of identity: ``chmod`` alone never changes a key.) +* **Symlinks are flattened.** A symlink inside a returned directory is stored + and restored as an ordinary file with the *target's* content. +* **Empty subdirectories are preserved**; modification times, ownership, and + other metadata are not. + +Lifetime of materialized paths +------------------------------ + +The temporary tree behind a hit lives exactly as long as some ``Path`` object +derived from it is referenced. Deriving (``p.parent``, ``p / "x"``, +``p.with_suffix(...)``) keeps the tree alive; converting to ``str`` does not. + +.. code-block:: python + + p = cached_fn() # hit -> temp file + loc = str(p) + del p # last reference gone ... + Path(loc).exists() # ... False: the temp tree was deleted + +Keep the ``Path`` object (or the container holding it) for as long as you need +the file. If you need the file at a stable location, copy it out: +``shutil.copy(p, dest)``. + +Paths nested inside containers +------------------------------ + +Whether a nested path is stored by content comes down to one thing: +**destructuring** — whether storage takes the surrounding container apart into +independently-stored children, or pickles it whole as one opaque value. Only +children reach the path machinery, so the list of containers fleche +destructures *is* the list of places a nested path gets content treatment. +That list is +:data:`~fleche.storage.destructuring._DESTRUCTURERS`: ``dict``, +``OrderedDict``, ``list``, ``tuple`` (**exact types** — see below), +``dataclasses`` and ``attrs`` classes. See :doc:`/storage/destructuring` for +what destructuring is and how ``remaining_depth`` shapes it, and +:ref:`extending-destructurer` for how to add your own container to the list. + +Within those, paths are found nested to any depth, as values *or* as dict +keys, and everything above about identity, materialization, and lifetime +applies to each one individually. Container structure is otherwise faithful +on a hit: lists and tuples keep their element order, and a mended dict keeps +the insertion order the stored value had. + +"Any depth" is not a figure of speech, and no storage setting narrows it. A +``Path`` matches no destructurer, so it takes the depth-∞ treatment every +opaque value gets (:doc:`/storage/destructuring`): it is always written out as +its own stored entry rather than inlined into the container above it — and +being written out is exactly what hands it to the content machinery. The +``remaining_depth`` knob only decides how eagerly *destructurable* nodes are +split into separate entries, so it cannot put a path out of reach however it is +set. + +Three caveats specific to nesting: + +* **Dict keys mend into new keys.** A ``Path`` used as a dictionary key comes + back as a materialized path at a new location — keys get the same + materialization, type, and lifetime treatment as values — and looking the + entry up by the *original* path object misses (``Path`` hashing is + location-based). If you intend to look entries up, key by something that + survives a round trip: the basename (``path.name``) or an + application-level identifier. +* **Aliasing is not preserved.** If the same path object appears twice in a + result, a hit materializes each occurrence separately: equal content, two + locations, ``is``-distinct objects. +* **Only listed container types are traversed.** Subclasses — including + ``defaultdict``, ``Counter``, and ``namedtuple`` — and other containers + (``set``, plain classes that are not dataclass/attrs) are stored **verbatim** + as opaque values. + +Opaque containers store paths by *location* +------------------------------------------- + +A path inside an opaque value (a ``namedtuple``, a ``set``, an arbitrary +object) is never destructured out of it, and so never reaches the content +machinery. The call is still *keyed* +correctly — the digest layer does look inside — but what is stored is the path +object itself, pointing at wherever the file was when it was saved. A hit +returns that original *location* (whether as the same object or an equal copy +is backend-dependent — rely on neither): if the file has since been deleted or +edited, the hit hands you a dangling or stale path, **without any warning**. + +Rule of thumb: return paths in plain dicts / lists / tuples / dataclasses. If +you need a custom container to participate, register a destructurer for it +with :func:`~fleche.storage.destructuring.register_destructurer` — see +:ref:`extending-destructurer`. + +.. _file-dedup: + +Deduplication +------------- + +File bodies are stored once per unique byte-content — shared across file +names, across directories, and with plain ``bytes`` values. Renaming a file +and re-caching adds a tiny name record, never a second copy of the bytes. If +you want *keys* (not just storage) to ignore the name too, return ``bytes`` +instead of a ``Path``; that is the content-only escape hatch. + +.. _file-remote-caches: + +Paths stop at a remote (SSH) cache +---------------------------------- + +Everything above assumes the cache and your process see the **same +filesystem**. A :class:`~fleche.remote.SshCache` does not: it forwards each +operation to a fleche running on another machine, and values travel by +cloudpickle — which reduces a ``Path`` to its *string*. Only the name would +arrive, and the remote would resolve it against its own filesystem, storing +whatever happens to sit there (or nothing) under a digest that no longer +matches yours. + +Rather than store something else under your key, fleche refuses: +:class:`~fleche.remote.RemotePathUnsupported` is raised when a path — bare or +nested in a container — would cross the wire. Because it is a +:class:`~fleche.storage.SaveError`, the usual degradations apply and your code +does not have to catch anything: + +* a path **argument** falls back to a digest-only reference. The call is still + keyed correctly (the digest is computed *here*, from your file), so lookups + hit and miss exactly as they should; only the file's bytes are not retrievable + from the remote record. The degradation is per argument: the path's + non-path siblings are stored normally and read back off the record as + values. +* a path **result** is rejected: the call runs, returns your file, and is + logged as not cached. +* **loading** a path stored on the remote raises too. The remote materializes + into a temp directory on *its* disk and can only send back the name, which + means nothing here. This is lazy — a record whose result is a path can still + be loaded and queried; only touching the path value raises. + +To share file content across machines, return the file's ``bytes``: they +travel and deduplicate normally. To keep full path semantics, put a local +cache layer in front of the remote one (see :doc:`/notebooks/CacheStack`) — the +local layer is where saves land, so paths never reach the wire. + +Quick reference +--------------- + +=============================================== ====================================== +You do A cache hit gives you +=============================================== ====================================== +Return ``Path`` to a file Copy at ``/`` +Return ``Path`` to a directory Tree at ``/``, faithful inside +Return paths in dict/list/tuple/dataclass Same container shape, each path a copy +Return ``bytes`` The bytes (no file, no name) +Use ``Path`` as dict key New key at new location +Return same path twice Two independent copies +Return path inside namedtuple/set/custom class The *original* location (may be stale) +Pass a nonexistent ``Path`` argument No caching: warns, always executes +Write into a directory you received Hit (keyed as passed); write not replayed +Return a ``Path`` through an ``SshCache`` Refused: runs, warns, not cached +=============================================== ====================================== + +See also +-------- + +* :doc:`/recipes/files_and_paths` — copy-paste recipes. +* :doc:`/dev/path_storage` — how storage implements this contract. +* :doc:`/digests/digests_as_args` — passing digests instead of values. diff --git a/docs/usage/helpers.rst b/docs/usage/helpers.rst index 39d98066..dd84458b 100644 --- a/docs/usage/helpers.rst +++ b/docs/usage/helpers.rst @@ -102,7 +102,9 @@ Functions Returning ``None`` Functions that return ``None`` are **never cached**. When a decorated function returns ``None``, ``fleche`` logs a ``WARNING`` and skips the save step entirely. Subsequent calls will execute the function again rather than -returning a cached value. +returning a cached value — including any side effects they have, which for a +cached function would otherwise happen on the cold call only (see +:doc:`purity`). This applies to all code paths, with one difference for ``.rerun()``: diff --git a/docs/usage/purity.rst b/docs/usage/purity.rst new file mode 100644 index 00000000..e70dfe0e --- /dev/null +++ b/docs/usage/purity.rst @@ -0,0 +1,101 @@ +.. _purity: + +Purity and Side Effects +======================= + +What fleche assumes about the functions you decorate, and what therefore does +*not* survive a cache hit. The rules here apply to every argument and result +type; :doc:`file_semantics` is the same contract seen through ``Path`` values. + +The assumption +-------------- + +fleche caches **pure** functions: the result is determined by the arguments, +and anything else the body does is incidental. A cache hit replays the +*result* and nothing else — so every effect a function has besides returning +a value happens on cold calls only. + +That is not a restriction fleche can check, and it does not raise if you break +it. It shows up as a function that behaves differently the second time you +call it. + +.. _argument-mutation: + +Arguments are keyed as passed +----------------------------- + +Argument content is captured **before** the body runs, so a function that +mutates its own argument is still recorded under the pre-call content and +honest repeat calls hit. The mutation itself is neither recorded nor +replayed: + +.. code-block:: python + + @fleche + def append_and_report(xs, n): + xs.append(n) + return len(xs) + + a = [1, 2] + append_and_report(a, 9) # 3 — body runs, and a is now [1, 2, 9] + + b = [1, 2] + append_and_report(b, 9) # 3 — cache hit; b is still [1, 2] + +Both calls return ``3``, because that is the recorded result. Only the first +one changed its argument. Nothing about this is specific to lists: a +``dict``, a numpy array, and a directory a function writes into all behave the +same way. + +Return what you mutate +~~~~~~~~~~~~~~~~~~~~~~ + +An argument that is mutated **and returned** is captured faithfully in its +final, post-mutation state — the result is stored after the body runs: + +.. code-block:: python + + @fleche + def append_and_return(xs, n): + xs.append(n) + return xs + + append_and_return([1, 2], 9) # [1, 2, 9] — body runs + append_and_return([1, 2], 9) # [1, 2, 9] — cache hit, same value + +If the mutation is the point of the function, return it. Treat arguments you +receive as read-only otherwise, and build results fresh. + +Other side effects +------------------ + +Everything else a body does — printing, logging, writing files outside the +returned value, sending a request, inserting a row — happens on the cold call +and never again: + +.. code-block:: python + + @fleche + def record(x): + db.insert(x) # runs once, ever + return x * 2 + +If an effect must happen on every call, it belongs outside the cached +function; keep the cached part to the computation whose result you want +stored. + +Two related cases +----------------- + +* A function returning ``None`` is never cached, so it re-executes every time + — including its side effects. See :ref:`none-not-cached`. +* Paths follow all of the above, with the mutation case made concrete: + a directory a function receives and writes into is recorded under its + pre-call tree, and the written file does not reappear on a hit. See + :doc:`file_semantics`. + +See also +-------- + +* :doc:`file_semantics` — the same contract for files and directories. +* :doc:`helpers` — ``.rerun()`` for forcing a cold call deliberately. diff --git a/docs/usage/tldr.rst b/docs/usage/tldr.rst index a17ee44d..3aa938cf 100644 --- a/docs/usage/tldr.rst +++ b/docs/usage/tldr.rst @@ -55,8 +55,14 @@ The essentials beyond that: expensive.rerun(1, 2) # force re-execution and overwrite the entry expensive.query().table() # stored calls for this function as a pandas DataFrame -That is all you need for everyday use. The rest of this section covers the -helper methods in depth (:doc:`helpers`), lazy loading of large cached -objects (:doc:`lazy_call`), and querying stored calls (:doc:`query`); -storage backends and configuration details live under -:doc:`/storage/configuration`. +Functions may also take and return :class:`~pathlib.Path` objects: files and +directories are cached by **content** (not by path string), and a cache hit +returns a freshly materialized copy under a temporary path. See +:doc:`file_semantics` for the exact contract. + +That is all you need for everyday use. The rest of this section covers what +fleche assumes about the functions you decorate (:doc:`purity`), the helper +methods in depth (:doc:`helpers`), the file/path contract +(:doc:`file_semantics`), lazy loading of large cached objects +(:doc:`lazy_call`), and querying stored calls (:doc:`query`); storage backends +and configuration details live under :doc:`/storage/configuration`. diff --git a/notebooks/Files.ipynb b/notebooks/Files.ipynb new file mode 100644 index 00000000..f76494f3 --- /dev/null +++ b/notebooks/Files.ipynb @@ -0,0 +1,601 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# Caching functions that read and write files\n", + "\n", + "`fleche` is a content-addressed cache. When a cached function takes or returns a\n", + "`pathlib.Path`, fleche stores the file's (or directory tree's) **contents** — not\n", + "just the path string — so results are portable and reproducible across machines.\n", + "\n", + "A path is keyed by its content, so:\n", + "\n", + "- two files with identical bytes share one storage entry (deduplication), and\n", + "- re-calling a function with the same file content is a cache hit, even if the\n", + " file lives at a different location.\n", + "\n", + "The in-memory cache (`cache(\"memory\")`) and every default value storage now carry\n", + "this behaviour out of the box — no custom storage composition required." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cell-01", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:48.374404Z", + "iopub.status.busy": "2026-08-06T19:58:48.374124Z", + "iopub.status.idle": "2026-08-06T19:58:49.119391Z", + "shell.execute_reply": "2026-08-06T19:58:49.117923Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['PerKeyLockMixin', 'DestructuringMixin', 'PathValueMixin', 'ValueMixin']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "from subprocess import run\n", + "\n", + "import fleche as fl\n", + "from fleche import fleche\n", + "\n", + "fl.cache(\"memory\") # activate a transient in-memory cache\n", + "c = fl.cache() # grab the active cache to introspect later\n", + "\n", + "# The default value storage stores Paths by content:\n", + "[k.__name__ for k in type(c.values).__mro__ if k.__name__.endswith(\"Mixin\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cell-02", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.122636Z", + "iopub.status.busy": "2026-08-06T19:58:49.122082Z", + "iopub.status.idle": "2026-08-06T19:58:49.128658Z", + "shell.execute_reply": "2026-08-06T19:58:49.126851Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/tmp/tmpyxng_i3t-fleche-files')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# A scratch directory for the files our functions produce.\n", + "WORK = Path(tempfile.mkdtemp(suffix=\"-fleche-files\"))\n", + "WORK" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Producing a file\n", + "\n", + "A cached function can create a file and return its `Path`. fleche stores the\n", + "bytes; on a cache hit it hands back a freshly materialized temporary `Path` with\n", + "the same contents. The `print` fires only when the body actually runs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "cell-04", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.132107Z", + "iopub.status.busy": "2026-08-06T19:58:49.131691Z", + "iopub.status.idle": "2026-08-06T19:58:49.141581Z", + "shell.execute_reply": "2026-08-06T19:58:49.140088Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'hello'\n", + "returned: PosixPath -> hellohello\n" + ] + } + ], + "source": [ + "@fleche\n", + "def write(text, repeat=1, name=\"out.txt\"):\n", + " print(\" [write] running:\", repr(text))\n", + " f = WORK / name\n", + " f.write_text(text * repeat)\n", + " return f\n", + "\n", + "f = write(\"hello\", 2)\n", + "print(\"returned:\", type(f).__name__, \"->\", f.read_text())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cell-05", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.144539Z", + "iopub.status.busy": "2026-08-06T19:58:49.144174Z", + "iopub.status.idle": "2026-08-06T19:58:49.151092Z", + "shell.execute_reply": "2026-08-06T19:58:49.149640Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "from cache: hellohello\n" + ] + } + ], + "source": [ + "# Same arguments -> cache hit -> the body does NOT run (no \"[write] running\").\n", + "again = write(\"hello\", 2)\n", + "print(\"from cache:\", again.read_text())" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "## Consuming a file\n", + "\n", + "A function can take a `Path` argument; fleche keys the call on the file's\n", + "content. Feeding it a path produced by another cached function chains the two." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cell-07", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.153583Z", + "iopub.status.busy": "2026-08-06T19:58:49.153313Z", + "iopub.status.idle": "2026-08-06T19:58:49.166569Z", + "shell.execute_reply": "2026-08-06T19:58:49.165422Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'a quick brown fox'\n", + " [wordcount] running: sentence.txt\n", + "count: 4\n", + "count again: 4\n" + ] + } + ], + "source": [ + "@fleche\n", + "def wordcount(path: Path):\n", + " print(\" [wordcount] running:\", path.name)\n", + " return len(path.read_text().split())\n", + "\n", + "print(\"count:\", wordcount(write(\"a quick brown fox\", 1, name=\"sentence.txt\")))\n", + "# Re-run: both `write` and `wordcount` are served from cache.\n", + "print(\"count again:\", wordcount(write(\"a quick brown fox\", 1, name=\"sentence.txt\")))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "## Directories\n", + "\n", + "Returning a directory `Path` stores the whole tree. On load it is rebuilt under\n", + "a temporary directory, so `iterdir()` / `rglob()` work exactly as before." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cell-09", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.169824Z", + "iopub.status.busy": "2026-08-06T19:58:49.169546Z", + "iopub.status.idle": "2026-08-06T19:58:49.183369Z", + "shell.execute_reply": "2026-08-06T19:58:49.181356Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [make_tree] running: alpha\n" + ] + }, + { + "data": { + "text/plain": [ + "['sub', 'sub/leaf.bin', 'top.txt']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "@fleche\n", + "def make_tree(seed):\n", + " print(\" [make_tree] running:\", seed)\n", + " d = WORK / f\"tree-{seed}\"\n", + " d.mkdir(exist_ok=True)\n", + " (d / \"top.txt\").write_text(seed)\n", + " (d / \"sub\").mkdir(exist_ok=True)\n", + " (d / \"sub\" / \"leaf.bin\").write_bytes(seed.encode() * 3)\n", + " return d\n", + "\n", + "@fleche\n", + "def total_bytes(d: Path):\n", + " print(\" [total_bytes] running:\", d.name)\n", + " return sum(p.stat().st_size for p in d.rglob(\"*\") if p.is_file())\n", + "\n", + "tree = make_tree(\"alpha\")\n", + "sorted(p.relative_to(tree).as_posix() for p in tree.rglob(\"*\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cell-10", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.186305Z", + "iopub.status.busy": "2026-08-06T19:58:49.186041Z", + "iopub.status.idle": "2026-08-06T19:58:49.212754Z", + "shell.execute_reply": "2026-08-06T19:58:49.211003Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [total_bytes] running: ff88b2cec3e828c231a8a9df977be4f4cfa2e938cc0cdc41aac1e03350f5af8b\n" + ] + }, + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# End-to-end cache hit: make_tree(\"alpha\") and total_bytes both come from cache.\n", + "total_bytes(make_tree(\"alpha\"))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Content-addressing\n", + "\n", + "Files are stored under the digest of their contents, so identical bodies are\n", + "stored once regardless of filename or which call produced them." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "cell-12", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.215720Z", + "iopub.status.busy": "2026-08-06T19:58:49.215476Z", + "iopub.status.idle": "2026-08-06T19:58:49.223612Z", + "shell.execute_reply": "2026-08-06T19:58:49.222050Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'shared body'\n", + " [write] running: 'shared body'\n", + "shared body stored once: True\n" + ] + } + ], + "source": [ + "# Two calls, different names, identical body -> the content is stored once.\n", + "write(\"shared body\", 1, name=\"left.txt\")\n", + "write(\"shared body\", 1, name=\"right.txt\")\n", + "\n", + "body_key = fl.digest.digest(b\"shared body\") # content blobs are plain bytes\n", + "print(\"shared body stored once:\", body_key in set(c.values.list()))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "cell-13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.227426Z", + "iopub.status.busy": "2026-08-06T19:58:49.227174Z", + "iopub.status.idle": "2026-08-06T19:58:49.232930Z", + "shell.execute_reply": "2026-08-06T19:58:49.231276Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FileBlob('out.txt', '785d68f8426805e292630852bdedb46dd56ac44dcb7047740d30704f3d84d4fa')\n", + "FileBlob('sentence.txt', '0f61b76af53fa2dc41528c3866206d22ceb3b7f560e7f42aacc006fc5ace228c')\n", + "DirectoryBlob({'leaf.bin': 'c70f6db1a5371bc6046fb5a040fd13bd5220c78908eac8126c1361daad854904'})\n", + "DirectoryBlob({'sub': '7aa02a10cd58fe7b0f16ef0e06b255d7842ce362687e489e941281d27761d95a', 'top.txt': '8eb42147b1727df4b082ebc0bdfc5fbaea064308411a4d801cae67c908ce4287'})\n", + "FileBlob('left.txt', '32cbd77d1dbff488cd42dc84ea72ebd47358fbf321412b35a5c1084e36f5b775')\n", + "FileBlob('right.txt', '32cbd77d1dbff488cd42dc84ea72ebd47358fbf321412b35a5c1084e36f5b775')\n" + ] + } + ], + "source": [ + "# What a `Path` actually becomes in storage: its content as plain `bytes` under\n", + "# its own digest, plus a small record pairing that content with a name. Note\n", + "# that left.txt and right.txt reference the *same* content digest.\n", + "for blob in c.values.storage.values():\n", + " if type(blob).__name__ in (\"FileBlob\", \"DirectoryBlob\"):\n", + " print(blob)\n" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## A real workflow: orchestrating shell scripts\n", + "\n", + "The motivating use case: wrap command-line tools that read and write files. Each\n", + "step runs in a working directory, produces files, and the whole pipeline is\n", + "cached by content.\n", + "\n", + "Each step returns the `subprocess.CompletedProcess` that `run()` produced. fleche\n", + "digests those directly — by `args`, `returncode`, `stdout`, and `stderr` — so the\n", + "captured output participates in the key with no extra setup. For a type fleche\n", + "does not know, `fl.digest.add_hook((TheType, fn))` is the extension point." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cell-15", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.235969Z", + "iopub.status.busy": "2026-08-06T19:58:49.235619Z", + "iopub.status.idle": "2026-08-06T19:58:49.245255Z", + "shell.execute_reply": "2026-08-06T19:58:49.243354Z" + } + }, + "outputs": [], + "source": [ + "@fleche\n", + "def shell(cwd, prog, args=(), stdin=b\"\"):\n", + " print(\" [shell] running:\", prog, *args)\n", + " ret = run([prog, *args], cwd=cwd, capture_output=True, input=stdin)\n", + " return cwd, ret\n", + "\n", + "@fleche\n", + "def pipeline(content):\n", + " print(\" [pipeline] running:\", content)\n", + " work = Path(tempfile.mkdtemp(suffix=\"-pipeline\"))\n", + " (work / \"input.txt\").write_bytes(content)\n", + " shell(work, \"cp\", [\"input.txt\", \"copy.txt\"]) # produce a file\n", + " _, upper = shell(work, \"tr\", [\"a-z\", \"A-Z\"], stdin=content) # capture stdout\n", + " (work / \"shout.txt\").write_bytes(upper.stdout)\n", + " return work" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cell-16", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.248014Z", + "iopub.status.busy": "2026-08-06T19:58:49.247696Z", + "iopub.status.idle": "2026-08-06T19:58:49.269994Z", + "shell.execute_reply": "2026-08-06T19:58:49.268579Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- first run ---\n", + " [pipeline] running: b'hello world'\n", + " [shell] running: cp input.txt copy.txt\n", + " [shell] running: tr a-z A-Z\n", + "produced: ['copy.txt', 'input.txt', 'shout.txt']\n", + "shout.txt: HELLO WORLD\n" + ] + } + ], + "source": [ + "print(\"--- first run ---\")\n", + "out = pipeline(b\"hello world\")\n", + "print(\"produced:\", sorted(p.name for p in out.iterdir()))\n", + "print(\"shout.txt:\", (out / \"shout.txt\").read_text())" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cell-17", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.272480Z", + "iopub.status.busy": "2026-08-06T19:58:49.272198Z", + "iopub.status.idle": "2026-08-06T19:58:49.280138Z", + "shell.execute_reply": "2026-08-06T19:58:49.277972Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- second run: fully cached (no body / shell prints) ---\n", + "same files: ['copy.txt', 'input.txt', 'shout.txt']\n" + ] + } + ], + "source": [ + "print(\"--- second run: fully cached (no body / shell prints) ---\")\n", + "out2 = pipeline(b\"hello world\")\n", + "print(\"same files:\", sorted(p.name for p in out2.iterdir()))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "cell-18", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.283069Z", + "iopub.status.busy": "2026-08-06T19:58:49.282802Z", + "iopub.status.idle": "2026-08-06T19:58:49.301525Z", + "shell.execute_reply": "2026-08-06T19:58:49.300406Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namemoduletimestarttimestopwalltime
6f89shell__main__2026-08-06 19:58:49.251591921+00:002026-08-06 19:58:49.259691+00:000.008099
a668shell__main__2026-08-06 19:58:49.261044025+00:002026-08-06 19:58:49.264387608+00:000.003344
\n", + "
" + ], + "text/plain": [ + " name module timestart \\\n", + "6f89 shell __main__ 2026-08-06 19:58:49.251591921+00:00 \n", + "a668 shell __main__ 2026-08-06 19:58:49.261044025+00:00 \n", + "\n", + " timestop walltime \n", + "6f89 2026-08-06 19:58:49.259691+00:00 0.008099 \n", + "a668 2026-08-06 19:58:49.264387608+00:00 0.003344 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Every shell invocation fleche recorded:\n", + "shell.query().table()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/PathsInContainers.ipynb b/notebooks/PathsInContainers.ipynb new file mode 100644 index 00000000..98c8849e --- /dev/null +++ b/notebooks/PathsInContainers.ipynb @@ -0,0 +1,704 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9050c7cf", + "metadata": {}, + "source": [ + "# Paths nested inside containers: the usual UX, and what to beware of\n", + "\n", + "[Files.ipynb](Files.ipynb) shows the sunny-day behaviour of `Path` arguments and\n", + "return values: files and directories are stored by **content** and rematerialized\n", + "under a temporary path on a cache hit.\n", + "\n", + "This notebook explores what happens when paths are *nested inside other values* —\n", + "dicts, lists, dataclasses — which fleche's `DestructuringMixin` takes apart and\n", + "reassembles (\"mends\") around the path machinery.\n", + "\n", + "The first half is the intended UX. The second half is what to watch out for, in\n", + "two distinct flavours:\n", + "\n", + "- **Beware** — consequences of what caching a pure function by content *means*.\n", + " A cached call is replayed by its **value**, so a file comes back as a copy: its\n", + " location, its identity, and whatever sat next to it on disk were never part of\n", + " that value. Code that leaned on them was relying on something the cache never\n", + " promised. These are not defects and they are not going to change.\n", + "- **Caveat** — real limits of the mending machinery, where a hit hands back\n", + " something less faithful than it could: paths used as dict keys, and paths\n", + " hidden inside containers fleche does not destructure.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1ba85e32", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:50.542910Z", + "iopub.status.busy": "2026-08-06T19:58:50.542683Z", + "iopub.status.idle": "2026-08-06T19:58:51.227501Z", + "shell.execute_reply": "2026-08-06T19:58:51.225855Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['PerKeyLockMixin', 'DestructuringMixin', 'PathValueMixin', 'ValueMixin']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import gc\n", + "import tempfile\n", + "from collections import Counter, defaultdict, namedtuple\n", + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "import fleche as fl\n", + "from fleche import fleche\n", + "\n", + "fl.cache(\"memory\") # transient in-memory cache\n", + "c = fl.cache()\n", + "\n", + "# DestructuringMixin sits *above* PathValueMixin: containers are taken apart\n", + "# first, and each nested Path is then stored by content as a blob.\n", + "[k.__name__ for k in type(c.values).__mro__ if k.__name__.endswith(\"Mixin\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6825eb50", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.230438Z", + "iopub.status.busy": "2026-08-06T19:58:51.229894Z", + "iopub.status.idle": "2026-08-06T19:58:51.235385Z", + "shell.execute_reply": "2026-08-06T19:58:51.233846Z" + } + }, + "outputs": [], + "source": [ + "# A scratch directory standing in for \"wherever your functions write their files\".\n", + "WORK = Path(tempfile.mkdtemp(suffix=\"-fleche-nested\"))\n", + "\n", + "def fresh(name, text):\n", + " p = WORK / name\n", + " p.write_text(text)\n", + " return p" + ] + }, + { + "cell_type": "markdown", + "id": "8841d5b6", + "metadata": {}, + "source": [ + "## The usual UX: containers of paths just work\n", + "\n", + "A cached function can return paths tucked inside dicts, lists, tuples, or\n", + "dataclasses. On a hit, the container is mended and every nested path comes back\n", + "freshly materialized, with its content and basename intact." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "96e644a1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.238073Z", + "iopub.status.busy": "2026-08-06T19:58:51.237818Z", + "iopub.status.idle": "2026-08-06T19:58:51.248360Z", + "shell.execute_reply": "2026-08-06T19:58:51.247018Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [fit] running: alpha\n", + "cold: PosixPath -> /tmp/tmp4rik9ggl-fleche-nested/alpha-fit.txt\n" + ] + } + ], + "source": [ + "@dataclass\n", + "class FitResult:\n", + " report: Path\n", + " score: float\n", + "\n", + "@fleche\n", + "def fit(seed):\n", + " print(\" [fit] running:\", seed)\n", + " return {\n", + " \"results\": [FitResult(fresh(f\"{seed}-fit.txt\", f\"loss={len(seed)}\"), 0.5)],\n", + " \"seed\": seed,\n", + " }\n", + "\n", + "cold = fit(\"alpha\")\n", + "print(\"cold:\", type(cold[\"results\"][0].report).__name__, \"->\", cold[\"results\"][0].report)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "74dac313", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.250981Z", + "iopub.status.busy": "2026-08-06T19:58:51.250755Z", + "iopub.status.idle": "2026-08-06T19:58:51.257133Z", + "shell.execute_reply": "2026-08-06T19:58:51.255733Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "warm: TempPath -> /tmp/tmpqu7_eme9fleche/alpha-fit.txt\n", + "content: loss=5 | name kept: alpha-fit.txt\n" + ] + } + ], + "source": [ + "warm = fit(\"alpha\") # no \"[fit] running\" -> served from cache\n", + "r = warm[\"results\"][0]\n", + "print(\"warm:\", type(r.report).__name__, \"->\", r.report)\n", + "print(\"content:\", r.report.read_text(), \"| name kept:\", r.report.name)" + ] + }, + { + "cell_type": "markdown", + "id": "78c368f4", + "metadata": {}, + "source": [ + "True content addressing: the *original* file can vanish entirely — the cache holds\n", + "the bytes, so hits keep working." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1a38c092", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.259707Z", + "iopub.status.busy": "2026-08-06T19:58:51.259397Z", + "iopub.status.idle": "2026-08-06T19:58:51.266926Z", + "shell.execute_reply": "2026-08-06T19:58:51.265381Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'loss=5'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cold[\"results\"][0].report.unlink() # delete the original on disk\n", + "again = fit(\"alpha\")\n", + "again[\"results\"][0].report.read_text() # still served, from stored content" + ] + }, + { + "cell_type": "markdown", + "id": "4db3a027", + "metadata": {}, + "source": [ + "## Beware: a hit gives you a *copy*, somewhere else\n", + "\n", + "The cold call returns whatever the function returned — the real location in\n", + "`WORK`, as a plain `Path`. A warm hit returns a `TempPath` in a fresh temporary\n", + "directory. Content is identical; **location is not**.\n", + "\n", + "The function's value is the file it returned, and a location is not part of a\n", + "file's content — so this is content addressing working exactly as advertised, not\n", + "a fidelity gap. The practical consequence: code that resolves *siblings* of a\n", + "returned path (`p.parent / \"meta.json\"`) works on the first call and breaks on\n", + "every hit, because only what was returned got captured — the sibling was never\n", + "part of the value.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7fe21179", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.269472Z", + "iopub.status.busy": "2026-08-06T19:58:51.269194Z", + "iopub.status.idle": "2026-08-06T19:58:51.278562Z", + "shell.execute_reply": "2026-08-06T19:58:51.277139Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [produce] running: beta\n", + "cold sibling exists: True\n", + "warm location: /tmp/tmp822dn7defleche/beta-data.csv\n", + "warm sibling exists: False\n" + ] + } + ], + "source": [ + "@fleche\n", + "def produce(seed):\n", + " print(\" [produce] running:\", seed)\n", + " fresh(f\"{seed}-meta.json\", '{\"version\": 1}') # sibling, NOT returned\n", + " return fresh(f\"{seed}-data.csv\", \"1,2,3\")\n", + "\n", + "p_cold = produce(\"beta\")\n", + "print(\"cold sibling exists:\", (p_cold.parent / \"beta-meta.json\").exists())\n", + "\n", + "p_warm = produce(\"beta\")\n", + "print(\"warm location:\", p_warm)\n", + "print(\"warm sibling exists:\", (p_warm.parent / \"beta-meta.json\").exists())" + ] + }, + { + "cell_type": "markdown", + "id": "a2950eff", + "metadata": {}, + "source": [ + "Only what is *returned* (or passed) is captured. If the sibling matters, return\n", + "it too — or return the whole directory." + ] + }, + { + "cell_type": "markdown", + "id": "8fcb480d", + "metadata": {}, + "source": [ + "## Beware: aliasing is not part of a value\n", + "\n", + "Return the *same* path twice and the cold result holds one object in two slots.\n", + "The warm hit mends each slot independently: two separate materializations, in two\n", + "different temp directories. Equal content, unequal (and non-identical) paths.\n", + "\n", + "Object identity is a property of one process's memory, not of the value being\n", + "cached — nothing about \"these two files are the same object\" survives a round\n", + "trip through storage, and nothing could. Compare content, never `is`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "67496526", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.281270Z", + "iopub.status.busy": "2026-08-06T19:58:51.281055Z", + "iopub.status.idle": "2026-08-06T19:58:51.291226Z", + "shell.execute_reply": "2026-08-06T19:58:51.289901Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [twice] running: gamma\n", + "cold: identical: True | equal: True\n", + "warm: identical: False | equal: False | same content: True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def twice(seed):\n", + " print(\" [twice] running:\", seed)\n", + " p = fresh(f\"{seed}-shared.txt\", seed * 2)\n", + " return [p, p]\n", + "\n", + "pc = twice(\"gamma\")\n", + "print(\"cold: identical:\", pc[0] is pc[1], \"| equal:\", pc[0] == pc[1])\n", + "\n", + "pw = twice(\"gamma\")\n", + "print(\"warm: identical:\", pw[0] is pw[1], \"| equal:\", pw[0] == pw[1],\n", + " \"| same content:\", pw[0].read_bytes() == pw[1].read_bytes())" + ] + }, + { + "cell_type": "markdown", + "id": "78097da1", + "metadata": {}, + "source": [ + "## Caveat: paths as dict *keys* mend into different keys\n", + "\n", + "Dict keys are destructured like values. A `Path` key comes back as a `TempPath`\n", + "at a new location — so the mended dict has a *different key* than the original,\n", + "and lookups by the original path miss. Use `str(path)` or a stable identifier as\n", + "the key if you intend to look things up by it." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "77ba67df", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.293639Z", + "iopub.status.busy": "2026-08-06T19:58:51.293387Z", + "iopub.status.idle": "2026-08-06T19:58:51.302382Z", + "shell.execute_reply": "2026-08-06T19:58:51.300962Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [index] running: delta\n", + "cold key: /tmp/tmp4rik9ggl-fleche-nested/delta-k.txt\n", + "warm key: /tmp/tmpbt0e_qsqfleche/delta-k.txt\n", + "kw[orig_key] works: False\n" + ] + } + ], + "source": [ + "@fleche\n", + "def index(seed):\n", + " print(\" [index] running:\", seed)\n", + " return {fresh(f\"{seed}-k.txt\", seed): \"metadata\"}\n", + "\n", + "kc = index(\"delta\")\n", + "orig_key = next(iter(kc))\n", + "\n", + "kw = index(\"delta\")\n", + "warm_key = next(iter(kw))\n", + "print(\"cold key:\", orig_key)\n", + "print(\"warm key:\", warm_key)\n", + "print(\"kw[orig_key] works:\", orig_key in kw)" + ] + }, + { + "cell_type": "markdown", + "id": "f5ea33da", + "metadata": {}, + "source": [ + "## Beware: each hit is its own copy, with temp-file lifetime\n", + "\n", + "Each hit copies the stored bytes into a new temporary directory (large files: mind\n", + "the churn). The temp tree lives exactly as long as some `TempPath` derived from\n", + "it is referenced — keep only a `str` of the location and the file is gone once the\n", + "path object is collected.\n", + "\n", + "Again this follows from the model rather than working against it: the cache owns\n", + "the content, and hands you a copy to use. It cannot know when you are finished\n", + "with that copy except by watching the reference you were given, so hold the\n", + "`Path` object for as long as you need the file, and `shutil.copy` it out if you\n", + "need it at a location of your own.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "88ba415b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.304989Z", + "iopub.status.busy": "2026-08-06T19:58:51.304725Z", + "iopub.status.idle": "2026-08-06T19:58:51.389293Z", + "shell.execute_reply": "2026-08-06T19:58:51.388102Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [artifact] running: epsilon\n", + "two hits, same location: False\n", + "file still there after dropping the TempPath: False\n", + "w2 unaffected (own temp dir): True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def artifact(seed):\n", + " print(\" [artifact] running:\", seed)\n", + " return fresh(f\"{seed}-art.txt\", seed)\n", + "\n", + "_ = artifact(\"epsilon\") # cold\n", + "w1 = artifact(\"epsilon\") # hit -> copy #1\n", + "w2 = artifact(\"epsilon\") # hit -> copy #2\n", + "print(\"two hits, same location:\", w1 == w2)\n", + "\n", + "location = str(w1) # keep only the string...\n", + "del w1\n", + "gc.collect()\n", + "print(\"file still there after dropping the TempPath:\", Path(location).exists())\n", + "print(\"w2 unaffected (own temp dir):\", w2.exists())" + ] + }, + { + "cell_type": "markdown", + "id": "d8593d07", + "metadata": {}, + "source": [ + "## Caveat: paths hidden in *opaque* containers are stored by location, not content\n", + "\n", + "Destructuring only recurses into what it knows: lists, tuples, dicts, dataclasses,\n", + "attrs classes. Everything else — namedtuples (deliberately treated as opaque),\n", + "sets, arbitrary objects with a `Path` attribute — is stored **verbatim**. The\n", + "nested path never reaches the content machinery: what is stored is the path\n", + "*object*, pointing at the original location.\n", + "\n", + "The call is still *keyed* correctly (the digest layer does recurse, hashing file\n", + "content), so hits and misses behave right. But a warm hit hands back the original\n", + "location — an **incompletely mended** result. If that file has been deleted,\n", + "moved, or edited since, the hit returns a dangling or stale path, silently." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "6e5db0b0", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.391523Z", + "iopub.status.busy": "2026-08-06T19:58:51.391305Z", + "iopub.status.idle": "2026-08-06T19:58:51.399526Z", + "shell.execute_reply": "2026-08-06T19:58:51.398213Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [bundle] running: zeta\n", + "warm type: PosixPath -> /tmp/tmp4rik9ggl-fleche-nested/zeta-nt.txt\n", + "points at the ORIGINAL location: True\n" + ] + } + ], + "source": [ + "Bundle = namedtuple(\"Bundle\", [\"out\", \"score\"])\n", + "\n", + "@fleche\n", + "def bundle(seed):\n", + " print(\" [bundle] running:\", seed)\n", + " return Bundle(fresh(f\"{seed}-nt.txt\", seed), 0.5)\n", + "\n", + "b_cold = bundle(\"zeta\")\n", + "b_warm = bundle(\"zeta\") # cache hit...\n", + "print(\"warm type:\", type(b_warm.out).__name__, \"->\", b_warm.out)\n", + "print(\"points at the ORIGINAL location:\", b_warm.out == b_cold.out)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5c4884f9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.401931Z", + "iopub.status.busy": "2026-08-06T19:58:51.401678Z", + "iopub.status.idle": "2026-08-06T19:58:51.406926Z", + "shell.execute_reply": "2026-08-06T19:58:51.405586Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hit returns: /tmp/tmp4rik9ggl-fleche-nested/zeta-nt.txt\n", + "exists: False <- dangling, no warning\n" + ] + } + ], + "source": [ + "# Now the original vanishes -- e.g. a scratch dir is cleaned up between sessions.\n", + "b_cold.out.unlink()\n", + "\n", + "b_stale = bundle(\"zeta\") # still a cache hit (keyed on content at save time)\n", + "print(\"hit returns:\", b_stale.out)\n", + "print(\"exists:\", b_stale.out.exists(), \" <- dangling, no warning\")" + ] + }, + { + "cell_type": "markdown", + "id": "5ede4d0b", + "metadata": {}, + "source": [ + "Compare with Edge 1's dict: the *destructured* container survived deletion of the\n", + "original because the content was captured. The namedtuple did not. Same story\n", + "for sets and custom non-dataclass objects.\n", + "\n", + "**Rule of thumb:** return paths in plain dicts/lists/tuples/dataclasses, not\n", + "smuggled inside opaque types." + ] + }, + { + "cell_type": "markdown", + "id": "ab85f64e", + "metadata": {}, + "source": [ + "## Caveat: container subclasses are opaque — deliberately\n", + "\n", + "Mending rebuilds containers via `type(value)()`, a contract subclasses\n", + "may repurpose: `defaultdict`'s first argument is a factory (would crash),\n", + "`Counter` *counts* its argument (would silently corrupt). Destructuring therefore\n", + "matches **exact types only** — `dict`, `OrderedDict`, `list`, `tuple` (plus\n", + "dataclasses/attrs, whose mending bypasses `__init__`). Everything else is stored\n", + "verbatim as an opaque value.\n", + "\n", + "So subclasses round-trip *as values* — but any path nested inside them follows\n", + "Edge 5's location semantics, not content addressing." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d19c337b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.409279Z", + "iopub.status.busy": "2026-08-06T19:58:51.409056Z", + "iopub.status.idle": "2026-08-06T19:58:51.417030Z", + "shell.execute_reply": "2026-08-06T19:58:51.415673Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [by_kind] running: eta\n", + "warm: defaultdict {'files': [PosixPath('/tmp/tmp4rik9ggl-fleche-nested/eta-dd.txt')]}\n", + "but the nested path is the ORIGINAL location: True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def by_kind(seed):\n", + " print(\" [by_kind] running:\", seed)\n", + " d = defaultdict(list)\n", + " d[\"files\"].append(fresh(f\"{seed}-dd.txt\", seed))\n", + " return d\n", + "\n", + "dd_cold = by_kind(\"eta\")\n", + "dd_warm = by_kind(\"eta\")\n", + "print(\"warm:\", type(dd_warm).__name__, dict(dd_warm))\n", + "print(\"but the nested path is the ORIGINAL location:\", dd_warm[\"files\"][0] == dd_cold[\"files\"][0])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ef2d0d13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:51.419207Z", + "iopub.status.busy": "2026-08-06T19:58:51.419003Z", + "iopub.status.idle": "2026-08-06T19:58:51.425975Z", + "shell.execute_reply": "2026-08-06T19:58:51.424622Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [tally] running: 5\n", + "cold: Counter({'b': 6, 'a': 5})\n", + "warm: Counter({'b': 6, 'a': 5}) <- opaque, so counts survive intact\n", + "Help on function register_destructurer in module fleche.storage.destructuring:\n", + "\n", + "register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None\n", + " Register a custom container destructurer.\n", + " \n", + " *pred(value)* should return ``True`` for values this destructurer handles.\n", + " *fn* must accept ``(intern, value)`` where *intern* is\n", + " :meth:`DestructuringMixin._intern_rec`. Entries are appended after the\n", + " built-in ones; first match wins, so registering a handler for an entirely\n", + " new container type is safe without displacing list/dict/dataclass/attrs.\n", + " The built-in predicates match exact types only, so a handler for a subclass\n", + " (e.g. ``defaultdict`` with a picklable factory) can also be registered\n", + " without conflict. Call before any :class:`DestructuringMixin` instance is\n", + " used.\n", + "\n" + ] + } + ], + "source": [ + "@fleche\n", + "def tally(seed):\n", + " print(\" [tally] running:\", seed)\n", + " return Counter({\"a\": seed, \"b\": seed + 1})\n", + "\n", + "print(\"cold:\", tally(5))\n", + "print(\"warm:\", tally(5), \" <- opaque, so counts survive intact\")\n", + "\n", + "# Custom destructurers for subclasses can be opted in later:\n", + "help(fl.storage.destructuring.register_destructurer)" + ] + }, + { + "cell_type": "markdown", + "id": "fea5f4b5", + "metadata": {}, + "source": [ + "## Summary: rules of thumb\n", + "\n", + "- **Works out of the box:** paths (files *and* directories) as arguments, return\n", + " values, or nested anywhere inside dicts / lists / tuples / dataclasses / attrs\n", + " classes — arbitrarily deep. Content-addressed, dedup'd, survives deletion of\n", + " the originals.\n", + "- **Beware — a hit is a copy, because a value is all that is cached:** returned\n", + " paths live in fresh temp directories. Don't resolve siblings, don't compare\n", + " locations, don't expect aliasing, and keep a reference to the `Path` object for\n", + " as long as you need the file. None of this is pending a fix; it is what\n", + " caching by content means.\n", + "- **Caveat — don't key dicts by `Path`** if you'll look them up afterwards — keys\n", + " mend into new locations. Use `str(path)` or a stable ID.\n", + "- **Caveat — don't hide paths in opaque containers** (namedtuples, sets, plain\n", + " classes, and any container *subclass* — only exact `dict` / `OrderedDict` /\n", + " `list` / `tuple` are destructured): they are stored by location and come back\n", + " stale or dangling after the original moves on. `register_destructurer` is the\n", + " opt-in door for well-behaved custom containers.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/fleche/caches.py b/src/fleche/caches.py index 16d4a93d..4067ed69 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -62,7 +62,10 @@ def prepare(self, call: Call) -> PreparedCall: Finish the returned :class:`~fleche.call.PreparedCall` with exactly one of :meth:`~fleche.call.PreparedCall.commit` or - :meth:`~fleche.call.PreparedCall.abandon`. + :meth:`~fleche.call.PreparedCall.abandon`. Until then the stashed + values are referenced by no record, and :meth:`Cache.gc` treats them as + roots (:func:`~fleche.call.in_flight_digests`) so a concurrent sweep + cannot evict the arguments of a running call. """ return PreparedCall(digested=call.digest(), cache=self) @@ -324,6 +327,8 @@ def prepare(self, call: Call) -> PreparedCall: # record cannot end up keyed on post-mutation content. No cache-level # lock: the keys are only known once the value storage has digested # each value, and value storages carry their own per-key locking. + # Registering the result as a gc root is PreparedCall's own doing, so + # it survives the rebinding CacheWrapper.prepare does. return PreparedCall(digested=call.stash(self.values), cache=self) def save(self, call: PreparedCall | Call) -> str: @@ -465,10 +470,46 @@ def gc(self) -> set[Digest]: ``values`` key outside the reachable set. Call records are left untouched. + Values belonging to a call that is between + :meth:`~fleche.caches.BaseCache.prepare` and + :meth:`~fleche.call.PreparedCall.commit` are referenced by no record + yet, so they are seeded into the reachable set from + :func:`~fleche.call.in_flight_digests` — otherwise a sweep during a + long function body would evict the arguments out from under the call, + and the eventual commit would file a record with dangling references. + That registry is **per-process**: a sweep here cannot see a call in + flight in another process, so avoid running ``gc()`` against a cache + that other processes are writing to concurrently. The concrete case + is :class:`~fleche.remote.SshCache` — the arguments are stashed by the + *server's* ``prepare``, and only the sealed record travels back, so a + sweep run on the server has nothing to key on while the client's body + runs. + + The three reads below are ordered so that a call finishing *during* the + sweep cannot fall between them. Candidates are listed first, so a value + stored later in the sweep is not a candidate at all. Then the in-flight + roots, then the records: :meth:`~fleche.call.PreparedCall.commit` + deregisters only *after* :meth:`save` has filed the record, so a call + that commits mid-sweep is seen by the registry read, the record read, or + both — never neither. Reading the records first instead leaves a real + hole, and a threaded test in ``test_gc.py`` reproduces it. + + What remains is the gap between storing a value and registering the call + that owns it, and only if it spans those first two reads. That is the + same window the one-shot :meth:`save` has always had between storing the + values and filing the record — bounded by a storage write rather than by + a function body — and closing it would need a lock shared by every + writer. It is real rather than theoretical: forcing that gap open makes + the eviction reproducible, which + ``test_gc_may_evict_a_value_stored_but_not_yet_registered`` pins. So a + sweep concurrent with writers is best-effort, and a sweep on an idle + cache is exact. + Returns: The set of digests that were evicted from value storage. """ - reachable: set[Digest] = set() + candidates = list(self.values.list()) + reachable: set[Digest] = call.in_flight_digests() for key in self.calls.list(): try: dc = self.calls.load(key) @@ -493,7 +534,7 @@ def gc(self) -> set[Digest]: frontier |= new evicted: set[Digest] = set() - for key in list(self.values.list()): + for key in candidates: if key not in reachable: try: self.values.evict(key) diff --git a/src/fleche/call.py b/src/fleche/call.py index 38355188..c391398a 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -1,4 +1,6 @@ import logging +import threading +import weakref from dataclasses import dataclass, field, replace from functools import lru_cache from typing import Any, Callable, get_type_hints, get_origin, get_args, Annotated @@ -331,6 +333,38 @@ def fetch(self, cache) -> "LazyCall": ) +# Prepared calls whose argument values are stored but not yet referenced by any +# record — the prepare -> commit window. Held weakly: a prepared call dropped +# without commit() or abandon() falls out on its own, and the registry never +# keeps a call (or its cache) alive. Keyed by ``id`` because ``PreparedCall`` +# is an unhashable dataclass; ``WeakValueDictionary`` only drops a key whose +# stored reference is still the dead one, so a recycled ``id`` is safe. The +# lock guards the snapshot below against a concurrent ``prepare`` resizing the +# mapping mid-iteration; weakref removals are already deferred while it is +# being iterated. +_IN_FLIGHT: "weakref.WeakValueDictionary[int, PreparedCall]" = weakref.WeakValueDictionary() +_IN_FLIGHT_LOCK = threading.Lock() + + +def in_flight_digests() -> set[Digest]: + """Return the value digests of every prepared-but-unfinished call. + + These are stored in value storage but reachable from no record yet, so a + reachability sweep (:meth:`fleche.caches.Cache.gc`) must treat them as + roots or it evicts the arguments of a call still running. Registration is + per-process: a call in flight in *another* process is not visible here. + """ + with _IN_FLIGHT_LOCK: + in_flight = list(_IN_FLIGHT.values()) + reachable: set[Digest] = set() + for prepared in in_flight: + digested = prepared.digested + if isinstance(digested.result, Digest): + reachable.add(digested.result) + reachable.update(v for v in digested.arguments.values() if isinstance(v, Digest)) + return reachable + + @dataclass class PreparedCall: """A call admitted to a cache: arguments stored, key sealed, result awaited. @@ -358,6 +392,14 @@ class PreparedCall: _result: Any = field(default=None, init=False, repr=False) _metadata: "dict | None" = field(default=None, init=False, repr=False) + def __post_init__(self) -> None: + # Registered here rather than in ``prepare`` so wrapper rebinding — + # ``replace(inner.prepare(call), cache=self)`` — registers the new + # object too; the inner one is garbage by then and drops out on its + # own. + with _IN_FLIGHT_LOCK: + _IN_FLIGHT[id(self)] = self + def commit(self, result: Any, metadata: dict | None = None) -> Digest: """Attach *result* and *metadata* to the record and file it. @@ -384,7 +426,14 @@ def commit(self, result: Any, metadata: dict | None = None) -> Digest: self._result = result if metadata is not None: self._metadata = metadata - return self.cache.save(self) + try: + return self.cache.save(self) + finally: + # Either the record now references the arguments, or the save was + # rejected and they are genuine orphans. Either way they no longer + # need protecting from a sweep. + with _IN_FLIGHT_LOCK: + _IN_FLIGHT.pop(id(self), None) def to_lookup_key(self) -> Digest: return self.digested.to_lookup_key() @@ -406,12 +455,16 @@ def abandon(self) -> None: """Release the call without recording it. Called when the function body raises or its result is not cacheable. - The default is a no-op: argument values stored by + The default is a no-op beyond bookkeeping: argument values stored by :meth:`~fleche.caches.BaseCache.prepare` are content-addressed orphans - that a later garbage collection sweep reclaims. Subclasses may hook + that a later garbage collection sweep reclaims — abandoning is what + releases them to it, since an unfinished call keeps them reachable + (:func:`in_flight_digests`). Subclasses may hook cleanup here. Idempotent; only :meth:`commit` is barred afterwards. """ self._finished = True + with _IN_FLIGHT_LOCK: + _IN_FLIGHT.pop(id(self), None) def __enter__(self) -> "PreparedCall": return self diff --git a/src/fleche/digest.py b/src/fleche/digest.py index 2eb40f87..8b051fe7 100644 --- a/src/fleche/digest.py +++ b/src/fleche/digest.py @@ -6,6 +6,8 @@ import numbers from numbers import Number import struct +import subprocess +from pathlib import Path import types import importlib.metadata from collections.abc import Iterable, Mapping @@ -18,6 +20,15 @@ logger = logging.getLogger("fleche.digest") +# Types that are ``Iterable`` but are matched by an arm *above* the generic +# ``Iterable`` arm in :func:`_digest_bytes`: each hashes its own buffer and +# never looks at the elements, so nothing reachable through one can influence +# a digest. Named here so a walker that mirrors ``digest`` — notably +# :func:`fleche.storage.paths.find_path` — can skip exactly the same types +# instead of re-deriving the list and drifting from it. +OPAQUE_ITERABLES = (np.ndarray, pd.DataFrame, pd.Series, pd.Index) + + class Indigestible(Exception): """Exception raised when an object cannot be digested.""" @@ -151,6 +162,29 @@ def digest(value: Any) -> Digest: return Digest(_digest_bytes(value).decode()) +def _path_content_digest(path) -> Digest: + """Content-only digest of a path: a file as its bytes, a directory as its + ``{name: child}`` tree (the directory's own name excluded). + + This mirrors exactly what :class:`~fleche.storage.paths.PathValueMixin` + stores for a file's content blob and for a :class:`DirectoryBlob`, so a + directory tree and a standalone file agree on their children's digests. A + *standalone* file additionally carries its basename (see the ``Path`` arm of + :func:`_digest_bytes`); inside a directory the name lives in the parent's + ``contents`` key, so only the content matters here. + """ + if path.is_file(): + return digest(path.read_bytes()) + if path.is_dir(): + contents = { + child.name: _path_content_digest(child) + for child in path.iterdir() + if child.is_file() or child.is_dir() + } + return digest(("DirectoryBlob", contents)) + raise Indigestible("Can only digest files and folders!") + + def _digest_bytes(value: Any) -> bytes: """ Returns bytes representing the SHA-256 digest of *value*. @@ -229,6 +263,39 @@ def _digest_bytes(value: Any) -> bytes: return _digest_bytes(value) case bytes(): m.update(value) + case Path(): + # A *file* is identified by (basename, content): its name (and + # extension) matter, but the content deduplicates. A *directory* is + # identified by its tree alone — its own (often incidental, e.g. a + # temp dir) root name is dropped, though child names are kept. Both + # mirror exactly what PathValueMixin stores, preserving the + # digest(path) == values.save(path) invariant on which cache lookups + # of path-valued arguments/results depend. The "FileBlob" / + # "DirectoryBlob" salts MUST match + # fleche.storage.paths.{FileBlob,DirectoryBlob}.__digest__. + # + # Reading can fail for reasons that have nothing to do with the + # value being unsuitable — permissions, EIO, a network mount that + # went away. Those degrade to `Indigestible` like every other + # case in this arm, so the wrapper runs the call uncached instead + # of crashing it before the body ever executes. A caller who + # genuinely needs the read to succeed will hit the same error on + # its own terms inside the function. + try: + if not value.exists(): + raise Indigestible("Only existing paths can be digested.") + if value.is_file(): + return _digest_bytes( + ("FileBlob", value.name, _path_content_digest(value)) + ) + elif value.is_dir(): + return _path_content_digest(value).encode() + else: + raise Indigestible("Can only digest files and folders!") + except OSError as e: + # `Indigestible` is not an OSError, so the raises above pass + # through untouched. + raise Indigestible(f"Could not read {value}: {e}") from None case np.ndarray(): m.update(_digest_bytes(value.dtype.str)) m.update(_digest_bytes(value.shape)) @@ -298,6 +365,12 @@ def _digest_bytes(value: Any) -> bytes: m.update(_digest_bytes(value.__func__)) case property(): m.update(_digest_bytes((value.fget, value.fset, value.fdel))) + case subprocess.CompletedProcess(): + m.update( + _digest_bytes( + (value.args, value.returncode, value.stdout, value.stderr) + ) + ) case _ if isinstance(value, type) and value.__module__ == 'builtins': # Digest a built-in type (int, str, list, …) by its qualified name. # Restricted to the builtins module; user-defined types remain Indigestible. diff --git a/src/fleche/remote.py b/src/fleche/remote.py index 0d9f0679..bb185edf 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -60,6 +60,7 @@ from .call import DigestedCall, LazyCall, PreparedCall, QueryCall from .digest import Digest from .storage.base import SaveError +from .storage.paths import find_path logger = logging.getLogger("fleche.remote") @@ -381,6 +382,35 @@ class RemoteConnectionError(RuntimeError): """Raised when the SSH subprocess cannot be reached or has died.""" +class RemotePathUnsupported(SaveError): + """Raised when a :class:`~pathlib.Path` would cross the SSH boundary. + + Path values are stored **by content** (see + :class:`~fleche.storage.paths.PathValueMixin`), but the RPC ships values + by cloudpickle, and a pickled ``Path`` is just its *string*. Letting one + through would hand the server a name to resolve against its **own** + filesystem: it stores whatever happens to sit at that location — a + different file, or nothing — under a digest that no longer matches the + client's ``digest(path)``. The mismatch silently breaks the + ``digest(x) == save_value(x)`` seal every lookup depends on, so the + record is filed under a key no client can ever reproduce. + + Subclasses :class:`~fleche.storage.base.SaveError` so the standard + save-side degradations apply and nothing has to special-case it: a path + *argument* falls back to a digest-only reference (correct local digest, + lookups still hit, only the bytes are unavailable remotely), and a path + *result* turns into :class:`~fleche.caches.Rejected` — the call runs and + returns normally, it just is not cached. On the **load** side it + propagates to the caller instead: a path the server materialized into + its own temp directory is meaningless here, and the guard's whole point + is to say so rather than hand back a dangling name. + + This is a guard, not a verdict: doing the path-to-blob reduction on the + *client* would make paths work over SSH with the seal intact. Tracked in + issue #829. + """ + + def _warn_on_version_skew(info: dict[str, Any]) -> None: """Log a warning when the remote's fleche or cloudpickle version differs. @@ -712,7 +742,9 @@ class _RemoteValues: _cache: "SshCache" def save(self, value: Any) -> Digest: - return self._cache._rpc("save_value", value) + # `save_value`, not `_rpc`, so the path guard applies to a committed + # result exactly as it does to a bare `save_value` call. + return self._cache.save_value(value) def _fetch_lazy_call(sc: "SshCache", dc: DigestedCall) -> LazyCall: @@ -762,6 +794,13 @@ class SshCache(BaseCache): ControlPersist in ``~/.ssh/config`` or via *ssh_options* to share the underlying connection across multiple fleche runs. + **Filesystem paths do not cross this cache.** Values travel by + cloudpickle, so a :class:`~pathlib.Path` would arrive as a bare string for + the remote to resolve against its own disk; both directions raise + :class:`RemotePathUnsupported` instead. Ship the file's ``bytes``, or put + a local layer in front of the remote one — see that exception and + :ref:`file-remote-caches`. + Args: host: SSH target, e.g. ``"user@host"`` or any alias from ``~/.ssh/config``. @@ -840,33 +879,93 @@ def _rpc(self, name: str, *args: Any) -> Any: raise Rejected(self, *args) return spec.unwrap(self, self._conn.call(name, *args)) - def save(self, call: PreparedCall | _call.Call) -> str: + @staticmethod + def _reject_path(value: Any, what: str) -> None: + """Raise :class:`RemotePathUnsupported` if *value* carries a ``Path``. + + Walks the value the way :func:`~fleche.digest.digest` does, so a path + hidden in a namedtuple or set is caught too — those still decide the + key, and the far side would resolve the same name against its own + filesystem. + """ + path = find_path(value) + if path is not None: + raise RemotePathUnsupported( + f"{what} carries the filesystem path {str(path)!r}, which cannot " + "cross an SshCache: values travel by cloudpickle, so only the " + "path string would arrive and the remote would resolve it " + "against its own filesystem. Return the file's `bytes` instead " + "to store its content, or keep path-valued calls in a local " + "cache layer." + ) + + def save(self, call: PreparedCall | DigestedCall | _call.Call) -> str: if isinstance(call, PreparedCall): # Recreate Cache.save's ending in two trips: ship the result value, # then file the plain digested record — a PreparedCall itself never # goes over the wire. A failure between the trips leaves the value # as a content-addressed orphan for gc, like any abandoned call. - digested = call.resolve(_RemoteValues(self)) + # `_RemoteValues.save` routes through `save_value`, so a path + # result is refused here rather than resolved against the + # server's filesystem. + try: + digested = call.resolve(_RemoteValues(self)) + except RemotePathUnsupported as e: + raise Rejected(e) from None return self._rpc("save", digested) + # A DigestedCall carries only digests; the degenerate live-`Call` + # form still carries values, and the server would stash them itself. + if isinstance(call, _call.Call): + try: + self._reject_path(dict(call.arguments), "the call's arguments") + self._reject_path(call.result, "the call's result") + except RemotePathUnsupported as e: + raise Rejected(e) from None return self._rpc("save", call) def load(self, key: str) -> LazyCall: return self._rpc("load", key) def load_value(self, key: str) -> Any: - return self._rpc("load_value", key) + # The server materializes a stored path into a temp directory on + # *its* filesystem and can only ship us the name — which points + # nowhere here, and which the server unlinks as soon as its own + # reference dies. Refuse it rather than return a dangling path. + value = self._rpc("load_value", key) + self._reject_path(value, f"the value at {key}") + return value + + def save_value(self, value: Any) -> Digest: + # One round trip per value (no batching yet). Values travel by + # cloudpickle, so a Path would ship its path *string*, not its + # content — see `RemotePathUnsupported` for why that is refused + # rather than silently stored against the server's filesystem. + self._reject_path(value, "the value") + return self._rpc("save_value", value) def prepare(self, call: _call.Call) -> PreparedCall: # One round trip: the whole call goes over so the remote stashes the # argument values before the body runs and seals the record, which - # comes back for the client to complete with ``save``. Values travel - # by cloudpickle, so a Path argument ships its path *string*, not its - # content — paths over SSH are unsupported. + # comes back for the client to complete with ``save``. self._ensure_handshake() if self._info_cache and self._info_cache.get("read_only", False): # Digest-only admission, as BaseCache: the body still runs, and the # commit's ``save`` is rejected locally without a round trip. return super().prepare(call) + if find_path(dict(call.arguments)) is not None: + # Shipping the live call would hand the server a path *string* to + # resolve against its own filesystem — exactly what + # `RemotePathUnsupported` exists to prevent, and this is the one + # RPC that carries argument *values* rather than digests. Stash + # the arguments one at a time instead: `_RemoteValues.save` routes + # each through `save_value`, so only the path arguments raise and + # `Call.stash` degrades *those* to a digest-only reference computed + # here — keeping the `digest(x) == save_value(x)` seal intact so + # lookups still hit — while their non-path siblings are stored + # normally and stay loadable off the record. Costs one round trip + # per argument rather than one for the call, which is the price of + # not degrading the whole call for one path. + return PreparedCall(digested=call.stash(_RemoteValues(self)), cache=self) return PreparedCall(digested=self._rpc("prepare", call), cache=self) def evict(self, key: str | Digest) -> None: diff --git a/src/fleche/storage/__init__.py b/src/fleche/storage/__init__.py index 291570f0..e602593b 100644 --- a/src/fleche/storage/__init__.py +++ b/src/fleche/storage/__init__.py @@ -19,7 +19,8 @@ get_storage_constructor, is_registered_storage, ) -from .destructuring import DestructuringMixin, register_destructurer +from .destructuring import DestructuringMixin, child_slots, register_destructurer +from .paths import PathValueMixin, TempPath, FileBlob, DirectoryBlob, find_path from .memory import MemoryBackend, ValueMemory, CallMemory from .void import VoidBackend, ValueVoid, CallVoid from .file import FileStorage @@ -47,7 +48,13 @@ "get_storage_constructor", "is_registered_storage", "DestructuringMixin", + "child_slots", "register_destructurer", + "PathValueMixin", + "TempPath", + "FileBlob", + "DirectoryBlob", + "find_path", "MemoryBackend", "ValueMemory", "CallMemory", diff --git a/src/fleche/storage/bagofholding_file.py b/src/fleche/storage/bagofholding_file.py index 2b49f975..43ffd5e8 100644 --- a/src/fleche/storage/bagofholding_file.py +++ b/src/fleche/storage/bagofholding_file.py @@ -9,6 +9,7 @@ from .base import SaveError, ValueMixin, CallMixin, register_storage from .thread_safe import PerKeyLockMixin from .destructuring import DestructuringMixin +from .paths import PathValueMixin from ..digest import Digest, DIGEST_LENGTH from pyiron_snippets.import_alarm import ImportAlarm @@ -371,7 +372,7 @@ def rebag(self, version_validator: VersionValidator = "none") -> None: @register_storage("bagofholding_hdf", kind="value") @dataclass(frozen=True) -class ValueBagOfHoldingH5File(PerKeyLockMixin, DestructuringMixin, ValueMixin, BagOfHoldingH5FileBackend): +class ValueBagOfHoldingH5File(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, BagOfHoldingH5FileBackend): def to_config(self) -> dict[str, Any]: return { "type": "bagofholding_hdf", diff --git a/src/fleche/storage/base.py b/src/fleche/storage/base.py index dae3077b..0079969a 100644 --- a/src/fleche/storage/base.py +++ b/src/fleche/storage/base.py @@ -348,6 +348,33 @@ def save(self, value: Any, key: Digest | None = None) -> Digest: ... @abstractmethod def load(self, key: Digest | str) -> Any: ... + def load_raw(self, key: Digest | str) -> Any: + """The entry exactly as stored, with no mending applied. + + ``load`` runs the whole mending chain — destructured children are + rewired back in, a stored path is materialized into a temp tree. That + is right for callers who want the *value*, and wrong for callers who + want the *record*: mending resolves child references away, so a + reference-graph walk (``gc``, ``count_reuses``) that reads through + ``load`` cannot see them and concludes the children are unreachable. + + Mending layers override ``load`` only, so this default is already raw + for storages that do none; :class:`ValueMixin` overrides it with the + bare backend read. + """ + return self.load(key) + + def _raw_sub_digests(self, raw: Any) -> set[Digest]: + """Digests directly referenced by a raw stored entry. + + The terminating case of a cooperative chain: every mixin that wraps + values in a record of its own reports that record's child references + here and delegates the rest upward, so a storage's reachable set is + the union over its layers. An entry no layer claims references + nothing. + """ + return set() + class ValueMixin(ValueStorage, StorageBackend): """Bridges :class:`~fleche.storage.base.ValueStorage` with :class:`~fleche.storage.base.StorageBackend` primitives. @@ -365,6 +392,9 @@ def save(self, value: Any, key: Digest | None = None) -> Digest: return self.put(value, key) def load(self, key: Digest | str) -> Any: + return self.load_raw(key) + + def load_raw(self, key: Digest | str) -> Any: with self._operation_context(key): key = self._normalize_key(key) logger.debug("Loading value with key %s", key) diff --git a/src/fleche/storage/destructuring.py b/src/fleche/storage/destructuring.py index 549e6e32..217778eb 100644 --- a/src/fleche/storage/destructuring.py +++ b/src/fleche/storage/destructuring.py @@ -1,6 +1,7 @@ import hashlib from abc import ABC, abstractmethod -from collections import Counter +from collections import Counter, OrderedDict +from collections.abc import Mapping import dataclasses as _dataclasses from dataclasses import dataclass from numbers import Number @@ -71,10 +72,17 @@ def get(storage, key): else: return key + @dataclass class DigestedIterable(Digested): items: list | tuple + def __repr__(self): + # Surface the inner container type (list / tuple / subclass) which is + # part of the wrapper's identity but invisible in a bare [...] / (...) repr. + inner = list.__repr__(list(self.items)) + return f"DigestedIterable[{type(self.items).__name__}]({inner})" + def underlying(self): return self.items @@ -95,32 +103,58 @@ def _rebuild_digest(cls, value: list | tuple, labels: tuple, children: tuple) -> @dataclass -class DigestedDict(Digested): - items: dict +class DigestedMapping(Digested): + """Marker for a destructured mapping, preserving the concrete mapping type. + + Reconstruction (:meth:`_rebuild_plain`, :meth:`mend`) rebuilds via + ``type(value)()``, so only mapping types + whose constructor accepts an iterable of pairs round-trip correctly + (``dict``, ``OrderedDict``, plain dict subclasses, ...). Types that break + this contract — e.g. ``defaultdict`` (first argument is a factory, raises) + or ``Counter`` (would *count* the pairs instead) — must not be sundered. + Which mappings are destructured is controlled by the exact-type allowlist + in ``_DESTRUCTURERS`` (currently ``dict`` and ``OrderedDict``); every other + mapping type is stored verbatim as an opaque value unless a handler is + registered via :func:`register_destructurer`. + """ + + items: Mapping + + def __repr__(self): + # The inner type (dict / OrderedDict / DirectoryBlob / ...) is part of the + # wrapper's identity but lost by ``dict``'s ``{...}`` repr. Surface it. + return f"DigestedMapping[{type(self.items).__name__}]({dict.__repr__(self.items)})" def underlying(self): return self.items - def mend(self, storage: 'DestructuringMixin') -> dict: - return {self.get(storage, k): self.get(storage, v) - for k, v in self.items.items()} + def mend(self, storage: 'DestructuringMixin') -> Mapping: + return type(self.items)( + (self.get(storage, k), self.get(storage, v)) # ty: ignore[too-many-positional-arguments] + for k, v in self.items.items() + ) @classmethod - def _slots(cls, value: dict) -> list[tuple[None, Any]]: + def _slots(cls, value: Mapping) -> list[tuple[None, Any]]: # keys and values interned as one flat sequence; _rebuild_* re-pairs them by # position using len(value) as the key/value split point. return [(None, k) for k in value] + [(None, v) for v in value.values()] @classmethod - def _rebuild_plain(cls, value: dict, labels: tuple, children: tuple) -> dict: + def _rebuild_plain(cls, value: Mapping, labels: tuple, children: tuple) -> Mapping: n = len(value) - return dict(zip(children[:n], children[n:])) + return type(value)(zip(children[:n], children[n:])) # ty: ignore[too-many-positional-arguments] @classmethod - def _rebuild_digest(cls, value: dict, labels: tuple, children: tuple) -> 'DigestedDict': + def _rebuild_digest(cls, value: Mapping, labels: tuple, children: tuple) -> 'DigestedMapping': return cls(cls._rebuild_plain(value, labels, children)) +# Backward-compatible alias: DigestedMapping was renamed from DigestedDict when it +# was generalized from dict to any Mapping. Keep the old name importable. +DigestedDict = DigestedMapping + + @dataclass class DigestedFields(Digested): """Common base for record-shaped value markers (dataclasses, attrs). @@ -195,13 +229,50 @@ def _field_items(value: Any) -> list[tuple[str, Any]]: _DESTRUCTURERS: list[tuple[Callable[[Any], bool], Callable]] = [ - (lambda v: isinstance(v, (list, tuple)), DigestedIterable.sunder), - (lambda v: isinstance(v, dict), DigestedDict.sunder), + # Exact types (not isinstance): reconstruction goes through + # ``type(value)()``, a contract subclasses may repurpose — + # defaultdict's first argument is a factory (crashes), Counter *counts* the + # pairs (silently wrong), namedtuples reject a single iterable. Subclasses + # are therefore stored verbatim as opaque values unless a handler is opted + # in via register_destructurer(). Dataclasses/attrs are exempt from this + # concern: their mend bypasses __init__ entirely. + (lambda v: type(v) in (list, tuple), DigestedIterable.sunder), # noqa: E721 + (lambda v: type(v) in (dict, OrderedDict), DigestedMapping.sunder), # noqa: E721 (lambda v: _dataclasses.is_dataclass(v) and not isinstance(v, type), DigestedDataclass.sunder), (_attrs.is_attrs_instance, DigestedAttrs.sunder), ] +def child_slots(value: Any) -> list[tuple[Any, Any]] | None: + """Enumerate the ``(label, child)`` pairs a save would recurse into. + + The read-only half of :meth:`DestructuringMixin._intern_rec`'s dispatch: + same leaf rules (numbers / strings / bytes / digests / namedtuples are + opaque), same ``_DESTRUCTURERS`` lookup, but it only *reports* the + children instead of interning them. ``None`` means "opaque leaf" — the + value has no children a destructuring save would look inside. + + Labels follow the underlying :meth:`Digested._slots`: field names for + dataclass / ``attrs`` instances, ``None`` for the positional slots of + lists, tuples, and mappings. + + A destructurer registered through :func:`register_destructurer` is only + visible here if it is a :class:`Digested` classmethod (i.e. exposes + ``_slots``); anything else is reported as an opaque leaf, since there is + no way to enumerate its children without also storing them. + """ + if isinstance(value, (digest.Digest, Number, str, bytes)): + return None + if isinstance(value, tuple) and DestructuringMixin._is_trojan_tuple(value): + return None + for pred, sunder_fn in _DESTRUCTURERS: + if pred(value): + owner = getattr(sunder_fn, "__self__", None) + slots = getattr(owner, "_slots", None) + return None if slots is None else slots(value) + return None + + def register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None: """Register a custom container destructurer. @@ -210,7 +281,10 @@ def register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None: :meth:`DestructuringMixin._intern_rec`. Entries are appended after the built-in ones; first match wins, so registering a handler for an entirely new container type is safe without displacing list/dict/dataclass/attrs. - Call before any :class:`DestructuringMixin` instance is used. + The built-in predicates match exact types only, so a handler for a subclass + (e.g. ``defaultdict`` with a picklable factory) can also be registered + without conflict. Call before any :class:`DestructuringMixin` instance is + used. """ _DESTRUCTURERS.append((pred, fn)) @@ -326,7 +400,7 @@ def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: match raw: case DigestedIterable(): return {i for i in raw.items if isinstance(i, digest.Digest)} - case DigestedDict(): + case DigestedMapping(): return { x for pair in raw.items.items() @@ -336,7 +410,8 @@ def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: case DigestedFields(): return {v for v in raw.fields.values() if isinstance(v, digest.Digest)} case _: - return set() + # Not one of ours — a layer below may still claim it. + return super()._raw_sub_digests(raw) def child_digests(self, key: digest.Digest | str) -> set[digest.Digest]: """Direct digest children of the raw entry stored at *key*. @@ -349,13 +424,13 @@ def child_digests(self, key: digest.Digest | str) -> set[digest.Digest]: Raises: KeyError: if *key* is not present in the underlying backend. """ - return self._raw_sub_digests(super().load(key)) + return self._raw_sub_digests(self.load_raw(key)) def count_reuses(self) -> Counter[digest.Digest]: """Return a counter of how many times each stored key is referenced as a sub-component. Scans every raw entry and tallies ``Digest`` back-references found inside - :class:`~fleche.storage.destructuring.DigestedIterable` and :class:`~fleche.storage.destructuring.DigestedDict` wrappers. A count of ``0`` + :class:`~fleche.storage.destructuring.DigestedIterable` and :class:`~fleche.storage.destructuring.DigestedMapping` wrappers. A count of ``0`` means the key is not pointed to by any other stored value (i.e. a top-level entry). A count greater than ``1`` indicates a sub-value shared between multiple parent containers. @@ -376,7 +451,7 @@ def count_reuses(self) -> Counter[digest.Digest]: """ counts: Counter[digest.Digest] = Counter({key: 0 for key in self.list()}) for key in list(counts): - for sub in self._raw_sub_digests(super().load(key)): + for sub in self._raw_sub_digests(self.load_raw(key)): if sub in counts: counts[sub] += 1 return counts diff --git a/src/fleche/storage/memory.py b/src/fleche/storage/memory.py index 67d738ed..e8e7f1bf 100644 --- a/src/fleche/storage/memory.py +++ b/src/fleche/storage/memory.py @@ -3,6 +3,7 @@ from .base import ValueMixin, CallMixin, StorageBackend, register_storage from .destructuring import DestructuringMixin +from .paths import PathValueMixin from .thread_safe import PerKeyLockMixin from ..digest import Digest from copy import deepcopy @@ -59,7 +60,7 @@ def from_config(cls, **kwargs) -> "MemoryBackend": @dataclass(frozen=True) -class ValueMemory(PerKeyLockMixin, DestructuringMixin, ValueMixin, MemoryBackend): +class ValueMemory(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, MemoryBackend): __hash__ = object.__hash__ def to_config(self) -> dict[str, Any]: diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py new file mode 100644 index 00000000..c14a9f89 --- /dev/null +++ b/src/fleche/storage/paths.py @@ -0,0 +1,309 @@ +import dataclasses +import os +import sys +from collections.abc import Iterable, Mapping +from numbers import Number +from pathlib import Path +from typing import Any +import tempfile +import weakref + +from . import base +from .. import _attrs +from .. import digest + + +def find_path(value: Any) -> Path | None: + """Return a :class:`~pathlib.Path` :func:`~fleche.digest.digest` would read, if any. + + Stops at the first ``Path`` reachable inside *value*, returning ``None`` + if there is none. Which one comes back when there are several is + unspecified: this answers "is there one", and the path itself is there to + name in the resulting error. + + This is a *predicate helper*, not part of the storage protocol: it exists + so a caller that cannot honour path semantics (notably + :class:`fleche.remote.SshCache`, where a path's meaning does not survive + the hop to another filesystem) can detect the situation up front instead + of silently storing something else. Shared cycles are visited once. + + **The walk deliberately mirrors** :func:`~fleche.digest.digest`, **not + destructuring**, and that difference is the whole point. Destructuring + treats namedtuples, sets, and frozensets as opaque, but ``digest`` + recurses into all of them and *reads the file* — so a path hidden in one + still decides the key. Locally that is harmless, because the process + computing the digest is the one holding the file. Across a machine + boundary it is not: the far side would digest the same name against its + own filesystem, which is exactly the ``digest(x) == save_value(x)`` break + the caller is trying to prevent. Mirroring destructuring here would let + ``Bundle(path, 0.5)`` through and reintroduce it. + + Mirroring means mirroring ``digest``'s *containers*, not a list of them: + the iterable arm below walks anything iterable, as ``digest`` does, and + skips only what ``digest`` itself never looks inside. An allowlist of + concrete types is the same bug in slower motion — it covers ``list`` and + ``tuple`` and lets a ``deque`` through. + """ + seen: set[int] = set() + stack: list[Any] = [value] + while stack: + item = stack.pop() + if isinstance(item, Path): + return item + if isinstance(item, (digest.Digest, Number, str, bytes, bytearray)): + continue + if id(item) in seen: + continue + seen.add(id(item)) + if isinstance(item, Mapping): + stack.extend(item.keys()) + stack.extend(item.values()) + elif dataclasses.is_dataclass(item) and not isinstance(item, type): + stack.extend(getattr(item, f.name) for f in dataclasses.fields(item)) + elif _attrs.is_attrs_instance(item): + stack.extend(v for _, v in _attrs.field_items(item)) + elif isinstance(item, Iterable): + # ``digest``'s ``Iterable`` arm walks *any* iterable, so this one + # has to as well: a path in a ``deque`` decides the key exactly as + # much as one in a list, and an allowlist of concrete types silently + # stops covering whatever a caller reaches for next. Three + # exclusions, each mirroring something ``digest`` does earlier: + # + # * :data:`~fleche.digest.OPAQUE_ITERABLES` — matched above the + # ``Iterable`` arm, hashing their own buffer without looking at + # elements, so no path inside one can reach a digest. + # * ``range`` — its elements are ``int`` by construction, and + # materializing ``range(10**9)`` onto the stack to learn that + # would be a denial of service. + # * one-shot iterators, which are their own ``__iter__``. Walking + # a generator consumes it; ``digest`` has already done so by the + # time a value could ship, so there is nothing left in one for us + # to find anyway. + if isinstance(item, (digest.OPAQUE_ITERABLES, range)): + continue + try: + if iter(item) is item: + continue + except TypeError: + continue + stack.extend(item) + return None + + +class TempPath(type(Path())): # ty: ignore[unsupported-base] + """ + A Path that deletes its backing temp tree when no references remain. + Paths derived via /, .parent, .with_suffix, etc. share the same + TemporaryDirectory and keep it alive collectively. + + On 3.12+ propagation rides ``with_segments``, the single hook pathlib + routes every derived path through. 3.11 has no such instance hook — its + derivation goes through the *classmethods* ``_from_parsed_parts`` / + ``_from_parts``, which never see the originating instance — so there the + class keeps a weak registry of live temp roots and re-attaches the + matching ``TemporaryDirectory`` to any path constructed under one. The + registry holds only weak references: instances alone keep a root alive, + so cleanup semantics are identical on both versions. + """ + + # str(root dir) -> TemporaryDirectory, weakly; used by the 3.11 branch only. + _live_roots: "weakref.WeakValueDictionary[str, tempfile.TemporaryDirectory]" = ( + weakref.WeakValueDictionary() + ) + + @classmethod + def mkdtemp( + cls, + ) -> "TempPath": + root = tempfile.TemporaryDirectory( + suffix="fleche", + ignore_cleanup_errors=True, + ) + obj = cls(root.name) + object.__setattr__(obj, "_temp_root", root) + if sys.version_info < (3, 12): + cls._live_roots[str(obj)] = root + return obj + + if sys.version_info >= (3, 12): + + def with_segments(self, *pathsegments): + new = super().with_segments(*pathsegments) + root = getattr(self, "_temp_root", None) + if root is not None: + object.__setattr__(new, "_temp_root", root) + return new + + else: + + @classmethod + def _adopt_live_root(cls, new: "TempPath") -> "TempPath": + path_str = str(new) + for root_str, root in list(cls._live_roots.items()): + if path_str == root_str or path_str.startswith(root_str + os.sep): + object.__setattr__(new, "_temp_root", root) + break + return new + + @classmethod + def _from_parsed_parts(cls, drv, root, parts): + return cls._adopt_live_root(super()._from_parsed_parts(drv, root, parts)) + + @classmethod + def _from_parts(cls, args): + return cls._adopt_live_root(super()._from_parts(args)) + + +class FileBlob: + """A stored file: a basename paired with a reference to its content. + + A file is identified by *(name, content)*. The content ``bytes`` are stored + once under their own content digest — so identical bodies deduplicate across + names, and even with plain ``bytes`` values — and this small record pairs + that content reference with the file's basename. On load it materializes at + ``/`` so ``.name`` / ``.suffix`` / ``.stem`` are faithful, and + a downstream consumer receives an ordinary ``Path`` with the right name. + + To store file content *without* a name (content only, maximal reuse), return + the plain ``bytes`` instead of a ``Path``. + + Plain ``__dict__`` (no ``__slots__``) so H5 can reconstruct it; the + ``"FileBlob"`` salt in :meth:`__digest__` MUST match ``fleche.digest``'s file + ``Path`` arm. + """ + + def __init__(self, name, content): + self.name = name + self.content = content # Digest of the file's content bytes + + def __eq__(self, other): + return isinstance(other, FileBlob) and (self.name, self.content) == ( + other.name, + other.content, + ) + + __hash__ = None # mutable; not intended as a dict key + + def __digest__(self): + return digest.digest(("FileBlob", self.name, self.content)) + + def __repr__(self): + return f"FileBlob({self.name!r}, {self.content!r})" + + +class DirectoryBlob: + """A stored directory: ``{name: content_ref}``, keyed by its tree alone. + + A directory's *root* name is **not** part of its identity — directories hash + by their content (the tree) — but its child names are (they are the dict + keys). Each child is a content reference: a file child to its content + ``bytes``, a subdirectory child to its own :class:`DirectoryBlob`. A + reloaded directory is therefore named by its digest, its children by their + real names. + + Stored verbatim by :class:`~fleche.storage.destructuring.DestructuringMixin` + (not a dict subclass, not a dataclass, so no match arm catches it). + Kept as a plain ``__dict__``-backed class (no ``__slots__``): some backends + reconstruct via ``obj.__dict__.update(state)`` (e.g. bagofholding's H5 + unpacker), which a slots-only object cannot satisfy. + """ + + def __init__(self, contents): + self.contents = dict(contents) + + def __eq__(self, other): + return isinstance(other, DirectoryBlob) and self.contents == other.contents + + __hash__ = None # mutable; not intended as a dict key + + def __digest__(self): + # Digest a (type_name, payload) tuple — the codebase idiom for custom + # digests. The "DirectoryBlob" element salts the hash so this is not + # digest-equal to a plain dict carrying the same {name: Digest} mapping. + return digest.digest(("DirectoryBlob", self.contents)) + + def __repr__(self): + return f"DirectoryBlob({self.contents!r})" + + +class PathValueMixin(base.ValueStorage): + """Convert :class:`~pathlib.Path` values to blobs (and back). + + A **file** is stored as its content ``bytes`` (deduplicated under the content + digest) wrapped in a :class:`FileBlob` carrying the basename — so files are + keyed by *(name, content)* and a cache hit returns a path with its real name. + A **directory** is stored as a :class:`DirectoryBlob` keyed by its tree only; + its root name is dropped (a reloaded directory is named by its digest, its + children by their real names). Plain ``bytes`` are the way to store file + content without a name. + + The traversal never goes through ``self.save`` / ``self.load`` — every + storage call uses ``super()`` — so this mixin composes cleanly with + :class:`~fleche.storage.destructuring.DestructuringMixin` above it without + load-context ambiguity. Compose **below** ``DestructuringMixin`` in the MRO + so ``super().save`` from Destructure's recursion lands here for nested Paths. + """ + + def save(self, value: Any, key: digest.Digest | None = None) -> digest.Digest: + if isinstance(value, Path): + if value.is_file(): + # Content deduplicates as plain bytes; the FileBlob record adds + # the name (the file's key is digest(name, content)). + content = super().save(value.read_bytes()) + return super().save(FileBlob(value.name, content), key) + if value.is_dir(): + return super().save(self._build(value), key) + return super().save(value, key) + + def _build(self, p: Path) -> DirectoryBlob: + contents = {} + for child in sorted(p.iterdir()): + if child.is_file(): + contents[child.name] = super().save(child.read_bytes()) + elif child.is_dir(): + contents[child.name] = super().save(self._build(child)) + return DirectoryBlob(contents) + + def load(self, key: digest.Digest | str) -> Any: + value = super().load(key) + if isinstance(value, FileBlob): + # Materialize at / so the basename round-trips. + target = TempPath.mkdtemp() / value.name + target.write_bytes(super().load(value.content)) + return target + if isinstance(value, DirectoryBlob): + # No stored root name — materialize under the digest (mangled root). + path = TempPath.mkdtemp() / str(key) + self._materialize(path, value) + return path + return value + + def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: + """Report the content blobs a stored path record points at. + + Without this a reachability walk sees a :class:`FileBlob` / + :class:`DirectoryBlob` as a childless leaf, so the ``bytes`` holding + the actual file content look unreferenced and ``gc`` reclaims them — + destroying every path-valued entry it sweeps past. The blobs are + precisely a *name plus references*, so the references have to be + declared here, at the layer that creates them. + """ + if isinstance(raw, FileBlob): + return {raw.content} + if isinstance(raw, DirectoryBlob): + return set(raw.contents.values()) + return super()._raw_sub_digests(raw) + + def _materialize(self, path: Path, blob: DirectoryBlob) -> None: + path.mkdir() + for name, child_ref in blob.contents.items(): + child = super().load(child_ref) + if isinstance(child, DirectoryBlob): + self._materialize(path / name, child) + elif isinstance(child, (bytes, bytearray)): + (path / name).write_bytes(child) + else: + raise TypeError( + f"directory entry {name!r} resolved to unexpected type " + f"{type(child).__name__}" + ) diff --git a/src/fleche/storage/pickle_file.py b/src/fleche/storage/pickle_file.py index a537015a..320532bd 100644 --- a/src/fleche/storage/pickle_file.py +++ b/src/fleche/storage/pickle_file.py @@ -8,6 +8,7 @@ from .base import ValueMixin, CallMixin, register_storage from .thread_safe import PerKeyLockMixin from .destructuring import DestructuringMixin +from .paths import PathValueMixin from ..security import get_secret_key, normalize_secret_key, SignedBytes, SignatureError from pyiron_snippets.import_alarm import ImportAlarm @@ -112,7 +113,7 @@ def decompress_all(self) -> None: @dataclass(frozen=True) -class ValuePickleFile(PerKeyLockMixin, DestructuringMixin, ValueMixin, PickleFileBackend): +class ValuePickleFile(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, PickleFileBackend): def to_config(self) -> dict[str, Any]: # `dumps`/`loads` are not config data: the `type` name is what selects # them again on the way back in, and it comes off the instance rather diff --git a/src/fleche/storage/void.py b/src/fleche/storage/void.py index 881414b2..1c63f2c3 100644 --- a/src/fleche/storage/void.py +++ b/src/fleche/storage/void.py @@ -27,6 +27,10 @@ def _contains(self, key: Digest) -> bool: return False +# ValueVoid is the degenerate "store nothing" value storage: it deliberately omits +# DestructuringMixin and, for the same reason, PathValueMixin. Since VoidBackend +# discards on put and raises KeyError on get, layering Path-to-blob conversion would +# only build-then-throw-away blobs and could never materialize a Path on load. @register_storage("void", kind="value") @dataclass(frozen=True) class ValueVoid(ValueMixin, VoidBackend): diff --git a/tests/integration/test_path_mutation.py b/tests/integration/test_path_mutation.py new file mode 100644 index 00000000..cec08e53 --- /dev/null +++ b/tests/integration/test_path_mutation.py @@ -0,0 +1,69 @@ +"""Path arguments are keyed as passed (two-phase save + content addressing). + +A function that writes into a directory it received used to be recorded under +the post-mutation tree and could never hit; with the two-phase save protocol +the identity is sealed before the body runs. See +docs/usage/file_semantics.rst, "Argument mutation". +""" +from pathlib import Path + +import fleche as fl +from fleche import fleche + + +def make_input(root: Path, name: str) -> Path: + d = root / name / "data" + d.mkdir(parents=True) + (d / "in.txt").write_text("same content") + return d + + +def test_mutating_path_consumer_hits(tmp_path): + with fl.cache("memory"): + runs = [] + + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return sorted(p.name for p in d.iterdir()) + + assert consume(make_input(tmp_path, "a")) == ["in.txt", "out.txt"] + d2 = make_input(tmp_path, "b") # identical tree, as passed + assert consume(d2) == ["in.txt", "out.txt"] + assert len(runs) == 1 # keyed on the pre-call tree: hit + assert not (d2 / "out.txt").exists() # the mutation is not replayed + + +def test_mutated_state_is_a_different_call(tmp_path): + with fl.cache("memory"): + runs = [] + + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return len(list(d.iterdir())) + + d1 = make_input(tmp_path, "a") + consume(d1) # cold; d1 now holds in+out + consume(d1) # post-mutation tree: honest miss, + assert len(runs) == 2 # not a false hit on the old record + + +def test_returned_mutated_argument_captured_in_final_state(tmp_path): + with fl.cache("memory"): + runs = [] + + @fleche + def stamp(d: Path) -> Path: + runs.append(1) + (d / "stamp.txt").write_text("stamped") + return d + + stamp(make_input(tmp_path, "a")) + warm = stamp(make_input(tmp_path, "b")) # hit, keyed on the input as passed + assert len(runs) == 1 + # ... but the result was captured at commit time: final, stamped state. + assert sorted(p.name for p in warm.iterdir()) == ["in.txt", "stamp.txt"] + assert (warm / "stamp.txt").read_text() == "stamped" diff --git a/tests/integration/test_paths_workflow.py b/tests/integration/test_paths_workflow.py new file mode 100644 index 00000000..a36f1118 --- /dev/null +++ b/tests/integration/test_paths_workflow.py @@ -0,0 +1,151 @@ +"""End-to-end workflow tests for Path-by-content caching. + +These mirror the canonical "functions that accept and produce files and +directories via paths" scenario (see ``notebooks/Files.ipynb``) but assert +*correctness* rather than merely "runs without error": cache hits are proven by +counting body executions, and content-addressed deduplication is proven by +inspecting the value store. + +Parametrized over an in-memory and an on-disk (pickle) cache so the opaque +``FileBlob`` / ``DirectoryBlob`` records are exercised through real +serialization, not just deepcopy. +""" + +from pathlib import Path + +import pytest + +from fleche import fleche, cache +from fleche.digest import digest +from fleche.caches import Cache +from fleche.storage import ( + ValueMemory, + CallMemory, + ValuePickleFile, + CallPickleFile, +) + + +@pytest.fixture(params=["memory", "cloudpickle"]) +def paths_cache(request, tmp_path): + if request.param == "memory": + return Cache(ValueMemory({}), CallMemory({})) + return Cache( + ValuePickleFile.with_cloudpickle(tmp_path / "values"), + CallPickleFile.with_cloudpickle(tmp_path / "calls"), + ) + + +def _relmap(root: Path) -> dict[str, bytes]: + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +def test_file_producer_runs_once_and_roundtrips(paths_cache, tmp_path): + """A cached file producer executes once; the cached result is a readable Path.""" + runs = [] + + @fleche + def produce(content): + runs.append(content) + f = tmp_path / f"out-{content}.txt" + f.write_text(content) + return f + + with cache(paths_cache): + first = produce("payload") + assert isinstance(first, Path) + assert first.read_text() == "payload" + + second = produce("payload") + assert isinstance(second, Path) + assert second.read_text() == "payload" + + assert runs == ["payload"], "second call should have been served from cache" + + +def test_directory_producer_roundtrips_tree(paths_cache, tmp_path): + """A cached directory producer round-trips a full nested tree by content.""" + runs = [] + + @fleche + def build(seed): + runs.append(seed) + d = tmp_path / f"tree-{seed}" + d.mkdir() + (d / "top.txt").write_text(seed) + (d / "sub").mkdir() + (d / "sub" / "leaf.bin").write_bytes(seed.encode() * 3) + return d + + expected = { + "top.txt": b"seed", + "sub/leaf.bin": b"seedseedseed", + } + + with cache(paths_cache): + first = build("seed") + assert _relmap(first) == expected + + second = build("seed") + assert second.is_dir() + assert _relmap(second) == expected + + assert runs == ["seed"] + + +def test_producer_consumer_chain_caches(paths_cache, tmp_path): + """A Path flowing producer -> consumer: both stages cache on repeat input.""" + writes, parses = [], [] + + @fleche + def write(content): + writes.append(content) + f = tmp_path / f"w-{content}.txt" + f.write_text(content) + return f + + @fleche + def parse(f: Path): + parses.append(f.name) + return f.read_text().upper() + + with cache(paths_cache): + assert parse(write("hi")) == "HI" + # write hits cache (same content) and so does parse (same file content), + # even though the second write returns a freshly materialized TempPath. + assert parse(write("hi")) == "HI" + + assert writes == ["hi"] + assert parses == ["w-hi.txt"] + + +def test_shared_file_body_is_deduplicated(paths_cache, tmp_path): + """A file body shared by two produced directories is reused, not re-stored.""" + shared = b"shared-body" + one, two = b"one-only", b"two-only" + + @fleche + def build(tag, unique): + d = tmp_path / f"d-{tag}" + d.mkdir() + (d / "shared").write_bytes(shared) + (d / "unique").write_bytes(unique) + return d + + with cache(paths_cache): + build("a", one) + keys_after_a = set(paths_cache.values.list()) + assert digest(shared) in keys_after_a, "shared body stored when first dir is produced" + + build("b", two) + keys_after_b = set(paths_cache.values.list()) + + new_keys = keys_after_b - keys_after_a + # The second directory's genuinely new content shows up... + assert digest(two) in new_keys + # ...but the shared body is content-addressed and reused, not re-stored. + assert digest(shared) not in new_keys diff --git a/tests/integration/test_remote.py b/tests/integration/test_remote.py index 1c2f136b..e70c0ed1 100644 --- a/tests/integration/test_remote.py +++ b/tests/integration/test_remote.py @@ -6,15 +6,19 @@ parsing on the server side, and the ``Popen`` handshake on the client side. """ +import logging import os import subprocess import sys import textwrap +from pathlib import Path import pytest +from fleche import cache, fleche from fleche.call import Call -from fleche.remote import SshCache, _Connection +from fleche.digest import digest +from fleche.remote import RemotePathUnsupported, SshCache, _Connection class _LocalSubprocessConnection(_Connection): @@ -165,3 +169,67 @@ def test_subprocess_named_cache(tmp_path): finally: sc_main.close() sc_alt.close() + + +# --------------------------------------------------------------------------- +# Paths stop at the SSH boundary +# --------------------------------------------------------------------------- + + +def test_relative_path_would_resolve_against_the_servers_own_cwd( + tmp_path, monkeypatch +): + """The divergence the path guard exists for, reproduced in one process. + + Client and server run in different working directories, so the same + relative name denotes a *different file* on each side — the single-machine + stand-in for "the remote is another filesystem". Only the path string + crosses the wire, so without the guard the server would digest and store + its own file under a key the client can never reproduce, and a later load + would hand back the wrong bytes. + """ + remote_root = tmp_path / "remote" + remote_root.mkdir() + client_root = tmp_path / "client" + client_root.mkdir() + (client_root / "data.txt").write_text("client bytes") + (remote_root / "data.txt").write_text("SERVER BYTES - a different file entirely") + # The two sides genuinely disagree about what "data.txt" contains. + assert digest(client_root / "data.txt") != digest(remote_root / "data.txt") + + monkeypatch.chdir(client_root) + sc = _build_remote(remote_root) + try: + with pytest.raises(RemotePathUnsupported): + sc.save_value(Path("data.txt")) + finally: + sc.close() + + +def test_path_returning_function_runs_uncached_against_a_remote(tmp_path, caplog): + """A rejected result must not break the call — it just isn't cached. + + ``Rejected`` is the cache's "I won't keep this" signal, which the + decorator logs and swallows, so the user still gets their file back. + """ + remote_root = tmp_path / "remote" + remote_root.mkdir() + sc = _build_remote(remote_root) + runs = [] + + @fleche + def produce(name): + runs.append(name) + f = tmp_path / f"{name}.txt" + f.write_text(name) + return f + + try: + with cache(sc), caplog.at_level(logging.WARNING): + assert produce("out").read_text() == "out" + assert produce("out").read_text() == "out" + finally: + sc.close() + + assert runs == ["out", "out"] # never cached, never wrong + assert any("rejected save" in r.message.lower() for r in caplog.records) diff --git a/tests/unit/caches/test_gc.py b/tests/unit/caches/test_gc.py index 2da9d424..c5729a78 100644 --- a/tests/unit/caches/test_gc.py +++ b/tests/unit/caches/test_gc.py @@ -2,7 +2,8 @@ import pytest -from fleche.call import Call +from fleche import call +from fleche.call import Call, PreparedCall from fleche.caches import Cache from fleche.digest import digest from fleche.storage.base import ValueMixin @@ -144,3 +145,303 @@ def test_gc_evicts_deeply_unreachable_structure(split_cache): assert orphan_leaf_only in evicted # Previously-live keys still present. assert reachable_before.issubset(set(split_cache.values.list())) + + +# ---- Path values: blob records reference their content, and gc must see it ---- + + +@pytest.fixture +def path_cache(): + return Cache(values=ValueMemory({}), calls=CallMemory({})) + + +@pytest.fixture +def a_file(tmp_path): + f = tmp_path / "data.txt" + f.write_text("file content") + return f + + +@pytest.fixture +def a_tree(tmp_path): + d = tmp_path / "tree" + (d / "sub").mkdir(parents=True) + (d / "top.txt").write_text("top") + (d / "sub" / "leaf.bin").write_bytes(b"leaf") + return d + + +@pytest.mark.parametrize("which", ["file", "tree", "both-nested"]) +def test_gc_keeps_the_content_behind_a_stored_path(path_cache, a_file, a_tree, which): + """A stored path is a name plus a *reference*; gc must follow the reference. + + The blob records carry the digests of the content ``bytes``. If the walk + cannot see them the content looks unreferenced, gc reclaims it, and the + entry is destroyed — the load then raises ``KeyError``, which the wrapper + reports as an ordinary cache miss, so the loss is silent. + """ + result = {"file": a_file, "dir": a_tree} if which == "both-nested" else ( + a_file if which == "file" else a_tree + ) + call = Call(name="f", arguments={"x": 1}, result=result) + key = path_cache.save(call) + + assert path_cache.gc() == set() + + loaded = path_cache.load(key).result + if which == "file": + assert loaded.read_text() == "file content" + elif which == "tree": + assert (loaded / "top.txt").read_text() == "top" + assert (loaded / "sub" / "leaf.bin").read_bytes() == b"leaf" + else: + assert loaded["file"].read_text() == "file content" + assert (loaded["dir"] / "sub" / "leaf.bin").read_bytes() == b"leaf" + + +def test_gc_still_evicts_an_orphaned_path(path_cache, a_file): + """The fix must not make path content unconditionally reachable.""" + orphan = path_cache.values.save(a_file) + assert orphan in path_cache.gc() + + +def test_child_digests_reports_a_blob_s_content(path_cache, a_file, a_tree): + """The blob layer declares its own references, unmended.""" + file_key = path_cache.values.save(a_file) + assert path_cache.values.child_digests(file_key), "FileBlob reported no children" + + tree_key = path_cache.values.save(a_tree) + children = path_cache.values.child_digests(tree_key) + assert len(children) == 2, "DirectoryBlob should reference both of its entries" + + +def test_reachability_walk_does_not_materialize_paths(path_cache, a_file, monkeypatch): + """Inspecting the graph must not build temp trees for every stored path. + + ``child_digests`` reads through ``load_raw``, so the path layer's + materialization is skipped: gc over a large cache would otherwise copy + every stored file to disk just to ask what it points at. + """ + from fleche.storage import paths as paths_mod + + path_cache.save(Call(name="f", arguments={"x": 1}, result=a_file)) + + calls = [] + real = paths_mod.TempPath.mkdtemp + monkeypatch.setattr( + paths_mod.TempPath, "mkdtemp", + classmethod(lambda cls: (calls.append(1), real())[1]), + ) + path_cache.gc() + path_cache.values.count_reuses() + assert calls == [], f"walk materialized {len(calls)} temp tree(s)" + + +# ---- Calls in flight: prepared arguments are roots until commit or abandon ---- + + +def test_gc_keeps_arguments_of_a_call_in_flight(gc_cache): + """A sweep during the function body must not evict the sealed arguments. + + ``prepare`` stores the arguments before the body runs, precisely so the + record cannot be keyed on post-mutation content — but for the whole + duration of the body no record references them yet. Without the in-flight + roots ``gc`` reclaims them as orphans and the eventual ``commit`` files a + record whose arguments are dangling digests. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + + assert gc_cache.gc() == set() # body "runs" here + + key = prepared.commit("a result") + assert gc_cache.load(key).arguments["x"] == "an argument" + + +def test_gc_evicts_arguments_of_an_abandoned_call(gc_cache): + """Abandoning releases the roots: the arguments are orphans again.""" + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + arg_key = digest("an argument") + assert arg_key in gc_cache.values.list() + + prepared.abandon() + + assert arg_key in gc_cache.gc() + + +def test_gc_evicts_arguments_after_commit_returns(gc_cache): + """Committing hands the roots over to the record, and holds nothing else. + + Once the record references them the registry must let go, or a long-lived + ``PreparedCall`` object would pin values forever. Evicting the record and + sweeping is the observable check. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + key = prepared.commit("a result") + gc_cache.evict(key) + + assert digest("an argument") in gc_cache.gc() + + +def test_gc_keeps_nested_arguments_of_a_call_in_flight(split_cache): + """In-flight roots seed the transitive walk, not just the top-level keys.""" + prepared = split_cache.prepare(Call(name="f", arguments={"x": [[1, 2], [3, 4]]})) + + assert split_cache.gc() == set() + + key = prepared.commit(None) + assert split_cache.load(key).arguments["x"] == [[1, 2], [3, 4]] + + +def test_gc_keeps_arguments_of_a_call_in_flight_through_a_wrapper(gc_cache): + """Wrapper rebinding must not drop the registration. + + ``CacheWrapper.prepare`` returns ``replace(inner.prepare(call), cache=self)`` + — a *new* object — so registering in ``prepare`` alone would leave only the + inner one registered, and that one is garbage the moment ``replace`` + returns. + """ + import gc as _gc + + from fleche.caches import RefreshingCache + + wrapper = RefreshingCache(gc_cache) + prepared = wrapper.prepare(Call(name="f", arguments={"x": "an argument"})) + _gc.collect() # drop the inner PreparedCall replace() left behind + + assert gc_cache.gc() == set() + assert digest("an argument") in gc_cache.values.list() + prepared.abandon() + + +def test_in_flight_registry_does_not_leak_dropped_calls(gc_cache): + """The registry is weak: a prepared call nobody finished falls out.""" + import gc as _gc + + from fleche.call import in_flight_digests + + gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + _gc.collect() + + assert in_flight_digests() == set() + + +def test_gc_is_safe_against_concurrent_prepares(gc_cache): + """Sweeping while other threads admit calls must not raise. + + Scoped to what the sweep actually guarantees: the registry is a plain + mapping that ``gc`` reads, so a concurrent ``prepare`` must not resize it + mid-iteration, and neither side may blow up. Value fidelity under a + concurrent sweep is *not* asserted here — a narrow window remains between + storing a value and registering the call that owns it, pinned deterministically + by ``test_gc_may_evict_a_value_stored_but_not_yet_registered`` below. An + earlier version of this test asserted fidelity and was flaky on CI for + exactly that reason. + """ + import threading + + stop = threading.Event() + errors: list[BaseException] = [] + + def preparer(i): + try: + while not stop.is_set(): + prepared = gc_cache.prepare(Call(name="f", arguments={"x": f"arg-{i}"})) + prepared.commit(i) + except BaseException as e: # pragma: no cover - only on a real race + errors.append(e) + + threads = [threading.Thread(target=preparer, args=(i,)) for i in range(4)] + for t in threads: + t.start() + try: + for _ in range(200): + gc_cache.gc() + finally: + stop.set() + for t in threads: + t.join() + + assert not errors, errors + + +def test_gc_keeps_a_call_that_commits_mid_sweep(gc_cache, monkeypatch): + """A call finishing *between* two of the sweep's reads must survive. + + ``commit`` deregisters only after ``save`` has filed the record, so the + registry read and the record read overlap in time — reading the roots + first means a call that commits in the gap is caught by the roots. Reading + the records first (the original order) leaves it covered by neither, and + its arguments are swept out from under a record that references them. + Forced rather than raced: the commit is driven from inside the roots read. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + committed = {} + + # Drive the commit from the *record* read, so it lands after that read + # whichever order the sweep uses. With the roots read first (current + # order) the registration was still live when the roots were taken, so the + # value survives; with the records read first it is already deregistered by + # the time the roots are taken, and nothing covers it. + # Patched on the class: the storages are frozen dataclasses, so the hook + # cannot go on the instance. + storage_cls = type(gc_cache.calls) + real_list = storage_cls.list + + def list_then_commit(self): + records = list(real_list(self)) + if "key" not in committed: + committed["key"] = prepared.commit("a result") + return records + + monkeypatch.setattr(storage_cls, "list", list_then_commit) + gc_cache.gc() + monkeypatch.undo() + + assert gc_cache.load(committed["key"]).arguments["x"] == "an argument" + + +def test_gc_may_evict_a_value_stored_but_not_yet_registered(gc_cache, monkeypatch): + """The one window the sweep does not close, pinned so it stays known. + + ``prepare`` stores the argument values and only then registers the call as + a gc root. A sweep that reads its candidates and roots inside that gap + sees the value but not its owner, and reclaims it; the later commit files a + record with a dangling reference. This is the same window the one-shot + ``save`` has always had between storing values and filing the record — + bounded by a storage write rather than by a function body — and closing it + would need a lock shared by every writer. + + Asserted so the limit is documented behaviour rather than folklore: if + someone closes it, this test should fail and be deleted. + """ + import threading + + started = threading.Event() + release = threading.Event() + real_post_init = PreparedCall.__post_init__ + + def stall_before_registering(self): + # Values are already stored; registration has not happened yet. + started.set() + release.wait(5) + real_post_init(self) + + monkeypatch.setattr(PreparedCall, "__post_init__", stall_before_registering) + + out = {} + + def worker(): + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + out["key"] = prepared.commit("a result") + + t = threading.Thread(target=worker) + t.start() + try: + assert started.wait(5) + evicted = gc_cache.gc() + finally: + release.set() + t.join() + + assert digest("an argument") in evicted + assert gc_cache.load(out["key"]).arguments["x"] == digest("an argument") diff --git a/tests/unit/digest/test_digest.py b/tests/unit/digest/test_digest.py index 4de054d2..2b80fc76 100644 --- a/tests/unit/digest/test_digest.py +++ b/tests/unit/digest/test_digest.py @@ -1,6 +1,8 @@ import cmath import datetime import struct +import subprocess +import sys import collections import collections.abc import types as types_module @@ -945,3 +947,65 @@ def test_non_builtin_type_raises_indigestible(t): """Non-builtin type objects (user-defined classes, dataclass classes) raise Indigestible.""" with pytest.raises(Indigestible): digest(t) + + +# --------------------------------------------------------------------------- +# subprocess.CompletedProcess +# --------------------------------------------------------------------------- + + +def _cp(args=("echo", "hi"), returncode=0, stdout=b"hi\n", stderr=b""): + return subprocess.CompletedProcess( + args=list(args), returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def test_completedprocess_hashes_by_its_fields(): + """Wrapping shell tools is a first-class use case; `run()`'s result must hash.""" + assert digest(_cp()) == digest(_cp()) + + +@pytest.mark.parametrize( + "changed", + [ + pytest.param({"args": ("echo", "bye")}, id="args"), + pytest.param({"returncode": 1}, id="returncode"), + pytest.param({"stdout": b"other\n"}, id="stdout"), + pytest.param({"stderr": b"boom\n"}, id="stderr"), + ], +) +def test_completedprocess_distinguishes_each_field(changed): + assert digest(_cp()) != digest(_cp(**changed)) + + +def test_completedprocess_is_not_digest_equal_to_its_field_tuple(): + """The type name salts the hash, as for every other arm.""" + cp = _cp() + assert digest(cp) != digest((cp.args, cp.returncode, cp.stdout, cp.stderr)) + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({"stdout": "hi\n", "stderr": ""}, id="text-mode-str"), + pytest.param({"stdout": None, "stderr": None}, id="streams-not-captured"), + ], +) +def test_completedprocess_handles_uncaptured_and_text_streams(kwargs): + """`capture_output=False` leaves None; `text=True` leaves str.""" + assert digest(_cp(**kwargs)) == digest(_cp(**kwargs)) + + +def test_completedprocess_from_a_real_run(): + a = subprocess.run([sys.executable, "-c", "print('x')"], capture_output=True) + b = subprocess.run([sys.executable, "-c", "print('x')"], capture_output=True) + c = subprocess.run([sys.executable, "-c", "print('y')"], capture_output=True) + assert digest(a) == digest(b) + assert digest(a) != digest(c) + + +def test_completedprocess_nested_in_a_result(): + """The notebook's shape: a function returning (path-ish, CompletedProcess).""" + cp = _cp() + assert digest(("workdir", cp)) == digest(("workdir", _cp())) + assert digest({"ret": cp}) != digest({"ret": _cp(returncode=2)}) diff --git a/tests/unit/digest/test_digest_paths.py b/tests/unit/digest/test_digest_paths.py new file mode 100644 index 00000000..714eab63 --- /dev/null +++ b/tests/unit/digest/test_digest_paths.py @@ -0,0 +1,267 @@ +"""Coverage for the ``Path`` arm in ``fleche.digest._digest_bytes``. + +Files digest on ``(basename, content)``; directories digest on their tree alone +(a directory's own root name is not part of its identity). +""" +from pathlib import Path + +import pytest + +from fleche.digest import digest, Indigestible + + +# ---- Non-existent paths ---- + + +def test_nonexistent_path_raises_indigestible(tmp_path): + """A Path that doesn't exist on disk cannot be digested.""" + missing = tmp_path / "does-not-exist" + with pytest.raises(Indigestible): + digest(missing) + + +# ---- File paths ---- + + +def test_file_path_digest_is_name_plus_content(tmp_path): + """A file Path digests on (basename, content), mirroring the stored FileBlob record. + + ``digest(path) == digest(FileBlob(name, digest(bytes)))`` — the + ``digest(path) == values.save(path)`` invariant cached lookups rely on — and + ``!= digest(bytes)`` so a file is distinct from a bare bytes value of the + same content (while still deduplicating its content blob; see storage tests). + """ + from fleche.storage.paths import FileBlob + + p = tmp_path / "f.txt" + p.write_bytes(b"hello world") + assert digest(p) == digest(FileBlob("f.txt", digest(b"hello world"))) + assert digest(p) != digest(b"hello world") + + +def test_file_path_digest_changes_with_contents(tmp_path): + """Different file bodies → different digests.""" + p = tmp_path / "f.txt" + p.write_bytes(b"alpha") + d1 = digest(p) + p.write_bytes(b"beta") + d2 = digest(p) + assert d1 != d2 + + +def test_file_path_digest_depends_on_filename(tmp_path): + """Files are keyed on (name, content): same body, different names → different digests. + + Same basename + same content → identical, regardless of the parent directory. + """ + a = tmp_path / "a.txt" + b = tmp_path / "b.txt" + a.write_bytes(b"same") + b.write_bytes(b"same") + assert digest(a) != digest(b) # name is part of a file's identity + + other = tmp_path / "sub" + other.mkdir() + c = other / "a.txt" + c.write_bytes(b"same") + assert digest(c) == digest(a) # same basename + content -> same digest + + +# ---- Directory paths ---- + + +def test_empty_directory_can_be_digested(tmp_path): + """An empty directory is digestible (does not raise).""" + digest(tmp_path) # smoke + + +def test_directory_digest_ignores_root_name(tmp_path): + """A directory hashes by its tree alone — its own root name does not matter.""" + d1 = tmp_path / "alpha" + d2 = tmp_path / "beta" + for d in (d1, d2): + d.mkdir() + (d / "x.txt").write_bytes(b"X") + (d / "sub").mkdir() + (d / "sub" / "y.bin").write_bytes(b"Y") + assert digest(d1) == digest(d2) # different root names, identical trees + + +def test_directory_digest_changes_with_filename(tmp_path): + """The directory arm hashes ``{name: child}``, so renaming a file changes the digest.""" + (tmp_path / "a.txt").write_bytes(b"x") + d1 = digest(tmp_path) + + # Rename and re-hash. + (tmp_path / "a.txt").rename(tmp_path / "b.txt") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_changes_with_file_contents(tmp_path): + """Mutating a child file changes the directory digest.""" + f = tmp_path / "f.txt" + f.write_bytes(b"v1") + d1 = digest(tmp_path) + f.write_bytes(b"v2") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_stable_across_iteration_order(tmp_path): + """Two directories with the same {name: bytes} payload share a digest. + + ``_digest_mapping`` sorts by key-digest, so filesystem ``iterdir`` order + must not leak into the result. + """ + d1 = tmp_path / "d1" + d2 = tmp_path / "d2" + d1.mkdir() + d2.mkdir() + # Write children in different orders. + (d1 / "a").write_bytes(b"A") + (d1 / "b").write_bytes(b"B") + (d2 / "b").write_bytes(b"B") + (d2 / "a").write_bytes(b"A") + assert digest(d1) == digest(d2) + + +def test_nested_directory_digest_recurses(tmp_path): + """Mutating a deeply nested file changes the root directory digest.""" + sub = tmp_path / "sub" / "deeper" + sub.mkdir(parents=True) + leaf = sub / "leaf.txt" + leaf.write_bytes(b"v1") + d1 = digest(tmp_path) + leaf.write_bytes(b"v2") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_differs_from_plain_dict(tmp_path): + """A directory digest does not collide with a plain dict of {name: path}. + + A directory hashes as the tuple ``("DirectoryBlob", {name: child_digest})``, + whereas a plain dict hashes as a bare mapping — different shapes and salts, + so they differ even though both ultimately reference the same child content. + """ + (tmp_path / "a.txt").write_bytes(b"hello") + plain = {"a.txt": tmp_path / "a.txt"} + assert digest(tmp_path) != digest(plain) + + +# ---- Non-file, non-directory paths ---- + + +def test_special_path_raises_indigestible(tmp_path): + """A path that exists but is neither a regular file nor a directory raises. + + GUESS: covers the ``else`` branch (e.g. symlinks to nowhere, sockets, + FIFOs). A dangling symlink is the most portable trigger: ``exists()`` + returns False for it (so we'd land in the not-exists branch first). Use a + FIFO instead — POSIX only, skip elsewhere. + """ + import os + fifo = tmp_path / "fifo" + try: + os.mkfifo(fifo) + except (AttributeError, NotImplementedError, OSError): + pytest.skip("FIFOs not supported on this platform") + with pytest.raises(Indigestible): + digest(fifo) + + +# ---- Unreadable paths degrade rather than escaping ---- +# +# A read can fail for reasons unrelated to the value's suitability — permissions, +# EIO, a network mount that vanished. Those must degrade to `Indigestible` like +# every other case in the arm, or the wrapper crashes the call before the body +# ever runs. Patched rather than chmod-ed so the tests are meaningful when the +# suite runs as root, where mode bits are not enforced. + + +def test_unreadable_file_degrades_to_indigestible(tmp_path, monkeypatch): + f = tmp_path / "secret.txt" + f.write_text("hidden") + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "read_bytes", boom) + with pytest.raises(Indigestible, match="Could not read"): + digest(f) + + +def test_unreadable_directory_degrades_to_indigestible(tmp_path, monkeypatch): + d = tmp_path / "locked" + d.mkdir() + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "iterdir", boom) + with pytest.raises(Indigestible, match="Could not read"): + digest(d) + + +def test_unreadable_child_degrades_the_whole_tree(tmp_path, monkeypatch): + """One unreadable file must not make the enclosing directory raise raw.""" + d = tmp_path / "tree" + d.mkdir() + (d / "ok.txt").write_text("fine") + (d / "bad.bin").write_bytes(b"x") + + real = Path.read_bytes + + def selective(self, *a, **kw): + if self.name == "bad.bin": + raise OSError(5, "Input/output error") + return real(self, *a, **kw) + + monkeypatch.setattr(Path, "read_bytes", selective) + with pytest.raises(Indigestible, match="Could not read"): + digest(d) + + +def test_wrapped_call_on_an_unreadable_path_runs_uncached(tmp_path, monkeypatch): + """The point of degrading: the body still runs, it just isn't cached. + + Before this, an `OSError` escaped `digest()` and killed the call before the + body executed — unlike every other undigestable argument, which warns and + falls through to an uncached run. + """ + from fleche import fleche, cache + from fleche.caches import Cache + from fleche.storage import CallMemory, ValueMemory + + f = tmp_path / "secret.txt" + f.write_text("hidden") + runs = [] + + @fleche + def consume(p: Path): + runs.append(p) + return "body ran" + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "read_bytes", boom) + c = Cache(ValueMemory({}), CallMemory({})) + with cache(c): + assert consume(f) == "body ran" + assert consume(f) == "body ran" + + assert len(runs) == 2, "an uncachable call must re-execute, not hit" + assert not c.calls.storage, "nothing should have been filed" + + +def test_readable_paths_are_unaffected(tmp_path): + """The guard must not swallow anything that genuinely works.""" + f = tmp_path / "a.txt" + f.write_text("content") + d = tmp_path / "d" + d.mkdir() + (d / "child.txt").write_text("content") + assert digest(f) and digest(d) + assert digest(f) != digest(d) diff --git a/tests/unit/storage/test_destructuring_storage.py b/tests/unit/storage/test_destructuring_storage.py index 1ca1cee5..b832ea3f 100644 --- a/tests/unit/storage/test_destructuring_storage.py +++ b/tests/unit/storage/test_destructuring_storage.py @@ -1,10 +1,10 @@ import pytest -from collections import namedtuple +from collections import namedtuple, Counter, defaultdict, OrderedDict from dataclasses import dataclass from hypothesis import given, settings, HealthCheck, strategies as st from fleche.storage import ValueMixin, DestructuringMixin from fleche.storage.memory import MemoryBackend -from fleche.storage.destructuring import DigestedIterable, DigestedDict, Digested +from fleche.storage.destructuring import DigestedIterable, DigestedMapping, Digested from fleche.digest import digest, Digest from tests.strategies import st_base_values, st_nested_values, st_key_values, namedtuples @@ -87,19 +87,19 @@ def test_digest_transparency_iterable_mixed(container, items): @given(st.dictionaries(st_key_values, st_base_values, min_size=1, max_size=6)) def test_digest_transparency_dict_all_digests(d): - """DigestedDict whose keys and values are all Digests hashes like the original dict.""" - dd = DigestedDict({digest(k): digest(v) for k, v in d.items()}) + """DigestedMapping whose keys and values are all Digests hashes like the original dict.""" + dd = DigestedMapping({digest(k): digest(v) for k, v in d.items()}) assert digest(dd) == digest(d) @given(st.dictionaries(st_key_values, st_base_values, min_size=2, max_size=6)) def test_digest_transparency_dict_mixed(d): - """DigestedDict with mixed plain and Digest entries still hashes like the original.""" + """DigestedMapping with mixed plain and Digest entries still hashes like the original.""" items = list(d.items()) # inline first key-value pair, digest the rest mixed = {items[0][0]: items[0][1]} mixed.update({digest(k): digest(v) for k, v in items[1:]}) - dd = DigestedDict(mixed) + dd = DigestedMapping(mixed) assert digest(dd) == digest(d) @@ -122,11 +122,11 @@ def test_digested_iterable_mend_roundtrip(container, ds, items): @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) @given(st.dictionaries(st.text(), st_base_values, min_size=1, max_size=6)) def test_digested_dict_mend_roundtrip(ds, d): - """DigestedDict stored by ds can be re-assembled via mend.""" + """DigestedMapping stored by ds can be re-assembled via mend.""" key = ds.save(d) # Access the raw stored value directly from the backend dict (bypasses mend) raw = ds.storage[key] - assert isinstance(raw, DigestedDict) + assert isinstance(raw, DigestedMapping) assert raw.mend(ds) == d @@ -257,7 +257,7 @@ def test_plain_container_stored_when_all_inline(container): def test_plain_dict_stored_when_all_inline(): - """When all dict entries are inlined, the dict is stored without a DigestedDict wrapper.""" + """When all dict entries are inlined, the dict is stored without a DigestedMapping wrapper.""" ds = make_ds(remaining_depth=10) data = {"a": 1, "b": [2, 3]} key = ds.save(data) @@ -452,3 +452,144 @@ def test_count_reuses_nonnegative(value): ds.save(value) hits = ds.count_reuses() assert all(v >= 0 for v in hits.values()) + + +# ---- Mapping subclass destructuring ---- +# DigestedMapping is generic over Mapping, mirroring how DigestedIterable is +# generic over list/tuple: ``sunder`` and ``mend`` reconstruct via +# ``type(value)(...)``. Only the exact types on the _DESTRUCTURERS allowlist +# (dict, OrderedDict) are destructured; subclasses may repurpose that +# constructor (defaultdict, Counter) and are stored verbatim as opaque values. + + +def test_ordereddict_roundtrip(ds): + """OrderedDict round-trips with its type and order preserved.""" + od = OrderedDict([("b", 1), ("a", 2), ("c", 3)]) + key = ds.save(od) + loaded = ds.load(key) + assert loaded == od + assert type(loaded) is OrderedDict + assert list(loaded) == ["b", "a", "c"] + + +def test_ordereddict_is_destructured(ds): + """OrderedDict is wrapped in a DigestedMapping whose items preserve the mapping type.""" + od = OrderedDict([("a", 1), ("b", [2, 3])]) + key = ds.save(od) + raw = ds.storage[key] + assert isinstance(raw, DigestedMapping) + assert type(raw.items) is OrderedDict + + +def test_ordereddict_digest_transparency(): + """DigestedMapping wrapping an OrderedDict hashes like the original OrderedDict.""" + od = OrderedDict([("a", 1), ("b", 2)]) + dd = DigestedMapping(OrderedDict((digest(k), digest(v)) for k, v in od.items())) + assert digest(dd) == digest(od) + + +def test_ordereddict_digest_distinct_from_dict(): + """OrderedDict and dict with the same entries hash differently — verify we preserve that.""" + od = OrderedDict([("a", 1), ("b", 2)]) + d = {"a": 1, "b": 2} + assert digest(od) != digest(d) + ds_local = make_ds() + assert ds_local.save(od) != ds_local.save(d) + + +def test_empty_ordereddict_roundtrip(ds): + """An empty OrderedDict is stored verbatim and round-trips as OrderedDict.""" + od = OrderedDict() + key = ds.save(od) + raw = ds.storage[key] + assert not isinstance(raw, Digested) + loaded = ds.load(key) + assert loaded == od + assert type(loaded) is OrderedDict + + +def test_ordereddict_mend_preserves_type(ds): + """The raw DigestedMapping's mend reconstructs an OrderedDict, not a plain dict.""" + od = OrderedDict([("a", 1), ("b", 2)]) + key = ds.save(od) + raw = ds.storage[key] + assert isinstance(raw, DigestedMapping) + mended = raw.mend(ds) + assert type(mended) is OrderedDict + assert mended == od + + +def test_ordereddict_nested_in_dict_roundtrip(ds): + """An OrderedDict nested in a regular dict is preserved through destructuring.""" + data = {"inner": OrderedDict([("x", 1), ("y", 2)]), "other": 42} + key = ds.save(data) + loaded = ds.load(key) + assert loaded == data + assert type(loaded["inner"]) is OrderedDict + assert list(loaded["inner"]) == ["x", "y"] + + +def test_dict_nested_in_ordereddict_roundtrip(ds): + """A plain dict value inside an OrderedDict round-trips with both types preserved.""" + data = OrderedDict([("a", {"x": 1, "y": 2}), ("b", 3)]) + key = ds.save(data) + loaded = ds.load(key) + assert loaded == data + assert type(loaded) is OrderedDict + assert type(loaded["a"]) is dict + + +# ---- Off-allowlist container types are opaque ---- + + +def test_defaultdict_is_opaque(ds): + """defaultdict is off the allowlist: its constructor wants a factory, not pairs.""" + dd = defaultdict(list, {"a": [1], "b": [2, 3]}) + key = ds.save(dd) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is defaultdict + assert loaded == dd + + +def test_counter_is_opaque(ds): + """Counter is off the allowlist: its constructor would count the pairs.""" + cnt = Counter({"a": 5, "b": 6}) + key = ds.save(cnt) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is Counter + assert loaded == cnt + + +def test_dict_subclass_is_opaque(ds): + """Even a well-behaved dict subclass is opaque until opted in explicitly.""" + + class MyDict(dict): + pass + + md = MyDict({"a": 1, "b": [2, 3]}) + key = ds.save(md) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is MyDict + assert loaded == md + + +def test_list_subclass_is_opaque(ds): + """List subclasses are opaque too — same exact-type rule as mappings.""" + + class MyList(list): + pass + + ml = MyList([1, [2, 3]]) + key = ds.save(ml) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is MyList + assert loaded == ml + + +# DirectoryBlob is now an opaque type owned by PathValueMixin (see +# tests/unit/storage/test_paths.py for its end-to-end coverage). DestructuringMixin +# leaves it alone — no match arm catches it — so it has no presence here. diff --git a/tests/unit/storage/test_paths.py b/tests/unit/storage/test_paths.py new file mode 100644 index 00000000..658aa05b --- /dev/null +++ b/tests/unit/storage/test_paths.py @@ -0,0 +1,496 @@ +import collections +import gc +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pytest + +from collections import namedtuple + +from fleche.digest import digest, Indigestible, OPAQUE_ITERABLES +from fleche.storage import ValueMixin, DestructuringMixin +from fleche.storage.memory import MemoryBackend +from fleche.storage.destructuring import DigestedMapping +from fleche.storage.paths import ( + TempPath, + PathValueMixin, + FileBlob, + DirectoryBlob, + find_path, +) + + +def test_mkdtemp_returns_temp_path(): + p = TempPath.mkdtemp() + assert isinstance(p, TempPath) + + +def test_mkdtemp_directory_exists(): + p = TempPath.mkdtemp() + assert p.exists() + assert p.is_dir() + + +def test_mkdtemp_cleanup_on_del(): + p = TempPath.mkdtemp() + path_str = str(p) + assert Path(path_str).exists() + del p + gc.collect() + assert not Path(path_str).exists() + + +def test_derived_path_shares_temp_root(): + p = TempPath.mkdtemp() + child = p / "subdir" + assert isinstance(child, TempPath) + assert getattr(child, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_derived_path_keeps_temp_dir_alive(): + p = TempPath.mkdtemp() + child = p / "child.txt" + path_str = str(p) + del p + gc.collect() + # child still holds a reference to _temp_root, so directory must still exist + assert Path(path_str).exists() + del child + gc.collect() + assert not Path(path_str).exists() + + +def test_cleanup_after_all_derived_refs_gone(): + p = TempPath.mkdtemp() + a = p / "a" + b = p / "b" / "c" + path_str = str(p) + del p + del a + gc.collect() + assert Path(path_str).exists() + del b + gc.collect() + assert not Path(path_str).exists() + + +def test_path_operations_work(): + p = TempPath.mkdtemp() + child = p / "file.txt" + child.write_text("hello") + assert child.read_text() == "hello" + + +def test_with_suffix_shares_temp_root(): + p = TempPath.mkdtemp() + suffixed = (p / "file").with_suffix(".txt") + assert isinstance(suffixed, TempPath) + assert getattr(suffixed, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_parent_shares_temp_root(): + p = TempPath.mkdtemp() + child = p / "a" / "b" + parent = child.parent + assert isinstance(parent, TempPath) + assert getattr(parent, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_multiple_mkdtemp_calls_independent(): + p1 = TempPath.mkdtemp() + p2 = TempPath.mkdtemp() + assert p1 != p2 + path1_str = str(p1) + del p1 + gc.collect() + assert not Path(path1_str).exists() + assert p2.exists() + + +def test_temp_path_is_path_subclass(): + p = TempPath.mkdtemp() + assert isinstance(p, Path) + + +def test_no_temp_root_on_plain_construction(): + # Constructing TempPath without mkdtemp should not set _temp_root + # (and should not raise on attribute access) + with tempfile.TemporaryDirectory() as tmpdir: + p = TempPath(tmpdir) + assert getattr(p, "_temp_root", None) is None + + +def test_derived_from_plain_has_no_temp_root(): + with tempfile.TemporaryDirectory() as tmpdir: + p = TempPath(tmpdir) + child = p / "child" + assert getattr(child, "_temp_root", None) is None + + +# ---- DirectoryBlob / FileBlob basics ---- + + +def test_directoryblob_is_opaque_to_destructuring(): + """DirectoryBlob is neither a dict subclass nor a dataclass — DestructuringMixin leaves it alone.""" + from dataclasses import is_dataclass + assert not isinstance(DirectoryBlob({}), dict) + assert not is_dataclass(DirectoryBlob({})) + + +def test_directoryblob_repr_distinguishes_from_dict(): + """Repr shows the wrapper class explicitly.""" + db = DirectoryBlob({"a": digest(b"x")}) + assert repr(db).startswith("DirectoryBlob(") + nested = DirectoryBlob({"sub": digest(b"y")}) + assert repr(nested).startswith("DirectoryBlob(") + + +def test_directoryblob_digest_distinct_from_plain_dict(): + """DirectoryBlob salts its digest with the class name so it never collides with a bare dict.""" + contents = {"a": digest(b"x"), "b": digest(b"y")} + db = DirectoryBlob(contents) + assert digest(db) != digest(contents) + + +def test_directoryblob_equality_and_unhashable(): + """DirectoryBlobs with equal contents are equal; instances are unhashable.""" + a = DirectoryBlob({"a": digest(b"x")}) + b = DirectoryBlob({"a": digest(b"x")}) + assert a == b + with pytest.raises(TypeError): + hash(a) + + +def test_fileblob_record_digests_on_name_and_content(): + """FileBlob pairs a basename with a content digest; both are part of its identity.""" + c = digest(b"hello") + assert digest(FileBlob("a.txt", c)) != digest(FileBlob("b.txt", c)) # name matters + assert digest(FileBlob("a.txt", c)) == digest(FileBlob("a.txt", c)) # stable + assert digest(FileBlob("a.txt", c)) != c # not the bare content + + +def test_fileblob_equality_repr_unhashable(): + c = digest(b"x") + assert FileBlob("a", c) == FileBlob("a", c) + assert FileBlob("a", c) != FileBlob("b", c) + assert repr(FileBlob("a", c)).startswith("FileBlob(") + with pytest.raises(TypeError): + hash(FileBlob("a", c)) + + +# ---- PathValueMixin + DestructuringMixin composition ---- +# MRO contract: DestructuringMixin sits above PathValueMixin so that +# Destructure's recursion (via super().save) lands here when it encounters a +# nested Path. PathValueMixin owns the directory traversal end-to-end and +# never re-enters self.save / self.load — every storage call uses super(), +# so there's no load-context ambiguity when other transform mixins compose +# below it. + + +@dataclass(frozen=True) +class PathDM(DestructuringMixin, PathValueMixin, ValueMixin, MemoryBackend): + """Test storage: Destructure -> PathValue -> ValueMixin -> MemoryBackend.""" + __hash__ = object.__hash__ + + +@pytest.fixture +def pds(): + return PathDM(storage={}) + + +def test_toplevel_directory_path_stored_as_opaque_directoryblob(pds, tmp_path): + """Saving a directory Path yields a DirectoryBlob stored verbatim (not destructured).""" + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "b.txt").write_bytes(b"world") + + key = pds.save(tmp_path) + raw = pds.storage[key] + assert type(raw) is DirectoryBlob + assert set(raw.contents) == {"a.txt", "b.txt"} + + +def test_toplevel_file_path_stored_as_fileblob_record(pds, tmp_path): + """A file Path stores as content bytes (deduped) plus a FileBlob(name, content) record.""" + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + key = pds.save(p) + raw = pds.storage[key] + assert type(raw) is FileBlob + assert raw.name == "f.txt" + # Content lives separately as plain bytes under its own content digest. + assert raw.content == digest(b"contents") + assert pds.storage[digest(b"contents")] == b"contents" + # The file's key is (name, content) == the path's own digest, not the raw bytes'. + assert key == digest(p) + assert key != digest(b"contents") + + +def test_toplevel_file_path_roundtrip_preserves_name(pds, tmp_path): + """Save → load of a file Path returns a path with the original bytes AND name.""" + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + loaded = pds.load(pds.save(p)) + assert isinstance(loaded, Path) + assert loaded.read_bytes() == b"contents" + assert loaded.name == "f.txt" # name preserved by default — no wrapping + assert loaded.suffix == ".txt" + + +def test_toplevel_directory_path_roundtrip(pds, tmp_path): + """Save → load of a directory Path materializes the tree; root name is mangled, children kept.""" + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.txt").write_bytes(b"world") + + key = pds.save(tmp_path) + loaded = pds.load(key) + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert loaded.name != tmp_path.name # directory root name is not preserved + assert (loaded / "a.txt").read_bytes() == b"hello" + assert (loaded / "sub" / "b.txt").read_bytes() == b"world" + + +def test_nested_path_inside_dict_is_converted_at_save_and_materialized_at_load(pds, tmp_path): + """Path nested in a container: outer dict destructured, inner Path materializes to disk on load. + + The motivating composition scenario: DestructuringMixin handles the outer + structure, PathValueMixin catches the Path via super().save from + Destructure's recursion and stores it as an opaque DirectoryBlob. + """ + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.txt").write_bytes(b"world") + + wrapper = {"tree": tmp_path, "label": "x"} + key = pds.save(wrapper) + + # The outer dict went through DestructuringMixin. + outer = pds.storage[key] + assert isinstance(outer, DigestedMapping) + + # The inner Path was stored as a verbatim DirectoryBlob (not destructured); + # the scalar "tree" key itself stays inline (depth 0 < remaining_depth). + tree_key = outer.items["tree"] + tree_raw = pds.storage[tree_key] + assert type(tree_raw) is DirectoryBlob, \ + f"DirectoryBlob should be stored opaquely, got {type(tree_raw).__name__}" + + # On load, the inner Path materializes back to a real filesystem path. + loaded = pds.load(key) + assert loaded["label"] == "x" + assert isinstance(loaded["tree"], Path) + assert (loaded["tree"] / "a.txt").read_bytes() == b"hello" + assert (loaded["tree"] / "sub" / "b.txt").read_bytes() == b"world" + + +def test_directory_path_file_children_stored_at_content_digest(pds, tmp_path): + """Each file in a saved directory ends up as plain bytes at its content digest.""" + a, b = b"alpha", b"beta" + (tmp_path / "a").write_bytes(a) + (tmp_path / "b").write_bytes(b) + + pds.save(tmp_path) + keys = set(pds.list()) + assert digest(a) in keys + assert digest(b) in keys + + +def test_directoryblob_contents_reference_children_by_digest(pds, tmp_path): + """The DirectoryBlob's ``contents`` maps filename → child content Digest (not inline bytes).""" + payload = b"shared-bytes" + (tmp_path / "f.txt").write_bytes(payload) + + key = pds.save(tmp_path) + raw = pds.storage[key] + assert isinstance(raw, DirectoryBlob) + assert raw.contents["f.txt"] == digest(payload) + + +def test_two_directories_share_file_storage_via_content_addressing(pds, tmp_path): + """Two distinct directories holding the same file body share its storage entry.""" + payload = b"shared" + d1 = tmp_path / "d1"; d1.mkdir(); (d1 / "x").write_bytes(payload) + d2 = tmp_path / "d2"; d2.mkdir(); (d2 / "y").write_bytes(payload) + + pds.save(d1) + keys_after_first = set(pds.list()) + pds.save(d2) + keys_after_second = set(pds.list()) + + # The new directory added its own DirectoryBlob entry but reused the file body. + new_keys = keys_after_second - keys_after_first + assert digest(payload) not in new_keys + assert digest(payload) in keys_after_first + + +def test_empty_directory_path_roundtrip(pds, tmp_path): + """Saving an empty directory yields an empty DirectoryBlob and round-trips to an empty dir.""" + key = pds.save(tmp_path) + raw = pds.storage[key] + assert type(raw) is DirectoryBlob + assert raw.contents == {} + + loaded = pds.load(key) + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert list(loaded.iterdir()) == [] + + +# --------------------------------------------------------------------------- +# find_path: does a save of this value reach a Path? +# --------------------------------------------------------------------------- + + +@dataclass +class _Fields: + a: object + b: object = None + + +_Bundle = namedtuple("_Bundle", ["out", "score"]) + + +class _Opaque: + """Not a container a destructuring save looks inside.""" + + def __init__(self, p): + self.p = p + + +def test_find_path_finds_a_bare_path(tmp_path): + assert find_path(tmp_path) is tmp_path + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: [0, [p]], id="nested-list"), + pytest.param(lambda p: (p,), id="tuple"), + pytest.param(lambda p: {"k": p}, id="dict-value"), + pytest.param(lambda p: {p: "v"}, id="dict-key"), + pytest.param(lambda p: _Fields(a=1, b={"deep": [p]}), id="dataclass-field"), + ], +) +def test_find_path_descends_the_containers_digest_descends(tmp_path, wrap): + assert find_path(wrap(tmp_path)) is tmp_path + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: _Bundle(p, 0.5), id="namedtuple"), + pytest.param(lambda p: {p}, id="set"), + pytest.param(lambda p: frozenset({p}), id="frozenset"), + pytest.param(lambda p: [_Bundle(p, 0.5)], id="namedtuple-in-list"), + ], +) +def test_find_path_descends_containers_destructuring_treats_as_opaque(tmp_path, wrap): + """The guard must be broader than destructuring, or it lets the bug back in. + + Destructuring stores a namedtuple / set verbatim, but ``digest`` recurses + into both and *reads the file* — so a path hidden in one still decides the + key. A caller that cannot honour path semantics across a machine boundary + has to see it, or the far side digests the same name against its own + filesystem and the ``digest(x) == save_value(x)`` seal breaks silently. + """ + assert find_path(wrap(tmp_path)) is tmp_path + + +@pytest.mark.parametrize( + "value", + [ + pytest.param([1, "two", b"three"], id="scalars"), + pytest.param({}, id="empty"), + pytest.param("/not/a/path/just/a/string", id="path-shaped-string"), + ], +) +def test_find_path_returns_none_without_a_path(value): + assert find_path(value) is None + + +def test_find_path_stops_where_digest_stops(tmp_path): + """A plain object is `Indigestible`, so no path inside it can decide a key. + + This is the boundary: ``find_path`` follows ``digest``, and ``digest`` + cannot see into an arbitrary object either — it raises rather than + reading the file. Nothing to warn about, so nothing to report. + """ + assert find_path(_Opaque(tmp_path)) is None + with pytest.raises(Indigestible): + digest(_Opaque(tmp_path)) + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: collections.deque([p]), id="deque"), + pytest.param(lambda p: [collections.deque([0, p])], id="deque-in-list"), + pytest.param(lambda p: collections.OrderedDict(k=p), id="ordereddict"), + pytest.param(lambda p: collections.defaultdict(list, k=p), id="defaultdict"), + pytest.param(lambda p: collections.Counter({p: 1}), id="counter-key"), + pytest.param(lambda p: collections.ChainMap({"k": p}), id="chainmap"), + pytest.param(lambda p: array_like(p), id="object-array-elementwise"), + ], +) +def test_find_path_descends_any_re_iterable_container(tmp_path, wrap): + """An allowlist of concrete types is the bug in slower motion. + + ``digest``'s ``Iterable`` arm walks *anything* iterable and reads the file, + so a path in a ``deque`` decides the key exactly as much as one in a list. + Enumerating ``list``/``tuple``/``set``/``frozenset`` covered the containers + that happened to be in mind and let the next one through. + """ + assert find_path(wrap(tmp_path)) is tmp_path + + +def array_like(p): + """A re-iterable custom container — the case no allowlist can anticipate.""" + + class _Rows: + def __init__(self, rows): + self._rows = rows + + def __iter__(self): + return iter(self._rows) + + return _Rows([0, p]) + + +def test_find_path_skips_iterables_digest_never_looks_inside(tmp_path): + """The exclusions have to be exactly the ones ``digest`` itself makes. + + A numpy object array is ``Iterable``, but ``digest`` matches it above the + ``Iterable`` arm and hashes its buffer — the elements never reach a digest, + so a path inside one cannot decide a key and there is nothing to report. + Walking it anyway would also mean walking every large numeric array. + """ + arr = np.array([tmp_path, 1], dtype=object) + assert find_path(arr) is None + assert isinstance(arr, OPAQUE_ITERABLES) + + +def test_find_path_does_not_materialize_a_huge_range(): + """``range`` holds ints by construction; walking one is pure cost. + + Guarded explicitly because the walk pushes elements onto a list, so a + large range would exhaust memory rather than merely be slow. + """ + assert find_path([range(10**9)]) is None + + +def test_find_path_does_not_consume_a_generator(tmp_path): + """Walking must not have the side effect of exhausting a one-shot iterable.""" + gen = (x for x in [tmp_path]) + find_path(gen) + assert list(gen) == [tmp_path] + + +def test_find_path_terminates_on_a_cycle(tmp_path): + cyclic = [1] + cyclic.append(cyclic) + assert find_path(cyclic) is None + cyclic.append(tmp_path) + assert find_path(cyclic) is tmp_path diff --git a/tests/unit/storage/test_paths_default_wiring.py b/tests/unit/storage/test_paths_default_wiring.py new file mode 100644 index 00000000..220e592e --- /dev/null +++ b/tests/unit/storage/test_paths_default_wiring.py @@ -0,0 +1,253 @@ +"""Path-by-content storage through the *default* value storages. + +The committed ``test_paths.py`` exercises a hand-built ``PathDM`` over +``MemoryBackend`` only. These tests prove that wiring ``PathValueMixin`` into +the real default classes (``ValueMemory``, ``ValuePickleFile``, +``ValueBagOfHoldingH5File``) actually works end-to-end — in particular that the +``FileBlob`` (name, content) records, ``DirectoryBlob`` trees, and plain-bytes +content survive each backend's serialization (pickle, H5), not just in-memory +deepcopy. +""" + +from pathlib import Path + +import pytest + +from fleche import fleche, cache +from fleche.digest import digest +from fleche.caches import Cache, DigestedDict +from fleche.storage import ( + ValueMemory, + ValuePickleFile, + ValueBagOfHoldingH5File, + CallMemory, + PathValueMixin, +) +from fleche.storage.destructuring import DigestedMapping + + +# ---- helpers ------------------------------------------------------------- + +def _make_tree(root: Path) -> Path: + """Create *root* and fill it with a small nested directory tree; return *root*.""" + root.mkdir(parents=True, exist_ok=True) + (root / "a.txt").write_bytes(b"hello") + sub = root / "sub" + sub.mkdir() + (sub / "b.bin").write_bytes(b"\x00\x01\x02world") + (sub / "deep").mkdir() + (sub / "deep" / "c").write_bytes(b"deep-bytes") + return root + + +def _relmap(root: Path) -> dict[str, bytes]: + """Map every file under *root* to ``{relative_posix_path: bytes}``.""" + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +# ---- the default value storages carry PathValueMixin --------------------- + +@pytest.mark.parametrize( + "cls", [ValueMemory, ValuePickleFile, ValueBagOfHoldingH5File] +) +def test_default_value_storage_has_path_mixin_in_mro(cls): + assert PathValueMixin in cls.__mro__ + names = [c.__name__ for c in cls.__mro__] + # PathValueMixin sits between DestructuringMixin and ValueMixin so + # Destructure's recursion lands on it, and it stores blobs via ValueMixin. + assert ( + names.index("DestructuringMixin") + < names.index("PathValueMixin") + < names.index("ValueMixin") + ) + + +# ---- round-trips across every real backend ------------------------------- + +def test_file_path_roundtrip(value_storage, tmp_path): + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + + key = value_storage.save(p) + # A file is keyed on (name, content); that equals the path's own digest. + assert key == digest(p) + + loaded = value_storage.load(key) + assert isinstance(loaded, Path) + assert loaded.read_bytes() == b"contents" + assert loaded.name == "f.txt" # name preserved by default — no wrapping + + +def test_directory_tree_roundtrip(value_storage, tmp_path): + src = _make_tree(tmp_path / "src") + expected = _relmap(src) + + key = value_storage.save(src) + loaded = value_storage.load(key) + + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert _relmap(loaded) == expected + + +def test_path_nested_in_dict_roundtrip(value_storage, tmp_path): + src = _make_tree(tmp_path / "src") + expected = _relmap(src) + + wrapper = {"tree": src, "label": "x", "n": 3} + key = value_storage.save(wrapper) + loaded = value_storage.load(key) + + assert loaded["label"] == "x" + assert loaded["n"] == 3 + assert isinstance(loaded["tree"], Path) + assert _relmap(loaded["tree"]) == expected + + +def test_content_addressed_file_dedup(value_storage, tmp_path): + """A file body shared by two directories is stored exactly once.""" + shared = b"shared-body" + one, two = b"one", b"two" + + d1 = tmp_path / "d1" + d1.mkdir() + (d1 / "x").write_bytes(shared) + (d1 / "y").write_bytes(one) + + d2 = tmp_path / "d2" + d2.mkdir() + (d2 / "x").write_bytes(shared) + (d2 / "z").write_bytes(two) + + k1 = value_storage.save(d1) + k2 = value_storage.save(d2) + + keys = set(value_storage.list()) + # File bodies are plain bytes keyed by content; only one copy of `shared`, + # and the two directory trees are distinct. + assert keys == { + digest(shared), + digest(one), + digest(two), + k1, + k2, + } + + +def test_save_load_save_is_idempotent(value_storage, tmp_path): + """Re-saving a materialized TempPath reproduces the same content key.""" + src = _make_tree(tmp_path / "src") + + key = value_storage.save(src) + loaded = value_storage.load(key) + key2 = value_storage.save(loaded) + + assert key2 == key + + +# ---- end-to-end through a Cache + @fleche -------------------------------- + +def test_cache_end_to_end_path_roundtrip(tmp_path): + """A cached function producing a Path: tree stored by content, cache hit on repeat.""" + runs = [] + + @fleche + def build(name, payload): + runs.append(name) + d = tmp_path / f"out-{name}" + d.mkdir() + (d / "data.txt").write_text(payload) + (d / "nested").mkdir() + (d / "nested" / "more.bin").write_bytes(payload.encode() * 2) + return d + + with cache(Cache(ValueMemory({}), CallMemory({}))): + first = build("a", "hello") + assert isinstance(first, Path) + assert (first / "data.txt").read_text() == "hello" + + second = build("a", "hello") + assert isinstance(second, Path) + assert (second / "data.txt").read_text() == "hello" + assert (second / "nested" / "more.bin").read_bytes() == b"hellohello" + + # The body ran exactly once; the second call was served from cache. + assert runs == ["a"] + + +def test_cache_path_argument_roundtrip(tmp_path): + """A cached function consuming a Path argument hits cache on repeat input.""" + parsed = [] + + @fleche + def parse(f: Path): + parsed.append(f.name) + return f.read_text().upper() + + p = tmp_path / "in.txt" + p.write_text("abc") + + with cache(Cache(ValueMemory({}), CallMemory({}))): + assert parse(p) == "ABC" + assert parse(p) == "ABC" + + # Same Path content => one execution. + assert parsed == ["in.txt"] + + +def test_downstream_consumer_needs_no_branching(tmp_path): + """A file keeps its name through the cache, so a plain consumer sees the right suffix. + + No NamedPath, no isinstance: ``inspect`` is an ordinary path consumer and its + ``.suffix`` check holds even when ``produce`` is served from cache. + """ + runs = [] + + @fleche + def produce(seed): + runs.append(("produce", seed)) + p = tmp_path / f"{seed}.json" + p.write_text(f'{{"seed": "{seed}"}}') + return p + + @fleche + def inspect(path): + runs.append(("inspect", path.suffix)) + assert path.suffix == ".json" + return path.stem + + with cache(Cache(ValueMemory({}), CallMemory({}))): + assert inspect(produce("alpha")) == "alpha" + assert inspect(produce("alpha")) == "alpha" # produce cached -> alpha.json + + assert runs == [("produce", "alpha"), ("inspect", ".json")] + + +def test_renaming_does_not_duplicate_the_content_blob(tmp_path): + """Renaming a file (same bytes, new name) adds only a record, never re-stores the body.""" + store = ValueMemory({}) + body = b"the unchanging body" * 100 + + original = tmp_path / "draft.txt" + original.write_bytes(body) + store.save(original) + keys_before = set(store.list()) + assert digest(body) in keys_before, "content body stored on first save" + + renamed_path = tmp_path / "final.txt" + original.rename(renamed_path) + store.save(renamed_path) + + added = set(store.list()) - keys_before + assert len(added) == 1 # just the (name, content) record + assert digest(body) not in added # body reused, not duplicated + + +# ---- backward-compat alias ---------------------------------------------- + +def test_digesteddict_alias_is_digestedmapping(): + assert DigestedDict is DigestedMapping diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 96afcb49..fe1ee4cd 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -5,11 +5,14 @@ only the transport is swapped. """ +import collections +import dataclasses import io import os import sys import threading import types +from typing import Any import pytest @@ -20,6 +23,7 @@ from fleche.digest import Digest, digest from fleche.remote import ( RemoteConnectionError, + RemotePathUnsupported, SshCache, _Connection, _dispatch, @@ -1067,3 +1071,204 @@ def test_run_server_serves_active_cache_over_stdio_until_eof(monkeypatch, cache_ assert info["cache"] == cache_to_config(expected_cache) # Clean EOF: no second frame queued. assert fake_stdout.read() == b"" + + +# --------------------------------------------------------------------------- +# Paths do not cross the wire +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _Holder: + payload: Any + + +@pytest.fixture +def a_file(tmp_path): + f = tmp_path / "data.txt" + f.write_text("client bytes") + return f + + +def test_save_value_rejects_a_bare_path(remote, server_cache, a_file): + """A Path would arrive as a bare string the server resolves itself.""" + with pytest.raises(RemotePathUnsupported) as excinfo: + remote.save_value(a_file) + assert str(a_file) in str(excinfo.value) + # Refused before the RPC: nothing reached the server. + assert not server_cache.values.storage + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: [1, p], id="list"), + pytest.param(lambda p: (p,), id="tuple"), + pytest.param(lambda p: {"a": {"b": p}}, id="nested-dict"), + pytest.param(lambda p: _Holder(payload=[p]), id="dataclass"), + ], +) +def test_save_value_rejects_a_nested_path(remote, server_cache, a_file, wrap): + """Destructuring walks into containers, so the guard has to as well.""" + with pytest.raises(RemotePathUnsupported): + remote.save_value(wrap(a_file)) + assert not server_cache.values.storage + + +def test_save_value_still_accepts_the_bytes_escape_hatch(remote, a_file): + """Returning ``bytes`` is the documented way to ship content remotely.""" + key = remote.save_value(a_file.read_bytes()) + assert remote.load_value(key) == b"client bytes" + + +def test_rejection_is_a_save_error(a_file): + """``SaveError`` is what the two-phase save protocol degrades on.""" + assert issubclass(RemotePathUnsupported, SaveError) + + +def test_prepare_degrades_a_path_argument_to_a_digest_only_reference(remote, a_file): + """The sealed key must still be the one a later lookup computes. + + This is the regression the guard buys: letting the path through made the + server digest *its* view of that name, so the record was filed under a + key no client could reproduce. + """ + call = Call(name="f", arguments={"p": a_file, "n": 3}) + prepared = remote.prepare(call) + assert prepared.to_lookup_key() == call.to_lookup_key() + # The argument survives as its (locally computed) digest, not as content. + assert prepared.digested.arguments["p"] == digest(a_file) + + +def test_saving_a_live_call_carrying_a_path_is_rejected(remote, server_cache, a_file): + """The one-shot form ships values too — the server would stash them.""" + with pytest.raises(Rejected): + remote.save(Call(name="f", arguments={"p": a_file}, result=1)) + with pytest.raises(Rejected): + remote.save(Call(name="f", arguments={"x": 1}, result=[a_file])) + assert not server_cache.calls.storage + + +def test_load_value_refuses_a_path_materialized_on_the_server( + remote, server_cache, a_file +): + """A path stored remotely comes back pointing into the server's temp dir. + + Materialization happens on the server's filesystem and only the *name* + travels back, so the client would be handed a dangling reference — one + the server unlinks as soon as its own reference dies. + """ + key = server_cache.values.save(a_file) # stored server-side, by content + with pytest.raises(RemotePathUnsupported) as excinfo: + remote.load_value(key) + assert key in str(excinfo.value) + + +def test_load_value_refuses_a_path_nested_in_a_container( + remote, server_cache, a_file +): + key = server_cache.values.save({"out": [a_file]}) + with pytest.raises(RemotePathUnsupported): + remote.load_value(key) + + +def test_lazy_call_only_trips_on_the_path_value(remote, server_cache, a_file): + """Records stay queryable; only touching the path value fails.""" + key = server_cache.save(Call(name="f", arguments={"x": 1}, result=a_file)) + lc = remote.load(key) + assert dict(lc.arguments) == {"x": 1} + with pytest.raises(RemotePathUnsupported): + lc.result + + +# --------------------------------------------------------------------------- +# The two-phase protocol is a values-over-the-wire path too +# --------------------------------------------------------------------------- + + +_Bundle = collections.namedtuple("_Bundle", ["out", "score"]) + + +def test_prepare_does_not_ship_a_call_carrying_a_path(remote, server_cache, a_file): + """`prepare` is the one RPC that sends argument *values*, not digests. + + Shipping the live call would hand the server a path string to resolve + against its own filesystem — the bug the guard exists to stop — so the + arguments are stashed one at a time instead. + """ + call = Call(name="f", arguments={"p": a_file, "n": 3}) + prepared = remote.prepare(call) + # Sealed locally: the key is still the one a later lookup computes. + assert prepared.to_lookup_key() == call.to_lookup_key() + assert prepared.digested.arguments["p"] == digest(a_file) + # Nothing about the path reached the server — not the record, not the + # content bytes it would have been split into. + assert digest(a_file) not in server_cache.values.storage + assert a_file.read_bytes() not in server_cache.values.storage.values() + + +def test_prepare_degrades_only_the_path_argument_not_its_siblings( + remote, server_cache, a_file +): + """One unshippable argument must not take the whole call down with it. + + The fallback used to be a blanket `BaseCache.prepare`, which digests + everything and stores nothing: `f(p: Path, payload)` lost `payload` too, + and `load(key).arguments["payload"]` came back a bare digest for no + reason. Only the path cannot cross. + """ + payload = {"rows": [1, 2, 3]} + call = Call(name="f", arguments={"p": a_file, "payload": payload}) + prepared = remote.prepare(call) + + # The sibling was stored on the remote and reads back as a value... + assert remote.load_value(prepared.digested.arguments["payload"]) == payload + # ...while the path is a digest-only reference computed here. + assert prepared.digested.arguments["p"] == digest(a_file) + assert digest(a_file) not in server_cache.values.storage + + # End to end: the filed record hands back the sibling, not a digest. + key = prepared.commit(1) + loaded = remote.load(key) + assert loaded.arguments["payload"] == payload + # The path argument stays a digest, which is the documented degradation: + # keyed correctly, content not retrievable from the remote. + assert loaded.arguments["p"] == digest(a_file) + + +def test_prepare_still_uses_the_remote_when_no_path_is_involved(remote, server_cache): + """The fallback must be narrow: ordinary calls keep the one-round-trip path.""" + prepared = remote.prepare(Call(name="f", arguments={"x": 1, "y": "two"})) + assert ( + prepared.to_lookup_key() + == Call(name="f", arguments={"x": 1, "y": "two"}).to_lookup_key() + ) + assert server_cache.values.storage # the remote stashed the arguments + + +def test_committing_a_path_result_is_rejected(remote, a_file): + """`commit` ships the result through `save`; a path must not slip through.""" + prepared = remote.prepare(Call(name="f", arguments={"x": 1})) + with pytest.raises(Rejected): + prepared.commit(a_file) + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: _Bundle(p, 0.5), id="namedtuple"), + pytest.param(lambda p: {p}, id="set"), + ], +) +def test_paths_hidden_in_destructuring_opaque_containers_are_refused( + remote, server_cache, a_file, wrap +): + """Destructuring stores these verbatim, but `digest` reads the file anyway. + + So the far side would digest the same name against its own filesystem — + the guard has to follow `digest`, not destructuring, or the seal breaks + exactly as it did before. + """ + with pytest.raises(RemotePathUnsupported): + remote.save_value(wrap(a_file)) + assert not server_cache.values.storage