From 2da4b104280372374ea8ccbe205cee780bd2bce0 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sun, 26 Jul 2026 10:43:34 -0400 Subject: [PATCH 01/27] feat(caches)!: seal call identity before the function body runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wrapper digested arguments twice from live state: once before the function body (the lookup key) and once after (when Cache.save stashed the call record). A function that mutated an argument — e.g. appended to a received list, or wrote into a received directory — was therefore recorded under the post-mutation content: honest repeat calls could never hit it, while a call passing the mutated state would false-hit a result computed from different input. Fleche commits pure functions: the recorded identity of a call is now sealed from the arguments as passed, via a two-phase save protocol: - cache.prepare(call) -> PreparedCall stores argument values and seals the lookup key before the body runs; PreparedCall.commit(result, metadata) stores the result as returned (a mutated argument passed back out is captured in its final state) and files the record; PreparedCall.abandon() releases without recording (body raised or result uncacheable). PreparedCall is also a context manager that abandons when the block exits uncommitted. - prepare is implemented once on BaseCache over a new save_value primitive (the write-side counterpart of load_value), which wrappers forward, CacheStack routes to stack[0], and SshCache sends over the wire; read-only caches admit digest-only (nothing written, and the commit after the body is rejected — matching the previous post-body rejection behavior). - Cache.save takes the fully digested record; a live Call is still accepted as the degenerate one-shot form (values static, nothing to drift), so post-hoc saving — transfers, redigest, external callers — keeps working unchanged. BREAKING CHANGE: calls of argument-mutating functions are now recorded under the pre-call argument state; caches populated by such functions will re-execute once and re-record under the correct key. Co-Authored-By: Claude Fable 5 --- src/fleche/caches.py | 114 +++++++++++++-- src/fleche/call.py | 83 ++++++++++- src/fleche/remote.py | 13 +- src/fleche/wrapper.py | 24 +++- tests/unit/call/test_prepared_call.py | 166 ++++++++++++++++++++++ tests/unit/config/test_cache_to_config.py | 3 + tests/unit/test_remote.py | 1 + 7 files changed, 382 insertions(+), 22 deletions(-) create mode 100644 tests/unit/call/test_prepared_call.py diff --git a/src/fleche/caches.py b/src/fleche/caches.py index 1d1fba1e..fe62e668 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -13,7 +13,7 @@ from .storage.base import _apply_shrink, _resolve_prefix, Intent, OperationContext from .storage.destructuring import HasChildDigests from .storage.thread_safe import PerKeyLockMixin, _PicklableRLock -from .call import Call, DigestedCall, LazyCall, QueryCall +from .call import Call, DigestedCall, LazyCall, PreparedCall, QueryCall from . import call from . import query @@ -49,7 +49,45 @@ def from_config(cls, config: "dict[str, Any] | list[dict[str, Any]]") -> "BaseCa return _config.cache_from_config(config) @abstractmethod - def save(self, call: Call) -> str: + def save_value(self, value: Any) -> "Digest": + """Store one value, returning its content digest. + + The write-side counterpart of :meth:`load_value`; also the primitive + :meth:`prepare` runs argument and result values through. + """ + ... + + def prepare(self, call: Call) -> PreparedCall: + """Admit *call* to this cache: store its arguments, seal its lookup key. + + The first half of the two-phase save protocol. Argument values go + through :meth:`save_value` *now* — before the function body runs — so + the recorded identity always describes the arguments as they were at + call time, even if the body later mutates them. Because ``digest(x) + == save_value(x)`` for every storable value, the sealed key equals + ``call.to_lookup_key()``. + + Finish the returned :class:`~fleche.call.PreparedCall` with exactly + one of :meth:`~fleche.call.PreparedCall.commit` (store the result, + file the record) or :meth:`~fleche.call.PreparedCall.abandon`. + + Argument values the storage refuses (``SaveError``) fall back to a + digest-only reference, as in :meth:`fleche.call.Call.stash`. + """ + digested = call._to_digested(self.save_value) + return PreparedCall( + call=call, digested=digested, key=digested.to_lookup_key(), cache=self + ) + + @abstractmethod + def save(self, call: DigestedCall | Call) -> str: + """File a call record. + + The primary form takes a fully digested record whose values were + already stored — see :meth:`fleche.call.Call.prepare` for the + two-phase protocol that does both in the right order. A live + :class:`Call` is also accepted as the degenerate one-shot form + (values static, stored on the spot).""" ... @abstractmethod @@ -294,14 +332,38 @@ def load_value(self, key): with self._operation_context(key): return self.values.load(key) - def save(self, call: Call) -> str: + def save_value(self, value: Any) -> Digest: + # No cache-level lock: the key is only known once the value storage + # has digested the value, and value storages carry their own per-key + # locking (PerKeyLockMixin) where they need it. + return self.values.save(value) + + def save(self, call: DigestedCall | Call) -> str: + # Record-only: argument and result values were already written to + # ``self.values`` by Call.prepare / PreparedCall.commit, whose digests + # this record carries. Writing them here instead would re-read mutable + # values (e.g. Path contents) *after* the function body ran and file + # the record under post-mutation content — the incoherence the + # two-phase protocol exists to prevent. + # + # A live Call (values not yet stored) is still accepted as the + # degenerate one-shot form: with no function body between digesting + # and filing there is nothing to drift, so stash-then-file is + # equivalent to prepare/commit here. Inlined rather than routed + # through prepare().commit() so one logical save does not re-enter + # subclass save() overrides a second time. Wrappers and stacks need + # no own shim — every save path lands here. + if isinstance(call, Call): + key = call.to_lookup_key() + with self._operation_context(key): + try: + digested = call.stash(self.values) + except storage.SaveError as e: + raise Rejected(e) + return self.calls.save(digested) key = call.to_lookup_key() with self._operation_context(key): - try: - digested = call.stash(self.values) - except storage.SaveError as e: - raise Rejected(e) - return self.calls.save(digested) + return self.calls.save(call) def load(self, key: str) -> LazyCall: with self._operation_context(key): @@ -392,8 +454,8 @@ def redigest(self) -> None: This may take time depending on cache size.""" for key in self.calls.list(): - call = self.load(key).fetch() - new_key = call.to_lookup_key() + loaded = self.load(key).fetch() + new_key = loaded.to_lookup_key() if new_key == key: continue # Hold the per-key locks for both the old and the new key so the @@ -407,7 +469,7 @@ def redigest(self) -> None: first, second = sorted((key, new_key)) with self._operation_context(first), self._operation_context(second): # instantiate values too - self.save(call) + self.save(loaded) self.evict(key) def gc(self) -> set[Digest]: @@ -468,7 +530,10 @@ class CacheWrapper(BaseCache): cache: BaseCache - def save(self, call: Call) -> str: + def save_value(self, value: Any) -> Digest: + return self.cache.save_value(value) + + def save(self, call: DigestedCall | Call) -> str: return self.cache.save(call) def load(self, key: str) -> LazyCall: @@ -518,12 +583,25 @@ class ReadOnlyMixin: short-circuits ``save``/``evict`` without a round-trip). """ - def save(self, call: Call): + def save(self, call: DigestedCall | Call): raise Rejected(self, call) def evict(self, key: str | Digest) -> None: raise Rejected("Cannot evict from a read-only cache", self, key) + def save_value(self, value: Any) -> Digest: + raise Rejected("Cannot save values to a read-only cache", self) + + def prepare(self, call: Call) -> "PreparedCall": + # Digest-only admission: a read-only cache stashes nothing, so the + # function body still runs with a correctly sealed key, and the + # eventual commit is rejected (save_value raises) — matching the + # behavior save() rejection produced before the two-phase protocol. + digested = call.digest() + return PreparedCall( + call=call, digested=digested, key=digested.to_lookup_key(), cache=self + ) + @dataclass(frozen=True) class ReadOnlyCache(ReadOnlyMixin, CacheWrapper): @@ -768,9 +846,15 @@ def __post_init__(self): if isinstance(c, CacheStack): raise ValueError("CacheStack cannot be nested inside another CacheStack") - def save(self, call: Call): + def save(self, call: DigestedCall | Call): self.stack[0].save(call) + def save_value(self, value: Any) -> Digest: + # Writes always land on stack[0] (matching save); reads that need the + # full fan-out go through load_value, which _MultiCache spreads over + # every member. + return self.stack[0].save_value(value) + @contextlib.contextmanager def _operation_context(self, key, *, intent: Intent = Intent.WRITE): # Saves always land on ``stack[0]``, so that is the only member that @@ -923,7 +1007,7 @@ def _enforce_size_limit(self) -> None: target = self._pick_eviction_target(list(self._keys)) self.evict(target) - def save(self, call: call.Call) -> str: + def save(self, call: call.DigestedCall | call.Call) -> str: with self._lock: key = super().save(call) self._keys.add(key) diff --git a/src/fleche/call.py b/src/fleche/call.py index 249a3265..ec7405c3 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -199,8 +199,13 @@ def to_lookup_key(self) -> "Digest": return digest.digest(call) def _to_digested(self, save_fn: Callable[[Any], Digest]) -> "DigestedCall": - """Generic conversion to DigestedCall using *save_fn* to handle each value.""" - result = save_fn(self.result) + """Generic conversion to DigestedCall using *save_fn* to handle each value. + + A ``None`` result stays ``None`` (an incomplete record awaiting its + result — see :meth:`fleche.caches.BaseCache.prepare`) rather than being + run through *save_fn*. + """ + result = None if self.result is None else save_fn(self.result) arguments: dict[str, Digest] = {} for k, v in self.arguments.items(): if isinstance(v, Digest): @@ -324,6 +329,80 @@ def fetch(self, cache) -> "LazyCall": ) +@dataclass +class PreparedCall: + """A call admitted to a cache: arguments stored, key sealed, result awaited. + + The middle state of a call's lifecycle, between :class:`Call` (live values, + nothing stored) and :class:`DigestedCall` (fully recorded). Created by + :meth:`fleche.caches.BaseCache.prepare`, which injects the cache whose + value storage received the arguments; finished by exactly one of + :meth:`commit` or :meth:`abandon`. + + Also usable as a context manager in synchronous code: leaving the block + without having committed abandons the call. + + .. code-block:: python + + with cache.prepare(call) as prepared: + prepared.commit(func(...)) # skipped on exception -> abandoned + """ + + call: Call + digested: DigestedCall + key: Digest + cache: Any + _finished: bool = field(default=False, init=False, repr=False) + + def commit(self, result: Any, metadata: dict | None = None) -> Digest: + """Store *result*, attach *metadata*, and file the call record. + + The second half of the two-phase save protocol: the result is written to + ``cache.values`` (capturing its content *as returned* — including any + argument the body mutated and passed back out), and the completed + :class:`DigestedCall` is filed under :attr:`key`. + + Args: + result: The function's return value. + metadata: Optional metadata mapping stored on the record. + + Returns: + Digest: the record key (equal to :attr:`key`). + + Raises: + fleche.caches.Rejected: if the result cannot be stored or the cache + refuses the record. + """ + from .storage.base import SaveError # lazy import: storage.base depends on call + try: + self.digested.result = self.cache.save_value(result) + except SaveError as e: + from .caches import Rejected # lazy import: caches depends on call + raise Rejected(e) from None + if metadata is not None: + self.digested.metadata = metadata + self._finished = True + return self.cache.save(self.digested) + + def abandon(self) -> None: + """Release the call without recording it. + + Called when the function body raises or its result is not cacheable. + The default is a no-op: argument values stored by :meth:`Call.prepare` + are content-addressed orphans that a later garbage collection sweep + reclaims. Subclasses may hook cleanup here. + """ + self._finished = True + + def __enter__(self) -> "PreparedCall": + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + if not self._finished: + self.abandon() + return False + + class LazyArguments(Mapping): def __init__(self, cache, arg_digests): self._cache = cache diff --git a/src/fleche/remote.py b/src/fleche/remote.py index e29471ae..a51fd189 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -57,7 +57,7 @@ from . import call as _call from .caches import BaseCache, Rejected -from .call import Call, DigestedCall, LazyCall, QueryCall +from .call import DigestedCall, LazyCall, QueryCall from .digest import Digest logger = logging.getLogger("fleche.remote") @@ -148,6 +148,7 @@ def _query_result(cache: BaseCache, args: tuple) -> tuple[DigestedCall, ...]: "save": _RemoteMethod(lambda cache, args: cache.save(*args)), "load": _RemoteMethod(lambda cache, args: _strip_cache(cache.load(*args))), "load_value": _RemoteMethod(lambda cache, args: cache.load_value(*args)), + "save_value": _RemoteMethod(lambda cache, args: cache.save_value(*args)), "evict": _RemoteMethod(lambda cache, args: cache.evict(*args), void=True), "contains": _RemoteMethod(lambda cache, args: cache.contains(*args)), "expand": _RemoteMethod(lambda cache, args: cache.expand(*args)), @@ -680,6 +681,7 @@ def _fetch_lazy_calls(sc: "SshCache", results: "tuple[DigestedCall, ...]") -> It "save": _ClientMethod(write=True), "load": _ClientMethod(unwrap=_fetch_lazy_call), "load_value": _ClientMethod(), + "save_value": _ClientMethod(write=True), "evict": _ClientMethod( write=True, reject_message="Cannot evict from a read-only remote cache" ), @@ -783,7 +785,7 @@ def _rpc(self, name: str, *args: Any) -> Any: raise Rejected(self, *args) return spec.unwrap(self, self._conn.call(name, *args)) - def save(self, call: Call) -> str: + def save(self, call: DigestedCall | _call.Call) -> str: return self._rpc("save", call) def load(self, key: str) -> LazyCall: @@ -792,6 +794,13 @@ def load(self, key: str) -> LazyCall: def load_value(self, key: str) -> Any: return self._rpc("load_value", key) + def save_value(self, value: Any) -> Digest: + # One round trip per value (no batching yet). Values travel by + # cloudpickle, so a Path value ships its path *string*, not its + # content — the same limitation the previous ship-the-whole-Call + # protocol had; paths over SSH remain unsupported. + return self._rpc("save_value", value) + def evict(self, key: str | Digest) -> None: self._rpc("evict", key) diff --git a/src/fleche/wrapper.py b/src/fleche/wrapper.py index 56e9ffe0..16133f2a 100644 --- a/src/fleche/wrapper.py +++ b/src/fleche/wrapper.py @@ -235,7 +235,20 @@ def _run_and_cache(): for m in active_meta: metadata[m.name] |= m.pre(replace(call, metadata={})) - result: _T = func(*args, **kwargs) + # Two-phase save: argument values are stored and the record's key + # sealed *before* the body runs, so the recorded identity is the + # arguments as passed — even if the body mutates them (e.g. writes + # into a directory it received). A read-only cache stashes + # nothing here (digest-only prepare) and rejects at commit time, + # so the call still runs and returns uncached, as it previously + # did when the post-body save was rejected. + prepared = cache.prepare(call) + + try: + result: _T = func(*args, **kwargs) + except BaseException: + prepared.abandon() + raise def _cache(future=None): if future is not None: @@ -245,7 +258,11 @@ def _cache(future=None): if future is None: call.result = result else: - call.result = future.result() + try: + call.result = future.result() + except BaseException: + prepared.abandon() + raise if call.result is None: if isinstance(cache, RefreshingCache): try: @@ -257,6 +274,7 @@ def _cache(future=None): logger.warning("Cache rejected evict: %s", e.args) else: logger.warning("Function returned None, not caching") + prepared.abandon() return None for m in active_meta: metadata[m.name] |= m.post( @@ -265,7 +283,7 @@ def _cache(future=None): try: call.metadata = metadata logger.debug("Saving result for %s with key %s", call.name, key) - cache.save(call) + prepared.commit(call.result, metadata) except Rejected as e: logger.warning("Cache rejected save: %s", e.args) return call.result diff --git a/tests/unit/call/test_prepared_call.py b/tests/unit/call/test_prepared_call.py new file mode 100644 index 00000000..2717d88b --- /dev/null +++ b/tests/unit/call/test_prepared_call.py @@ -0,0 +1,166 @@ +"""Two-phase save protocol: Call.prepare / PreparedCall.commit / abandon. + +The protocol exists to fix one incoherence: the wrapper digests arguments +*before* the function body runs (the lookup key), but the old save path +re-digested them from their live values *after* — so a function that mutated +an argument was recorded under post-mutation content and could never be found +by an honest repeat call. Under the two-phase protocol the recorded identity +is sealed at prepare time. +""" +import pytest + +from fleche import fleche +import fleche as fl +from fleche.call import Call +from fleche.caches import Cache, Rejected +from fleche.digest import digest +from fleche.storage.memory import ValueMemory, CallMemory + + +@pytest.fixture +def cache(): + return Cache(ValueMemory({}), CallMemory({})) + + +def make_call(**arguments): + return Call(name="f", arguments=arguments, module="m", version="1") + + +# ---- PreparedCall lifecycle ---- + + +def test_prepare_seals_lookup_key(cache): + call = make_call(x=1, y="a") + prepared = cache.prepare(call) + assert prepared.key == call.to_lookup_key() + + +def test_prepare_stores_arguments_before_commit(cache): + call = make_call(x=[1, 2]) + cache.prepare(call) + assert cache.values.load(digest([1, 2])) == [1, 2] + + +def test_commit_files_record_under_sealed_key(cache): + call = make_call(x=1) + prepared = cache.prepare(call) + key = prepared.commit("result", {"meta": {"k": "v"}}) + assert key == prepared.key + loaded = cache.load(key) + assert loaded.result == "result" + assert loaded.metadata == {"meta": {"k": "v"}} + + +def test_commit_captures_result_at_commit_time(cache): + """A mutated argument passed back out is stored in its *final* state.""" + xs = [1, 2] + call = make_call(x=xs) + prepared = cache.prepare(call) + xs.append(3) # the "function body" mutates the argument + key = prepared.commit(xs) + loaded = cache.load(key) + assert loaded.result == [1, 2, 3] # result: final state + assert loaded.arguments["x"] == [1, 2] # argument: initial state + + +def test_abandon_leaves_no_record(cache): + call = make_call(x=1) + prepared = cache.prepare(call) + prepared.abandon() + assert not cache.contains(prepared.key) + + +def test_context_manager_abandons_without_commit(cache): + call = make_call(x=1) + with cache.prepare(call) as prepared: + pass + assert not cache.contains(prepared.key) + + +def test_context_manager_commit_sticks(cache): + call = make_call(x=1) + with cache.prepare(call) as prepared: + prepared.commit(42) + assert cache.load(prepared.key).result == 42 + + +def test_context_manager_does_not_swallow_exceptions(cache): + call = make_call(x=1) + with pytest.raises(RuntimeError): + with cache.prepare(call) as prepared: + raise RuntimeError("body failed") + assert not cache.contains(prepared.key) + + +def test_readonly_prepare_is_digest_only_and_commit_rejects(cache): + """A read-only cache admits the call without writing anything; the + rejection lands at commit time, after the body would have run.""" + call = make_call(x=[1, 2]) + prepared = cache.readonly().prepare(call) + assert prepared.key == call.to_lookup_key() + assert list(cache.values.list()) == [] # nothing was stashed + with pytest.raises(Rejected): + prepared.commit(42) + assert not cache.contains(prepared.key) + + +def test_key_matches_one_shot_save(cache): + """prepare/commit and the legacy one-shot save file under the same key.""" + call = make_call(x={"a": 1}) + call.result = "r" + one_shot = Cache(ValueMemory({}), CallMemory({})) + assert cache.prepare(call).commit("r") == one_shot.save(call) + + +# ---- end-to-end: argument mutation no longer corrupts identity ---- + + +def test_mutating_consumer_hits_on_honest_repeat(): + """A function that mutates its argument is keyed on the *pre-call* state: + a repeat call with the same initial state is a hit.""" + fl.cache("memory") + runs = [] + + @fleche + def consume(xs: list): + runs.append(1) + xs.append(99) + return sum(xs) + + assert consume([1, 2]) == 102 + assert consume([1, 2]) == 102 + assert len(runs) == 1 + + +def test_no_mislabeled_entry_for_mutated_state(): + """The post-mutation argument state is a *different* call and recomputes — + the old behavior filed the first call under this state (a false hit that + returned a result computed from different input).""" + fl.cache("memory") + runs = [] + + @fleche + def consume(xs: list): + runs.append(1) + xs.append(99) + return len(xs) + + assert consume([1, 2]) == 3 + assert consume([1, 2, 99]) == 4 # miss: its own honest computation + assert len(runs) == 2 + + +def test_body_exception_leaves_no_record(): + fl.cache("memory") + runs = [] + + @fleche + def boom(x): + runs.append(1) + raise ValueError("no") + + for _ in range(2): + with pytest.raises(ValueError): + boom(1) + assert len(runs) == 2 # nothing cached, body ran twice + assert not boom.contains(1) diff --git a/tests/unit/config/test_cache_to_config.py b/tests/unit/config/test_cache_to_config.py index dc15177a..78584e95 100644 --- a/tests/unit/config/test_cache_to_config.py +++ b/tests/unit/config/test_cache_to_config.py @@ -112,6 +112,9 @@ def load(self, key): def load_value(self, key): raise KeyError(key) + def save_value(self, value): + return "" + def evict(self, key): pass diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 80e02c7a..0ccf5da2 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -229,6 +229,7 @@ def test_dispatch_covers_every_registered_method(): "save", "load", "load_value", + "save_value", "evict", "contains", "expand", From a37ef218ccf2d7012d8974cd4c7bd787f20f5165 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sun, 21 Jun 2026 19:42:22 -0400 Subject: [PATCH 02/27] feat(storage): generalize DigestedDict to DigestedMapping Rename DigestedDict to DigestedMapping and make it generic over any Mapping rather than only dict, preserving the concrete mapping type (e.g. OrderedDict) and subclass identity through a save/load round-trip. A backward-compatible DigestedDict alias is kept (re-exported from fleche.caches and fleche.storage.destructuring) so existing imports keep working. Co-Authored-By: Claude Opus 4.8 --- src/fleche/caches.py | 4 +- src/fleche/storage/destructuring.py | 74 ++++++-- .../storage/test_destructuring_storage.py | 159 +++++++++++++++++- 3 files changed, 212 insertions(+), 25 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index fe62e668..7a214532 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -28,7 +28,9 @@ class Rejected(Exception): # backwards compat imports # from breaking introduced in 0.4.0 DigestedIterable = storage.destructuring.DigestedIterable -DigestedDict = storage.destructuring.DigestedDict +DigestedMapping = storage.destructuring.DigestedMapping +# DigestedDict was renamed to DigestedMapping; keep the old name re-exported. +DigestedDict = storage.destructuring.DigestedMapping class BaseCache(OperationContext): diff --git a/src/fleche/storage/destructuring.py b/src/fleche/storage/destructuring.py index 9677fff6..b2b682ff 100644 --- a/src/fleche/storage/destructuring.py +++ b/src/fleche/storage/destructuring.py @@ -1,6 +1,7 @@ import hashlib from abc import ABC, abstractmethod -from collections import Counter +from collections import Counter, OrderedDict +from collections.abc import Mapping import dataclasses as _dataclasses from dataclasses import dataclass from numbers import Number @@ -71,10 +72,17 @@ def get(storage, key): else: return key + @dataclass class DigestedIterable(Digested): items: list | tuple + def __repr__(self): + # Surface the inner container type (list / tuple / subclass) which is + # part of the wrapper's identity but invisible in a bare [...] / (...) repr. + inner = list.__repr__(list(self.items)) + return f"DigestedIterable[{type(self.items).__name__}]({inner})" + def underlying(self): return self.items @@ -95,32 +103,58 @@ def _rebuild_digest(cls, value: list | tuple, labels: tuple, children: tuple) -> @dataclass -class DigestedDict(Digested): - items: dict +class DigestedMapping(Digested): + """Marker for a destructured mapping, preserving the concrete mapping type. + + Reconstruction (:meth:`_rebuild_plain`, :meth:`mend`) rebuilds via + ``type(value)()``, so only mapping types + whose constructor accepts an iterable of pairs round-trip correctly + (``dict``, ``OrderedDict``, plain dict subclasses, ...). Types that break + this contract — e.g. ``defaultdict`` (first argument is a factory, raises) + or ``Counter`` (would *count* the pairs instead) — must not be sundered. + Which mappings are destructured is controlled by the exact-type allowlist + in ``_DESTRUCTURERS`` (currently ``dict`` and ``OrderedDict``); every other + mapping type is stored verbatim as an opaque value unless a handler is + registered via :func:`register_destructurer`. + """ + + items: Mapping + + def __repr__(self): + # The inner type (dict / OrderedDict / DirectoryBlob / ...) is part of the + # wrapper's identity but lost by ``dict``'s ``{...}`` repr. Surface it. + return f"DigestedMapping[{type(self.items).__name__}]({dict.__repr__(self.items)})" def underlying(self): return self.items - def mend(self, storage: 'DestructuringMixin') -> dict: - return {self.get(storage, k): self.get(storage, v) - for k, v in self.items.items()} + def mend(self, storage: 'DestructuringMixin') -> Mapping: + return type(self.items)( + (self.get(storage, k), self.get(storage, v)) + for k, v in self.items.items() + ) @classmethod - def _slots(cls, value: dict) -> list[tuple[None, Any]]: + def _slots(cls, value: Mapping) -> list[tuple[None, Any]]: # keys and values interned as one flat sequence; _rebuild_* re-pairs them by # position using len(value) as the key/value split point. return [(None, k) for k in value] + [(None, v) for v in value.values()] @classmethod - def _rebuild_plain(cls, value: dict, labels: tuple, children: tuple) -> dict: + def _rebuild_plain(cls, value: Mapping, labels: tuple, children: tuple) -> Mapping: n = len(value) - return dict(zip(children[:n], children[n:])) + return type(value)(zip(children[:n], children[n:])) @classmethod - def _rebuild_digest(cls, value: dict, labels: tuple, children: tuple) -> 'DigestedDict': + def _rebuild_digest(cls, value: Mapping, labels: tuple, children: tuple) -> 'DigestedMapping': return cls(cls._rebuild_plain(value, labels, children)) +# Backward-compatible alias: DigestedMapping was renamed from DigestedDict when it +# was generalized from dict to any Mapping. Keep the old name importable. +DigestedDict = DigestedMapping + + @dataclass class DigestedFields(Digested): """Common base for record-shaped value markers (dataclasses, attrs). @@ -195,8 +229,15 @@ def _field_items(value: Any) -> list[tuple[str, Any]]: _DESTRUCTURERS: list[tuple[Callable[[Any], bool], Callable]] = [ - (lambda v: isinstance(v, (list, tuple)), DigestedIterable.sunder), - (lambda v: isinstance(v, dict), DigestedDict.sunder), + # Exact types (not isinstance): reconstruction goes through + # ``type(value)()``, a contract subclasses may repurpose — + # defaultdict's first argument is a factory (crashes), Counter *counts* the + # pairs (silently wrong), namedtuples reject a single iterable. Subclasses + # are therefore stored verbatim as opaque values unless a handler is opted + # in via register_destructurer(). Dataclasses/attrs are exempt from this + # concern: their mend bypasses __init__ entirely. + (lambda v: type(v) in (list, tuple), DigestedIterable.sunder), # noqa: E721 + (lambda v: type(v) in (dict, OrderedDict), DigestedMapping.sunder), # noqa: E721 (lambda v: _dataclasses.is_dataclass(v) and not isinstance(v, type), DigestedDataclass.sunder), (_attrs.is_attrs_instance, DigestedAttrs.sunder), ] @@ -210,7 +251,10 @@ def register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None: :meth:`DestructuringMixin._intern_rec`. Entries are appended after the built-in ones; first match wins, so registering a handler for an entirely new container type is safe without displacing list/dict/dataclass/attrs. - Call before any :class:`DestructuringMixin` instance is used. + The built-in predicates match exact types only, so a handler for a subclass + (e.g. ``defaultdict`` with a picklable factory) can also be registered + without conflict. Call before any :class:`DestructuringMixin` instance is + used. """ _DESTRUCTURERS.append((pred, fn)) @@ -326,7 +370,7 @@ def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: match raw: case DigestedIterable(): return {i for i in raw.items if isinstance(i, digest.Digest)} - case DigestedDict(): + case DigestedMapping(): return { x for pair in raw.items.items() @@ -355,7 +399,7 @@ def count_reuses(self) -> Counter[digest.Digest]: """Return a counter of how many times each stored key is referenced as a sub-component. Scans every raw entry and tallies ``Digest`` back-references found inside - :class:`DigestedIterable` and :class:`DigestedDict` wrappers. A count of ``0`` + :class:`DigestedIterable` and :class:`DigestedMapping` wrappers. A count of ``0`` means the key is not pointed to by any other stored value (i.e. a top-level entry). A count greater than ``1`` indicates a sub-value shared between multiple parent containers. diff --git a/tests/unit/storage/test_destructuring_storage.py b/tests/unit/storage/test_destructuring_storage.py index 1ca1cee5..b832ea3f 100644 --- a/tests/unit/storage/test_destructuring_storage.py +++ b/tests/unit/storage/test_destructuring_storage.py @@ -1,10 +1,10 @@ import pytest -from collections import namedtuple +from collections import namedtuple, Counter, defaultdict, OrderedDict from dataclasses import dataclass from hypothesis import given, settings, HealthCheck, strategies as st from fleche.storage import ValueMixin, DestructuringMixin from fleche.storage.memory import MemoryBackend -from fleche.storage.destructuring import DigestedIterable, DigestedDict, Digested +from fleche.storage.destructuring import DigestedIterable, DigestedMapping, Digested from fleche.digest import digest, Digest from tests.strategies import st_base_values, st_nested_values, st_key_values, namedtuples @@ -87,19 +87,19 @@ def test_digest_transparency_iterable_mixed(container, items): @given(st.dictionaries(st_key_values, st_base_values, min_size=1, max_size=6)) def test_digest_transparency_dict_all_digests(d): - """DigestedDict whose keys and values are all Digests hashes like the original dict.""" - dd = DigestedDict({digest(k): digest(v) for k, v in d.items()}) + """DigestedMapping whose keys and values are all Digests hashes like the original dict.""" + dd = DigestedMapping({digest(k): digest(v) for k, v in d.items()}) assert digest(dd) == digest(d) @given(st.dictionaries(st_key_values, st_base_values, min_size=2, max_size=6)) def test_digest_transparency_dict_mixed(d): - """DigestedDict with mixed plain and Digest entries still hashes like the original.""" + """DigestedMapping with mixed plain and Digest entries still hashes like the original.""" items = list(d.items()) # inline first key-value pair, digest the rest mixed = {items[0][0]: items[0][1]} mixed.update({digest(k): digest(v) for k, v in items[1:]}) - dd = DigestedDict(mixed) + dd = DigestedMapping(mixed) assert digest(dd) == digest(d) @@ -122,11 +122,11 @@ def test_digested_iterable_mend_roundtrip(container, ds, items): @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) @given(st.dictionaries(st.text(), st_base_values, min_size=1, max_size=6)) def test_digested_dict_mend_roundtrip(ds, d): - """DigestedDict stored by ds can be re-assembled via mend.""" + """DigestedMapping stored by ds can be re-assembled via mend.""" key = ds.save(d) # Access the raw stored value directly from the backend dict (bypasses mend) raw = ds.storage[key] - assert isinstance(raw, DigestedDict) + assert isinstance(raw, DigestedMapping) assert raw.mend(ds) == d @@ -257,7 +257,7 @@ def test_plain_container_stored_when_all_inline(container): def test_plain_dict_stored_when_all_inline(): - """When all dict entries are inlined, the dict is stored without a DigestedDict wrapper.""" + """When all dict entries are inlined, the dict is stored without a DigestedMapping wrapper.""" ds = make_ds(remaining_depth=10) data = {"a": 1, "b": [2, 3]} key = ds.save(data) @@ -452,3 +452,144 @@ def test_count_reuses_nonnegative(value): ds.save(value) hits = ds.count_reuses() assert all(v >= 0 for v in hits.values()) + + +# ---- Mapping subclass destructuring ---- +# DigestedMapping is generic over Mapping, mirroring how DigestedIterable is +# generic over list/tuple: ``sunder`` and ``mend`` reconstruct via +# ``type(value)(...)``. Only the exact types on the _DESTRUCTURERS allowlist +# (dict, OrderedDict) are destructured; subclasses may repurpose that +# constructor (defaultdict, Counter) and are stored verbatim as opaque values. + + +def test_ordereddict_roundtrip(ds): + """OrderedDict round-trips with its type and order preserved.""" + od = OrderedDict([("b", 1), ("a", 2), ("c", 3)]) + key = ds.save(od) + loaded = ds.load(key) + assert loaded == od + assert type(loaded) is OrderedDict + assert list(loaded) == ["b", "a", "c"] + + +def test_ordereddict_is_destructured(ds): + """OrderedDict is wrapped in a DigestedMapping whose items preserve the mapping type.""" + od = OrderedDict([("a", 1), ("b", [2, 3])]) + key = ds.save(od) + raw = ds.storage[key] + assert isinstance(raw, DigestedMapping) + assert type(raw.items) is OrderedDict + + +def test_ordereddict_digest_transparency(): + """DigestedMapping wrapping an OrderedDict hashes like the original OrderedDict.""" + od = OrderedDict([("a", 1), ("b", 2)]) + dd = DigestedMapping(OrderedDict((digest(k), digest(v)) for k, v in od.items())) + assert digest(dd) == digest(od) + + +def test_ordereddict_digest_distinct_from_dict(): + """OrderedDict and dict with the same entries hash differently — verify we preserve that.""" + od = OrderedDict([("a", 1), ("b", 2)]) + d = {"a": 1, "b": 2} + assert digest(od) != digest(d) + ds_local = make_ds() + assert ds_local.save(od) != ds_local.save(d) + + +def test_empty_ordereddict_roundtrip(ds): + """An empty OrderedDict is stored verbatim and round-trips as OrderedDict.""" + od = OrderedDict() + key = ds.save(od) + raw = ds.storage[key] + assert not isinstance(raw, Digested) + loaded = ds.load(key) + assert loaded == od + assert type(loaded) is OrderedDict + + +def test_ordereddict_mend_preserves_type(ds): + """The raw DigestedMapping's mend reconstructs an OrderedDict, not a plain dict.""" + od = OrderedDict([("a", 1), ("b", 2)]) + key = ds.save(od) + raw = ds.storage[key] + assert isinstance(raw, DigestedMapping) + mended = raw.mend(ds) + assert type(mended) is OrderedDict + assert mended == od + + +def test_ordereddict_nested_in_dict_roundtrip(ds): + """An OrderedDict nested in a regular dict is preserved through destructuring.""" + data = {"inner": OrderedDict([("x", 1), ("y", 2)]), "other": 42} + key = ds.save(data) + loaded = ds.load(key) + assert loaded == data + assert type(loaded["inner"]) is OrderedDict + assert list(loaded["inner"]) == ["x", "y"] + + +def test_dict_nested_in_ordereddict_roundtrip(ds): + """A plain dict value inside an OrderedDict round-trips with both types preserved.""" + data = OrderedDict([("a", {"x": 1, "y": 2}), ("b", 3)]) + key = ds.save(data) + loaded = ds.load(key) + assert loaded == data + assert type(loaded) is OrderedDict + assert type(loaded["a"]) is dict + + +# ---- Off-allowlist container types are opaque ---- + + +def test_defaultdict_is_opaque(ds): + """defaultdict is off the allowlist: its constructor wants a factory, not pairs.""" + dd = defaultdict(list, {"a": [1], "b": [2, 3]}) + key = ds.save(dd) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is defaultdict + assert loaded == dd + + +def test_counter_is_opaque(ds): + """Counter is off the allowlist: its constructor would count the pairs.""" + cnt = Counter({"a": 5, "b": 6}) + key = ds.save(cnt) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is Counter + assert loaded == cnt + + +def test_dict_subclass_is_opaque(ds): + """Even a well-behaved dict subclass is opaque until opted in explicitly.""" + + class MyDict(dict): + pass + + md = MyDict({"a": 1, "b": [2, 3]}) + key = ds.save(md) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is MyDict + assert loaded == md + + +def test_list_subclass_is_opaque(ds): + """List subclasses are opaque too — same exact-type rule as mappings.""" + + class MyList(list): + pass + + ml = MyList([1, [2, 3]]) + key = ds.save(ml) + assert not isinstance(ds.storage[key], Digested) + loaded = ds.load(key) + assert type(loaded) is MyList + assert loaded == ml + + +# DirectoryBlob is now an opaque type owned by PathValueMixin (see +# tests/unit/storage/test_paths.py for its end-to-end coverage). DestructuringMixin +# leaves it alone — no match arm catches it — so it has no presence here. From d917df3263f05de1d48bb0ba9cc889dbd01d8ad7 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sun, 21 Jun 2026 19:42:22 -0400 Subject: [PATCH 03/27] feat(storage): cache filesystem paths by content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store and retrieve pathlib.Path values by their content rather than as opaque path strings, so caching a function that takes or returns a Path is portable and reproducible across machines. Following git's split — content is content-addressed, names live in trees: - a file is keyed on (name, content): it keeps its name and extension on a cache hit, and its content (plain bytes) deduplicates — shared across names and with bytes values. To key on content alone, return bytes. - a directory is keyed on its tree alone; its incidental root name is dropped (a reloaded directory is named by its digest, its children by their real names). PathValueMixin owns the traversal and is wired into the default value storages (memory, pickle, H5) between DestructuringMixin and ValueMixin: a file becomes a FileBlob(name, content) record, a directory a DirectoryBlob tree, and content lives in plain bytes blobs. The digest Path arm mirrors storage so digest(path) == values.save(path) — the invariant cached lookups of path arguments and results depend on. Co-Authored-By: Claude Opus 4.8 --- src/fleche/digest.py | 44 +++ src/fleche/storage/__init__.py | 5 + src/fleche/storage/bagofholding_file.py | 3 +- src/fleche/storage/memory.py | 3 +- src/fleche/storage/paths.py | 212 +++++++++++ src/fleche/storage/pickle_file.py | 3 +- src/fleche/storage/void.py | 4 + tests/integration/test_paths_workflow.py | 151 ++++++++ tests/unit/digest/test_digest_paths.py | 171 +++++++++ tests/unit/storage/test_paths.py | 329 ++++++++++++++++++ .../unit/storage/test_paths_default_wiring.py | 253 ++++++++++++++ 11 files changed, 1175 insertions(+), 3 deletions(-) create mode 100644 src/fleche/storage/paths.py create mode 100644 tests/integration/test_paths_workflow.py create mode 100644 tests/unit/digest/test_digest_paths.py create mode 100644 tests/unit/storage/test_paths.py create mode 100644 tests/unit/storage/test_paths_default_wiring.py diff --git a/src/fleche/digest.py b/src/fleche/digest.py index 91a80884..f4a36537 100644 --- a/src/fleche/digest.py +++ b/src/fleche/digest.py @@ -6,6 +6,7 @@ import numbers from numbers import Number import struct +from pathlib import Path import types import importlib.metadata from collections.abc import Iterable, Mapping @@ -154,6 +155,29 @@ def digest(value: Any) -> Digest: return Digest(_digest_bytes(value).decode()) +def _path_content_digest(path) -> Digest: + """Content-only digest of a path: a file as its bytes, a directory as its + ``{name: child}`` tree (the directory's own name excluded). + + This mirrors exactly what :class:`~fleche.storage.paths.PathValueMixin` + stores for a file's content blob and for a :class:`DirectoryBlob`, so a + directory tree and a standalone file agree on their children's digests. A + *standalone* file additionally carries its basename (see the ``Path`` arm of + :func:`_digest_bytes`); inside a directory the name lives in the parent's + ``contents`` key, so only the content matters here. + """ + if path.is_file(): + return digest(path.read_bytes()) + if path.is_dir(): + contents = { + child.name: _path_content_digest(child) + for child in path.iterdir() + if child.is_file() or child.is_dir() + } + return digest(("DirectoryBlob", contents)) + raise Indigestible("Can only digest files and folders!") + + def _digest_bytes(value: Any) -> bytes: """ Returns bytes representing the SHA-256 digest of *value*. @@ -232,6 +256,26 @@ def _digest_bytes(value: Any) -> bytes: return _digest_bytes(value) case bytes(): m.update(value) + case Path(): + # A *file* is identified by (basename, content): its name (and + # extension) matter, but the content deduplicates. A *directory* is + # identified by its tree alone — its own (often incidental, e.g. a + # temp dir) root name is dropped, though child names are kept. Both + # mirror exactly what PathValueMixin stores, preserving the + # digest(path) == values.save(path) invariant on which cache lookups + # of path-valued arguments/results depend. The "FileBlob" / + # "DirectoryBlob" salts MUST match + # fleche.storage.paths.{FileBlob,DirectoryBlob}.__digest__. + if not value.exists(): + raise Indigestible("Only existing paths can be digested.") + if value.is_file(): + return _digest_bytes( + ("FileBlob", value.name, _path_content_digest(value)) + ) + elif value.is_dir(): + return _path_content_digest(value).encode() + else: + raise Indigestible("Can only digest files and folders!") case np.ndarray(): m.update(_digest_bytes(value.dtype.str)) m.update(_digest_bytes(value.shape)) diff --git a/src/fleche/storage/__init__.py b/src/fleche/storage/__init__.py index de1e4ea0..59ab51e7 100644 --- a/src/fleche/storage/__init__.py +++ b/src/fleche/storage/__init__.py @@ -17,6 +17,7 @@ CallMixin, ) from .destructuring import DestructuringMixin, register_destructurer +from .paths import PathValueMixin, TempPath, FileBlob, DirectoryBlob from .memory import ValueMemory, CallMemory from .void import ValueVoid, CallVoid from .file import FileStorage @@ -38,6 +39,10 @@ "CallMixin", "DestructuringMixin", "register_destructurer", + "PathValueMixin", + "TempPath", + "FileBlob", + "DirectoryBlob", "ValueMemory", "CallMemory", "ValueVoid", diff --git a/src/fleche/storage/bagofholding_file.py b/src/fleche/storage/bagofholding_file.py index dea6af0d..3a39050e 100644 --- a/src/fleche/storage/bagofholding_file.py +++ b/src/fleche/storage/bagofholding_file.py @@ -8,6 +8,7 @@ from .base import SaveError, ValueMixin, CallMixin from .thread_safe import PerKeyLockMixin from .destructuring import DestructuringMixin +from .paths import PathValueMixin from ..digest import Digest, DIGEST_LENGTH from pyiron_snippets.import_alarm import ImportAlarm @@ -314,7 +315,7 @@ def rebag(self, version_validator: VersionValidator = "none") -> None: @dataclass(frozen=True) -class ValueBagOfHoldingH5File(PerKeyLockMixin, DestructuringMixin, ValueMixin, BagOfHoldingH5FileBackend): ... +class ValueBagOfHoldingH5File(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, BagOfHoldingH5FileBackend): ... @dataclass(frozen=True) class CallBagOfHoldingH5File(PerKeyLockMixin, CallMixin, BagOfHoldingH5FileBackend): ... diff --git a/src/fleche/storage/memory.py b/src/fleche/storage/memory.py index 29255c71..3afae4c2 100644 --- a/src/fleche/storage/memory.py +++ b/src/fleche/storage/memory.py @@ -3,6 +3,7 @@ from .base import ValueMixin, CallMixin, StorageBackend from .destructuring import DestructuringMixin +from .paths import PathValueMixin from .thread_safe import PerKeyLockMixin from ..digest import Digest from copy import deepcopy @@ -50,7 +51,7 @@ def _evict(self, key: Digest) -> None: @dataclass(frozen=True) -class ValueMemory(PerKeyLockMixin, DestructuringMixin, ValueMixin, MemoryBackend): +class ValueMemory(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, MemoryBackend): __hash__ = object.__hash__ @dataclass(frozen=True) diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py new file mode 100644 index 00000000..6d0e13b8 --- /dev/null +++ b/src/fleche/storage/paths.py @@ -0,0 +1,212 @@ +import os +import sys +from pathlib import Path +from typing import Any +import tempfile +import weakref + +from . import base +from .. import digest + + +class TempPath(type(Path())): + """ + A Path that deletes its backing temp tree when no references remain. + Paths derived via /, .parent, .with_suffix, etc. share the same + TemporaryDirectory and keep it alive collectively. + + On 3.12+ propagation rides ``with_segments``, the single hook pathlib + routes every derived path through. 3.11 has no such instance hook — its + derivation goes through the *classmethods* ``_from_parsed_parts`` / + ``_from_parts``, which never see the originating instance — so there the + class keeps a weak registry of live temp roots and re-attaches the + matching ``TemporaryDirectory`` to any path constructed under one. The + registry holds only weak references: instances alone keep a root alive, + so cleanup semantics are identical on both versions. + """ + + # str(root dir) -> TemporaryDirectory, weakly; used by the 3.11 branch only. + _live_roots: "weakref.WeakValueDictionary[str, tempfile.TemporaryDirectory]" = ( + weakref.WeakValueDictionary() + ) + + @classmethod + def mkdtemp( + cls, + ) -> "TempPath": + root = tempfile.TemporaryDirectory( + suffix="fleche", + ignore_cleanup_errors=True, + ) + obj = cls(root.name) + object.__setattr__(obj, "_temp_root", root) + if sys.version_info < (3, 12): + cls._live_roots[str(obj)] = root + return obj + + if sys.version_info >= (3, 12): + + def with_segments(self, *pathsegments): + new = super().with_segments(*pathsegments) + root = getattr(self, "_temp_root", None) + if root is not None: + object.__setattr__(new, "_temp_root", root) + return new + + else: + + @classmethod + def _adopt_live_root(cls, new: "TempPath") -> "TempPath": + path_str = str(new) + for root_str, root in list(cls._live_roots.items()): + if path_str == root_str or path_str.startswith(root_str + os.sep): + object.__setattr__(new, "_temp_root", root) + break + return new + + @classmethod + def _from_parsed_parts(cls, drv, root, parts): + return cls._adopt_live_root(super()._from_parsed_parts(drv, root, parts)) + + @classmethod + def _from_parts(cls, args): + return cls._adopt_live_root(super()._from_parts(args)) + + +class FileBlob: + """A stored file: a basename paired with a reference to its content. + + A file is identified by *(name, content)*. The content ``bytes`` are stored + once under their own content digest — so identical bodies deduplicate across + names, and even with plain ``bytes`` values — and this small record pairs + that content reference with the file's basename. On load it materializes at + ``/`` so ``.name`` / ``.suffix`` / ``.stem`` are faithful, and + a downstream consumer receives an ordinary ``Path`` with the right name. + + To store file content *without* a name (content only, maximal reuse), return + the plain ``bytes`` instead of a ``Path``. + + Plain ``__dict__`` (no ``__slots__``) so H5 can reconstruct it; the + ``"FileBlob"`` salt in :meth:`__digest__` MUST match ``fleche.digest``'s file + ``Path`` arm. + """ + + def __init__(self, name, content): + self.name = name + self.content = content # Digest of the file's content bytes + + def __eq__(self, other): + return isinstance(other, FileBlob) and (self.name, self.content) == ( + other.name, + other.content, + ) + + __hash__ = None # mutable; not intended as a dict key + + def __digest__(self): + return digest.digest(("FileBlob", self.name, self.content)) + + def __repr__(self): + return f"FileBlob({self.name!r}, {self.content!r})" + + +class DirectoryBlob: + """A stored directory: ``{name: content_ref}``, keyed by its tree alone. + + A directory's *root* name is **not** part of its identity — directories hash + by their content (the tree) — but its child names are (they are the dict + keys). Each child is a content reference: a file child to its content + ``bytes``, a subdirectory child to its own :class:`DirectoryBlob`. A + reloaded directory is therefore named by its digest, its children by their + real names. + + Stored verbatim by :class:`~fleche.storage.destructuring.DestructuringMixin` + (not a dict subclass, not a dataclass, so no match arm catches it). + Kept as a plain ``__dict__``-backed class (no ``__slots__``): some backends + reconstruct via ``obj.__dict__.update(state)`` (e.g. bagofholding's H5 + unpacker), which a slots-only object cannot satisfy. + """ + + def __init__(self, contents): + self.contents = dict(contents) + + def __eq__(self, other): + return isinstance(other, DirectoryBlob) and self.contents == other.contents + + __hash__ = None # mutable; not intended as a dict key + + def __digest__(self): + # Digest a (type_name, payload) tuple — the codebase idiom for custom + # digests. The "DirectoryBlob" element salts the hash so this is not + # digest-equal to a plain dict carrying the same {name: Digest} mapping. + return digest.digest(("DirectoryBlob", self.contents)) + + def __repr__(self): + return f"DirectoryBlob({self.contents!r})" + + +class PathValueMixin(base.ValueStorage): + """Convert :class:`~pathlib.Path` values to blobs (and back). + + A **file** is stored as its content ``bytes`` (deduplicated under the content + digest) wrapped in a :class:`FileBlob` carrying the basename — so files are + keyed by *(name, content)* and a cache hit returns a path with its real name. + A **directory** is stored as a :class:`DirectoryBlob` keyed by its tree only; + its root name is dropped (a reloaded directory is named by its digest, its + children by their real names). Plain ``bytes`` are the way to store file + content without a name. + + The traversal never goes through ``self.save`` / ``self.load`` — every + storage call uses ``super()`` — so this mixin composes cleanly with + :class:`~fleche.storage.destructuring.DestructuringMixin` above it without + load-context ambiguity. Compose **below** ``DestructuringMixin`` in the MRO + so ``super().save`` from Destructure's recursion lands here for nested Paths. + """ + + def save(self, value: Any, key: digest.Digest | None = None) -> digest.Digest: + if isinstance(value, Path): + if value.is_file(): + # Content deduplicates as plain bytes; the FileBlob record adds + # the name (the file's key is digest(name, content)). + content = super().save(value.read_bytes()) + return super().save(FileBlob(value.name, content), key) + if value.is_dir(): + return super().save(self._build(value), key) + return super().save(value, key) + + def _build(self, p: Path) -> DirectoryBlob: + contents = {} + for child in sorted(p.iterdir()): + if child.is_file(): + contents[child.name] = super().save(child.read_bytes()) + elif child.is_dir(): + contents[child.name] = super().save(self._build(child)) + return DirectoryBlob(contents) + + def load(self, key: digest.Digest | str) -> Any: + value = super().load(key) + if isinstance(value, FileBlob): + # Materialize at / so the basename round-trips. + target = TempPath.mkdtemp() / value.name + target.write_bytes(super().load(value.content)) + return target + if isinstance(value, DirectoryBlob): + # No stored root name — materialize under the digest (mangled root). + path = TempPath.mkdtemp() / str(key) + self._materialize(path, value) + return path + return value + + def _materialize(self, path: Path, blob: DirectoryBlob) -> None: + path.mkdir() + for name, child_ref in blob.contents.items(): + child = super().load(child_ref) + if isinstance(child, DirectoryBlob): + self._materialize(path / name, child) + elif isinstance(child, (bytes, bytearray)): + (path / name).write_bytes(child) + else: + raise TypeError( + f"directory entry {name!r} resolved to unexpected type " + f"{type(child).__name__}" + ) diff --git a/src/fleche/storage/pickle_file.py b/src/fleche/storage/pickle_file.py index b83b52a0..3750e794 100644 --- a/src/fleche/storage/pickle_file.py +++ b/src/fleche/storage/pickle_file.py @@ -10,6 +10,7 @@ from .base import ValueMixin, CallMixin from .thread_safe import PerKeyLockMixin from .destructuring import DestructuringMixin +from .paths import PathValueMixin from ..security import get_secret_key, normalize_secret_key, SignedBytes, SignatureError from pyiron_snippets.import_alarm import ImportAlarm @@ -113,7 +114,7 @@ def decompress_all(self) -> None: @dataclass(frozen=True) -class ValuePickleFile(PerKeyLockMixin, DestructuringMixin, ValueMixin, PickleFileBackend): ... +class ValuePickleFile(PerKeyLockMixin, DestructuringMixin, PathValueMixin, ValueMixin, PickleFileBackend): ... @dataclass(frozen=True) class CallPickleFile(PerKeyLockMixin, CallMixin, PickleFileBackend): ... diff --git a/src/fleche/storage/void.py b/src/fleche/storage/void.py index 967ae70d..02821d84 100644 --- a/src/fleche/storage/void.py +++ b/src/fleche/storage/void.py @@ -27,6 +27,10 @@ def _contains(self, key: Digest) -> bool: return False +# ValueVoid is the degenerate "store nothing" value storage: it deliberately omits +# DestructuringMixin and, for the same reason, PathValueMixin. Since VoidBackend +# discards on put and raises KeyError on get, layering Path-to-blob conversion would +# only build-then-throw-away blobs and could never materialize a Path on load. @dataclass(frozen=True) class ValueVoid(ValueMixin, VoidBackend): ... diff --git a/tests/integration/test_paths_workflow.py b/tests/integration/test_paths_workflow.py new file mode 100644 index 00000000..a36f1118 --- /dev/null +++ b/tests/integration/test_paths_workflow.py @@ -0,0 +1,151 @@ +"""End-to-end workflow tests for Path-by-content caching. + +These mirror the canonical "functions that accept and produce files and +directories via paths" scenario (see ``notebooks/Files.ipynb``) but assert +*correctness* rather than merely "runs without error": cache hits are proven by +counting body executions, and content-addressed deduplication is proven by +inspecting the value store. + +Parametrized over an in-memory and an on-disk (pickle) cache so the opaque +``FileBlob`` / ``DirectoryBlob`` records are exercised through real +serialization, not just deepcopy. +""" + +from pathlib import Path + +import pytest + +from fleche import fleche, cache +from fleche.digest import digest +from fleche.caches import Cache +from fleche.storage import ( + ValueMemory, + CallMemory, + ValuePickleFile, + CallPickleFile, +) + + +@pytest.fixture(params=["memory", "cloudpickle"]) +def paths_cache(request, tmp_path): + if request.param == "memory": + return Cache(ValueMemory({}), CallMemory({})) + return Cache( + ValuePickleFile.with_cloudpickle(tmp_path / "values"), + CallPickleFile.with_cloudpickle(tmp_path / "calls"), + ) + + +def _relmap(root: Path) -> dict[str, bytes]: + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +def test_file_producer_runs_once_and_roundtrips(paths_cache, tmp_path): + """A cached file producer executes once; the cached result is a readable Path.""" + runs = [] + + @fleche + def produce(content): + runs.append(content) + f = tmp_path / f"out-{content}.txt" + f.write_text(content) + return f + + with cache(paths_cache): + first = produce("payload") + assert isinstance(first, Path) + assert first.read_text() == "payload" + + second = produce("payload") + assert isinstance(second, Path) + assert second.read_text() == "payload" + + assert runs == ["payload"], "second call should have been served from cache" + + +def test_directory_producer_roundtrips_tree(paths_cache, tmp_path): + """A cached directory producer round-trips a full nested tree by content.""" + runs = [] + + @fleche + def build(seed): + runs.append(seed) + d = tmp_path / f"tree-{seed}" + d.mkdir() + (d / "top.txt").write_text(seed) + (d / "sub").mkdir() + (d / "sub" / "leaf.bin").write_bytes(seed.encode() * 3) + return d + + expected = { + "top.txt": b"seed", + "sub/leaf.bin": b"seedseedseed", + } + + with cache(paths_cache): + first = build("seed") + assert _relmap(first) == expected + + second = build("seed") + assert second.is_dir() + assert _relmap(second) == expected + + assert runs == ["seed"] + + +def test_producer_consumer_chain_caches(paths_cache, tmp_path): + """A Path flowing producer -> consumer: both stages cache on repeat input.""" + writes, parses = [], [] + + @fleche + def write(content): + writes.append(content) + f = tmp_path / f"w-{content}.txt" + f.write_text(content) + return f + + @fleche + def parse(f: Path): + parses.append(f.name) + return f.read_text().upper() + + with cache(paths_cache): + assert parse(write("hi")) == "HI" + # write hits cache (same content) and so does parse (same file content), + # even though the second write returns a freshly materialized TempPath. + assert parse(write("hi")) == "HI" + + assert writes == ["hi"] + assert parses == ["w-hi.txt"] + + +def test_shared_file_body_is_deduplicated(paths_cache, tmp_path): + """A file body shared by two produced directories is reused, not re-stored.""" + shared = b"shared-body" + one, two = b"one-only", b"two-only" + + @fleche + def build(tag, unique): + d = tmp_path / f"d-{tag}" + d.mkdir() + (d / "shared").write_bytes(shared) + (d / "unique").write_bytes(unique) + return d + + with cache(paths_cache): + build("a", one) + keys_after_a = set(paths_cache.values.list()) + assert digest(shared) in keys_after_a, "shared body stored when first dir is produced" + + build("b", two) + keys_after_b = set(paths_cache.values.list()) + + new_keys = keys_after_b - keys_after_a + # The second directory's genuinely new content shows up... + assert digest(two) in new_keys + # ...but the shared body is content-addressed and reused, not re-stored. + assert digest(shared) not in new_keys diff --git a/tests/unit/digest/test_digest_paths.py b/tests/unit/digest/test_digest_paths.py new file mode 100644 index 00000000..b314e6f3 --- /dev/null +++ b/tests/unit/digest/test_digest_paths.py @@ -0,0 +1,171 @@ +"""Coverage for the ``Path`` arm in ``fleche.digest._digest_bytes``. + +Files digest on ``(basename, content)``; directories digest on their tree alone +(a directory's own root name is not part of its identity). +""" +from pathlib import Path + +import pytest + +from fleche.digest import digest, Indigestible + + +# ---- Non-existent paths ---- + + +def test_nonexistent_path_raises_indigestible(tmp_path): + """A Path that doesn't exist on disk cannot be digested.""" + missing = tmp_path / "does-not-exist" + with pytest.raises(Indigestible): + digest(missing) + + +# ---- File paths ---- + + +def test_file_path_digest_is_name_plus_content(tmp_path): + """A file Path digests on (basename, content), mirroring the stored FileBlob record. + + ``digest(path) == digest(FileBlob(name, digest(bytes)))`` — the + ``digest(path) == values.save(path)`` invariant cached lookups rely on — and + ``!= digest(bytes)`` so a file is distinct from a bare bytes value of the + same content (while still deduplicating its content blob; see storage tests). + """ + from fleche.storage.paths import FileBlob + + p = tmp_path / "f.txt" + p.write_bytes(b"hello world") + assert digest(p) == digest(FileBlob("f.txt", digest(b"hello world"))) + assert digest(p) != digest(b"hello world") + + +def test_file_path_digest_changes_with_contents(tmp_path): + """Different file bodies → different digests.""" + p = tmp_path / "f.txt" + p.write_bytes(b"alpha") + d1 = digest(p) + p.write_bytes(b"beta") + d2 = digest(p) + assert d1 != d2 + + +def test_file_path_digest_depends_on_filename(tmp_path): + """Files are keyed on (name, content): same body, different names → different digests. + + Same basename + same content → identical, regardless of the parent directory. + """ + a = tmp_path / "a.txt" + b = tmp_path / "b.txt" + a.write_bytes(b"same") + b.write_bytes(b"same") + assert digest(a) != digest(b) # name is part of a file's identity + + other = tmp_path / "sub" + other.mkdir() + c = other / "a.txt" + c.write_bytes(b"same") + assert digest(c) == digest(a) # same basename + content -> same digest + + +# ---- Directory paths ---- + + +def test_empty_directory_can_be_digested(tmp_path): + """An empty directory is digestible (does not raise).""" + digest(tmp_path) # smoke + + +def test_directory_digest_ignores_root_name(tmp_path): + """A directory hashes by its tree alone — its own root name does not matter.""" + d1 = tmp_path / "alpha" + d2 = tmp_path / "beta" + for d in (d1, d2): + d.mkdir() + (d / "x.txt").write_bytes(b"X") + (d / "sub").mkdir() + (d / "sub" / "y.bin").write_bytes(b"Y") + assert digest(d1) == digest(d2) # different root names, identical trees + + +def test_directory_digest_changes_with_filename(tmp_path): + """The directory arm hashes ``{name: child}``, so renaming a file changes the digest.""" + (tmp_path / "a.txt").write_bytes(b"x") + d1 = digest(tmp_path) + + # Rename and re-hash. + (tmp_path / "a.txt").rename(tmp_path / "b.txt") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_changes_with_file_contents(tmp_path): + """Mutating a child file changes the directory digest.""" + f = tmp_path / "f.txt" + f.write_bytes(b"v1") + d1 = digest(tmp_path) + f.write_bytes(b"v2") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_stable_across_iteration_order(tmp_path): + """Two directories with the same {name: bytes} payload share a digest. + + ``_digest_mapping`` sorts by key-digest, so filesystem ``iterdir`` order + must not leak into the result. + """ + d1 = tmp_path / "d1" + d2 = tmp_path / "d2" + d1.mkdir() + d2.mkdir() + # Write children in different orders. + (d1 / "a").write_bytes(b"A") + (d1 / "b").write_bytes(b"B") + (d2 / "b").write_bytes(b"B") + (d2 / "a").write_bytes(b"A") + assert digest(d1) == digest(d2) + + +def test_nested_directory_digest_recurses(tmp_path): + """Mutating a deeply nested file changes the root directory digest.""" + sub = tmp_path / "sub" / "deeper" + sub.mkdir(parents=True) + leaf = sub / "leaf.txt" + leaf.write_bytes(b"v1") + d1 = digest(tmp_path) + leaf.write_bytes(b"v2") + d2 = digest(tmp_path) + assert d1 != d2 + + +def test_directory_digest_differs_from_plain_dict(tmp_path): + """A directory digest does not collide with a plain dict of {name: path}. + + A directory hashes as the tuple ``("DirectoryBlob", {name: child_digest})``, + whereas a plain dict hashes as a bare mapping — different shapes and salts, + so they differ even though both ultimately reference the same child content. + """ + (tmp_path / "a.txt").write_bytes(b"hello") + plain = {"a.txt": tmp_path / "a.txt"} + assert digest(tmp_path) != digest(plain) + + +# ---- Non-file, non-directory paths ---- + + +def test_special_path_raises_indigestible(tmp_path): + """A path that exists but is neither a regular file nor a directory raises. + + GUESS: covers the ``else`` branch (e.g. symlinks to nowhere, sockets, + FIFOs). A dangling symlink is the most portable trigger: ``exists()`` + returns False for it (so we'd land in the not-exists branch first). Use a + FIFO instead — POSIX only, skip elsewhere. + """ + import os + fifo = tmp_path / "fifo" + try: + os.mkfifo(fifo) + except (AttributeError, NotImplementedError, OSError): + pytest.skip("FIFOs not supported on this platform") + with pytest.raises(Indigestible): + digest(fifo) diff --git a/tests/unit/storage/test_paths.py b/tests/unit/storage/test_paths.py new file mode 100644 index 00000000..8247344a --- /dev/null +++ b/tests/unit/storage/test_paths.py @@ -0,0 +1,329 @@ +import gc +import tempfile +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from fleche.digest import digest +from fleche.storage import ValueMixin, DestructuringMixin +from fleche.storage.memory import MemoryBackend +from fleche.storage.destructuring import DigestedMapping +from fleche.storage.paths import TempPath, PathValueMixin, FileBlob, DirectoryBlob + + +def test_mkdtemp_returns_temp_path(): + p = TempPath.mkdtemp() + assert isinstance(p, TempPath) + + +def test_mkdtemp_directory_exists(): + p = TempPath.mkdtemp() + assert p.exists() + assert p.is_dir() + + +def test_mkdtemp_cleanup_on_del(): + p = TempPath.mkdtemp() + path_str = str(p) + assert Path(path_str).exists() + del p + gc.collect() + assert not Path(path_str).exists() + + +def test_derived_path_shares_temp_root(): + p = TempPath.mkdtemp() + child = p / "subdir" + assert isinstance(child, TempPath) + assert getattr(child, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_derived_path_keeps_temp_dir_alive(): + p = TempPath.mkdtemp() + child = p / "child.txt" + path_str = str(p) + del p + gc.collect() + # child still holds a reference to _temp_root, so directory must still exist + assert Path(path_str).exists() + del child + gc.collect() + assert not Path(path_str).exists() + + +def test_cleanup_after_all_derived_refs_gone(): + p = TempPath.mkdtemp() + a = p / "a" + b = p / "b" / "c" + path_str = str(p) + del p + del a + gc.collect() + assert Path(path_str).exists() + del b + gc.collect() + assert not Path(path_str).exists() + + +def test_path_operations_work(): + p = TempPath.mkdtemp() + child = p / "file.txt" + child.write_text("hello") + assert child.read_text() == "hello" + + +def test_with_suffix_shares_temp_root(): + p = TempPath.mkdtemp() + suffixed = (p / "file").with_suffix(".txt") + assert isinstance(suffixed, TempPath) + assert getattr(suffixed, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_parent_shares_temp_root(): + p = TempPath.mkdtemp() + child = p / "a" / "b" + parent = child.parent + assert isinstance(parent, TempPath) + assert getattr(parent, "_temp_root", None) is getattr(p, "_temp_root", None) + + +def test_multiple_mkdtemp_calls_independent(): + p1 = TempPath.mkdtemp() + p2 = TempPath.mkdtemp() + assert p1 != p2 + path1_str = str(p1) + del p1 + gc.collect() + assert not Path(path1_str).exists() + assert p2.exists() + + +def test_temp_path_is_path_subclass(): + p = TempPath.mkdtemp() + assert isinstance(p, Path) + + +def test_no_temp_root_on_plain_construction(): + # Constructing TempPath without mkdtemp should not set _temp_root + # (and should not raise on attribute access) + with tempfile.TemporaryDirectory() as tmpdir: + p = TempPath(tmpdir) + assert getattr(p, "_temp_root", None) is None + + +def test_derived_from_plain_has_no_temp_root(): + with tempfile.TemporaryDirectory() as tmpdir: + p = TempPath(tmpdir) + child = p / "child" + assert getattr(child, "_temp_root", None) is None + + +# ---- DirectoryBlob / FileBlob basics ---- + + +def test_directoryblob_is_opaque_to_destructuring(): + """DirectoryBlob is neither a dict subclass nor a dataclass — DestructuringMixin leaves it alone.""" + from dataclasses import is_dataclass + assert not isinstance(DirectoryBlob({}), dict) + assert not is_dataclass(DirectoryBlob({})) + + +def test_directoryblob_repr_distinguishes_from_dict(): + """Repr shows the wrapper class explicitly.""" + db = DirectoryBlob({"a": digest(b"x")}) + assert repr(db).startswith("DirectoryBlob(") + nested = DirectoryBlob({"sub": digest(b"y")}) + assert repr(nested).startswith("DirectoryBlob(") + + +def test_directoryblob_digest_distinct_from_plain_dict(): + """DirectoryBlob salts its digest with the class name so it never collides with a bare dict.""" + contents = {"a": digest(b"x"), "b": digest(b"y")} + db = DirectoryBlob(contents) + assert digest(db) != digest(contents) + + +def test_directoryblob_equality_and_unhashable(): + """DirectoryBlobs with equal contents are equal; instances are unhashable.""" + a = DirectoryBlob({"a": digest(b"x")}) + b = DirectoryBlob({"a": digest(b"x")}) + assert a == b + with pytest.raises(TypeError): + hash(a) + + +def test_fileblob_record_digests_on_name_and_content(): + """FileBlob pairs a basename with a content digest; both are part of its identity.""" + c = digest(b"hello") + assert digest(FileBlob("a.txt", c)) != digest(FileBlob("b.txt", c)) # name matters + assert digest(FileBlob("a.txt", c)) == digest(FileBlob("a.txt", c)) # stable + assert digest(FileBlob("a.txt", c)) != c # not the bare content + + +def test_fileblob_equality_repr_unhashable(): + c = digest(b"x") + assert FileBlob("a", c) == FileBlob("a", c) + assert FileBlob("a", c) != FileBlob("b", c) + assert repr(FileBlob("a", c)).startswith("FileBlob(") + with pytest.raises(TypeError): + hash(FileBlob("a", c)) + + +# ---- PathValueMixin + DestructuringMixin composition ---- +# MRO contract: DestructuringMixin sits above PathValueMixin so that +# Destructure's recursion (via super().save) lands here when it encounters a +# nested Path. PathValueMixin owns the directory traversal end-to-end and +# never re-enters self.save / self.load — every storage call uses super(), +# so there's no load-context ambiguity when other transform mixins compose +# below it. + + +@dataclass(frozen=True) +class PathDM(DestructuringMixin, PathValueMixin, ValueMixin, MemoryBackend): + """Test storage: Destructure -> PathValue -> ValueMixin -> MemoryBackend.""" + __hash__ = object.__hash__ + + +@pytest.fixture +def pds(): + return PathDM(storage={}) + + +def test_toplevel_directory_path_stored_as_opaque_directoryblob(pds, tmp_path): + """Saving a directory Path yields a DirectoryBlob stored verbatim (not destructured).""" + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "b.txt").write_bytes(b"world") + + key = pds.save(tmp_path) + raw = pds.storage[key] + assert type(raw) is DirectoryBlob + assert set(raw.contents) == {"a.txt", "b.txt"} + + +def test_toplevel_file_path_stored_as_fileblob_record(pds, tmp_path): + """A file Path stores as content bytes (deduped) plus a FileBlob(name, content) record.""" + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + key = pds.save(p) + raw = pds.storage[key] + assert type(raw) is FileBlob + assert raw.name == "f.txt" + # Content lives separately as plain bytes under its own content digest. + assert raw.content == digest(b"contents") + assert pds.storage[digest(b"contents")] == b"contents" + # The file's key is (name, content) == the path's own digest, not the raw bytes'. + assert key == digest(p) + assert key != digest(b"contents") + + +def test_toplevel_file_path_roundtrip_preserves_name(pds, tmp_path): + """Save → load of a file Path returns a path with the original bytes AND name.""" + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + loaded = pds.load(pds.save(p)) + assert isinstance(loaded, Path) + assert loaded.read_bytes() == b"contents" + assert loaded.name == "f.txt" # name preserved by default — no wrapping + assert loaded.suffix == ".txt" + + +def test_toplevel_directory_path_roundtrip(pds, tmp_path): + """Save → load of a directory Path materializes the tree; root name is mangled, children kept.""" + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.txt").write_bytes(b"world") + + key = pds.save(tmp_path) + loaded = pds.load(key) + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert loaded.name != tmp_path.name # directory root name is not preserved + assert (loaded / "a.txt").read_bytes() == b"hello" + assert (loaded / "sub" / "b.txt").read_bytes() == b"world" + + +def test_nested_path_inside_dict_is_converted_at_save_and_materialized_at_load(pds, tmp_path): + """Path nested in a container: outer dict destructured, inner Path materializes to disk on load. + + The motivating composition scenario: DestructuringMixin handles the outer + structure, PathValueMixin catches the Path via super().save from + Destructure's recursion and stores it as an opaque DirectoryBlob. + """ + (tmp_path / "a.txt").write_bytes(b"hello") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.txt").write_bytes(b"world") + + wrapper = {"tree": tmp_path, "label": "x"} + key = pds.save(wrapper) + + # The outer dict went through DestructuringMixin. + outer = pds.storage[key] + assert isinstance(outer, DigestedMapping) + + # The inner Path was stored as a verbatim DirectoryBlob (not destructured); + # the scalar "tree" key itself stays inline (depth 0 < remaining_depth). + tree_key = outer.items["tree"] + tree_raw = pds.storage[tree_key] + assert type(tree_raw) is DirectoryBlob, \ + f"DirectoryBlob should be stored opaquely, got {type(tree_raw).__name__}" + + # On load, the inner Path materializes back to a real filesystem path. + loaded = pds.load(key) + assert loaded["label"] == "x" + assert isinstance(loaded["tree"], Path) + assert (loaded["tree"] / "a.txt").read_bytes() == b"hello" + assert (loaded["tree"] / "sub" / "b.txt").read_bytes() == b"world" + + +def test_directory_path_file_children_stored_at_content_digest(pds, tmp_path): + """Each file in a saved directory ends up as plain bytes at its content digest.""" + a, b = b"alpha", b"beta" + (tmp_path / "a").write_bytes(a) + (tmp_path / "b").write_bytes(b) + + pds.save(tmp_path) + keys = set(pds.list()) + assert digest(a) in keys + assert digest(b) in keys + + +def test_directoryblob_contents_reference_children_by_digest(pds, tmp_path): + """The DirectoryBlob's ``contents`` maps filename → child content Digest (not inline bytes).""" + payload = b"shared-bytes" + (tmp_path / "f.txt").write_bytes(payload) + + key = pds.save(tmp_path) + raw = pds.storage[key] + assert isinstance(raw, DirectoryBlob) + assert raw.contents["f.txt"] == digest(payload) + + +def test_two_directories_share_file_storage_via_content_addressing(pds, tmp_path): + """Two distinct directories holding the same file body share its storage entry.""" + payload = b"shared" + d1 = tmp_path / "d1"; d1.mkdir(); (d1 / "x").write_bytes(payload) + d2 = tmp_path / "d2"; d2.mkdir(); (d2 / "y").write_bytes(payload) + + pds.save(d1) + keys_after_first = set(pds.list()) + pds.save(d2) + keys_after_second = set(pds.list()) + + # The new directory added its own DirectoryBlob entry but reused the file body. + new_keys = keys_after_second - keys_after_first + assert digest(payload) not in new_keys + assert digest(payload) in keys_after_first + + +def test_empty_directory_path_roundtrip(pds, tmp_path): + """Saving an empty directory yields an empty DirectoryBlob and round-trips to an empty dir.""" + key = pds.save(tmp_path) + raw = pds.storage[key] + assert type(raw) is DirectoryBlob + assert raw.contents == {} + + loaded = pds.load(key) + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert list(loaded.iterdir()) == [] diff --git a/tests/unit/storage/test_paths_default_wiring.py b/tests/unit/storage/test_paths_default_wiring.py new file mode 100644 index 00000000..220e592e --- /dev/null +++ b/tests/unit/storage/test_paths_default_wiring.py @@ -0,0 +1,253 @@ +"""Path-by-content storage through the *default* value storages. + +The committed ``test_paths.py`` exercises a hand-built ``PathDM`` over +``MemoryBackend`` only. These tests prove that wiring ``PathValueMixin`` into +the real default classes (``ValueMemory``, ``ValuePickleFile``, +``ValueBagOfHoldingH5File``) actually works end-to-end — in particular that the +``FileBlob`` (name, content) records, ``DirectoryBlob`` trees, and plain-bytes +content survive each backend's serialization (pickle, H5), not just in-memory +deepcopy. +""" + +from pathlib import Path + +import pytest + +from fleche import fleche, cache +from fleche.digest import digest +from fleche.caches import Cache, DigestedDict +from fleche.storage import ( + ValueMemory, + ValuePickleFile, + ValueBagOfHoldingH5File, + CallMemory, + PathValueMixin, +) +from fleche.storage.destructuring import DigestedMapping + + +# ---- helpers ------------------------------------------------------------- + +def _make_tree(root: Path) -> Path: + """Create *root* and fill it with a small nested directory tree; return *root*.""" + root.mkdir(parents=True, exist_ok=True) + (root / "a.txt").write_bytes(b"hello") + sub = root / "sub" + sub.mkdir() + (sub / "b.bin").write_bytes(b"\x00\x01\x02world") + (sub / "deep").mkdir() + (sub / "deep" / "c").write_bytes(b"deep-bytes") + return root + + +def _relmap(root: Path) -> dict[str, bytes]: + """Map every file under *root* to ``{relative_posix_path: bytes}``.""" + return { + p.relative_to(root).as_posix(): p.read_bytes() + for p in sorted(root.rglob("*")) + if p.is_file() + } + + +# ---- the default value storages carry PathValueMixin --------------------- + +@pytest.mark.parametrize( + "cls", [ValueMemory, ValuePickleFile, ValueBagOfHoldingH5File] +) +def test_default_value_storage_has_path_mixin_in_mro(cls): + assert PathValueMixin in cls.__mro__ + names = [c.__name__ for c in cls.__mro__] + # PathValueMixin sits between DestructuringMixin and ValueMixin so + # Destructure's recursion lands on it, and it stores blobs via ValueMixin. + assert ( + names.index("DestructuringMixin") + < names.index("PathValueMixin") + < names.index("ValueMixin") + ) + + +# ---- round-trips across every real backend ------------------------------- + +def test_file_path_roundtrip(value_storage, tmp_path): + p = tmp_path / "f.txt" + p.write_bytes(b"contents") + + key = value_storage.save(p) + # A file is keyed on (name, content); that equals the path's own digest. + assert key == digest(p) + + loaded = value_storage.load(key) + assert isinstance(loaded, Path) + assert loaded.read_bytes() == b"contents" + assert loaded.name == "f.txt" # name preserved by default — no wrapping + + +def test_directory_tree_roundtrip(value_storage, tmp_path): + src = _make_tree(tmp_path / "src") + expected = _relmap(src) + + key = value_storage.save(src) + loaded = value_storage.load(key) + + assert isinstance(loaded, Path) + assert loaded.is_dir() + assert _relmap(loaded) == expected + + +def test_path_nested_in_dict_roundtrip(value_storage, tmp_path): + src = _make_tree(tmp_path / "src") + expected = _relmap(src) + + wrapper = {"tree": src, "label": "x", "n": 3} + key = value_storage.save(wrapper) + loaded = value_storage.load(key) + + assert loaded["label"] == "x" + assert loaded["n"] == 3 + assert isinstance(loaded["tree"], Path) + assert _relmap(loaded["tree"]) == expected + + +def test_content_addressed_file_dedup(value_storage, tmp_path): + """A file body shared by two directories is stored exactly once.""" + shared = b"shared-body" + one, two = b"one", b"two" + + d1 = tmp_path / "d1" + d1.mkdir() + (d1 / "x").write_bytes(shared) + (d1 / "y").write_bytes(one) + + d2 = tmp_path / "d2" + d2.mkdir() + (d2 / "x").write_bytes(shared) + (d2 / "z").write_bytes(two) + + k1 = value_storage.save(d1) + k2 = value_storage.save(d2) + + keys = set(value_storage.list()) + # File bodies are plain bytes keyed by content; only one copy of `shared`, + # and the two directory trees are distinct. + assert keys == { + digest(shared), + digest(one), + digest(two), + k1, + k2, + } + + +def test_save_load_save_is_idempotent(value_storage, tmp_path): + """Re-saving a materialized TempPath reproduces the same content key.""" + src = _make_tree(tmp_path / "src") + + key = value_storage.save(src) + loaded = value_storage.load(key) + key2 = value_storage.save(loaded) + + assert key2 == key + + +# ---- end-to-end through a Cache + @fleche -------------------------------- + +def test_cache_end_to_end_path_roundtrip(tmp_path): + """A cached function producing a Path: tree stored by content, cache hit on repeat.""" + runs = [] + + @fleche + def build(name, payload): + runs.append(name) + d = tmp_path / f"out-{name}" + d.mkdir() + (d / "data.txt").write_text(payload) + (d / "nested").mkdir() + (d / "nested" / "more.bin").write_bytes(payload.encode() * 2) + return d + + with cache(Cache(ValueMemory({}), CallMemory({}))): + first = build("a", "hello") + assert isinstance(first, Path) + assert (first / "data.txt").read_text() == "hello" + + second = build("a", "hello") + assert isinstance(second, Path) + assert (second / "data.txt").read_text() == "hello" + assert (second / "nested" / "more.bin").read_bytes() == b"hellohello" + + # The body ran exactly once; the second call was served from cache. + assert runs == ["a"] + + +def test_cache_path_argument_roundtrip(tmp_path): + """A cached function consuming a Path argument hits cache on repeat input.""" + parsed = [] + + @fleche + def parse(f: Path): + parsed.append(f.name) + return f.read_text().upper() + + p = tmp_path / "in.txt" + p.write_text("abc") + + with cache(Cache(ValueMemory({}), CallMemory({}))): + assert parse(p) == "ABC" + assert parse(p) == "ABC" + + # Same Path content => one execution. + assert parsed == ["in.txt"] + + +def test_downstream_consumer_needs_no_branching(tmp_path): + """A file keeps its name through the cache, so a plain consumer sees the right suffix. + + No NamedPath, no isinstance: ``inspect`` is an ordinary path consumer and its + ``.suffix`` check holds even when ``produce`` is served from cache. + """ + runs = [] + + @fleche + def produce(seed): + runs.append(("produce", seed)) + p = tmp_path / f"{seed}.json" + p.write_text(f'{{"seed": "{seed}"}}') + return p + + @fleche + def inspect(path): + runs.append(("inspect", path.suffix)) + assert path.suffix == ".json" + return path.stem + + with cache(Cache(ValueMemory({}), CallMemory({}))): + assert inspect(produce("alpha")) == "alpha" + assert inspect(produce("alpha")) == "alpha" # produce cached -> alpha.json + + assert runs == [("produce", "alpha"), ("inspect", ".json")] + + +def test_renaming_does_not_duplicate_the_content_blob(tmp_path): + """Renaming a file (same bytes, new name) adds only a record, never re-stores the body.""" + store = ValueMemory({}) + body = b"the unchanging body" * 100 + + original = tmp_path / "draft.txt" + original.write_bytes(body) + store.save(original) + keys_before = set(store.list()) + assert digest(body) in keys_before, "content body stored on first save" + + renamed_path = tmp_path / "final.txt" + original.rename(renamed_path) + store.save(renamed_path) + + added = set(store.list()) - keys_before + assert len(added) == 1 # just the (name, content) record + assert digest(body) not in added # body reused, not duplicated + + +# ---- backward-compat alias ---------------------------------------------- + +def test_digesteddict_alias_is_digestedmapping(): + assert DigestedDict is DigestedMapping From 4f8198e4ada4141fbfe2fa3420b7d233bb70acc1 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sun, 21 Jun 2026 19:42:22 -0400 Subject: [PATCH 04/27] docs(paths): document path caching with recipes, a dev guide, and a notebook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a light recipes page and a path-storage dev-guide deep dive — content addressing, the digest(path) == values.save(path) invariant, deduplication, and the name-aware-files / content-only-directories model — wire both into the toctree, and add a runnable Files.ipynb walkthrough registered in the notebook integration test. Co-Authored-By: Claude Opus 4.8 --- docs/dev/path_storage.rst | 101 +++++++++ docs/index.rst | 8 + docs/notebooks/Files.ipynb | 1 + docs/recipes/files_and_paths.rst | 71 +++++++ notebooks/Files.ipynb | 305 ++++++++++++++++++++++++++++ tests/integration/test_notebooks.py | 1 + 6 files changed, 487 insertions(+) create mode 100644 docs/dev/path_storage.rst create mode 120000 docs/notebooks/Files.ipynb create mode 100644 docs/recipes/files_and_paths.rst create mode 100644 notebooks/Files.ipynb diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst new file mode 100644 index 00000000..f070cec6 --- /dev/null +++ b/docs/dev/path_storage.rst @@ -0,0 +1,101 @@ +Path Storage Internals +====================== + +How ``fleche`` stores :class:`~pathlib.Path` values — single files and whole +directory trees — by content. For the practical version, see +:doc:`/recipes/files_and_paths`. + +The model, in one line +---------------------- + +``fleche`` follows git's split: **content is content-addressed; names live in +trees.** Concretely — + +* a **file** is identified by ``(basename, content)`` — its name matters, its + bytes deduplicate; +* a **directory** is identified by its **tree alone** — child names are part of + it, but the directory's own root name is not (a reloaded directory is named by + its digest); +* plain **bytes** are anonymous file content — return them when you don't want a + name to enter the cache key. + +:class:`~fleche.storage.paths.PathValueMixin` owns the traversal. In the default +value storages it sits between ``DestructuringMixin`` and ``ValueMixin`` in the +method-resolution order, so a bare ``Path`` (or one nested inside a list, dict, +or dataclass) is intercepted on save and materialized again on load. + +How a file is stored +-------------------- + +A file is split into two pieces so that content deduplicates while the name is +still part of the key: + +* the bytes are saved as **plain** ``bytes`` under their content digest — shared + by every file (and every ``bytes`` value) with the same content; +* a small :class:`~fleche.storage.paths.FileBlob` record pairs the basename with + a *reference* to that content blob, and is keyed + ``digest(("FileBlob", name, content_digest))``. + +On load the content is materialized at ``/`` and returned as an +ordinary path, so ``.name`` / ``.suffix`` / ``.stem`` are faithful and a consumer +needs no special-casing. Because the bytes live in a shared, content-keyed blob +and only the tiny ``FileBlob`` differs per name, **renaming a file never +duplicates its body** — a rename adds one record and reuses the blob. + +How a directory is stored +------------------------- + +A directory is a :class:`~fleche.storage.paths.DirectoryBlob`: a +``{name: content_ref}`` mapping keyed by ``digest(("DirectoryBlob", contents))``. +A file child is referenced by its content bytes; a subdirectory child by its own +``DirectoryBlob``. The directory's **own** name never enters this — two trees +with identical contents under different root names hash identically. On load the +tree is rebuilt under ``/`` (hashed root, faithful children). + +The ``digest(path) == values.save(path)`` invariant +--------------------------------------------------- + +This is the load-bearing property. ``fleche`` *looks up* a cached call by +``digest(arguments)`` but *stores* it under ``values.save(arguments)``; the two +must agree, or a call that takes or returns a path would never hit. The ``Path`` +arm of :func:`~fleche.digest.digest` therefore mirrors storage exactly — a file +as ``digest(("FileBlob", name, digest(bytes)))``, a directory as the content-only +tree — with the ``"FileBlob"`` / ``"DirectoryBlob"`` salts matching +:class:`~fleche.storage.paths.FileBlob` and +:class:`~fleche.storage.paths.DirectoryBlob`'s ``__digest__``. + +Deduplication +------------- + +Everything bottoms out in content-keyed ``bytes`` blobs, so an identical body is +stored once — across names, across directories, and across plain ``bytes`` +values. Only the small ``FileBlob`` / ``DirectoryBlob`` records (references, not +bytes) differ. + +Salting (the tuple idiom) +------------------------- + +``FileBlob`` and ``DirectoryBlob`` salt their digests with the class name via the +tuple idiom from :doc:`custom_digests`. This keeps a ``DirectoryBlob`` from +colliding with a plain ``dict`` carrying the same ``{name: digest}`` mapping, and +a ``FileBlob`` record from colliding with an unrelated value of the same shape. +Note that file *content* needs no such marker: it is plain ``bytes``, and the +"this is a file" information lives in the ``FileBlob`` record (top level) or the +parent ``DirectoryBlob`` entry (inside a tree), never in the content blob itself. + +Choosing content-only +--------------------- + +There is no separate "anonymous path" type: if you want a file's content without +its name in the key, return the ``bytes``. There is deliberately no way to make +a *directory's* root name significant — directories are trees, and their root +name is treated as incidental (typically a temp dir). If a root name carries +meaning, name a *child* meaningfully instead, or wrap the tree's identity in your +own value. + +See also +-------- + +* :doc:`/recipes/files_and_paths` — copy-paste recipes. +* :doc:`/notebooks/Files` — a runnable walkthrough. +* :doc:`custom_digests` — the tuple-digest idiom these blobs use. diff --git a/docs/index.rst b/docs/index.rst index 896f736d..7814fd8f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,12 @@ Welcome to the **Fleche** library documentation. usage/lazy_call usage/query +.. toctree:: + :maxdepth: 2 + :caption: Recipes + + recipes/files_and_paths + .. toctree:: :maxdepth: 2 :caption: Digests @@ -62,6 +68,7 @@ Welcome to the **Fleche** library documentation. :caption: Development dev/custom_digests + dev/path_storage dev/developer dev/ssh_cache @@ -75,6 +82,7 @@ Welcome to the **Fleche** library documentation. notebooks/SecureStorage notebooks/CacheStack notebooks/ConcurrentExecution + notebooks/Files .. toctree:: :maxdepth: 2 diff --git a/docs/notebooks/Files.ipynb b/docs/notebooks/Files.ipynb new file mode 120000 index 00000000..542734c5 --- /dev/null +++ b/docs/notebooks/Files.ipynb @@ -0,0 +1 @@ +../../notebooks/Files.ipynb \ No newline at end of file diff --git a/docs/recipes/files_and_paths.rst b/docs/recipes/files_and_paths.rst new file mode 100644 index 00000000..65f48d13 --- /dev/null +++ b/docs/recipes/files_and_paths.rst @@ -0,0 +1,71 @@ +Caching Functions that Work with Files +====================================== + +Short, copy-paste recipes for caching functions that produce or consume files +and directories. For *why* any of this works, see :doc:`/dev/path_storage`; for +a runnable walkthrough, see the :doc:`/notebooks/Files` notebook. + +Return a file from a cached function +------------------------------------ + +Just return the :class:`~pathlib.Path`. ``fleche`` stores the file's *contents* +(not the path string), keyed on its ``(name, content)`` — so the cache is +portable across machines, and a cache hit comes back as a path with the **same +name and extension**. + +.. code-block:: python + + from pathlib import Path + from fleche import fleche + + @fleche + def render(text) -> Path: + out = Path("report.pdf") + out.write_text(text) + return out + +Downstream code needs nothing special — a cache hit is an ordinary ``Path``: + +.. code-block:: python + + @fleche + def count_pages(doc: Path) -> int: + assert doc.suffix == ".pdf" # still true on a cache hit + return ... + + count_pages(render("hello")) + +Don't care about the name? Return ``bytes`` +------------------------------------------- + +If the filename is irrelevant and you only care about the content, return the +``bytes`` instead of a ``Path``. Content-only values deduplicate maximally — +the same bytes under different would-be names share one cache entry. + +.. code-block:: python + + @fleche + def serialize(obj) -> bytes: + return pickle.dumps(obj) + +Return a directory +------------------ + +Return the directory ``Path``; the whole tree round-trips, and its children keep +their names. + +.. code-block:: python + + @fleche + def build(src) -> Path: + out = Path("build") + out.mkdir() + (out / "result.bin").write_bytes(compile(src)) + return out + +.. note:: + + A directory is identified by its *tree*, not its root name — so a reloaded + directory's own ``.name`` is a hash (its children's names are faithful). + Don't rely on the top-level directory name surviving a cache hit; do rely on + everything inside it. diff --git a/notebooks/Files.ipynb b/notebooks/Files.ipynb new file mode 100644 index 00000000..e3e72b12 --- /dev/null +++ b/notebooks/Files.ipynb @@ -0,0 +1,305 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# Caching functions that read and write files\n", + "\n", + "`fleche` is a content-addressed cache. When a cached function takes or returns a\n", + "`pathlib.Path`, fleche stores the file's (or directory tree's) **contents** \u2014 not\n", + "just the path string \u2014 so results are portable and reproducible across machines.\n", + "\n", + "A path is keyed by its content, so:\n", + "\n", + "- two files with identical bytes share one storage entry (deduplication), and\n", + "- re-calling a function with the same file content is a cache hit, even if the\n", + " file lives at a different location.\n", + "\n", + "The in-memory cache (`cache(\"memory\")`) and every default value storage now carry\n", + "this behaviour out of the box \u2014 no custom storage composition required." + ] + }, + { + "cell_type": "code", + "id": "cell-01", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "import tempfile\n", + "from pathlib import Path\n", + "from subprocess import run, CompletedProcess\n", + "\n", + "import fleche as fl\n", + "from fleche import fleche\n", + "\n", + "fl.cache(\"memory\") # activate a transient in-memory cache\n", + "c = fl.cache() # grab the active cache to introspect later\n", + "\n", + "# The default value storage stores Paths by content:\n", + "[k.__name__ for k in type(c.values).__mro__ if k.__name__.endswith(\"Mixin\")]" + ] + }, + { + "cell_type": "code", + "id": "cell-02", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# A scratch directory for the files our functions produce.\n", + "WORK = Path(tempfile.mkdtemp(suffix=\"-fleche-files\"))\n", + "WORK" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Producing a file\n", + "\n", + "A cached function can create a file and return its `Path`. fleche stores the\n", + "bytes; on a cache hit it hands back a freshly materialized temporary `Path` with\n", + "the same contents. The `print` fires only when the body actually runs." + ] + }, + { + "cell_type": "code", + "id": "cell-04", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "@fleche\n", + "def write(text, repeat=1, name=\"out.txt\"):\n", + " print(\" [write] running:\", repr(text))\n", + " f = WORK / name\n", + " f.write_text(text * repeat)\n", + " return f\n", + "\n", + "f = write(\"hello\", 2)\n", + "print(\"returned:\", type(f).__name__, \"->\", f.read_text())" + ] + }, + { + "cell_type": "code", + "id": "cell-05", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# Same arguments -> cache hit -> the body does NOT run (no \"[write] running\").\n", + "again = write(\"hello\", 2)\n", + "print(\"from cache:\", again.read_text())" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "## Consuming a file\n", + "\n", + "A function can take a `Path` argument; fleche keys the call on the file's\n", + "content. Feeding it a path produced by another cached function chains the two." + ] + }, + { + "cell_type": "code", + "id": "cell-07", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "@fleche\n", + "def wordcount(path: Path):\n", + " print(\" [wordcount] running:\", path.name)\n", + " return len(path.read_text().split())\n", + "\n", + "print(\"count:\", wordcount(write(\"a quick brown fox\", 1, name=\"sentence.txt\")))\n", + "# Re-run: both `write` and `wordcount` are served from cache.\n", + "print(\"count again:\", wordcount(write(\"a quick brown fox\", 1, name=\"sentence.txt\")))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "## Directories\n", + "\n", + "Returning a directory `Path` stores the whole tree. On load it is rebuilt under\n", + "a temporary directory, so `iterdir()` / `rglob()` work exactly as before." + ] + }, + { + "cell_type": "code", + "id": "cell-09", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "@fleche\n", + "def make_tree(seed):\n", + " print(\" [make_tree] running:\", seed)\n", + " d = WORK / f\"tree-{seed}\"\n", + " d.mkdir(exist_ok=True)\n", + " (d / \"top.txt\").write_text(seed)\n", + " (d / \"sub\").mkdir(exist_ok=True)\n", + " (d / \"sub\" / \"leaf.bin\").write_bytes(seed.encode() * 3)\n", + " return d\n", + "\n", + "@fleche\n", + "def total_bytes(d: Path):\n", + " print(\" [total_bytes] running:\", d.name)\n", + " return sum(p.stat().st_size for p in d.rglob(\"*\") if p.is_file())\n", + "\n", + "tree = make_tree(\"alpha\")\n", + "sorted(p.relative_to(tree).as_posix() for p in tree.rglob(\"*\"))" + ] + }, + { + "cell_type": "code", + "id": "cell-10", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# End-to-end cache hit: make_tree(\"alpha\") and total_bytes both come from cache.\n", + "total_bytes(make_tree(\"alpha\"))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Content-addressing\n", + "\n", + "Files are stored under the digest of their contents, so identical bodies are\n", + "stored once regardless of filename or which call produced them." + ] + }, + { + "cell_type": "code", + "id": "cell-12", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# Two calls, different names, identical body -> the content is stored once.\n", + "write(\"shared body\", 1, name=\"left.txt\")\n", + "write(\"shared body\", 1, name=\"right.txt\")\n", + "\n", + "body_key = fl.digest.digest(b\"shared body\") # content blobs are plain bytes\n", + "print(\"shared body stored once:\", body_key in set(c.values.list()))" + ] + }, + { + "cell_type": "code", + "id": "cell-13", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# Introspect the recorded calls of any cached function.\n", + "write.query().table()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## A real workflow: orchestrating shell scripts\n", + "\n", + "The motivating use case: wrap command-line tools that read and write files. Each\n", + "step runs in a working directory, produces files, and the whole pipeline is\n", + "cached by content.\n", + "\n", + "A `subprocess.CompletedProcess` isn't digestible out of the box, so we register a\n", + "digest hook describing how to fingerprint one." + ] + }, + { + "cell_type": "code", + "id": "cell-15", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "def digest_completedprocess(cp):\n", + " return fl.digest.digest((type(cp).__name__, cp.args, cp.returncode, cp.stdout, cp.stderr))\n", + "\n", + "fl.digest.add_hook((CompletedProcess, digest_completedprocess))\n", + "\n", + "@fleche\n", + "def shell(cwd, prog, args=(), stdin=b\"\"):\n", + " print(\" [shell] running:\", prog, *args)\n", + " ret = run([prog, *args], cwd=cwd, capture_output=True, input=stdin)\n", + " return cwd, ret\n", + "\n", + "@fleche\n", + "def pipeline(content):\n", + " print(\" [pipeline] running:\", content)\n", + " work = Path(tempfile.mkdtemp(suffix=\"-pipeline\"))\n", + " (work / \"input.txt\").write_bytes(content)\n", + " shell(work, \"cp\", [\"input.txt\", \"copy.txt\"]) # produce a file\n", + " _, upper = shell(work, \"tr\", [\"a-z\", \"A-Z\"], stdin=content) # capture stdout\n", + " (work / \"shout.txt\").write_bytes(upper.stdout)\n", + " return work" + ] + }, + { + "cell_type": "code", + "id": "cell-16", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "print(\"--- first run ---\")\n", + "out = pipeline(b\"hello world\")\n", + "print(\"produced:\", sorted(p.name for p in out.iterdir()))\n", + "print(\"shout.txt:\", (out / \"shout.txt\").read_text())" + ] + }, + { + "cell_type": "code", + "id": "cell-17", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "print(\"--- second run: fully cached (no body / shell prints) ---\")\n", + "out2 = pipeline(b\"hello world\")\n", + "print(\"same files:\", sorted(p.name for p in out2.iterdir()))" + ] + }, + { + "cell_type": "code", + "id": "cell-18", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# Every shell invocation fleche recorded:\n", + "shell.query().table()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/integration/test_notebooks.py b/tests/integration/test_notebooks.py index 5c8fbb2c..e08dd491 100644 --- a/tests/integration/test_notebooks.py +++ b/tests/integration/test_notebooks.py @@ -9,6 +9,7 @@ "StorageBackends.ipynb", "SecureStorage.ipynb", "CacheStack.ipynb", + "Files.ipynb", ] From 2cdb745d8da9158febf9217ea2d41c4468f71168 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sat, 25 Jul 2026 15:25:49 -0400 Subject: [PATCH 05/27] docs(storage): add notebook on paths nested inside containers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion to Files.ipynb: shows the intended UX of Path values nested in dicts/lists/dataclasses, then the edge cases — hits changing location and type, aliasing loss, Path dict-keys mending into new keys, per-hit materialization and temp-file lifetime, opaque containers storing paths by location, and the exact-type destructuring allowlist. Co-Authored-By: Claude Fable 5 --- notebooks/PathsInContainers.ipynb | 681 ++++++++++++++++++++++++++++++ 1 file changed, 681 insertions(+) create mode 100644 notebooks/PathsInContainers.ipynb diff --git a/notebooks/PathsInContainers.ipynb b/notebooks/PathsInContainers.ipynb new file mode 100644 index 00000000..65ed0006 --- /dev/null +++ b/notebooks/PathsInContainers.ipynb @@ -0,0 +1,681 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9050c7cf", + "metadata": {}, + "source": [ + "# Paths nested inside containers: the usual UX, and the edges\n", + "\n", + "[Files.ipynb](Files.ipynb) shows the sunny-day behaviour of `Path` arguments and\n", + "return values: files and directories are stored by **content** and rematerialized\n", + "under a temporary path on a cache hit.\n", + "\n", + "This notebook explores what happens when paths are *nested inside other values* —\n", + "dicts, lists, dataclasses — which fleche's `DestructuringMixin` takes apart and\n", + "reassembles (\"mends\") around the path machinery. The first half is the intended\n", + "UX; the second half collects the edge cases where a cache hit is **not** a faithful\n", + "replay of the original call: location changes, aliasing loss, path dict-keys, and\n", + "container types whose mending is incomplete or outright broken." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "1ba85e32", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:11.833017Z", + "iopub.status.busy": "2026-07-25T19:23:11.832839Z", + "iopub.status.idle": "2026-07-25T19:23:12.221170Z", + "shell.execute_reply": "2026-07-25T19:23:12.220562Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['PerKeyLockMixin', 'DestructuringMixin', 'PathValueMixin', 'ValueMixin']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import gc\n", + "import tempfile\n", + "from collections import Counter, defaultdict, namedtuple\n", + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "\n", + "import fleche as fl\n", + "from fleche import fleche\n", + "\n", + "fl.cache(\"memory\") # transient in-memory cache\n", + "c = fl.cache()\n", + "\n", + "# DestructuringMixin sits *above* PathValueMixin: containers are taken apart\n", + "# first, and each nested Path is then stored by content as a blob.\n", + "[k.__name__ for k in type(c.values).__mro__ if k.__name__.endswith(\"Mixin\")]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "6825eb50", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.222514Z", + "iopub.status.busy": "2026-07-25T19:23:12.222262Z", + "iopub.status.idle": "2026-07-25T19:23:12.224735Z", + "shell.execute_reply": "2026-07-25T19:23:12.224328Z" + } + }, + "outputs": [], + "source": [ + "# A scratch directory standing in for \"wherever your functions write their files\".\n", + "WORK = Path(tempfile.mkdtemp(suffix=\"-fleche-nested\"))\n", + "\n", + "def fresh(name, text):\n", + " p = WORK / name\n", + " p.write_text(text)\n", + " return p" + ] + }, + { + "cell_type": "markdown", + "id": "8841d5b6", + "metadata": {}, + "source": [ + "## The usual UX: containers of paths just work\n", + "\n", + "A cached function can return paths tucked inside dicts, lists, tuples, or\n", + "dataclasses. On a hit, the container is mended and every nested path comes back\n", + "freshly materialized, with its content and basename intact." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "96e644a1", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.226046Z", + "iopub.status.busy": "2026-07-25T19:23:12.225909Z", + "iopub.status.idle": "2026-07-25T19:23:12.231118Z", + "shell.execute_reply": "2026-07-25T19:23:12.230608Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [fit] running: alpha\n", + "cold: PosixPath -> /tmp/claude-1000/tmpddupkllp-fleche-nested/alpha-fit.txt\n" + ] + } + ], + "source": [ + "@dataclass\n", + "class FitResult:\n", + " report: Path\n", + " score: float\n", + "\n", + "@fleche\n", + "def fit(seed):\n", + " print(\" [fit] running:\", seed)\n", + " return {\n", + " \"results\": [FitResult(fresh(f\"{seed}-fit.txt\", f\"loss={len(seed)}\"), 0.5)],\n", + " \"seed\": seed,\n", + " }\n", + "\n", + "cold = fit(\"alpha\")\n", + "print(\"cold:\", type(cold[\"results\"][0].report).__name__, \"->\", cold[\"results\"][0].report)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "74dac313", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.232200Z", + "iopub.status.busy": "2026-07-25T19:23:12.232055Z", + "iopub.status.idle": "2026-07-25T19:23:12.234825Z", + "shell.execute_reply": "2026-07-25T19:23:12.234336Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "warm: TempPath -> /tmp/claude-1000/tmphvfdmrplfleche/alpha-fit.txt\n", + "content: loss=5 | name kept: alpha-fit.txt\n" + ] + } + ], + "source": [ + "warm = fit(\"alpha\") # no \"[fit] running\" -> served from cache\n", + "r = warm[\"results\"][0]\n", + "print(\"warm:\", type(r.report).__name__, \"->\", r.report)\n", + "print(\"content:\", r.report.read_text(), \"| name kept:\", r.report.name)" + ] + }, + { + "cell_type": "markdown", + "id": "78c368f4", + "metadata": {}, + "source": [ + "True content addressing: the *original* file can vanish entirely — the cache holds\n", + "the bytes, so hits keep working." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1a38c092", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.236049Z", + "iopub.status.busy": "2026-07-25T19:23:12.235897Z", + "iopub.status.idle": "2026-07-25T19:23:12.239485Z", + "shell.execute_reply": "2026-07-25T19:23:12.239067Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'loss=5'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cold[\"results\"][0].report.unlink() # delete the original on disk\n", + "again = fit(\"alpha\")\n", + "again[\"results\"][0].report.read_text() # still served, from stored content" + ] + }, + { + "cell_type": "markdown", + "id": "4db3a027", + "metadata": {}, + "source": [ + "## Edge 1: a hit changes *where* (and what type) the path is\n", + "\n", + "The cold call returns whatever the function returned — the real location in\n", + "`WORK`, as a plain `Path`. A warm hit returns a `TempPath` in a fresh temporary\n", + "directory. Content is identical; **location is not**.\n", + "\n", + "The classic footgun: code that resolves *siblings* of a returned path\n", + "(`p.parent / \"meta.json\"`) works on the first call and breaks on every hit,\n", + "because the materialized file sits alone in its temp directory." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7fe21179", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.240747Z", + "iopub.status.busy": "2026-07-25T19:23:12.240600Z", + "iopub.status.idle": "2026-07-25T19:23:12.245110Z", + "shell.execute_reply": "2026-07-25T19:23:12.244626Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [produce] running: beta\n", + "cold sibling exists: True\n", + "warm location: /tmp/claude-1000/tmptm88ttu4fleche/beta-data.csv\n", + "warm sibling exists: False\n" + ] + } + ], + "source": [ + "@fleche\n", + "def produce(seed):\n", + " print(\" [produce] running:\", seed)\n", + " fresh(f\"{seed}-meta.json\", '{\"version\": 1}') # sibling, NOT returned\n", + " return fresh(f\"{seed}-data.csv\", \"1,2,3\")\n", + "\n", + "p_cold = produce(\"beta\")\n", + "print(\"cold sibling exists:\", (p_cold.parent / \"beta-meta.json\").exists())\n", + "\n", + "p_warm = produce(\"beta\")\n", + "print(\"warm location:\", p_warm)\n", + "print(\"warm sibling exists:\", (p_warm.parent / \"beta-meta.json\").exists())" + ] + }, + { + "cell_type": "markdown", + "id": "a2950eff", + "metadata": {}, + "source": [ + "Only what is *returned* (or passed) is captured. If the sibling matters, return\n", + "it too — or return the whole directory." + ] + }, + { + "cell_type": "markdown", + "id": "8fcb480d", + "metadata": {}, + "source": [ + "## Edge 2: aliasing is not preserved\n", + "\n", + "Return the *same* path twice and the cold result holds one object in two slots.\n", + "The warm hit mends each slot independently: two separate materializations, in two\n", + "different temp directories. Equal content, unequal (and non-identical) paths." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "67496526", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.246344Z", + "iopub.status.busy": "2026-07-25T19:23:12.246201Z", + "iopub.status.idle": "2026-07-25T19:23:12.251214Z", + "shell.execute_reply": "2026-07-25T19:23:12.250671Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [twice] running: gamma\n", + "cold: identical: True | equal: True\n", + "warm: identical: False | equal: False | same content: True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def twice(seed):\n", + " print(\" [twice] running:\", seed)\n", + " p = fresh(f\"{seed}-shared.txt\", seed * 2)\n", + " return [p, p]\n", + "\n", + "pc = twice(\"gamma\")\n", + "print(\"cold: identical:\", pc[0] is pc[1], \"| equal:\", pc[0] == pc[1])\n", + "\n", + "pw = twice(\"gamma\")\n", + "print(\"warm: identical:\", pw[0] is pw[1], \"| equal:\", pw[0] == pw[1],\n", + " \"| same content:\", pw[0].read_bytes() == pw[1].read_bytes())" + ] + }, + { + "cell_type": "markdown", + "id": "78097da1", + "metadata": {}, + "source": [ + "## Edge 3: paths as dict *keys* mend into different keys\n", + "\n", + "Dict keys are destructured like values. A `Path` key comes back as a `TempPath`\n", + "at a new location — so the mended dict has a *different key* than the original,\n", + "and lookups by the original path miss. Use `str(path)` or a stable identifier as\n", + "the key if you intend to look things up by it." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "77ba67df", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.252600Z", + "iopub.status.busy": "2026-07-25T19:23:12.252431Z", + "iopub.status.idle": "2026-07-25T19:23:12.258461Z", + "shell.execute_reply": "2026-07-25T19:23:12.257826Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [index] running: delta\n", + "cold key: /tmp/claude-1000/tmpddupkllp-fleche-nested/delta-k.txt\n", + "warm key: /tmp/claude-1000/tmpxk4r0zg6fleche/delta-k.txt\n", + "kw[orig_key] works: False\n" + ] + } + ], + "source": [ + "@fleche\n", + "def index(seed):\n", + " print(\" [index] running:\", seed)\n", + " return {fresh(f\"{seed}-k.txt\", seed): \"metadata\"}\n", + "\n", + "kc = index(\"delta\")\n", + "orig_key = next(iter(kc))\n", + "\n", + "kw = index(\"delta\")\n", + "warm_key = next(iter(kw))\n", + "print(\"cold key:\", orig_key)\n", + "print(\"warm key:\", warm_key)\n", + "print(\"kw[orig_key] works:\", orig_key in kw)" + ] + }, + { + "cell_type": "markdown", + "id": "f5ea33da", + "metadata": {}, + "source": [ + "## Edge 4: every hit materializes a fresh copy, with temp-file lifetime\n", + "\n", + "Each hit copies the stored bytes into a new temporary directory (large files: mind\n", + "the churn). The temp tree lives exactly as long as some `TempPath` derived from\n", + "it is referenced — keep only a `str` of the location and the file is gone once the\n", + "path object is collected." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "88ba415b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.259737Z", + "iopub.status.busy": "2026-07-25T19:23:12.259570Z", + "iopub.status.idle": "2026-07-25T19:23:12.314448Z", + "shell.execute_reply": "2026-07-25T19:23:12.313281Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [artifact] running: epsilon\n", + "two hits, same location: False\n", + "file still there after dropping the TempPath: False\n", + "w2 unaffected (own temp dir): True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def artifact(seed):\n", + " print(\" [artifact] running:\", seed)\n", + " return fresh(f\"{seed}-art.txt\", seed)\n", + "\n", + "_ = artifact(\"epsilon\") # cold\n", + "w1 = artifact(\"epsilon\") # hit -> copy #1\n", + "w2 = artifact(\"epsilon\") # hit -> copy #2\n", + "print(\"two hits, same location:\", w1 == w2)\n", + "\n", + "location = str(w1) # keep only the string...\n", + "del w1\n", + "gc.collect()\n", + "print(\"file still there after dropping the TempPath:\", Path(location).exists())\n", + "print(\"w2 unaffected (own temp dir):\", w2.exists())" + ] + }, + { + "cell_type": "markdown", + "id": "d8593d07", + "metadata": {}, + "source": [ + "## Edge 5: paths hidden in *opaque* containers are stored by location, not content\n", + "\n", + "Destructuring only recurses into what it knows: lists, tuples, dicts, dataclasses,\n", + "attrs classes. Everything else — namedtuples (deliberately treated as opaque),\n", + "sets, arbitrary objects with a `Path` attribute — is stored **verbatim**. The\n", + "nested path never reaches the content machinery: what is stored is the path\n", + "*object*, pointing at the original location.\n", + "\n", + "The call is still *keyed* correctly (the digest layer does recurse, hashing file\n", + "content), so hits and misses behave right. But a warm hit hands back the original\n", + "location — an **incompletely mended** result. If that file has been deleted,\n", + "moved, or edited since, the hit returns a dangling or stale path, silently." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "6e5db0b0", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.316163Z", + "iopub.status.busy": "2026-07-25T19:23:12.316065Z", + "iopub.status.idle": "2026-07-25T19:23:12.319779Z", + "shell.execute_reply": "2026-07-25T19:23:12.319316Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [bundle] running: zeta\n", + "warm type: PosixPath -> /tmp/claude-1000/tmpddupkllp-fleche-nested/zeta-nt.txt\n", + "points at the ORIGINAL location: True\n" + ] + } + ], + "source": [ + "Bundle = namedtuple(\"Bundle\", [\"out\", \"score\"])\n", + "\n", + "@fleche\n", + "def bundle(seed):\n", + " print(\" [bundle] running:\", seed)\n", + " return Bundle(fresh(f\"{seed}-nt.txt\", seed), 0.5)\n", + "\n", + "b_cold = bundle(\"zeta\")\n", + "b_warm = bundle(\"zeta\") # cache hit...\n", + "print(\"warm type:\", type(b_warm.out).__name__, \"->\", b_warm.out)\n", + "print(\"points at the ORIGINAL location:\", b_warm.out == b_cold.out)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "5c4884f9", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.323854Z", + "iopub.status.busy": "2026-07-25T19:23:12.323678Z", + "iopub.status.idle": "2026-07-25T19:23:12.326503Z", + "shell.execute_reply": "2026-07-25T19:23:12.326114Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hit returns: /tmp/claude-1000/tmpddupkllp-fleche-nested/zeta-nt.txt\n", + "exists: False <- dangling, no warning\n" + ] + } + ], + "source": [ + "# Now the original vanishes -- e.g. a scratch dir is cleaned up between sessions.\n", + "b_cold.out.unlink()\n", + "\n", + "b_stale = bundle(\"zeta\") # still a cache hit (keyed on content at save time)\n", + "print(\"hit returns:\", b_stale.out)\n", + "print(\"exists:\", b_stale.out.exists(), \" <- dangling, no warning\")" + ] + }, + { + "cell_type": "markdown", + "id": "5ede4d0b", + "metadata": {}, + "source": [ + "Compare with Edge 1's dict: the *destructured* container survived deletion of the\n", + "original because the content was captured. The namedtuple did not. Same story\n", + "for sets and custom non-dataclass objects.\n", + "\n", + "**Rule of thumb:** return paths in plain dicts/lists/tuples/dataclasses, not\n", + "smuggled inside opaque types." + ] + }, + { + "cell_type": "markdown", + "id": "ab85f64e", + "metadata": {}, + "source": [ + "## Edge 6: container subclasses are opaque — deliberately\n", + "\n", + "Mending rebuilds containers via `type(value)()`, a contract subclasses\n", + "may repurpose: `defaultdict`'s first argument is a factory (would crash),\n", + "`Counter` *counts* its argument (would silently corrupt). Destructuring therefore\n", + "matches **exact types only** — `dict`, `OrderedDict`, `list`, `tuple` (plus\n", + "dataclasses/attrs, whose mending bypasses `__init__`). Everything else is stored\n", + "verbatim as an opaque value.\n", + "\n", + "So subclasses round-trip *as values* — but any path nested inside them follows\n", + "Edge 5's location semantics, not content addressing." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d19c337b", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.328117Z", + "iopub.status.busy": "2026-07-25T19:23:12.328034Z", + "iopub.status.idle": "2026-07-25T19:23:12.330973Z", + "shell.execute_reply": "2026-07-25T19:23:12.330638Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [by_kind] running: eta\n", + "warm: defaultdict {'files': [PosixPath('/tmp/claude-1000/tmpddupkllp-fleche-nested/eta-dd.txt')]}\n", + "but the nested path is the ORIGINAL location: True\n" + ] + } + ], + "source": [ + "@fleche\n", + "def by_kind(seed):\n", + " print(\" [by_kind] running:\", seed)\n", + " d = defaultdict(list)\n", + " d[\"files\"].append(fresh(f\"{seed}-dd.txt\", seed))\n", + " return d\n", + "\n", + "dd_cold = by_kind(\"eta\")\n", + "dd_warm = by_kind(\"eta\")\n", + "print(\"warm:\", type(dd_warm).__name__, dict(dd_warm))\n", + "print(\"but the nested path is the ORIGINAL location:\", dd_warm[\"files\"][0] == dd_cold[\"files\"][0])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ef2d0d13", + "metadata": { + "execution": { + "iopub.execute_input": "2026-07-25T19:23:12.332260Z", + "iopub.status.busy": "2026-07-25T19:23:12.332187Z", + "iopub.status.idle": "2026-07-25T19:23:12.335016Z", + "shell.execute_reply": "2026-07-25T19:23:12.334601Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [tally] running: 5\n", + "cold: Counter({'b': 6, 'a': 5})\n", + "warm: Counter({'b': 6, 'a': 5}) <- opaque, so counts survive intact\n", + "Help on function register_destructurer in module fleche.storage.destructuring:\n", + "\n", + "register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None\n", + " Register a custom container destructurer.\n", + "\n", + " *pred(value)* should return ``True`` for values this destructurer handles.\n", + " *fn* must accept ``(intern, value)`` where *intern* is\n", + " :meth:`DestructuringMixin._intern_rec`. Entries are appended after the\n", + " built-in ones; first match wins, so registering a handler for an entirely\n", + " new container type is safe without displacing list/dict/dataclass/attrs.\n", + " The built-in predicates match exact types only, so a handler for a subclass\n", + " (e.g. ``defaultdict`` with a picklable factory) can also be registered\n", + " without conflict. Call before any :class:`DestructuringMixin` instance is\n", + " used.\n", + "\n" + ] + } + ], + "source": [ + "@fleche\n", + "def tally(seed):\n", + " print(\" [tally] running:\", seed)\n", + " return Counter({\"a\": seed, \"b\": seed + 1})\n", + "\n", + "print(\"cold:\", tally(5))\n", + "print(\"warm:\", tally(5), \" <- opaque, so counts survive intact\")\n", + "\n", + "# Custom destructurers for subclasses can be opted in later:\n", + "help(fl.storage.destructuring.register_destructurer)" + ] + }, + { + "cell_type": "markdown", + "id": "fea5f4b5", + "metadata": {}, + "source": [ + "## Summary: rules of thumb\n", + "\n", + "- **Works out of the box:** paths (files *and* directories) as arguments, return\n", + " values, or nested anywhere inside dicts / lists / tuples / dataclasses / attrs\n", + " classes — arbitrarily deep. Content-addressed, dedup'd, survives deletion of\n", + " the originals.\n", + "- **A hit is a copy, not a replay:** returned paths live in fresh temp\n", + " directories. Don't resolve siblings, don't compare locations, don't expect\n", + " aliasing, and keep a reference to the `Path` object for as long as you need the\n", + " file.\n", + "- **Don't key dicts by `Path`** if you'll look them up afterwards — keys mend\n", + " into new locations. Use `str(path)` or a stable ID.\n", + "- **Don't hide paths in opaque containers** (namedtuples, sets, plain classes,\n", + " and any container *subclass* — only exact `dict` / `OrderedDict` / `list` /\n", + " `tuple` are destructured): they are stored by location and come back stale or\n", + " dangling after the original moves on. `register_destructurer` is the opt-in\n", + " door for well-behaved custom containers." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 2fed8360588db25c2327ff6d8eb3e093f7500f33 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sat, 25 Jul 2026 16:02:27 -0400 Subject: [PATCH 06/27] docs(paths): add black-box-tested file semantics contract page New docs/usage/file_semantics.rst spells out the full user-facing contract for Path values: call identity ((basename, content) for files, tree-only for directories, content-keyed bytes), materialization on hits (fresh temp copies, faithful basenames, deterministic digest root names, order preservation), lifetime of materialized paths, nesting in destructured containers (dict keys, aliasing, exact-type allowlist), opaque-container location semantics, the do-not-mutate-arguments rule, and fidelity limits (permissions, symlinks, metadata). The page was validated black-box: agents restricted to reading only this page wrote assertion scripts against it in three rounds; every failed or ambiguous expectation was folded back into the text. The mutation rule and fidelity limits were discovered this way. Also cross-links the page from the TL;DR, the files recipe, and the path storage internals page. Co-Authored-By: Claude Fable 5 --- docs/dev/path_storage.rst | 1 + docs/index.rst | 1 + docs/recipes/files_and_paths.rst | 13 +- docs/usage/file_semantics.rst | 222 +++++++++++++++++++++++++++++++ docs/usage/tldr.rst | 13 +- 5 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 docs/usage/file_semantics.rst diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index f070cec6..9b40db5d 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -96,6 +96,7 @@ own value. See also -------- +* :doc:`/usage/file_semantics` — the user-facing contract this implements. * :doc:`/recipes/files_and_paths` — copy-paste recipes. * :doc:`/notebooks/Files` — a runnable walkthrough. * :doc:`custom_digests` — the tuple-digest idiom these blobs use. diff --git a/docs/index.rst b/docs/index.rst index 7814fd8f..394d79a2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -32,6 +32,7 @@ Welcome to the **Fleche** library documentation. usage/tldr usage/helpers + usage/file_semantics usage/lazy_call usage/query diff --git a/docs/recipes/files_and_paths.rst b/docs/recipes/files_and_paths.rst index 65f48d13..93e5f245 100644 --- a/docs/recipes/files_and_paths.rst +++ b/docs/recipes/files_and_paths.rst @@ -2,8 +2,17 @@ Caching Functions that Work with Files ====================================== Short, copy-paste recipes for caching functions that produce or consume files -and directories. For *why* any of this works, see :doc:`/dev/path_storage`; for -a runnable walkthrough, see the :doc:`/notebooks/Files` notebook. +and directories. For the precise contract (identity, materialization, +lifetime, nesting), see :doc:`/usage/file_semantics`; for *why* any of this +works, see :doc:`/dev/path_storage`; for a runnable walkthrough, see the +:doc:`/notebooks/Files` notebook. + +.. note:: + + A cache hit returns a **fresh temporary copy**, not the original location. + The copy lives as long as you hold a ``Path`` object pointing into it — + copy it out (``shutil.copy``) if you need it at a stable place. Details: + :doc:`/usage/file_semantics`. Return a file from a cached function ------------------------------------ diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst new file mode 100644 index 00000000..25e297ee --- /dev/null +++ b/docs/usage/file_semantics.rst @@ -0,0 +1,222 @@ +File and Path Semantics +======================= + +What happens, precisely, when a :class:`~pathlib.Path` crosses the cache +boundary — as an argument, as a return value, or nested inside one. This page +is the contract: everything here is intended behavior you may rely on. For +copy-paste recipes see :doc:`/recipes/files_and_paths`; for the storage design +see :doc:`/dev/path_storage`. + +The model in three sentences +---------------------------- + +A **file** is identified by its ``(basename, content)`` — where it lives never +matters, what it is called and what is in it always do. A **directory** is +identified by its tree alone — child names and contents, recursively — while its +own root name is ignored. A cache hit does not give you back the original +location: it **materializes a fresh copy** under a temporary directory and hands +you the path to that. + +Identity: what makes two calls "the same call" +---------------------------------------------- + +Arguments are turned into a lookup key by digesting them; for paths the digest +follows the model above. + +* ``f(Path("a/data.csv"))`` and ``f(Path("b/data.csv"))`` are the **same call** + if both files have the same bytes — location is not part of the key. +* ``f(Path("data.csv"))`` and ``f(Path("data.tsv"))`` are **different calls** + even with identical bytes — the basename is part of a file's key. +* Editing a file's content changes the key: the next call is a miss and + recomputes. +* For a **directory** argument, renaming or moving the directory itself does + *not* change the key, but renaming or moving anything **inside** it does. +* ``bytes`` arguments are content-keyed too: two calls passing equal bytes are + the same call, regardless of where the bytes came from. +* Two distinct arguments whose digests agree are interchangeable: a hit can be + served for a path argument your process never saw before, as long as name and + bytes match. + +The same identity governs return values: results are stored by content, so two +functions returning identical files share one stored body (see +:ref:`file-dedup`). + +Argument mutation +~~~~~~~~~~~~~~~~~ + +A call is keyed on its arguments **as passed**: argument content is captured +*before* the function body runs. A function that mutates its own argument — +most commonly, writing an output file *into* a directory it received — is +still recorded under the pre-call content, so honest repeat calls hit. The +mutation itself, however, is neither recorded nor replayed: a cache hit +leaves the argument untouched, so a side effect on the input happens on cold +calls only. + +fleche caches *pure* functions. What a function does to its arguments +without passing it back out is invisible to the cache — treat received paths +as read-only and write outputs to a fresh directory (``tempfile.mkdtemp``). +A mutated argument that *is* returned is captured faithfully in its final, +post-mutation state: if the mutation is the point, return it. + +Only *content* changes count as mutation: permissions are not part of +identity (see :ref:`fidelity-limits`), so a ``chmod`` on a received path is +harmless. + +Nonexistent paths +~~~~~~~~~~~~~~~~~ + +A path that does not exist on disk has no content and therefore **no digest**. +Passing one to a cached function does not raise: fleche logs a warning +(``"No hash for argument: ..."``) and **runs the function uncached** — every +call executes, nothing is stored or looked up. If you meant "an output +location the function should write to", pass the location as a ``str`` or +annotate the parameter :class:`~fleche.Ignored` (``dest: Ignored[Path]``) so it +stays out of the key. + +Cache hits materialize copies +----------------------------- + +On a hit, fleche rebuilds the file (or tree) from stored content in a **fresh +temporary directory** and returns that path. Consequences you should design +for: + +* **The location changes.** The cold (computing) call returns the function's + return value untouched — same objects, same locations, same aliasing; every + subsequent hit returns a path under a new temporary directory. Never + compare returned paths by location or store ``str(path)`` as a stable + identifier. +* **The type changes.** Hits return a ``Path`` subclass that manages the + temporary tree's lifetime. The subclass is an implementation detail — do not + import it or dispatch on its name; rely only on ``isinstance(p, Path)``, and + do not assume ``type(p) is PosixPath``. +* **File names are faithful.** A file materializes as ``/``: + ``.name``, ``.suffix``, and ``.stem`` are the same cold and warm. +* **Directory root names are not.** A directory materializes as + ``/``: its own ``.name`` is a hex digest on a hit. The digest + name is deterministic — hits for the same content always carry the same root + name (under differing temporary parents) — but treat it as opaque. + Everything *inside* — child names, nesting, contents — is faithful. +* **Every hit is a fresh copy.** Each hit materializes under its own private + temporary directory, so two hits for the same call return two different + locations with equal content. For large files this means disk traffic per + hit; hold on to the result rather than re-calling in a loop. +* **Siblings do not come along.** Only the returned (or passed) path is + captured. A hit's ``p.parent`` contains nothing but ``p`` itself — code + like ``p.parent / "meta.json"`` works on the cold call and fails on hits. + Return the sibling too, or return the whole directory. +* **The original is not needed.** Once stored, hits are served from cache + content; the file the cold call returned can be deleted, moved, or edited + without affecting later hits. + +.. _fidelity-limits: + +Fidelity limits +~~~~~~~~~~~~~~~ + +What is stored — and thus what a hit restores — is **names and bytes, nothing +else**: + +* **Permissions are not preserved.** A materialized file has default + permissions; in particular the executable bit is lost, so a returned script + is not runnable on a hit without a fresh ``chmod``. (Permissions are also + not part of identity: ``chmod`` alone never changes a key.) +* **Symlinks are flattened.** A symlink inside a returned directory is stored + and restored as an ordinary file with the *target's* content. +* **Empty subdirectories are preserved**; modification times, ownership, and + other metadata are not. + +Lifetime of materialized paths +------------------------------ + +The temporary tree behind a hit lives exactly as long as some ``Path`` object +derived from it is referenced. Deriving (``p.parent``, ``p / "x"``, +``p.with_suffix(...)``) keeps the tree alive; converting to ``str`` does not. + +.. code-block:: python + + p = cached_fn() # hit -> temp file + loc = str(p) + del p # last reference gone ... + Path(loc).exists() # ... False: the temp tree was deleted + +Keep the ``Path`` object (or the container holding it) for as long as you need +the file. If you need the file at a stable location, copy it out: +``shutil.copy(p, dest)``. + +Paths nested inside containers +------------------------------ + +Paths are found and content-stored inside the containers fleche takes apart: +``dict``, ``OrderedDict``, ``list``, ``tuple`` (**exact types** — see below), +``dataclasses`` and ``attrs`` classes — nested to any depth, as values *or* as +dict keys. Everything above about identity, materialization, and lifetime +applies to each nested path individually. Container structure is otherwise +faithful on a hit: lists and tuples keep their element order, and a mended +dict keeps the insertion order the stored value had. + +Three caveats specific to nesting: + +* **Dict keys mend into new keys.** A ``Path`` used as a dictionary key comes + back as a materialized path at a new location — keys get the same + materialization, type, and lifetime treatment as values — and looking the + entry up by the *original* path object misses (``Path`` hashing is + location-based). If you intend to look entries up, key by something that + survives a round trip: the basename (``path.name``) or an + application-level identifier. +* **Aliasing is not preserved.** If the same path object appears twice in a + result, a hit materializes each occurrence separately: equal content, two + locations, ``is``-distinct objects. +* **Only listed container types are traversed.** Subclasses — including + ``defaultdict``, ``Counter``, and ``namedtuple`` — and other containers + (``set``, plain classes that are not dataclass/attrs) are stored **verbatim** + as opaque values. + +Opaque containers store paths by *location* +------------------------------------------- + +A path inside an opaque value (a ``namedtuple``, a ``set``, an arbitrary +object) never reaches the content machinery. The call is still *keyed* +correctly — the digest layer does look inside — but what is stored is the path +object itself, pointing at wherever the file was when it was saved. A hit +returns that original *location* (whether as the same object or an equal copy +is backend-dependent — rely on neither): if the file has since been deleted or +edited, the hit hands you a dangling or stale path, **without any warning**. + +Rule of thumb: return paths in plain dicts / lists / tuples / dataclasses. If +you need a custom container to participate, see +:func:`~fleche.storage.destructuring.register_destructurer`. + +.. _file-dedup: + +Deduplication +------------- + +File bodies are stored once per unique byte-content — shared across file +names, across directories, and with plain ``bytes`` values. Renaming a file +and re-caching adds a tiny name record, never a second copy of the bytes. If +you want *keys* (not just storage) to ignore the name too, return ``bytes`` +instead of a ``Path``; that is the content-only escape hatch. + +Quick reference +--------------- + +=============================================== ====================================== +You do A cache hit gives you +=============================================== ====================================== +Return ``Path`` to a file Copy at ``/`` +Return ``Path`` to a directory Tree at ``/``, faithful inside +Return paths in dict/list/tuple/dataclass Same container shape, each path a copy +Return ``bytes`` The bytes (no file, no name) +Use ``Path`` as dict key New key at new location +Return same path twice Two independent copies +Return path inside namedtuple/set/custom class The *original* location (may be stale) +Pass a nonexistent ``Path`` argument No caching: warns, always executes +Write into a directory you received Hit (keyed as passed); write not replayed +=============================================== ====================================== + +See also +-------- + +* :doc:`/recipes/files_and_paths` — copy-paste recipes. +* :doc:`/dev/path_storage` — how storage implements this contract. +* :doc:`/digests/digests_as_args` — passing digests instead of values. diff --git a/docs/usage/tldr.rst b/docs/usage/tldr.rst index a17ee44d..e3332e9c 100644 --- a/docs/usage/tldr.rst +++ b/docs/usage/tldr.rst @@ -55,8 +55,13 @@ The essentials beyond that: expensive.rerun(1, 2) # force re-execution and overwrite the entry expensive.query().table() # stored calls for this function as a pandas DataFrame +Functions may also take and return :class:`~pathlib.Path` objects: files and +directories are cached by **content** (not by path string), and a cache hit +returns a freshly materialized copy under a temporary path. See +:doc:`file_semantics` for the exact contract. + That is all you need for everyday use. The rest of this section covers the -helper methods in depth (:doc:`helpers`), lazy loading of large cached -objects (:doc:`lazy_call`), and querying stored calls (:doc:`query`); -storage backends and configuration details live under -:doc:`/storage/configuration`. +helper methods in depth (:doc:`helpers`), the file/path contract +(:doc:`file_semantics`), lazy loading of large cached objects +(:doc:`lazy_call`), and querying stored calls (:doc:`query`); storage backends +and configuration details live under :doc:`/storage/configuration`. From 6ed35808c3bb17830edfd6b7b0a8ecf474b8010f Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Sun, 26 Jul 2026 10:47:51 -0400 Subject: [PATCH 07/27] test(paths): pin pre-call keying for mutated Path arguments Integration coverage for the two-phase save protocol meeting path storage: a function writing into a received directory hits on honest repeats, the mutated tree is a distinct call, and a mutated argument passed back out is captured in its final state. Co-Authored-By: Claude Fable 5 --- tests/integration/test_path_mutation.py | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/integration/test_path_mutation.py diff --git a/tests/integration/test_path_mutation.py b/tests/integration/test_path_mutation.py new file mode 100644 index 00000000..25103cc8 --- /dev/null +++ b/tests/integration/test_path_mutation.py @@ -0,0 +1,69 @@ +"""Path arguments are keyed as passed (two-phase save + content addressing). + +A function that writes into a directory it received used to be recorded under +the post-mutation tree and could never hit; with the two-phase save protocol +the identity is sealed before the body runs. See +docs/usage/file_semantics.rst, "Argument mutation". +""" +from pathlib import Path + +import fleche as fl +from fleche import fleche + + +def make_input(root: Path, name: str) -> Path: + d = root / name / "data" + d.mkdir(parents=True) + (d / "in.txt").write_text("same content") + return d + + +def test_mutating_path_consumer_hits(tmp_path): + fl.cache("memory") + runs = [] + + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return sorted(p.name for p in d.iterdir()) + + assert consume(make_input(tmp_path, "a")) == ["in.txt", "out.txt"] + d2 = make_input(tmp_path, "b") # identical tree, as passed + assert consume(d2) == ["in.txt", "out.txt"] + assert len(runs) == 1 # keyed on the pre-call tree: hit + assert not (d2 / "out.txt").exists() # the mutation is not replayed + + +def test_mutated_state_is_a_different_call(tmp_path): + fl.cache("memory") + runs = [] + + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return len(list(d.iterdir())) + + d1 = make_input(tmp_path, "a") + consume(d1) # cold; d1 now holds in+out + consume(d1) # post-mutation tree: honest miss, + assert len(runs) == 2 # not a false hit on the old record + + +def test_returned_mutated_argument_captured_in_final_state(tmp_path): + fl.cache("memory") + runs = [] + + @fleche + def stamp(d: Path) -> Path: + runs.append(1) + (d / "stamp.txt").write_text("stamped") + return d + + stamp(make_input(tmp_path, "a")) + warm = stamp(make_input(tmp_path, "b")) # hit, keyed on the input as passed + assert len(runs) == 1 + # ... but the result was captured at commit time: final, stamped state. + assert sorted(p.name for p in warm.iterdir()) == ["in.txt", "stamp.txt"] + assert (warm / "stamp.txt").read_text() == "stamped" From b8b386538c5d11d66c4076111c5b3f513a6bbab4 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:49:11 +0000 Subject: [PATCH 08/27] refactor(caches): drop save_value, seal via values storage in prepare Address review on #793: - `BaseCache.save_value` is gone (and with it the new abstract method on `BaseCache`, which broke the API for existing subclasses). `Cache.prepare` now hands its whole value storage to `Call.stash`, which already leaves an unknown result as `None`. - `PreparedCall.commit` no longer stores the result itself: it attaches the live value to the record and lets `Cache.save` store it at the end, next to the one-shot `Call` path. - `PreparedCall` drops the redundant `call` and `key` fields; callers that need the key have the `Call` or the `DigestedCall` at hand. - `BaseCache.prepare` keeps a concrete digest-only default (key sealed, nothing written), so third-party caches keep working and `ReadOnlyMixin` only restates it to win the MRO over `CacheWrapper`. Wrappers and stacks delegate storage inward and rebind the cache so the commit runs through their own `save` policy. - Over SSH, `save_value` (one round trip per value) is replaced by a single `prepare` RPC that stashes the arguments remotely and returns the sealed record; read-only remotes short-circuit to the local digest-only admission. - `CacheStack.save` now returns the record key, which `commit` hands back. Co-authored-by: Marvin Poul <2719909+pmrv@users.noreply.github.com> --- src/fleche/caches.py | 140 ++++++++++------------ src/fleche/call.py | 34 +++--- src/fleche/remote.py | 29 +++-- tests/unit/call/test_prepared_call.py | 52 ++++++-- tests/unit/config/test_cache_to_config.py | 3 - tests/unit/test_remote.py | 2 +- 6 files changed, 143 insertions(+), 117 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index fe62e668..406a2fa6 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -48,46 +48,37 @@ def from_config(cls, config: "dict[str, Any] | list[dict[str, Any]]") -> "BaseCa from . import config as _config return _config.cache_from_config(config) - @abstractmethod - def save_value(self, value: Any) -> "Digest": - """Store one value, returning its content digest. - - The write-side counterpart of :meth:`load_value`; also the primitive - :meth:`prepare` runs argument and result values through. - """ - ... - def prepare(self, call: Call) -> PreparedCall: - """Admit *call* to this cache: store its arguments, seal its lookup key. + """Admit *call* to this cache: seal its lookup key before the body runs. - The first half of the two-phase save protocol. Argument values go - through :meth:`save_value` *now* — before the function body runs — so - the recorded identity always describes the arguments as they were at - call time, even if the body later mutates them. Because ``digest(x) - == save_value(x)`` for every storable value, the sealed key equals - ``call.to_lookup_key()``. + The first half of the two-phase save protocol. Caches that own a + value storage (:class:`Cache`) override this to stash the argument + values *now* — before the function body runs — so the recorded + identity describes the arguments as they were at call time, even if + the body later mutates them. + + This base implementation is the digest-only admission used by caches + that cannot (or must not) write ahead of the body — read-only views, + aggregates without their own value storage: the key is sealed, but + nothing is written and whether a record survives is decided by + :meth:`save` at commit time. Because ``digest(x) == values.save(x)`` + for every storable value, both forms seal the same key. Finish the returned :class:`~fleche.call.PreparedCall` with exactly one of :meth:`~fleche.call.PreparedCall.commit` (store the result, file the record) or :meth:`~fleche.call.PreparedCall.abandon`. - - Argument values the storage refuses (``SaveError``) fall back to a - digest-only reference, as in :meth:`fleche.call.Call.stash`. """ - digested = call._to_digested(self.save_value) - return PreparedCall( - call=call, digested=digested, key=digested.to_lookup_key(), cache=self - ) + return PreparedCall(digested=call.digest(), cache=self) @abstractmethod def save(self, call: DigestedCall | Call) -> str: """File a call record. - The primary form takes a fully digested record whose values were - already stored — see :meth:`fleche.call.Call.prepare` for the - two-phase protocol that does both in the right order. A live - :class:`Call` is also accepted as the degenerate one-shot form - (values static, stored on the spot).""" + Takes either a live :class:`Call` — values are stored on the spot, the + one-shot form — or a :class:`DigestedCall` whose argument values were + already stored by :meth:`prepare`, in which case only its result (if + any) still needs storing. See :meth:`prepare` for the two-phase + protocol that does both in the right order.""" ... @abstractmethod @@ -332,37 +323,35 @@ def load_value(self, key): with self._operation_context(key): return self.values.load(key) - def save_value(self, value: Any) -> Digest: - # No cache-level lock: the key is only known once the value storage - # has digested the value, and value storages carry their own per-key - # locking (PerKeyLockMixin) where they need it. - return self.values.save(value) + def prepare(self, call: Call) -> PreparedCall: + # Stash the arguments *now*, before the function body runs, so the + # record cannot end up keyed on post-mutation content. ``stash`` + # leaves the still-unknown result as ``None``; ``save`` below stores + # it once ``commit`` fills it in. No cache-level lock: the keys are + # only known once the value storage has digested each value, and value + # storages carry their own per-key locking where they need it. + return PreparedCall(digested=call.stash(self.values), cache=self) def save(self, call: DigestedCall | Call) -> str: - # Record-only: argument and result values were already written to - # ``self.values`` by Call.prepare / PreparedCall.commit, whose digests - # this record carries. Writing them here instead would re-read mutable - # values (e.g. Path contents) *after* the function body ran and file - # the record under post-mutation content — the incoherence the - # two-phase protocol exists to prevent. - # - # A live Call (values not yet stored) is still accepted as the - # degenerate one-shot form: with no function body between digesting - # and filing there is nothing to drift, so stash-then-file is - # equivalent to prepare/commit here. Inlined rather than routed - # through prepare().commit() so one logical save does not re-enter - # subclass save() overrides a second time. Wrappers and stacks need - # no own shim — every save path lands here. - if isinstance(call, Call): - key = call.to_lookup_key() - with self._operation_context(key): - try: - digested = call.stash(self.values) - except storage.SaveError as e: - raise Rejected(e) - return self.calls.save(digested) key = call.to_lookup_key() with self._operation_context(key): + try: + if isinstance(call, Call): + # One-shot form: nothing was stored ahead of time. With no + # function body between digesting and filing there is + # nothing to drift, so stash-then-file is equivalent to + # prepare/commit here. + call = call.stash(self.values) + elif call.result is not None and not isinstance(call.result, Digest): + # Second half of the two-phase protocol: the arguments are + # already digests (stored by ``prepare``), only the result + # arrives live from ``PreparedCall.commit``. Not re-saving + # the arguments here is the point — reading them again + # *after* the body ran would file the record under + # post-mutation content. + call = replace(call, result=self.values.save(call.result)) + except storage.SaveError as e: + raise Rejected(e) return self.calls.save(call) def load(self, key: str) -> LazyCall: @@ -530,8 +519,11 @@ class CacheWrapper(BaseCache): cache: BaseCache - def save_value(self, value: Any) -> Digest: - return self.cache.save_value(value) + def prepare(self, call: Call) -> PreparedCall: + # The inner cache decides how the arguments are stored; rebinding the + # cache makes the eventual commit go through *this* wrapper's ``save``, + # so wrapper policy (read-only, filtering, size limits) still applies. + return replace(self.cache.prepare(call), cache=self) def save(self, call: DigestedCall | Call) -> str: return self.cache.save(call) @@ -589,18 +581,14 @@ def save(self, call: DigestedCall | Call): def evict(self, key: str | Digest) -> None: raise Rejected("Cannot evict from a read-only cache", self, key) - def save_value(self, value: Any) -> Digest: - raise Rejected("Cannot save values to a read-only cache", self) - def prepare(self, call: Call) -> "PreparedCall": - # Digest-only admission: a read-only cache stashes nothing, so the - # function body still runs with a correctly sealed key, and the - # eventual commit is rejected (save_value raises) — matching the - # behavior save() rejection produced before the two-phase protocol. - digested = call.digest() - return PreparedCall( - call=call, digested=digested, key=digested.to_lookup_key(), cache=self - ) + # Digest-only admission (the BaseCache default, restated because this + # mixin is base-free and must beat CacheWrapper.prepare in the MRO): a + # read-only cache stashes nothing, so the function body still runs with + # a correctly sealed key and the eventual commit is rejected by + # ``save`` above — the behavior save() rejection produced before the + # two-phase protocol, minus the wasted writes. + return PreparedCall(digested=call.digest(), cache=self) @dataclass(frozen=True) @@ -846,14 +834,16 @@ def __post_init__(self): if isinstance(c, CacheStack): raise ValueError("CacheStack cannot be nested inside another CacheStack") - def save(self, call: DigestedCall | Call): - self.stack[0].save(call) + def save(self, call: DigestedCall | Call) -> str: + # Returning the key (rather than dropping it, as this used to) keeps + # the BaseCache contract that PreparedCall.commit hands back to callers. + return self.stack[0].save(call) - def save_value(self, value: Any) -> Digest: - # Writes always land on stack[0] (matching save); reads that need the - # full fan-out go through load_value, which _MultiCache spreads over - # every member. - return self.stack[0].save_value(value) + def prepare(self, call: Call) -> PreparedCall: + # Writes always land on stack[0] (matching save), so that is where the + # arguments are stashed; rebinding the cache routes the commit back + # through this stack's ``save``. + return replace(self.stack[0].prepare(call), cache=self) @contextlib.contextmanager def _operation_context(self, key, *, intent: Intent = Intent.WRITE): diff --git a/src/fleche/call.py b/src/fleche/call.py index ec7405c3..5ea74073 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -266,7 +266,11 @@ class DigestedCall: name: str arguments: dict[str, "digest.Digest"] - result: "digest.Digest | None" = None + # Normally a Digest pointer, or None while the result is still unknown + # (a record prepared before its function body ran). Between + # :meth:`PreparedCall.commit` and the cache's ``save`` it briefly holds the + # live return value, which ``save`` stores and replaces with its Digest. + result: "digest.Digest | Any" = None metadata: dict[str, dict[str, Any]] = field(default_factory=dict) module: str | None = None version: str | int | None = None @@ -348,37 +352,30 @@ class PreparedCall: prepared.commit(func(...)) # skipped on exception -> abandoned """ - call: Call digested: DigestedCall - key: Digest cache: Any _finished: bool = field(default=False, init=False, repr=False) def commit(self, result: Any, metadata: dict | None = None) -> Digest: - """Store *result*, attach *metadata*, and file the call record. + """Attach *result* and *metadata* to the record and file it. - The second half of the two-phase save protocol: the result is written to - ``cache.values`` (capturing its content *as returned* — including any - argument the body mutated and passed back out), and the completed - :class:`DigestedCall` is filed under :attr:`key`. + The second half of the two-phase save protocol. The result is handed + to the cache live, so it is stored *as returned* — including any + argument the body mutated and passed back out — while the arguments + keep the digests sealed at prepare time. Args: result: The function's return value. metadata: Optional metadata mapping stored on the record. Returns: - Digest: the record key (equal to :attr:`key`). + Digest: the record key. Raises: fleche.caches.Rejected: if the result cannot be stored or the cache refuses the record. """ - from .storage.base import SaveError # lazy import: storage.base depends on call - try: - self.digested.result = self.cache.save_value(result) - except SaveError as e: - from .caches import Rejected # lazy import: caches depends on call - raise Rejected(e) from None + self.digested.result = result if metadata is not None: self.digested.metadata = metadata self._finished = True @@ -388,9 +385,10 @@ def abandon(self) -> None: """Release the call without recording it. Called when the function body raises or its result is not cacheable. - The default is a no-op: argument values stored by :meth:`Call.prepare` - are content-addressed orphans that a later garbage collection sweep - reclaims. Subclasses may hook cleanup here. + The default is a no-op: argument values stored by + :meth:`~fleche.caches.BaseCache.prepare` are content-addressed orphans + that a later garbage collection sweep reclaims. Subclasses may hook + cleanup here. """ self._finished = True diff --git a/src/fleche/remote.py b/src/fleche/remote.py index a51fd189..245cdeca 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -57,7 +57,7 @@ from . import call as _call from .caches import BaseCache, Rejected -from .call import DigestedCall, LazyCall, QueryCall +from .call import DigestedCall, LazyCall, PreparedCall, QueryCall from .digest import Digest logger = logging.getLogger("fleche.remote") @@ -148,7 +148,10 @@ def _query_result(cache: BaseCache, args: tuple) -> tuple[DigestedCall, ...]: "save": _RemoteMethod(lambda cache, args: cache.save(*args)), "load": _RemoteMethod(lambda cache, args: _strip_cache(cache.load(*args))), "load_value": _RemoteMethod(lambda cache, args: cache.load_value(*args)), - "save_value": _RemoteMethod(lambda cache, args: cache.save_value(*args)), + # Only the sealed record travels back; the server-side PreparedCall is + # dropped (abandon is a no-op) and the client finishes the protocol with a + # plain ``save`` once the body has run. + "prepare": _RemoteMethod(lambda cache, args: cache.prepare(*args).digested), "evict": _RemoteMethod(lambda cache, args: cache.evict(*args), void=True), "contains": _RemoteMethod(lambda cache, args: cache.contains(*args)), "expand": _RemoteMethod(lambda cache, args: cache.expand(*args)), @@ -681,7 +684,9 @@ def _fetch_lazy_calls(sc: "SshCache", results: "tuple[DigestedCall, ...]") -> It "save": _ClientMethod(write=True), "load": _ClientMethod(unwrap=_fetch_lazy_call), "load_value": _ClientMethod(), - "save_value": _ClientMethod(write=True), + # Read-only remotes never reach this entry: `SshCache.prepare` short-circuits + # to the local digest-only admission before dispatching. + "prepare": _ClientMethod(), "evict": _ClientMethod( write=True, reject_message="Cannot evict from a read-only remote cache" ), @@ -794,12 +799,18 @@ def load(self, key: str) -> LazyCall: def load_value(self, key: str) -> Any: return self._rpc("load_value", key) - def save_value(self, value: Any) -> Digest: - # One round trip per value (no batching yet). Values travel by - # cloudpickle, so a Path value ships its path *string*, not its - # content — the same limitation the previous ship-the-whole-Call - # protocol had; paths over SSH remain unsupported. - return self._rpc("save_value", value) + def prepare(self, call: _call.Call) -> PreparedCall: + # One round trip: the whole call goes over so the remote stashes the + # argument values before the body runs and seals the record, which + # comes back for the client to complete with ``save``. Values travel + # by cloudpickle, so a Path argument ships its path *string*, not its + # content — paths over SSH remain unsupported, as before. + self._ensure_handshake() + if self._info_cache and self._info_cache.get("read_only", False): + # Digest-only admission, as BaseCache: the body still runs, and the + # commit's ``save`` is rejected locally without a round trip. + return super().prepare(call) + return PreparedCall(digested=self._rpc("prepare", call), cache=self) def evict(self, key: str | Digest) -> None: self._rpc("evict", key) diff --git a/tests/unit/call/test_prepared_call.py b/tests/unit/call/test_prepared_call.py index 2717d88b..526fc7d7 100644 --- a/tests/unit/call/test_prepared_call.py +++ b/tests/unit/call/test_prepared_call.py @@ -1,4 +1,4 @@ -"""Two-phase save protocol: Call.prepare / PreparedCall.commit / abandon. +"""Two-phase save protocol: Cache.prepare / PreparedCall.commit / abandon. The protocol exists to fix one incoherence: the wrapper digests arguments *before* the function body runs (the lookup key), but the old save path @@ -12,7 +12,7 @@ from fleche import fleche import fleche as fl from fleche.call import Call -from fleche.caches import Cache, Rejected +from fleche.caches import Cache, CacheStack, RefreshingCache, Rejected from fleche.digest import digest from fleche.storage.memory import ValueMemory, CallMemory @@ -32,7 +32,7 @@ def make_call(**arguments): def test_prepare_seals_lookup_key(cache): call = make_call(x=1, y="a") prepared = cache.prepare(call) - assert prepared.key == call.to_lookup_key() + assert prepared.digested.to_lookup_key() == call.to_lookup_key() def test_prepare_stores_arguments_before_commit(cache): @@ -45,7 +45,7 @@ def test_commit_files_record_under_sealed_key(cache): call = make_call(x=1) prepared = cache.prepare(call) key = prepared.commit("result", {"meta": {"k": "v"}}) - assert key == prepared.key + assert key == call.to_lookup_key() loaded = cache.load(key) assert loaded.result == "result" assert loaded.metadata == {"meta": {"k": "v"}} @@ -67,29 +67,29 @@ def test_abandon_leaves_no_record(cache): call = make_call(x=1) prepared = cache.prepare(call) prepared.abandon() - assert not cache.contains(prepared.key) + assert not cache.contains(call.to_lookup_key()) def test_context_manager_abandons_without_commit(cache): call = make_call(x=1) with cache.prepare(call) as prepared: pass - assert not cache.contains(prepared.key) + assert not cache.contains(call.to_lookup_key()) def test_context_manager_commit_sticks(cache): call = make_call(x=1) with cache.prepare(call) as prepared: prepared.commit(42) - assert cache.load(prepared.key).result == 42 + assert cache.load(call.to_lookup_key()).result == 42 def test_context_manager_does_not_swallow_exceptions(cache): call = make_call(x=1) with pytest.raises(RuntimeError): - with cache.prepare(call) as prepared: + with cache.prepare(call): raise RuntimeError("body failed") - assert not cache.contains(prepared.key) + assert not cache.contains(call.to_lookup_key()) def test_readonly_prepare_is_digest_only_and_commit_rejects(cache): @@ -97,11 +97,11 @@ def test_readonly_prepare_is_digest_only_and_commit_rejects(cache): rejection lands at commit time, after the body would have run.""" call = make_call(x=[1, 2]) prepared = cache.readonly().prepare(call) - assert prepared.key == call.to_lookup_key() + assert prepared.digested.to_lookup_key() == call.to_lookup_key() assert list(cache.values.list()) == [] # nothing was stashed with pytest.raises(Rejected): prepared.commit(42) - assert not cache.contains(prepared.key) + assert not cache.contains(call.to_lookup_key()) def test_key_matches_one_shot_save(cache): @@ -112,6 +112,36 @@ def test_key_matches_one_shot_save(cache): assert cache.prepare(call).commit("r") == one_shot.save(call) +# ---- wrappers and stacks: storage from the inner cache, policy from the outer ---- + + +def test_stack_prepares_on_stack0_and_commits_through_the_stack(cache): + """Arguments are stashed where the stack's saves land; the commit still + goes through the stack itself (its ``save`` policy, not stack[0]'s).""" + second = Cache(ValueMemory({}), CallMemory({})) + stack = CacheStack([cache, second]) + call = make_call(x=[1, 2]) + prepared = stack.prepare(call) + assert prepared.cache is stack + key = prepared.commit([1, 2, 3]) + assert key == call.to_lookup_key() + assert cache.load(key).result == [1, 2, 3] + assert cache.load(key).arguments["x"] == [1, 2] + assert not second.contains(key) + + +def test_wrapper_prepare_commits_through_the_wrapper(cache): + """A wrapper delegates storage to its inner cache but keeps its own save + policy on the commit — here the refresh wrapper's write-through.""" + wrapper = RefreshingCache(cache) + call = make_call(x=[1, 2]) + prepared = wrapper.prepare(call) + assert prepared.cache is wrapper + key = prepared.commit("r") + assert cache.load(key).result == "r" + assert cache.load(key).arguments["x"] == [1, 2] + + # ---- end-to-end: argument mutation no longer corrupts identity ---- diff --git a/tests/unit/config/test_cache_to_config.py b/tests/unit/config/test_cache_to_config.py index 78584e95..dc15177a 100644 --- a/tests/unit/config/test_cache_to_config.py +++ b/tests/unit/config/test_cache_to_config.py @@ -112,9 +112,6 @@ def load(self, key): def load_value(self, key): raise KeyError(key) - def save_value(self, value): - return "" - def evict(self, key): pass diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 0ccf5da2..b993bbf5 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -229,7 +229,7 @@ def test_dispatch_covers_every_registered_method(): "save", "load", "load_value", - "save_value", + "prepare", "evict", "contains", "expand", From 402522948fc169d01a1278412a04cdedc8338477 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:13:21 +0000 Subject: [PATCH 09/27] fix(call): enforce single-shot PreparedCall.commit without mutating the record commit now builds the filed record with dataclasses.replace, so the live result is never parked on the prepared record after save, and raises RuntimeError on a second commit or a commit after abandon (abandon stays idempotent). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- src/fleche/call.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/fleche/call.py b/src/fleche/call.py index 5ea74073..43cbba08 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -374,12 +374,17 @@ def commit(self, result: Any, metadata: dict | None = None) -> Digest: Raises: fleche.caches.Rejected: if the result cannot be stored or the cache refuses the record. + RuntimeError: if this call was already committed or abandoned. """ - self.digested.result = result - if metadata is not None: - self.digested.metadata = metadata + if self._finished: + raise RuntimeError("PreparedCall already committed or abandoned") self._finished = True - return self.cache.save(self.digested) + digested = replace( + self.digested, + result=result, + metadata=self.digested.metadata if metadata is None else metadata, + ) + return self.cache.save(digested) def abandon(self) -> None: """Release the call without recording it. @@ -388,7 +393,7 @@ def abandon(self) -> None: The default is a no-op: argument values stored by :meth:`~fleche.caches.BaseCache.prepare` are content-addressed orphans that a later garbage collection sweep reclaims. Subclasses may hook - cleanup here. + cleanup here. Idempotent; only :meth:`commit` is barred afterwards. """ self._finished = True From ab77c07c95f58b567e465b935260473371250388 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:13:21 +0000 Subject: [PATCH 10/27] fix(wrapper): run uncached when the cache rejects prepare Mirror the existing Rejected handling on save: a cache that refuses admission logs a warning and the call executes uncached instead of failing before the body runs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- src/fleche/wrapper.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/fleche/wrapper.py b/src/fleche/wrapper.py index 16133f2a..c6ad826a 100644 --- a/src/fleche/wrapper.py +++ b/src/fleche/wrapper.py @@ -235,14 +235,16 @@ def _run_and_cache(): for m in active_meta: metadata[m.name] |= m.pre(replace(call, metadata={})) - # Two-phase save: argument values are stored and the record's key - # sealed *before* the body runs, so the recorded identity is the - # arguments as passed — even if the body mutates them (e.g. writes - # into a directory it received). A read-only cache stashes - # nothing here (digest-only prepare) and rejects at commit time, - # so the call still runs and returns uncached, as it previously - # did when the post-body save was rejected. - prepared = cache.prepare(call) + # Seal the call's identity (and stash its argument values) before + # the body runs, so mutations the body makes to its arguments + # cannot leak into the recorded key. A read-only cache seals + # without writing and rejects at commit; the call still runs and + # returns uncached. + try: + prepared = cache.prepare(call) + except Rejected as e: + logger.warning("Cache rejected prepare: %s", e.args) + return func(*args, **kwargs) try: result: _T = func(*args, **kwargs) From 404bff77a0856e70db2d3828ab46f067aea51389 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:13:22 +0000 Subject: [PATCH 11/27] test: cover remote prepare, commit guards, and hermetic e2e caches - SshCache.prepare: live wire round trip (arguments stashed server-side before the body, mutation-coherent commit) and the read-only local digest-only admission with zero RPCs. - PreparedCall: double commit and commit-after-abandon raise; commit leaves the prepared record unmutated. - The end-to-end tests now run on a private Cache instance: populating the shared named-memory singleton (load_cache_config caches it in _live_caches) leaked into any later test that round-trips that cache's config, e.g. test_run_server_serves_active_cache_over_stdio_until_eof. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- tests/unit/call/test_prepared_call.py | 51 +++++++++++++++++-------- tests/unit/test_remote.py | 55 ++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/tests/unit/call/test_prepared_call.py b/tests/unit/call/test_prepared_call.py index 526fc7d7..1e00fcae 100644 --- a/tests/unit/call/test_prepared_call.py +++ b/tests/unit/call/test_prepared_call.py @@ -92,6 +92,27 @@ def test_context_manager_does_not_swallow_exceptions(cache): assert not cache.contains(call.to_lookup_key()) +def test_commit_is_exactly_once(cache): + prepared = cache.prepare(make_call(x=1)) + prepared.commit(1) + with pytest.raises(RuntimeError): + prepared.commit(2) + + +def test_abandon_is_idempotent_but_bars_commit(cache): + prepared = cache.prepare(make_call(x=1)) + prepared.abandon() + prepared.abandon() + with pytest.raises(RuntimeError): + prepared.commit(1) + + +def test_commit_does_not_mutate_the_prepared_record(cache): + prepared = cache.prepare(make_call(x=1)) + prepared.commit("r") + assert prepared.digested.result is None + + def test_readonly_prepare_is_digest_only_and_commit_rejects(cache): """A read-only cache admits the call without writing anything; the rejection lands at commit time, after the body would have run.""" @@ -145,10 +166,9 @@ def test_wrapper_prepare_commits_through_the_wrapper(cache): # ---- end-to-end: argument mutation no longer corrupts identity ---- -def test_mutating_consumer_hits_on_honest_repeat(): +def test_mutating_consumer_hits_on_honest_repeat(cache): """A function that mutates its argument is keyed on the *pre-call* state: a repeat call with the same initial state is a hit.""" - fl.cache("memory") runs = [] @fleche @@ -157,16 +177,16 @@ def consume(xs: list): xs.append(99) return sum(xs) - assert consume([1, 2]) == 102 - assert consume([1, 2]) == 102 + with fl.cache(cache): + assert consume([1, 2]) == 102 + assert consume([1, 2]) == 102 assert len(runs) == 1 -def test_no_mislabeled_entry_for_mutated_state(): +def test_no_mislabeled_entry_for_mutated_state(cache): """The post-mutation argument state is a *different* call and recomputes — the old behavior filed the first call under this state (a false hit that returned a result computed from different input).""" - fl.cache("memory") runs = [] @fleche @@ -175,13 +195,13 @@ def consume(xs: list): xs.append(99) return len(xs) - assert consume([1, 2]) == 3 - assert consume([1, 2, 99]) == 4 # miss: its own honest computation + with fl.cache(cache): + assert consume([1, 2]) == 3 + assert consume([1, 2, 99]) == 4 # miss: its own honest computation assert len(runs) == 2 -def test_body_exception_leaves_no_record(): - fl.cache("memory") +def test_body_exception_leaves_no_record(cache): runs = [] @fleche @@ -189,8 +209,9 @@ def boom(x): runs.append(1) raise ValueError("no") - for _ in range(2): - with pytest.raises(ValueError): - boom(1) - assert len(runs) == 2 # nothing cached, body ran twice - assert not boom.contains(1) + with fl.cache(cache): + for _ in range(2): + with pytest.raises(ValueError): + boom(1) + assert len(runs) == 2 # nothing cached, body ran twice + assert not boom.contains(1) diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index b993bbf5..0ae59ae8 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -17,7 +17,7 @@ from fleche.call import Call, QueryCall from fleche.caches import Cache, Rejected from fleche.config import cache_to_config, load_cache_config -from fleche.digest import Digest +from fleche.digest import Digest, digest from fleche.remote import ( RemoteConnectionError, SshCache, @@ -303,6 +303,59 @@ def test_read_only_false_for_writable_remote(remote): assert remote.read_only is False +# --------------------------------------------------------------------------- +# Two-phase save protocol over the wire +# --------------------------------------------------------------------------- + + +def test_prepare_commit_round_trip(remote, server_cache): + """`prepare` stashes the arguments server-side before the body runs; + `commit` ships the live result and files the record under the sealed key.""" + xs = [1, 2] + c = Call(name="f", arguments={"xs": xs}, module="m", version="1") + sealed = c.to_lookup_key() + prepared = remote.prepare(c) + assert prepared.cache is remote + assert prepared.digested.to_lookup_key() == sealed + # The argument value is already on the server before any commit. + assert server_cache.values.load(digest([1, 2])) == [1, 2] + xs.append(3) # the "function body" mutates the argument + key = prepared.commit(xs) + assert key == sealed # NOT c.to_lookup_key(): that drifted with the mutation + lc = remote.load(key) + assert lc.result == [1, 2, 3] # result: as returned + assert lc.arguments["xs"] == [1, 2] # argument: as passed + + +def test_read_only_prepare_is_local_and_commit_rejects(): + """`prepare` against a read-only remote seals the key locally — no RPC, + nothing stashed — and the commit's `save` is rejected without a round trip.""" + server = Cache(ValueMemory({}), CallMemory({})).readonly() + sc = _make_remote(server) + try: + # Trigger the info fetch once so the read_only flag is cached. + assert sc.read_only is True + + rpc_calls = [] + original_call = sc._conn.call + + def spy(method, *args, **kwargs): + rpc_calls.append(method) + return original_call(method, *args, **kwargs) + + sc._conn.call = spy # type: ignore[method-assign] + + c = Call(name="f", arguments={"xs": [1, 2]}, module="m", version="1") + prepared = sc.prepare(c) + assert prepared.digested.to_lookup_key() == c.to_lookup_key() + assert list(server.cache.values.list()) == [] # nothing stashed + with pytest.raises(Rejected): + prepared.commit(42) + assert rpc_calls == [], f"unexpected RPCs: {rpc_calls}" + finally: + sc.close() + + def test_reconnect_invalidates_info_cache(remote, server_cache): """`reconnect()` drops the cached info so the next read re-fetches.""" info1 = remote.info() From a611788bfa91f82c8fefecf9e06021e752b1a37b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:38:13 +0000 Subject: [PATCH 12/27] refactor(caches): carry the pending result on PreparedCall until save stores it Combines the #827 sketch with the replace-based commit: DigestedCall goes back to holding digests only, PreparedCall carries the live result and metadata privately, and Cache.save stores the result when handed the PreparedCall itself. PreparedCall pickles without its process-local cache binding so a commit against SshCache ships only the sealed record and the pending result. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- src/fleche/caches.py | 57 +++++++++++++++++++++++--------------------- src/fleche/call.py | 33 ++++++++++++++++--------- src/fleche/remote.py | 6 ++++- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index 406a2fa6..17b88626 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -13,7 +13,7 @@ from .storage.base import _apply_shrink, _resolve_prefix, Intent, OperationContext from .storage.destructuring import HasChildDigests from .storage.thread_safe import PerKeyLockMixin, _PicklableRLock -from .call import Call, DigestedCall, LazyCall, PreparedCall, QueryCall +from .call import Call, LazyCall, PreparedCall, QueryCall from . import call from . import query @@ -71,14 +71,13 @@ def prepare(self, call: Call) -> PreparedCall: return PreparedCall(digested=call.digest(), cache=self) @abstractmethod - def save(self, call: DigestedCall | Call) -> str: + def save(self, call: PreparedCall | Call) -> str: """File a call record. Takes either a live :class:`Call` — values are stored on the spot, the - one-shot form — or a :class:`DigestedCall` whose argument values were - already stored by :meth:`prepare`, in which case only its result (if - any) still needs storing. See :meth:`prepare` for the two-phase - protocol that does both in the right order.""" + one-shot form — or a :class:`~fleche.call.PreparedCall` whose argument + values were already stored by :meth:`prepare` and whose pending result + is stored now, as returned.""" ... @abstractmethod @@ -332,27 +331,33 @@ def prepare(self, call: Call) -> PreparedCall: # storages carry their own per-key locking where they need it. return PreparedCall(digested=call.stash(self.values), cache=self) - def save(self, call: DigestedCall | Call) -> str: + def save(self, call: PreparedCall | Call) -> str: key = call.to_lookup_key() with self._operation_context(key): try: if isinstance(call, Call): - # One-shot form: nothing was stored ahead of time. With no - # function body between digesting and filing there is - # nothing to drift, so stash-then-file is equivalent to - # prepare/commit here. - call = call.stash(self.values) - elif call.result is not None and not isinstance(call.result, Digest): - # Second half of the two-phase protocol: the arguments are - # already digests (stored by ``prepare``), only the result - # arrives live from ``PreparedCall.commit``. Not re-saving - # the arguments here is the point — reading them again - # *after* the body ran would file the record under - # post-mutation content. - call = replace(call, result=self.values.save(call.result)) + # One-shot form: nothing was stored ahead of time, and with + # no function body between digesting and filing there is + # nothing to drift. + digested = call.stash(self.values) + elif isinstance(call, PreparedCall): + # Committed result, stored as returned; the arguments must + # NOT be re-saved here — reading them after the body ran is + # exactly the post-mutation keying prepare exists to + # prevent. + digested = replace( + call.digested, + result=self.values.save(call._result), + metadata=call.digested.metadata + if call._metadata is None + else call._metadata, + ) + else: + # Already fully digested: file as-is. + digested = call except storage.SaveError as e: raise Rejected(e) - return self.calls.save(call) + return self.calls.save(digested) def load(self, key: str) -> LazyCall: with self._operation_context(key): @@ -525,7 +530,7 @@ def prepare(self, call: Call) -> PreparedCall: # so wrapper policy (read-only, filtering, size limits) still applies. return replace(self.cache.prepare(call), cache=self) - def save(self, call: DigestedCall | Call) -> str: + def save(self, call: PreparedCall | Call) -> str: return self.cache.save(call) def load(self, key: str) -> LazyCall: @@ -575,7 +580,7 @@ class ReadOnlyMixin: short-circuits ``save``/``evict`` without a round-trip). """ - def save(self, call: DigestedCall | Call): + def save(self, call: PreparedCall | Call): raise Rejected(self, call) def evict(self, key: str | Digest) -> None: @@ -834,9 +839,7 @@ def __post_init__(self): if isinstance(c, CacheStack): raise ValueError("CacheStack cannot be nested inside another CacheStack") - def save(self, call: DigestedCall | Call) -> str: - # Returning the key (rather than dropping it, as this used to) keeps - # the BaseCache contract that PreparedCall.commit hands back to callers. + def save(self, call: PreparedCall | Call) -> str: return self.stack[0].save(call) def prepare(self, call: Call) -> PreparedCall: @@ -997,7 +1000,7 @@ def _enforce_size_limit(self) -> None: target = self._pick_eviction_target(list(self._keys)) self.evict(target) - def save(self, call: call.DigestedCall | call.Call) -> str: + def save(self, call: call.PreparedCall | call.Call) -> str: with self._lock: key = super().save(call) self._keys.add(key) diff --git a/src/fleche/call.py b/src/fleche/call.py index 43cbba08..3a7c872d 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -266,11 +266,9 @@ class DigestedCall: name: str arguments: dict[str, "digest.Digest"] - # Normally a Digest pointer, or None while the result is still unknown - # (a record prepared before its function body ran). Between - # :meth:`PreparedCall.commit` and the cache's ``save`` it briefly holds the - # live return value, which ``save`` stores and replaces with its Digest. - result: "digest.Digest | Any" = None + # None while the result is still unknown (a record prepared before its + # function body ran). + result: "digest.Digest | None" = None metadata: dict[str, dict[str, Any]] = field(default_factory=dict) module: str | None = None version: str | int | None = None @@ -355,6 +353,10 @@ class PreparedCall: digested: DigestedCall cache: Any _finished: bool = field(default=False, init=False, repr=False) + # The live result and metadata between commit and the cache's save; + # DigestedCall itself only ever holds digests. + _result: Any = field(default=None, init=False, repr=False) + _metadata: "dict | None" = field(default=None, init=False, repr=False) def commit(self, result: Any, metadata: dict | None = None) -> Digest: """Attach *result* and *metadata* to the record and file it. @@ -379,12 +381,21 @@ def commit(self, result: Any, metadata: dict | None = None) -> Digest: if self._finished: raise RuntimeError("PreparedCall already committed or abandoned") self._finished = True - digested = replace( - self.digested, - result=result, - metadata=self.digested.metadata if metadata is None else metadata, - ) - return self.cache.save(digested) + self._result = result + if metadata is not None: + self._metadata = metadata + return self.cache.save(self) + + def to_lookup_key(self) -> Digest: + return self.digested.to_lookup_key() + + def __getstate__(self) -> dict: + # The cache binding is process-local (it may hold live locks or an SSH + # connection); a PreparedCall on the wire carries only the record and + # the pending result. + state = self.__dict__.copy() + state["cache"] = None + return state def abandon(self) -> None: """Release the call without recording it. diff --git a/src/fleche/remote.py b/src/fleche/remote.py index 245cdeca..2ca57216 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -790,7 +790,11 @@ def _rpc(self, name: str, *args: Any) -> Any: raise Rejected(self, *args) return spec.unwrap(self, self._conn.call(name, *args)) - def save(self, call: DigestedCall | _call.Call) -> str: + def save(self, call: PreparedCall | _call.Call) -> str: + # A PreparedCall pickles without its cache binding (see + # ``PreparedCall.__getstate__``), so only the sealed record and the + # pending result travel; the server's cache stores the result and files + # the record. return self._rpc("save", call) def load(self, key: str) -> LazyCall: From 5d5a67a5d5fd8b9b53eca530e12025c005c8189e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:38:13 +0000 Subject: [PATCH 13/27] fix(wrapper): degrade to uncached on any prepare failure Policy rejections keep their warning; actual errors (storage faults, lost connections) are logged with traceback and the body still runs. Also drop the dead call.metadata assignment left over from the commit protocol switch. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- src/fleche/wrapper.py | 7 ++++++- tests/unit/call/test_prepared_call.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/fleche/wrapper.py b/src/fleche/wrapper.py index c6ad826a..4f3dedd3 100644 --- a/src/fleche/wrapper.py +++ b/src/fleche/wrapper.py @@ -245,6 +245,12 @@ def _run_and_cache(): except Rejected as e: logger.warning("Cache rejected prepare: %s", e.args) return func(*args, **kwargs) + except Exception: + # Not a policy rejection but an actual error (storage fault, + # lost connection, ...): still prefer running the call uncached + # over failing before the body ever ran. + logger.warning("Cache failed to prepare call", exc_info=True) + return func(*args, **kwargs) try: result: _T = func(*args, **kwargs) @@ -283,7 +289,6 @@ def _cache(future=None): metadata[m.name], replace(call, metadata={}) ) try: - call.metadata = metadata logger.debug("Saving result for %s with key %s", call.name, key) prepared.commit(call.result, metadata) except Rejected as e: diff --git a/tests/unit/call/test_prepared_call.py b/tests/unit/call/test_prepared_call.py index 1e00fcae..5dd9bfe5 100644 --- a/tests/unit/call/test_prepared_call.py +++ b/tests/unit/call/test_prepared_call.py @@ -201,6 +201,30 @@ def consume(xs: list): assert len(runs) == 2 +@pytest.mark.parametrize("exc", [Rejected("no admission"), OSError("storage down")]) +def test_prepare_failure_degrades_to_uncached(exc): + """A prepare that fails — policy rejection or actual error — must not stop + the call: the body runs and the result comes back uncached.""" + + class Broken(Cache): + def prepare(self, call): + raise exc + + broken = Broken(ValueMemory({}), CallMemory({})) + runs = [] + + @fleche + def f(x): + runs.append(1) + return x + 1 + + with fl.cache(broken): + assert f(1) == 2 + assert f(1) == 2 + assert len(runs) == 2 # never cached: prepare fails on every call + assert list(broken.calls.list()) == [] + + def test_body_exception_leaves_no_record(cache): runs = [] From b23ada933b71aeaf4b3b0a103a4638d181f3a736 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 17:03:53 +0000 Subject: [PATCH 14/27] refactor(remote): ship the committed result and record in two trips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SshCache.save now recreates Cache.save's ending for a PreparedCall: a write-gated save_value RPC stores the result value remotely, then the plain digested record is filed — only Call and DigestedCall ever cross the wire, so PreparedCall.__getstate__ goes away. The resolve step shared by both cache types moves onto PreparedCall. Server-side, save_value routes through prepare with a synthetic never-committed call, so wrappers and stacks direct the value to the same storage their saves use without any new public cache method. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- src/fleche/caches.py | 8 +------- src/fleche/call.py | 19 ++++++++++++------- src/fleche/remote.py | 35 +++++++++++++++++++++++++++++------ tests/unit/test_remote.py | 21 ++++++++++++++++++++- 4 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index 17b88626..a81469e3 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -345,13 +345,7 @@ def save(self, call: PreparedCall | Call) -> str: # NOT be re-saved here — reading them after the body ran is # exactly the post-mutation keying prepare exists to # prevent. - digested = replace( - call.digested, - result=self.values.save(call._result), - metadata=call.digested.metadata - if call._metadata is None - else call._metadata, - ) + digested = call.resolve(self.values.save(call._result)) else: # Already fully digested: file as-is. digested = call diff --git a/src/fleche/call.py b/src/fleche/call.py index 3a7c872d..1f1bbf35 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -389,13 +389,18 @@ def commit(self, result: Any, metadata: dict | None = None) -> Digest: def to_lookup_key(self) -> Digest: return self.digested.to_lookup_key() - def __getstate__(self) -> dict: - # The cache binding is process-local (it may hold live locks or an SSH - # connection); a PreparedCall on the wire carries only the record and - # the pending result. - state = self.__dict__.copy() - state["cache"] = None - return state + def resolve(self, result: Digest) -> DigestedCall: + """Return the final record with the pending result resolved to *result*. + + The shared ending of the two-phase save: the cache stores + :attr:`_result` wherever its values live and hands the digest back + here; pending metadata is applied at the same time. + """ + return replace( + self.digested, + result=result, + metadata=self.digested.metadata if self._metadata is None else self._metadata, + ) def abandon(self) -> None: """Release the call without recording it. diff --git a/src/fleche/remote.py b/src/fleche/remote.py index 2ca57216..e31a89a7 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -141,6 +141,22 @@ def _query_result(cache: BaseCache, args: tuple) -> tuple[DigestedCall, ...]: return tuple(_strip_cache(lc) for lc in cache.query(template)) +def _save_value(cache: BaseCache, value: Any) -> Digest: + """Store a bare *value* in the served cache and return its digest. + + The write-side counterpart of ``load_value``, existing only in the wire + protocol. Routed through :meth:`~fleche.caches.BaseCache.prepare` with a + synthetic, never-committed call, so wrappers and stacks direct the value + to the same storage their saves use; the value is kept (content-addressed) + while no call record is ever filed. + """ + prepared = cache.prepare( + _call.Call(name="fleche.remote:save_value", arguments={"value": value}) + ) + prepared.abandon() + return prepared.digested.arguments["value"] + + # Single inventory of every method `SshCache` forwards across the wire; the # client-side stubs near `SshCache` call these names via `self._conn.call(...)`. # Adding a new RPC-exposed cache method only requires one entry here. @@ -148,9 +164,10 @@ def _query_result(cache: BaseCache, args: tuple) -> tuple[DigestedCall, ...]: "save": _RemoteMethod(lambda cache, args: cache.save(*args)), "load": _RemoteMethod(lambda cache, args: _strip_cache(cache.load(*args))), "load_value": _RemoteMethod(lambda cache, args: cache.load_value(*args)), + "save_value": _RemoteMethod(lambda cache, args: _save_value(cache, *args)), # Only the sealed record travels back; the server-side PreparedCall is - # dropped (abandon is a no-op) and the client finishes the protocol with a - # plain ``save`` once the body has run. + # dropped (abandon is a no-op) and the client finishes the protocol with + # ``save_value`` + ``save`` once the body has run. "prepare": _RemoteMethod(lambda cache, args: cache.prepare(*args).digested), "evict": _RemoteMethod(lambda cache, args: cache.evict(*args), void=True), "contains": _RemoteMethod(lambda cache, args: cache.contains(*args)), @@ -684,6 +701,9 @@ def _fetch_lazy_calls(sc: "SshCache", results: "tuple[DigestedCall, ...]") -> It "save": _ClientMethod(write=True), "load": _ClientMethod(unwrap=_fetch_lazy_call), "load_value": _ClientMethod(), + # Write-gated so a commit against a read-only remote is rejected locally + # before the first trip. + "save_value": _ClientMethod(write=True), # Read-only remotes never reach this entry: `SshCache.prepare` short-circuits # to the local digest-only admission before dispatching. "prepare": _ClientMethod(), @@ -791,10 +811,13 @@ def _rpc(self, name: str, *args: Any) -> Any: return spec.unwrap(self, self._conn.call(name, *args)) def save(self, call: PreparedCall | _call.Call) -> str: - # A PreparedCall pickles without its cache binding (see - # ``PreparedCall.__getstate__``), so only the sealed record and the - # pending result travel; the server's cache stores the result and files - # the record. + if isinstance(call, PreparedCall): + # Recreate Cache.save's ending in two trips: ship the result value, + # then file the plain digested record — a PreparedCall itself never + # goes over the wire. A failure between the trips leaves the value + # as a content-addressed orphan for gc, like any abandoned call. + digested = call.resolve(self._rpc("save_value", call._result)) + return self._rpc("save", digested) return self._rpc("save", call) def load(self, key: str) -> LazyCall: diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 0ae59ae8..96653633 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -229,6 +229,7 @@ def test_dispatch_covers_every_registered_method(): "save", "load", "load_value", + "save_value", "prepare", "evict", "contains", @@ -310,7 +311,19 @@ def test_read_only_false_for_writable_remote(remote): def test_prepare_commit_round_trip(remote, server_cache): """`prepare` stashes the arguments server-side before the body runs; - `commit` ships the live result and files the record under the sealed key.""" + `commit` ships the result value and then the plain digested record — + a PreparedCall itself never goes over the wire.""" + from fleche.call import PreparedCall + + shipped = [] + original_call = remote._conn.call + + def spy(method, *args, **kwargs): + shipped.append((method, args)) + return original_call(method, *args, **kwargs) + + remote._conn.call = spy # type: ignore[method-assign] + xs = [1, 2] c = Call(name="f", arguments={"xs": xs}, module="m", version="1") sealed = c.to_lookup_key() @@ -322,6 +335,12 @@ def test_prepare_commit_round_trip(remote, server_cache): xs.append(3) # the "function body" mutates the argument key = prepared.commit(xs) assert key == sealed # NOT c.to_lookup_key(): that drifted with the mutation + # The commit took two trips (result value, then the record), and nothing + # PreparedCall-shaped ever crossed the wire. + assert [m for m, _ in shipped if m in ("save_value", "save")] == ["save_value", "save"] + assert not any( + isinstance(a, PreparedCall) for _, args in shipped for a in args + ) lc = remote.load(key) assert lc.result == [1, 2, 3] # result: as returned assert lc.arguments["xs"] == [1, 2] # argument: as passed From ae092dd31c60a53daa99ad09404b144920d638dd Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Thu, 6 Aug 2026 16:30:27 -0400 Subject: [PATCH 15/27] docs(caches): trim protocol comments to their invariants (#830) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the remaining comment trims flagged in the #793 review (the rest already landed with #825). Narration and before/after history go; invariants stay: - `BaseCache.prepare` docstring: 20 → 15 lines; keeps the override contract, the digest-only base semantics, and the `digest(x) == values.save(x)` key-equality invariant. - `Cache.prepare`: drops the result-lifecycle narration (documented on `PreparedCall`/`save`); keeps the pre-body stash rationale and the no-cache-level-lock explanation. - `ReadOnlyMixin.prepare`: drops "the behavior save() rejection produced before the two-phase protocol, minus the wasted writes"; keeps the MRO reason and the seal-without-writing behavior. - `SshCache.prepare`: drops the ", as before" tail; keeps the Path-over-SSH caveat. Comment-only diff. Affected test dirs pass (191 tests), `ty check src/` clean. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk --- _Generated by [Claude Code](https://claude.ai/code/session_01QT4DxU9dVALrMvff7setnk)_ Co-authored-by: Claude --- src/fleche/caches.py | 38 +++++++++++++++----------------------- src/fleche/remote.py | 2 +- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index a81469e3..9c45c45b 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -53,20 +53,16 @@ def prepare(self, call: Call) -> PreparedCall: The first half of the two-phase save protocol. Caches that own a value storage (:class:`Cache`) override this to stash the argument - values *now* — before the function body runs — so the recorded - identity describes the arguments as they were at call time, even if - the body later mutates them. - - This base implementation is the digest-only admission used by caches - that cannot (or must not) write ahead of the body — read-only views, - aggregates without their own value storage: the key is sealed, but - nothing is written and whether a record survives is decided by - :meth:`save` at commit time. Because ``digest(x) == values.save(x)`` - for every storable value, both forms seal the same key. + values now, so the recorded identity describes the arguments as they + were at call time, even if the body later mutates them. This base + implementation is the digest-only admission for caches that cannot + (or must not) write ahead of the body — read-only views, aggregates + without their own storage; ``digest(x) == values.save(x)``, so both + forms seal the same key. Finish the returned :class:`~fleche.call.PreparedCall` with exactly - one of :meth:`~fleche.call.PreparedCall.commit` (store the result, - file the record) or :meth:`~fleche.call.PreparedCall.abandon`. + one of :meth:`~fleche.call.PreparedCall.commit` or + :meth:`~fleche.call.PreparedCall.abandon`. """ return PreparedCall(digested=call.digest(), cache=self) @@ -324,11 +320,9 @@ def load_value(self, key): def prepare(self, call: Call) -> PreparedCall: # Stash the arguments *now*, before the function body runs, so the - # record cannot end up keyed on post-mutation content. ``stash`` - # leaves the still-unknown result as ``None``; ``save`` below stores - # it once ``commit`` fills it in. No cache-level lock: the keys are - # only known once the value storage has digested each value, and value - # storages carry their own per-key locking where they need it. + # record cannot end up keyed on post-mutation content. No cache-level + # lock: the keys are only known once the value storage has digested + # each value, and value storages carry their own per-key locking. return PreparedCall(digested=call.stash(self.values), cache=self) def save(self, call: PreparedCall | Call) -> str: @@ -581,12 +575,10 @@ def evict(self, key: str | Digest) -> None: raise Rejected("Cannot evict from a read-only cache", self, key) def prepare(self, call: Call) -> "PreparedCall": - # Digest-only admission (the BaseCache default, restated because this - # mixin is base-free and must beat CacheWrapper.prepare in the MRO): a - # read-only cache stashes nothing, so the function body still runs with - # a correctly sealed key and the eventual commit is rejected by - # ``save`` above — the behavior save() rejection produced before the - # two-phase protocol, minus the wasted writes. + # Digest-only admission, restated from BaseCache because this mixin is + # base-free and must beat CacheWrapper.prepare in the MRO: nothing is + # stashed, the key is still sealed, and the commit is rejected by + # ``save`` above — the body runs and returns uncached. return PreparedCall(digested=call.digest(), cache=self) diff --git a/src/fleche/remote.py b/src/fleche/remote.py index e31a89a7..ef71cedb 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -831,7 +831,7 @@ def prepare(self, call: _call.Call) -> PreparedCall: # argument values before the body runs and seals the record, which # comes back for the client to complete with ``save``. Values travel # by cloudpickle, so a Path argument ships its path *string*, not its - # content — paths over SSH remain unsupported, as before. + # content — paths over SSH are unsupported. self._ensure_handshake() if self._info_cache and self._info_cache.get("read_only", False): # Digest-only admission, as BaseCache: the body still runs, and the From e2fe718dd54ab9ea0fa75e153f0d860c3feddfa7 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Thu, 6 Aug 2026 16:31:17 -0400 Subject: [PATCH 16/27] docs: split the purity contract out of file semantics; digest CompletedProcess (#831) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the four inline review threads on #797. - Move the general "fleche caches pure functions" contract — arguments keyed as passed, mutation and other side effects not replayed — out of file_semantics into a new usage/purity page. Verified it is not path-specific: a mutated list argument behaves identically. - Reframe "Paths nested inside containers" around destructuring, since only destructured children reach the path machinery, so _DESTRUCTURERS is the list of places a nested path gets content treatment. Also document why "any depth" holds at any remaining_depth: a Path matches no destructurer, so it is always written out as its own entry rather than inlined. - Move the CompletedProcess digester out of notebooks/Files.ipynb into digest.py as a match arm (args + returncode + stdout + stderr). Purely additive: those values raised Indigestible before, so no stored digest changes and no hash_version bump. - Commit Files.ipynb and PathsInContainers.ipynb executed, and wire PathsInContainers into docs/notebooks/, the toctree, and test_notebooks.py. Replace the generic query cell in Files.ipynb with the value store's own view of a stored path. Reframe the PathsInContainers "Edge N" sections as Beware (consequences of caching by value) vs Caveat (real limits of the mending machinery). --- agents/DEVELOPING.md | 5 +- docs/index.rst | 2 + docs/notebooks/PathsInContainers.ipynb | 1 + docs/usage/file_semantics.rst | 66 ++-- docs/usage/helpers.rst | 6 +- docs/usage/purity.rst | 101 +++++++ docs/usage/tldr.rst | 5 +- notebooks/Files.ipynb | 400 +++++++++++++++++++++---- notebooks/PathsInContainers.ipynb | 201 +++++++------ src/fleche/digest.py | 7 + tests/integration/test_notebooks.py | 1 + tests/unit/digest/test_digest.py | 64 ++++ 12 files changed, 689 insertions(+), 170 deletions(-) create mode 120000 docs/notebooks/PathsInContainers.ipynb create mode 100644 docs/usage/purity.rst diff --git a/agents/DEVELOPING.md b/agents/DEVELOPING.md index 3b25ae54..98a77385 100644 --- a/agents/DEVELOPING.md +++ b/agents/DEVELOPING.md @@ -174,8 +174,8 @@ Shared fixtures (in `fixtures.py`): - `benchmarks/` — `benchmark_{digest,integration,storage}.py`, `run_benchmarks.py`, `compare_results.py` (diff two `results.csv` runs), `utils.py`, `profile_digest_types.py` (per-type cProfile harness), `results.csv`. - `devnotes/storage-hierarchy.{dot,md,svg}` — rendered inheritance diagram for the storage classes. -- `docs/` — Sphinx sources, grouped by topic: root holds `index`, `installation`, `parallel_execution`; `usage/` holds `tldr`, `helpers`, `lazy_call`, `query` (`tldr` first, PR #745); `digests/` holds `digests_as_args`, `digest_equivalence`, `entry_points` (added by PR #752 as the dedicated fleche-ase / third-party plug-in page); `storage/` holds `configuration`, `cache_stack`, `security`; `dev/` holds `custom_digests`, `developer`, `ssh_cache`. `docs/notebooks/` is **symlinks** into `../../notebooks/` (six entries — `Caches` and `TransferWorkflow` are not exposed in docs); the `rendernb.yml` workflow re-executes `notebooks/*.ipynb` in place when a PR carries the `rendernb` label. -- `notebooks/` — usage examples (`GettingStarted`, `Caches`, `CacheStack`, `StorageBackends`, `SecureStorage`, `ConcurrentExecution`, `ExtraMethods`, `TransferWorkflow`); five of these (all except `Caches`, `ConcurrentExecution`, `TransferWorkflow`) are executed by `tests/integration/test_notebooks.py`. +- `docs/` — Sphinx sources, grouped by topic: root holds `index`, `installation`, `parallel_execution`; `usage/` holds `tldr`, `purity`, `helpers`, `file_semantics`, `lazy_call`, `query` (`tldr` first, PR #745; `purity` carries the general "fleche caches pure functions" contract — arguments keyed as passed, mutation and other side effects not replayed — that `file_semantics` states for paths); `digests/` holds `digests_as_args`, `digest_equivalence`, `entry_points` (added by PR #752 as the dedicated fleche-ase / third-party plug-in page); `storage/` holds `configuration`, `cache_stack`, `security`; `dev/` holds `custom_digests`, `developer`, `path_storage`, `ssh_cache`; `recipes/` holds `files_and_paths`. `docs/notebooks/` is **symlinks** into `../../notebooks/` (eight entries — `Caches` and `TransferWorkflow` are not exposed in docs); the `rendernb.yml` workflow re-executes `notebooks/*.ipynb` in place when a PR carries the `rendernb` label. +- `notebooks/` — usage examples (`GettingStarted`, `Caches`, `CacheStack`, `StorageBackends`, `SecureStorage`, `ConcurrentExecution`, `ExtraMethods`, `Files`, `PathsInContainers`, `TransferWorkflow`); seven of these (all except `Caches`, `ConcurrentExecution`, `TransferWorkflow`) are executed by `tests/integration/test_notebooks.py`. Committed **with outputs** — `rendernb.yml` re-executes them in place on a PR labelled `rendernb`, so an edited notebook should be re-run before it lands. - `.github/workflows/` — CI: `tests.yml` (PR sweep across 3.11–3.14 + `sql-backends` job that boots Postgres 16 + MariaDB 11 service containers and sets `FLECHE_TEST_{POSTGRES,MYSQL}_URL`), `ty.yml`, `test-minimum-deps.yml` (PR #645, installs every direct dep at its declared floor via `uv pip install --resolution lowest-direct -e ".[tests]"` and runs the suite — `uv pip install` rather than `uv sync` because the universal lock floats deps up through cross-extra constraints and would mask the real minimums), `benchmarks.yml`/`benchmarks-main.yml`/`updatebenchmarks.yml`, `perf-triage.yml` (PR #628 — Haiku reads the PR diff/description and adds the `benchmark` label when the change touches a hot path — that label is the existing `benchmarks.yml` trigger), `rendernb.yml` (re-executes `notebooks/*.ipynb` on PRs labelled `rendernb`), `release-please.yml`, `pypi-publish.yml` (trusted-publisher upload triggered by `release: published`), `claude.yaml` + `claude_ci_details.yaml` (the latter exposes CI status as a tool the in-PR Claude can call). Releases use **release-please** (`release-please-config.json`, `.release-please-manifest.json`) — release PRs are opened automatically from conventional-commit history on `main`. ## Design themes / open scope (issue tracker) @@ -217,6 +217,7 @@ Cheat sheet of what's been considered. Issue numbers are the entry points — fe - `CachePool(ReadOnlyMixin, _MultiCache)` — a read-only unordered collection of caches (PR #689). Extracted the `CacheStack` read fan-out into a shared `_MultiCache(BaseCache)` base (`_first_hit`/`_collect`/`_foreach` + `contains`/`load_value`/`expand`/`_shrink`/`_query`) so `CacheStack` and `CachePool` share one implementation. `save`/`evict` raise `Rejected`; config dispatches `{"pool": [...]}` alongside the existing list→`CacheStack` and `type="ssh"`→`SshCache` paths. Pinned in `tests/unit/caches/test_cache_pool.py`. - Cache-level race fixes on top of #569: #217 → PR #629, #451 → PR #631, #452 → PR #630, #485 → PR #627. Regression-pinned in `tests/regression/test_issue_{217,451,452,485}.py`; concurrency stress tests share `run_workers()` from `tests/fixtures.py` (PR #633). - Digest dispatch grew a `builtins`-type arm and a `not isinstance(value, type)` guard on the dataclass/attrs arms so `digest(int)` / `digest(SomeDataclass)` (the class object) stop raising — PR #651, closes #469. +- `digest()` grew a `subprocess.CompletedProcess` arm (args + returncode + stdout + stderr) so functions wrapping command-line tools hash without a user hook — `run()`'s result is neither a dataclass nor iterable, so it was `Indigestible` before. Purely additive (those values raised previously), so no `hash_version` bump. Pinned in `tests/unit/digest/test_digest.py::test_completedprocess_*`; `notebooks/Files.ipynb` used to register this by hand via `add_hook` and no longer does. - `digest()` grew explicit `pd.DataFrame` / `pd.Series` / `pd.Index` arms so pandas inputs hash by content (columns/name + dtype + index + `hash_pandas_object`), not by the column names yielded by `iter()` — PR #675. Silent change with no `hash_version` bump (previous pandas cache entries are now unreachable). Pinned in `tests/unit/digest/test_digest.py::test_pandas_{dataframe,series,index}_hashes_by_content`. - `SizeLimitedMixin` swapped its raw `threading.RLock` for `_PicklableRLock` so `SizeLimitedCache` survives `pickle.dumps` (PR #664, closes #663). - `MetaData.keys` is uniformly a `@property` across every built-in (`Runtime`/`Environment`/`Git`/`Tags`) instead of the prior mix of class-level dict attributes for the zero-arg three plus a property for `Tags` (PR #690, closes #637). The abstract-property-overridden-by-class-attribute smell on the ABC is gone. PR #763 (closes #738) then single-sourced the static-schema built-ins by moving the shared schema to a `_keys: ClassVar[dict[str, type]]` class attribute on the ABC (default `{}`) with `MetaData.keys` a concrete property returning `self._keys`; `Runtime`/`Environment`/`Git` now declare `_keys` once (dropping their per-class `keys` override), while `Tags` keeps overriding `keys` directly since its schema is per-instance. Drift guard: `tests/unit/metadata/test_metadata.py::test_builtin_metadata_pre_post_keys_match_schema` asserts `set(pre_output) | set(post_output) == set(cls._keys)` for each of the three static built-ins. `name` is still set by `@configurable` (`Runtime`/`Environment`/`Git`) or as a manual dataclass attribute (`Tags`) — unchanged. diff --git a/docs/index.rst b/docs/index.rst index 394d79a2..41b9581e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,7 @@ Welcome to the **Fleche** library documentation. :caption: Using Fleche usage/tldr + usage/purity usage/helpers usage/file_semantics usage/lazy_call @@ -84,6 +85,7 @@ Welcome to the **Fleche** library documentation. notebooks/CacheStack notebooks/ConcurrentExecution notebooks/Files + notebooks/PathsInContainers .. toctree:: :maxdepth: 2 diff --git a/docs/notebooks/PathsInContainers.ipynb b/docs/notebooks/PathsInContainers.ipynb new file mode 120000 index 00000000..6382d357 --- /dev/null +++ b/docs/notebooks/PathsInContainers.ipynb @@ -0,0 +1 @@ +../../notebooks/PathsInContainers.ipynb \ No newline at end of file diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst index 25e297ee..d8d5d50a 100644 --- a/docs/usage/file_semantics.rst +++ b/docs/usage/file_semantics.rst @@ -44,19 +44,20 @@ functions returning identical files share one stored body (see Argument mutation ~~~~~~~~~~~~~~~~~ -A call is keyed on its arguments **as passed**: argument content is captured -*before* the function body runs. A function that mutates its own argument — -most commonly, writing an output file *into* a directory it received — is -still recorded under the pre-call content, so honest repeat calls hit. The -mutation itself, however, is neither recorded nor replayed: a cache hit -leaves the argument untouched, so a side effect on the input happens on cold -calls only. - -fleche caches *pure* functions. What a function does to its arguments -without passing it back out is invisible to the cache — treat received paths -as read-only and write outputs to a fresh directory (``tempfile.mkdtemp``). -A mutated argument that *is* returned is captured faithfully in its final, -post-mutation state: if the mutation is the point, return it. +Arguments are keyed **as passed** — captured before the body runs — and a +mutation the body performs on one is neither recorded nor replayed. That is +a general rule (:ref:`argument-mutation`); its most common instance here is a +function that writes an output file *into* a directory it received. The call +is recorded under the directory's pre-call tree, so honest repeat calls hit, +but the written file does not reappear on a hit — it exists only on cold +calls. + +Returning what you wrote to is the normal shape, not a workaround for this: a +function that writes into a directory and hands that directory back is recorded +with it in its final, post-mutation state, and that is the intended way to +produce files. What is not replayed is a write to something you never return — +so a path that is pure *input* should be treated as read-only, with new files +written somewhere you do return (``tempfile.mkdtemp``). Only *content* changes count as mutation: permissions are not part of identity (see :ref:`fidelity-limits`), so a ``chmod`` on a received path is @@ -146,13 +147,30 @@ the file. If you need the file at a stable location, copy it out: Paths nested inside containers ------------------------------ -Paths are found and content-stored inside the containers fleche takes apart: -``dict``, ``OrderedDict``, ``list``, ``tuple`` (**exact types** — see below), -``dataclasses`` and ``attrs`` classes — nested to any depth, as values *or* as -dict keys. Everything above about identity, materialization, and lifetime -applies to each nested path individually. Container structure is otherwise -faithful on a hit: lists and tuples keep their element order, and a mended -dict keeps the insertion order the stored value had. +Whether a nested path is stored by content comes down to one thing: +**destructuring** — whether storage takes the surrounding container apart into +independently-stored children, or pickles it whole as one opaque value. Only +children reach the path machinery, so the list of containers fleche +destructures *is* the list of places a nested path gets content treatment. +That list is +:data:`~fleche.storage.destructuring._DESTRUCTURERS`: ``dict``, +``OrderedDict``, ``list``, ``tuple`` (**exact types** — see below), +``dataclasses`` and ``attrs`` classes. See :ref:`extending-destructurer` for +the mechanism and how to add your own container to it. + +Within those, paths are found nested to any depth, as values *or* as dict +keys, and everything above about identity, materialization, and lifetime +applies to each one individually. Container structure is otherwise faithful +on a hit: lists and tuples keep their element order, and a mended dict keeps +the insertion order the stored value had. + +"Any depth" is not a figure of speech, and no storage setting narrows it. A +``Path`` matches no destructurer, so it is always written out as its own stored +entry rather than inlined into the container above it — and being written out is +exactly what hands it to the content machinery. The +``remaining_depth`` knob only decides how eagerly *destructurable* nodes are +split into separate entries, so it cannot put a path out of reach however it is +set. Three caveats specific to nesting: @@ -175,7 +193,8 @@ Opaque containers store paths by *location* ------------------------------------------- A path inside an opaque value (a ``namedtuple``, a ``set``, an arbitrary -object) never reaches the content machinery. The call is still *keyed* +object) is never destructured out of it, and so never reaches the content +machinery. The call is still *keyed* correctly — the digest layer does look inside — but what is stored is the path object itself, pointing at wherever the file was when it was saved. A hit returns that original *location* (whether as the same object or an equal copy @@ -183,8 +202,9 @@ is backend-dependent — rely on neither): if the file has since been deleted or edited, the hit hands you a dangling or stale path, **without any warning**. Rule of thumb: return paths in plain dicts / lists / tuples / dataclasses. If -you need a custom container to participate, see -:func:`~fleche.storage.destructuring.register_destructurer`. +you need a custom container to participate, register a destructurer for it +with :func:`~fleche.storage.destructuring.register_destructurer` — see +:ref:`extending-destructurer`. .. _file-dedup: diff --git a/docs/usage/helpers.rst b/docs/usage/helpers.rst index 001da91e..4df64fb1 100644 --- a/docs/usage/helpers.rst +++ b/docs/usage/helpers.rst @@ -38,7 +38,7 @@ Attempts to load the result of a specific call from the cache. If the result is Returns ``True`` if the result for the given call is already present in the cache, ``False`` otherwise. ``.query(*args, metadata={}, **kwargs)`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Returns matching cached calls from the active cache. Any argument passed as ``None`` acts as a wildcard, matching any stored value for that parameter. The ``metadata`` keyword argument accepts a dictionary of metadata tags to further filter results (e.g., ``metadata={"tags": {"project": "alpha"}}``). @@ -102,7 +102,9 @@ Functions Returning ``None`` Functions that return ``None`` are **never cached**. When a decorated function returns ``None``, ``fleche`` logs a ``WARNING`` and skips the save step entirely. Subsequent calls will execute the function again rather than -returning a cached value. +returning a cached value — including any side effects they have, which for a +cached function would otherwise happen on the cold call only (see +:doc:`purity`). This applies to all code paths, with one difference for ``.rerun()``: diff --git a/docs/usage/purity.rst b/docs/usage/purity.rst new file mode 100644 index 00000000..e70dfe0e --- /dev/null +++ b/docs/usage/purity.rst @@ -0,0 +1,101 @@ +.. _purity: + +Purity and Side Effects +======================= + +What fleche assumes about the functions you decorate, and what therefore does +*not* survive a cache hit. The rules here apply to every argument and result +type; :doc:`file_semantics` is the same contract seen through ``Path`` values. + +The assumption +-------------- + +fleche caches **pure** functions: the result is determined by the arguments, +and anything else the body does is incidental. A cache hit replays the +*result* and nothing else — so every effect a function has besides returning +a value happens on cold calls only. + +That is not a restriction fleche can check, and it does not raise if you break +it. It shows up as a function that behaves differently the second time you +call it. + +.. _argument-mutation: + +Arguments are keyed as passed +----------------------------- + +Argument content is captured **before** the body runs, so a function that +mutates its own argument is still recorded under the pre-call content and +honest repeat calls hit. The mutation itself is neither recorded nor +replayed: + +.. code-block:: python + + @fleche + def append_and_report(xs, n): + xs.append(n) + return len(xs) + + a = [1, 2] + append_and_report(a, 9) # 3 — body runs, and a is now [1, 2, 9] + + b = [1, 2] + append_and_report(b, 9) # 3 — cache hit; b is still [1, 2] + +Both calls return ``3``, because that is the recorded result. Only the first +one changed its argument. Nothing about this is specific to lists: a +``dict``, a numpy array, and a directory a function writes into all behave the +same way. + +Return what you mutate +~~~~~~~~~~~~~~~~~~~~~~ + +An argument that is mutated **and returned** is captured faithfully in its +final, post-mutation state — the result is stored after the body runs: + +.. code-block:: python + + @fleche + def append_and_return(xs, n): + xs.append(n) + return xs + + append_and_return([1, 2], 9) # [1, 2, 9] — body runs + append_and_return([1, 2], 9) # [1, 2, 9] — cache hit, same value + +If the mutation is the point of the function, return it. Treat arguments you +receive as read-only otherwise, and build results fresh. + +Other side effects +------------------ + +Everything else a body does — printing, logging, writing files outside the +returned value, sending a request, inserting a row — happens on the cold call +and never again: + +.. code-block:: python + + @fleche + def record(x): + db.insert(x) # runs once, ever + return x * 2 + +If an effect must happen on every call, it belongs outside the cached +function; keep the cached part to the computation whose result you want +stored. + +Two related cases +----------------- + +* A function returning ``None`` is never cached, so it re-executes every time + — including its side effects. See :ref:`none-not-cached`. +* Paths follow all of the above, with the mutation case made concrete: + a directory a function receives and writes into is recorded under its + pre-call tree, and the written file does not reappear on a hit. See + :doc:`file_semantics`. + +See also +-------- + +* :doc:`file_semantics` — the same contract for files and directories. +* :doc:`helpers` — ``.rerun()`` for forcing a cold call deliberately. diff --git a/docs/usage/tldr.rst b/docs/usage/tldr.rst index e3332e9c..3aa938cf 100644 --- a/docs/usage/tldr.rst +++ b/docs/usage/tldr.rst @@ -60,8 +60,9 @@ directories are cached by **content** (not by path string), and a cache hit returns a freshly materialized copy under a temporary path. See :doc:`file_semantics` for the exact contract. -That is all you need for everyday use. The rest of this section covers the -helper methods in depth (:doc:`helpers`), the file/path contract +That is all you need for everyday use. The rest of this section covers what +fleche assumes about the functions you decorate (:doc:`purity`), the helper +methods in depth (:doc:`helpers`), the file/path contract (:doc:`file_semantics`), lazy loading of large cached objects (:doc:`lazy_call`), and querying stored calls (:doc:`query`); storage backends and configuration details live under :doc:`/storage/configuration`. diff --git a/notebooks/Files.ipynb b/notebooks/Files.ipynb index e3e72b12..f76494f3 100644 --- a/notebooks/Files.ipynb +++ b/notebooks/Files.ipynb @@ -8,8 +8,8 @@ "# Caching functions that read and write files\n", "\n", "`fleche` is a content-addressed cache. When a cached function takes or returns a\n", - "`pathlib.Path`, fleche stores the file's (or directory tree's) **contents** \u2014 not\n", - "just the path string \u2014 so results are portable and reproducible across machines.\n", + "`pathlib.Path`, fleche stores the file's (or directory tree's) **contents** — not\n", + "just the path string — so results are portable and reproducible across machines.\n", "\n", "A path is keyed by its content, so:\n", "\n", @@ -18,19 +18,37 @@ " file lives at a different location.\n", "\n", "The in-memory cache (`cache(\"memory\")`) and every default value storage now carry\n", - "this behaviour out of the box \u2014 no custom storage composition required." + "this behaviour out of the box — no custom storage composition required." ] }, { "cell_type": "code", + "execution_count": 1, "id": "cell-01", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:48.374404Z", + "iopub.status.busy": "2026-08-06T19:58:48.374124Z", + "iopub.status.idle": "2026-08-06T19:58:49.119391Z", + "shell.execute_reply": "2026-08-06T19:58:49.117923Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['PerKeyLockMixin', 'DestructuringMixin', 'PathValueMixin', 'ValueMixin']" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import tempfile\n", "from pathlib import Path\n", - "from subprocess import run, CompletedProcess\n", + "from subprocess import run\n", "\n", "import fleche as fl\n", "from fleche import fleche\n", @@ -44,10 +62,28 @@ }, { "cell_type": "code", + "execution_count": 2, "id": "cell-02", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.122636Z", + "iopub.status.busy": "2026-08-06T19:58:49.122082Z", + "iopub.status.idle": "2026-08-06T19:58:49.128658Z", + "shell.execute_reply": "2026-08-06T19:58:49.126851Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/tmp/tmpyxng_i3t-fleche-files')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# A scratch directory for the files our functions produce.\n", "WORK = Path(tempfile.mkdtemp(suffix=\"-fleche-files\"))\n", @@ -68,10 +104,26 @@ }, { "cell_type": "code", + "execution_count": 3, "id": "cell-04", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.132107Z", + "iopub.status.busy": "2026-08-06T19:58:49.131691Z", + "iopub.status.idle": "2026-08-06T19:58:49.141581Z", + "shell.execute_reply": "2026-08-06T19:58:49.140088Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'hello'\n", + "returned: PosixPath -> hellohello\n" + ] + } + ], "source": [ "@fleche\n", "def write(text, repeat=1, name=\"out.txt\"):\n", @@ -86,10 +138,25 @@ }, { "cell_type": "code", + "execution_count": 4, "id": "cell-05", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.144539Z", + "iopub.status.busy": "2026-08-06T19:58:49.144174Z", + "iopub.status.idle": "2026-08-06T19:58:49.151092Z", + "shell.execute_reply": "2026-08-06T19:58:49.149640Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "from cache: hellohello\n" + ] + } + ], "source": [ "# Same arguments -> cache hit -> the body does NOT run (no \"[write] running\").\n", "again = write(\"hello\", 2)\n", @@ -109,10 +176,28 @@ }, { "cell_type": "code", + "execution_count": 5, "id": "cell-07", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.153583Z", + "iopub.status.busy": "2026-08-06T19:58:49.153313Z", + "iopub.status.idle": "2026-08-06T19:58:49.166569Z", + "shell.execute_reply": "2026-08-06T19:58:49.165422Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'a quick brown fox'\n", + " [wordcount] running: sentence.txt\n", + "count: 4\n", + "count again: 4\n" + ] + } + ], "source": [ "@fleche\n", "def wordcount(path: Path):\n", @@ -137,10 +222,35 @@ }, { "cell_type": "code", + "execution_count": 6, "id": "cell-09", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.169824Z", + "iopub.status.busy": "2026-08-06T19:58:49.169546Z", + "iopub.status.idle": "2026-08-06T19:58:49.183369Z", + "shell.execute_reply": "2026-08-06T19:58:49.181356Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [make_tree] running: alpha\n" + ] + }, + { + "data": { + "text/plain": [ + "['sub', 'sub/leaf.bin', 'top.txt']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "@fleche\n", "def make_tree(seed):\n", @@ -163,10 +273,35 @@ }, { "cell_type": "code", + "execution_count": 7, "id": "cell-10", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.186305Z", + "iopub.status.busy": "2026-08-06T19:58:49.186041Z", + "iopub.status.idle": "2026-08-06T19:58:49.212754Z", + "shell.execute_reply": "2026-08-06T19:58:49.211003Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [total_bytes] running: ff88b2cec3e828c231a8a9df977be4f4cfa2e938cc0cdc41aac1e03350f5af8b\n" + ] + }, + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# End-to-end cache hit: make_tree(\"alpha\") and total_bytes both come from cache.\n", "total_bytes(make_tree(\"alpha\"))" @@ -185,10 +320,27 @@ }, { "cell_type": "code", + "execution_count": 8, "id": "cell-12", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.215720Z", + "iopub.status.busy": "2026-08-06T19:58:49.215476Z", + "iopub.status.idle": "2026-08-06T19:58:49.223612Z", + "shell.execute_reply": "2026-08-06T19:58:49.222050Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " [write] running: 'shared body'\n", + " [write] running: 'shared body'\n", + "shared body stored once: True\n" + ] + } + ], "source": [ "# Two calls, different names, identical body -> the content is stored once.\n", "write(\"shared body\", 1, name=\"left.txt\")\n", @@ -200,13 +352,37 @@ }, { "cell_type": "code", + "execution_count": 9, "id": "cell-13", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.227426Z", + "iopub.status.busy": "2026-08-06T19:58:49.227174Z", + "iopub.status.idle": "2026-08-06T19:58:49.232930Z", + "shell.execute_reply": "2026-08-06T19:58:49.231276Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FileBlob('out.txt', '785d68f8426805e292630852bdedb46dd56ac44dcb7047740d30704f3d84d4fa')\n", + "FileBlob('sentence.txt', '0f61b76af53fa2dc41528c3866206d22ceb3b7f560e7f42aacc006fc5ace228c')\n", + "DirectoryBlob({'leaf.bin': 'c70f6db1a5371bc6046fb5a040fd13bd5220c78908eac8126c1361daad854904'})\n", + "DirectoryBlob({'sub': '7aa02a10cd58fe7b0f16ef0e06b255d7842ce362687e489e941281d27761d95a', 'top.txt': '8eb42147b1727df4b082ebc0bdfc5fbaea064308411a4d801cae67c908ce4287'})\n", + "FileBlob('left.txt', '32cbd77d1dbff488cd42dc84ea72ebd47358fbf321412b35a5c1084e36f5b775')\n", + "FileBlob('right.txt', '32cbd77d1dbff488cd42dc84ea72ebd47358fbf321412b35a5c1084e36f5b775')\n" + ] + } + ], "source": [ - "# Introspect the recorded calls of any cached function.\n", - "write.query().table()" + "# What a `Path` actually becomes in storage: its content as plain `bytes` under\n", + "# its own digest, plus a small record pairing that content with a name. Note\n", + "# that left.txt and right.txt reference the *same* content digest.\n", + "for blob in c.values.storage.values():\n", + " if type(blob).__name__ in (\"FileBlob\", \"DirectoryBlob\"):\n", + " print(blob)\n" ] }, { @@ -220,22 +396,26 @@ "step runs in a working directory, produces files, and the whole pipeline is\n", "cached by content.\n", "\n", - "A `subprocess.CompletedProcess` isn't digestible out of the box, so we register a\n", - "digest hook describing how to fingerprint one." + "Each step returns the `subprocess.CompletedProcess` that `run()` produced. fleche\n", + "digests those directly — by `args`, `returncode`, `stdout`, and `stderr` — so the\n", + "captured output participates in the key with no extra setup. For a type fleche\n", + "does not know, `fl.digest.add_hook((TheType, fn))` is the extension point." ] }, { "cell_type": "code", + "execution_count": 10, "id": "cell-15", - "metadata": {}, - "execution_count": null, + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.235969Z", + "iopub.status.busy": "2026-08-06T19:58:49.235619Z", + "iopub.status.idle": "2026-08-06T19:58:49.245255Z", + "shell.execute_reply": "2026-08-06T19:58:49.243354Z" + } + }, "outputs": [], "source": [ - "def digest_completedprocess(cp):\n", - " return fl.digest.digest((type(cp).__name__, cp.args, cp.returncode, cp.stdout, cp.stderr))\n", - "\n", - "fl.digest.add_hook((CompletedProcess, digest_completedprocess))\n", - "\n", "@fleche\n", "def shell(cwd, prog, args=(), stdin=b\"\"):\n", " print(\" [shell] running:\", prog, *args)\n", @@ -255,10 +435,30 @@ }, { "cell_type": "code", + "execution_count": 11, "id": "cell-16", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.248014Z", + "iopub.status.busy": "2026-08-06T19:58:49.247696Z", + "iopub.status.idle": "2026-08-06T19:58:49.269994Z", + "shell.execute_reply": "2026-08-06T19:58:49.268579Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- first run ---\n", + " [pipeline] running: b'hello world'\n", + " [shell] running: cp input.txt copy.txt\n", + " [shell] running: tr a-z A-Z\n", + "produced: ['copy.txt', 'input.txt', 'shout.txt']\n", + "shout.txt: HELLO WORLD\n" + ] + } + ], "source": [ "print(\"--- first run ---\")\n", "out = pipeline(b\"hello world\")\n", @@ -268,10 +468,26 @@ }, { "cell_type": "code", + "execution_count": 12, "id": "cell-17", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.272480Z", + "iopub.status.busy": "2026-08-06T19:58:49.272198Z", + "iopub.status.idle": "2026-08-06T19:58:49.280138Z", + "shell.execute_reply": "2026-08-06T19:58:49.277972Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- second run: fully cached (no body / shell prints) ---\n", + "same files: ['copy.txt', 'input.txt', 'shout.txt']\n" + ] + } + ], "source": [ "print(\"--- second run: fully cached (no body / shell prints) ---\")\n", "out2 = pipeline(b\"hello world\")\n", @@ -280,10 +496,81 @@ }, { "cell_type": "code", + "execution_count": 13, "id": "cell-18", - "metadata": {}, - "execution_count": null, - "outputs": [], + "metadata": { + "execution": { + "iopub.execute_input": "2026-08-06T19:58:49.283069Z", + "iopub.status.busy": "2026-08-06T19:58:49.282802Z", + "iopub.status.idle": "2026-08-06T19:58:49.301525Z", + "shell.execute_reply": "2026-08-06T19:58:49.300406Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
namemoduletimestarttimestopwalltime
6f89shell__main__2026-08-06 19:58:49.251591921+00:002026-08-06 19:58:49.259691+00:000.008099
a668shell__main__2026-08-06 19:58:49.261044025+00:002026-08-06 19:58:49.264387608+00:000.003344
\n", + "
" + ], + "text/plain": [ + " name module timestart \\\n", + "6f89 shell __main__ 2026-08-06 19:58:49.251591921+00:00 \n", + "a668 shell __main__ 2026-08-06 19:58:49.261044025+00:00 \n", + "\n", + " timestop walltime \n", + "6f89 2026-08-06 19:58:49.259691+00:00 0.008099 \n", + "a668 2026-08-06 19:58:49.264387608+00:00 0.003344 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Every shell invocation fleche recorded:\n", "shell.query().table()" @@ -297,7 +584,16 @@ "name": "python3" }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" } }, "nbformat": 4, diff --git a/notebooks/PathsInContainers.ipynb b/notebooks/PathsInContainers.ipynb index 65ed0006..98c8849e 100644 --- a/notebooks/PathsInContainers.ipynb +++ b/notebooks/PathsInContainers.ipynb @@ -5,7 +5,7 @@ "id": "9050c7cf", "metadata": {}, "source": [ - "# Paths nested inside containers: the usual UX, and the edges\n", + "# Paths nested inside containers: the usual UX, and what to beware of\n", "\n", "[Files.ipynb](Files.ipynb) shows the sunny-day behaviour of `Path` arguments and\n", "return values: files and directories are stored by **content** and rematerialized\n", @@ -13,10 +13,19 @@ "\n", "This notebook explores what happens when paths are *nested inside other values* —\n", "dicts, lists, dataclasses — which fleche's `DestructuringMixin` takes apart and\n", - "reassembles (\"mends\") around the path machinery. The first half is the intended\n", - "UX; the second half collects the edge cases where a cache hit is **not** a faithful\n", - "replay of the original call: location changes, aliasing loss, path dict-keys, and\n", - "container types whose mending is incomplete or outright broken." + "reassembles (\"mends\") around the path machinery.\n", + "\n", + "The first half is the intended UX. The second half is what to watch out for, in\n", + "two distinct flavours:\n", + "\n", + "- **Beware** — consequences of what caching a pure function by content *means*.\n", + " A cached call is replayed by its **value**, so a file comes back as a copy: its\n", + " location, its identity, and whatever sat next to it on disk were never part of\n", + " that value. Code that leaned on them was relying on something the cache never\n", + " promised. These are not defects and they are not going to change.\n", + "- **Caveat** — real limits of the mending machinery, where a hit hands back\n", + " something less faithful than it could: paths used as dict keys, and paths\n", + " hidden inside containers fleche does not destructure.\n" ] }, { @@ -25,10 +34,10 @@ "id": "1ba85e32", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:11.833017Z", - "iopub.status.busy": "2026-07-25T19:23:11.832839Z", - "iopub.status.idle": "2026-07-25T19:23:12.221170Z", - "shell.execute_reply": "2026-07-25T19:23:12.220562Z" + "iopub.execute_input": "2026-08-06T19:58:50.542910Z", + "iopub.status.busy": "2026-08-06T19:58:50.542683Z", + "iopub.status.idle": "2026-08-06T19:58:51.227501Z", + "shell.execute_reply": "2026-08-06T19:58:51.225855Z" } }, "outputs": [ @@ -67,10 +76,10 @@ "id": "6825eb50", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.222514Z", - "iopub.status.busy": "2026-07-25T19:23:12.222262Z", - "iopub.status.idle": "2026-07-25T19:23:12.224735Z", - "shell.execute_reply": "2026-07-25T19:23:12.224328Z" + "iopub.execute_input": "2026-08-06T19:58:51.230438Z", + "iopub.status.busy": "2026-08-06T19:58:51.229894Z", + "iopub.status.idle": "2026-08-06T19:58:51.235385Z", + "shell.execute_reply": "2026-08-06T19:58:51.233846Z" } }, "outputs": [], @@ -102,10 +111,10 @@ "id": "96e644a1", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.226046Z", - "iopub.status.busy": "2026-07-25T19:23:12.225909Z", - "iopub.status.idle": "2026-07-25T19:23:12.231118Z", - "shell.execute_reply": "2026-07-25T19:23:12.230608Z" + "iopub.execute_input": "2026-08-06T19:58:51.238073Z", + "iopub.status.busy": "2026-08-06T19:58:51.237818Z", + "iopub.status.idle": "2026-08-06T19:58:51.248360Z", + "shell.execute_reply": "2026-08-06T19:58:51.247018Z" } }, "outputs": [ @@ -114,7 +123,7 @@ "output_type": "stream", "text": [ " [fit] running: alpha\n", - "cold: PosixPath -> /tmp/claude-1000/tmpddupkllp-fleche-nested/alpha-fit.txt\n" + "cold: PosixPath -> /tmp/tmp4rik9ggl-fleche-nested/alpha-fit.txt\n" ] } ], @@ -142,10 +151,10 @@ "id": "74dac313", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.232200Z", - "iopub.status.busy": "2026-07-25T19:23:12.232055Z", - "iopub.status.idle": "2026-07-25T19:23:12.234825Z", - "shell.execute_reply": "2026-07-25T19:23:12.234336Z" + "iopub.execute_input": "2026-08-06T19:58:51.250981Z", + "iopub.status.busy": "2026-08-06T19:58:51.250755Z", + "iopub.status.idle": "2026-08-06T19:58:51.257133Z", + "shell.execute_reply": "2026-08-06T19:58:51.255733Z" } }, "outputs": [ @@ -153,7 +162,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "warm: TempPath -> /tmp/claude-1000/tmphvfdmrplfleche/alpha-fit.txt\n", + "warm: TempPath -> /tmp/tmpqu7_eme9fleche/alpha-fit.txt\n", "content: loss=5 | name kept: alpha-fit.txt\n" ] } @@ -180,10 +189,10 @@ "id": "1a38c092", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.236049Z", - "iopub.status.busy": "2026-07-25T19:23:12.235897Z", - "iopub.status.idle": "2026-07-25T19:23:12.239485Z", - "shell.execute_reply": "2026-07-25T19:23:12.239067Z" + "iopub.execute_input": "2026-08-06T19:58:51.259707Z", + "iopub.status.busy": "2026-08-06T19:58:51.259397Z", + "iopub.status.idle": "2026-08-06T19:58:51.266926Z", + "shell.execute_reply": "2026-08-06T19:58:51.265381Z" } }, "outputs": [ @@ -209,15 +218,18 @@ "id": "4db3a027", "metadata": {}, "source": [ - "## Edge 1: a hit changes *where* (and what type) the path is\n", + "## Beware: a hit gives you a *copy*, somewhere else\n", "\n", "The cold call returns whatever the function returned — the real location in\n", "`WORK`, as a plain `Path`. A warm hit returns a `TempPath` in a fresh temporary\n", "directory. Content is identical; **location is not**.\n", "\n", - "The classic footgun: code that resolves *siblings* of a returned path\n", - "(`p.parent / \"meta.json\"`) works on the first call and breaks on every hit,\n", - "because the materialized file sits alone in its temp directory." + "The function's value is the file it returned, and a location is not part of a\n", + "file's content — so this is content addressing working exactly as advertised, not\n", + "a fidelity gap. The practical consequence: code that resolves *siblings* of a\n", + "returned path (`p.parent / \"meta.json\"`) works on the first call and breaks on\n", + "every hit, because only what was returned got captured — the sibling was never\n", + "part of the value.\n" ] }, { @@ -226,10 +238,10 @@ "id": "7fe21179", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.240747Z", - "iopub.status.busy": "2026-07-25T19:23:12.240600Z", - "iopub.status.idle": "2026-07-25T19:23:12.245110Z", - "shell.execute_reply": "2026-07-25T19:23:12.244626Z" + "iopub.execute_input": "2026-08-06T19:58:51.269472Z", + "iopub.status.busy": "2026-08-06T19:58:51.269194Z", + "iopub.status.idle": "2026-08-06T19:58:51.278562Z", + "shell.execute_reply": "2026-08-06T19:58:51.277139Z" } }, "outputs": [ @@ -239,7 +251,7 @@ "text": [ " [produce] running: beta\n", "cold sibling exists: True\n", - "warm location: /tmp/claude-1000/tmptm88ttu4fleche/beta-data.csv\n", + "warm location: /tmp/tmp822dn7defleche/beta-data.csv\n", "warm sibling exists: False\n" ] } @@ -273,11 +285,15 @@ "id": "8fcb480d", "metadata": {}, "source": [ - "## Edge 2: aliasing is not preserved\n", + "## Beware: aliasing is not part of a value\n", "\n", "Return the *same* path twice and the cold result holds one object in two slots.\n", "The warm hit mends each slot independently: two separate materializations, in two\n", - "different temp directories. Equal content, unequal (and non-identical) paths." + "different temp directories. Equal content, unequal (and non-identical) paths.\n", + "\n", + "Object identity is a property of one process's memory, not of the value being\n", + "cached — nothing about \"these two files are the same object\" survives a round\n", + "trip through storage, and nothing could. Compare content, never `is`.\n" ] }, { @@ -286,10 +302,10 @@ "id": "67496526", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.246344Z", - "iopub.status.busy": "2026-07-25T19:23:12.246201Z", - "iopub.status.idle": "2026-07-25T19:23:12.251214Z", - "shell.execute_reply": "2026-07-25T19:23:12.250671Z" + "iopub.execute_input": "2026-08-06T19:58:51.281270Z", + "iopub.status.busy": "2026-08-06T19:58:51.281055Z", + "iopub.status.idle": "2026-08-06T19:58:51.291226Z", + "shell.execute_reply": "2026-08-06T19:58:51.289901Z" } }, "outputs": [ @@ -323,7 +339,7 @@ "id": "78097da1", "metadata": {}, "source": [ - "## Edge 3: paths as dict *keys* mend into different keys\n", + "## Caveat: paths as dict *keys* mend into different keys\n", "\n", "Dict keys are destructured like values. A `Path` key comes back as a `TempPath`\n", "at a new location — so the mended dict has a *different key* than the original,\n", @@ -337,10 +353,10 @@ "id": "77ba67df", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.252600Z", - "iopub.status.busy": "2026-07-25T19:23:12.252431Z", - "iopub.status.idle": "2026-07-25T19:23:12.258461Z", - "shell.execute_reply": "2026-07-25T19:23:12.257826Z" + "iopub.execute_input": "2026-08-06T19:58:51.293639Z", + "iopub.status.busy": "2026-08-06T19:58:51.293387Z", + "iopub.status.idle": "2026-08-06T19:58:51.302382Z", + "shell.execute_reply": "2026-08-06T19:58:51.300962Z" } }, "outputs": [ @@ -349,8 +365,8 @@ "output_type": "stream", "text": [ " [index] running: delta\n", - "cold key: /tmp/claude-1000/tmpddupkllp-fleche-nested/delta-k.txt\n", - "warm key: /tmp/claude-1000/tmpxk4r0zg6fleche/delta-k.txt\n", + "cold key: /tmp/tmp4rik9ggl-fleche-nested/delta-k.txt\n", + "warm key: /tmp/tmpbt0e_qsqfleche/delta-k.txt\n", "kw[orig_key] works: False\n" ] } @@ -376,12 +392,18 @@ "id": "f5ea33da", "metadata": {}, "source": [ - "## Edge 4: every hit materializes a fresh copy, with temp-file lifetime\n", + "## Beware: each hit is its own copy, with temp-file lifetime\n", "\n", "Each hit copies the stored bytes into a new temporary directory (large files: mind\n", "the churn). The temp tree lives exactly as long as some `TempPath` derived from\n", "it is referenced — keep only a `str` of the location and the file is gone once the\n", - "path object is collected." + "path object is collected.\n", + "\n", + "Again this follows from the model rather than working against it: the cache owns\n", + "the content, and hands you a copy to use. It cannot know when you are finished\n", + "with that copy except by watching the reference you were given, so hold the\n", + "`Path` object for as long as you need the file, and `shutil.copy` it out if you\n", + "need it at a location of your own.\n" ] }, { @@ -390,10 +412,10 @@ "id": "88ba415b", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.259737Z", - "iopub.status.busy": "2026-07-25T19:23:12.259570Z", - "iopub.status.idle": "2026-07-25T19:23:12.314448Z", - "shell.execute_reply": "2026-07-25T19:23:12.313281Z" + "iopub.execute_input": "2026-08-06T19:58:51.304989Z", + "iopub.status.busy": "2026-08-06T19:58:51.304725Z", + "iopub.status.idle": "2026-08-06T19:58:51.389293Z", + "shell.execute_reply": "2026-08-06T19:58:51.388102Z" } }, "outputs": [ @@ -431,7 +453,7 @@ "id": "d8593d07", "metadata": {}, "source": [ - "## Edge 5: paths hidden in *opaque* containers are stored by location, not content\n", + "## Caveat: paths hidden in *opaque* containers are stored by location, not content\n", "\n", "Destructuring only recurses into what it knows: lists, tuples, dicts, dataclasses,\n", "attrs classes. Everything else — namedtuples (deliberately treated as opaque),\n", @@ -451,10 +473,10 @@ "id": "6e5db0b0", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.316163Z", - "iopub.status.busy": "2026-07-25T19:23:12.316065Z", - "iopub.status.idle": "2026-07-25T19:23:12.319779Z", - "shell.execute_reply": "2026-07-25T19:23:12.319316Z" + "iopub.execute_input": "2026-08-06T19:58:51.391523Z", + "iopub.status.busy": "2026-08-06T19:58:51.391305Z", + "iopub.status.idle": "2026-08-06T19:58:51.399526Z", + "shell.execute_reply": "2026-08-06T19:58:51.398213Z" } }, "outputs": [ @@ -463,7 +485,7 @@ "output_type": "stream", "text": [ " [bundle] running: zeta\n", - "warm type: PosixPath -> /tmp/claude-1000/tmpddupkllp-fleche-nested/zeta-nt.txt\n", + "warm type: PosixPath -> /tmp/tmp4rik9ggl-fleche-nested/zeta-nt.txt\n", "points at the ORIGINAL location: True\n" ] } @@ -488,10 +510,10 @@ "id": "5c4884f9", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.323854Z", - "iopub.status.busy": "2026-07-25T19:23:12.323678Z", - "iopub.status.idle": "2026-07-25T19:23:12.326503Z", - "shell.execute_reply": "2026-07-25T19:23:12.326114Z" + "iopub.execute_input": "2026-08-06T19:58:51.401931Z", + "iopub.status.busy": "2026-08-06T19:58:51.401678Z", + "iopub.status.idle": "2026-08-06T19:58:51.406926Z", + "shell.execute_reply": "2026-08-06T19:58:51.405586Z" } }, "outputs": [ @@ -499,7 +521,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "hit returns: /tmp/claude-1000/tmpddupkllp-fleche-nested/zeta-nt.txt\n", + "hit returns: /tmp/tmp4rik9ggl-fleche-nested/zeta-nt.txt\n", "exists: False <- dangling, no warning\n" ] } @@ -531,7 +553,7 @@ "id": "ab85f64e", "metadata": {}, "source": [ - "## Edge 6: container subclasses are opaque — deliberately\n", + "## Caveat: container subclasses are opaque — deliberately\n", "\n", "Mending rebuilds containers via `type(value)()`, a contract subclasses\n", "may repurpose: `defaultdict`'s first argument is a factory (would crash),\n", @@ -550,10 +572,10 @@ "id": "d19c337b", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.328117Z", - "iopub.status.busy": "2026-07-25T19:23:12.328034Z", - "iopub.status.idle": "2026-07-25T19:23:12.330973Z", - "shell.execute_reply": "2026-07-25T19:23:12.330638Z" + "iopub.execute_input": "2026-08-06T19:58:51.409279Z", + "iopub.status.busy": "2026-08-06T19:58:51.409056Z", + "iopub.status.idle": "2026-08-06T19:58:51.417030Z", + "shell.execute_reply": "2026-08-06T19:58:51.415673Z" } }, "outputs": [ @@ -562,7 +584,7 @@ "output_type": "stream", "text": [ " [by_kind] running: eta\n", - "warm: defaultdict {'files': [PosixPath('/tmp/claude-1000/tmpddupkllp-fleche-nested/eta-dd.txt')]}\n", + "warm: defaultdict {'files': [PosixPath('/tmp/tmp4rik9ggl-fleche-nested/eta-dd.txt')]}\n", "but the nested path is the ORIGINAL location: True\n" ] } @@ -587,10 +609,10 @@ "id": "ef2d0d13", "metadata": { "execution": { - "iopub.execute_input": "2026-07-25T19:23:12.332260Z", - "iopub.status.busy": "2026-07-25T19:23:12.332187Z", - "iopub.status.idle": "2026-07-25T19:23:12.335016Z", - "shell.execute_reply": "2026-07-25T19:23:12.334601Z" + "iopub.execute_input": "2026-08-06T19:58:51.419207Z", + "iopub.status.busy": "2026-08-06T19:58:51.419003Z", + "iopub.status.idle": "2026-08-06T19:58:51.425975Z", + "shell.execute_reply": "2026-08-06T19:58:51.424622Z" } }, "outputs": [ @@ -605,7 +627,7 @@ "\n", "register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None\n", " Register a custom container destructurer.\n", - "\n", + " \n", " *pred(value)* should return ``True`` for values this destructurer handles.\n", " *fn* must accept ``(intern, value)`` where *intern* is\n", " :meth:`DestructuringMixin._intern_rec`. Entries are appended after the\n", @@ -643,17 +665,18 @@ " values, or nested anywhere inside dicts / lists / tuples / dataclasses / attrs\n", " classes — arbitrarily deep. Content-addressed, dedup'd, survives deletion of\n", " the originals.\n", - "- **A hit is a copy, not a replay:** returned paths live in fresh temp\n", - " directories. Don't resolve siblings, don't compare locations, don't expect\n", - " aliasing, and keep a reference to the `Path` object for as long as you need the\n", - " file.\n", - "- **Don't key dicts by `Path`** if you'll look them up afterwards — keys mend\n", - " into new locations. Use `str(path)` or a stable ID.\n", - "- **Don't hide paths in opaque containers** (namedtuples, sets, plain classes,\n", - " and any container *subclass* — only exact `dict` / `OrderedDict` / `list` /\n", - " `tuple` are destructured): they are stored by location and come back stale or\n", - " dangling after the original moves on. `register_destructurer` is the opt-in\n", - " door for well-behaved custom containers." + "- **Beware — a hit is a copy, because a value is all that is cached:** returned\n", + " paths live in fresh temp directories. Don't resolve siblings, don't compare\n", + " locations, don't expect aliasing, and keep a reference to the `Path` object for\n", + " as long as you need the file. None of this is pending a fix; it is what\n", + " caching by content means.\n", + "- **Caveat — don't key dicts by `Path`** if you'll look them up afterwards — keys\n", + " mend into new locations. Use `str(path)` or a stable ID.\n", + "- **Caveat — don't hide paths in opaque containers** (namedtuples, sets, plain\n", + " classes, and any container *subclass* — only exact `dict` / `OrderedDict` /\n", + " `list` / `tuple` are destructured): they are stored by location and come back\n", + " stale or dangling after the original moves on. `register_destructurer` is the\n", + " opt-in door for well-behaved custom containers.\n" ] } ], @@ -673,7 +696,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.11" + "version": "3.11.15" } }, "nbformat": 4, diff --git a/src/fleche/digest.py b/src/fleche/digest.py index f4a36537..fc7dcf6e 100644 --- a/src/fleche/digest.py +++ b/src/fleche/digest.py @@ -6,6 +6,7 @@ import numbers from numbers import Number import struct +import subprocess from pathlib import Path import types import importlib.metadata @@ -345,6 +346,12 @@ def _digest_bytes(value: Any) -> bytes: m.update(_digest_bytes(value.__func__)) case property(): m.update(_digest_bytes((value.fget, value.fset, value.fdel))) + case subprocess.CompletedProcess(): + m.update( + _digest_bytes( + (value.args, value.returncode, value.stdout, value.stderr) + ) + ) case _ if isinstance(value, type) and value.__module__ == 'builtins': # Digest a built-in type (int, str, list, …) by its qualified name. # Restricted to the builtins module; user-defined types remain Indigestible. diff --git a/tests/integration/test_notebooks.py b/tests/integration/test_notebooks.py index e08dd491..6a58daeb 100644 --- a/tests/integration/test_notebooks.py +++ b/tests/integration/test_notebooks.py @@ -10,6 +10,7 @@ "SecureStorage.ipynb", "CacheStack.ipynb", "Files.ipynb", + "PathsInContainers.ipynb", ] diff --git a/tests/unit/digest/test_digest.py b/tests/unit/digest/test_digest.py index 4de054d2..2b80fc76 100644 --- a/tests/unit/digest/test_digest.py +++ b/tests/unit/digest/test_digest.py @@ -1,6 +1,8 @@ import cmath import datetime import struct +import subprocess +import sys import collections import collections.abc import types as types_module @@ -945,3 +947,65 @@ def test_non_builtin_type_raises_indigestible(t): """Non-builtin type objects (user-defined classes, dataclass classes) raise Indigestible.""" with pytest.raises(Indigestible): digest(t) + + +# --------------------------------------------------------------------------- +# subprocess.CompletedProcess +# --------------------------------------------------------------------------- + + +def _cp(args=("echo", "hi"), returncode=0, stdout=b"hi\n", stderr=b""): + return subprocess.CompletedProcess( + args=list(args), returncode=returncode, stdout=stdout, stderr=stderr + ) + + +def test_completedprocess_hashes_by_its_fields(): + """Wrapping shell tools is a first-class use case; `run()`'s result must hash.""" + assert digest(_cp()) == digest(_cp()) + + +@pytest.mark.parametrize( + "changed", + [ + pytest.param({"args": ("echo", "bye")}, id="args"), + pytest.param({"returncode": 1}, id="returncode"), + pytest.param({"stdout": b"other\n"}, id="stdout"), + pytest.param({"stderr": b"boom\n"}, id="stderr"), + ], +) +def test_completedprocess_distinguishes_each_field(changed): + assert digest(_cp()) != digest(_cp(**changed)) + + +def test_completedprocess_is_not_digest_equal_to_its_field_tuple(): + """The type name salts the hash, as for every other arm.""" + cp = _cp() + assert digest(cp) != digest((cp.args, cp.returncode, cp.stdout, cp.stderr)) + + +@pytest.mark.parametrize( + "kwargs", + [ + pytest.param({"stdout": "hi\n", "stderr": ""}, id="text-mode-str"), + pytest.param({"stdout": None, "stderr": None}, id="streams-not-captured"), + ], +) +def test_completedprocess_handles_uncaptured_and_text_streams(kwargs): + """`capture_output=False` leaves None; `text=True` leaves str.""" + assert digest(_cp(**kwargs)) == digest(_cp(**kwargs)) + + +def test_completedprocess_from_a_real_run(): + a = subprocess.run([sys.executable, "-c", "print('x')"], capture_output=True) + b = subprocess.run([sys.executable, "-c", "print('x')"], capture_output=True) + c = subprocess.run([sys.executable, "-c", "print('y')"], capture_output=True) + assert digest(a) == digest(b) + assert digest(a) != digest(c) + + +def test_completedprocess_nested_in_a_result(): + """The notebook's shape: a function returning (path-ish, CompletedProcess).""" + cp = _cp() + assert digest(("workdir", cp)) == digest(("workdir", _cp())) + assert digest({"ret": cp}) != digest({"ret": _cp(returncode=2)}) From 4e2083e7028925fbe753fbd0a329cee6dcfa4f75 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 20:48:49 +0000 Subject: [PATCH 17/27] fix(ci): silence the three ty diagnostics this branch introduces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ty check src/` passes on `main` but reports three diagnostics here, so the ty workflow fails for every PR into this branch (and now for the branch itself). Both are false positives on code that is correct at runtime: - `DigestedMapping.mend` / `_rebuild_plain` call `type(value)(...)`, which ty narrows to `type[Mapping]`. `Mapping` is an ABC with no `__init__`, so the call resolves to `object.__init__` and the argument reads as one positional too many. The runtime type is always a concrete `dict`/`OrderedDict`. - `TempPath(type(Path()))` uses a dynamic base, through which ty cannot resolve an MRO. The indirection is required while Python 3.11 is supported — bare `Path` only became subclassable in 3.12. Suppressed rather than reworked: a `cast` on the two constructors would trade a checker complaint for an indirection, and the `TempPath` base cannot change until the 3.11 floor moves. Comments are anchored on the exact lines ty reports, which is where it binds them. `ty check src/` (pinned 0.0.62, as CI runs it): All checks passed! Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- src/fleche/storage/destructuring.py | 4 ++-- src/fleche/storage/paths.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fleche/storage/destructuring.py b/src/fleche/storage/destructuring.py index b2b682ff..5054957f 100644 --- a/src/fleche/storage/destructuring.py +++ b/src/fleche/storage/destructuring.py @@ -130,7 +130,7 @@ def underlying(self): def mend(self, storage: 'DestructuringMixin') -> Mapping: return type(self.items)( - (self.get(storage, k), self.get(storage, v)) + (self.get(storage, k), self.get(storage, v)) # ty: ignore[too-many-positional-arguments] for k, v in self.items.items() ) @@ -143,7 +143,7 @@ def _slots(cls, value: Mapping) -> list[tuple[None, Any]]: @classmethod def _rebuild_plain(cls, value: Mapping, labels: tuple, children: tuple) -> Mapping: n = len(value) - return type(value)(zip(children[:n], children[n:])) + return type(value)(zip(children[:n], children[n:])) # ty: ignore[too-many-positional-arguments] @classmethod def _rebuild_digest(cls, value: Mapping, labels: tuple, children: tuple) -> 'DigestedMapping': diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py index 6d0e13b8..66738775 100644 --- a/src/fleche/storage/paths.py +++ b/src/fleche/storage/paths.py @@ -9,7 +9,7 @@ from .. import digest -class TempPath(type(Path())): +class TempPath(type(Path())): # ty: ignore[unsupported-base] """ A Path that deletes its backing temp tree when no references remain. Paths derived via /, .parent, .with_suffix, etc. share the same From 0ddf18f59f1893d6e68c0b78ae4b25194cb052c4 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Thu, 6 Aug 2026 16:59:18 -0400 Subject: [PATCH 18/27] fix(remote): refuse Path values at the SshCache boundary (#828) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers your comment on #797, ["Interactions with ssh cache unclear"](https://github.com/pmrv/fleche/pull/797#issuecomment-5198004797). Targets `temppath`. The four **inline** review threads on #797 moved to #831, which has since **merged** — `temppath` was merged back in here (`f938975`, no conflicts), so this PR is the `SshCache` × `Path` work only. ## What the interaction actually was Probed with a real `python -m fleche remote --serve` subprocess, client and server in different working directories so the same relative name denotes different files on each side. Three behaviours, all silent: | | before | |---|---| | server *can* see something at that name | stores **its** bytes, returns **its** digest — `digest(x) == save_value(x)` broken, so the record is filed under a key no client recomputes, and a load returns the wrong content | | server can see nothing there | `Indigestible` surfaces from inside an RPC | | loading a path stored remotely | server materializes into a temp dir on **its** disk and sends back only the name — dangling here, and unlinked there as soon as its own reference dies | The last one is not fixable by holding a reference: `PurePath.__reduce__` rebuilds from `parts` alone, so `TempPath`'s `_temp_root` is dropped in transit and `_live_roots` is per-process. The `save_value` comment already said "paths over SSH remain unsupported" — nothing enforced it. ## What this does Refuses, in both directions, via a new `RemotePathUnsupported`. It subclasses `SaveError`, so the two-phase-save degradations already in the codebase carry it and no caller special-cases anything: - path **argument** → digest-only reference, **digest computed locally**. The seal stays intact, lookups hit and miss correctly; only the bytes aren't retrievable from the remote record. - path **result** → `Rejected`. The call runs, returns the file, is logged as not cached. - **load** → raises, lazily. A record whose result is a path still loads and queries; only touching the path value raises. Detection is `storage.paths.find_path`, walking a value through a new `storage.destructuring.child_slots` — the read-only half of `_intern_rec`'s dispatch — so nesting is covered exactly as far as a destructuring save reaches, and no further (a path inside an opaque object is still stored by location, same as locally). Escape hatches, both documented: return `bytes`, or put a local layer in front of the remote one so saves never reach the wire. ## Follow-up (second commit) Per your review, the problem itself is tracked as **#829** rather than left as a comment. The shape is already implied by `PathValueMixin`: it reduces a path to `bytes` plus a `FileBlob`/`DirectoryBlob`, all of which ship fine, and those blobs' `__digest__` is *defined* to match the `Path` digest arm — so running the reduction **client-side** keeps the seal intact by construction, and needs no server change for saves (a `FileBlob` isn't a `Path`, so it falls straight through the server's path layer). The one genuinely new verb is an unmended `load_value` that returns the blob instead of materializing it on the server. Recorded in `RemotePathUnsupported`, `dev/path_storage`, and the DEVELOPING design-theme cheat sheet. ## Docs - `usage/file_semantics.rst`: new "Paths stop at a remote (SSH) cache" contract section + a quick-reference row. - `dev/path_storage.rst`: the mechanism, and the #829 sketch. - `recipes/files_and_paths.rst`: a recipe. `SshCache`'s own docstring too. ## Tests `tests/unit/test_remote.py` — bare and nested (list/tuple/nested-dict/dataclass) rejection with nothing reaching the server; the `bytes` hatch still working; `prepare()` keeping `prepared.key == call.to_lookup_key()` for a path argument (the regression the guard buys); live-`Call` rejection; load-side rejection incl. nested; a `LazyCall` whose arguments still read fine and only `.result` raises. `tests/unit/storage/test_paths.py` — `find_path` descent, opaque-leaf boundary, cycle termination. `tests/integration/test_remote.py` — the divergent-cwd reproduction above, and a path-returning `@fleche` function running uncached against a remote instead of breaking. Re-verified after merging `temppath`: **1788 passed / 11 skipped**, docs build succeeds. `ty check src/` reports **3** diagnostics — all already on `temppath`, none from this PR; see [the #831 thread](https://github.com/pmrv/fleche/pull/831#issuecomment-5208623452) for the diagnosis and a verified fix, still pending your call on where it should land. --------- Co-authored-by: Claude --- agents/DEVELOPING.md | 2 +- docs/dev/path_storage.rst | 47 ++++++++++++ docs/recipes/files_and_paths.rst | 18 +++++ docs/usage/file_semantics.rst | 36 +++++++++ src/fleche/remote.py | 81 +++++++++++++++++++- src/fleche/storage/__init__.py | 6 +- src/fleche/storage/destructuring.py | 30 ++++++++ src/fleche/storage/paths.py | 33 ++++++++ tests/integration/test_remote.py | 70 ++++++++++++++++- tests/unit/storage/test_paths.py | 73 +++++++++++++++++- tests/unit/test_remote.py | 115 +++++++++++++++++++++++++++- 11 files changed, 500 insertions(+), 11 deletions(-) diff --git a/agents/DEVELOPING.md b/agents/DEVELOPING.md index 98a77385..23b54605 100644 --- a/agents/DEVELOPING.md +++ b/agents/DEVELOPING.md @@ -185,7 +185,7 @@ Cheat sheet of what's been considered. Issue numbers are the entry points — fe **Active design themes** - **Cache thread-safety / concurrency** (umbrella #444). Enabling refactor #569 is **fully landed** (PRs #601, #604, #622): `BaseCache(OperationContext)` and `Cache(PerKeyLockMixin, BaseCache)` — lock mixins now attach at the cache layer too. Storage layer was already thread-safe (per-key locks; #213, #449 closed). Cache-level races whose **fixes landed**: #217 (back-fill via PR #629, issue closed), #452 (transfer contains→save TOCTOU via PR #630, issue closed), #485 (in-flight dedup window via PR #627, issue closed). #451 (redigest atomicity) — PR #631 shipped the cross-key lock but the **issue is still open** pending follow-up review; cite the PR, not "#451 closed". Still open: `gc` #450, `expand` lock scope #453. (Wrapper check→execute→save #448 closed not-planned.) `BackgroundSaveMixin` (#447) is the planned vehicle for moving disk I/O off the hot path while `save()` keeps returning a key synchronously. - **Performance hot-spots** (#625, #440). One open digest hot-spot: `_digest_bytes` returns hex-encoded bytes that double parent SHA256 input at every nesting level — fix needs a `hash_version` bump + `Cache.redigest` migration. BLAKE3 tree-hashing was the structural alternative; the hash-function recon issue (#614, **closed 2026-06-02**) concluded with **blake2b(digest_size=32)** as the cheaper migration (stdlib, ~35-40% faster on tree workloads, 64-char hex preserved so `DIGEST_LENGTH` / SQL schema / `D()` pass-through all stay put). #615 is the open Option-A switchover; semver-breaking (invalidates persistent keys; users run `redigest()` once on upgrade). `Perf audit:` issues are refreshed periodically — check the latest one (currently #625, refreshed 2026-07-02) for current numbers; previous audits #527 (2026-05-28), and the older ones it supersedes, are closed. The 2026-06-11/18/25 runs all pin PR #622's cache-layer `PerKeyLockMixin` double-lock penalty (×1.54–2.0 on memory-backend miss/save across all data_heavy workloads — hit/contains unchanged); the storage-layer side of the same double-lock motivated the `init=False, hash=False` storage-dict option in #634's review, which the `__init_subclass__` guard in PR #683 sidestepped without addressing the perf side, so the double-lock cost is still open. The 2026-06-18 run's ×13 H5+Sql data_heavy miss spike did **not** recur on 2026-06-25 (7.8 ms vs 8.3 ms) — confirmed I/O jitter, not a code regression — but `BagOfHoldingH5File` still pays a structural ×5 over `PickleFile` on save (full HDF5 file open/close per `put`/`get`); fix candidates are h5py SWMR / pooled file handles in `bagofholding_file.py`. Hot-spot #2 from earlier audits (per-element `sha256()` allocation in recursive iterables/dicts) was retracted on review — the per-element context is load-bearing for the Merkle-tree property and has no cheap fix. SQL evict hot-spot from the 2026-05-21 audit was resolved by PR #535 (bulk DELETE in place of ORM materialisation), and the earlier BagOfHolding double-open by PR #616. -- **Distributed / remote caching** (#552; #551 **landed**). Still open: #552 — a `TieredValues` + `GlobusValues` cold tier for HPC value blobs that `Cache.query()` never touches, only `Cache.load_value()` on a hot miss. +- **Distributed / remote caching** (#552; #551 **landed**). `SshCache` refuses `Path` values outright (`RemotePathUnsupported`, a `SaveError` subclass so a path argument degrades to a locally-computed digest-only reference and a path result becomes `Rejected`) — a pickled `Path` is only its string, so the server was resolving it against its own filesystem and filing records under keys no client recomputes. **#829** tracks making them actually work: run the path→blob reduction client-side (the blobs' `__digest__` already matches the `Path` digest arm) plus one new unmended `load_value` verb. Still open: #552 — a `TieredValues` + `GlobusValues` cold tier for HPC value blobs that `Cache.query()` never touches, only `Cache.load_value()` on a hot miss. - **Config redesign** (#568). Next-generation TOML/YAML schema with top-level `value`/`call`/`stash`/`metadata` namespaces and cross-file named references. Supersedes the just-landed merged-discovery model (PR #553) once the schema firms up; expect a `hash_version`-style migration story. - **Metadata extensions** (#567). `Resources` metadata (peak memory, CPU time) is the next built-in to add on top of the `Runtime` / `Environment` / `Git` / `Tags` set. A standalone `Version` metadata (PR #573) was deliberately folded into `Environment`, which now records `fleche_version` + `python_version` — so a separate version namespace is *not* the direction. - **Path / file handling** (#516, #517, #522, #33). Pushing toward first-class `Path` support where files are transparently stashed and restored. Open questions: digest of empty/missing files (#517), unifying `FilePath`/`DirectoryPath` under `Path` (#33). `isolate=True` removal is in flight via PR #523 (closes #522). diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index 9b40db5d..9f757140 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -93,6 +93,53 @@ name is treated as incidental (typically a temp dir). If a root name carries meaning, name a *child* meaningfully instead, or wrap the tree's identity in your own value. +Why paths stop at the SSH boundary +---------------------------------- + +:class:`~fleche.remote.SshCache` is the one cache where "store this path" +cannot mean what it means everywhere else, because the two sides do not share +a filesystem. Values cross the wire by cloudpickle, and a pickled ``Path`` is +only its string — so a path handed to the remote is a *name it resolves +itself*. Three things follow, and each of them is silent: + +* ``save_value`` on the remote runs :class:`PathValueMixin` against the + **server's** disk. If a file happens to sit at that name it is stored, under + the digest of *those* bytes; ``digest(path) == values.save(path)`` — the + invariant the seal in :meth:`~fleche.caches.BaseCache.prepare` rests on — + is broken, and the record lands under a key no client ever recomputes. +* if nothing sits there, the server raises ``Indigestible`` from the middle of + an RPC. +* ``load_value`` materializes into a temp directory on the **server** and can + only ship the name back. The :class:`TempPath` guard does not survive the + hop either — ``PurePath.__reduce__`` reconstructs from ``parts`` alone, so + the ``_temp_root`` attribute is dropped and the class-level ``_live_roots`` + registry is per-process — so the server frees the tree as soon as its own + reference dies, and the client is left with a dangling name for a filesystem + it cannot see. + +So :class:`~fleche.remote.SshCache` refuses instead, via +:func:`~fleche.storage.paths.find_path` (which walks a value exactly the way a +destructuring save does, using +:func:`~fleche.storage.destructuring.child_slots`) and +:class:`~fleche.remote.RemotePathUnsupported`. That exception subclasses +:class:`~fleche.storage.SaveError` so the existing two-phase-save degradations +carry it: a path argument becomes a digest-only reference **whose digest was +computed locally**, which keeps the seal intact and lookups correct, and a path +result becomes :class:`~fleche.caches.Rejected`, so the call runs uncached +rather than wrongly cached. See :ref:`file-remote-caches` for the user-facing +version. + +Making paths genuinely work over SSH is a separate feature, tracked in +`issue #829 `_. The shape is +already visible above: :meth:`PathValueMixin.save` reduces a path to ``bytes`` +plus a :class:`FileBlob` / :class:`DirectoryBlob`, all of which ship fine, and +those blobs' ``__digest__`` is *defined* to match the ``Path`` arm — so running +the reduction **client-side** keeps the seal intact by construction. The one +new piece is on the load side: an unmended ``load_value`` that hands back the +blob instead of materializing it on the server, so the client materializes into +its own temp directory and owns the :class:`TempPath` lifetime. That is a +change to the RPC surface, not to this module. + See also -------- diff --git a/docs/recipes/files_and_paths.rst b/docs/recipes/files_and_paths.rst index 93e5f245..0c97b0f2 100644 --- a/docs/recipes/files_and_paths.rst +++ b/docs/recipes/files_and_paths.rst @@ -78,3 +78,21 @@ their names. directory's own ``.name`` is a hash (its children's names are faithful). Don't rely on the top-level directory name surviving a cache hit; do rely on everything inside it. + +Sharing files with a remote (SSH) cache +--------------------------------------- + +Return ``bytes``. A :class:`~pathlib.Path` cannot cross a +:class:`~fleche.remote.SshCache` — only the path string would travel, and the +remote would resolve it against its own filesystem — so fleche refuses it and +runs the call uncached. Content ships and deduplicates normally: + +.. code-block:: python + + @fleche + def render(text) -> bytes: # not `-> Path` + return _render_to_pdf(text) + +If you want path semantics *and* a remote cache, put a local layer in front of +it: saves land in the local layer, so paths never reach the wire. See +:ref:`file-remote-caches` for the full contract. diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst index d8d5d50a..a2c081ba 100644 --- a/docs/usage/file_semantics.rst +++ b/docs/usage/file_semantics.rst @@ -217,6 +217,41 @@ and re-caching adds a tiny name record, never a second copy of the bytes. If you want *keys* (not just storage) to ignore the name too, return ``bytes`` instead of a ``Path``; that is the content-only escape hatch. +.. _file-remote-caches: + +Paths stop at a remote (SSH) cache +---------------------------------- + +Everything above assumes the cache and your process see the **same +filesystem**. A :class:`~fleche.remote.SshCache` does not: it forwards each +operation to a fleche running on another machine, and values travel by +cloudpickle — which reduces a ``Path`` to its *string*. Only the name would +arrive, and the remote would resolve it against its own filesystem, storing +whatever happens to sit there (or nothing) under a digest that no longer +matches yours. + +Rather than store something else under your key, fleche refuses: +:class:`~fleche.remote.RemotePathUnsupported` is raised when a path — bare or +nested in a container — would cross the wire. Because it is a +:class:`~fleche.storage.SaveError`, the usual degradations apply and your code +does not have to catch anything: + +* a path **argument** falls back to a digest-only reference. The call is still + keyed correctly (the digest is computed *here*, from your file), so lookups + hit and miss exactly as they should; only the file's bytes are not retrievable + from the remote record. +* a path **result** is rejected: the call runs, returns your file, and is + logged as not cached. +* **loading** a path stored on the remote raises too. The remote materializes + into a temp directory on *its* disk and can only send back the name, which + means nothing here. This is lazy — a record whose result is a path can still + be loaded and queried; only touching the path value raises. + +To share file content across machines, return the file's ``bytes``: they +travel and deduplicate normally. To keep full path semantics, put a local +cache layer in front of the remote one (see :doc:`/notebooks/CacheStack`) — the +local layer is where saves land, so paths never reach the wire. + Quick reference --------------- @@ -232,6 +267,7 @@ Return same path twice Two independent copies Return path inside namedtuple/set/custom class The *original* location (may be stale) Pass a nonexistent ``Path`` argument No caching: warns, always executes Write into a directory you received Hit (keyed as passed); write not replayed +Return a ``Path`` through an ``SshCache`` Refused: runs, warns, not cached =============================================== ====================================== See also diff --git a/src/fleche/remote.py b/src/fleche/remote.py index a51fd189..71494f41 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -59,6 +59,8 @@ from .caches import BaseCache, Rejected from .call import DigestedCall, LazyCall, QueryCall from .digest import Digest +from .storage.base import SaveError +from .storage.paths import find_path logger = logging.getLogger("fleche.remote") @@ -346,6 +348,35 @@ class RemoteConnectionError(RuntimeError): """Raised when the SSH subprocess cannot be reached or has died.""" +class RemotePathUnsupported(SaveError): + """Raised when a :class:`~pathlib.Path` would cross the SSH boundary. + + Path values are stored **by content** (see + :class:`~fleche.storage.paths.PathValueMixin`), but the RPC ships values + by cloudpickle, and a pickled ``Path`` is just its *string*. Letting one + through would hand the server a name to resolve against its **own** + filesystem: it stores whatever happens to sit at that location — a + different file, or nothing — under a digest that no longer matches the + client's ``digest(path)``. The mismatch silently breaks the + ``digest(x) == save_value(x)`` seal every lookup depends on, so the + record is filed under a key no client can ever reproduce. + + Subclasses :class:`~fleche.storage.base.SaveError` so the standard + save-side degradations apply and nothing has to special-case it: a path + *argument* falls back to a digest-only reference (correct local digest, + lookups still hit, only the bytes are unavailable remotely), and a path + *result* turns into :class:`~fleche.caches.Rejected` — the call runs and + returns normally, it just is not cached. On the **load** side it + propagates to the caller instead: a path the server materialized into + its own temp directory is meaningless here, and the guard's whole point + is to say so rather than hand back a dangling name. + + This is a guard, not a verdict: doing the path-to-blob reduction on the + *client* would make paths work over SSH with the seal intact. Tracked in + issue #829. + """ + + def _warn_on_version_skew(info: dict[str, Any]) -> None: """Log a warning when the remote's fleche or cloudpickle version differs. @@ -707,6 +738,13 @@ class SshCache(BaseCache): ControlPersist in ``~/.ssh/config`` or via *ssh_options* to share the underlying connection across multiple fleche runs. + **Filesystem paths do not cross this cache.** Values travel by + cloudpickle, so a :class:`~pathlib.Path` would arrive as a bare string for + the remote to resolve against its own disk; both directions raise + :class:`RemotePathUnsupported` instead. Ship the file's ``bytes``, or put + a local layer in front of the remote one — see that exception and + :ref:`file-remote-caches`. + Args: host: SSH target, e.g. ``"user@host"`` or any alias from ``~/.ssh/config``. @@ -785,20 +823,55 @@ def _rpc(self, name: str, *args: Any) -> Any: raise Rejected(self, *args) return spec.unwrap(self, self._conn.call(name, *args)) + @staticmethod + def _reject_path(value: Any, what: str) -> None: + """Raise :class:`RemotePathUnsupported` if *value* carries a ``Path``. + + Scans the same object graph a destructuring save would walk, so a + path nested in a list, dict, dataclass, or ``attrs`` instance is + caught too — those reach the server's value storage exactly like a + bare one and fail the same way. + """ + path = find_path(value) + if path is not None: + raise RemotePathUnsupported( + f"{what} carries the filesystem path {str(path)!r}, which cannot " + "cross an SshCache: values travel by cloudpickle, so only the " + "path string would arrive and the remote would resolve it " + "against its own filesystem. Return the file's `bytes` instead " + "to store its content, or keep path-valued calls in a local " + "cache layer." + ) + def save(self, call: DigestedCall | _call.Call) -> str: + # A DigestedCall carries only digests; the degenerate live-`Call` + # form still carries values, and the server would stash them itself. + if isinstance(call, _call.Call): + try: + self._reject_path(dict(call.arguments), "the call's arguments") + self._reject_path(call.result, "the call's result") + except RemotePathUnsupported as e: + raise Rejected(e) from None return self._rpc("save", call) def load(self, key: str) -> LazyCall: return self._rpc("load", key) def load_value(self, key: str) -> Any: - return self._rpc("load_value", key) + # The server materializes a stored path into a temp directory on + # *its* filesystem and can only ship us the name — which points + # nowhere here, and which the server unlinks as soon as its own + # reference dies. Refuse it rather than return a dangling path. + value = self._rpc("load_value", key) + self._reject_path(value, f"the value at {key}") + return value def save_value(self, value: Any) -> Digest: # One round trip per value (no batching yet). Values travel by - # cloudpickle, so a Path value ships its path *string*, not its - # content — the same limitation the previous ship-the-whole-Call - # protocol had; paths over SSH remain unsupported. + # cloudpickle, so a Path would ship its path *string*, not its + # content — see `RemotePathUnsupported` for why that is refused + # rather than silently stored against the server's filesystem. + self._reject_path(value, "the value") return self._rpc("save_value", value) def evict(self, key: str | Digest) -> None: diff --git a/src/fleche/storage/__init__.py b/src/fleche/storage/__init__.py index 59ab51e7..db6b5cba 100644 --- a/src/fleche/storage/__init__.py +++ b/src/fleche/storage/__init__.py @@ -16,8 +16,8 @@ CallStorage, CallMixin, ) -from .destructuring import DestructuringMixin, register_destructurer -from .paths import PathValueMixin, TempPath, FileBlob, DirectoryBlob +from .destructuring import DestructuringMixin, child_slots, register_destructurer +from .paths import PathValueMixin, TempPath, FileBlob, DirectoryBlob, find_path from .memory import ValueMemory, CallMemory from .void import ValueVoid, CallVoid from .file import FileStorage @@ -38,11 +38,13 @@ "CallStorage", "CallMixin", "DestructuringMixin", + "child_slots", "register_destructurer", "PathValueMixin", "TempPath", "FileBlob", "DirectoryBlob", + "find_path", "ValueMemory", "CallMemory", "ValueVoid", diff --git a/src/fleche/storage/destructuring.py b/src/fleche/storage/destructuring.py index 5054957f..4320db6c 100644 --- a/src/fleche/storage/destructuring.py +++ b/src/fleche/storage/destructuring.py @@ -243,6 +243,36 @@ def _field_items(value: Any) -> list[tuple[str, Any]]: ] +def child_slots(value: Any) -> list[tuple[Any, Any]] | None: + """Enumerate the ``(label, child)`` pairs a save would recurse into. + + The read-only half of :meth:`DestructuringMixin._intern_rec`'s dispatch: + same leaf rules (numbers / strings / bytes / digests / namedtuples are + opaque), same ``_DESTRUCTURERS`` lookup, but it only *reports* the + children instead of interning them. ``None`` means "opaque leaf" — the + value has no children a destructuring save would look inside. + + Labels follow the underlying :meth:`Digested._slots`: field names for + dataclass / ``attrs`` instances, ``None`` for the positional slots of + lists, tuples, and mappings. + + A destructurer registered through :func:`register_destructurer` is only + visible here if it is a :class:`Digested` classmethod (i.e. exposes + ``_slots``); anything else is reported as an opaque leaf, since there is + no way to enumerate its children without also storing them. + """ + if isinstance(value, (digest.Digest, Number, str, bytes)): + return None + if isinstance(value, tuple) and DestructuringMixin._is_trojan_tuple(value): + return None + for pred, sunder_fn in _DESTRUCTURERS: + if pred(value): + owner = getattr(sunder_fn, "__self__", None) + slots = getattr(owner, "_slots", None) + return None if slots is None else slots(value) + return None + + def register_destructurer(pred: Callable[[Any], bool], fn: Callable) -> None: """Register a custom container destructurer. diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py index 66738775..313b8c0d 100644 --- a/src/fleche/storage/paths.py +++ b/src/fleche/storage/paths.py @@ -6,9 +6,42 @@ import weakref from . import base +from . import destructuring from .. import digest +def find_path(value: Any) -> Path | None: + """Return a :class:`~pathlib.Path` a save of *value* would reach, if any. + + Walks *value* exactly the way a destructuring save does — via + :func:`~fleche.storage.destructuring.child_slots`, so lists, tuples, + dicts, dataclasses, and ``attrs`` instances are descended into and + everything else is a leaf — and stops at the first ``Path`` it reaches, + returning ``None`` if the value carries no path at all. Which path comes + back when there are several is unspecified: this answers "is there one", + and the path itself is there to name in the resulting error. + + This is a *predicate helper*, not part of the storage protocol: it exists + so a caller that cannot honour path semantics (notably + :class:`fleche.remote.SshCache`, where a path's meaning does not survive + the hop to another filesystem) can detect the situation up front instead + of silently storing something else. Shared cycles are visited once. + """ + seen: set[int] = set() + stack: list[Any] = [value] + while stack: + item = stack.pop() + if isinstance(item, Path): + return item + if id(item) in seen: + continue + seen.add(id(item)) + slots = destructuring.child_slots(item) + if slots: + stack.extend(child for _, child in slots) + return None + + class TempPath(type(Path())): # ty: ignore[unsupported-base] """ A Path that deletes its backing temp tree when no references remain. diff --git a/tests/integration/test_remote.py b/tests/integration/test_remote.py index 1c2f136b..e70c0ed1 100644 --- a/tests/integration/test_remote.py +++ b/tests/integration/test_remote.py @@ -6,15 +6,19 @@ parsing on the server side, and the ``Popen`` handshake on the client side. """ +import logging import os import subprocess import sys import textwrap +from pathlib import Path import pytest +from fleche import cache, fleche from fleche.call import Call -from fleche.remote import SshCache, _Connection +from fleche.digest import digest +from fleche.remote import RemotePathUnsupported, SshCache, _Connection class _LocalSubprocessConnection(_Connection): @@ -165,3 +169,67 @@ def test_subprocess_named_cache(tmp_path): finally: sc_main.close() sc_alt.close() + + +# --------------------------------------------------------------------------- +# Paths stop at the SSH boundary +# --------------------------------------------------------------------------- + + +def test_relative_path_would_resolve_against_the_servers_own_cwd( + tmp_path, monkeypatch +): + """The divergence the path guard exists for, reproduced in one process. + + Client and server run in different working directories, so the same + relative name denotes a *different file* on each side — the single-machine + stand-in for "the remote is another filesystem". Only the path string + crosses the wire, so without the guard the server would digest and store + its own file under a key the client can never reproduce, and a later load + would hand back the wrong bytes. + """ + remote_root = tmp_path / "remote" + remote_root.mkdir() + client_root = tmp_path / "client" + client_root.mkdir() + (client_root / "data.txt").write_text("client bytes") + (remote_root / "data.txt").write_text("SERVER BYTES - a different file entirely") + # The two sides genuinely disagree about what "data.txt" contains. + assert digest(client_root / "data.txt") != digest(remote_root / "data.txt") + + monkeypatch.chdir(client_root) + sc = _build_remote(remote_root) + try: + with pytest.raises(RemotePathUnsupported): + sc.save_value(Path("data.txt")) + finally: + sc.close() + + +def test_path_returning_function_runs_uncached_against_a_remote(tmp_path, caplog): + """A rejected result must not break the call — it just isn't cached. + + ``Rejected`` is the cache's "I won't keep this" signal, which the + decorator logs and swallows, so the user still gets their file back. + """ + remote_root = tmp_path / "remote" + remote_root.mkdir() + sc = _build_remote(remote_root) + runs = [] + + @fleche + def produce(name): + runs.append(name) + f = tmp_path / f"{name}.txt" + f.write_text(name) + return f + + try: + with cache(sc), caplog.at_level(logging.WARNING): + assert produce("out").read_text() == "out" + assert produce("out").read_text() == "out" + finally: + sc.close() + + assert runs == ["out", "out"] # never cached, never wrong + assert any("rejected save" in r.message.lower() for r in caplog.records) diff --git a/tests/unit/storage/test_paths.py b/tests/unit/storage/test_paths.py index 8247344a..0bf18951 100644 --- a/tests/unit/storage/test_paths.py +++ b/tests/unit/storage/test_paths.py @@ -9,7 +9,13 @@ from fleche.storage import ValueMixin, DestructuringMixin from fleche.storage.memory import MemoryBackend from fleche.storage.destructuring import DigestedMapping -from fleche.storage.paths import TempPath, PathValueMixin, FileBlob, DirectoryBlob +from fleche.storage.paths import ( + TempPath, + PathValueMixin, + FileBlob, + DirectoryBlob, + find_path, +) def test_mkdtemp_returns_temp_path(): @@ -327,3 +333,68 @@ def test_empty_directory_path_roundtrip(pds, tmp_path): assert isinstance(loaded, Path) assert loaded.is_dir() assert list(loaded.iterdir()) == [] + + +# --------------------------------------------------------------------------- +# find_path: does a save of this value reach a Path? +# --------------------------------------------------------------------------- + + +@dataclass +class _Fields: + a: object + b: object = None + + +class _Opaque: + """Not a container a destructuring save looks inside.""" + + def __init__(self, p): + self.p = p + + +def test_find_path_finds_a_bare_path(tmp_path): + assert find_path(tmp_path) is tmp_path + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: [0, [p]], id="nested-list"), + pytest.param(lambda p: (p,), id="tuple"), + pytest.param(lambda p: {"k": p}, id="dict-value"), + pytest.param(lambda p: {p: "v"}, id="dict-key"), + pytest.param(lambda p: _Fields(a=1, b={"deep": [p]}), id="dataclass-field"), + ], +) +def test_find_path_descends_the_containers_a_save_descends(tmp_path, wrap): + assert find_path(wrap(tmp_path)) is tmp_path + + +@pytest.mark.parametrize( + "value", + [ + pytest.param([1, "two", b"three"], id="scalars"), + pytest.param({}, id="empty"), + pytest.param("/not/a/path/just/a/string", id="path-shaped-string"), + ], +) +def test_find_path_returns_none_without_a_path(value): + assert find_path(value) is None + + +def test_find_path_stops_at_opaque_values(tmp_path): + """A Path inside a plain object is stored as part of that object. + + ``find_path`` mirrors the destructuring walk exactly: what it does not + descend into, path storage does not intercept either. + """ + assert find_path(_Opaque(tmp_path)) is None + + +def test_find_path_terminates_on_a_cycle(tmp_path): + cyclic = [1] + cyclic.append(cyclic) + assert find_path(cyclic) is None + cyclic.append(tmp_path) + assert find_path(cyclic) is tmp_path diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 0ccf5da2..745a819d 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -5,11 +5,13 @@ only the transport is swapped. """ +import dataclasses import io import os import sys import threading import types +from typing import Any import pytest @@ -17,9 +19,10 @@ from fleche.call import Call, QueryCall from fleche.caches import Cache, Rejected from fleche.config import cache_to_config, load_cache_config -from fleche.digest import Digest +from fleche.digest import Digest, digest from fleche.remote import ( RemoteConnectionError, + RemotePathUnsupported, SshCache, _Connection, _dispatch, @@ -29,7 +32,7 @@ _write_frame, serve, ) -from fleche.storage import CallMemory, ValueMemory +from fleche.storage import CallMemory, SaveError, ValueMemory class _PipeConnection(_Connection): @@ -933,3 +936,111 @@ def test_run_server_serves_active_cache_over_stdio_until_eof(monkeypatch, cache_ assert info["cache"] == cache_to_config(expected_cache) # Clean EOF: no second frame queued. assert fake_stdout.read() == b"" + + +# --------------------------------------------------------------------------- +# Paths do not cross the wire +# --------------------------------------------------------------------------- + + +@dataclasses.dataclass +class _Holder: + payload: Any + + +@pytest.fixture +def a_file(tmp_path): + f = tmp_path / "data.txt" + f.write_text("client bytes") + return f + + +def test_save_value_rejects_a_bare_path(remote, server_cache, a_file): + """A Path would arrive as a bare string the server resolves itself.""" + with pytest.raises(RemotePathUnsupported) as excinfo: + remote.save_value(a_file) + assert str(a_file) in str(excinfo.value) + # Refused before the RPC: nothing reached the server. + assert not server_cache.values.storage + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: [1, p], id="list"), + pytest.param(lambda p: (p,), id="tuple"), + pytest.param(lambda p: {"a": {"b": p}}, id="nested-dict"), + pytest.param(lambda p: _Holder(payload=[p]), id="dataclass"), + ], +) +def test_save_value_rejects_a_nested_path(remote, server_cache, a_file, wrap): + """Destructuring walks into containers, so the guard has to as well.""" + with pytest.raises(RemotePathUnsupported): + remote.save_value(wrap(a_file)) + assert not server_cache.values.storage + + +def test_save_value_still_accepts_the_bytes_escape_hatch(remote, a_file): + """Returning ``bytes`` is the documented way to ship content remotely.""" + key = remote.save_value(a_file.read_bytes()) + assert remote.load_value(key) == b"client bytes" + + +def test_rejection_is_a_save_error(a_file): + """``SaveError`` is what the two-phase save protocol degrades on.""" + assert issubclass(RemotePathUnsupported, SaveError) + + +def test_prepare_degrades_a_path_argument_to_a_digest_only_reference(remote, a_file): + """The sealed key must still be the one a later lookup computes. + + This is the regression the guard buys: letting the path through made the + server digest *its* view of that name, so the record was filed under a + key no client could reproduce. + """ + call = Call(name="f", arguments={"p": a_file, "n": 3}) + prepared = remote.prepare(call) + assert prepared.key == call.to_lookup_key() + # The argument survives as its (locally computed) digest, not as content. + assert prepared.digested.arguments["p"] == digest(a_file) + + +def test_saving_a_live_call_carrying_a_path_is_rejected(remote, server_cache, a_file): + """The one-shot form ships values too — the server would stash them.""" + with pytest.raises(Rejected): + remote.save(Call(name="f", arguments={"p": a_file}, result=1)) + with pytest.raises(Rejected): + remote.save(Call(name="f", arguments={"x": 1}, result=[a_file])) + assert not server_cache.calls.storage + + +def test_load_value_refuses_a_path_materialized_on_the_server( + remote, server_cache, a_file +): + """A path stored remotely comes back pointing into the server's temp dir. + + Materialization happens on the server's filesystem and only the *name* + travels back, so the client would be handed a dangling reference — one + the server unlinks as soon as its own reference dies. + """ + key = server_cache.save_value(a_file) # stored server-side, by content + with pytest.raises(RemotePathUnsupported) as excinfo: + remote.load_value(key) + assert key in str(excinfo.value) + + +def test_load_value_refuses_a_path_nested_in_a_container( + remote, server_cache, a_file +): + key = server_cache.save_value({"out": [a_file]}) + with pytest.raises(RemotePathUnsupported): + remote.load_value(key) + + +def test_lazy_call_only_trips_on_the_path_value(remote, server_cache, a_file): + """Records stay queryable; only touching the path value fails.""" + key = server_cache.save(Call(name="f", arguments={"x": 1}, result=a_file)) + lc = remote.load(key) + assert dict(lc.arguments) == {"x": 1} + with pytest.raises(RemotePathUnsupported): + lc.result From 8fe8227c1f2de070b0ecd8f4fa9607cf33caeca9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 22:00:58 +0000 Subject: [PATCH 19/27] docs(paths): explain why remaining_depth cannot reach a nested path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `PathValueMixin` only sees values `DestructuringMixin` decided to write out — an inlined value rides inside its parent's `Digested` wrapper and is pickled with it, never handed down the MRO. So "a nested path is always stored by content" holds only because a path can never be inlined: it matches no destructurer, so its depth stays `float("inf")` and `inf < remaining_depth` is false at every setting. That is the mechanism behind the guarantee, not a quirk beside it, and it was previously undocumented. Records the propagation too, since a parent's depth is `1 + max(child_depths)`: every container between the root and a path is written out as its own entry, while sibling subtrees inline exactly as they would without the path. One path therefore costs its own nesting depth in extra entries, not the size of the structure around it — so tuning `remaining_depth` on a path-heavy workload changes how the non-path parts are packed and nothing else. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- docs/dev/path_storage.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index 9f757140..3637fa13 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -64,6 +64,42 @@ tree — with the ``"FileBlob"`` / ``"DirectoryBlob"`` salts matching :class:`~fleche.storage.paths.FileBlob` and :class:`~fleche.storage.paths.DirectoryBlob`'s ``__digest__``. +Why ``remaining_depth`` cannot reach a path +------------------------------------------- + +``PathValueMixin`` only ever sees a value that +:class:`~fleche.storage.destructuring.DestructuringMixin` decided to **write +out** — an inlined value is carried inside its parent's ``Digested`` wrapper and +pickled with it, never handed down the MRO. So "a nested path is always stored +by content" holds only if a path can never be inlined, and that is arranged +rather than hoped for: a ``Path`` matches no destructurer, so ``_intern_rec`` +leaves its depth at ``float("inf")``, and ``inf < remaining_depth`` is false for +every setting. + +That is the enforcement mechanism, not a side effect of one. It also +propagates, since a parent's depth is ``1 + max(child_depths)``: every container +between the root and a path inherits ``inf`` and is written out as its own +entry too. Sibling subtrees are untouched and inline exactly as they would +without the path. + +``[[[1, 2], [3, path]], [4, 5]]`` at ``remaining_depth=10`` — a setting that +collapses the same structure without the path into a *single* entry: + +.. code-block:: text + + bytes b'xyz' + FileBlob FileBlob('data.txt', ) + DigestedIterable [3, ] <- innermost, own entry + DigestedIterable [[1, 2], ] <- own entry; [1, 2] inlined + DigestedIterable [, [4, 5]] <- root; [4, 5] inlined + +So one path costs *its own nesting depth* in extra entries, not the size of the +structure around it, and the scalars beside it inline under the usual rules +(``[1, 3, 4, path]`` stores as ``DigestedIterable([1, 3, 4, ])``, not as +four separate slots). Tuning ``remaining_depth`` for a path-heavy workload +therefore changes how the *non-path* parts are packed and nothing else — the +content addressing of the paths is not on the table. + Deduplication ------------- From 28df4cd87898f2d294ee4df157b81e116def8003 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 18:57:12 +0000 Subject: [PATCH 20/27] fix(storage): keep path content reachable from the gc walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding 1 on #797: `Cache.gc()` permanently destroyed every path-valued entry it swept past. A stored path is a name plus a *reference*: the content lives under its own digest and the `FileBlob`/`DirectoryBlob` record points at it. The reachability walk never saw that reference, so the content looked orphaned and `gc()` reclaimed it, leaving a record pointing at nothing. The next `load(key).result` raised `KeyError`, which the wrapper reports as an ordinary cache miss — so a routine maintenance sweep silently emptied the cache of every file and directory in it. Worse for being advertised: `PreparedCall.abandon()` tells users to run gc to reclaim orphans. Two causes, both fixed: - The blobs declared no children. `_raw_sub_digests` is now a cooperative chain: `DestructuringMixin` reports its `Digested` wrappers and delegates anything else down the MRO, `PathValueMixin` reports a `FileBlob`'s content and a `DirectoryBlob`'s entries, and `ValueStorage` terminates it. A storage's reachable set is the union over its layers instead of whatever the top layer happens to answer. - The walk read through `load`, which mends. Mending resolves child references away — a materialized `Path` no longer knows which blob it came from — so the walk could not have seen them even once declared. It now reads `ValueStorage.load_raw`, the same read with every mending layer skipped (layers override `load` only). That also drops the second half of the finding: `gc` and `count_reuses` no longer copy every stored file into a temp tree purely to ask what it points at. Pinned in tests/unit/caches/test_gc.py: content survives gc for a file, a directory tree, and both nested in a dict; an orphaned path is still evicted; blobs report their children; and the walk materializes nothing. Five of the six fail against the previous source. 1869 passed, 11 skipped; `ty check src/` clean; docs build clean, with the invariant written up in dev/path_storage. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- docs/dev/path_storage.rst | 28 +++++++++ src/fleche/storage/base.py | 30 ++++++++++ src/fleche/storage/destructuring.py | 7 ++- src/fleche/storage/paths.py | 16 +++++ tests/unit/caches/test_gc.py | 90 +++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+), 3 deletions(-) diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index 3637fa13..175cae03 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -100,6 +100,34 @@ four separate slots). Tuning ``remaining_depth`` for a path-heavy workload therefore changes how the *non-path* parts are packed and nothing else — the content addressing of the paths is not on the table. +Blobs must declare their references +----------------------------------- + +Storing a path by content splits it in two: the ``bytes`` live under their own +content digest, and the :class:`~fleche.storage.paths.FileBlob` / +:class:`~fleche.storage.paths.DirectoryBlob` record holds a *reference* to +them. That makes the blob a node in the value store's reference graph, and +anything walking that graph — :meth:`~fleche.caches.Cache.gc`, +:meth:`~fleche.storage.destructuring.DestructuringMixin.count_reuses` — has to +be told so. A blob that reports no children looks like a leaf, its content +looks unreferenced, and ``gc`` reclaims the bytes out from under it; the entry +survives as a record pointing at nothing, and the next load raises +``KeyError``, which the wrapper reports as an ordinary cache miss. Silent data +loss, triggered by routine maintenance. + +:meth:`PathValueMixin._raw_sub_digests` therefore declares them, and each +mixin that wraps values in a record of its own does the same for its own +wrappers, delegating anything it does not recognise down the MRO — so a +storage's reachable set is the union over its layers rather than whichever +layer happens to answer first. + +The walk reads through :meth:`~fleche.storage.base.ValueStorage.load_raw` +rather than ``load``, for two reasons. Correctness: ``load`` mends, and +mending resolves child references away — a materialized ``Path`` no longer +knows which blob it came from. Cost: mending a stored path means copying its +whole tree into a temp directory, so a graph walk through ``load`` would +rebuild every stored file on disk purely to ask what it points at. + Deduplication ------------- diff --git a/src/fleche/storage/base.py b/src/fleche/storage/base.py index dae3077b..0079969a 100644 --- a/src/fleche/storage/base.py +++ b/src/fleche/storage/base.py @@ -348,6 +348,33 @@ def save(self, value: Any, key: Digest | None = None) -> Digest: ... @abstractmethod def load(self, key: Digest | str) -> Any: ... + def load_raw(self, key: Digest | str) -> Any: + """The entry exactly as stored, with no mending applied. + + ``load`` runs the whole mending chain — destructured children are + rewired back in, a stored path is materialized into a temp tree. That + is right for callers who want the *value*, and wrong for callers who + want the *record*: mending resolves child references away, so a + reference-graph walk (``gc``, ``count_reuses``) that reads through + ``load`` cannot see them and concludes the children are unreachable. + + Mending layers override ``load`` only, so this default is already raw + for storages that do none; :class:`ValueMixin` overrides it with the + bare backend read. + """ + return self.load(key) + + def _raw_sub_digests(self, raw: Any) -> set[Digest]: + """Digests directly referenced by a raw stored entry. + + The terminating case of a cooperative chain: every mixin that wraps + values in a record of its own reports that record's child references + here and delegates the rest upward, so a storage's reachable set is + the union over its layers. An entry no layer claims references + nothing. + """ + return set() + class ValueMixin(ValueStorage, StorageBackend): """Bridges :class:`~fleche.storage.base.ValueStorage` with :class:`~fleche.storage.base.StorageBackend` primitives. @@ -365,6 +392,9 @@ def save(self, value: Any, key: Digest | None = None) -> Digest: return self.put(value, key) def load(self, key: Digest | str) -> Any: + return self.load_raw(key) + + def load_raw(self, key: Digest | str) -> Any: with self._operation_context(key): key = self._normalize_key(key) logger.debug("Loading value with key %s", key) diff --git a/src/fleche/storage/destructuring.py b/src/fleche/storage/destructuring.py index ea86ab4c..217778eb 100644 --- a/src/fleche/storage/destructuring.py +++ b/src/fleche/storage/destructuring.py @@ -410,7 +410,8 @@ def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: case DigestedFields(): return {v for v in raw.fields.values() if isinstance(v, digest.Digest)} case _: - return set() + # Not one of ours — a layer below may still claim it. + return super()._raw_sub_digests(raw) def child_digests(self, key: digest.Digest | str) -> set[digest.Digest]: """Direct digest children of the raw entry stored at *key*. @@ -423,7 +424,7 @@ def child_digests(self, key: digest.Digest | str) -> set[digest.Digest]: Raises: KeyError: if *key* is not present in the underlying backend. """ - return self._raw_sub_digests(super().load(key)) + return self._raw_sub_digests(self.load_raw(key)) def count_reuses(self) -> Counter[digest.Digest]: """Return a counter of how many times each stored key is referenced as a sub-component. @@ -450,7 +451,7 @@ def count_reuses(self) -> Counter[digest.Digest]: """ counts: Counter[digest.Digest] = Counter({key: 0 for key in self.list()}) for key in list(counts): - for sub in self._raw_sub_digests(super().load(key)): + for sub in self._raw_sub_digests(self.load_raw(key)): if sub in counts: counts[sub] += 1 return counts diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py index e037c9ed..d2104a14 100644 --- a/src/fleche/storage/paths.py +++ b/src/fleche/storage/paths.py @@ -255,6 +255,22 @@ def load(self, key: digest.Digest | str) -> Any: return path return value + def _raw_sub_digests(self, raw: Any) -> set[digest.Digest]: + """Report the content blobs a stored path record points at. + + Without this a reachability walk sees a :class:`FileBlob` / + :class:`DirectoryBlob` as a childless leaf, so the ``bytes`` holding + the actual file content look unreferenced and ``gc`` reclaims them — + destroying every path-valued entry it sweeps past. The blobs are + precisely a *name plus references*, so the references have to be + declared here, at the layer that creates them. + """ + if isinstance(raw, FileBlob): + return {raw.content} + if isinstance(raw, DirectoryBlob): + return set(raw.contents.values()) + return super()._raw_sub_digests(raw) + def _materialize(self, path: Path, blob: DirectoryBlob) -> None: path.mkdir() for name, child_ref in blob.contents.items(): diff --git a/tests/unit/caches/test_gc.py b/tests/unit/caches/test_gc.py index 2da9d424..d851b5df 100644 --- a/tests/unit/caches/test_gc.py +++ b/tests/unit/caches/test_gc.py @@ -144,3 +144,93 @@ def test_gc_evicts_deeply_unreachable_structure(split_cache): assert orphan_leaf_only in evicted # Previously-live keys still present. assert reachable_before.issubset(set(split_cache.values.list())) + + +# ---- Path values: blob records reference their content, and gc must see it ---- + + +@pytest.fixture +def path_cache(): + return Cache(values=ValueMemory({}), calls=CallMemory({})) + + +@pytest.fixture +def a_file(tmp_path): + f = tmp_path / "data.txt" + f.write_text("file content") + return f + + +@pytest.fixture +def a_tree(tmp_path): + d = tmp_path / "tree" + (d / "sub").mkdir(parents=True) + (d / "top.txt").write_text("top") + (d / "sub" / "leaf.bin").write_bytes(b"leaf") + return d + + +@pytest.mark.parametrize("which", ["file", "tree", "both-nested"]) +def test_gc_keeps_the_content_behind_a_stored_path(path_cache, a_file, a_tree, which): + """A stored path is a name plus a *reference*; gc must follow the reference. + + The blob records carry the digests of the content ``bytes``. If the walk + cannot see them the content looks unreferenced, gc reclaims it, and the + entry is destroyed — the load then raises ``KeyError``, which the wrapper + reports as an ordinary cache miss, so the loss is silent. + """ + result = {"file": a_file, "dir": a_tree} if which == "both-nested" else ( + a_file if which == "file" else a_tree + ) + call = Call(name="f", arguments={"x": 1}, result=result) + key = path_cache.save(call) + + assert path_cache.gc() == set() + + loaded = path_cache.load(key).result + if which == "file": + assert loaded.read_text() == "file content" + elif which == "tree": + assert (loaded / "top.txt").read_text() == "top" + assert (loaded / "sub" / "leaf.bin").read_bytes() == b"leaf" + else: + assert loaded["file"].read_text() == "file content" + assert (loaded["dir"] / "sub" / "leaf.bin").read_bytes() == b"leaf" + + +def test_gc_still_evicts_an_orphaned_path(path_cache, a_file): + """The fix must not make path content unconditionally reachable.""" + orphan = path_cache.values.save(a_file) + assert orphan in path_cache.gc() + + +def test_child_digests_reports_a_blob_s_content(path_cache, a_file, a_tree): + """The blob layer declares its own references, unmended.""" + file_key = path_cache.values.save(a_file) + assert path_cache.values.child_digests(file_key), "FileBlob reported no children" + + tree_key = path_cache.values.save(a_tree) + children = path_cache.values.child_digests(tree_key) + assert len(children) == 2, "DirectoryBlob should reference both of its entries" + + +def test_reachability_walk_does_not_materialize_paths(path_cache, a_file, monkeypatch): + """Inspecting the graph must not build temp trees for every stored path. + + ``child_digests`` reads through ``load_raw``, so the path layer's + materialization is skipped: gc over a large cache would otherwise copy + every stored file to disk just to ask what it points at. + """ + from fleche.storage import paths as paths_mod + + path_cache.save(Call(name="f", arguments={"x": 1}, result=a_file)) + + calls = [] + real = paths_mod.TempPath.mkdtemp + monkeypatch.setattr( + paths_mod.TempPath, "mkdtemp", + classmethod(lambda cls: (calls.append(1), real())[1]), + ) + path_cache.gc() + path_cache.values.count_reuses() + assert calls == [], f"walk materialized {len(calls)} temp tree(s)" From 59f17e61b250b2e81dd35740d6caf841662f71e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 21:01:41 +0000 Subject: [PATCH 21/27] fix(digest): degrade unreadable paths to Indigestible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding 5 on #797. The `Path` arm mapped "does not exist" and "neither file nor directory" to `Indigestible`, but an `OSError` from actually reading the content escaped raw: >>> digest(Path("/proc/self/mem")) OSError: [Errno 5] Input/output error That killed the wrapped call before the body ever ran, which is not how any other undigestable argument behaves — they warn and fall through to an uncached run. A file being unreadable is a caching problem; it is not a reason to fail a call the function might well handle, and if the function does need to read it, it raises on its own terms inside the body where the traceback is meaningful. Read errors now degrade like the rest of the arm. The wrap covers the directory walk too, so one unreadable child no longer makes the enclosing tree raise. `Indigestible` is not an `OSError`, so the deliberate raises inside the block pass through untouched. [fleche] No hash for argument: Could not read /proc/self/mem: [Errno 5] ... call 1 -> body ran call 2 -> body ran body executed 2x (uncached), records stored: 0 Tests patch the read rather than chmod-ing, so they stay meaningful when the suite runs as root and mode bits are not enforced: unreadable file, unreadable directory, one unreadable child of a readable tree, the end-to-end uncached run, and a guard that readable paths still digest. Four of the five fail against the previous source. `usage/file_semantics` renames "Nonexistent paths" to "Paths with no readable content" and states the widened contract. 1874 passed, 11 skipped; `ty check src/` clean; docs build clean. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- docs/usage/file_semantics.rst | 28 +++++--- src/fleche/digest.py | 33 ++++++--- tests/unit/digest/test_digest_paths.py | 96 ++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 20 deletions(-) diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst index a2c081ba..fd6bdd78 100644 --- a/docs/usage/file_semantics.rst +++ b/docs/usage/file_semantics.rst @@ -63,16 +63,24 @@ Only *content* changes count as mutation: permissions are not part of identity (see :ref:`fidelity-limits`), so a ``chmod`` on a received path is harmless. -Nonexistent paths -~~~~~~~~~~~~~~~~~ - -A path that does not exist on disk has no content and therefore **no digest**. -Passing one to a cached function does not raise: fleche logs a warning -(``"No hash for argument: ..."``) and **runs the function uncached** — every -call executes, nothing is stored or looked up. If you meant "an output -location the function should write to", pass the location as a ``str`` or -annotate the parameter :class:`~fleche.Ignored` (``dest: Ignored[Path]``) so it -stays out of the key. +Paths with no readable content +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A path fleche cannot read has no content and therefore **no digest**. That +covers one that does not exist, one that is neither a file nor a directory, and +one whose content the process cannot actually read — no permission, an I/O +error, a network mount that went away. + +None of these raise. fleche logs a warning (``"No hash for argument: ..."``) +and **runs the function uncached**: every call executes, nothing is stored or +looked up. An unreadable file is a caching problem, not a reason to fail a call +the function itself might well handle — and if the function does need to read +it, it will raise on its own terms, inside the body, where the traceback means +something. + +If you meant "an output location the function should write to", pass the +location as a ``str`` or annotate the parameter :class:`~fleche.Ignored` +(``dest: Ignored[Path]``) so it stays out of the key. Cache hits materialize copies ----------------------------- diff --git a/src/fleche/digest.py b/src/fleche/digest.py index 0d95a32c..996122a8 100644 --- a/src/fleche/digest.py +++ b/src/fleche/digest.py @@ -267,16 +267,29 @@ def _digest_bytes(value: Any) -> bytes: # of path-valued arguments/results depend. The "FileBlob" / # "DirectoryBlob" salts MUST match # fleche.storage.paths.{FileBlob,DirectoryBlob}.__digest__. - if not value.exists(): - raise Indigestible("Only existing paths can be digested.") - if value.is_file(): - return _digest_bytes( - ("FileBlob", value.name, _path_content_digest(value)) - ) - elif value.is_dir(): - return _path_content_digest(value).encode() - else: - raise Indigestible("Can only digest files and folders!") + # + # Reading can fail for reasons that have nothing to do with the + # value being unsuitable — permissions, EIO, a network mount that + # went away. Those degrade to `Indigestible` like every other + # case in this arm, so the wrapper runs the call uncached instead + # of crashing it before the body ever executes. A caller who + # genuinely needs the read to succeed will hit the same error on + # its own terms inside the function. + try: + if not value.exists(): + raise Indigestible("Only existing paths can be digested.") + if value.is_file(): + return _digest_bytes( + ("FileBlob", value.name, _path_content_digest(value)) + ) + elif value.is_dir(): + return _path_content_digest(value).encode() + else: + raise Indigestible("Can only digest files and folders!") + except OSError as e: + # `Indigestible` is not an OSError, so the raises above pass + # through untouched. + raise Indigestible(f"Could not read {value}: {e}") from None case np.ndarray(): m.update(_digest_bytes(value.dtype.str)) m.update(_digest_bytes(value.shape)) diff --git a/tests/unit/digest/test_digest_paths.py b/tests/unit/digest/test_digest_paths.py index b314e6f3..714eab63 100644 --- a/tests/unit/digest/test_digest_paths.py +++ b/tests/unit/digest/test_digest_paths.py @@ -169,3 +169,99 @@ def test_special_path_raises_indigestible(tmp_path): pytest.skip("FIFOs not supported on this platform") with pytest.raises(Indigestible): digest(fifo) + + +# ---- Unreadable paths degrade rather than escaping ---- +# +# A read can fail for reasons unrelated to the value's suitability — permissions, +# EIO, a network mount that vanished. Those must degrade to `Indigestible` like +# every other case in the arm, or the wrapper crashes the call before the body +# ever runs. Patched rather than chmod-ed so the tests are meaningful when the +# suite runs as root, where mode bits are not enforced. + + +def test_unreadable_file_degrades_to_indigestible(tmp_path, monkeypatch): + f = tmp_path / "secret.txt" + f.write_text("hidden") + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "read_bytes", boom) + with pytest.raises(Indigestible, match="Could not read"): + digest(f) + + +def test_unreadable_directory_degrades_to_indigestible(tmp_path, monkeypatch): + d = tmp_path / "locked" + d.mkdir() + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "iterdir", boom) + with pytest.raises(Indigestible, match="Could not read"): + digest(d) + + +def test_unreadable_child_degrades_the_whole_tree(tmp_path, monkeypatch): + """One unreadable file must not make the enclosing directory raise raw.""" + d = tmp_path / "tree" + d.mkdir() + (d / "ok.txt").write_text("fine") + (d / "bad.bin").write_bytes(b"x") + + real = Path.read_bytes + + def selective(self, *a, **kw): + if self.name == "bad.bin": + raise OSError(5, "Input/output error") + return real(self, *a, **kw) + + monkeypatch.setattr(Path, "read_bytes", selective) + with pytest.raises(Indigestible, match="Could not read"): + digest(d) + + +def test_wrapped_call_on_an_unreadable_path_runs_uncached(tmp_path, monkeypatch): + """The point of degrading: the body still runs, it just isn't cached. + + Before this, an `OSError` escaped `digest()` and killed the call before the + body executed — unlike every other undigestable argument, which warns and + falls through to an uncached run. + """ + from fleche import fleche, cache + from fleche.caches import Cache + from fleche.storage import CallMemory, ValueMemory + + f = tmp_path / "secret.txt" + f.write_text("hidden") + runs = [] + + @fleche + def consume(p: Path): + runs.append(p) + return "body ran" + + def boom(self, *a, **kw): + raise PermissionError(13, "Permission denied") + + monkeypatch.setattr(Path, "read_bytes", boom) + c = Cache(ValueMemory({}), CallMemory({})) + with cache(c): + assert consume(f) == "body ran" + assert consume(f) == "body ran" + + assert len(runs) == 2, "an uncachable call must re-execute, not hit" + assert not c.calls.storage, "nothing should have been filed" + + +def test_readable_paths_are_unaffected(tmp_path): + """The guard must not swallow anything that genuinely works.""" + f = tmp_path / "a.txt" + f.write_text("content") + d = tmp_path / "d" + d.mkdir() + (d / "child.txt").write_text("content") + assert digest(f) and digest(d) + assert digest(f) != digest(d) From a9e1a1361f19512dd0b48b853a5ac6f4efd5eebc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 21:30:07 +0000 Subject: [PATCH 22/27] fix(caches): keep in-flight call arguments reachable from gc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `prepare` stores the argument values before the function body runs, so the record cannot be keyed on post-mutation content. For the whole duration of the body, though, no record references them: a `gc()` in that window reclaims them as orphans, and the eventual `commit` files a record whose arguments are dangling digests — `load(key).arguments[...]` hands back a bare `Digest`. Before the two-phase protocol the window was the few microseconds inside `save()`; it is now as long as the function takes. Prepared-but-unfinished calls are now gc roots. `PreparedCall` registers itself in a weak, process-wide registry on construction and drops out on `commit` (the record references the values by then, or the save was rejected and they are genuine orphans) or on `abandon`; `Cache.gc` seeds its reachable set from `call.in_flight_digests()`, so the existing transitive walk covers nested arguments too. Registration lives in `__post_init__` rather than in `prepare` because `CacheWrapper.prepare` rebinds via `replace(...)`, which builds a *new* object — registering in `prepare` would leave only the inner one registered, and that one is garbage the moment `replace` returns. The registry holds weak references, so a prepared call dropped without commit or abandon frees its roots on collection instead of pinning them forever. The registry is per-process, which `gc`'s docstring now states: a sweep run on an `SshCache` server cannot see the client call whose arguments its own `prepare` stashed. Fixes finding 2 of the review on #797. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- src/fleche/caches.py | 40 +++++++++++- src/fleche/call.py | 59 ++++++++++++++++- tests/unit/caches/test_gc.py | 122 +++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 5 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index 16d4a93d..b0d52a84 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -62,7 +62,10 @@ def prepare(self, call: Call) -> PreparedCall: Finish the returned :class:`~fleche.call.PreparedCall` with exactly one of :meth:`~fleche.call.PreparedCall.commit` or - :meth:`~fleche.call.PreparedCall.abandon`. + :meth:`~fleche.call.PreparedCall.abandon`. Until then the stashed + values are referenced by no record, and :meth:`Cache.gc` treats them as + roots (:func:`~fleche.call.in_flight_digests`) so a concurrent sweep + cannot evict the arguments of a running call. """ return PreparedCall(digested=call.digest(), cache=self) @@ -324,6 +327,8 @@ def prepare(self, call: Call) -> PreparedCall: # record cannot end up keyed on post-mutation content. No cache-level # lock: the keys are only known once the value storage has digested # each value, and value storages carry their own per-key locking. + # Registering the result as a gc root is PreparedCall's own doing, so + # it survives the rebinding CacheWrapper.prepare does. return PreparedCall(digested=call.stash(self.values), cache=self) def save(self, call: PreparedCall | Call) -> str: @@ -465,6 +470,29 @@ def gc(self) -> set[Digest]: ``values`` key outside the reachable set. Call records are left untouched. + Values belonging to a call that is between + :meth:`~fleche.caches.BaseCache.prepare` and + :meth:`~fleche.call.PreparedCall.commit` are referenced by no record + yet, so they are seeded into the reachable set from + :func:`~fleche.call.in_flight_digests` — otherwise a sweep during a + long function body would evict the arguments out from under the call, + and the eventual commit would file a record with dangling references. + That registry is **per-process**: a sweep here cannot see a call in + flight in another process, so avoid running ``gc()`` against a cache + that other processes are writing to concurrently. The concrete case + is :class:`~fleche.remote.SshCache` — the arguments are stashed by the + *server's* ``prepare``, and only the sealed record travels back, so a + sweep run on the server has nothing to key on while the client's body + runs. + + Within a process the roots are read *after* the eviction candidates are + listed, which leaves one narrow window: a value stored by a concurrent + ``prepare`` that has not registered its call yet. That is the same + window the one-shot :meth:`save` has always had between storing the + values and filing the record — bounded by a storage write rather than + by a function body — and closing it would need a lock shared by every + writer. + Returns: The set of digests that were evicted from value storage. """ @@ -480,6 +508,14 @@ def gc(self) -> set[Digest]: if isinstance(v, Digest): reachable.add(v) + # Snapshot the candidates before reading the in-flight roots: a value + # stored after this line is not a candidate at all, so the only gap + # left is a store that already happened but whose call has not + # registered yet. Listing after the read would widen that to every + # call admitted during the walk. + candidates = list(self.values.list()) + reachable |= call.in_flight_digests() + if isinstance(self.values, HasChildDigests): frontier = set(reachable) while frontier: @@ -493,7 +529,7 @@ def gc(self) -> set[Digest]: frontier |= new evicted: set[Digest] = set() - for key in list(self.values.list()): + for key in candidates: if key not in reachable: try: self.values.evict(key) diff --git a/src/fleche/call.py b/src/fleche/call.py index 38355188..c391398a 100644 --- a/src/fleche/call.py +++ b/src/fleche/call.py @@ -1,4 +1,6 @@ import logging +import threading +import weakref from dataclasses import dataclass, field, replace from functools import lru_cache from typing import Any, Callable, get_type_hints, get_origin, get_args, Annotated @@ -331,6 +333,38 @@ def fetch(self, cache) -> "LazyCall": ) +# Prepared calls whose argument values are stored but not yet referenced by any +# record — the prepare -> commit window. Held weakly: a prepared call dropped +# without commit() or abandon() falls out on its own, and the registry never +# keeps a call (or its cache) alive. Keyed by ``id`` because ``PreparedCall`` +# is an unhashable dataclass; ``WeakValueDictionary`` only drops a key whose +# stored reference is still the dead one, so a recycled ``id`` is safe. The +# lock guards the snapshot below against a concurrent ``prepare`` resizing the +# mapping mid-iteration; weakref removals are already deferred while it is +# being iterated. +_IN_FLIGHT: "weakref.WeakValueDictionary[int, PreparedCall]" = weakref.WeakValueDictionary() +_IN_FLIGHT_LOCK = threading.Lock() + + +def in_flight_digests() -> set[Digest]: + """Return the value digests of every prepared-but-unfinished call. + + These are stored in value storage but reachable from no record yet, so a + reachability sweep (:meth:`fleche.caches.Cache.gc`) must treat them as + roots or it evicts the arguments of a call still running. Registration is + per-process: a call in flight in *another* process is not visible here. + """ + with _IN_FLIGHT_LOCK: + in_flight = list(_IN_FLIGHT.values()) + reachable: set[Digest] = set() + for prepared in in_flight: + digested = prepared.digested + if isinstance(digested.result, Digest): + reachable.add(digested.result) + reachable.update(v for v in digested.arguments.values() if isinstance(v, Digest)) + return reachable + + @dataclass class PreparedCall: """A call admitted to a cache: arguments stored, key sealed, result awaited. @@ -358,6 +392,14 @@ class PreparedCall: _result: Any = field(default=None, init=False, repr=False) _metadata: "dict | None" = field(default=None, init=False, repr=False) + def __post_init__(self) -> None: + # Registered here rather than in ``prepare`` so wrapper rebinding — + # ``replace(inner.prepare(call), cache=self)`` — registers the new + # object too; the inner one is garbage by then and drops out on its + # own. + with _IN_FLIGHT_LOCK: + _IN_FLIGHT[id(self)] = self + def commit(self, result: Any, metadata: dict | None = None) -> Digest: """Attach *result* and *metadata* to the record and file it. @@ -384,7 +426,14 @@ def commit(self, result: Any, metadata: dict | None = None) -> Digest: self._result = result if metadata is not None: self._metadata = metadata - return self.cache.save(self) + try: + return self.cache.save(self) + finally: + # Either the record now references the arguments, or the save was + # rejected and they are genuine orphans. Either way they no longer + # need protecting from a sweep. + with _IN_FLIGHT_LOCK: + _IN_FLIGHT.pop(id(self), None) def to_lookup_key(self) -> Digest: return self.digested.to_lookup_key() @@ -406,12 +455,16 @@ def abandon(self) -> None: """Release the call without recording it. Called when the function body raises or its result is not cacheable. - The default is a no-op: argument values stored by + The default is a no-op beyond bookkeeping: argument values stored by :meth:`~fleche.caches.BaseCache.prepare` are content-addressed orphans - that a later garbage collection sweep reclaims. Subclasses may hook + that a later garbage collection sweep reclaims — abandoning is what + releases them to it, since an unfinished call keeps them reachable + (:func:`in_flight_digests`). Subclasses may hook cleanup here. Idempotent; only :meth:`commit` is barred afterwards. """ self._finished = True + with _IN_FLIGHT_LOCK: + _IN_FLIGHT.pop(id(self), None) def __enter__(self) -> "PreparedCall": return self diff --git a/tests/unit/caches/test_gc.py b/tests/unit/caches/test_gc.py index d851b5df..03916d38 100644 --- a/tests/unit/caches/test_gc.py +++ b/tests/unit/caches/test_gc.py @@ -234,3 +234,125 @@ def test_reachability_walk_does_not_materialize_paths(path_cache, a_file, monkey path_cache.gc() path_cache.values.count_reuses() assert calls == [], f"walk materialized {len(calls)} temp tree(s)" + + +# ---- Calls in flight: prepared arguments are roots until commit or abandon ---- + + +def test_gc_keeps_arguments_of_a_call_in_flight(gc_cache): + """A sweep during the function body must not evict the sealed arguments. + + ``prepare`` stores the arguments before the body runs, precisely so the + record cannot be keyed on post-mutation content — but for the whole + duration of the body no record references them yet. Without the in-flight + roots ``gc`` reclaims them as orphans and the eventual ``commit`` files a + record whose arguments are dangling digests. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + + assert gc_cache.gc() == set() # body "runs" here + + key = prepared.commit("a result") + assert gc_cache.load(key).arguments["x"] == "an argument" + + +def test_gc_evicts_arguments_of_an_abandoned_call(gc_cache): + """Abandoning releases the roots: the arguments are orphans again.""" + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + arg_key = digest("an argument") + assert arg_key in gc_cache.values.list() + + prepared.abandon() + + assert arg_key in gc_cache.gc() + + +def test_gc_evicts_arguments_after_commit_returns(gc_cache): + """Committing hands the roots over to the record, and holds nothing else. + + Once the record references them the registry must let go, or a long-lived + ``PreparedCall`` object would pin values forever. Evicting the record and + sweeping is the observable check. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + key = prepared.commit("a result") + gc_cache.evict(key) + + assert digest("an argument") in gc_cache.gc() + + +def test_gc_keeps_nested_arguments_of_a_call_in_flight(split_cache): + """In-flight roots seed the transitive walk, not just the top-level keys.""" + prepared = split_cache.prepare(Call(name="f", arguments={"x": [[1, 2], [3, 4]]})) + + assert split_cache.gc() == set() + + key = prepared.commit(None) + assert split_cache.load(key).arguments["x"] == [[1, 2], [3, 4]] + + +def test_gc_keeps_arguments_of_a_call_in_flight_through_a_wrapper(gc_cache): + """Wrapper rebinding must not drop the registration. + + ``CacheWrapper.prepare`` returns ``replace(inner.prepare(call), cache=self)`` + — a *new* object — so registering in ``prepare`` alone would leave only the + inner one registered, and that one is garbage the moment ``replace`` + returns. + """ + import gc as _gc + + from fleche.caches import RefreshingCache + + wrapper = RefreshingCache(gc_cache) + prepared = wrapper.prepare(Call(name="f", arguments={"x": "an argument"})) + _gc.collect() # drop the inner PreparedCall replace() left behind + + assert gc_cache.gc() == set() + assert digest("an argument") in gc_cache.values.list() + prepared.abandon() + + +def test_in_flight_registry_does_not_leak_dropped_calls(gc_cache): + """The registry is weak: a prepared call nobody finished falls out.""" + import gc as _gc + + from fleche.call import in_flight_digests + + gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + _gc.collect() + + assert in_flight_digests() == set() + + +def test_gc_is_safe_against_concurrent_prepares(gc_cache): + """Sweeping while other threads admit calls must not raise or lose values. + + The registry is a plain mapping read by ``gc``; without the lock a + concurrent ``prepare`` resizes it mid-iteration. + """ + import threading + + stop = threading.Event() + errors: list[BaseException] = [] + + def preparer(i): + try: + while not stop.is_set(): + prepared = gc_cache.prepare(Call(name="f", arguments={"x": f"arg-{i}"})) + key = prepared.commit(i) + assert gc_cache.load(key).arguments["x"] == f"arg-{i}" + except BaseException as e: # pragma: no cover - only on a real race + errors.append(e) + + threads = [threading.Thread(target=preparer, args=(i,)) for i in range(4)] + for t in threads: + t.start() + try: + for _ in range(50): + gc_cache.gc() + finally: + stop.set() + for t in threads: + t.join() + + assert not errors, errors From ff168a0e7f6295c6752b6f0370a4ccc7adfa347a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 22:02:58 +0000 Subject: [PATCH 23/27] fix(caches): order gc's reads so a mid-sweep commit cannot slip through MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The threaded test added with the in-flight roots caught a second, older race, on CI first and then locally in ~3 of 8 runs: `gc` read the call records *before* the in-flight registry, so a call that committed between those two reads was covered by neither. Its record had not been listed and its registration was already gone, so its arguments were evicted under a record that references them — the same dangling-argument symptom the in-flight roots were added to prevent, just through a narrower door. The reads are now ordered candidates, roots, records. `commit` deregisters only after `save` has filed the record, so with the registry read before the records, a call finishing mid-sweep is seen by one read or the other, never neither. Listing the candidates first means a value stored later in the sweep is not a candidate at all. What is left is the gap between storing a value and registering the call that owns it, and only when it spans the first two reads — the same window the one-shot `save` has always had between storing values and filing the record. `gc`'s docstring states the ordering and what it does and does not cover. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- src/fleche/caches.py | 31 ++++++++++++++++--------------- tests/unit/caches/test_gc.py | 12 +++++++++--- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index b0d52a84..a7b2f023 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -485,18 +485,27 @@ def gc(self) -> set[Digest]: sweep run on the server has nothing to key on while the client's body runs. - Within a process the roots are read *after* the eviction candidates are - listed, which leaves one narrow window: a value stored by a concurrent - ``prepare`` that has not registered its call yet. That is the same - window the one-shot :meth:`save` has always had between storing the - values and filing the record — bounded by a storage write rather than - by a function body — and closing it would need a lock shared by every + The three reads below are ordered so that a call finishing *during* the + sweep cannot fall between them. Candidates are listed first, so a value + stored later in the sweep is not a candidate at all. Then the in-flight + roots, then the records: :meth:`~fleche.call.PreparedCall.commit` + deregisters only *after* :meth:`save` has filed the record, so a call + that commits mid-sweep is seen by the registry read, the record read, or + both — never neither. Reading the records first instead leaves a real + hole, and a threaded test in ``test_gc.py`` reproduces it. + + What remains is the gap between storing a value and registering the call + that owns it, and only if it spans those first two reads. That is the + same window the one-shot :meth:`save` has always had between storing the + values and filing the record — bounded by a storage write rather than by + a function body — and closing it would need a lock shared by every writer. Returns: The set of digests that were evicted from value storage. """ - reachable: set[Digest] = set() + candidates = list(self.values.list()) + reachable: set[Digest] = call.in_flight_digests() for key in self.calls.list(): try: dc = self.calls.load(key) @@ -508,14 +517,6 @@ def gc(self) -> set[Digest]: if isinstance(v, Digest): reachable.add(v) - # Snapshot the candidates before reading the in-flight roots: a value - # stored after this line is not a candidate at all, so the only gap - # left is a store that already happened but whose call has not - # registered yet. Listing after the read would widen that to every - # call admitted during the walk. - candidates = list(self.values.list()) - reachable |= call.in_flight_digests() - if isinstance(self.values, HasChildDigests): frontier = set(reachable) while frontier: diff --git a/tests/unit/caches/test_gc.py b/tests/unit/caches/test_gc.py index 03916d38..4d9effd5 100644 --- a/tests/unit/caches/test_gc.py +++ b/tests/unit/caches/test_gc.py @@ -327,8 +327,14 @@ def test_in_flight_registry_does_not_leak_dropped_calls(gc_cache): def test_gc_is_safe_against_concurrent_prepares(gc_cache): """Sweeping while other threads admit calls must not raise or lose values. - The registry is a plain mapping read by ``gc``; without the lock a - concurrent ``prepare`` resizes it mid-iteration. + Two things have to hold. The registry is a plain mapping read by ``gc``, + so a concurrent ``prepare`` must not resize it mid-iteration. And the + sweep's three reads — candidates, roots, records — must be ordered so a + call that *commits* between two of them is still seen by one: reading the + records before the roots leaves a call that finishes in that gap covered + by neither, and its arguments are evicted under a record that references + them. Both failures show up here as an argument that came back a bare + digest. """ import threading @@ -348,7 +354,7 @@ def preparer(i): for t in threads: t.start() try: - for _ in range(50): + for _ in range(200): gc_cache.gc() finally: stop.set() From 4c10b927624f3dec74b0a50388aa2511eaf52707 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 23:01:44 +0000 Subject: [PATCH 24/27] fix(remote): degrade only the path arguments, not the whole call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SshCache.prepare` is the one RPC that carries argument *values* rather than digests, so a call carrying a path cannot be shipped whole — the server would resolve the path string against its own filesystem. The guard fell back to `BaseCache.prepare`, which digests everything and stores nothing: for `f(p: Path, payload)` the payload was never stored either, and `load(key).arguments["payload"]` came back a bare digest for no reason. That contradicted both the comment on the fallback ("each argument goes through save_value") and `RemotePathUnsupported`'s documented degradation, which promises only the path's bytes go missing. The arguments are now stashed one at a time through `_RemoteValues`, whose `save` routes each through `save_value`. Only the path arguments raise `RemotePathUnsupported`, and `Call.stash`'s existing per-argument `SaveError` fallback degrades exactly those to a digest-only reference computed locally — so the `digest(x) == save_value(x)` seal still holds and lookups still hit, while the siblings are stored and stay loadable off the record. Costs one round trip per argument instead of one for the call, on this branch only. That is the price of not degrading every argument for the sake of one path. Fixes finding 3 of the review on #797. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- docs/dev/path_storage.rst | 12 ++++++++++- docs/usage/file_semantics.rst | 4 +++- src/fleche/remote.py | 16 ++++++++------ tests/unit/test_remote.py | 39 +++++++++++++++++++++++++++++++---- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index 175cae03..c80f0da0 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -190,7 +190,17 @@ destructuring save does, using carry it: a path argument becomes a digest-only reference **whose digest was computed locally**, which keeps the seal intact and lookups correct, and a path result becomes :class:`~fleche.caches.Rejected`, so the call runs uncached -rather than wrongly cached. See :ref:`file-remote-caches` for the user-facing +rather than wrongly cached. + +The degradation is deliberately per *argument*, not per call. ``prepare`` is +the one RPC that carries argument values rather than digests, so a call +carrying a path cannot be shipped whole; it stashes the arguments one at a +time through :class:`~fleche.remote._RemoteValues` instead, and +:meth:`~fleche.call.Call.stash`'s existing per-argument ``SaveError`` fallback +does the rest — only the path arguments are digested-and-not-stored, and their +siblings are stored and remain loadable off the record. Falling back to the +blanket digest-only admission instead would cost every other argument for the +sake of one path, which is neither what the guard promises nor necessary. See :ref:`file-remote-caches` for the user-facing version. Making paths genuinely work over SSH is a separate feature, tracked in diff --git a/docs/usage/file_semantics.rst b/docs/usage/file_semantics.rst index fd6bdd78..b51c6338 100644 --- a/docs/usage/file_semantics.rst +++ b/docs/usage/file_semantics.rst @@ -247,7 +247,9 @@ does not have to catch anything: * a path **argument** falls back to a digest-only reference. The call is still keyed correctly (the digest is computed *here*, from your file), so lookups hit and miss exactly as they should; only the file's bytes are not retrievable - from the remote record. + from the remote record. The degradation is per argument: the path's + non-path siblings are stored normally and read back off the record as + values. * a path **result** is rejected: the call runs, returns your file, and is logged as not cached. * **loading** a path stored on the remote raises too. The remote materializes diff --git a/src/fleche/remote.py b/src/fleche/remote.py index 8cfb6376..bb185edf 100644 --- a/src/fleche/remote.py +++ b/src/fleche/remote.py @@ -956,12 +956,16 @@ def prepare(self, call: _call.Call) -> PreparedCall: # Shipping the live call would hand the server a path *string* to # resolve against its own filesystem — exactly what # `RemotePathUnsupported` exists to prevent, and this is the one - # RPC that carries argument *values* rather than digests. Fall - # back to the local two-phase prepare, where each argument goes - # through `save_value` and a path degrades to a digest-only - # reference computed *here*, keeping the - # `digest(x) == save_value(x)` seal intact so lookups still hit. - return super().prepare(call) + # RPC that carries argument *values* rather than digests. Stash + # the arguments one at a time instead: `_RemoteValues.save` routes + # each through `save_value`, so only the path arguments raise and + # `Call.stash` degrades *those* to a digest-only reference computed + # here — keeping the `digest(x) == save_value(x)` seal intact so + # lookups still hit — while their non-path siblings are stored + # normally and stay loadable off the record. Costs one round trip + # per argument rather than one for the call, which is the price of + # not degrading the whole call for one path. + return PreparedCall(digested=call.stash(_RemoteValues(self)), cache=self) return PreparedCall(digested=self._rpc("prepare", call), cache=self) def evict(self, key: str | Digest) -> None: diff --git a/tests/unit/test_remote.py b/tests/unit/test_remote.py index 39c099d1..12e0e407 100644 --- a/tests/unit/test_remote.py +++ b/tests/unit/test_remote.py @@ -1162,16 +1162,47 @@ def test_prepare_does_not_ship_a_call_carrying_a_path(remote, server_cache, a_fi """`prepare` is the one RPC that sends argument *values*, not digests. Shipping the live call would hand the server a path string to resolve - against its own filesystem — the bug the guard exists to stop — so it has - to fall back to the local two-phase prepare instead. + against its own filesystem — the bug the guard exists to stop — so the + arguments are stashed one at a time instead. """ call = Call(name="f", arguments={"p": a_file, "n": 3}) prepared = remote.prepare(call) # Sealed locally: the key is still the one a later lookup computes. assert prepared.to_lookup_key() == call.to_lookup_key() assert prepared.digested.arguments["p"] == digest(a_file) - # Nothing about the path reached the server. - assert not server_cache.values.storage + # Nothing about the path reached the server — not the record, not the + # content bytes it would have been split into. + assert digest(a_file) not in server_cache.values.storage + assert a_file.read_bytes() not in server_cache.values.storage.values() + + +def test_prepare_degrades_only_the_path_argument_not_its_siblings( + remote, server_cache, a_file +): + """One unshippable argument must not take the whole call down with it. + + The fallback used to be a blanket `BaseCache.prepare`, which digests + everything and stores nothing: `f(p: Path, payload)` lost `payload` too, + and `load(key).arguments["payload"]` came back a bare digest for no + reason. Only the path cannot cross. + """ + payload = {"rows": [1, 2, 3]} + call = Call(name="f", arguments={"p": a_file, "payload": payload}) + prepared = remote.prepare(call) + + # The sibling was stored on the remote and reads back as a value... + assert remote.load_value(prepared.digested.arguments["payload"]) == payload + # ...while the path is a digest-only reference computed here. + assert prepared.digested.arguments["p"] == digest(a_file) + assert digest(a_file) not in server_cache.values.storage + + # End to end: the filed record hands back the sibling, not a digest. + key = prepared.commit(1) + loaded = remote.load(key) + assert loaded.arguments["payload"] == payload + # The path argument stays a digest, which is the documented degradation: + # keyed correctly, content not retrievable from the remote. + assert loaded.arguments["p"] == digest(a_file) def test_prepare_still_uses_the_remote_when_no_path_is_involved(remote, server_cache): From 4d6ded1c12d02dc97bc6c80a9d975592177253cc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 23:32:57 +0000 Subject: [PATCH 25/27] fix(storage): walk any re-iterable container when looking for a path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `find_path` descended into exactly list/tuple/set/frozenset, but `digest`'s `Iterable` arm walks *anything* iterable and reads the file it finds. A path in a `deque` therefore decided the cache key while bypassing the SshCache guard entirely: the digest was computed here from the local file, the live value shipped over the wire, and the server re-digested the same name against its own filesystem — the `digest(x) == save_value(x)` break the guard exists to prevent. An allowlist of concrete types is that bug in slower motion: it covered the containers that happened to be in mind and let the next one through. The iterable arm now walks whatever `digest` walks, and skips only what `digest` itself never looks inside: * `OPAQUE_ITERABLES` (ndarray, DataFrame, Series, Index), which `digest` matches *above* its `Iterable` arm and hashes by their own buffer — no element inside one can influence a digest. Named in `fleche.digest` rather than re-derived here, so the mirror cannot drift. * `range`, whose elements are `int` by construction; the walk pushes onto a list, so `range(10**9)` would exhaust memory to learn nothing. * one-shot iterators, identified by `iter(x) is x`. Walking a generator consumes it, and `digest` has already drained one by the time a value could ship, so there is nothing in it left to find. Also drops a stale claim in `path_storage.rst` that `find_path` walks the way a destructuring save does — it mirrors `digest`, which is the whole point. Fixes finding 4 of the review on #797. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- docs/dev/path_storage.rst | 10 +++--- src/fleche/digest.py | 9 +++++ src/fleche/storage/paths.py | 33 ++++++++++++++--- tests/unit/storage/test_paths.py | 62 +++++++++++++++++++++++++++++++- 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/docs/dev/path_storage.rst b/docs/dev/path_storage.rst index c80f0da0..8319b633 100644 --- a/docs/dev/path_storage.rst +++ b/docs/dev/path_storage.rst @@ -182,10 +182,12 @@ itself*. Three things follow, and each of them is silent: it cannot see. So :class:`~fleche.remote.SshCache` refuses instead, via -:func:`~fleche.storage.paths.find_path` (which walks a value exactly the way a -destructuring save does, using -:func:`~fleche.storage.destructuring.child_slots`) and -:class:`~fleche.remote.RemotePathUnsupported`. That exception subclasses +:func:`~fleche.storage.paths.find_path` and +:class:`~fleche.remote.RemotePathUnsupported`. ``find_path`` mirrors +:func:`~fleche.digest.digest`, **not** destructuring: what makes a path +dangerous here is that it decides the key, and ``digest`` reads files inside +namedtuples, sets, and arbitrary iterables that a destructuring save stores +verbatim. Mirroring the narrower walk would let ``Bundle(path, 0.5)`` cross. That exception subclasses :class:`~fleche.storage.SaveError` so the existing two-phase-save degradations carry it: a path argument becomes a digest-only reference **whose digest was computed locally**, which keeps the seal intact and lookups correct, and a path diff --git a/src/fleche/digest.py b/src/fleche/digest.py index 996122a8..99f7149d 100644 --- a/src/fleche/digest.py +++ b/src/fleche/digest.py @@ -20,6 +20,15 @@ logger = logging.getLogger("fleche.digest") +# Types that are ``Iterable`` but are matched by an arm *above* the generic +# ``Iterable`` arm in :func:`_digest_bytes`: each hashes its own buffer and +# never looks at the elements, so nothing reachable through one can influence +# a digest. Named here so a walker that mirrors ``digest`` — notably +# :func:`fleche.storage.paths.find_path` — can skip exactly the same types +# instead of re-deriving the list and drifting from it. +OPAQUE_ITERABLES = (np.ndarray, pd.DataFrame, pd.Series, pd.Index) + + class Indigestible(Exception): """Exception raised when an object cannot be digested.""" diff --git a/src/fleche/storage/paths.py b/src/fleche/storage/paths.py index d2104a14..c14a9f89 100644 --- a/src/fleche/storage/paths.py +++ b/src/fleche/storage/paths.py @@ -37,6 +37,12 @@ def find_path(value: Any) -> Path | None: own filesystem, which is exactly the ``digest(x) == save_value(x)`` break the caller is trying to prevent. Mirroring destructuring here would let ``Bundle(path, 0.5)`` through and reintroduce it. + + Mirroring means mirroring ``digest``'s *containers*, not a list of them: + the iterable arm below walks anything iterable, as ``digest`` does, and + skips only what ``digest`` itself never looks inside. An allowlist of + concrete types is the same bug in slower motion — it covers ``list`` and + ``tuple`` and lets a ``deque`` through. """ seen: set[int] = set() stack: list[Any] = [value] @@ -57,11 +63,28 @@ def find_path(value: Any) -> Path | None: elif _attrs.is_attrs_instance(item): stack.extend(v for _, v in _attrs.field_items(item)) elif isinstance(item, Iterable): - # Covers list/tuple/set/frozenset and namedtuples — everything the - # ``Iterable`` arm of ``_digest_bytes`` walks. Guarded because an - # arbitrary iterable may be a one-shot generator; digesting one - # consumes it, and so would we. - if not isinstance(item, (list, tuple, set, frozenset)): + # ``digest``'s ``Iterable`` arm walks *any* iterable, so this one + # has to as well: a path in a ``deque`` decides the key exactly as + # much as one in a list, and an allowlist of concrete types silently + # stops covering whatever a caller reaches for next. Three + # exclusions, each mirroring something ``digest`` does earlier: + # + # * :data:`~fleche.digest.OPAQUE_ITERABLES` — matched above the + # ``Iterable`` arm, hashing their own buffer without looking at + # elements, so no path inside one can reach a digest. + # * ``range`` — its elements are ``int`` by construction, and + # materializing ``range(10**9)`` onto the stack to learn that + # would be a denial of service. + # * one-shot iterators, which are their own ``__iter__``. Walking + # a generator consumes it; ``digest`` has already done so by the + # time a value could ship, so there is nothing left in one for us + # to find anyway. + if isinstance(item, (digest.OPAQUE_ITERABLES, range)): + continue + try: + if iter(item) is item: + continue + except TypeError: continue stack.extend(item) return None diff --git a/tests/unit/storage/test_paths.py b/tests/unit/storage/test_paths.py index 6283db1f..658aa05b 100644 --- a/tests/unit/storage/test_paths.py +++ b/tests/unit/storage/test_paths.py @@ -1,13 +1,15 @@ +import collections import gc import tempfile from dataclasses import dataclass from pathlib import Path +import numpy as np import pytest from collections import namedtuple -from fleche.digest import digest, Indigestible +from fleche.digest import digest, Indigestible, OPAQUE_ITERABLES from fleche.storage import ValueMixin, DestructuringMixin from fleche.storage.memory import MemoryBackend from fleche.storage.destructuring import DigestedMapping @@ -421,6 +423,64 @@ def test_find_path_stops_where_digest_stops(tmp_path): digest(_Opaque(tmp_path)) +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda p: collections.deque([p]), id="deque"), + pytest.param(lambda p: [collections.deque([0, p])], id="deque-in-list"), + pytest.param(lambda p: collections.OrderedDict(k=p), id="ordereddict"), + pytest.param(lambda p: collections.defaultdict(list, k=p), id="defaultdict"), + pytest.param(lambda p: collections.Counter({p: 1}), id="counter-key"), + pytest.param(lambda p: collections.ChainMap({"k": p}), id="chainmap"), + pytest.param(lambda p: array_like(p), id="object-array-elementwise"), + ], +) +def test_find_path_descends_any_re_iterable_container(tmp_path, wrap): + """An allowlist of concrete types is the bug in slower motion. + + ``digest``'s ``Iterable`` arm walks *anything* iterable and reads the file, + so a path in a ``deque`` decides the key exactly as much as one in a list. + Enumerating ``list``/``tuple``/``set``/``frozenset`` covered the containers + that happened to be in mind and let the next one through. + """ + assert find_path(wrap(tmp_path)) is tmp_path + + +def array_like(p): + """A re-iterable custom container — the case no allowlist can anticipate.""" + + class _Rows: + def __init__(self, rows): + self._rows = rows + + def __iter__(self): + return iter(self._rows) + + return _Rows([0, p]) + + +def test_find_path_skips_iterables_digest_never_looks_inside(tmp_path): + """The exclusions have to be exactly the ones ``digest`` itself makes. + + A numpy object array is ``Iterable``, but ``digest`` matches it above the + ``Iterable`` arm and hashes its buffer — the elements never reach a digest, + so a path inside one cannot decide a key and there is nothing to report. + Walking it anyway would also mean walking every large numeric array. + """ + arr = np.array([tmp_path, 1], dtype=object) + assert find_path(arr) is None + assert isinstance(arr, OPAQUE_ITERABLES) + + +def test_find_path_does_not_materialize_a_huge_range(): + """``range`` holds ints by construction; walking one is pure cost. + + Guarded explicitly because the walk pushes elements onto a list, so a + large range would exhaust memory rather than merely be slow. + """ + assert find_path([range(10**9)]) is None + + def test_find_path_does_not_consume_a_generator(tmp_path): """Walking must not have the side effect of exhausting a one-shot iterable.""" gen = (x for x in [tmp_path]) From 58a0cfe0d2ac626b537bdbbcea62eaa17b8fa610 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 00:12:05 +0000 Subject: [PATCH 26/27] test(paths): stop leaking the sticky cache out of the mutation tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fl.cache("memory")` sets the active cache immediately and only restores the previous one if the returned context manager is entered. Called bare, the memory cache stayed active for the rest of the session, so every test that ran afterwards shared it — order-dependent flakiness, and runs-counting tests could see stale hits from an unrelated module. Proved it before fixing, with a probe test either side of this module: assert fl.cache() is before -> FAILED (Cache(values=ValueMemory(...))) Each test now uses the `with` form, matching `test_paths_workflow.py` and `test_prepared_call.py`. Nothing depended on the leak: 1851 passed, 4 skipped. Fixes finding 7 of the review on #797. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- tests/integration/test_path_mutation.py | 76 ++++++++++++------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/integration/test_path_mutation.py b/tests/integration/test_path_mutation.py index 25103cc8..cec08e53 100644 --- a/tests/integration/test_path_mutation.py +++ b/tests/integration/test_path_mutation.py @@ -19,51 +19,51 @@ def make_input(root: Path, name: str) -> Path: def test_mutating_path_consumer_hits(tmp_path): - fl.cache("memory") - runs = [] + with fl.cache("memory"): + runs = [] - @fleche - def consume(d: Path): - runs.append(1) - (d / "out.txt").write_text("produced") - return sorted(p.name for p in d.iterdir()) + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return sorted(p.name for p in d.iterdir()) - assert consume(make_input(tmp_path, "a")) == ["in.txt", "out.txt"] - d2 = make_input(tmp_path, "b") # identical tree, as passed - assert consume(d2) == ["in.txt", "out.txt"] - assert len(runs) == 1 # keyed on the pre-call tree: hit - assert not (d2 / "out.txt").exists() # the mutation is not replayed + assert consume(make_input(tmp_path, "a")) == ["in.txt", "out.txt"] + d2 = make_input(tmp_path, "b") # identical tree, as passed + assert consume(d2) == ["in.txt", "out.txt"] + assert len(runs) == 1 # keyed on the pre-call tree: hit + assert not (d2 / "out.txt").exists() # the mutation is not replayed def test_mutated_state_is_a_different_call(tmp_path): - fl.cache("memory") - runs = [] + with fl.cache("memory"): + runs = [] - @fleche - def consume(d: Path): - runs.append(1) - (d / "out.txt").write_text("produced") - return len(list(d.iterdir())) + @fleche + def consume(d: Path): + runs.append(1) + (d / "out.txt").write_text("produced") + return len(list(d.iterdir())) - d1 = make_input(tmp_path, "a") - consume(d1) # cold; d1 now holds in+out - consume(d1) # post-mutation tree: honest miss, - assert len(runs) == 2 # not a false hit on the old record + d1 = make_input(tmp_path, "a") + consume(d1) # cold; d1 now holds in+out + consume(d1) # post-mutation tree: honest miss, + assert len(runs) == 2 # not a false hit on the old record def test_returned_mutated_argument_captured_in_final_state(tmp_path): - fl.cache("memory") - runs = [] - - @fleche - def stamp(d: Path) -> Path: - runs.append(1) - (d / "stamp.txt").write_text("stamped") - return d - - stamp(make_input(tmp_path, "a")) - warm = stamp(make_input(tmp_path, "b")) # hit, keyed on the input as passed - assert len(runs) == 1 - # ... but the result was captured at commit time: final, stamped state. - assert sorted(p.name for p in warm.iterdir()) == ["in.txt", "stamp.txt"] - assert (warm / "stamp.txt").read_text() == "stamped" + with fl.cache("memory"): + runs = [] + + @fleche + def stamp(d: Path) -> Path: + runs.append(1) + (d / "stamp.txt").write_text("stamped") + return d + + stamp(make_input(tmp_path, "a")) + warm = stamp(make_input(tmp_path, "b")) # hit, keyed on the input as passed + assert len(runs) == 1 + # ... but the result was captured at commit time: final, stamped state. + assert sorted(p.name for p in warm.iterdir()) == ["in.txt", "stamp.txt"] + assert (warm / "stamp.txt").read_text() == "stamped" From 35c3aae5dc03b58bc43ff73f53d3838168a8b1dd Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 18:16:14 +0000 Subject: [PATCH 27/27] test(gc): scope the concurrent-sweep test to what the sweep guarantees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The threaded test added with the in-flight roots asserted value fidelity under a concurrent `gc()`, which is stronger than the code promises, and it went red on CI twice (3.12 and 3.14) while passing locally. The failure was genuine, not a flaky runner: `gc` closes the wide windows but not the gap between `prepare` storing an argument value and registering the call that owns it. Confirmed the mechanism rather than assuming it — holding that gap open makes the eviction deterministic: gc evicted the stored argument: True committed record hands back : 'b979068f6d84...' (bare digest) So the test was wrong, not the fix. It now asserts only what is guaranteed — that a sweep concurrent with writers raises nothing and the registry survives being read while other threads resize it — and two deterministic tests replace the raced assertion: * `test_gc_keeps_a_call_that_commits_mid_sweep` drives a commit from inside the record read, pinning the ordering fix from ff168a0. Verified to have teeth: it fails with the reads in the old order and passes in the new one, without depending on timing. * `test_gc_may_evict_a_value_stored_but_not_yet_registered` pins the window that stays open, so the limit is documented behaviour rather than folklore. If someone closes it, that test fails and should be deleted. `gc`'s docstring now says the residue is reproducible, and states the practical consequence plainly: a sweep concurrent with writers is best-effort, a sweep on an idle cache is exact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RascUYS7JgCMHHpXsYbPTf --- src/fleche/caches.py | 6 +- tests/unit/caches/test_gc.py | 109 ++++++++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 14 deletions(-) diff --git a/src/fleche/caches.py b/src/fleche/caches.py index a7b2f023..4067ed69 100644 --- a/src/fleche/caches.py +++ b/src/fleche/caches.py @@ -499,7 +499,11 @@ def gc(self) -> set[Digest]: same window the one-shot :meth:`save` has always had between storing the values and filing the record — bounded by a storage write rather than by a function body — and closing it would need a lock shared by every - writer. + writer. It is real rather than theoretical: forcing that gap open makes + the eviction reproducible, which + ``test_gc_may_evict_a_value_stored_but_not_yet_registered`` pins. So a + sweep concurrent with writers is best-effort, and a sweep on an idle + cache is exact. Returns: The set of digests that were evicted from value storage. diff --git a/tests/unit/caches/test_gc.py b/tests/unit/caches/test_gc.py index 4d9effd5..c5729a78 100644 --- a/tests/unit/caches/test_gc.py +++ b/tests/unit/caches/test_gc.py @@ -2,7 +2,8 @@ import pytest -from fleche.call import Call +from fleche import call +from fleche.call import Call, PreparedCall from fleche.caches import Cache from fleche.digest import digest from fleche.storage.base import ValueMixin @@ -325,16 +326,16 @@ def test_in_flight_registry_does_not_leak_dropped_calls(gc_cache): def test_gc_is_safe_against_concurrent_prepares(gc_cache): - """Sweeping while other threads admit calls must not raise or lose values. - - Two things have to hold. The registry is a plain mapping read by ``gc``, - so a concurrent ``prepare`` must not resize it mid-iteration. And the - sweep's three reads — candidates, roots, records — must be ordered so a - call that *commits* between two of them is still seen by one: reading the - records before the roots leaves a call that finishes in that gap covered - by neither, and its arguments are evicted under a record that references - them. Both failures show up here as an argument that came back a bare - digest. + """Sweeping while other threads admit calls must not raise. + + Scoped to what the sweep actually guarantees: the registry is a plain + mapping that ``gc`` reads, so a concurrent ``prepare`` must not resize it + mid-iteration, and neither side may blow up. Value fidelity under a + concurrent sweep is *not* asserted here — a narrow window remains between + storing a value and registering the call that owns it, pinned deterministically + by ``test_gc_may_evict_a_value_stored_but_not_yet_registered`` below. An + earlier version of this test asserted fidelity and was flaky on CI for + exactly that reason. """ import threading @@ -345,8 +346,7 @@ def preparer(i): try: while not stop.is_set(): prepared = gc_cache.prepare(Call(name="f", arguments={"x": f"arg-{i}"})) - key = prepared.commit(i) - assert gc_cache.load(key).arguments["x"] == f"arg-{i}" + prepared.commit(i) except BaseException as e: # pragma: no cover - only on a real race errors.append(e) @@ -362,3 +362,86 @@ def preparer(i): t.join() assert not errors, errors + + +def test_gc_keeps_a_call_that_commits_mid_sweep(gc_cache, monkeypatch): + """A call finishing *between* two of the sweep's reads must survive. + + ``commit`` deregisters only after ``save`` has filed the record, so the + registry read and the record read overlap in time — reading the roots + first means a call that commits in the gap is caught by the roots. Reading + the records first (the original order) leaves it covered by neither, and + its arguments are swept out from under a record that references them. + Forced rather than raced: the commit is driven from inside the roots read. + """ + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + committed = {} + + # Drive the commit from the *record* read, so it lands after that read + # whichever order the sweep uses. With the roots read first (current + # order) the registration was still live when the roots were taken, so the + # value survives; with the records read first it is already deregistered by + # the time the roots are taken, and nothing covers it. + # Patched on the class: the storages are frozen dataclasses, so the hook + # cannot go on the instance. + storage_cls = type(gc_cache.calls) + real_list = storage_cls.list + + def list_then_commit(self): + records = list(real_list(self)) + if "key" not in committed: + committed["key"] = prepared.commit("a result") + return records + + monkeypatch.setattr(storage_cls, "list", list_then_commit) + gc_cache.gc() + monkeypatch.undo() + + assert gc_cache.load(committed["key"]).arguments["x"] == "an argument" + + +def test_gc_may_evict_a_value_stored_but_not_yet_registered(gc_cache, monkeypatch): + """The one window the sweep does not close, pinned so it stays known. + + ``prepare`` stores the argument values and only then registers the call as + a gc root. A sweep that reads its candidates and roots inside that gap + sees the value but not its owner, and reclaims it; the later commit files a + record with a dangling reference. This is the same window the one-shot + ``save`` has always had between storing values and filing the record — + bounded by a storage write rather than by a function body — and closing it + would need a lock shared by every writer. + + Asserted so the limit is documented behaviour rather than folklore: if + someone closes it, this test should fail and be deleted. + """ + import threading + + started = threading.Event() + release = threading.Event() + real_post_init = PreparedCall.__post_init__ + + def stall_before_registering(self): + # Values are already stored; registration has not happened yet. + started.set() + release.wait(5) + real_post_init(self) + + monkeypatch.setattr(PreparedCall, "__post_init__", stall_before_registering) + + out = {} + + def worker(): + prepared = gc_cache.prepare(Call(name="f", arguments={"x": "an argument"})) + out["key"] = prepared.commit("a result") + + t = threading.Thread(target=worker) + t.start() + try: + assert started.wait(5) + evicted = gc_cache.gc() + finally: + release.set() + t.join() + + assert digest("an argument") in evicted + assert gc_cache.load(out["key"]).arguments["x"] == digest("an argument")