diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9ca5ea00..3111edd92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,20 +148,17 @@ jobs: save-if: false - run: cargo test --workspace --all-targets --all-features --exclude dolos-minibf --exclude dolos-minikupo --exclude dolos-trp - # `dolos-snapshot` carries two default-off halves — the OCI transport (`oci`) - # and the backfill daemon (`backfill`) — and every job - # above builds them the one way the `dolos` binary does, with both on. A - # `cfg` that only compiles under that combination passes all of them and - # breaks a build nobody here runs. This checks the combinations the crate - # promises instead, at `cargo check` depth: the guard is that each compiles, - # not that it passes a suite the jobs above already run. + # `dolos-snapshot` carries one default-off half — the backfill daemon + # (`backfill`) — and every job above builds it the one way the `dolos` + # binary does, with it on (the default `mithril` feature forwards to it). + # Feature unification means no workspace job ever constructs the + # backfill-off build, so a `cfg` mistake there passes everything and breaks + # only a consumer. This checks that one configuration, at `cargo check` + # depth: the guard is that it compiles, not that it passes a suite the jobs + # above already run. snapshot-features: - name: Check (dolos-snapshot ${{ matrix.features }}) + name: Check (dolos-snapshot default features) runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - features: [oci, backfill, "oci,backfill"] steps: - uses: actions/checkout@v4 with: @@ -171,7 +168,7 @@ jobs: with: shared-key: test-ubuntu-latest save-if: false - - run: cargo check -p dolos-snapshot --all-targets --features ${{ matrix.features }} + - run: cargo check -p dolos-snapshot --all-targets # These sleep on a real relay rather than compute, so they are slow and only # as reliable as the relay. Run them on merges to main; smoke stays on the diff --git a/.github/workflows/registry.yml b/.github/workflows/registry.yml deleted file mode 100644 index 407345e0d..000000000 --- a/.github/workflows/registry.yml +++ /dev/null @@ -1,89 +0,0 @@ -# The #[ignore]d registry suites, in a workflow of their own. Stelae is a -# supporting module of this repository and is expected to move to its own -# repository one day (ADR-004 names the extraction condition); keeping its -# registry gate out of ci.yml keeps the main workflow about this repo's own -# purpose, and lets the gate travel with the crate when it goes. The two -# dolos-snapshot suites ride along because they exercise the same round trip -# from the profile's side; they stay behind when the crate leaves. -name: Registry - -on: - push: - branches: [main] - pull_request: - -permissions: - contents: read - -concurrency: - group: registry-${{ github.ref }} - cancel-in-progress: true - -env: - CARGO_TERM_COLOR: always - -jobs: - # Each test spawns a real OCI registry itself (`docker run`, torn down on - # the way out), so there is no `services:` block to configure and nothing - # here to keep in step with the fixtures. Ubuntu only: the runner with a - # Docker daemon, and the transport is platform-independent. The suites stay - # #[ignore]d so plain `cargo test` is green without a container runtime; - # this job is what makes "ignored" mean "runs here" instead of "runs when - # somebody remembers". No retries: a flake that is not the fixture's is a - # finding about the registry, and it belongs in a PR rather than in a retry - # loop. AGENTS.md ("Code Verification Requirements") mirrors these - # commands; the two must not drift. - registry: - name: Registry round trip (${{ matrix.image }}) - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - image: - # The Docker Official registry, via the ECR Public mirror of the - # same image so the job never spends Docker Hub's anonymous pull - # allowance from a shared runner IP: 2.x is what most operators - # still run, 3.0 is the OCI 1.1 release. - - public.ecr.aws/docker/library/registry:2 - - public.ecr.aws/docker/library/registry:3 - # zot, an OCI-native registry. Pinned: `latest` is a moving - # answer to "which registry did this pass against". - - ghcr.io/project-zot/zot-linux-amd64:v2.1.20 - env: - STELAE_TEST_REGISTRY_IMAGE: ${{ matrix.image }} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@1.91 - # The three matrix legs compile identical artifacts (the image is a - # runtime knob), so they deliberately share one cache; the first leg - # to finish saves it. - - uses: Swatinem/rust-cache@v2 - - # Pulled in its own step so an image that cannot be fetched fails as - # infrastructure, distinguishable from a fixture or suite failure. The - # pull backs off and tries again because ECR Public *throttles* - # anonymous pulls per source IP — runner IPs are shared and the three - # legs start at once — and a throttle, unlike Docker Hub's six-hour - # quota, clears in seconds. The suites below still run exactly once: - # the no-retry rule is about the round trip, and this step exists to - # keep infrastructure out of it. - - name: Pull the registry image - run: | - for attempt in 1 2 3 4 5; do - docker pull "$STELAE_TEST_REGISTRY_IMAGE" && exit 0 - echo "pull attempt ${attempt} failed; backing off" - sleep $((attempt * 30)) - done - exit 1 - - - name: Stelae transport round trip - run: cargo test -p stelae --all-features --test oci -- --ignored --nocapture --test-threads=1 - - - name: Snapshot publish over a registry - run: cargo test -p dolos-snapshot --features oci --test publish -- --ignored --nocapture --test-threads=1 - - - name: Snapshot restore over a registry - run: cargo test -p dolos-snapshot --features oci --test restore_registry -- --ignored --nocapture --test-threads=1 diff --git a/AGENTS.md b/AGENTS.md index 6f672cb76..81ae052f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -333,21 +333,25 @@ All agents working on this repository must verify their modifications by running 4. **Registry round trip** (requires Docker): the `#[ignore]`d suites that spawn a real OCI registry ```bash - cargo test -p stelae --all-features --test oci -- --ignored --test-threads=1 - cargo test -p dolos-snapshot --features oci --test publish -- --ignored --test-threads=1 - cargo test -p dolos-snapshot --features oci --test restore_registry -- --ignored --test-threads=1 + cargo test -p dolos-snapshot --test publish -- --ignored --test-threads=1 + cargo test -p dolos-snapshot --test restore_registry -- --ignored --test-threads=1 ``` Each test spawns its own registry container via `docker run` and tears it down on the way out; the suites are `#[ignore]`d so plain `cargo test` - stays green without a container runtime. Run them when touching - `crates/stelae/src/oci.rs`, the manifest shape, or `crates/snapshot`'s - registry publish/restore paths. `STELAE_TEST_REGISTRY_IMAGE` selects the - server; the `Registry` workflow (`.github/workflows/registry.yml` — its - own workflow, so the gate can travel with a future extraction of - `crates/stelae`) runs these suites on Linux (with `--nocapture`) against - `registry:2`, `registry:3` and a pinned `zot`, so the round trip against a - real registry never depends on someone remembering to run it. + stays green without a container runtime. Run them when touching the + stelae pin or `crates/snapshot`'s registry publish/restore paths; + `STELAE_TEST_REGISTRY_IMAGE` selects the server. + + These are local verification tools, deliberately not a CI job here. + Registry interaction — transport and publish lifecycle — is implemented + by the stelae crates, so testing that integration in CI is + `github.com/txpipe/stelae`'s responsibility, and its `Registry` workflow + runs against `registry:2`, `registry:3` and a pinned `zot`. Dolos's test + subject is the profile, and the profile is transport-blind by + construction: the directory-transport suites in the workspace gate cover + it, and these two suites exist to double-check the composition when the + seam itself is in question. ### Code Quality Standards diff --git a/Cargo.lock b/Cargo.lock index e241079a3..a01fe39b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5380,7 +5380,8 @@ checksum = "5c0e04424e733e69714ca1bbb9204c1a57f09f5493439520f9f68c132ad25eec" [[package]] name = "stelae" -version = "1.7.0-alpha.1" +version = "0.1.0" +source = "git+https://github.com/txpipe/stelae?tag=v0.1.0#23496793cb1f86240f03e008f342f65238ffbb6e" dependencies = [ "bytes", "futures-util", @@ -5389,12 +5390,10 @@ dependencies = [ "minicbor 0.26.4", "oci-client", "reqwest 0.13.4", - "rustls", "serde", "serde_jcs", "serde_json", "sha2 0.10.9", - "stats_alloc", "tempfile", "thiserror 2.0.18", "tokio", @@ -5403,14 +5402,14 @@ dependencies = [ [[package]] name = "stelae-driver" -version = "1.7.0-alpha.1" +version = "0.1.0" +source = "git+https://github.com/txpipe/stelae?tag=v0.1.0#23496793cb1f86240f03e008f342f65238ffbb6e" dependencies = [ "fs4", "minicbor 0.26.4", "serde", "serde_json", "stelae", - "tempfile", "thiserror 2.0.18", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 55a4a3b14..a2ccab019 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ dolos-core = { path = "crates/core" } dolos-cardano = { path = "crates/cardano", features = ["include-genesis"] } dolos-redb3 = { path = "crates/redb3" } dolos-fjall = { path = "crates/fjall" } -dolos-snapshot = { path = "crates/snapshot", features = ["oci"] } +dolos-snapshot = { path = "crates/snapshot" } dolos-mithril = { path = "crates/mithril", optional = true } dolos-minibf = { path = "crates/minibf", optional = true } dolos-minikupo = { path = "crates/minikupo", optional = true } @@ -201,8 +201,6 @@ members = [ "crates/flatfiles", "crates/mithril", "crates/snapshot", - "crates/stelae", - "crates/stelae-driver", "crates/testing", "crates/minibf", "crates/redb3", @@ -262,19 +260,12 @@ opentelemetry_sdk = { version = "0.32.1", features = ["rt-tokio"] } opentelemetry-otlp = { version = "0.32.0", features = ["grpc-tonic"] } tracing-opentelemetry = "0.33" -# Stelae (crates/stelae). `minicbor` is pinned to the 0.26 line that Pallas -# already pulls in, so the protocol crate adds no new CBOR implementation to the -# tree and encodes against the same codec the Dolos profile will use. +# `minicbor` is pinned to the 0.26 line that Pallas already pulls in, so the +# Dolos profile (crates/snapshot) encodes against the same codec the stelae +# protocol crates do — the same pallas-alignment convention the stelae +# repository states in its own manifests. Moving this line is a +# byte-compatibility question, not a dependency bump. minicbor = { version = "0.26", features = ["std"] } -serde_jcs = "0.2.0" -sha2 = "0.10.9" -zstd = "0.13.3" - -# The restore path's disk preflight, and nothing else. `default-features = -# false` drops the file-locking half of the crate, leaving the free-space -# query, which is a `statvfs`/`GetDiskFreeSpaceExW` call over `rustix` and -# `windows-sys` — both already in this tree. -fs4 = { version = "1.1.0", default-features = false } [workspace.metadata.release] push = false diff --git a/adrs/004_stelae_snapshots.md b/adrs/004_stelae_snapshots.md index d95a45129..f52f3ff0a 100644 --- a/adrs/004_stelae_snapshots.md +++ b/adrs/004_stelae_snapshots.md @@ -4,6 +4,20 @@ Proposed +> **This is the decision record, not the specification.** It states the +> problem — the friction of the tarball-shaped snapshot — and the adoption +> of Stelae: the decision, its rationale, its limitations, and the +> alternatives it displaced. The normative text lives elsewhere, split along +> the protocol/profile boundary the decision itself drew: the **protocol** +> in [`SPEC.md` of +> `txpipe/stelae`](https://github.com/txpipe/stelae/blob/main/SPEC.md), and +> the **Dolos profile** in +> [`crates/snapshot/PROFILE.md`](../crates/snapshot/PROFILE.md). Older +> references to this ADR's implementation sections resolve there — PROFILE.md +> preserves the section names. The protocol version this tree implements is +> the tag `crates/snapshot/Cargo.toml` pins, whatever `main`'s spec says +> since. + ## Context - Dolos snapshots are currently a gzip tarball of the raw `archive/`, `state/` and `index/` database directories, uploaded to publicly accessible storage (Cloudflare R2) and addressed by a URL template (`https://dolos-snapshots.txpipe.cloud/${VERSION}/${NETWORK}/${VARIANT}/${POINT}.tar.gz`). There is no manifest, no checksum and no signature; the only integrity check is that gzip/tar fail on corrupt data. @@ -110,234 +124,40 @@ None of those four goals is Cardano-specific, and neither are the mechanisms tha - Pros: one crate, one vocabulary, no extension machinery to design or test. - Cons: a third-party publisher has no collision-free namespace and would have to fork the spec; the Dolos context absorbs decisions (framing, attestation, transport) unrelated to a data node; and extraction later means renaming every media type, tag and identifier already published. The boundary costs one crate today and is irreversible-cheap only before implementation starts. -## Implementation Details - -### Naming, profiles and media types - -Envelope types are protocol-owned and shared by every profile; payload types are vendor-owned: - -| Role | Media type | Owner | -|---|---|---| -| Artifact type (manifest) | `application/vnd.stelae.stele.v1` | protocol | -| Config blob (the inscription) | `application/vnd.stelae.inscription.v1+json` | protocol | -| Signature (referrer artifact) | `application/vnd.stelae.signature.v1` | protocol | -| Layer payloads | `application/vnd.{vendor}.stele.{kind}.v{n}+{codec}` | vendor | -| — Dolos profile | `application/vnd.dolos.stele.{blocks\|indexes\|log-{ns}\|state-{ns}\|digests}.v1+zstd` | Dolos | - -Normative rules for coexistence: - -1. Payload media types must carry a vendor slot the publisher controls. `vnd.stelae.*` is reserved for envelope types and is never a payload type — Stelae defines no payload format. -2. Profile names are reverse-DNS and vendor-owned; this profile is `io.txpipe.dolos.cardano`, version 1. The short token in media types (`dolos`) follows IANA `vnd.` custom. -3. The protocol never parses layer bodies or a profile's opaque objects. An unknown profile name, or a profile major version the client does not implement, is a clean refusal — never a partial or misinterpreted restore. A layer whose **kind** the client does not implement is deliberately not one of those: the client cannot store what it does not model, so it skips the layer and reports the skip, and only a `required: true` in that layer's `scope` turns the skip back into a refusal naming the kind and the scope. The publish side keeps the strict rule in both cases — a publisher that cannot build a kind must not chain onto a stele carrying it, since the alternatives are dropping the layer from the repository silently or attesting bytes it never read. `required` lives in the profile-owned `scope` and not in an OCI annotation, so it is signed planning input rather than unsigned transport metadata; the protocol carries it and never reads it. It is one-way — a kind published as required forever constrains readers older than it — so marking one is an ADR-level act and rare by construction. -4. One repository per (profile, dataset). Sharing a registry namespace is safe: discovery filters on the common `artifactType`, the `profile` field discriminates, and tags are rendered by the profile. -5. Signatures are generic and cover the inscription digest, which itself binds the profile — so signing and verification tooling is shared across vendors. - -Rule 3 answers for a kind the reader does not *know*. It says nothing about a kind the publisher no longer *carries*, and `required: true` cannot be stretched to cover one: `required` is a property of a layer, and a retired kind has no layer to put it on. Absence is already meaningful in this format — a `log-{ns}` layer exists if and only if it holds a record, and a restore passes over a kind it does not recognise — so a reader that still models `log-member-rewards`, finds no such layer, and reports a clean restore has just built a node with no reward history and no way to have noticed. - -**A profile therefore declares the namespaces it defines, and a retirement is declared rather than inferred.** `parameters.schemas` carries an entry for every namespace the profile version defines; a namespace it has retired keeps its entry at revision `0`, which is not a schema revision and reads as "this version defines no records here". A restore compares that map against the namespaces it models, before a store is opened: an entry that is missing or zero for one it models refuses the restore and names the namespace. Only presence is judged, never the revision's value — a revision the reader has not seen describes bytes it can still parse, and gating on it would make every additive append breaking, which is exactly what the `.v{x}` contract below exists to avoid. - -Like `required`, the rule binds forward and not backward: it constrains readers from the version that implements it onward, and cannot reach the ones already deployed. What protects those, for the four namespaces retired so far, is that every one of them was also a *state* namespace, and the state tip's completeness check refuses a stele missing a kind it expects. A log-only namespace would have had no such backstop, and that is the case this rule exists for. Retiring a namespace is an ADR-level act, for the same reason marking one `required` is. - -Rule 3's skip is available at layer granularity and at no finer one. Index **dimensions** stay fail-closed: `indexes` is a single layer per epoch, so an unknown dimension surfaces mid-stream — record by record, inside a layer the plan has already committed to restoring — where skipping it would be silent data loss rather than a visible plan-time choice, and where the store cannot look the name up in any case (it keeps a hash of the name, not the name). Changing the dimension set therefore remains a media-type-version event. The same reasoning is why a new *namespace* is additive and a new dimension is not: a namespace arrives as its own `log-{ns}` or `state-{ns}` layer, which a plan can decline; a dimension arrives inside one. - -### Layer formats - -All layers are zstd-compressed CBOR sequences (RFC 8742). Deterministic encoding profile pinned by the spec: shortest-form integers, definite lengths only, no floats, no tags. Every layer starts with a protocol-defined header record that makes the blob self-describing even when detached from its registry: - -``` -[format_version = 1, profile: tstr, kind: tstr, scope: any] -``` - -`scope` is opaque to the protocol. The Dolos profile encodes `[network_magic, epoch, start_slot, end_slot]` for epoch layers, `[network_magic, epoch, shard]` for every state layer — one shape across all fourteen kinds, single-blob namespaces included, whose one layer is shard 0 — and `[network_magic, epoch, last_immutable]` for the digests layer. - -The state layers carry **two roles over that one header shape**, and only the *descriptor* scope tells them apart: a tip is `{"shard": n}`, a retained dump is `{"epoch": E, "shard": n}`. The header is deliberately blind to the distinction, and that is what makes the dump a publish cuts at `sequence == E` the tip's own bytes rather than a copy of them — same header, same records, one `diffId`, one blob under two descriptors. See "State history" below. - -Content records per kind (Dolos profile): - -| Kind | Record | Order | Restore write path | -|---|---|---|---| -| `blocks` (per epoch) | `[slot, hash: bytes(32), body: bytes]`, body = raw wire CBOR verbatim | ascending slot, stream order for same-slot (Byron EBB) | `ArchiveWriter::apply` | -| `indexes` (per epoch) | tags: `[0, dimension: tstr, key_hash: bytes(8), slot]` with `key_hash = xxh3_64(key)` BE — except dimension `metadata`, see below; exact: `[1, kind: tstr, key: bytes, slot]` for block-hash/block-number/tx | sorted, deduped | new `IndexWriter::append_prehashed` | -| `log-{ns}` (per epoch, per log namespace, omitted when empty) | `[log_key: bytes(40), value: bytes]`, value = stored EntityValue verbatim | `log_key` | `ArchiveWriter::write_log` into the namespace the kind names | -| `state-{ns}` (tip or retained dump, per state namespace, `scope.shard` = 0..`parameters.shards[ns]`-1) | `[key: bytes, value: bytes]` | `key`; shard = first nibble of `key[0]` for a 16-way namespace, 0 for a single blob | dispatch on the kind: `state-utxos` → chunked `StateWriter::apply_utxoset`, else `write_entity` into the namespace the kind names | -| `digests` (tip, optional) | `[immutable_number, chunk: bytes(32), primary: bytes(32), secondary: bytes(32)]`, each sha256 over the raw file bytes | ascending `immutable_number` | none — verification metadata, not written to stores | - -One exception to the tag hashing rule is normative for `indexes` v1: records in dimension `metadata` carry the logical u64 metadata label **verbatim** (big-endian) in `key_hash`, never hashed. The index stores keep metadata labels as raw labels rather than hashes, and the layer ships the stored form — that is the whole point of the pre-hashed design. `parameters.indexKeyHash` therefore describes every dimension *except* `metadata`. A publisher that hashes metadata labels produces structurally valid records that restore cleanly but can never be matched by a metadata query; conformance tooling must check this dimension specifically (#1149 tracks whether a future media-type version unifies the rule). - -State namespaces: the 16 entity namespaces from `dolos_cardano::model::build_schema()` (key = 32-byte `EntityKey` verbatim, value = stored minicbor verbatim) plus `utxos` (key = `tx_hash(32) ‖ output_index(4, BE)`, value = CBOR `[era: uint, body: bytes]`). The chain point lives in the inscription's `position`, not in a layer. Live-UTxO index dimensions (`utxo::*`) are not shipped; they are rebuilt at restore via `index_delta_from_utxo_delta`. - -State kinds: one per state namespace, and the set is closed — 14 of them, spelled `state-` followed by the namespace with `_` rewritten to `-`, by the same rule and for the same reasons as the log kinds below. The namespace is therefore **not** in the record — it is the layer — which is what puts the fail-closed edge of a breaking change on exactly the namespace that broke, and lets a reader skip a namespace this profile does not define at the transport rather than choking on one shared layer. The shard count is **specification, never tuning**: `utxos`, `accounts`, `assets` and `datums` split 16 ways, every other namespace is a single blob, and `parameters.shards` reports the map so a reader never has to discover it from the data. Re-sharding a namespace is a media-type-version event for that namespace's kind. Every shard of every kind is published, empty ones included, so tip completeness is structural: a restore requires all 14 kinds and, per kind, exactly the shards its count promises. - -**State history: retained dumps at configured epochs, plus the moving tip.** A stele's state is the tip — the ledger as of `sequence`, swapped whole by every publish — and, for each epoch a publisher retains, an immutable **dump** of the state as of that epoch. The two are the same kinds, the same records and the same shard geometry; a dump differs from a tip in its descriptor scope, which names the epoch, and in nothing else. - -- **The retained set is configuration, not derivation.** `snapshot.state_epochs` names it. Era boundaries are one sensible criterion and cherry-picked epochs another; which epochs are worth a dump is operational, so nothing derives the list from the chain summary. The list is strictly ascending and never names epoch 0, and it is refused where it is read rather than where a dump is cut — it reaches `parameters` before any layer is written. Publishers are expected to keep it around 10–20: per-epoch dumps were rejected outright (~46k manifest descriptors on mainnet against a ~12k ceiling). -- **The list is signed input.** It is echoed verbatim into `parameters.stateEpochs`, so two publishers of one network configured differently produce different parameters, different inscription digests, and a divergence an operator reads out of a one-field diff instead of hunting through layers for. It is therefore **normative per network** and belongs pinned wherever the default repository is pinned: a publisher quietly running a different list self-ejects from co-signing. -- **Production rule.** At the publish where `sequence` equals a retained epoch E, E's dump is cut from the tip: one walk of the store, one sink per shard, one blob, and the transport attests the result a second time under the dump's scope. Nothing is compressed twice and nothing crosses the wire twice. At a publish standing past E, the dump is adopted from the predecessor by the same scope equality every immutable layer uses. A dump for a past epoch that no predecessor carries is a **warning and a shorter stele**, never a failed publish: this publish's stores hold the tip, and the state as of an epoch it has moved past is not in them to be written. Producing one is a backfill run's job. -- **Inheritance.** The rule "no state layer is ever inherited" was about the tip role, in two independent ways, and both still hold of it: the tip changes every publish, and its scope names no epoch, so scope equality could not tell one publish's shard from another's. A dump's scope does name its epoch, so it inherits, checkpoints and resumes exactly as a `blocks` layer does — including through the resumption record an interrupted publish leaves. -- **Adoption is per shard.** A predecessor that can hand over fifteen of a dump's sixteen shards hands over fifteen. Adopting is an act — it puts the blob in the transport — so all-or-nothing would mean either leaving blobs the inscription does not describe (a refused seal) or refusing a publish over history that was never load-bearing. -- **Restore reports dumps and does not consume them.** A restore builds a node standing at `sequence`, so the tip is what it reads; dumps are recorded in the plan, excluded from the byte accounting and from the disk preflight, and reported. Tip completeness is checked on the tip alone. Bootstrapping *at* a retained epoch — consuming a dump as the tip — is deliberately out of scope here and nothing forecloses it: the restore write path takes a kind and a descriptor, and a dump's are the tip's. - -Log kinds: one per namespace the ledger writes epoch-boundary logs under, and the set is closed — `log-account-epochs`, `log-epochs`, `log-stakes`. `account-epochs` is one `(account, epoch)` record holding everything an account did in an epoch; it replaced `account-stakes`, `leader-rewards`, `member-rewards` and `pool-deposit-refunds`, which were four identically-keyed namespaces and are now the profile's four retired ones (ADR-0027). The kind token is `log-` followed by the namespace with `_` rewritten to `-`, since a media type's kind token admits hyphens and not underscores; the mapping is injective, and a publisher spells the kinds out rather than composing them, so a namespace rename cannot silently rename a published kind. The namespace is therefore **not** in the record — it is the layer — and a restore writes a layer's records into the namespace its kind names. A publisher that finds logs under a namespace no kind covers **fails the publish** naming the namespace: the format has no layer to carry them, and shipping a snapshot that silently omits a slice of the ledger is the failure that costs most and shows least. Log layers wear the epoch scope unchanged, so per-(kind, scope) inheritance works across the split with no new scope shape. - -Empty log layers are omitted: **a `log-{ns}` layer exists if and only if it holds at least one record.** The rule is content-determined and not writer discretion — that is what keeps it deterministic across publishers, since two honest publishers agree about whether a layer exists exactly because they agree about whether a record does. Byron alone sheds ~1,200 empty blobs to it. Absence is normative and never a defect: a restore that finds no `log-stakes` for an epoch restores that epoch with no stake logs, exactly as the ledger had none. Every other kind keeps the arity it had — `blocks` and `indexes` are one layer per epoch whatever the window holds, and every state layer is always present, empty or not. - -#### Compatibility contract - -Within a media-type version, `.v{x}` is a **contract on record content, not an exact-byte pin** (decision 0026). Writers MAY append numbered optional or `#[cbor(default)]` fields; readers MUST skip fields they do not know; field indexes are never renumbered, removed or repurposed. Every append moves that namespace's `schemas` revision — additive included, because `schemas` is what keeps the inscription a full pin of the bytes rather than a compatibility gate. `v{x}` bumps are reserved for breaking changes. - -Enums are the contract's hard edge and the one place the rule inverts. A minicbor enum refuses a variant index it has never heard of, so **adding a variant to any enum reachable from a record is reader-breaking within `v{x}`**, whatever the field policy says: it requires a media-type version bump on the kinds that carry it, or an explicit ADR waiver. - -The contract is enforced by `crates/snapshot/tests/field_registry.rs` and its `tests/registry/` data module, which is where a record shape is actually pinned. Per namespace it holds a **canary** — a fully-populated value, every `Option` `Some` and every collection non-empty — and the hex of its encoding through the production encode path, at that namespace's current `schemas` revision. The suite then asserts, on every build: - -- the canary still encodes to its pinned bytes, so a renumbering, a removal or a silent width change is a build failure rather than a moved digest with no explanation; -- the registry's current revision equals `SCHEMA_REVS[ns]`, so a field appended without its revision bump — or a bump with nothing pinning it — fails in either direction; -- every **retained** revision still decodes under today's decoder, which is the reader tolerance the contract promises, asserted rather than assumed. Retained canaries are append-only: never edited, never deleted. A change that makes one undecodable is breaking by definition. -- every reachable enum's variant table is pinned per variant, including the Pallas enums the records embed — a dependency upgrade that renumbered `Relay` or `DRep` would otherwise change published bytes with nothing in this repository to notice; -- the codec's own tolerance behaviour (unknown trailing field skipped, index gap null-padded, missing trailing field defaulted) holds against the pinned `minicbor` version, so a codec upgrade that changed any of the three breaks in a test rather than in a published stele; -- the same value encodes to the same bytes across repeated construction. A failure there is never a re-pin: it is an encoding-determinism defect, and it breaks the cross-party digest identity this whole document rests on. - -The `digests` layer covers the immutable files fully contained in the stele's block range: `lastImmutable` is derived from the boundary slot and the chunk geometry observed in the chain — canonical, never dependent on aggregator state at publish time. Digest values equal Mithril Cardano DB v2's merkle leaves (hex-decoded), so any Mithril certificate whose beacon covers `lastImmutable` can verify them via the aggregator's digest route and a merkle proof. The certificate reference is deliberately *not* part of the inscription: certificates are produced on the aggregator's cadence, so two independent publishers at the same boundary would reference different certificates — including one would break cross-publisher determinism, while the digest values themselves are byte-stable properties of the chain. - -### The cut point and the boundary sliver - -A stele is cut by syncing with `chain.stop_epoch = E`, and the halt is **one block past the epoch boundary**, not on it. The sync crosses the boundary — Ewrap closes epoch E-1, Estart opens epoch E — and then applies the first block of epoch E before stopping. That block is what makes the stele addressable: Estart alone leaves the cursor a bare `ChainPoint::Slot` carrying no block hash (`estart::commit_finalize`), and `position.point` must carry one for a stele to be verifiable against a chain. The anchoring block may itself sit exactly on `epoch_start(E)`, in which case the sliver is one slot wide. - -So the epoch windows a stele covers are `0..=E`: epochs `0..E-1` **complete**, plus epoch E's **boundary sliver** — the window that opens at `epoch_start(E)` and closes at the anchoring block. The sliver is normative, not an artifact of where a publisher happened to stop. +## Adoption -It is load-bearing because of how boundary data is keyed. **All of epoch X's boundary logs key at `epoch_start(X)`**: Ewrap writes the *ending* epoch's closing logs and its completed `EpochState` at `epoch_start(X)` when X is the epoch it closes, and Estart writes the *starting* epoch's opening account logs at `epoch_start(X)` when X is the epoch it opens. One temporal key per epoch, and nothing straddles two windows. Epoch E's estart logs therefore live inside the sliver and nowhere else: a stele that dropped the sliver in the name of a clean boundary would ship a state tip whose opening logs are in no layer at all, and a restored node would never regenerate them. +The specification this ADR proposed lives in two normative documents, and +this record deliberately duplicates neither: -`sequence`, the immutable tag's E, and `position.epoch` are consequently **one number**: the epoch the cursor stands in, which is also the last epoch the layers cover. The operator-facing form of the same rule is *configure the epoch you want the node to start in* — `stop_epoch = E` produces the stele tagged `epoch-E`. +- **The protocol** — framing, the inscription and its history invariant, the + manifest and its agreement rules, the transport and what it requires of a + host, restore planning — is [`SPEC.md` in + `txpipe/stelae`](https://github.com/txpipe/stelae/blob/main/SPEC.md), the + repository the `stelae` and `stelae-driver` crates extracted to. +- **The Dolos profile** — layer kinds and record shapes, scopes, state + history, `position`/`parameters`, the compatibility contract, the cut + point, and the publish/restore pipelines — is + [`crates/snapshot/PROFILE.md`](../crates/snapshot/PROFILE.md), beside the + `dolos-snapshot` crate that implements it. -### OCI layout and the inscription - -- Repository per (profile, network) — e.g. `ghcr.io/txpipe/dolos-snapshots/mainnet`; tags `epoch-E` (E = the newly started epoch, equal to `sequence` and to `position.epoch`; layers cover epochs `0..E-1` complete plus epoch E's boundary sliver — see "The cut point and the boundary sliver" above) and `latest`. The protocol requires an immutable tag per sequence plus a moving `latest`; the profile renders the strings. -- `artifactType: application/vnd.stelae.stele.v1`; layer media types per the table above; three annotations per layer, named in "The manifest" below — one of them normative, the other two informational. -- Config blob (`application/vnd.stelae.inscription.v1+json`), canonical JSON per RFC 8785. Generic keys plus three profile-owned opaque objects — `position`, `parameters` and each layer's `scope`: - -```json -{ "schema": 1, - "profile": {"name": "io.txpipe.dolos.cardano", "version": 1}, - "sequence": 550, - "position": { "network": {"magic": 764824073, "name": "mainnet"}, - "point": {"slot": 152236812, "hash": "…"}, - "epoch": 550 }, - "parameters": { "indexKeyHash": "xxh3-64", - "shards": {"accounts": 16, "assets": 16, "datums": 16, "utxos": 16, "…": 1}, - "schemas": {"accounts": 1, "utxos": 1, "…": 1}, - "stateEpochs": [208, 236, 290, 365] }, - "compression": {"algo": "zstd", "level": 9}, - "history": [ - {"sequence": 548, "inscriptionDigest": "sha256:…"}, - {"sequence": 549, "inscriptionDigest": "sha256:…"} ], - "layers": [ - {"kind": "blocks", "mediaType": "application/vnd.dolos.stele.blocks.v1+zstd", - "diffId": "sha256:…", "records": 21600, "uncompressedSize": 43210000, - "scope": {"epoch": 0, "startSlot": 0, "endSlot": 21599}}, - {"kind": "state-utxos", "mediaType": "application/vnd.dolos.stele.state-utxos.v1+zstd", - "diffId": "sha256:…", "records": 812345, "uncompressedSize": 402653184, - "scope": {"shard": 0}}, - {"kind": "state-pools", "mediaType": "application/vnd.dolos.stele.state-pools.v1+zstd", - "diffId": "sha256:…", "records": 3210, "uncompressedSize": 1048576, - "scope": {"shard": 0}}, - {"kind": "state-pools", "mediaType": "application/vnd.dolos.stele.state-pools.v1+zstd", - "diffId": "sha256:…", "records": 2980, "uncompressedSize": 972800, - "scope": {"epoch": 365, "shard": 0}}, - {"kind": "digests", "mediaType": "application/vnd.dolos.stele.digests.v1+zstd", - "diffId": "sha256:…", "records": 6188, "uncompressedSize": 618800, - "scope": {"lastImmutable": 6187}} ] } -``` - -`parameters` is the profile's compatibility declaration. Three of its four values are a consequence of publisher code rather than a free choice: `indexKeyHash` names the hash behind the pre-hashed index keys; `shards` is the per-namespace shard map above; and `schemas` is a per-namespace revision of the *record content* — the stored minicbor a `state-{ns}` or `log-{ns}` layer carries verbatim — which moves when that namespace's stored shape changes, plus one entry at revision `0` per retired namespace, per the removed-kind rule above. Thirteen of the fourteen live revisions are 1; `epochs` is at 2, the first bump the format has taken (see Limitations), and the four retired namespaces sit alongside them at 0. Every live revision is pinned by a canary in `crates/snapshot/tests/field_registry.rs`, which fails the build when a record's field table moves without its revision, or the other way round. The split between the two is deliberate: a change to how a layer is *framed* moves that kind's media type and fails closed at the transport, while a change to what a record *contains* moves its schema revision, which a reader consults to decide whether it can interpret what it can already parse. The fourth, `stateEpochs`, is the exception that proves the rule: it is the publisher's configured retained set, and it is here precisely *because* it is a choice — declaring it is what turns a configuration difference between two publishers into a visible parameters difference instead of a silently divergent history. - -`sequence` is the protocol's ordering key; the Dolos profile sets it to the epoch. `diffId` = sha256 of the uncompressed CBOR sequence. `position`, `parameters` and `scope` are canonicalized by JCS like every other key, so determinism holds without the protocol interpreting them; verifiers reject unknown *generic* top-level keys, so extension happens only inside those three objects. Determinism and signing are defined only over this document's sha256. Signatures are Ed25519 over the inscription digest, pushed as OCI referrer artifacts (`application/vnd.stelae.signature.v1`, cosign-compatible envelope where convenient). Restore verifies registry blob digests (transport integrity) and diffIds (canonical identity). - -`history` embeds the digest of every previously published inscription, so the latest signed inscription transitively attests the entire publication history (~80 bytes per sequence, ~50 KB after 600 epochs — negligible for a config blob). This makes attestation outlive blob retention: a stele whose blobs have long been garbage-collected can still be verified by anyone holding a copy, because the copy carries its own inscription (the OCI config blob) — check that inscription's digest against the `history` of the latest signed one, then the layers against its diffIds. No external trusted storage of attestations is required. - -History invariant: `history` contains exactly one entry per published sequence, contiguous from the network's first published sequence (pinned per network alongside the default repository) up to `sequence - 1`, in strictly ascending order — no gaps, no duplicates. JCS canonicalizes object keys but preserves array order, so the ordering is normative. Verifiers reject inscriptions that violate the invariant. This is a reproducibility requirement as much as a safety one: independent publishers converge on byte-identical inscriptions only if the publication schedule and history encoding are canonical; an independent party reproduces the digest chain naturally by computing each boundary inscription while replaying the chain. If the list ever outgrows the inscription, or succinct append-only consistency proofs become a requirement, the designated evolution is a sequence-indexed Merkle Mountain Range commitment (`{root, size}`) — a schema-versioned change that can be built retroactively from the flat list. - -Note: a side-effect of anchoring identity on uncompressed content digests is that layer *content* can be mirrored over any content-addressed transport (e.g. IPFS) — or re-compressed with a different algorithm — and still be verified against the same signed inscription via diffIds. Consumption is stricter than verification: the restore client expects the canonical zstd blobs referenced by the OCI manifest, so re-encoded mirrors serve archival and verification, not direct restore. This is a property of the format, not a requirement of the protocol; the OCI registry remains the canonical distribution channel. - -#### The manifest - -A stele in a registry is one OCI image manifest, and its shape is closed: a conforming publisher writes exactly the fields below, and a conforming client refuses anything else. - -- `schemaVersion: 2`; `mediaType: application/vnd.oci.image.manifest.v1+json`; `artifactType: application/vnd.stelae.stele.v1`. -- `config` is the inscription's descriptor: `mediaType` is `application/vnd.stelae.inscription.v1+json`, `digest` is the sha256 of the canonical inscription bytes — the same digest independent parties reproduce and sign — and `size` is those bytes' length. -- `layers`: one descriptor per inscription layer, **in inscription order**. Each carries the layer's `mediaType` exactly as the inscription states it, the compressed blob's `digest` and `size`, and the three annotations below. -- No `subject` and no manifest-level `annotations`. - -The manifest bytes are canonical JSON per RFC 8785, through the same canonicalizer as the inscription, and are pushed verbatim: the protocol has one answer to "what are the bytes of this JSON document", not two that agree until they do not. - -The per-layer annotation keys are reverse-DNS under `stelae.store`, a domain TxPipe owns: - -| Key | Status | Value | -| --- | --- | --- | -| `store.stelae.layer.diffId` | **normative** | the layer's `diffId`, exactly as the inscription states it | -| `store.stelae.layer.kind` | informational | the layer's profile-defined kind | -| `store.stelae.layer.scope` | informational | the layer's scope object as stringified canonical JSON (annotation values are strings) | - -`store.stelae.layer.diffId` is the identity→blob map — the thing a registry hands over for free and a directory has to rebuild by decompressing every blob. A client that does not read it cannot fetch a layer; it is the one annotation a reader must understand. The other two exist so a human or a generic registry tool can see what a blob covers without fetching the config blob, and a client may ignore them. - -#### Manifest–inscription agreement - -The manifest and the inscription are two views of one stele — the inscription holds identity, the manifest holds transport — and a disagreement between them, in either direction, is a refusal, never a preference. - -A publisher refuses to build a manifest — before anything is pushed — when the inscription describes a layer that was never written, or a layer was written that the inscription does not describe: a blob nothing attests must not be published. - -A client refuses a manifest — before any blob is fetched — when: - -- `artifactType` is missing. This fails closed by choice: a registry that strips the OCI 1.1 discovery field has published something a client cannot recognize as a stele, and reading it anyway would make the discovery contract advisory. -- `artifactType` is present and is not `application/vnd.stelae.stele.v1`. -- the config descriptor's media type is not the inscription's. -- the manifest's layer count differs from the inscription's. -- a layer carries no `store.stelae.layer.diffId` annotation, so nothing says which layer it holds. -- a layer's `diffId` annotation disagrees with the inscription's layer *at that position*. Positional correspondence is a check of its own: a manifest carrying the right blobs in the wrong order passes the map and fails the order. -- a layer's media type disagrees with the inscription's at that position. - -#### The manifest size ceiling - -A manifest past **4 MiB** (`stelae::MANIFEST_SIZE_LIMIT`) is refused before the push. The figure is not a limit the OCI specification imposes; it is the ceiling registries converge on, and the refusal is measured on the exact canonical bytes that would have been pushed, so it names the document and its layer count instead of arriving later as a registry's `413`. - -The arithmetic is counted in layers, because layers are what the ceiling counts: a descriptor with its annotations costs ~350 bytes, so the ceiling falls near 12,000 layers. A mainnet stele is bounded above by ~600 epochs × 5 per-epoch kinds (`blocks`, `indexes` and the three `log-{ns}`), plus the state tip's 74 layers (4 namespaces × 16 shards + 10 single blobs), plus 74 more for every retained state dump — at 20 retained epochs, the ceiling of what a publisher is expected to configure, that is ~4,554 layers and a manifest of roughly 1.6 MB, still comfortably inside the ceiling. The bound is loose in the direction that helps: the log kinds are omitted when empty, and Byron's ~200 epochs carry no reward or stake logs at all, so the realized count sits near ~4,150. **This is the arithmetic that bounds the retained list**, and the reason per-epoch dumps were rejected: ~580 of them would be ~43,000 state layers on their own, more than three times the ceiling. (The Rationale's "~1,700 manifest descriptors" is decision-time sizing of the pre-split artifact; this paragraph is the authoritative count, and it counts layers rather than epochs.) - -#### What the transport requires of its host - -- **A process that opens a registry client must have installed a process-default rustls `CryptoProvider` first.** The transport ships no crypto backend of its own (`reqwest/rustls-no-provider`): the backend the client library would otherwise pick, `aws-lc-rs`, wants `cmake` on every build machine — the dependency this workspace already goes out of its way to avoid — so it stays out of the tree and the choice of provider moves to the program. In Dolos, `main()` installs `ring`. Omitting the install is a panic when the registry client opens, not a link error. -- **Authentication is the host's decision, in one of three shapes.** The client is opened with credentials its caller supplies — anonymous, a bearer token, or an HTTP Basic pair — and never sources them itself. Which identity a program authenticates as is that program's credential policy, and where it keeps its credentials is that program's deployment: a protocol library that read an environment variable would be deciding both on its host's behalf, and naming the variable would freeze that decision into a published API. **So this specification names no environment variable and no configuration key**, and `stelae::oci::Options::auth` is the whole of the interface. Dolos's own answer is under "CLI and configuration" below. - - Anonymous remains legitimate and is what a genuinely public repository wants. It is not what a registry that authenticates every request wants, and that is the deployment Dolos is heading for: read access to a stele repository is free and identity-less, and still credentialed. +What remains below is what only this repository can say: how Dolos adopted +the protocol — where the code lives, what the operator surface is, and the +phases the work shipped in. ### Code layout -Four crates, all workspace members until the extraction. The Stelae half is two of them — the protocol a third party implements from and the profile-generic lifecycle machinery — and the boundary is checkable: **`cargo tree -e normal --all-features` for `stelae` and `stelae-driver` must contain no `dolos-*` package**, so extracting the pair is a directory move rather than a refactor. +The extraction has happened: the protocol crate (`stelae`) and the +profile-generic lifecycle machinery (`stelae-driver`) live in +[`github.com/txpipe/stelae`](https://github.com/txpipe/stelae), history +preserved, and this workspace consumes them as one pinned git tag (both +crates version in lockstep; the pin lives in `crates/snapshot/Cargo.toml` +and nowhere else). Their module layout is documented in that repository; +"no `dolos-*` dependency" is enforced there by its cargo-deny bans — the +boundary this section once asked contributors to keep is structural now. -```text -crates/stelae/ # package `stelae` — the wire protocol, zero dolos deps - lib.rs # errors, protocol constants, envelope media types - frame.rs # deterministic CBOR-seq record read/write, Limits - codec.rs # fixed-arity decode helpers for layer content records - inscription.rs # schema, JCS encode/verify, digest, history invariant (history_for) - profile.rs # Profile trait, layer-kind registry, media-type & tag naming rules - digest.rs # streaming sha256 + zstd (diffId + blob digest in one pass) - layer.rs # reading a layer without holding it - plan.rs # progress file, resume, remaining-bytes accounting - progress.rs # Observer: what a transfer says about itself while running - transport.rs # the SteleReader/SteleWriter seam and the blob index - dir.rs # a stele on a local filesystem - oci.rs # feature `oci`: push with blob-skip, pull missing-only, tags, referrers - tests/toy_profile.rs # a second, trivial profile — proves the core carries no Dolos assumption - -crates/stelae-driver/ # package `stelae-driver` — profile-generic lifecycle, zero dolos deps - lib.rs # the driver's Error - profile.rs # DriverProfile: the dataset policy stelae::Profile deliberately refuses - predecessor.rs # Predecessor/First: what a publish follows and may carry forward - publish.rs # the chained-publish lifecycle: open, Tuning, Publishing, Chained, standing - restore.rs # Budget/Checkpoint/Outlook: restore bounds and the resume checkpoint - preflight.rs # one free-space policy, in both directions - reporting.rs # counting layers and records for the two drivers to report - retry.rs # bounded patience for an external that fails in bursts - digests.rs # the digests-layer codec (Cardano immutable-DB file hashes) +What remains here is the profile side: +```text crates/snapshot/ # package `dolos-snapshot` — the io.txpipe.dolos.cardano profile lib.rs # DolosProfile, driver re-exports, profile constants, error mapping namespaces.rs # the closed set of state namespaces a Dolos stele carries @@ -396,28 +216,6 @@ Two shapes are refusals rather than precedence rules, checked once the configura The resolution is `dolos::common::stele_registry_auth`, a pure function of `[stelae.registry]` that hands the answer to the transport as a value. Another host embedding `stelae` decides its own credential sources, and this specification constrains none of them. -### Publisher pipeline - -1. Restore the publisher node from the previous stele (self-hosting delta pull; first run via Mithril). -2. Sync with `chain.stop_epoch = E` until `StopEpochReached` — the state crosses the boundary and lands on the **first block of epoch E**, the block that gives `position.point` a hash. -3. `dolos snapshot publish` — only the newly closed epoch's layers and epoch E's boundary sliver upload; fresh state layers + inscription; tag `epoch-E`, move `latest`. On networks with a Mithril aggregator, fetch the immutable-file digest list from the aggregator's digest route, verify it against a certificate, and write the `digests` layer for the files within the boundary. -4. Determinism job: an independent runner that synced by any means runs `dolos snapshot digest` and alerts on inscription mismatch. -5. Matching verifiers sign and push referrer signatures; clients enforce k-of-n. - -Registry hygiene: keep a trailing window of `epoch-E` tags (e.g. 12); untagged state blobs are reclaimed by registry GC; epoch blobs remain referenced by later manifests. Trust evidence for reclaimed steles survives in the `history` of every later inscription. - -### Restore pipeline - -1. Resolve tag → manifest → inscription; verify its digest, schema, profile name and major version, network magic and signatures. -2. Plan which layers to consume — no kind is mandatory: ledger-only nodes skip `blocks`/`indexes`/`log-{ns}`, and a layer of a kind this client does not implement is skipped too, reported alongside the epochs `sync.max_history` drops — unless its `scope` marks it `required`, which refuses the restore before a store is opened; a future Mithril-sourced mode fetches block data from an aggregator instead of `blocks` layers, verifying each immutable file against the `digests` layer before the usual decode→append import. Plan the epoch range from `sync.max_history`; diff against the progress file (`/.snapshot-restore.json`, records inscription digest + completed layer diffIds) for `--continue`. Preflight: sum the `uncompressedSize` of the planned layers and fail early if free space at `storage.path` is insufficient; derive download progress and time-remaining estimates from the compressed blob sizes of the layers that remain to be fetched — excluding layers already completed per the progress file or already present locally — so resumed and deduplicated restores report correct totals. -3. Open stores; `IndexStore::initialize_schema()`. -4. Per epoch (checkpointed): fetch + verify `blocks` and each `log-{ns}` the epoch carries → archive appends, commit; fetch `indexes` → pre-hashed appends, commit. -5. State tip: fetch every shard of every `state-{ns}` kind (parallelizable) → dispatch on the kind; `set_cursor(position.point)` last so `has_existing_data()` only ever sees complete restores; commit. -6. Rebuild live-UTxO indexes: `iter_utxos()` → `index_delta_from_utxo_delta` chunks; final chunk aligns the index cursor. -7. Delete progress file; existing `seed_wal_from_state` reseeds the WAL; the daemon chain-syncs the partial current epoch. - -Steps 1–2 and the fetch/verify half of steps 4–5 are protocol code; the store writes are profile code. - ### Development phases **1a. Stelae core** — `crates/stelae`: framing, inscription (schema, JCS, digest, history invariant), the `Profile` trait and naming rules, streaming digest/compression, signatures. Verified by CBOR-seq roundtrip and write→read→write byte-identity property tests, a JCS inscription golden test, history-invariant tests (gap/duplicate/out-of-order → reject), fail-closed tests (unknown generic key, unknown profile, higher profile major), and a toy non-Dolos profile exercising the full path. diff --git a/crates/snapshot/Cargo.toml b/crates/snapshot/Cargo.toml index aa5ab8a7e..22dea95d1 100644 --- a/crates/snapshot/Cargo.toml +++ b/crates/snapshot/Cargo.toml @@ -5,13 +5,6 @@ version.workspace = true edition.workspace = true [features] -# Publishing into an OCI registry (`src/registry.rs`). Default-off, and it -# forwards to `stelae/oci` rather than adding anything of its own: a build that -# only writes steles to a directory keeps the dependency tree it had, which is -# the whole reason the protocol's transport is behind a feature in the first -# place. `dolos`'s own `registry` feature is what turns this on. -oci = ["stelae/oci", "stelae-driver/oci"] - # The backfill daemon (`src/backfill.rs`): the publisher loop that composes # the mithril fetch with the publish lifecycle. Default-off, and the only # reason this crate ever pulls the aggregator client, pallas or a tokio @@ -25,12 +18,20 @@ backfill = [ "dep:tokio-util", ] -# This crate is the *profile* half of the Stelae boundary: it may depend on -# `stelae` and on `dolos-*`, never the other way around. See +# This crate is the *profile* half of the Stelae boundary: it depends on the +# stelae crates and on `dolos-*`, never the other way around — structural now +# that the protocol lives in its own repository. This is the workspace's one +# pin point: the CLI and root lib reach protocol and driver types through +# this crate's re-exports, and a stelae release lands here as one tag bump +# for both crates (they version in lockstep; never pin a branch). See # adrs/004_stelae_snapshots.md, "Code layout". [dependencies] -stelae = { path = "../stelae" } -stelae-driver = { path = "../stelae-driver" } +# The protocol crates keep their own default-off `oci` feature for third +# parties; this profile publishes into registries as a matter of course, so +# it turns the transport on unconditionally rather than forwarding a feature +# nothing in dolos ever left off. +stelae = { git = "https://github.com/txpipe/stelae", tag = "v0.1.0", features = ["oci"] } +stelae-driver = { git = "https://github.com/txpipe/stelae", tag = "v0.1.0", features = ["oci"] } dolos-core = { path = "../core" } dolos-cardano = { path = "../cardano" } diff --git a/crates/snapshot/PROFILE.md b/crates/snapshot/PROFILE.md new file mode 100644 index 000000000..d4eecb0e7 --- /dev/null +++ b/crates/snapshot/PROFILE.md @@ -0,0 +1,196 @@ +# The `io.txpipe.dolos.cardano` profile + +This document is the normative specification of the Dolos profile of the +[Stelae protocol](https://github.com/txpipe/stelae/blob/main/SPEC.md): the +layer kinds, record shapes, scopes and parameters a `vnd.dolos` stele +carries, and the pipelines that produce and consume one. It began as the +implementation half of `adrs/004_stelae_snapshots.md`, which remains the +decision record — the problem, the adoption of Stelae, and the alternatives — +and now specifies nothing this document covers. Section names are preserved +from the ADR, so older citations land on the same headings here. The crate +beside this file, `dolos-snapshot`, is the implementation. + +An independent party reproducing or verifying a Dolos stele reads two +documents: `SPEC.md` for the envelope — framing, inscription, manifest, +transport — and this one for every byte inside it. + +## Identity + +- Profile name `io.txpipe.dolos.cardano`, version 1; the media-type vendor + token is `dolos` (IANA `vnd.` custom). +- Payload media types: `application/vnd.dolos.stele.{blocks|indexes|log-{ns}|state-{ns}|digests}.v1+zstd`. +- One repository per network — e.g. `ghcr.io/txpipe/dolos-snapshots/mainnet`. +- Tags: immutable `epoch-E` per sequence plus a moving `latest`, where E is + the newly started epoch — equal to `sequence` and to `position.epoch`; the + layers cover epochs `0..E-1` complete plus epoch E's boundary sliver (see + "The cut point and the boundary sliver"). + +### Kinds, skips and retirement + +The protocol's coexistence rules (SPEC.md, "Profiles, naming and media +types") govern a kind the reader does not implement: rule 3 lets a restore +skip it and report the skip, unless the layer's `scope` marks it +`required`. This profile's own history puts flesh on both edges of that +rule: + +Rule 3 answers for a kind the reader does not *know*. It says nothing about a kind the publisher no longer *carries*, and `required: true` cannot be stretched to cover one: `required` is a property of a layer, and a retired kind has no layer to put it on. Absence is already meaningful in this format — a `log-{ns}` layer exists if and only if it holds a record, and a restore passes over a kind it does not recognise — so a reader that still models `log-member-rewards`, finds no such layer, and reports a clean restore has just built a node with no reward history and no way to have noticed. + +**A profile therefore declares the namespaces it defines, and a retirement is declared rather than inferred.** `parameters.schemas` carries an entry for every namespace the profile version defines; a namespace it has retired keeps its entry at revision `0`, which is not a schema revision and reads as "this version defines no records here". A restore compares that map against the namespaces it models, before a store is opened: an entry that is missing or zero for one it models refuses the restore and names the namespace. The gate is presence, with revision `0` reading as absence per the sentinel above; a *live* revision's value is never compared — a revision the reader has not seen describes bytes it can still parse, and gating on it would make every additive append breaking, which is exactly what the `.v{x}` contract below exists to avoid. + +Like `required`, the rule binds forward and not backward: it constrains readers from the version that implements it onward, and cannot reach the ones already deployed. What protects those, for the four namespaces retired so far, is that every one of them was also a *state* namespace, and the state tip's completeness check refuses a stele missing a kind it expects. A log-only namespace would have had no such backstop, and that is the case this rule exists for. Retiring a namespace is a spec-level act, for the same reason marking one `required` is. + +Rule 3's skip is available at layer granularity and at no finer one. Index **dimensions** stay fail-closed: `indexes` is a single layer per epoch, so an unknown dimension surfaces mid-stream — record by record, inside a layer the plan has already committed to restoring — where skipping it would be silent data loss rather than a visible plan-time choice, and where the store cannot look the name up in any case (it keeps a hash of the name, not the name). Changing the dimension set therefore remains a media-type-version event. The same reasoning is why a new *namespace* is additive and a new dimension is not: a namespace arrives as its own `log-{ns}` or `state-{ns}` layer, which a plan can decline; a dimension arrives inside one. + +### Layer formats + +Framing is the protocol's (SPEC.md, "Layer format"): zstd-compressed CBOR sequences under the pinned deterministic encoding profile, each opening with the protocol-defined header record: + +```text +[format_version = 1, profile: tstr, kind: tstr, scope: any] +``` + +`scope` is opaque to the protocol. The Dolos profile encodes `[network_magic, epoch, start_slot, end_slot]` for epoch layers, `[network_magic, epoch, shard]` for every state layer — one shape across all fourteen kinds, single-blob namespaces included, whose one layer is shard 0 — and `[network_magic, epoch, last_immutable]` for the digests layer. + +The state layers carry **two roles over that one header shape**, and only the *descriptor* scope tells them apart: a tip is `{"shard": n}`, a retained dump is `{"epoch": E, "shard": n}`. The header is deliberately blind to the distinction, and that is what makes the dump a publish cuts at `sequence == E` the tip's own bytes rather than a copy of them — same header, same records, one `diffId`, one blob under two descriptors. See "State history" below. + +Content records per kind (Dolos profile): + +| Kind | Record | Order | Restore write path | +|---|---|---|---| +| `blocks` (per epoch) | `[slot, hash: bytes(32), body: bytes]`, body = raw wire CBOR verbatim | ascending slot, stream order for same-slot (Byron EBB) | `ArchiveWriter::apply` | +| `indexes` (per epoch) | tags: `[0, dimension: tstr, key_hash: bytes(8), slot]` with `key_hash = xxh3_64(key)` BE — except dimension `metadata`, see below; exact: `[1, kind: tstr, key: bytes, slot]` for block-hash/block-number/tx | sorted, deduped | new `IndexWriter::append_prehashed` | +| `log-{ns}` (per epoch, per log namespace, omitted when empty) | `[log_key: bytes(40), value: bytes]`, value = stored EntityValue verbatim | `log_key` | `ArchiveWriter::write_log` into the namespace the kind names | +| `state-{ns}` (tip or retained dump, per state namespace, `scope.shard` = 0..`parameters.shards[ns]`-1) | `[key: bytes, value: bytes]` | `key`; shard = first nibble of `key[0]` for a 16-way namespace, 0 for a single blob | dispatch on the kind: `state-utxos` → chunked `StateWriter::apply_utxoset`, else `write_entity` into the namespace the kind names | +| `digests` (tip, optional) | `[immutable_number, chunk: bytes(32), primary: bytes(32), secondary: bytes(32)]`, each sha256 over the raw file bytes | ascending `immutable_number` | none — verification metadata, not written to stores | + +One exception to the tag hashing rule is normative for `indexes` v1: records in dimension `metadata` carry the logical u64 metadata label **verbatim** (big-endian) in `key_hash`, never hashed. The index stores keep metadata labels as raw labels rather than hashes, and the layer ships the stored form — that is the whole point of the pre-hashed design. `parameters.indexKeyHash` therefore describes every dimension *except* `metadata`. A publisher that hashes metadata labels produces structurally valid records that restore cleanly but can never be matched by a metadata query; conformance tooling must check this dimension specifically (#1149 tracks whether a future media-type version unifies the rule). + +State namespaces: the thirteen entity namespaces from `dolos_cardano::model::build_schema()` (key = 32-byte `EntityKey` verbatim, value = stored minicbor verbatim) plus `utxos` (key = `tx_hash(32) ‖ output_index(4, BE)`, value = CBOR `[era: uint, body: bytes]`). The chain point lives in the inscription's `position`, not in a layer. Live-UTxO index dimensions (`utxo::*`) are not shipped; they are rebuilt at restore via `index_delta_from_utxo_delta`. + +State kinds: one per state namespace, and the set is closed — 14 of them, spelled `state-` followed by the namespace with `_` rewritten to `-`, by the same rule and for the same reasons as the log kinds below. The namespace is therefore **not** in the record — it is the layer — which is what puts the fail-closed edge of a breaking change on exactly the namespace that broke, and lets a reader skip a namespace this profile does not define at the transport rather than choking on one shared layer. The shard count is **specification, never tuning**: `utxos`, `accounts`, `assets` and `datums` split 16 ways, every other namespace is a single blob, and `parameters.shards` reports the map so a reader never has to discover it from the data. Re-sharding a namespace is a media-type-version event for that namespace's kind. Every shard of every kind is published, empty ones included, so tip completeness is structural: a restore requires all 14 kinds and, per kind, exactly the shards its count promises. + +**State history: retained dumps at configured epochs, plus the moving tip.** A stele's state is the tip — the ledger as of `sequence`, swapped whole by every publish — and, for each epoch a publisher retains, an immutable **dump** of the state as of that epoch. The two are the same kinds, the same records and the same shard geometry; a dump differs from a tip in its descriptor scope, which names the epoch, and in nothing else. + +- **The retained set is configuration, not derivation.** `snapshot.state_epochs` names it. Era boundaries are one sensible criterion and cherry-picked epochs another; which epochs are worth a dump is operational, so nothing derives the list from the chain summary. The list is strictly ascending and never names epoch 0, and it is refused where it is read rather than where a dump is cut — it reaches `parameters` before any layer is written. Publishers are expected to keep it around 10–20: per-epoch dumps were rejected outright (~46k manifest descriptors on mainnet against a ~12k ceiling). +- **The list is signed input.** It is echoed verbatim into `parameters.stateEpochs`, so two publishers of one network configured differently produce different parameters, different inscription digests, and a divergence an operator reads out of a one-field diff instead of hunting through layers for. It is therefore **normative per network** and belongs pinned wherever the default repository is pinned: a publisher quietly running a different list self-ejects from co-signing. +- **Production rule.** At the publish where `sequence` equals a retained epoch E, E's dump is cut from the tip: one walk of the store, one sink per shard, one blob, and the transport attests the result a second time under the dump's scope. Nothing is compressed twice and nothing crosses the wire twice. At a publish standing past E, the dump is adopted from the predecessor by the same scope equality every immutable layer uses. A dump for a past epoch that no predecessor carries is a **warning and a shorter stele**, never a failed publish: this publish's stores hold the tip, and the state as of an epoch it has moved past is not in them to be written. Producing one is a backfill run's job. +- **Inheritance.** The rule "no state layer is ever inherited" was about the tip role, in two independent ways, and both still hold of it: the tip changes every publish, and its scope names no epoch, so scope equality could not tell one publish's shard from another's. A dump's scope does name its epoch, so it inherits, checkpoints and resumes exactly as a `blocks` layer does — including through the resumption record an interrupted publish leaves. +- **Adoption is per shard.** A predecessor that can hand over fifteen of a dump's sixteen shards hands over fifteen. Adopting is an act — it puts the blob in the transport — so all-or-nothing would mean either leaving blobs the inscription does not describe (a refused seal) or refusing a publish over history that was never load-bearing. +- **Restore reports dumps and does not consume them.** A restore builds a node standing at `sequence`, so the tip is what it reads; dumps are recorded in the plan, excluded from the byte accounting and from the disk preflight, and reported. Tip completeness is checked on the tip alone. Bootstrapping *at* a retained epoch — consuming a dump as the tip — is deliberately out of scope here and nothing forecloses it: the restore write path takes a kind and a descriptor, and a dump's are the tip's. + +Log kinds: one per namespace the ledger writes epoch-boundary logs under, and the set is closed — `log-account-epochs`, `log-epochs`, `log-stakes`. `account-epochs` is one `(account, epoch)` record holding everything an account did in an epoch; it replaced `account-stakes`, `leader-rewards`, `member-rewards` and `pool-deposit-refunds`, which were four identically-keyed namespaces and are now the profile's four retired ones (ADR-0027). The kind token is `log-` followed by the namespace with `_` rewritten to `-`, since a media type's kind token admits hyphens and not underscores; the mapping is injective, and a publisher spells the kinds out rather than composing them, so a namespace rename cannot silently rename a published kind. The namespace is therefore **not** in the record — it is the layer — and a restore writes a layer's records into the namespace its kind names. A publisher that finds logs under a namespace no kind covers **fails the publish** naming the namespace: the format has no layer to carry them, and shipping a snapshot that silently omits a slice of the ledger is the failure that costs most and shows least. Log layers wear the epoch scope unchanged, so per-(kind, scope) inheritance works across the split with no new scope shape. + +Empty log layers are omitted: **a `log-{ns}` layer exists if and only if it holds at least one record.** The rule is content-determined and not writer discretion — that is what keeps it deterministic across publishers, since two honest publishers agree about whether a layer exists exactly because they agree about whether a record does. Byron alone sheds ~1,200 empty blobs to it. Absence is normative and never a defect: a restore that finds no `log-stakes` for an epoch restores that epoch with no stake logs, exactly as the ledger had none. Every other kind keeps the arity it had — `blocks` and `indexes` are one layer per epoch whatever the window holds, and every state layer is always present, empty or not. + +#### Compatibility contract + +Within a media-type version, `.v{x}` is a **contract on record content, not an exact-byte pin** (decision 0026). Writers MAY append numbered optional or `#[cbor(default)]` fields; readers MUST skip fields they do not know; field indexes are never renumbered, removed or repurposed. Every append moves that namespace's `schemas` revision — additive included, because `schemas` is what keeps the inscription a full pin of the bytes rather than a compatibility gate. `v{x}` bumps are reserved for breaking changes. + +Enums are the contract's hard edge and the one place the rule inverts. A minicbor enum refuses a variant index it has never heard of, so **adding a variant to any enum reachable from a record is reader-breaking within `v{x}`**, whatever the field policy says: it requires a media-type version bump on the kinds that carry it, or an explicit waiver recorded in this document. + +The contract is enforced by `crates/snapshot/tests/field_registry.rs` and its `tests/registry/` data module, which is where a record shape is actually pinned. Per namespace it holds a **canary** — a fully-populated value, every `Option` `Some` and every collection non-empty — and the hex of its encoding through the production encode path, at that namespace's current `schemas` revision. The suite then asserts, on every build: + +- the canary still encodes to its pinned bytes, so a renumbering, a removal or a silent width change is a build failure rather than a moved digest with no explanation; +- the registry's current revision equals `SCHEMA_REVS[ns]`, so a field appended without its revision bump — or a bump with nothing pinning it — fails in either direction; +- every **retained** revision still decodes under today's decoder, which is the reader tolerance the contract promises, asserted rather than assumed. Retained canaries are append-only: never edited, never deleted. A change that makes one undecodable is breaking by definition. +- every reachable enum's variant table is pinned per variant, including the Pallas enums the records embed — a dependency upgrade that renumbered `Relay` or `DRep` would otherwise change published bytes with nothing in this repository to notice; +- the codec's own tolerance behaviour (unknown trailing field skipped, index gap null-padded, missing trailing field defaulted) holds against the pinned `minicbor` version, so a codec upgrade that changed any of the three breaks in a test rather than in a published stele; +- the same value encodes to the same bytes across repeated construction. A failure there is never a re-pin: it is an encoding-determinism defect, and it breaks the cross-party digest identity this whole document rests on. + +The `digests` layer covers the immutable files fully contained in the stele's block range: `lastImmutable` is derived from the boundary slot and the chunk geometry observed in the chain — canonical, never dependent on aggregator state at publish time. Digest values equal Mithril Cardano DB v2's merkle leaves (hex-decoded), so any Mithril certificate whose beacon covers `lastImmutable` can verify them via the aggregator's digest route and a merkle proof. The certificate reference is deliberately *not* part of the inscription: certificates are produced on the aggregator's cadence, so two independent publishers at the same boundary would reference different certificates — including one would break cross-publisher determinism, while the digest values themselves are byte-stable properties of the chain. + +### The cut point and the boundary sliver + +A stele is cut by syncing with `chain.stop_epoch = E`, and the halt is **one block past the epoch boundary**, not on it. The sync crosses the boundary — Ewrap closes epoch E-1, Estart opens epoch E — and then applies the first block of epoch E before stopping. That block is what makes the stele addressable: Estart alone leaves the cursor a bare `ChainPoint::Slot` carrying no block hash (`estart::commit_finalize`), and `position.point` must carry one for a stele to be verifiable against a chain. The anchoring block may itself sit exactly on `epoch_start(E)`, in which case the sliver is one slot wide. + +So the epoch windows a stele covers are `0..=E`: epochs `0..E-1` **complete**, plus epoch E's **boundary sliver** — the window that opens at `epoch_start(E)` and closes at the anchoring block. The sliver is normative, not an artifact of where a publisher happened to stop. + +It is load-bearing because of how boundary data is keyed. **All of epoch X's boundary logs key at `epoch_start(X)`**: Ewrap writes the *ending* epoch's closing logs and its completed `EpochState` at `epoch_start(X)` when X is the epoch it closes, and Estart writes the *starting* epoch's opening account logs at `epoch_start(X)` when X is the epoch it opens. One temporal key per epoch, and nothing straddles two windows. Epoch E's estart logs therefore live inside the sliver and nowhere else: a stele that dropped the sliver in the name of a clean boundary would ship a state tip whose opening logs are in no layer at all, and a restored node would never regenerate them. + +`sequence`, the immutable tag's E, and `position.epoch` are consequently **one number**: the epoch the cursor stands in, which is also the last epoch the layers cover. The operator-facing form of the same rule is *configure the epoch you want the node to start in* — `stop_epoch = E` produces the stele tagged `epoch-E`. + +### Position, parameters and the inscription + +The inscription's generic shape, canonicalization and history invariant are +the protocol's (SPEC.md, "The inscription"). What this profile owns are the +three opaque objects — `position`, `parameters` and each layer's `scope` — +and the meaning of `sequence`: + +```json +{ "schema": 1, + "profile": {"name": "io.txpipe.dolos.cardano", "version": 1}, + "sequence": 550, + "position": { "network": {"magic": 764824073, "name": "mainnet"}, + "point": {"slot": 152236812, "hash": "…"}, + "epoch": 550 }, + "parameters": { "indexKeyHash": "xxh3-64", + "shards": {"accounts": 16, "assets": 16, "datums": 16, "utxos": 16, "…": 1}, + "schemas": {"accounts": 1, "utxos": 1, "…": 1}, + "stateEpochs": [208, 236, 290, 365] }, + "compression": {"algo": "zstd", "level": 9}, + "history": [ + {"sequence": 548, "inscriptionDigest": "sha256:…"}, + {"sequence": 549, "inscriptionDigest": "sha256:…"} ], + "layers": [ + {"kind": "blocks", "mediaType": "application/vnd.dolos.stele.blocks.v1+zstd", + "diffId": "sha256:…", "records": 21600, "uncompressedSize": 43210000, + "scope": {"epoch": 0, "startSlot": 0, "endSlot": 21599}}, + {"kind": "state-utxos", "mediaType": "application/vnd.dolos.stele.state-utxos.v1+zstd", + "diffId": "sha256:…", "records": 812345, "uncompressedSize": 402653184, + "scope": {"shard": 0}}, + {"kind": "state-pools", "mediaType": "application/vnd.dolos.stele.state-pools.v1+zstd", + "diffId": "sha256:…", "records": 3210, "uncompressedSize": 1048576, + "scope": {"shard": 0}}, + {"kind": "state-pools", "mediaType": "application/vnd.dolos.stele.state-pools.v1+zstd", + "diffId": "sha256:…", "records": 2980, "uncompressedSize": 972800, + "scope": {"epoch": 365, "shard": 0}}, + {"kind": "digests", "mediaType": "application/vnd.dolos.stele.digests.v1+zstd", + "diffId": "sha256:…", "records": 6188, "uncompressedSize": 618800, + "scope": {"lastImmutable": 6187}} ] } +``` + +`parameters` is the profile's compatibility declaration. Three of its four values are a consequence of publisher code rather than a free choice: `indexKeyHash` names the hash behind the pre-hashed index keys; `shards` is the per-namespace shard map above; and `schemas` is a per-namespace revision of the *record content* — the stored minicbor a `state-{ns}` or `log-{ns}` layer carries verbatim — which moves when that namespace's stored shape changes, plus one entry at revision `0` per retired namespace, per the removed-kind rule above. Thirteen of the fourteen live revisions are 1; `epochs` is at 2, the first bump the format has taken (ADR-004, Limitations), and the four retired namespaces sit alongside them at 0. Every live revision is pinned by a canary in `crates/snapshot/tests/field_registry.rs`, which fails the build when a record's field table moves without its revision, or the other way round. The split between the two is deliberate: a change to how a layer is *framed* moves that kind's media type and fails closed at the transport, while a change to what a record *contains* moves its schema revision, which a reader consults to decide whether it can interpret what it can already parse. The fourth, `stateEpochs`, is the exception that proves the rule: it is the publisher's configured retained set, and it is here precisely *because* it is a choice — declaring it is what turns a configuration difference between two publishers into a visible parameters difference instead of a silently divergent history. + +`sequence` is the protocol's ordering key; this profile sets it to the epoch. The three opaque objects are canonicalized by JCS like every generic key (SPEC.md), so determinism holds without the protocol interpreting them — which is why every value in them must itself be deterministic, the property the compatibility contract above enforces. + +### The manifest size arithmetic + +The 4 MiB ceiling and its measurement are the protocol's (SPEC.md, "The +manifest size ceiling"); what is profile-owned is the arithmetic that keeps a Dolos stele inside the +ceiling, and it stays here: a descriptor with its annotations costs ~350 +bytes, so the ceiling falls near 12,000 layers. A mainnet stele is bounded +above by ~600 epochs × 5 per-epoch kinds (`blocks`, `indexes` and the three +`log-{ns}`), plus the state tip's 74 layers (4 namespaces × 16 shards + 10 +single blobs), plus 74 more for every retained state dump — at 20 retained +epochs, the ceiling of what a publisher is expected to configure, that is +~4,554 layers and a manifest of roughly 1.6 MB, comfortably inside. The +bound is loose in the direction that helps: the log kinds are omitted when +empty, and Byron's ~200 epochs carry no reward or stake logs at all, so the +realized count sits near ~4,150. **This is the arithmetic that bounds the +retained list**, and the reason per-epoch dumps were rejected: ~580 of them +would be ~43,000 state layers on their own, more than three times the +ceiling. (The Rationale's "~1,700 manifest descriptors" is decision-time +sizing of the pre-split artifact; this paragraph is the authoritative count, +and it counts layers rather than epochs.) + +### Publisher pipeline + +1. Restore the publisher node from the previous stele (self-hosting delta pull; first run via Mithril). +2. Sync with `chain.stop_epoch = E` until `StopEpochReached` — the state crosses the boundary and lands on the **first block of epoch E**, the block that gives `position.point` a hash. +3. `dolos snapshot publish` — only the newly closed epoch's layers and epoch E's boundary sliver upload; fresh state layers + inscription; tag `epoch-E`, move `latest`. On networks with a Mithril aggregator, fetch the immutable-file digest list from the aggregator's digest route, verify it against a certificate, and write the `digests` layer for the files within the boundary. +4. Determinism job: an independent runner that synced by any means runs `dolos snapshot digest` and alerts on inscription mismatch. +5. Matching verifiers sign and push referrer signatures; clients enforce k-of-n. + +Registry hygiene: keep a trailing window of `epoch-E` tags (e.g. 12); untagged state blobs are reclaimed by registry GC; epoch blobs remain referenced by later manifests. Trust evidence for reclaimed steles survives in the `history` of every later inscription. + +### Restore pipeline + +1. Resolve tag → manifest → inscription; verify its digest, schema, profile name and major version, network magic and signatures. +2. Plan which layers to consume — no kind is mandatory: ledger-only nodes skip `blocks`/`indexes`/`log-{ns}`, and a layer of a kind this client does not implement is skipped too, reported alongside the epochs `sync.max_history` drops — unless its `scope` marks it `required`, which refuses the restore before a store is opened; a future Mithril-sourced mode fetches block data from an aggregator instead of `blocks` layers, verifying each immutable file against the `digests` layer before the usual decode→append import. Plan the epoch range from `sync.max_history`; diff against the progress file (`/.snapshot-restore.json`, records inscription digest + completed layer diffIds) for `--continue`. Preflight: sum the `uncompressedSize` of the planned layers and fail early if free space at `storage.path` is insufficient; derive download progress and time-remaining estimates from the compressed blob sizes of the layers that remain to be fetched — excluding layers already completed per the progress file or already present locally — so resumed and deduplicated restores report correct totals. +3. Open stores; `IndexStore::initialize_schema()`. +4. Per epoch (checkpointed): fetch + verify `blocks` and each `log-{ns}` the epoch carries → archive appends, commit; fetch `indexes` → pre-hashed appends, commit. +5. State tip: fetch every shard of every `state-{ns}` kind (parallelizable) → dispatch on the kind; `set_cursor(position.point)` last so `has_existing_data()` only ever sees complete restores; commit. +6. Rebuild live-UTxO indexes: `iter_utxos()` → `index_delta_from_utxo_delta` chunks; final chunk aligns the index cursor. +7. Delete progress file; existing `seed_wal_from_state` reseeds the WAL; the daemon chain-syncs the partial current epoch. + +Steps 1–2 and the fetch/verify half of steps 4–5 are protocol code; the store writes are profile code. diff --git a/crates/snapshot/src/layers/state.rs b/crates/snapshot/src/layers/state.rs index cf9666f60..e186ac0af 100644 --- a/crates/snapshot/src/layers/state.rs +++ b/crates/snapshot/src/layers/state.rs @@ -15,11 +15,11 @@ //! //! ## One record shape, including for UTxOs //! -//! ADR-004 treats the UTxO set as namespace [`crate::UTXOS`] beside the sixteen -//! entity namespaces, rather than as a special layer kind. That is what keeps -//! the format's state vocabulary to a single record, and it makes the planned -//! refactor folding UTxOs into the entity system (#1042) invisible from -//! outside: the day `utxos` becomes an ordinary namespace, nothing in this +//! ADR-004 treats the UTxO set as namespace [`crate::UTXOS`] beside the +//! thirteen entity namespaces, rather than as a special layer kind. That is +//! what keeps the format's state vocabulary to a single record, and it makes +//! the planned refactor folding UTxOs into the entity system (#1042) invisible +//! from outside: the day `utxos` becomes an ordinary namespace, nothing in this //! file changes. //! //! The namespace still governs the *codec parameters* — the key width above diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index 4e57364b2..edbf6e73c 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -8,8 +8,8 @@ //! the tag a sequence renders as, what goes in `position`, `parameters` and //! each layer's `scope`, and the byte-exact codec for every record shape. //! -//! The normative specification is `adrs/004_stelae_snapshots.md`; this crate is -//! that document's record table (§"Layer formats") made executable. +//! The normative specification is `PROFILE.md` beside this crate; this crate +//! is that document's record table (§"Layer formats") made executable. //! //! ## What is here, and what is deliberately not //! @@ -70,9 +70,7 @@ pub mod layers; pub mod namespaces; pub mod node; pub mod planning; -#[cfg(feature = "oci")] pub mod publisher; -#[cfg(feature = "oci")] pub mod registry; pub mod restore; diff --git a/crates/snapshot/src/node.rs b/crates/snapshot/src/node.rs index c6d8e876b..c1fc1dc2a 100644 --- a/crates/snapshot/src/node.rs +++ b/crates/snapshot/src/node.rs @@ -28,7 +28,6 @@ pub fn scratch_dir(config: &StorageConfig, chosen: Option<&Path>) -> PathBuf { } } -#[cfg(feature = "oci")] mod auth { use dolos_core::config::StelaeConfig; @@ -97,7 +96,6 @@ mod auth { } } -#[cfg(feature = "oci")] pub use auth::{registry_auth, OFFICIAL_REGISTRY_PASSWORD}; #[cfg(test)] @@ -131,7 +129,6 @@ mod tests { } } - #[cfg(feature = "oci")] mod auth { use dolos_core::config::{StelaeConfig, StelaeRegistryConfig}; diff --git a/crates/snapshot/src/restore.rs b/crates/snapshot/src/restore.rs index f7829f12a..b5ed7d70e 100644 --- a/crates/snapshot/src/restore.rs +++ b/crates/snapshot/src/restore.rs @@ -6,7 +6,7 @@ //! //! ## The order is the specification //! -//! ADR-004 §"Restore pipeline" fixes the sequence, and it is not an +//! PROFILE.md §"Restore pipeline" fixes the sequence, and it is not an //! implementation preference — each step exists because of what the one before //! it established: //! @@ -45,8 +45,8 @@ //! the tip is never checkpointed — so the partial-`utxo::*` node is repairable //! by an operator who resumes, where before it could only be thrown away. What //! it does not answer is whether `set_cursor` should move *after* step 6. That -//! is ADR-004's ordering, it is an open question with its owner, and nothing -//! here reorders the pipeline to pre-empt it. +//! is the profile spec's ordering, it is an open question with its owner, and +//! nothing here reorders the pipeline to pre-empt it. //! //! ## Resume, and where the checkpoint goes //! @@ -768,7 +768,7 @@ fn scope_uint(descriptor: &LayerDescriptor, field: &str) -> Result { /// Name of the progress file inside a node's storage directory. /// -/// ADR-004's, spelled exactly as it spells it. "Snapshot" is this profile's +/// PROFILE.md's, spelled exactly as it spells it. "Snapshot" is this profile's /// word for a stele — Dolos says `dolos snapshot`, the protocol says *stele* — /// which is why the name lives here and not in `stelae`, whose /// [`stelae::plan::RestoreProgress`] takes a path a caller chose. @@ -1405,7 +1405,6 @@ pub const UNRESTORED_KINDS: [&str; 1] = [DIGESTS]; /// that decide what to do with existing data are handled a layer above, and a /// source rejected any later would have cost the operator the node they still /// had. -#[cfg(feature = "oci")] #[derive(Debug, Clone)] pub enum Source { /// A stele directory on this filesystem. @@ -1414,7 +1413,6 @@ pub enum Source { Repo(crate::registry::Repository), } -#[cfg(feature = "oci")] impl std::str::FromStr for Source { type Err = String; @@ -1446,7 +1444,7 @@ impl std::str::FromStr for Source { } } -#[cfg(all(test, feature = "oci"))] +#[cfg(test)] mod source_tests { use std::path::PathBuf; diff --git a/crates/snapshot/tests/common/mod.rs b/crates/snapshot/tests/common/mod.rs index 3c8b50724..a198c4c09 100644 --- a/crates/snapshot/tests/common/mod.rs +++ b/crates/snapshot/tests/common/mod.rs @@ -350,8 +350,8 @@ pub fn all_layers() -> Vec<(&'static str, Box, Vec)> { /// Read a layer both ways and insist the two paths agree. /// -/// The same discipline `crates/stelae/tests/toy_profile.rs` applies to the toy -/// profile: a layer that reads back through one path and not the other is a +/// The same discipline the stelae repo's `tests/toy_profile.rs` applies to the +/// toy profile: a layer that reads back through one path and not the other is a /// determinism bug in the format, and this profile is the first real one to put /// that to the test. pub fn read_both_ways( diff --git a/crates/snapshot/tests/goldens.rs b/crates/snapshot/tests/goldens.rs index 36594c63c..493595c0b 100644 --- a/crates/snapshot/tests/goldens.rs +++ b/crates/snapshot/tests/goldens.rs @@ -395,7 +395,7 @@ fn a_complete_stele_reads_back_and_reproduces_its_digest() { read.check_profile(&DolosProfile).unwrap(); // Every layer streams back under the *default* record ceiling — the - // confirmation `crates/stelae`'s streaming reader was left waiting for from + // confirmation the stelae crate's streaming reader was left waiting for from // its first real profile. let index = stele.blob_index().unwrap(); assert_eq!(index.len(), GOLDEN_LAYERS.len()); diff --git a/crates/snapshot/tests/node/mod.rs b/crates/snapshot/tests/node/mod.rs index c92db72db..d736da03a 100644 --- a/crates/snapshot/tests/node/mod.rs +++ b/crates/snapshot/tests/node/mod.rs @@ -108,7 +108,6 @@ impl Blank { // The re-export mirrors the module's own rule stated above: every test binary // compiles this file in full, so the suites that never open a registry see an // import they do not use. -#[cfg(feature = "oci")] #[allow(unused_imports)] pub use registry_node::Node; @@ -119,7 +118,6 @@ pub use registry_node::Node; /// stele the one suite publishes is what the other verifies, and a second /// copy of the fixture would be a second answer to "where do the two plans /// stand". -#[cfg(feature = "oci")] mod registry_node { use dolos_core::{BlockHash, ChainPoint, Domain as _}; use dolos_snapshot::{ diff --git a/crates/snapshot/tests/publish.rs b/crates/snapshot/tests/publish.rs index 67300db23..666029308 100644 --- a/crates/snapshot/tests/publish.rs +++ b/crates/snapshot/tests/publish.rs @@ -6,12 +6,12 @@ //! nothing — run it with: //! //! ```text -//! cargo test -p dolos-snapshot --features oci --test publish -- --ignored --nocapture +//! cargo test -p dolos-snapshot --test publish -- --ignored --nocapture //! ``` //! //! `STELAE_TEST_REGISTRY_IMAGE` chooses the server (default `registry:2`), the -//! same knob `crates/stelae/tests/oci.rs` uses, so this suite can be pointed at -//! another implementation. +//! same knob the stelae repo's `tests/oci.rs` uses, so this suite can be +//! pointed at another implementation. //! //! ## The properties //! @@ -55,8 +55,6 @@ //! rather than inherited. Reuse across a sequence is a property of a publisher //! that stops on epoch boundaries, which is what ADR-004's pipeline does. -#![cfg(feature = "oci")] - mod node; mod registry_fixture; mod watcher; diff --git a/crates/snapshot/tests/registry_fixture/mod.rs b/crates/snapshot/tests/registry_fixture/mod.rs index 5bf726a62..228647ae2 100644 --- a/crates/snapshot/tests/registry_fixture/mod.rs +++ b/crates/snapshot/tests/registry_fixture/mod.rs @@ -4,9 +4,9 @@ //! published by one is what the other reads and a second copy would be a second //! answer to "what is a registry, for a test". //! -//! **Still a second copy of `Fixture` in `crates/stelae/tests/oci.rs`**, and -//! deliberately. `stelae` must never depend on a `dolos-*` package — that is -//! the boundary ADR-004 sets and `cargo tree` checks — so a fixture *that* +//! **Still a second copy of `Fixture` in the stelae repo's `tests/oci.rs`**, +//! and deliberately. `stelae` must never depend on a `dolos-*` package — that +//! is the boundary ADR-004 sets and `cargo tree` checks — so a fixture *that* //! suite could import too would have to live somewhere neither crate owns. Two //! copies across the boundary is the cheaper of the two prices; two copies //! inside one crate is not, which is why this file exists. Keep the two in diff --git a/crates/snapshot/tests/restore_registry.rs b/crates/snapshot/tests/restore_registry.rs index e530a1cc9..53d1c929c 100644 --- a/crates/snapshot/tests/restore_registry.rs +++ b/crates/snapshot/tests/restore_registry.rs @@ -6,11 +6,11 @@ //! nothing — run it with: //! //! ```text -//! cargo test -p dolos-snapshot --features oci --test restore_registry -- --ignored --nocapture +//! cargo test -p dolos-snapshot --test restore_registry -- --ignored --nocapture //! ``` //! //! `STELAE_TEST_REGISTRY_IMAGE` chooses the server (default `registry:2`), the -//! same knob `tests/publish.rs` and `crates/stelae/tests/oci.rs` use. +//! same knob `tests/publish.rs` and the stelae repo's `tests/oci.rs` use. //! //! ## The four properties //! @@ -41,8 +41,6 @@ //! after a wall-clock delay would sometimes interrupt nothing over a loopback //! registry — the fixture stele is kilobytes — and pass for the wrong reason. -#![cfg(feature = "oci")] - mod node; mod registry_fixture; mod watcher; diff --git a/crates/snapshot/tests/snapshot_verify.rs b/crates/snapshot/tests/snapshot_verify.rs index 3bf2a61e8..921bd4a87 100644 --- a/crates/snapshot/tests/snapshot_verify.rs +++ b/crates/snapshot/tests/snapshot_verify.rs @@ -6,7 +6,7 @@ //! nothing — run it with: //! //! ```text -//! cargo test -p dolos-snapshot --features oci --test snapshot_verify -- --ignored --nocapture +//! cargo test -p dolos-snapshot --test snapshot_verify -- --ignored --nocapture //! ``` //! //! ## What this suite proves @@ -32,8 +32,6 @@ //! therefore a manifest (or config blob) rewritten under `latest` — the shape //! of attack a verifier actually faces. -#![cfg(feature = "oci")] - mod node; mod registry_fixture; diff --git a/crates/stelae-driver/Cargo.toml b/crates/stelae-driver/Cargo.toml deleted file mode 100644 index b83cffa02..000000000 --- a/crates/stelae-driver/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "stelae-driver" -description = "Profile-generic driver machinery for the Stelae snapshot protocol" -version.workspace = true -edition.workspace = true - -[features] -# Forwarded to `stelae/oci` so a driver built for a registry-capable binary and -# the protocol crate under it agree on one feature, rather than each consumer -# having to name both. Nothing here is `#[cfg]`-gated on it yet. -oci = ["stelae/oci"] - -# The other half of the Stelae boundary from `stelae` itself: this crate holds -# the lifecycle machinery a publisher or a restorer needs whatever it is -# publishing, and like the protocol crate it must never depend on a `dolos-*` -# package. `cargo tree -p stelae-driver -e normal --all-features` shows the -# boundary holding. See adrs/004_stelae_snapshots.md, "Code layout". -[dependencies] -stelae = { path = "../stelae" } - -fs4.workspace = true -minicbor.workspace = true -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tracing.workspace = true - -[dev-dependencies] -tempfile = "3.20.0" diff --git a/crates/stelae-driver/src/digests.rs b/crates/stelae-driver/src/digests.rs deleted file mode 100644 index 55a400b43..000000000 --- a/crates/stelae-driver/src/digests.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! The `digests` layer: sha256 of every Cardano immutable-DB file the stele -//! covers. -//! -//! `[immutable_number, chunk: bytes(32), primary: bytes(32), secondary: -//! bytes(32)]`, ascending `immutable_number`. -//! -//! ## Carries no restorable data -//! -//! Nothing here is written to a store. The layer exists so that block data -//! obtained from *somewhere else* — a Mithril aggregator, a mirror, a relay -//! replay — can be checked against the stele's signed inscription before it is -//! imported. It is also the enabler for a future Mithril-sourced restore mode, -//! which is why the digests are exactly Mithril Cardano DB v2's merkle leaves: -//! sha256 over the raw `.chunk`/`.primary`/`.secondary` file bytes, so a -//! certificate whose beacon covers `lastImmutable` verifies them by merkle -//! proof. -//! -//! The certificate itself is deliberately *not* part of a stele. Certificates -//! are produced on the aggregator's cadence, so two publishers at the same -//! boundary would reference different ones and stop reproducing each other's -//! inscription; the digest values, by contrast, are byte-stable properties of -//! the chain. -//! -//! The layer is optional — a network without a Mithril aggregator simply has no -//! `digests` layer, and the inscription says so. - -use stelae::{ - codec::{close, fixed, open, uint}, - frame::{self, CanonicalCbor}, - Digest, -}; - -use crate::Error; - -/// The layer kind these records travel in. -/// -/// Spelled here rather than in the profile that lists it, so the codec and the -/// name its refusals quote cannot drift apart; a profile that carries the layer -/// names this constant in its own kind table. -pub const DIGESTS: &str = "digests"; - -/// The three files of one immutable chunk, by content. -/// -/// Uses [`stelae::Digest`] rather than a fourth thirty-two-byte newtype: these -/// are sha256 over bytes, which is what that type is, and it already prints and -/// parses in the `sha256:…` form the rest of a stele uses. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ImmutableDigests { - pub immutable_number: u64, - pub chunk: Digest, - pub primary: Digest, - pub secondary: Digest, -} - -pub fn encode(record: &ImmutableDigests) -> Result { - Ok(frame::encode(|e| { - e.array(4)? - .u64(record.immutable_number)? - .bytes(record.chunk.as_bytes())? - .bytes(record.primary.as_bytes())? - .bytes(record.secondary.as_bytes())?; - Ok(()) - })?) -} - -pub fn decode(bytes: &[u8]) -> Result { - let mut decoder = minicbor::Decoder::new(bytes); - - open(DIGESTS, &mut decoder, 4)?; - - let immutable_number = uint(DIGESTS, "immutable_number", &mut decoder)?; - let chunk = fixed::<32>(DIGESTS, "chunk", &mut decoder)?; - let primary = fixed::<32>(DIGESTS, "primary", &mut decoder)?; - let secondary = fixed::<32>(DIGESTS, "secondary", &mut decoder)?; - - close(DIGESTS, &decoder, bytes)?; - - Ok(ImmutableDigests { - immutable_number, - chunk: Digest::from_bytes(chunk), - primary: Digest::from_bytes(primary), - secondary: Digest::from_bytes(secondary), - }) -} - -/// Strictly ascending `immutable_number` — one record per immutable file set. -#[derive(Debug, Default, Clone, Copy)] -pub struct OrderCheck { - last: Option, -} - -impl OrderCheck { - pub fn check(&mut self, record: &ImmutableDigests) -> Result<(), Error> { - if let Some(last) = self.last { - if record.immutable_number <= last { - return Err(Error::out_of_order( - DIGESTS, - format!( - "immutable {} follows immutable {last}", - record.immutable_number - ), - )); - } - } - - self.last = Some(record.immutable_number); - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(number: u64) -> ImmutableDigests { - ImmutableDigests { - immutable_number: number, - chunk: Digest::from_bytes([0x11; 32]), - primary: Digest::from_bytes([0x22; 32]), - secondary: Digest::from_bytes([0x33; 32]), - } - } - - #[test] - fn round_trips() { - for number in [0u64, 6187, u32::MAX as u64 + 1] { - let original = record(number); - let encoded = encode(&original).unwrap(); - - assert_eq!(decode(encoded.as_bytes()).unwrap(), original); - } - } - - #[test] - fn a_wrong_width_digest_is_refused() { - let wire = frame::encode(|e| { - e.array(4)? - .u64(1)? - .bytes(&[0u8; 32])? - .bytes(&[0u8; 20])? - .bytes(&[0u8; 32])?; - Ok(()) - }) - .unwrap(); - - let err = decode(wire.as_bytes()).unwrap_err(); - assert!(matches!(err, Error::MalformedRecord { .. }), "{err:?}"); - } - - #[test] - fn ordering_is_strictly_ascending() { - let mut order = OrderCheck::default(); - - order.check(&record(0)).unwrap(); - order.check(&record(1)).unwrap(); - - for backwards in [record(1), record(0)] { - let err = OrderCheck { last: Some(1) }.check(&backwards).unwrap_err(); - assert!(matches!(err, Error::OutOfOrder { .. }), "{err:?}"); - } - } -} diff --git a/crates/stelae-driver/src/lib.rs b/crates/stelae-driver/src/lib.rs deleted file mode 100644 index 69ab074e3..000000000 --- a/crates/stelae-driver/src/lib.rs +++ /dev/null @@ -1,274 +0,0 @@ -//! # Driver machinery for the Stelae protocol -//! -//! [`stelae`] is the protocol: framing, canonicalization, digests and the -//! naming rules. A *profile* — `dolos-snapshot` is this workspace's one — says -//! what a stele contains. Between them sits the work every publisher and every -//! restorer does whatever it is moving: sizing a volume before a run starts, -//! counting layers and records for whoever is watching, reading where a node -//! stands against a repository's newest stele, keying a layer by its kind and -//! scope. -//! -//! None of that is protocol and none of it is profile, so it lives here rather -//! than in either. No type here is a node's or a profile's — the boundary this -//! crate keeps is the same one `stelae` keeps, and the manifest states it. -//! -//! ## Module map -//! -//! - [`preflight`] — the free-space policy a publish and a restore share. -//! - [`reporting`] — the layer and record arithmetic behind -//! [`stelae::progress`]. -//! - [`digests`] — the codec for the `digests` layer kind. Its records are -//! sha256 over immutable-database files, which is a Cardano shape described -//! in Cardano words; the code depends on nothing but this crate and the -//! protocol, which is why it sits here. -//! - [`profile`] — [`DriverProfile`], the little a lifecycle has to ask a -//! profile that the protocol's own trait deliberately does not answer. -//! - [`predecessor`] — the publish a publish follows, and what it may carry -//! forward from it. -//! - [`publish`] — the chained-publish lifecycle against a repository, behind -//! the `oci` feature because that is where a repository lives. -//! - [`retry`] — the bounded patience a run spends on an external that fails in -//! bursts. -//! - [`Standing`] — where a node stands against a repository's latest stele. -//! - [`scope_key`] — the pair that identifies one layer. - -pub mod digests; -pub mod predecessor; -pub mod preflight; -pub mod profile; -#[cfg(feature = "oci")] -pub mod publish; -pub mod reporting; -pub mod restore; -pub mod retry; - -pub use predecessor::{First, Predecessor}; -pub use profile::DriverProfile; - -/// Errors raised by the driver. -/// -/// No variant carries a profile's or a node's types: a driver failure is about -/// a volume, a record's shape or a chain of sequences, and a profile wraps this -/// enum in its own rather than the other way round. -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("stelae error: {0}")] - Stelae(stelae::Error), - - /// A volume that cannot hold what the run is about to put on it, refused - /// before the run starts. - /// - /// Raised only from a number that was actually measured against free space - /// that was actually read — everything else warns and proceeds. One - /// variant for both directions because it is one policy; see - /// [`preflight`]. There is deliberately no flag that overrides it: a - /// scratch directory pointed at a bigger volume is the escape hatch. - #[error("not enough space: {0}")] - NotEnoughSpace(String), - - #[error("malformed {kind} record: {reason}")] - MalformedRecord { kind: &'static str, reason: String }, - - #[error("{kind} records are out of order: {reason}")] - OutOfOrder { kind: &'static str, reason: String }, - - /// A field of the inscription a profile owns — `position` or a layer's - /// `scope` — is not a shape that canonicalizes. - #[error("the inscription's {field} is not the shape this profile writes: {reason}")] - MalformedInscription { field: String, reason: String }, - - /// A predecessor describing a different dataset than the one being - /// published, as [`DriverProfile::check_same_dataset`] judged it. - /// - /// The two identities are the profile's own — this crate carries the - /// numbers and composes no sentence about what they name — so a profile - /// that spells the refusal itself keeps the message it always had, the way - /// every other shared refusal here does. - #[error("this stele describes dataset {found}, but this node is configured for {expected}")] - DatasetMismatch { expected: u64, found: u64 }, - - /// A publish that would not extend the repository's chain. Raised by - /// [`stelae::inscription::history_for`] and carried here unchanged. - #[error( - "this repository's latest stele is sequence {latest} and this publish is sequence \ - {publishing}: {reason}" - )] - HistoryBreak { - latest: u64, - publishing: u64, - reason: String, - }, -} - -/// Protocol refusals this crate also names keep their own variant rather than -/// arriving wrapped. -/// -/// [`stelae::codec`] raises `MalformedRecord` and -/// [`stelae::inscription::history_for`] raises `HistoryBreak`; both were this -/// crate's errors before they moved down into the protocol, and both are -/// matched on by callers. Flattening keeps the variant a caller matches and the -/// message an operator reads exactly what they were, at the cost of a `match` -/// arm per shared refusal — which is the direction a move that is supposed to -/// change nothing observable should pay in. -impl From for Error { - fn from(error: stelae::Error) -> Self { - match error { - stelae::Error::MalformedRecord { kind, reason } => { - Self::MalformedRecord { kind, reason } - } - stelae::Error::HistoryBreak { - latest, - publishing, - reason, - } => Self::HistoryBreak { - latest, - publishing, - reason, - }, - other => Self::Stelae(other), - } - } -} - -impl Error { - pub(crate) fn out_of_order(kind: &'static str, reason: impl Into) -> Self { - Self::OutOfOrder { - kind, - reason: reason.into(), - } - } - - pub(crate) fn malformed_inscription( - field: impl Into, - reason: impl Into, - ) -> Self { - Self::MalformedInscription { - field: field.into(), - reason: reason.into(), - } - } -} - -/// Where a node stands relative to the newest stele already published. -/// -/// The comparison a publisher on a timer needs *before* anything is built, and -/// both halves of it are already in hand: the sequence a repository's latest -/// stele carries, and the sequence the node's cursor derived. Without it the -/// ordinary case — nothing has closed since last time — arrives as the -/// [`Error::HistoryBreak`] refusal a skipped sequence does, and a job on a -/// timer cannot tell the two apart. -/// -/// A pure comparison over two numbers rather than a method on a transport, so -/// the cases can be checked without one. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Standing { - /// Nothing has been published; this stele would start the chain. - Empty, - /// The published chain has already reached this node. Not an error: a - /// publisher whose node has not entered a new epoch has nothing to do. - UpToDate { latest: u64 }, - /// The chain ends exactly one sequence back; this stele extends it. - Next { latest: u64 }, - /// The node is further ahead than one sequence, so a publish would leave a - /// gap. `distance` is how far — the number the refusal reports alongside - /// both sequences, because "you skipped some" and "you skipped forty" are - /// different incidents. - Ahead { latest: u64, distance: u64 }, -} - -impl Standing { - /// Read a node at `sequence` against a repository whose latest stele is - /// `latest`. - pub fn read(latest: Option, sequence: u64) -> Self { - let Some(latest) = latest else { - return Self::Empty; - }; - - match sequence.checked_sub(latest) { - None | Some(0) => Self::UpToDate { latest }, - Some(1) => Self::Next { latest }, - Some(distance) => Self::Ahead { latest, distance }, - } - } - - /// Whether a publish should go ahead. - pub fn publishable(&self) -> bool { - matches!(self, Self::Empty | Self::Next { .. }) - } -} - -/// The pair that identifies one layer: its kind, and the canonical encoding of -/// its profile-owned scope. -/// -/// Canonical rather than [`serde_json::Value`] equality, because two scopes are -/// one layer exactly when they are the same bytes inside the canonical -/// document — the only sense of "the same scope" the protocol has. -/// -/// One function rather than three, and that is the point of it being here -/// instead of beside any one caller. Every table keyed this way is compared -/// against another table keyed this way: the predecessor's inheritable layers -/// against what a publish asks for, an interrupted publish's record against the -/// same, a reproduction's layers against the published ones. Three copies of -/// four lines would agree until one of them was corrected, and the failure that -/// follows is silent — a layer rebuilt instead of inherited, or a divergence -/// reported between two documents that say the same thing. -pub fn scope_key(kind: &str, scope: &serde_json::Value) -> Result<(String, String), Error> { - let canonical = stelae::inscription::canonical_json(scope)?; - - let canonical = String::from_utf8(canonical) - .map_err(|e| Error::malformed_inscription("layer scope", e.to_string()))?; - - Ok((kind.to_owned(), canonical)) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The four readings of a repository a publisher on a timer meets, and the - /// one that used to arrive as a refusal. - #[test] - fn a_repository_is_read_as_empty_current_next_or_ahead() { - assert_eq!(Standing::read(None, 500), Standing::Empty); - - // The ordinary case for a job that runs more often than epochs close. - assert_eq!( - Standing::read(Some(500), 500), - Standing::UpToDate { latest: 500 } - ); - - // And a node genuinely behind the repository, which is up to date in - // the only sense this comparison is for: there is nothing to publish. - assert_eq!( - Standing::read(Some(501), 500), - Standing::UpToDate { latest: 501 } - ); - - assert_eq!( - Standing::read(Some(499), 500), - Standing::Next { latest: 499 } - ); - - assert_eq!( - Standing::read(Some(497), 500), - Standing::Ahead { - latest: 497, - distance: 3 - } - ); - - for standing in [Standing::Empty, Standing::Next { latest: 1 }] { - assert!(standing.publishable(), "{standing:?}"); - } - - for standing in [ - Standing::UpToDate { latest: 1 }, - Standing::Ahead { - latest: 1, - distance: 2, - }, - ] { - assert!(!standing.publishable(), "{standing:?}"); - } - } -} diff --git a/crates/stelae-driver/src/predecessor.rs b/crates/stelae-driver/src/predecessor.rs deleted file mode 100644 index 8be286e42..000000000 --- a/crates/stelae-driver/src/predecessor.rs +++ /dev/null @@ -1,122 +0,0 @@ -//! The publish a publish follows, and what it may carry forward from it. -//! -//! The seam an export walks against: one trait, and the two implementations -//! that need nothing but a document. The transport-shaped one — the publish -//! this one follows *in a repository* — is [`crate::publish::Chained`]. - -use stelae::inscription::{HistoryEntry, LayerDescriptor}; - -use crate::Error; - -/// The publish this one follows. -/// -/// Two questions, one concept: what a new inscription attests about the steles -/// before it, and which of their layers it may carry forward rather than build -/// again. Both are answers only the previous publish has, and holding them -/// together is what lets a publisher rebuild everything while still chaining — -/// the `--rebuild` case, which suppresses [`Predecessor::adopt`] and leaves -/// [`Predecessor::history`] exactly as it was. -/// -/// The publish this one follows can be **this publish, interrupted**, which is -/// what [`Predecessor::landed`] is for: a stele that never got sealed left -/// layers behind that a restart may carry forward on exactly the terms a -/// predecessor's do. Nothing here decides where that is written down — that is -/// the implementor's, as `adopt` already is. -/// -/// `Sync`, because an export drives its layer producers from a pool of -/// threads and each producer asks these questions for its own layers. An -/// implementor that keeps state — an adoption counter, a resumption record — -/// keeps it behind its own synchronization. -pub trait Predecessor: Sync { - /// The history the new inscription carries: every prior publication, - /// contiguous and ascending, ending at `sequence - 1`. - /// - /// Assembling it is the implementor's business, and the protocol holds it - /// to the invariant when the document is validated - /// (`stelae::inscription`). - fn history(&self) -> &[HistoryEntry]; - - /// The descriptor to adopt for a layer of `kind` at `scope`, or `None` to - /// build it from the stores. - /// - /// An implementation that answers `Some` **has already arranged for the - /// transport to carry the layer's blob**; all that is left for an export - /// is to not walk the store. That ordering is why this returns a descriptor - /// rather than a boolean: the answer and the arrangement are one act, and - /// an export that reused a descriptor whose blob nothing carried would - /// publish a manifest with a hole in it. - /// - /// The default reuses nothing, which is what makes [`First`] one line and - /// what a transport with no notion of "already there" — a directory — - /// wants. - fn adopt( - &self, - kind: &str, - scope: &serde_json::Value, - ) -> Result, Error> { - let _ = (kind, scope); - - Ok(None) - } - - /// Whether a layer of `kind` at `scope` would be carried forward rather - /// than built — [`Predecessor::adopt`]'s question, asked without arranging - /// anything. - /// - /// Separate because `adopt` *acts*: it puts the blob in the transport and - /// counts the layer as reused. Forecasting how many layers a publish will - /// write has to ask the same question without taking either step, and a - /// forecast that called `adopt` would double-count every layer it looked - /// at. - /// - /// It may answer `true` where `adopt` will later answer `None`, in exactly - /// one case: a layer an interrupted publish recorded, whose blob the - /// repository has since dropped. That is only discovered by reaching for - /// it, which is the step this deliberately does not take — so this is the - /// honest forecast and `adopt` is the outcome. - /// - /// The default reuses nothing, matching [`Predecessor::adopt`]'s. - fn carried_forward(&self, kind: &str, scope: &serde_json::Value) -> Result { - let _ = (kind, scope); - - Ok(false) - } - - /// Note that `descriptor`'s layer is in the transport and will be in the - /// manifest, whether it was built here or adopted. - /// - /// Called once per epoch layer, the moment it lands, so an implementor - /// writing it down leaves a record that means "this layer is up" rather - /// than "this layer was attempted" — the same boundary - /// a restore's checkpoint records on its side. Layers land from - /// concurrent producers, so calls interleave; the record is keyed by kind - /// and scope, never by arrival order. The state shards are deliberately - /// never offered: they describe a moving tip, and a restart must rebuild - /// them. - /// - /// A failure here **fails the publish**. Recording is not a courtesy: a - /// record that silently stopped being written would cost the hours it - /// exists to save, at the moment nobody is watching. - /// - /// The default does nothing, which is what a publish with no host behind it - /// — a directory, a reproduction — wants. - fn landed(&self, descriptor: &LayerDescriptor) -> Result<(), Error> { - let _ = descriptor; - - Ok(()) - } -} - -/// The first stele of a repository: no history, nothing to inherit. -/// -/// The protocol permits an empty history at any sequence, so this is not only -/// the very first publish — it is every publish into a directory, which has no -/// way to be asked what it already holds. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct First; - -impl Predecessor for First { - fn history(&self) -> &[HistoryEntry] { - &[] - } -} diff --git a/crates/stelae-driver/src/preflight.rs b/crates/stelae-driver/src/preflight.rs deleted file mode 100644 index 32e148251..000000000 --- a/crates/stelae-driver/src/preflight.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! One free-space policy, in both directions. -//! -//! A publish and a restore have opposite shapes and the same problem: each -//! needs room on a volume before it starts, and the operator finds out hours in -//! when it does not have it. The peaks differ — a publish holds all sixteen -//! shard sinks open across one walk of the store, a restore stages one layer at -//! a time and drops it — but the *policy* over those numbers is one thing, and -//! it lives here rather than in each driver so the two cannot come to disagree. -//! -//! ## The policy -//! -//! - **A measured shortfall refuses**, naming the volume and how far short it -//! is. Only what was actually measured can refuse. -//! - **What cannot be measured warns and proceeds** — free space that will not -//! read, or a need nothing could size (a first publish has no predecessor to -//! size from). -//! -//! There is no override flag. A `--scratch-dir` pointed at a bigger volume is -//! the escape hatch, and a better one than a flag that turns the check off. -//! -//! ## Needs that share a volume are summed -//! -//! A restore's destination and its staging directory are the same pool of free -//! bytes whenever the scratch directory sits on the storage filesystem — which -//! the default, `/scratch`, guarantees. Checking each against the -//! whole of that pool would pass two needs that together do not fit, so -//! [`check`] groups by volume first and compares the sum. - -use std::path::{Path, PathBuf}; - -use crate::Error; - -/// One demand a run makes on a filesystem, before it makes it. -#[derive(Debug, Clone)] -pub struct Need { - /// What the bytes are for, phrased to read as the subject of a refusal: - /// "*restoring it* needs at least N bytes at …". - what: String, - /// Where they land. Need not exist yet — [`check`] measures the nearest - /// ancestor that does, which is the shape a fresh node's storage path and - /// an uncreated scratch directory both have. - path: PathBuf, - size: Sizing, -} - -/// How many bytes a need is, or why nothing could say. -#[derive(Debug, Clone)] -enum Sizing { - /// A floor, and always a floor: every number here is compressed or - /// uncompressed bytes as some document states them, and a store keeps - /// indexes and slack of its own. Under-stating is the safe direction for a - /// check whose job is to catch the obviously-doomed run. - Bytes(u64), - /// Nothing could size it, and why. A warning, never a refusal. - Unknown(String), -} - -impl Need { - /// A need of `bytes` at `path`. - pub fn of(what: impl Into, path: impl Into, bytes: u64) -> Self { - Self { - what: what.into(), - path: path.into(), - size: Sizing::Bytes(bytes), - } - } - - /// A need nothing could size, and the reason to tell the operator. - pub fn unsized_because( - what: impl Into, - path: impl Into, - why: impl Into, - ) -> Self { - Self { - what: what.into(), - path: path.into(), - size: Sizing::Unknown(why.into()), - } - } - - /// `bytes` when it is `Some`, and a warning carrying `why` when it is not. - /// - /// The shape almost every caller has: a size read out of a document that - /// may not carry one. - pub fn or_unsized( - what: impl Into, - path: impl Into, - bytes: Option, - why: impl Into, - ) -> Self { - match bytes { - Some(bytes) => Self::of(what, path, bytes), - None => Self::unsized_because(what, path, why), - } - } -} - -/// Refuse a run whose measured needs cannot fit; warn about the rest. -/// -/// The single entry point for both directions. Needs landing on one volume are -/// summed before the comparison — see the module documentation. -pub fn check(needs: &[Need]) -> Result<(), Error> { - let mut sized: Vec<(&Need, u64, PathBuf)> = Vec::new(); - - for need in needs { - match &need.size { - Sizing::Unknown(why) => tracing::warn!( - path = %need.path.display(), - "could not size {}; skipping the free-space check: {why}", - need.what, - ), - Sizing::Bytes(bytes) => match probe_of(&need.path) { - Some(probe) => sized.push((need, *bytes, probe)), - // A run that cannot measure the disk is not a run that should - // refuse to start, but it is one whose operator should hear - // about it. - None => tracing::warn!( - path = %need.path.display(), - "could not determine free space; skipping the free-space check for {}", - need.what, - ), - }, - } - } - - for group in group_by_volume(&sized) { - let probe = &sized[group[0]].2; - - let available = match fs4::available_space(probe) { - Ok(available) => available, - Err(e) => { - tracing::warn!( - path = %probe.display(), - "could not determine free space; skipping the free-space check: {e}" - ); - continue; - } - }; - - // Saturating because these are two documents' numbers added together - // and nothing bounds their sum; a total that pins at `u64::MAX` refuses - // for the same reason the true total would. - let required = group - .iter() - .fold(0u64, |total, i| total.saturating_add(sized[*i].1)); - - if required > available { - let parts: Vec<&(&Need, u64, PathBuf)> = group.iter().map(|i| &sized[*i]).collect(); - - return Err(shortfall(&parts, required, available)); - } - } - - Ok(()) -} - -/// The refusal, naming the volume and how far short it is. -/// -/// Two phrasings, because two needs sharing a volume is not the same incident -/// as one need overflowing it: an operator reading "restoring it needs" when -/// the staging is half the number would go looking in the wrong place, so the -/// shared case breaks the total down by need and by path before it states it. -fn shortfall(parts: &[&(&Need, u64, PathBuf)], required: u64, available: u64) -> Error { - let short = required - available; - - match parts { - [(only, _, _)] => Error::NotEnoughSpace(format!( - "{} needs at least {required} bytes at {}, which has {available} free — {short} bytes \ - short", - only.what, - only.path.display(), - )), - many => { - let breakdown: Vec = many - .iter() - .map(|(need, bytes, _)| { - format!("{} ({bytes} bytes at {})", need.what, need.path.display()) - }) - .collect(); - - Error::NotEnoughSpace(format!( - "{} share one volume and together need at least {required} bytes, which has \ - {available} free — {short} bytes short", - breakdown.join(" and "), - )) - } - } -} - -/// The nearest existing ancestor of `path`, which is what the filesystem can be -/// asked about. -/// -/// A fresh node's `storage.path` and an uncreated scratch directory are both -/// about to exist, and neither can be `stat`ed yet; the volume they will land -/// on is the one holding the deepest ancestor that does exist. `None` means -/// nothing in the chain answered, which is the case the caller warns about. -fn probe_of(path: &Path) -> Option { - let mut probe = path; - - loop { - if probe.metadata().is_ok() { - return Some(probe.to_path_buf()); - } - - probe = probe.parent()?; - } -} - -/// Partition needs into the volumes they draw on, as indices into `sized`. -fn group_by_volume(sized: &[(&Need, u64, PathBuf)]) -> Vec> { - let mut groups: Vec> = Vec::new(); - - for (i, (_, _, probe)) in sized.iter().enumerate() { - match groups - .iter_mut() - .find(|group| same_volume(&sized[group[0]].2, probe)) - { - Some(group) => group.push(i), - None => groups.push(vec![i]), - } - } - - groups -} - -/// Whether two existing paths draw on the same pool of free bytes. -/// -/// On Unix this is the filesystem's device id, which is exact in both -/// directions: a dedicated mount *under* `storage.path` reads as a different -/// volume, and two paths on one filesystem read as the same however differently -/// they are spelled — so `--scratch-dir` pointed elsewhere on the same disk is -/// still summed. -/// -/// Elsewhere stable Rust exposes no device id, so the test is the canonical -/// path's **prefix** — the drive letter or UNC share, which is the coarsest -/// thing Windows calls a volume. Two directories on `C:` are one pool however -/// they are spelled and whether or not either contains the other, which is what -/// containment alone would have got wrong for two siblings. What it still -/// cannot see is a volume *mounted into a folder* on another drive: those read -/// as one pool and are two, so the need is over-stated. For a refusal with no -/// override that is the direction that gets reported, rather than the direction -/// that gets discovered at hour eight. -#[cfg(unix)] -fn same_volume(a: &Path, b: &Path) -> bool { - use std::os::unix::fs::MetadataExt as _; - - match (a.metadata(), b.metadata()) { - (Ok(a), Ok(b)) => a.dev() == b.dev(), - // Both were probed a moment ago, so this is a directory that went away - // mid-check. Treating it as its own volume drops it out of every sum, - // which under-states rather than over-states. - _ => false, - } -} - -#[cfg(not(unix))] -fn same_volume(a: &Path, b: &Path) -> bool { - let (a, b) = match (a.canonicalize(), b.canonicalize()) { - (Ok(a), Ok(b)) => (a, b), - _ => return false, - }; - - match (a.components().next(), b.components().next()) { - (Some(std::path::Component::Prefix(a)), Some(std::path::Component::Prefix(b))) => a == b, - // No prefix to compare — not a shape a canonical Windows path has. - // Fall back to the narrowest honest answer. - _ => a == b, - } -} - -/// Every need here is a proportion of the free space the test measured, and -/// deliberately a coarse one: `check()` takes its own measurement of the same -/// volume, so an assertion pinned within a few bytes of what the test read is a -/// race against whatever else the machine is doing — under `cargo test`'s -/// threads, that includes the other tests in this module creating and dropping -/// temporary directories. A fixed byte cushion would not do: runner free space -/// differs by orders of magnitude across hosts, and only a proportion is -/// generous on all of them. -#[cfg(test)] -mod tests { - use super::*; - - /// The whole of the free-space policy, over needs that share one volume. - /// - /// Both halves in one test because they are one rule: what was measured - /// decides, and what was not is a warning that changes nothing. The sizes - /// are taken from the volume the test is running on, so the assertions hold - /// on any host. - #[test] - fn a_measured_shortfall_refuses_and_an_unmeasurable_need_does_not() { - let temp = tempfile::tempdir().unwrap(); - let available = fs4::available_space(temp.path()).unwrap(); - - check(&[Need::of("restoring it", temp.path(), available / 4)]).unwrap(); - - let err = check(&[Need::of("restoring it", temp.path(), available / 2 * 3)]).unwrap_err(); - assert!(matches!(err, Error::NotEnoughSpace(_)), "{err:?}"); - - // Nothing sized it, so nothing refuses — however impossible the run. - check(&[Need::unsized_because( - "staging this publish", - temp.path(), - "this repository holds no stele to size from", - )]) - .unwrap(); - } - - /// Two needs on one volume are one need against one pool. - /// - /// Each of these fits on its own and the pair does not, so a check that - /// passed them separately would pass this and a check that sums them - /// refuses it. - /// - /// Both shapes the pair can take, because they are not the same test on - /// every platform: `/scratch` is what the default makes of - /// every restore that names no directory, and a *sibling* is what - /// `--scratch-dir` next to the storage path makes of one that does. - /// Containment answers the first and not the second. - #[test] - fn needs_sharing_a_volume_are_summed() { - let root = tempfile::tempdir().unwrap(); - let storage = root.path().join("data"); - std::fs::create_dir(&storage).unwrap(); - - // A sibling only counts as one if it exists: an absent directory is - // measured through its parent, which here is an *ancestor* of the - // storage path and so would prove nothing about siblings. - let beside = root.path().join("staging"); - std::fs::create_dir(&beside).unwrap(); - - let available = fs4::available_space(&storage).unwrap(); - // One of these fits with a quarter of the pool to spare; the pair asks - // half again as much as the whole of it. - let each = available / 4 * 3; - - for scratch in [storage.join("scratch"), beside] { - check(&[Need::of("restoring it", &storage, each)]).unwrap(); - check(&[Need::of("staging the layers it pulls", &scratch, each)]).unwrap(); - - let err = check(&[ - Need::of("restoring it", &storage, each), - Need::of("staging the layers it pulls", &scratch, each), - ]) - .unwrap_err(); - - let Error::NotEnoughSpace(message) = &err else { - panic!("{err:?}"); - }; - - assert!(message.contains("share one volume"), "{message}"); - - for (what, path) in [ - ("restoring it", storage.clone()), - ("staging the layers it pulls", scratch.clone()), - ] { - let part = format!("{what} ({each} bytes at {})", path.display()); - assert!(message.contains(&part), "{part:?} missing from {message:?}"); - } - } - } - - /// A directory that does not exist yet is measured through its parent — - /// the shape a fresh node's storage path and an uncreated scratch - /// directory both have. - #[test] - fn a_path_that_does_not_exist_yet_is_measured_through_its_parent() { - let temp = tempfile::tempdir().unwrap(); - let missing = temp.path().join("not").join("created").join("yet"); - - assert_eq!(probe_of(&missing).as_deref(), Some(temp.path())); - assert!(same_volume(temp.path(), &probe_of(&missing).unwrap())); - - check(&[Need::of("restoring it", &missing, 1)]).unwrap(); - } -} diff --git a/crates/stelae-driver/src/profile.rs b/crates/stelae-driver/src/profile.rs deleted file mode 100644 index fde74eb1f..000000000 --- a/crates/stelae-driver/src/profile.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! What the lifecycle has to ask a profile. - -use stelae::inscription::Inscription; - -use crate::Error; - -/// The questions the publish and restore lifecycle asks of a profile. -/// -/// [`stelae::Profile`] answers what the *protocol* needs — naming, kinds, tags, -/// the record ceiling — and deliberately has no hook for anything -/// dataset-shaped. The lifecycle needs a little more than that and strictly -/// less than a dataset: which kinds an epoch produces, which kinds carry the -/// tip, whether a layer may be carried forward, and whether two documents -/// describe the same dataset at all. None of those answers reaches into a -/// store, a chain or a node, which is why they can live on a companion trait -/// here rather than growing the protocol's. -/// -/// Everything crossing this boundary in either direction is opaque: a `scope` -/// and a `position` are [`serde_json::Value`], composed by the profile and -/// never composed here. The driver reads a document's shape only through the -/// implementor. -pub trait DriverProfile: stelae::Profile { - /// The kinds a closed window always produces a layer for, plus the sparse - /// ones it may. - /// - /// The set a publish enumerates when it asks what it could carry forward - /// rather than build. - fn epoch_kinds(&self) -> &[&str]; - - /// The subset of [`DriverProfile::epoch_kinds`] a window produces - /// unconditionally. - /// - /// Split from the whole because the layer arithmetic is made of the two - /// arities: the dense kinds multiply out by the number of windows, and the - /// sparse ones have to be counted against the data. - fn dense_epoch_kinds(&self) -> &[&str]; - - /// Whether `kind` carries the dataset's tip rather than one window of its - /// history. - /// - /// Kind classification like [`DriverProfile::epoch_kinds`], and asked for - /// one reason: the staging arithmetic sizes the two halves of a stele - /// differently. A tip is rewritten whole by every publish, so all of it is - /// staged together and every such layer sums; anything else is staged a few - /// at a time and only the largest few count. [`is_inheritable`] cannot - /// stand in for this — it answers a different question, and a stele has - /// layers that are neither inheritable nor tip. - /// - /// [`is_inheritable`]: DriverProfile::is_inheritable - fn is_state_kind(&self, kind: &str) -> bool; - - /// Whether a layer of `kind` at `scope` may be carried forward from an - /// earlier publish rather than built again. - /// - /// A question about the scope as well as the kind, and the one rule three - /// callers share: the predecessor's manifest, an interrupted publish's - /// record, and the note a landed layer leaves. - fn is_inheritable(&self, kind: &str, scope: &serde_json::Value) -> bool; - - /// Refuse a predecessor that describes a different dataset than the one - /// being published. - /// - /// `previous` is the stele being chained to; `position` is the document the - /// new stele will carry. Both halves are the profile's own shape, so what - /// "the same dataset" means is the profile's to decide — the driver only - /// knows that a repository holding two of them is a fault, and refuses - /// before anything is built. - fn check_same_dataset( - &self, - previous: &Inscription, - position: &serde_json::Value, - ) -> Result<(), Error>; -} diff --git a/crates/stelae-driver/src/publish.rs b/crates/stelae-driver/src/publish.rs deleted file mode 100644 index 9ba9eba38..000000000 --- a/crates/stelae-driver/src/publish.rs +++ /dev/null @@ -1,966 +0,0 @@ -//! The chained-publish lifecycle against a repository. -//! -//! Everything a publisher does once it is standing in front of a *repository* -//! rather than a transport: what the new stele says about the ones already in -//! it, and what it may take from them. Generic over -//! [`DriverProfile`][crate::DriverProfile] throughout — every document that -//! crosses this module is composed by a profile and only compared here. -//! -//! What is *not* here is where any of it lives on a host: a resumption record's -//! path is handed in, never derived. A node's storage layout is the profile's. -//! -//! See the profile-side module that wraps this for the publish rules -//! themselves — the chain-or-refuse contract, what a reused layer asserts, and -//! what an interrupted publish leaves behind. - -use std::{ - collections::BTreeMap, - io::Write as _, - num::NonZeroUsize, - path::{Path, PathBuf}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Mutex, - }, -}; - -use serde::{Deserialize, Serialize}; -use stelae::{ - inscription::{Compression, HistoryEntry, Inscription, LayerDescriptor}, - oci::{Auth, Options, Registry, Repository, Stele, DEFAULT_CONCURRENCY}, - transport::WrittenLayer, - Digest, SteleReader as _, -}; - -use crate::{scope_key, DriverProfile, Error, Predecessor, Standing}; - -/// Open a repository in a registry. -/// -/// Here rather than at the call site so a node's binary keeps never naming the -/// protocol crate — the same property a profile's own publish and restore entry -/// points hold for a directory. -/// -/// `insecure` speaks plaintext HTTP. It is for a registry on a loopback address -/// or a mirror inside a cluster, and for nothing that is reachable from outside -/// one. -/// -/// `auth` is who to authenticate as, decided by the caller. A node resolves it -/// from its own configuration and environment; nothing here goes looking, for -/// the reason `stelae::oci` states one layer down and this crate has no more -/// standing to override than that one does. -/// -/// `scratch_dir` is where layers are staged, in both directions. The transport -/// creates it when the first layer needs it, so it need not exist yet. -/// -/// `tuning` is the publish path's concurrency and the one check an operator may -/// want back; [`Tuning::default`] is what every caller that is not publishing -/// wants. -/// -/// **Never call any of this from inside an async context.** The transport owns -/// a runtime and enters it with `block_on`; `stelae::oci`'s module -/// documentation states the rule and the reason. -pub fn open( - repository: &Repository, - insecure: bool, - auth: Auth, - scratch_dir: PathBuf, - tuning: Tuning, -) -> Result { - Ok(Registry::open( - repository, - Options { - insecure, - scratch_dir: Some(scratch_dir), - auth, - concurrency: tuning.concurrency.map_or(DEFAULT_CONCURRENCY, Into::into), - verify_adopted: tuning.verify_adopted, - // Not an operator's knob and deliberately not one: the number that - // absorbs a registry's transient `5xx` is the transport's own - // measurement, and an outage longer than it is answered a level up, - // where `snapshot backfill` re-runs the whole publish rather than - // dying into a pod restart. - attempts: stelae::oci::DEFAULT_ATTEMPTS, - // Nor these, for a related reason: both are facts about the - // registry this publishes to and about the pod the publisher runs - // in, measured rather than chosen, and neither moves when an - // operator changes how fast a publish goes. `upload_memory` in - // particular is what keeps `--concurrency` from being a claim on - // memory — raising one does not raise the other. - monolithic_max: stelae::oci::DEFAULT_MONOLITHIC_MAX, - upload_memory: stelae::oci::DEFAULT_UPLOAD_MEMORY, - }, - )?) -} - -/// What an operator may set about *how* a publish moves, as against where it -/// goes. -/// -/// Separated from [`open`]'s other arguments because it is the only one of them -/// a caller can leave alone: a repository, a credential and a staging directory -/// are facts a publish cannot be run without, and these two are a default and -/// an escape hatch. A restore or an inspection passes [`Tuning::default`] and -/// means it. -#[derive(Debug, Clone, Copy, Default)] -pub struct Tuning { - /// How many layer round trips run at once; `None` is - /// [`DEFAULT_CONCURRENCY`]. - pub concurrency: Option, - - /// Re-prove that the registry still holds each blob carried forward out of - /// the predecessor's manifest. See [`stelae::oci::Options::verify_adopted`] - /// for why this is off. - pub verify_adopted: bool, -} - -/// Where a publish is going, and what the host running it knows about itself. -/// -/// The publish-side counterpart of a restore's node-side facts, and it holds -/// the transport for the same reason that one holds a storage path: these are -/// the facts a publish is *given*, as against the ones it derives. Threading -/// them separately is what took [`publish`] to the edge of its signature, and -/// they have never been supplied from different places. -/// -/// The record's path is owned rather than borrowed, and the type is `Clone` -/// rather than `Copy` because of it. Both are deliberate: a builder that took a -/// `&Path` would accept a node's *storage directory* as readily as its record, -/// silently, and the two are one `join` apart. -#[derive(Clone)] -pub struct Publishing<'a> { - /// The repository being published into, already opened. - pub registry: &'a Registry, - - /// Where the resumption record is kept, handed in rather than derived: - /// a node's storage layout is the profile's and this crate never composes - /// one. `None` for a caller with no node behind it, which records nothing - /// and resumes nothing. - pub record_path: Option, - - /// The operator's `--rebuild`: build every layer, inherit none, and start - /// the record over. - pub rebuild: bool, -} - -impl<'a> Publishing<'a> { - /// A publish into `registry` that keeps no record and rebuilds nothing. - pub fn new(registry: &'a Registry) -> Self { - Self { - registry, - record_path: None, - rebuild: false, - } - } - - /// The same publish, recording what it finishes at `record_path`. - pub fn recording_in(self, record_path: impl Into) -> Self { - Self { - record_path: Some(record_path.into()), - ..self - } - } - - /// The same publish, with the operator's `--rebuild`. - pub fn rebuilding(self, rebuild: bool) -> Self { - Self { rebuild, ..self } - } -} - -/// What a record has to agree with before a single layer in it is adopted. -/// -/// Everything a recorded layer's bytes and address depend on that is *not* in -/// the layer's own key. The key is the kind plus the descriptor scope, and that -/// scope names an epoch and a slot window and nothing else — so: -/// -/// - **the repository**, because a blob digest is an address in one repository -/// and means nothing in another; -/// - **the dataset**, because one sequence of two different datasets is two -/// different sets of bytes under one key. A descriptor scope names a window -/// and not a dataset — the layer's own header record carries that — so this -/// is the only place the record can hold it; -/// - **the parameters and the compression**, which are the inscription's own -/// statement of how its layers were built. A binary that changed either would -/// rebuild a recorded layer into different bytes, and the record would be -/// offering an answer to a question nobody is asking any more. -/// -/// A mismatch in any of them makes the record a fresh one for the origin at -/// hand. Nothing is repaired and nothing is merged: the layers it named are -/// still in the registry, and the publish that wants them will build them -/// again and find them there. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct Origin { - /// The repository the layers were uploaded to, as the transport names it. - pub repository: String, - /// The dataset the layers were built from, as the profile identifies it. - /// - /// Opaque here: this crate compares it and never reads it. A profile that - /// numbers its datasets one way and a profile that numbers them another - /// both fit, which is the whole reason the field is not named after - /// either. - pub dataset_id: u64, - /// The inscription parameters the layers were built under. - pub parameters: serde_json::Value, - /// The compression they were built with. - pub compression: Compression, -} - -impl Origin { - /// What a publish into `registry` of the dataset `dataset_id` identifies, - /// under `parameters` and `compression`, records under. - /// - /// Rebuilt from parts rather than read off a plan: every one of them is the - /// profile's own statement about the publish. - pub fn of( - registry: &Registry, - dataset_id: u64, - parameters: serde_json::Value, - compression: Compression, - ) -> Self { - Self { - repository: registry.repository().to_string(), - dataset_id, - parameters, - compression, - } - } - - /// The fields `self` and `other` disagree on, in the order [`Origin`] - /// states them. - /// - /// One of the four ways to differ changes the repository, so a refusal - /// that reported the two repository names alone would read, in the other - /// three, as though the two publishes matched. Names rather than values: - /// `parameters` is arbitrary JSON, and what an operator needs from the - /// event is which knob moved between the two runs. - pub fn differences_from(&self, other: &Self) -> Vec<&'static str> { - [ - (self.repository != other.repository, "repository"), - (self.dataset_id != other.dataset_id, "dataset id"), - (self.parameters != other.parameters, "parameters"), - (self.compression != other.compression, "compression"), - ] - .into_iter() - .filter_map(|(differs, field)| differs.then_some(field)) - .collect() - } -} - -/// The epoch layers an interrupted publish got as far as uploading. -/// -/// Written after each layer's upload succeeds and deleted once the stele is -/// sealed, so a record that exists describes a publish that did not finish. See -/// the module documentation for what it is and — more to the point — what it is -/// not. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct PublishRecord { - pub origin: Origin, - - /// The layers whose blobs are up, each as the transport measured it. - /// - /// The whole [`WrittenLayer`] rather than the digest pair the adoption - /// needs: the descriptor is what the new manifest has to state about the - /// layer, and a record that held only its identity would have to invent the - /// rest. In the canonical order of their keys, so the same progress is the - /// same bytes. - pub layers: Vec, -} - -impl PublishRecord { - /// Read the record at `path`, or `None` if there is none. - /// - /// **Only absence is `None`**, on a restore's progress file's reasoning - /// turned around: a file that exists and does not parse is an - /// error rather than an empty resume, because reading it as "nothing has - /// been uploaded" silently costs the rebuild this file exists to avoid. - /// `--rebuild` is how an operator asks for that outcome on purpose. - pub fn load(path: &Path) -> Result, Error> { - let raw = match read_file(path) { - Ok(raw) => raw, - Err(e) => return Err(Error::Stelae(e)), - }; - - let Some(raw) = raw else { - return Ok(None); - }; - - Ok(Some( - serde_json::from_slice(&raw).map_err(|e| Error::Stelae(e.into()))?, - )) - } - - /// Delete the record at `path`. A file that is not there is not an error. - pub fn remove(path: &Path) -> Result<(), Error> { - match std::fs::remove_file(path) { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(Error::Stelae(e.into())), - } - } - - /// Write the record at `path`, atomically. - /// - /// Through a temporary sibling and a rename, for the reason - /// `stelae::plan::RestoreProgress::save` states on its side: the failure - /// this file exists to survive is a process that stops mid-write, and a - /// half-written record would be refused by [`PublishRecord::load`] — - /// correctly, and uselessly, since the publish it described would then have - /// to start over. - pub fn save(&self, path: &Path) -> Result<(), Error> { - save_atomically( - path, - &serde_json::to_vec(self).map_err(stelae::Error::from)?, - ) - .map_err(Error::Stelae) - } - - /// The layers this record offers, keyed the way [`Chained`] looks them up. - /// - /// An [`Origin`] the caller is not publishing under offers nothing: see - /// [`Origin`] for why a mismatch is a fresh start rather than a merge. - pub fn table( - &self, - origin: &Origin, - profile: &dyn DriverProfile, - ) -> Result, Error> { - if &self.origin != origin { - tracing::info!( - differs = %self.origin.differences_from(origin).join(", "), - recorded = %self.origin.repository, - publishing = %origin.repository, - "a resumption record was left by a publish this one does not continue; \ - every layer will be rebuilt" - ); - - return Ok(BTreeMap::new()); - } - - let mut table = BTreeMap::new(); - - for layer in &self.layers { - // The same filter `inheritable_layers` applies to a predecessor's - // manifest, applied again on the way in: a record naming a state - // *tip* shard is a record nothing wrote, and honouring one would - // carry a stale tip into a manifest. A retained dump's scope names - // its epoch, so it passes here for the same reason an epoch layer - // does. - if !profile.is_inheritable(&layer.descriptor.kind, &layer.descriptor.scope) { - continue; - } - - table.insert( - scope_key(&layer.descriptor.kind, &layer.descriptor.scope)?, - layer.clone(), - ); - } - - Ok(table) - } -} - -fn read_file(path: &Path) -> Result>, stelae::Error> { - match std::fs::read(path) { - Ok(raw) => Ok(Some(raw)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(e.into()), - } -} - -fn save_atomically(path: &Path, bytes: &[u8]) -> Result<(), stelae::Error> { - let name = path - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(); - - let staging = path.with_file_name(format!(".{name}.{}.tmp", std::process::id())); - - // Scoped so the handle is closed before the rename: Windows refuses to - // rename a file that is still open. - { - let mut file = std::fs::File::create(&staging)?; - - file.write_all(bytes)?; - - // Before the rename, not after: a rename that lands pointing at bytes - // the page cache has not written yet is the same truncated file by - // another route. - file.sync_all()?; - } - - std::fs::rename(&staging, path)?; - - Ok(()) -} - -/// Where this node stands relative to what `registry` already holds. -/// -/// One read of the moving tag, and no store is touched. It is what turns "the -/// node has not entered a new epoch" from the same refusal a skipped epoch -/// raises into an answer a job on a timer can act on — see [`Standing`]. -/// -/// Cheap enough to ask before every publish: a manifest pull against the -/// moving tag, which [`publish`] and [`preview`] are each about to make anyway. -/// Asking twice is one HTTP round trip against the alternative, which is -/// threading the answer out of a call that has already started building. -/// -/// **Never call this from inside an async context.** See [`open`]. -pub fn standing( - registry: &Registry, - profile: &dyn DriverProfile, - sequence: u64, - position: &serde_json::Value, -) -> Result { - let latest = registry - .latest(profile)? - .map(|stele| stele.read_inscription()) - .transpose()?; - - if let Some(previous) = &latest { - // The same refusal a publish makes, made before the report rather than - // after it: a repository holding another dataset's chain is not "up to - // date" with this node in any sense worth reporting. - profile.check_same_dataset(previous, position)?; - } - - Ok(Standing::read( - latest.map(|previous| previous.sequence), - sequence, - )) -} - -/// The publish this one follows, in a repository — which may be itself. -/// -/// Holds the history it hands to the new inscription and the two tables of -/// layers it is willing to let the new stele carry forward rather than build, -/// both keyed by the pair that decides it: the layer's kind and the canonical -/// encoding of its profile-owned scope. -/// -/// The tables answer the same question from different standing. `inheritable` -/// is the *predecessor's manifest* — a stele the repository serves, so a layer -/// missing from it is a fault. `resumable` is *this publish's own record* of an -/// attempt that died before it could write a manifest — a note, so a layer -/// missing from the registry is only a rebuild. The manifest is consulted -/// first, because a repository that states it holds a layer needs no note to -/// say so. -pub struct Chained<'a> { - registry: &'a Registry, - /// Held rather than threaded, because [`Predecessor::landed`] is asked by - /// the export's producer pool and has nowhere to take one from. - profile: &'a (dyn DriverProfile + Sync), - source: Option<&'a Stele>, - predecessor: Option<(u64, Digest)>, - history: Vec, - inheritable: BTreeMap<(String, String), LayerDescriptor>, - resumable: BTreeMap<(String, String), WrittenLayer>, - record: Option, - /// Atomic, like the record's lock below: [`export::export`] asks a - /// predecessor about layers from a pool of producer threads. - adopted: AtomicUsize, -} - -/// The resumption record this publish is writing, open. -/// -/// Seeded with what it inherits, so the file is the whole of what is up rather -/// than the whole of what *this attempt* put up: an attempt that adopts twenty -/// layers and adds one, then dies, has to leave twenty-one behind or the third -/// attempt pays for the difference. -/// -/// The lock is held across the file write as well as the map insert: each -/// write rewrites the whole record, so two producers landing layers at once -/// must serialize on the file or the later write would drop the earlier -/// layer. -struct Recording { - path: PathBuf, - origin: Origin, - layers: Mutex>, -} - -impl<'a> Chained<'a> { - /// The publish `sequence` follows in `publishing`'s repository. - /// - /// Every input is a part rather than a plan: the sequence being published, - /// the `position` document the new stele will carry — read only by the - /// profile, through [`DriverProfile::check_same_dataset`] — and the - /// [`Origin`] the profile records under. - pub fn new( - profile: &'a (dyn DriverProfile + Sync), - publishing: Publishing<'a>, - latest: Option<&'a Stele>, - sequence: u64, - position: &serde_json::Value, - origin: Origin, - ) -> Result { - let Publishing { - registry, - record_path, - rebuild, - } = publishing; - - let inscription = latest.map(|stele| stele.read_inscription()).transpose()?; - - if let Some(previous) = &inscription { - // The pull that fetched this checked as a reader; this is the - // publish side, which inherits the chain and must attest it. - previous.check_profile_strict(profile)?; - profile.check_same_dataset(previous, position)?; - } - - let history = stelae::inscription::history_for(inscription.as_ref(), sequence)?; - - let predecessor = inscription - .as_ref() - .map(|previous| Ok::<_, Error>((previous.sequence, previous.digest()?))) - .transpose()?; - - // Built only when it can be used. `rebuild` is the publisher choosing - // to reproduce rather than inherit, and it stops here rather than at - // `adopt` so that nothing downstream has to remember it was set. - let inheritable = match (rebuild, &inscription) { - (false, Some(previous)) => inheritable_layers(previous, profile)?, - _ => BTreeMap::new(), - }; - - // Read on the same terms, and it is the *honouring* that `rebuild` - // gates rather than the reading — the asymmetry a restore's checkpoint - // states, for the same reason. A publisher that asked to rebuild gets a - // record that starts empty and overwrites whatever was there, so - // nothing an earlier attempt believed can survive the run that was - // meant to settle it. - let resumable = match (rebuild, record_path.as_deref()) { - (false, Some(record_path)) => match PublishRecord::load(record_path)? { - Some(record) => record.table(&origin, profile)?, - None => BTreeMap::new(), - }, - _ => BTreeMap::new(), - }; - - if !resumable.is_empty() { - tracing::info!( - layers = resumable.len(), - "an interrupted publish left epoch layers in this repository; \ - they will be carried forward rather than rebuilt" - ); - } - - let record = record_path.map(|record_path| Recording { - path: record_path, - origin, - layers: Mutex::new(resumable.clone()), - }); - - Ok(Self { - registry, - profile, - source: latest.filter(|_| !rebuild), - predecessor, - history, - inheritable, - resumable, - record, - adopted: AtomicUsize::new(0), - }) - } - - /// The stele this one chains to, if the repository holds one. - pub fn predecessor(&self) -> Option<(u64, Digest)> { - self.predecessor - } - - /// How many layers this publish carried forward rather than built. - pub fn adopted(&self) -> usize { - self.adopted.load(Ordering::Relaxed) - } - - /// Delete the resumption record. - /// - /// Called once the stele is sealed and never before it: before the seal it - /// would be a record that outlived neither the run nor its usefulness. - pub fn forget_record(&self) -> Result<(), Error> { - match &self.record { - Some(record) => PublishRecord::remove(&record.path), - None => Ok(()), - } - } -} - -impl Predecessor for Chained<'_> { - fn history(&self) -> &[HistoryEntry] { - &self.history - } - - /// What [`preview`] reports, and it spends no `HEAD`: the promise a dry run - /// makes is about what the scopes permit. The record's own gate — that the - /// registry still holds the blob — runs in [`Predecessor::adopt`] and can - /// turn one of these into a rebuild, which is the direction a dry run is - /// allowed to be wrong in. - fn carried_forward(&self, kind: &str, scope: &serde_json::Value) -> Result { - let key = scope_key(kind, scope)?; - - Ok(self.inheritable.contains_key(&key) || self.resumable.contains_key(&key)) - } - - fn adopt( - &self, - kind: &str, - scope: &serde_json::Value, - ) -> Result, Error> { - let key = scope_key(kind, scope)?; - - // The arrangement and the answer are one act, in both branches: by the - // time this returns a descriptor, the transport is already carrying the - // blob, and the `HEAD` that proves the registry still has it has already - // happened. - if let (Some(source), Some(descriptor)) = (self.source, self.inheritable.get(&key)) { - self.registry.adopt_layer(source, descriptor.clone())?; - self.adopted.fetch_add(1, Ordering::Relaxed); - - return Ok(Some(descriptor.clone())); - } - - let Some(recorded) = self.resumable.get(&key) else { - return Ok(None); - }; - - if !self.registry.adopt_carried(recorded.clone())? { - tracing::warn!( - kind, - %scope, - "a recorded layer's blob is no longer in the repository; rebuilding it" - ); - - return Ok(None); - } - - self.adopted.fetch_add(1, Ordering::Relaxed); - - Ok(Some(recorded.descriptor.clone())) - } - - fn landed(&self, descriptor: &LayerDescriptor) -> Result<(), Error> { - let Some(record) = &self.record else { - return Ok(()); - }; - - if !self - .profile - .is_inheritable(&descriptor.kind, &descriptor.scope) - { - return Ok(()); - } - - let Some(written) = self.registry.carried(&descriptor.diff_id) else { - tracing::warn!( - kind = descriptor.kind, - scope = %descriptor.scope, - "this layer is not in the transport, so nothing was recorded for it; \ - an interrupted publish will rebuild it" - ); - - return Ok(()); - }; - - // The transport is asked for its *measurement* and not for its - // descriptor. `carried` finds a layer by `diffId`, and one `diffId` can - // now wear two descriptors — the dump a publish cuts out of its own tip - // is the tip's bytes — so recording what the lookup returned verbatim - // would file the dump under the tip's scope, where nothing looks for it - // and where it would be discarded on the way back in. - let written = WrittenLayer { - descriptor: descriptor.clone(), - digests: written.digests, - }; - - // Held across the save below, not just the insert — see [`Recording`]. - let mut layers = record - .layers - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - - layers.insert(scope_key(&descriptor.kind, &descriptor.scope)?, written); - - PublishRecord { - origin: record.origin.clone(), - layers: layers.values().cloned().collect(), - } - .save(&record.path) - } -} - -/// The layers a new stele may inherit from `previous`, keyed by kind and scope. -/// -/// [`DriverProfile::is_inheritable`] decides, and it decides on the scope as -/// well as the kind. A state *tip* shard is out: it changes every publish, and -/// — independently — its descriptor scope names no epoch, so scope equality -/// could not tell one publish's shard from another's. A retained state dump is -/// in: it is a closed epoch's state under a scope that names that epoch. -/// `digests` has no source in this slice. -/// -/// Two layers of one kind claiming one scope, described differently in any -/// respect, is a refusal rather than a first-wins: it means the stele being -/// chained to describes the same window twice and disagrees with itself about -/// what is in it, and inheriting either answer would publish that disagreement -/// forward. "In any respect" and not "with different identities", because -/// `records` and `uncompressed_size` are determined by the bytes a `diff_id` -/// names — so a disagreement about them under one identity is the same -/// contradiction wearing a quieter shape. -pub fn inheritable_layers( - previous: &Inscription, - profile: &dyn DriverProfile, -) -> Result, Error> { - let mut inheritable = BTreeMap::new(); - - for layer in &previous.layers { - if !profile.is_inheritable(&layer.kind, &layer.scope) { - continue; - } - - let key = scope_key(&layer.kind, &layer.scope)?; - - if let Some(existing) = inheritable.get(&key) { - let existing: &LayerDescriptor = existing; - - // The whole descriptor, not the identity alone. `records` and - // `uncompressed_size` are functions of the bytes `diff_id` names, - // so two descriptors sharing an identity and disagreeing about - // either are a stele contradicting itself just as surely as two - // identities would be — and `adopt_layer` carries - // `uncompressed_size` forward into a stele that never reads the - // bytes that would settle it. - if existing != layer { - // Spelled out rather than named by identity alone: the two can - // now differ while sharing a `diff_id`, and a message printing - // one digest twice would describe nothing. - let describe = |layer: &LayerDescriptor| { - format!( - "{} ({} records, {} bytes)", - layer.diff_id, layer.records, layer.uncompressed_size, - ) - }; - - return Err(Error::malformed_inscription( - format!("layers[{}]", layer.kind), - format!( - "sequence {} describes {} twice at one scope, as {} and as {}", - previous.sequence, - layer.kind, - describe(existing), - describe(layer), - ), - )); - } - - continue; - } - - inheritable.insert(key, layer.clone()); - } - - Ok(inheritable) -} - -/// What a publish holds staged on disk at its peak. -/// -/// Not the size of the stele: a publish never has the whole document on disk at -/// once. What it does have at once is a *tip* pass — the profile's export keeps -/// one state kind's shard sinks open across a single walk of its namespace — -/// plus whatever other layers are in flight beside it. Summing *every* state -/// layer rather than the widest kind's is deliberately conservative: the peak -/// it prices is the pre-split one, an upper bound on any per-kind pass. -/// -/// Which layers are the tip is [`DriverProfile::is_state_kind`]'s answer and -/// not one this crate could reach: it is the one place the arithmetic has to -/// know something about a kind beyond its name. -/// -/// "Layers beside it" is a handful and not one, because the transport uploads -/// concurrently: a layer's staging file lives until its own round trip lands, -/// so as many finished layers as the transport has permits can sit on disk at -/// once alongside the pass. How many that is comes from -/// [`stelae::oci::Registry::concurrency`] — asked of the transport rather than -/// assumed here — and *which* ones is the largest that many, since the peak is -/// the worst arrangement and nothing orders the uploads. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct StagingPeak { - /// Every state layer, summed — the conservative reading above. - pub state_bytes: u64, - /// The non-state layers that can be staged at once, largest first: as many - /// as the transport uploads concurrently, or all of them where the stele - /// has fewer than that. Empty on a stele that is all state, and on a - /// [`StagingPeak::default`] nothing was measured into. - pub concurrent_other_bytes: Vec, - /// Layers the predecessor's manifest stated no size for. Non-zero makes - /// every number here a floor rather than an estimate. - pub unsized_layers: usize, -} - -impl StagingPeak { - /// Compressed bytes the scratch volume has to hold at once. - /// - /// Saturating throughout, because these are a manifest's numbers and a - /// manifest is not this node's document: a wrapped sum would be a *small* - /// need, which is the one direction a refusal must never be wrong in. - pub fn bytes(&self) -> u64 { - self.concurrent_other_bytes - .iter() - .fold(self.state_bytes, |total, bytes| { - total.saturating_add(*bytes) - }) - } - - /// The largest non-state layer, which is the first of the ones staged at - /// once. - pub fn largest_other_bytes(&self) -> u64 { - self.concurrent_other_bytes.first().copied().unwrap_or(0) - } -} - -/// Size the staging peak of the next publish from the stele before it. -/// -/// A proxy, and deliberately so: the numbers are the *predecessor's* layers, -/// not this publish's, because this publish's layers do not exist until it -/// builds them and sizing them would mean building them. One sequence of drift -/// is what the proxy costs, against a check that otherwise cannot exist at all -/// — and the refuse/warn split is what keeps it honest, since a shortfall -/// against the last publish's sizes is still a measured shortfall. -/// -/// Off the manifest the pull already fetched, and no per-layer `HEAD`: the -/// promise a profile's dry run makes about touching nothing holds here too. -/// -/// `Ok(None)` is a first publish — no predecessor, nothing to size from — which -/// [`preflight`] turns into a warning rather than a refusal. -/// -/// **Never call this from inside an async context.** See [`open`]. -pub fn staging_peak( - registry: &Registry, - profile: &dyn DriverProfile, -) -> Result, Error> { - let Some(previous) = registry.latest(profile)? else { - return Ok(None); - }; - - let inscription = previous.read_inscription()?; - let blobs = previous.blob_index()?; - - let mut peak = StagingPeak::default(); - let mut others = Vec::new(); - - for descriptor in &inscription.layers { - match previous.compressed_size(&blobs, descriptor)? { - None => peak.unsized_layers += 1, - // Every state layer adds — the conservative sum-all-state rule the - // type documents. - Some(bytes) if profile.is_state_kind(&descriptor.kind) => { - peak.state_bytes = peak.state_bytes.saturating_add(bytes) - } - Some(bytes) => others.push(bytes), - } - } - - // The largest as many as the transport stages at once, and no more: a stele - // with fewer non-state layers than the transport has permits cannot put - // more of them on disk than it has. - others.sort_unstable_by(|a, b| b.cmp(a)); - others.truncate(registry.concurrency()); - - peak.concurrent_other_bytes = others; - - Ok(Some(peak)) -} - -/// Refuse a publish whose scratch volume cannot hold what it stages at once. -/// -/// The publish side of [`crate::preflight`]'s one policy, which a profile's -/// restore is the other side of: a measured shortfall refuses, naming the -/// volume and the shortfall, and what cannot be sized warns and proceeds. -/// -/// The volume is the transport's own — [`stelae::oci::Registry::scratch_dir`] — -/// so the directory sized here and the directory written to cannot come apart. -/// A transport opened without one stages in the platform temporary directory, -/// which it does not name, so there is nothing to size and nothing to refuse; -/// [`open`] always sets one, so no caller that came through it reaches that -/// case. -/// -/// **Never call this from inside an async context.** See [`open`]. -pub fn preflight(registry: &Registry, profile: &dyn DriverProfile) -> Result<(), Error> { - let Some(scratch_dir) = registry.scratch_dir() else { - tracing::warn!( - "this transport stages in the platform temporary directory, which it does not name; \ - skipping the free-space check for {STAGING}" - ); - - return Ok(()); - }; - - crate::preflight::check(&[staging_need(staging_peak(registry, profile)?, scratch_dir)]) -} - -/// What the staging asks of its volume, as [`crate::preflight`] takes it. -/// -/// Split out of [`preflight`] so the demand and the transport that produced it -/// can be tested apart: sizing a peak needs a registry, and deciding what a -/// peak means for a volume needs none. -const STAGING: &str = "staging this publish"; - -fn staging_need(peak: Option, scratch_dir: &Path) -> crate::preflight::Need { - let Some(peak) = peak else { - return crate::preflight::Need::unsized_because( - STAGING, - scratch_dir, - "this repository holds no stele to size the staging from", - ); - }; - - if peak.unsized_layers > 0 { - tracing::warn!( - unsized_layers = peak.unsized_layers, - "the predecessor's manifest states no size for some of its layers; the staging \ - estimate is a floor" - ); - } - - crate::preflight::Need::of(STAGING, scratch_dir, peak.bytes()) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The publish half of the one policy, over a peak this test supplies. - /// - /// The other half of the chain — that the peak is the predecessor's tip - /// shards plus its largest other layer — needs a registry to state a - /// manifest, so it lives with a profile that can build one. Between them - /// the composition [`preflight`] performs is covered: this end decides, - /// that end measures. - #[test] - fn a_measured_staging_shortfall_refuses_and_a_first_publish_does_not() { - let temp = tempfile::tempdir().unwrap(); - - let check = |peak| -> Result<(), Error> { - crate::preflight::check(&[staging_need(peak, temp.path())]) - }; - - // No predecessor, so nothing sized it, so nothing refuses. - check(None).unwrap(); - - check(Some(StagingPeak::default())).unwrap(); - - // A peak no volume holds. Both halves are set, so the refusal is on - // their sum and not on either alone. - let err = check(Some(StagingPeak { - state_bytes: u64::MAX / 2, - concurrent_other_bytes: vec![u64::MAX / 4, u64::MAX / 4], - unsized_layers: 0, - })) - .unwrap_err(); - - let Error::NotEnoughSpace(message) = &err else { - panic!("{err:?}"); - }; - - assert!(message.contains(STAGING), "{message}"); - assert!( - message.contains(&temp.path().display().to_string()), - "{message}" - ); - assert!(message.contains("short"), "{message}"); - } -} diff --git a/crates/stelae-driver/src/reporting.rs b/crates/stelae-driver/src/reporting.rs deleted file mode 100644 index 4b73c2346..000000000 --- a/crates/stelae-driver/src/reporting.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Counting layers and records for the two drivers to report. -//! -//! Nothing here is protocol: [`stelae::progress`] defines the seam and the -//! events, and this is the small amount of arithmetic a *driver* has to do to -//! fill one in — which layer of how many is in flight, and how often a long -//! scan is worth mentioning. -//! -//! Shared by a profile's export and restore drivers because the two count the -//! same thing and a second copy would be a second answer to "how far along is -//! this". Both types are call-scoped: they live on a stack frame for the length -//! of one publish or one restore, and neither outlives it. - -use stelae::progress::{Event, Observer, Outcome}; - -/// Records reported in one go. -/// -/// Not one event per record. A mainnet state shard is tens of millions of them, -/// and a bar redrawn per record is resolution nobody can see bought at a -/// virtual call per record; this is fine enough that a bar still moves several -/// times a second on the slowest layer in the profile. -const RECORD_CADENCE: u64 = 4096; - -/// Where a driver is in its run of layers. -/// -/// Positions are handed out by [`Cursor::open`] and quoted back to -/// [`Cursor::close`] rather than tracked as "the current layer", because a -/// driver may hold several open at once — the export's state pass keeps all -/// sixteen shard sinks open across one walk of the store — and a single cursor -/// would report the last one opened as the one that finished. -/// -/// The position counter is atomic because the export drives its layer -/// producers from a pool of threads, and every producer announces through the -/// one cursor. Positions are display order, nothing else: the inscription -/// lists layers by its own rule, so two runs that announce in different -/// interleavings still publish the same document. -pub struct Cursor<'a> { - observer: &'a Observer, - next: std::sync::atomic::AtomicUsize, - total: usize, -} - -impl<'a> Cursor<'a> { - pub fn new(observer: &'a Observer, total: usize) -> Self { - Self { - observer, - next: std::sync::atomic::AtomicUsize::new(0), - total, - } - } - - /// Announce a layer and take its position in the run. - pub fn open(&self, kind: &str, scope: &serde_json::Value) -> usize { - let index = self.next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - self.observer.emit(Event::LayerStarted { - index, - total: self.total, - kind, - scope, - }); - - index - } - - /// Close the layer `index` was handed out for. - pub fn close(&self, index: usize, kind: &str, outcome: Outcome) { - self.observer.emit(Event::LayerFinished { - index, - total: self.total, - kind, - outcome, - }); - } - - /// A record counter reporting to the same place. - pub fn records(&self) -> Records<'a> { - Records { - observer: self.observer, - pending: 0, - } - } - - /// Layers announced so far — what a caller cross-checks its own total - /// against once the run is over. - pub fn opened(&self) -> usize { - self.next.load(std::sync::atomic::Ordering::Relaxed) - } -} - -/// Records counted since the last time anyone was told about them. -/// -/// [`Records::flush`] is explicit rather than a `Drop`, because the moment that -/// matters is *before* the layer closes: an observer that saw a layer finish -/// and then received records for it would have to know which layer they -/// belonged to, and the seam deliberately does not carry that. -pub struct Records<'a> { - observer: &'a Observer, - pending: u64, -} - -impl Records<'_> { - pub fn tick(&mut self) { - self.pending += 1; - - if self.pending >= RECORD_CADENCE { - self.flush(); - } - } - - pub fn flush(&mut self) { - if self.pending > 0 { - self.observer.emit(Event::Records(self.pending)); - self.pending = 0; - } - } -} - -#[cfg(test)] -mod tests { - use std::sync::{Arc, Mutex}; - - use super::*; - use stelae::progress::Progress; - - #[derive(Default)] - struct Recorder(Mutex>); - - impl Progress for Recorder { - fn on(&self, event: Event<'_>) { - let line = match event { - Event::LayerStarted { index, kind, .. } => format!("open {index} {kind}"), - Event::LayerFinished { - index, - kind, - outcome, - .. - } => format!("close {index} {kind} {outcome:?}"), - Event::Records(n) => format!("records {n}"), - other => format!("{other:?}"), - }; - - self.0.lock().unwrap().push(line); - } - } - - /// The shape the state pass needs: positions handed out up front, closed in - /// whatever order the sinks finish, and never confused with each other. - #[test] - fn positions_survive_layers_held_open_together() { - let recorder = Arc::new(Recorder::default()); - let observer = Observer::new(recorder.clone()); - let cursor = Cursor::new(&observer, 2); - - let scope = serde_json::json!({}); - let first = cursor.open("state", &scope); - let second = cursor.open("state", &scope); - - cursor.close(second, "state", Outcome::Transferred); - cursor.close(first, "state", Outcome::Transferred); - - assert_eq!(cursor.opened(), 2); - assert_eq!( - *recorder.0.lock().unwrap(), - vec![ - "open 0 state", - "open 1 state", - "close 1 state Transferred", - "close 0 state Transferred", - ] - ); - } - - /// A cadence that reported nothing until a layer ended would leave the bar - /// still for exactly the layers it exists for, and one that reported a - /// trailing zero would tell a renderer records moved when none did. - #[test] - fn records_report_on_the_cadence_and_once_at_the_end() { - let recorder = Arc::new(Recorder::default()); - let observer = Observer::new(recorder.clone()); - let cursor = Cursor::new(&observer, 1); - - let mut records = cursor.records(); - - for _ in 0..RECORD_CADENCE + 3 { - records.tick(); - } - - records.flush(); - records.flush(); - - assert_eq!( - *recorder.0.lock().unwrap(), - vec![format!("records {RECORD_CADENCE}"), "records 3".to_owned()] - ); - } -} diff --git a/crates/stelae-driver/src/restore.rs b/crates/stelae-driver/src/restore.rs deleted file mode 100644 index a9c9ea66c..000000000 --- a/crates/stelae-driver/src/restore.rs +++ /dev/null @@ -1,251 +0,0 @@ -//! Restore planning shapes and the resume checkpoint. -//! -//! The profile owns a restore's selection — a layer's `scope` is opaque to the -//! protocol, so only the profile can read an epoch out of one — and the stores -//! it writes into. What lives here is the part that is the same whatever is -//! being restored: how much a restore holds at once ([`Budget`]), what it still -//! has to do ([`Outlook`]), and where it records what it has finished -//! ([`Checkpoint`]). -//! -//! The checkpoint's rule is [`stelae::plan::Resume`]'s — a layer is done when -//! its `diffId` is recorded, which is a fact about bytes and not about the -//! stele they were published in. *Which* layers may be skipped at all is the -//! profile's half of the split, and stays with it: this type is only ever -//! asked about the layers a profile chose to ask about. - -use std::path::PathBuf; - -use stelae::{ - frame::Limits, - inscription::LayerDescriptor, - plan::{Remaining, RestoreProgress, Resume}, - progress::Outcome, - Digest, -}; -use tracing::info; - -use crate::Error; - -/// What a restore holds at once. -/// -/// A store writer batches until `commit` and a layer arrives as a stream, so -/// nothing bounds a restore's memory except these numbers. Both commit ceilings -/// are needed and neither subsumes the other: an index record is tens of bytes -/// and only a count bounds it, while one epoch of blocks can run to gigabytes -/// and only a byte budget bounds that. -/// -/// There is deliberately no `Default`: the read limits are the publishing -/// profile's ceilings, not the protocol's defaults — a restore that read under -/// a tighter limit than the publisher wrote under would refuse that profile's -/// own steles — so the profile supplies its budget. -#[derive(Debug, Clone, Copy)] -pub struct Budget { - /// Per-record and window bounds on the layer read itself. - pub limits: Limits, - /// Records accumulated before a write batch is committed. - pub commit_records: usize, - /// Bytes accumulated before a write batch is committed. - pub commit_bytes: usize, -} - -/// What a restore is about to do, once the stele has been read. -/// -/// Returned alongside the profile's plan so a caller can report the -/// *remaining* download rather than the original one — the whole point of the -/// accounting on a resumed run. -#[derive(Debug, Clone, Copy)] -pub struct Outlook { - /// Layers still to fetch, and what they weigh compressed. - pub remaining: Remaining, - /// Layers an earlier attempt had already committed. - pub inherited: usize, -} - -/// Where a restore records what it has finished, and what it inherits. -/// -/// One value rather than three arguments, because the three are one idea: the -/// file, the set of layers it says are done, and the identity of the stele -/// being restored into it. -pub struct Checkpoint { - path: PathBuf, - resume: Resume, - progress: RestoreProgress, -} - -impl Checkpoint { - /// Open the checkpoint at `path` for restoring the stele `identity`. - /// - /// The path is the caller's: the driver never derives where a node keeps - /// its progress file. `resume` gates whether anything on disk is - /// *honoured* — not merely whether it is read. A restore that is not - /// resuming is starting over: it takes an empty [`Resume`] and its first - /// checkpoint overwrites whatever was there. - /// - /// That asymmetry is deliberate. A progress file that outlived the stores - /// beside it would name layers whose data is gone, and honouring one - /// nobody asked to honour would skip them onto empty stores — a node - /// missing a slice of data that nothing would report. The rule here means - /// even a file that somehow survived its stores cannot do that damage. - pub fn open(path: PathBuf, identity: Digest, resume: bool) -> Result { - let existing = match resume { - true => RestoreProgress::load(&path)?, - false => None, - }; - - let resume = Resume::from_progress(existing.as_ref()); - - // The new identity, the old completions. The completions are what the - // resume rule is about — content, not the document that described it — - // and the digest is what tells a later reader which stele a - // half-finished restore was aimed at. - let progress = RestoreProgress { - inscription_digest: identity, - completed: existing.map(|p| p.completed).unwrap_or_default(), - }; - - Ok(Self { - path, - resume, - progress, - }) - } - - /// A restore that checkpoints nowhere. - /// - /// For a caller driving a restore without a node behind it — the test - /// suites, above all, which compare store sets rather than resumes. - pub fn none() -> Self { - Self { - path: PathBuf::new(), - resume: Resume::none(), - progress: RestoreProgress::new(Digest::from_bytes([0; 32])), - } - } - - /// What this checkpoint inherits, for the remaining-bytes accounting. - pub fn resume(&self) -> &Resume { - &self.resume - } - - /// Read `descriptor`'s layer unless an earlier attempt already committed - /// it. - /// - /// The one place a layer is decided about, so that the skip and the - /// checkpoint cannot drift apart. `fetch` runs to completion — the caller - /// commits before it returns — and only then is the layer recorded, which - /// is what makes the record mean "committed" rather than "attempted". - /// Returns the outcome alongside the value, rather than leaving a caller - /// to ask the resume the same question a second time: what an observer or - /// a summary reports has to be this decision and not a re-derivation of - /// it. - pub fn fetch>( - &mut self, - descriptor: &LayerDescriptor, - fetch: impl FnOnce() -> Result, - ) -> Result<(T, Outcome), E> { - if self.resume.is_done(&descriptor.diff_id) { - info!( - kind = descriptor.kind, - scope = %descriptor.scope, - "skipping a layer an earlier attempt completed" - ); - - return Ok((T::default(), Outcome::Skipped)); - } - - let out = fetch()?; - - self.record(descriptor.diff_id)?; - - Ok((out, Outcome::Transferred)) - } - - fn record(&mut self, diff_id: Digest) -> Result<(), Error> { - if self.path.as_os_str().is_empty() { - return Ok(()); - } - - self.progress.record(diff_id); - self.progress.save(&self.path)?; - - Ok(()) - } - - /// Delete the progress file. - /// - /// For the moment the restore is *finished* — which is the caller's call, - /// not the last `fetch`'s: whatever work follows the final layer is - /// exactly the window a kept progress file lets an operator repair by - /// resuming. - pub fn clear(&self) -> Result<(), Error> { - if self.path.as_os_str().is_empty() { - return Ok(()); - } - - RestoreProgress::remove(&self.path)?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn descriptor(byte: u8) -> LayerDescriptor { - LayerDescriptor { - kind: "blocks".into(), - media_type: "application/cbor-seq".into(), - diff_id: Digest::from_bytes([byte; 32]), - records: 1, - uncompressed_size: 1, - scope: serde_json::json!({"epoch": 7}), - } - } - - #[test] - fn a_resumed_checkpoint_skips_what_it_recorded() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("progress.json"); - let identity = Digest::from_bytes([1; 32]); - - let mut first = Checkpoint::open(path.clone(), identity, false).unwrap(); - let (_, outcome) = first.fetch::<(), Error>(&descriptor(2), || Ok(())).unwrap(); - assert!(matches!(outcome, Outcome::Transferred)); - - let mut second = Checkpoint::open(path, identity, true).unwrap(); - let (_, outcome) = second - .fetch::<(), Error>(&descriptor(2), || panic!("must not refetch")) - .unwrap(); - assert!(matches!(outcome, Outcome::Skipped)); - } - - #[test] - fn a_fresh_checkpoint_ignores_what_is_on_disk() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("progress.json"); - let identity = Digest::from_bytes([1; 32]); - - let mut first = Checkpoint::open(path.clone(), identity, false).unwrap(); - first.fetch::<(), Error>(&descriptor(2), || Ok(())).unwrap(); - - let mut fresh = Checkpoint::open(path, identity, false).unwrap(); - let mut fetched = false; - fresh - .fetch::<(), Error>(&descriptor(2), || { - fetched = true; - Ok(()) - }) - .unwrap(); - assert!(fetched); - } - - #[test] - fn a_checkpoint_that_goes_nowhere_records_nothing() { - let mut none = Checkpoint::none(); - none.fetch::<(), Error>(&descriptor(2), || Ok(())).unwrap(); - assert!(none.resume().is_empty()); - - none.clear().unwrap(); - } -} diff --git a/crates/stelae-driver/src/retry.rs b/crates/stelae-driver/src/retry.rs deleted file mode 100644 index 7f24288d7..000000000 --- a/crates/stelae-driver/src/retry.rs +++ /dev/null @@ -1,199 +0,0 @@ -//! Bounded patience for an external that fails in bursts. -//! -//! A registry round trip and an aggregator fetch fail the same way — briefly, -//! and for reasons neither end can classify — so the policy that absorbs them -//! is one policy rather than one per caller. - -use std::time::Duration; - -/// Attempts a transient-prone external gets before its failure is fatal. -/// -/// Bounded on purpose. The preprod G1 backfill measured why the retry exists — -/// four container exits between 06:39Z and 08:53Z on 2026-08-23, each one -/// paying a store restore, a window re-download and the in-flight epoch's -/// re-replay for what a half-minute wait would have absorbed — and the same -/// measurement is why it is not open-ended: a misconfigured aggregator or a -/// repository the credentials cannot read has to keep failing, and keep -/// failing while whoever launched the run is still watching. -pub const RETRY_ATTEMPTS: u32 = 4; - -/// The first wait between attempts; each later one doubles it. Three waits of -/// 5s, 10s and 20s put the ceiling at 35 seconds of patience. -pub const RETRY_BASE_DELAY: Duration = Duration::from_secs(5); - -/// Sleep, unless a shutdown is requested first. `false` means it was. -/// -/// Sliced rather than slept in one call so a signal that arrives during a -/// backoff is honoured at the next slice instead of at the end of the wait — -/// the driver's whole shutdown budget is a container's SIGTERM grace period, -/// which a 20-second sleep would eat. -fn sleep_unless_aborted(delay: Duration, abort: &dyn Fn() -> bool) -> bool { - const SLICE: Duration = Duration::from_millis(250); - - let mut left = delay; - - while !left.is_zero() { - if abort() { - return false; - } - - let slice = left.min(SLICE); - std::thread::sleep(slice); - left -= slice; - } - - !abort() -} - -/// Run `op`, retrying a failure with exponential backoff, then let it be fatal. -/// -/// The last attempt's error is returned untouched, so every caller keeps the -/// diagnostic it had before the retry was wrapped around it — the retry moves -/// where the fatal path is reached, never what it says. Nothing here decides a -/// failure is transient: the classification the alternative would need does not -/// exist at these seams (an aggregator's errors arrive as opaque strings), and -/// guessing it wrong reinstates exactly the fatal exits this is here to absorb. -/// What bounds the patience is [`RETRY_ATTEMPTS`], not a judgement about the -/// error. -/// -/// `abort` is polled between and during the waits, so a shutdown ends the run -/// on the failure in hand rather than after the remaining backoff. Callers with -/// no shutdown to observe pass `&|| false`. -/// -/// Only for operations that are safe to simply run again: reads, and downloads -/// whose destination is rewritten from the same arguments. -pub fn transient(what: &str, abort: &dyn Fn() -> bool, op: F) -> Result -where - F: FnMut() -> Result, - E: std::fmt::Display, -{ - bounded(what, RETRY_ATTEMPTS, RETRY_BASE_DELAY, abort, op) -} - -/// [`transient`] with its two constants spelled out, so a caller under test can -/// exercise the loop without waiting out a real backoff. -pub fn bounded( - what: &str, - attempts: u32, - base_delay: Duration, - abort: &dyn Fn() -> bool, - mut op: F, -) -> Result -where - F: FnMut() -> Result, - E: std::fmt::Display, -{ - let mut delay = base_delay; - - for attempt in 1..attempts { - match op() { - Ok(value) => return Ok(value), - Err(err) => { - if abort() { - return Err(err); - } - - tracing::warn!( - what, - attempt, - remaining = attempts - attempt, - backoff_secs = delay.as_secs(), - error = %err, - "transient failure; retrying", - ); - - if !sleep_unless_aborted(delay, abort) { - return Err(err); - } - - delay = delay.saturating_mul(2); - } - } - } - - op() -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The retry loop, with the real backoff replaced by none of it. - fn retried( - abort: &dyn Fn() -> bool, - op: impl FnMut() -> Result, - ) -> Result { - bounded("a test", RETRY_ATTEMPTS, Duration::ZERO, abort, op) - } - - #[test] - fn a_call_that_succeeds_is_made_once() { - let mut calls = 0; - - let result: Result = retried(&|| false, || { - calls += 1; - Ok(7) - }); - - assert_eq!(result.unwrap(), 7); - assert_eq!(calls, 1, "a success must not be retried"); - } - - #[test] - fn a_transient_failure_is_absorbed() { - let mut calls = 0; - - let result: Result = retried(&|| false, || { - calls += 1; - - if calls < 3 { - Err("the aggregator hung up".to_owned()) - } else { - Ok(7) - } - }); - - assert_eq!(result.unwrap(), 7); - assert_eq!(calls, 3, "the loop stops at the first success"); - } - - #[test] - fn patience_is_bounded_and_the_last_error_is_the_one_raised() { - let mut calls = 0; - - let result: Result = retried(&|| false, || { - calls += 1; - Err(format!("attempt {calls} failed")) - }); - - assert_eq!( - calls, RETRY_ATTEMPTS as usize, - "a persistent failure must still reach the fatal path", - ); - - assert_eq!( - result.unwrap_err(), - format!("attempt {RETRY_ATTEMPTS} failed"), - "the caller keeps the diagnostic the final attempt produced", - ); - } - - #[test] - fn a_shutdown_ends_the_run_on_the_failure_in_hand() { - let mut calls = 0; - - let result: Result = retried(&|| true, || { - calls += 1; - Err("interrupted".to_owned()) - }); - - assert_eq!(calls, 1, "a requested shutdown is not waited out"); - assert_eq!(result.unwrap_err(), "interrupted"); - } - - #[test] - fn a_shutdown_during_a_backoff_cuts_the_wait_short() { - assert!(!sleep_unless_aborted(Duration::from_secs(60), &|| true)); - assert!(sleep_unless_aborted(Duration::ZERO, &|| false)); - } -} diff --git a/crates/stelae/Cargo.toml b/crates/stelae/Cargo.toml deleted file mode 100644 index 1fba18c7c..000000000 --- a/crates/stelae/Cargo.toml +++ /dev/null @@ -1,131 +0,0 @@ -[package] -name = "stelae" -description = "Stelae: a deterministic, content-addressed snapshot protocol" -version.workspace = true -edition.workspace = true - -[features] -# OCI registry transport (`src/oci.rs`). Default-off, so a build that does not -# publish or restore over a registry keeps the dependency tree it had: the -# protocol's format, framing and digests need no HTTP client and no async -# runtime, and the two are a large part of this tree when they are pulled in. -# -# The boundary check does not relax for a feature: `cargo tree -p stelae -e -# normal --all-features` must still match nothing `^dolos(-|$)`. -oci = [ - "dep:bytes", - "dep:futures-util", - "dep:http", - "dep:oci-client", - "dep:reqwest", - "dep:tempfile", - "dep:tokio", -] - -# This crate is the Stelae protocol. It must never depend on a `dolos-*` -# package, so that extracting it later is a directory move rather than a -# refactor. `cargo tree -p stelae -e normal` shows the boundary holding. -# See adrs/004_stelae_snapshots.md, "Code layout". -[dependencies] -hex.workspace = true -minicbor.workspace = true -serde.workspace = true -serde_jcs.workspace = true -serde_json.workspace = true -sha2.workspace = true -thiserror.workspace = true -zstd.workspace = true - -# --- feature `oci` --------------------------------------------------------- -# -# `oci-client` is the oras-project registry client, taken with none of its own -# TLS features. Both of them are pure feature-forwarding — `rustls-tls = -# ["reqwest/rustls", "jsonwebtoken/aws_lc_rs"]`, `native-tls = -# ["reqwest/native-tls", "jsonwebtoken/rust_crypto"]` — and no line of the -# crate is `#[cfg]`-gated on `rustls-tls` at all. So the feature does not -# decide whether the client speaks TLS; it decides which crypto backend two of -# its dependencies are built with, and that choice is made below instead. -# -# It has to be made somewhere, because `rustls-tls` reaches `aws-lc-sys` -# twice over — through `jsonwebtoken/aws_lc_rs` for registry auth tokens and -# through `reqwest 0.13`'s `rustls`, which fans out to `rustls/aws-lc-rs` — -# and `aws-lc-sys` requires `cmake` to build, the very dependency the root -# package's `mithril-client` entry goes out of its way to avoid. Defeating one -# path leaves the other, so both are answered here — the `reqwest` one by -# naming a different feature, the `jsonwebtoken` one by needing nothing. -# -# The `jsonwebtoken` half needs nothing to replace it. `oci-client` reaches -# for that crate in exactly one place outside its own tests — -# `dangerous::insecure_decode`, reading a bearer token's `exp` claim to decide -# when the token cache should refetch — and that call is base64 and serde. It -# never signs and never verifies, so it never asks for a crypto provider, and -# the backend `rustls-tls` was selecting was paying for capability the client -# does not use. Leaving `jsonwebtoken` with no backend is a configuration -# `oci-client` supports rather than a hole punched in it: nothing in the crate -# is `#[cfg]`-gated on having one. -# -# The alternative the mithril precedent suggests, `jsonwebtoken/rust_crypto`, -# was tried first and rejected on evidence: it pulls `rsa 0.9`, which carries -# RUSTSEC-2023-0071 with no fixed version, so `cargo deny check advisories` -# fails from that commit onward — a standing tax on every later PR, bought for -# an RSA implementation that no reachable line calls. -# -# `test-registry` is dropped for its own reason: it exists for the crate's own -# test suite. -oci-client = { version = "0.17", optional = true, default-features = false } -# Named for `oci-client`, not for this crate: nothing here calls it. Cargo -# unifies features across the graph, so declaring it chooses the TLS `reqwest` -# that `oci-client` is built against without this crate ever naming the -# aws-lc-rs one. `default-features = false`, matching what `oci-client` itself -# asks for, so this adds one capability and no surface. -# -# `rustls-no-provider` is rustls with no crypto provider wired in — the same -# trade the root package already makes for `mithril-client`. It buys a smaller -# tree by moving a compile-time guarantee to a runtime one: a process that -# opens a `Registry` must have installed a process-default provider first. -# That precondition is stated in `oci.rs`'s module documentation, beside the -# async-context rule, and the tests here install `ring` rather than inheriting -# one. -reqwest = { version = "0.13", optional = true, default-features = false, features = [ - "rustls-no-provider", -] } -# One runtime, owned by the transport and entered with `block_on` at each call, -# so the protocol stays synchronous. See `oci.rs`. -# -# `sync` and `time` are named here and not at the workspace root because they -# are this crate's need and nobody else's: the publish path bounds its -# concurrent layer round trips with a `Semaphore`, and the permit is what also -# bounds how many staged layers can exist at once, while a round trip the -# registry answered with a `5xx` waits out a backoff before it is made again. -tokio = { workspace = true, optional = true, features = ["sync", "time"] } -# `push_blob_stream` takes a `Stream`, which is how a layer is uploaded from -# its staging file without ever being held. -futures-util = { workspace = true, optional = true } -bytes = { version = "1", optional = true } -# Solely to name the manifest's `Content-Type`. The manifest is pushed as the -# exact canonical bytes this crate produced — the ones the golden freezes and -# the size ceiling measured — which means `push_manifest_raw`, which takes a -# `HeaderValue`. `http` is the same one `reqwest` re-exports. -http = { version = "1", optional = true } -# Staging on the way up, and the pulled blob on the way down. Both are files -# for the same reason: a blob's digest is only known once its last byte has -# gone past, in either direction. -tempfile = { version = "3.20.0", optional = true } - -[dev-dependencies] -# The peak-allocation test instruments the global allocator, the same idiom the -# root package's tests/memory.rs uses for store iteration. A dev-dependency, so -# it does not appear in `cargo tree -p stelae -e normal` and the boundary check -# above is unaffected. -stats_alloc = "0.1" -tempfile = "3.20.0" -# The registry tests install `ring` as the process-default crypto provider, -# which is the precondition `oci.rs` puts on a caller and the reason `ring` is -# named here and nowhere else in this manifest: choosing a provider is the -# program's job, and under `cargo test` this crate's test binary is the -# program. A dev-dependency, so `cargo tree -p stelae -e normal -# --all-features` still matches no provider at all. -rustls = { version = "0.23", default-features = false, features = [ - "ring", - "std", -] } diff --git a/crates/stelae/src/codec.rs b/crates/stelae/src/codec.rs deleted file mode 100644 index 0459b6e49..000000000 --- a/crates/stelae/src/codec.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Fixed-arity decode helpers for a layer's content records. -//! -//! A profile's records are CBOR arrays of a known width, read out of bytes -//! [`crate::frame`] has already validated against the deterministic profile. -//! What is left to check is *shape* — the field count, each field's type and -//! width, and that nothing trails the record — and every profile checks it the -//! same way, so the helpers live here rather than once per profile. -//! -//! ## Why decoding does not re-validate canonical form -//! -//! Records reach a profile's `decode` from [`crate::dir::Layer::records`] or -//! [`crate::LayerReader::next_record`], both of which have already validated -//! every byte against the deterministic profile — that is the framing layer's -//! job and it is not repeated here. - -use minicbor::Decoder; - -use crate::Error; - -/// Open a record's outer array, insisting on a definite length of `expected`. -pub fn open(kind: &'static str, decoder: &mut Decoder<'_>, expected: u64) -> Result<(), Error> { - let fields = decoder - .array() - .map_err(|e| Error::malformed(kind, format!("expected an array: {e}")))? - .ok_or_else(|| Error::malformed(kind, "indefinite-length array"))?; - - if fields != expected { - return Err(Error::malformed( - kind, - format!("expected {expected} fields, found {fields}"), - )); - } - - Ok(()) -} - -/// Insist the record ended where the array did. -/// -/// A CBOR sequence has no frame markers, so a record with a tail would be read -/// as one item by the framing layer and as a different, shorter item here — two -/// readers disagreeing about the same bytes, which is how a diffId stops -/// meaning anything. -pub fn close(kind: &'static str, decoder: &Decoder<'_>, bytes: &[u8]) -> Result<(), Error> { - let read = decoder.position(); - - if read != bytes.len() { - return Err(Error::malformed( - kind, - format!("{} trailing byte(s) after the record", bytes.len() - read), - )); - } - - Ok(()) -} - -pub fn uint(kind: &'static str, field: &str, decoder: &mut Decoder<'_>) -> Result { - decoder - .u64() - .map_err(|e| Error::malformed(kind, format!("{field}: {e}"))) -} - -pub fn text<'b>( - kind: &'static str, - field: &str, - decoder: &mut Decoder<'b>, -) -> Result<&'b str, Error> { - decoder - .str() - .map_err(|e| Error::malformed(kind, format!("{field}: {e}"))) -} - -pub fn blob<'b>( - kind: &'static str, - field: &str, - decoder: &mut Decoder<'b>, -) -> Result<&'b [u8], Error> { - decoder - .bytes() - .map_err(|e| Error::malformed(kind, format!("{field}: {e}"))) -} - -/// A byte string of exactly `N` bytes. -/// -/// Width is checked here rather than by a lossy conversion downstream: a -/// fixed-width key type typically converts from a slice by zero-padding or -/// truncating, so a wrong-width field would become a valid-looking key that no -/// lookup can ever reach. -pub fn fixed( - kind: &'static str, - field: &str, - decoder: &mut Decoder<'_>, -) -> Result<[u8; N], Error> { - let raw = blob(kind, field, decoder)?; - - raw.try_into().map_err(|_| { - Error::malformed( - kind, - format!("{field}: expected {N} bytes, found {}", raw.len()), - ) - }) -} diff --git a/crates/stelae/src/digest.rs b/crates/stelae/src/digest.rs deleted file mode 100644 index 8611ab730..000000000 --- a/crates/stelae/src/digest.rs +++ /dev/null @@ -1,556 +0,0 @@ -//! Digests and the layer compression pipeline. -//! -//! A layer has two digests and they answer different questions: -//! -//! - **`diffId`** — sha256 over the *uncompressed* CBOR sequence. This is -//! identity. It is what the inscription lists, what independent publishers -//! reproduce, and what a signature ultimately covers. -//! - **blob digest** — sha256 over the *zstd-compressed* bytes. This is -//! transport. It is what an OCI registry addresses the blob by, and it is not -//! stable across zstd versions or levels, which is precisely why it cannot be -//! the identity anchor (ADR-004, "Determinism is anchored on uncompressed -//! bytes"). -//! -//! [`LayerWriter`] produces both in a single pass over the data: bytes are -//! hashed on the way in, compressed, and hashed again on the way out. Nothing -//! is buffered, so a layer of any size costs one sequential scan. - -use std::io::{ErrorKind, Read, Write}; - -use sha2::{Digest as _, Sha256}; - -use crate::Error; - -/// A sha256 digest, rendered as `sha256:<64 hex chars>` wherever it is written -/// down — the OCI spelling, so inscription values paste straight into registry -/// tooling. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Digest([u8; 32]); - -impl Digest { - pub const ALGORITHM: &'static str = "sha256"; - - pub fn from_bytes(bytes: [u8; 32]) -> Self { - Self(bytes) - } - - /// sha256 of a byte slice. - pub fn compute(data: impl AsRef<[u8]>) -> Self { - let mut hasher = Sha256::new(); - hasher.update(data.as_ref()); - Self(hasher.finalize().into()) - } - - pub fn as_bytes(&self) -> &[u8; 32] { - &self.0 - } - - /// The bare hex encoding, without the `sha256:` prefix. This is the file - /// name an OCI image layout uses under `blobs/sha256/`. - pub fn to_hex(&self) -> String { - hex::encode(self.0) - } -} - -impl std::fmt::Display for Digest { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}:{}", Self::ALGORITHM, hex::encode(self.0)) - } -} - -impl std::str::FromStr for Digest { - type Err = Error; - - fn from_str(s: &str) -> Result { - let invalid = |reason: &str| Error::InvalidDigest { - value: s.to_owned(), - reason: reason.to_owned(), - }; - - let (algorithm, hex_digits) = s - .split_once(':') - .ok_or_else(|| invalid("expected `:`"))?; - - if algorithm != Self::ALGORITHM { - return Err(invalid("only sha256 is defined by this protocol version")); - } - - if hex_digits.len() != 64 { - return Err(invalid("expected 64 hex digits")); - } - - if hex_digits - .bytes() - .any(|b| !b.is_ascii_digit() && !(b'a'..=b'f').contains(&b)) - { - return Err(invalid("expected lowercase hex digits")); - } - - let mut bytes = [0u8; 32]; - hex::decode_to_slice(hex_digits, &mut bytes).map_err(|e| invalid(&e.to_string()))?; - - Ok(Self(bytes)) - } -} - -impl serde::Serialize for Digest { - fn serialize(&self, serializer: S) -> Result { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> serde::Deserialize<'de> for Digest { - fn deserialize>(deserializer: D) -> Result { - let raw = String::deserialize(deserializer)?; - raw.parse().map_err(serde::de::Error::custom) - } -} - -/// What one pass over a layer blob establishes. -/// -/// Serializable for the reason [`crate::transport::WrittenLayer`] is: the two -/// digests and the two sizes are one measurement, and a host writing part of it -/// down is a host that can read back a pair that never described one layer. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct LayerDigests { - /// sha256 over the uncompressed CBOR sequence — the layer's identity. - pub diff_id: Digest, - /// sha256 over the compressed bytes — how a registry addresses the blob. - pub blob_digest: Digest, - pub uncompressed_size: u64, - pub compressed_size: u64, -} - -/// A writer that hashes what it is given, compresses it, and hashes the result. -/// -/// Write the uncompressed CBOR sequence into it; [`LayerWriter::finish`] -/// returns the sink together with both digests and both sizes. -pub struct LayerWriter { - encoder: zstd::stream::write::Encoder<'static, Tap>, - hasher: Sha256, - uncompressed_size: u64, -} - -impl LayerWriter { - /// Wrap `sink`, compressing at `level`. - /// - /// The level is transport policy: it changes the blob digest and the bytes - /// on the wire, never the `diffId`. It is recorded in the inscription's - /// `compression` so publishers converge on identical blobs in practice, but - /// verification never depends on that. - pub fn new(sink: W, level: i32) -> Result { - Ok(Self { - encoder: zstd::stream::write::Encoder::new(Tap::new(sink), level)?, - hasher: Sha256::new(), - uncompressed_size: 0, - }) - } - - pub fn finish(self) -> Result<(W, LayerDigests), Error> { - let diff_id = Digest(self.hasher.finalize().into()); - let tap = self.encoder.finish()?; - let (sink, blob_digest, compressed_size) = tap.finish(); - - Ok(( - sink, - LayerDigests { - diff_id, - blob_digest, - uncompressed_size: self.uncompressed_size, - compressed_size, - }, - )) - } -} - -impl Write for LayerWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let written = self.encoder.write(buf)?; - self.hasher.update(&buf[..written]); - self.uncompressed_size += written as u64; - Ok(written) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.encoder.flush() - } -} - -/// sha256 and byte count of a stream, with no interpretation of its content. -/// -/// Used to check a stored blob against the digest it is named by, which must -/// hold whether or not the blob turns out to be a readable layer. -pub fn digest_reader(mut source: R) -> Result<(Digest, u64), Error> { - let mut hasher = Sha256::new(); - let mut buffer = [0u8; 64 * 1024]; - let mut total = 0u64; - - loop { - let read = read_uninterrupted(&mut source, &mut buffer)?; - if read == 0 { - break; - } - hasher.update(&buffer[..read]); - total += read as u64; - } - - Ok((Digest(hasher.finalize().into()), total)) -} - -/// Read a compressed layer blob, computing both digests without holding the -/// uncompressed content. -/// -/// Deliberately unbounded: the decompressed bytes go to [`std::io::sink`], so a -/// blob with a hostile compression ratio costs time here but never memory. -/// Callers that keep the content want [`read_blob`], which requires a ceiling. -pub fn scan_blob(source: R) -> Result { - digest_blob(source, &mut std::io::sink(), None) -} - -/// Read a compressed layer blob, returning its uncompressed bytes and both -/// digests. -/// -/// The uncompressed content is buffered, so this is for callers that intend to -/// walk the records anyway. Verification-only callers want [`scan_blob`]. -/// -/// `max_uncompressed` bounds what is buffered, and is not optional. Content -/// addressing does not help here: whoever produced the blob also chose the -/// digest it is named by, so nothing about a well-formed, correctly named file -/// bounds what it expands to. A caller reading a layer passes the size its -/// descriptor claims — it is going to refuse a disagreement anyway, and -/// refusing it during decompression rather than after costs nothing. -pub fn read_blob( - source: R, - max_uncompressed: u64, -) -> Result<(Vec, LayerDigests), Error> { - let mut content = Vec::new(); - let digests = digest_blob(source, &mut content, Some(max_uncompressed))?; - Ok((content, digests)) -} - -fn digest_blob( - source: R, - sink: &mut W, - max_uncompressed: Option, -) -> Result { - let mut tap = Tap::new(source); - let mut uncompressed_size = 0u64; - let mut hasher = Sha256::new(); - - { - let mut decoder = zstd::stream::read::Decoder::new(&mut tap)?; - let mut buffer = [0u8; 64 * 1024]; - - loop { - let read = read_uninterrupted(&mut decoder, &mut buffer)?; - if read == 0 { - break; - } - - uncompressed_size += read as u64; - - // Checked before the write, so the ceiling bounds what the sink is - // ever asked to hold rather than overshooting it by a buffer. - if let Some(limit) = max_uncompressed { - if uncompressed_size > limit { - return Err(Error::DecompressedTooLarge { limit }); - } - } - - hasher.update(&buffer[..read]); - sink.write_all(&buffer[..read])?; - } - } - - // Decoding runs to EOF, so the tap has already seen the whole blob; the - // drain is a safety net in case a future decoder stops at a frame boundary - // instead. Either way the blob digest must cover every byte of the file, not - // only the part decompression happened to need. - std::io::copy(&mut tap, &mut std::io::sink())?; - - let (_, blob_digest, compressed_size) = tap.finish(); - - Ok(LayerDigests { - diff_id: Digest(hasher.finalize().into()), - blob_digest, - uncompressed_size, - compressed_size, - }) -} - -/// [`Read::read`], with `ErrorKind::Interrupted` treated as "read again". -/// -/// A bare `read` propagates `Interrupted`; only `std`'s convenience readers -/// (`io::copy`, `read_exact`, `read_to_end`) retry it, which is why the habit -/// is easy to miss in a hand-written loop. Every loop in this crate that drives -/// a source to EOF goes through here instead, because the sources are no longer -/// local files: an `oci://` publish or restore reads a network stream for hours -/// on a process that installs signal handlers, and a spurious mid-restore -/// failure there costs the whole run and reads as corruption. -/// -/// Unbounded, like `std`'s own readers: a source that returns `Interrupted` -/// forever is a source that never delivers a byte, and inventing a retry -/// ceiling here would turn "no progress" into a different error rather than -/// into progress. -pub(crate) fn read_uninterrupted( - source: &mut R, - buffer: &mut [u8], -) -> std::io::Result { - loop { - match source.read(buffer) { - Err(e) if e.kind() == ErrorKind::Interrupted => continue, - other => return other, - } - } -} - -/// Hashes and counts every byte that passes through, in either direction. -/// -/// Shared with [`crate::layer`], which puts the same tap under a streaming read -/// so that the buffered and streaming paths compute the blob digest from -/// identical bytes rather than from two similar-looking loops. -pub(crate) struct Tap { - inner: T, - hasher: Sha256, - bytes: u64, -} - -impl Tap { - pub(crate) fn new(inner: T) -> Self { - Self { - inner, - hasher: Sha256::new(), - bytes: 0, - } - } - - pub(crate) fn finish(self) -> (T, Digest, u64) { - ( - self.inner, - Digest(self.hasher.finalize().into()), - self.bytes, - ) - } -} - -impl Write for Tap { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let written = self.inner.write(buf)?; - self.hasher.update(&buf[..written]); - self.bytes += written as u64; - Ok(written) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } -} - -impl Read for Tap { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let read = self.inner.read(buf)?; - self.hasher.update(&buf[..read]); - self.bytes += read as u64; - Ok(read) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::frame::{encode, SeqWriter}; - - fn sample_sequence() -> Vec { - let mut writer = SeqWriter::new(Vec::new()); - - for i in 0..256u64 { - let record = encode(|e| { - e.array(2)? - .u64(i)? - .str("a repeated payload that compresses")?; - Ok(()) - }) - .unwrap(); - writer.write_record(&record).unwrap(); - } - - writer.into_inner() - } - - fn write_layer(content: &[u8], level: i32) -> (Vec, LayerDigests) { - let mut writer = LayerWriter::new(Vec::new(), level).unwrap(); - writer.write_all(content).unwrap(); - writer.finish().unwrap() - } - - /// Known-answer check against the sha256 of the empty string, so the digest - /// type is anchored to something outside this crate. - #[test] - fn digest_display_and_parse() { - let digest = Digest::compute([]); - assert_eq!( - digest.to_string(), - "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - ); - - let parsed: Digest = digest.to_string().parse().unwrap(); - assert_eq!(parsed, digest); - - for bad in [ - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "sha512:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "sha256:e3b0", - "sha256:E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", - ] { - assert!(bad.parse::().is_err(), "{bad} should not parse"); - } - } - - #[test] - fn one_pass_yields_both_digests() { - let content = sample_sequence(); - let (blob, digests) = write_layer(&content, 9); - - assert_eq!(digests.diff_id, Digest::compute(&content)); - assert_eq!(digests.blob_digest, Digest::compute(&blob)); - assert_eq!(digests.uncompressed_size, content.len() as u64); - assert_eq!(digests.compressed_size, blob.len() as u64); - assert!(digests.compressed_size < digests.uncompressed_size); - } - - #[test] - fn read_back_reproduces_the_digests() { - let content = sample_sequence(); - let (blob, written) = write_layer(&content, 9); - - let (read_content, read_digests) = - read_blob(blob.as_slice(), content.len() as u64).unwrap(); - assert_eq!(read_content, content); - assert_eq!(read_digests, written); - - let scanned = scan_blob(blob.as_slice()).unwrap(); - assert_eq!(scanned, written); - } - - /// Unknown #3 of the implementation plan: identity must survive compression - /// variance. The same records at three zstd levels produce one `diffId` and - /// three different blob digests — so a publisher who compresses differently - /// still reproduces the inscription, while the registry still addresses - /// each blob by its own bytes. - #[test] - fn identity_survives_compression_variance() { - let content = sample_sequence(); - - let results: Vec = [1, 9, 19] - .into_iter() - .map(|level| write_layer(&content, level).1) - .collect(); - - let diff_ids: std::collections::BTreeSet<_> = results.iter().map(|d| d.diff_id).collect(); - assert_eq!(diff_ids.len(), 1, "diffId must not depend on zstd level"); - assert_eq!( - diff_ids.into_iter().next().unwrap(), - Digest::compute(&content) - ); - - let blob_digests: std::collections::BTreeSet<_> = - results.iter().map(|d| d.blob_digest).collect(); - assert_eq!( - blob_digests.len(), - 3, - "the three levels should produce three distinct blobs; \ - if they ever collide the test has stopped proving anything" - ); - - for digests in &results { - assert_eq!(digests.uncompressed_size, content.len() as u64); - } - - // And every one of them decompresses back to the same bytes. - for level in [1, 9, 19] { - let (blob, _) = write_layer(&content, level); - let (round_tripped, _) = read_blob(blob.as_slice(), content.len() as u64).unwrap(); - assert_eq!(round_tripped, content); - } - } - - #[test] - fn empty_layer_content_is_well_defined() { - let (blob, digests) = write_layer(&[], 9); - assert_eq!(digests.diff_id, Digest::compute([])); - assert_eq!(digests.uncompressed_size, 0); - - let (content, read) = read_blob(blob.as_slice(), 0).unwrap(); - assert!(content.is_empty()); - assert_eq!(read, digests); - } - - /// A blob that has been altered must never read back as the layer it claims - /// to be. Padding is refused by the decoder (it looks for a further frame), - /// truncation leaves the frame incomplete, and a flipped byte either fails - /// to decode or yields a different `diffId` — never a silent match. - #[test] - fn a_tampered_blob_never_reads_back_clean() { - let content = sample_sequence(); - let (blob, digests) = write_layer(&content, 9); - - let mut padded = blob.clone(); - padded.extend_from_slice(b"appended"); - assert!(scan_blob(padded.as_slice()).is_err(), "padded blob"); - - assert!( - scan_blob(&blob[..blob.len() - 4]).is_err(), - "truncated blob" - ); - - let mut flipped = blob.clone(); - let middle = flipped.len() / 2; - flipped[middle] ^= 0xff; - if let Ok(read) = scan_blob(flipped.as_slice()) { - assert_ne!(read.diff_id, digests.diff_id, "corrupted blob"); - } - } - - /// A blob is content-addressed, but its producer chose both its bytes and - /// the digest naming it — so nothing about a well-formed file bounds what - /// it expands to. `read_blob` stops at the ceiling instead of - /// allocating whatever the stream asks for, and stops *during* - /// decompression: the buffer never grows past the limit even though the - /// blob is far larger. - #[test] - fn read_blob_refuses_to_expand_past_its_ceiling() { - // Compresses to a few hundred bytes; expands to 8 MiB. - let content = vec![0u8; 8 * 1024 * 1024]; - let (blob, digests) = write_layer(&content, 9); - assert!(blob.len() < content.len() / 1000, "needs a real ratio"); - - let err = read_blob(blob.as_slice(), 64 * 1024).unwrap_err(); - assert!( - matches!(err, Error::DecompressedTooLarge { limit: 65536 }), - "{err:?}" - ); - - // Exactly at the ceiling is fine; one byte under is not. - let (at_limit, _) = read_blob(blob.as_slice(), digests.uncompressed_size).unwrap(); - assert_eq!(at_limit.len(), content.len()); - - assert!(read_blob(blob.as_slice(), digests.uncompressed_size - 1).is_err()); - - // `scan_blob` holds nothing, so it stays unbounded on the same blob. - assert_eq!(scan_blob(blob.as_slice()).unwrap(), digests); - } - - /// The blob digest and the compressed size cover the whole file, which is - /// what lets a stele directory detect tampering by comparing a blob's name - /// with its content. - #[test] - fn blob_digest_covers_every_byte() { - let content = sample_sequence(); - let (blob, _) = write_layer(&content, 9); - - let read = scan_blob(blob.as_slice()).unwrap(); - assert_eq!(read.compressed_size, blob.len() as u64); - assert_eq!(read.blob_digest, Digest::compute(&blob)); - } -} diff --git a/crates/stelae/src/dir.rs b/crates/stelae/src/dir.rs deleted file mode 100644 index f13c00eda..000000000 --- a/crates/stelae/src/dir.rs +++ /dev/null @@ -1,614 +0,0 @@ -//! A stele on a local filesystem. -//! -//! ```text -//! / -//! inscription.json canonical JSON, byte-for-byte -//! blobs/sha256/ one compressed layer per file -//! ``` -//! -//! This is the smallest complete stele: enough to write one, read it back and -//! verify it end to end without a registry. It exists for two reasons. -//! -//! It is the **seam** OCI transport slots into. Blob paths are the OCI image -//! layout's (`blobs//`, named by the digest of the stored -//! bytes), so adding `oci-layout` and `index.json` later puts files *beside* -//! these rather than moving them. -//! -//! And it makes a stele **inspectable by hand** from the first commit of the -//! format — `zstd -d < blobs/sha256/ | cbor2diag` prints the records, -//! which is worth a great deal while a spec is young. -//! -//! It is not `dolos snapshot publish --output-dir`: that command belongs to the -//! Dolos profile and carries its own layer-selection and progress semantics. -//! -//! ## The one thing this layout cannot do -//! -//! An inscription lists `diffId`s — identity — and deliberately not compressed -//! digests, which are transport and live in the OCI manifest. Without a -//! manifest there is no map from a layer descriptor to the file holding it, so -//! [`SteleDir::blob_index`] rebuilds it by scanning: every blob is decompressed -//! once and indexed by the `diffId` it yields. That is a full verification pass -//! over the stele, which is the right cost for a fixture and the wrong one for -//! a registry — where the manifest supplies the map for free. -//! -//! ## Where the seam moved to -//! -//! [`LayerSpec`], [`WrittenLayer`] and [`BlobIndex`] are re-exports: they are -//! the vocabulary of [`crate::transport`], which this module was the first and -//! for a while the only implementation of. They keep their old paths so that -//! `stelae::dir::BlobIndex` still resolves. - -use std::{ - collections::BTreeMap, - fs, io, - io::Write, - path::{Path, PathBuf}, -}; - -pub use crate::transport::{BlobIndex, LayerSpec, WrittenLayer}; - -use crate::{ - digest::{digest_reader, read_blob, scan_blob, LayerDigests, LayerWriter}, - frame::{CanonicalCbor, LayerHeader, Limits, SeqReader, SeqWriter}, - inscription::LayerDescriptor, - layer::{check_header, check_identity, check_record_count, LayerReader}, - profile::Profile, - transport::{open_layer, RecordSink, SteleReader, SteleWriter}, - Digest, Error, Inscription, -}; - -/// File name of the inscription at the root of a stele directory. -pub const INSCRIPTION_FILE: &str = "inscription.json"; - -/// Directory holding content-addressed blobs, in OCI image-layout shape. -pub const BLOBS_DIR: &str = "blobs"; - -/// A layer read back from disk, verified against its descriptor. -pub struct Layer { - header: LayerHeader, - content: Vec, - header_len: usize, - digests: LayerDigests, -} - -impl std::fmt::Debug for Layer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Layer") - .field("header", &self.header) - .field("digests", &self.digests) - .finish_non_exhaustive() - } -} - -impl Layer { - pub fn header(&self) -> &LayerHeader { - &self.header - } - - pub fn digests(&self) -> &LayerDigests { - &self.digests - } - - /// The profile's content records, header excluded. Each is validated - /// canonical as it is yielded. - pub fn records(&self) -> SeqReader<'_> { - SeqReader::new(&self.content[self.header_len..]) - } - - /// The encoded header record, as it appears at the head of the sequence. - pub fn header_bytes(&self) -> &[u8] { - &self.content[..self.header_len] - } - - /// The whole uncompressed sequence, header record included. This is exactly - /// the byte string the `diffId` covers. - pub fn as_bytes(&self) -> &[u8] { - &self.content - } -} - -/// A layer of a stele directory being written, one record at a time. -/// -/// The directory's implementation of [`RecordSink`], and the mirror of -/// [`LayerReader`] on the write side. Nothing is buffered: records are framed, -/// hashed and compressed on the way past — [`LayerWriter`] was always a -/// one-pass writer — so a layer of any size costs the compressor's window and -/// one record, whatever the profile is publishing. -/// -/// A sink owns its staging file rather than borrowing the directory, so several -/// can be open at once; the counter in [`SteleDir::layer_sink`] keeps their -/// staging names apart. -/// -/// ## Nothing exists until `finish` -/// -/// A layer's name is the digest of its own compressed bytes, so it cannot be -/// known before the last record is written. Until then the layer is a staging -/// file, invisible to [`SteleDir::blob_index`], and a sink dropped without -/// [`RecordSink::finish`] takes it with it — an export that fails halfway -/// leaves no partial layer behind. Only `finish` puts a blob in the stele. -pub struct LayerSink { - /// Declared before `staging` on purpose: fields drop in declaration order, - /// so the file handle is closed before the file it names is unlinked. On - /// Windows that is the difference between removing an abandoned staging - /// file and failing to. - sequence: SeqWriter>, - staging: Staging, - root: PathBuf, - kind: String, - media_type: String, - scope: serde_json::Value, -} - -impl RecordSink for LayerSink { - fn write_record(&mut self, record: &CanonicalCbor) -> Result<(), Error> { - self.sequence.write_record(record) - } - - fn records(&self) -> u64 { - self.sequence.count() - } - - /// Close the layer: finish the compressed frame, name the blob by its own - /// digest and hand back the descriptor to put in the inscription. - /// - /// The rename is what publishes the layer, and it is the last thing that - /// happens. A failure anywhere in here leaves the stele exactly as it was: - /// the staging file is removed on the way out, the same as for a sink that - /// was simply dropped. - /// - /// ## A blob that is already there is a success, not a rename - /// - /// The destination is named by the digest of the very bytes about to be - /// moved onto it, so a destination that already exists already holds those - /// bytes — that is what content addressing means. Renaming anyway rewrites - /// a file with its own contents: hundreds of megabytes of I/O for a mainnet - /// state shard, and on Windows a failure. Every reader here opens a blob - /// with a plain [`fs::File::open`], without `FILE_SHARE_DELETE`, so - /// replacing a blob that a concurrent [`SteleReader::stream_layer`] holds - /// open is a sharing violation there and silently fine on Unix. So the blob - /// on disk is kept, the staging file goes the way an abandoned one does, - /// and the caller gets exactly the [`WrittenLayer`] it would have got had - /// it been first: `descriptor` and `digests` are computed from the record - /// stream, never from the rename. - /// - /// The existing blob is **trusted on its name, not re-read**. This writer - /// computed that digest itself, out of the bytes it had just written, a - /// microsecond earlier, and re-reading a file that may be gigabytes to - /// confirm what content addressing already asserts would put a second full - /// pass on the write path. That is a deliberate divergence from the - /// neighbour: [`SteleDir::blob_index`] re-digests every blob it finds, at a - /// cost its own documentation calls out. It is a fixture scanner rebuilding - /// a map with no manifest to lean on and no knowledge of who wrote what, - /// which is not the position of a writer holding its own digest. - /// - /// The window between the test and the rename is a TOCTOU, and an unguarded - /// one on purpose: the path *is* the digest, so the only racer that can - /// create it is another writer of byte-identical content, and both orders - /// leave the same blob. A lock here would serialize every layer write to - /// arbitrate between two callers who agree. - fn finish(self) -> Result { - let Self { - sequence, - mut staging, - root, - kind, - media_type, - scope, - } = self; - - let count = sequence.count(); - let (file, digests) = sequence.into_inner().finish()?; - file.sync_all()?; - drop(file); - - // Named by the digest of the bytes stored, per the OCI image layout. - let blob = blob_path(&root, &digests.blob_digest); - - // Already published, by this run or an earlier one: leave it alone and - // let `staging` take the duplicate with it on the way out. - if !blob.exists() { - fs::rename(&staging.path, &blob)?; - staging.published(); - } - - Ok(WrittenLayer { - descriptor: LayerDescriptor { - kind, - media_type, - diff_id: digests.diff_id, - records: count, - uncompressed_size: digests.uncompressed_size, - scope, - }, - digests, - }) - } -} - -/// The staging file a layer is written into before it has a name. -/// -/// Removing it is a `Drop` rather than a step in [`LayerSink::finish`] because -/// the case that matters is the one nobody writes code for: a producer that -/// hits an error mid-layer and returns. Nothing *reads* an abandoned staging -/// file — it sits beside `sha256/` and [`SteleDir::blob_index`] only considers -/// digest-named entries inside it — but a mainnet state shard is hundreds of -/// megabytes, and a failed export leaving sixteen of them on the disk is its -/// own incident. -struct Staging { - path: PathBuf, - published: bool, -} - -impl Staging { - fn new(path: PathBuf) -> Self { - Self { - path, - published: false, - } - } - - /// The file has been renamed to its content-addressed name: nothing is left - /// at the staging path, and an unlink of it later could only ever hit - /// somebody else's. - fn published(&mut self) { - self.published = true; - } -} - -impl Drop for Staging { - fn drop(&mut self) { - if !self.published { - // The sink has already failed or been abandoned; a failure to - // remove the file has nobody left to report it to, and the file is - // inert either way. - let _ = fs::remove_file(&self.path); - } - } -} - -/// Where a blob of `digest` lives under `root`, per the OCI image layout. -/// -/// One definition, used both to write a layer and to find it again — a stele -/// whose writer and reader disagreed about the path would be unreadable by -/// itself and perfectly readable by nobody. -fn blob_path(root: &Path, digest: &Digest) -> PathBuf { - root.join(BLOBS_DIR) - .join(Digest::ALGORITHM) - .join(digest.to_hex()) -} - -/// A stele directory. -pub struct SteleDir { - root: PathBuf, -} - -impl SteleDir { - /// Create the directory skeleton, failing if a stele is already there. - pub fn create(root: impl Into) -> Result { - let root = root.into(); - - if root.join(INSCRIPTION_FILE).exists() { - return Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - format!("{INSCRIPTION_FILE} already exists in {}", root.display()), - ))); - } - - fs::create_dir_all(root.join(BLOBS_DIR).join(Digest::ALGORITHM))?; - - Ok(Self { root }) - } - - /// Open an existing stele directory. - pub fn open(root: impl Into) -> Result { - let root = root.into(); - - if !root.join(INSCRIPTION_FILE).is_file() { - return Err(Error::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("no {INSCRIPTION_FILE} in {}", root.display()), - ))); - } - - Ok(Self { root }) - } - - pub fn root(&self) -> &Path { - &self.root - } - - pub fn blob_path(&self, digest: &Digest) -> PathBuf { - blob_path(&self.root, digest) - } -} - -impl SteleWriter for SteleDir { - type Sink = LayerSink; - - /// Open a layer and stream records into it. - /// - /// The header record is written here, before the handle is returned, so a - /// sink is always a well-formed layer in progress; the media type comes - /// from the profile and is validated against the naming rules first, so a - /// profile that claims a name it does not own is refused before anything is - /// created on disk. - /// - /// See [`LayerSink`] for what a sink that is never finished leaves behind. - fn layer_sink( - &self, - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - ) -> Result { - // A layer's file name is its digest, which is not known until the last - // byte is written, so it is staged first. The counter keeps two writers - // of the same kind apart — a profile sharding one logical layer into - // many is the normal case, not an edge one. Staging sits beside - // `sha256/` rather than in it, so a half-written layer is never mistaken - // for a blob. - static STAGING: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - - let staging = Staging::new(self.root.join(BLOBS_DIR).join(format!( - ".staging-{}-{}", - std::process::id(), - STAGING.fetch_add(1, std::sync::atomic::Ordering::Relaxed) - ))); - - // From here on every `?` unwinds through `staging`, which removes the - // file it named. - let (sequence, media_type) = open_layer(profile, spec, level, || { - Ok(fs::File::create(&staging.path)?) - })?; - - Ok(LayerSink { - sequence, - staging, - root: self.root.clone(), - kind: spec.kind.clone(), - media_type, - scope: spec.scope.clone(), - }) - } - - /// Write the inscription in canonical form and return its digest — the - /// stele's identity. - /// - /// A directory names nothing, so the profile goes unused here: there are no - /// tags to render and the file has one name the layout fixes. - fn seal(&self, _profile: &dyn Profile, inscription: &Inscription) -> Result { - let canonical = inscription.canonicalize()?; - - let mut file = fs::File::create(self.root.join(INSCRIPTION_FILE))?; - file.write_all(&canonical)?; - file.sync_all()?; - - Ok(Digest::compute(&canonical)) - } -} - -impl SteleReader for SteleDir { - type Blob = fs::File; - - /// Read and verify the inscription. - /// - /// The stored bytes must *be* the canonical encoding, not merely parse to - /// the same content: the file is what a verifier hashes, so a re-indented - /// copy carries a digest nobody else computes and is rejected rather than - /// silently repaired. - fn read_inscription(&self) -> Result { - let raw = fs::read(self.root.join(INSCRIPTION_FILE))?; - let inscription = Inscription::parse(&raw)?; - - if inscription.canonicalize()? != raw { - return Err(Error::NonCanonicalInscription); - } - - Ok(inscription) - } - - /// Rebuild the `diffId` → blob map by scanning and verifying every blob. - /// - /// Two distinct checks, in this order, because conflating them is how - /// corruption gets skipped as "not a layer": - /// - /// 1. **Content addressing** — a file named by a digest must hash to that - /// digest. This holds for every blob in the directory, layer or not, and - /// a mismatch is corruption and fails the whole index. - /// 2. **Readability** — only then is the blob decompressed. A file that is - /// not a zstd frame is simply not a layer and is skipped, which leaves - /// room for a future OCI layout's manifest and config blobs beside - /// these. - /// - /// Costs one raw pass plus one decompressing pass per blob. That is the - /// fixture's price for having no manifest; see the module documentation. - fn blob_index(&self) -> Result { - let mut index = BTreeMap::new(); - let dir = self.root.join(BLOBS_DIR).join(Digest::ALGORITHM); - - for entry in fs::read_dir(&dir)? { - let entry = entry?; - let path = entry.path(); - - if !entry.file_type()?.is_file() { - continue; - } - - // A file whose name is not a digest was not put here by this - // protocol; leave it alone. - let Some(name) = path.file_name().and_then(|n| n.to_str()) else { - continue; - }; - - let Ok(expected) = format!("{}:{name}", Digest::ALGORITHM).parse::() else { - continue; - }; - - let (actual, _) = digest_reader(fs::File::open(&path)?)?; - - if actual != expected { - return Err(Error::DigestMismatch { - subject: format!("blob {name}"), - expected: expected.to_string(), - actual: actual.to_string(), - }); - } - - match scan_blob(fs::File::open(&path)?) { - Ok(digests) => { - index.insert(digests.diff_id, digests.blob_digest); - } - // Content-addressed and intact, but not a compressed layer. - // - // Only the kinds zstd raises for input it cannot decode count - // as "not a layer": the bindings map every libzstd error code - // to `Other`, and a frame that ends early surfaces as - // `UnexpectedEof`. A `PermissionDenied` or a device error is a - // real failure, and skipping on it would drop a blob that does - // exist from the index — resurfacing later as a `LayerNotFound` - // that points at the wrong problem. - Err(Error::Io(e)) - if matches!( - e.kind(), - io::ErrorKind::InvalidData - | io::ErrorKind::UnexpectedEof - | io::ErrorKind::Other - ) => - { - continue - } - Err(e) => return Err(e), - } - } - - Ok(index.into_iter().collect()) - } - - /// Ask the filesystem how big the blob holding this layer is. - /// - /// One `stat`, and deliberately not a read: the compressed size is the - /// length of the file the blob is stored in, and a directory needs no - /// manifest to learn it. This is *not* the double-read a directory pays for - /// [`SteleDir::blob_index`] — nothing is opened and nothing is - /// decompressed. - /// - /// A layer the index does not place is `None` rather than an error, because - /// the caller asking this is estimating a download and not performing one. - /// The refusal belongs to [`SteleDir::stream_layer`], which is where the - /// missing layer actually stops something. - fn compressed_size( - &self, - index: &BlobIndex, - descriptor: &LayerDescriptor, - ) -> Result, Error> { - let Some(blob) = index.blob_for(&descriptor.diff_id) else { - return Ok(None); - }; - - match fs::metadata(self.blob_path(&blob)) { - Ok(metadata) => Ok(Some(metadata.len())), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(e.into()), - } - } - - /// Stream one layer's records without holding it. - /// - /// The same verification as [`SteleDir::read_layer`] — the checks are one - /// implementation, called from both — but spread across the read: the - /// header on construction, the decompression ceiling as the stream - /// advances, and the identity digest, size and record count in - /// [`LayerReader::finish`]. Records are therefore consumable *before* the - /// layer is proven; see the [`crate::layer`] module documentation for the - /// discipline that requires of a consumer. - fn stream_layer( - &self, - index: &BlobIndex, - profile: &dyn Profile, - descriptor: &LayerDescriptor, - limits: Limits, - ) -> Result, Error> { - let path = self.blob_of(index, descriptor)?; - - LayerReader::new(fs::File::open(&path)?, profile, descriptor, limits) - } -} - -impl SteleDir { - /// Locate the blob holding a layer, or say which layer is missing. - fn blob_of(&self, index: &BlobIndex, descriptor: &LayerDescriptor) -> Result { - let blob_digest = - index - .blob_for(&descriptor.diff_id) - .ok_or_else(|| Error::LayerNotFound { - kind: descriptor.kind.clone(), - diff_id: descriptor.diff_id.to_string(), - })?; - - Ok(self.blob_path(&blob_digest)) - } - - /// Read one layer, verifying it against its descriptor and its own header. - /// - /// Everything the descriptor claims is checked: the identity digest, the - /// uncompressed size, the record count, and that the header record inside - /// the blob names the same profile and kind. A layer that disagrees with - /// the document that points at it is refused. - /// - /// The layer is held whole, which is what makes this a fixture: the - /// descriptor's `uncompressedSize` is allocated outright, and on a Dolos - /// state shard that is 402 MB. Callers that only need to walk the records - /// want [`SteleDir::stream_layer`], which checks exactly the same claims - /// out of a bounded window. - pub fn read_layer( - &self, - index: &BlobIndex, - profile: &dyn Profile, - descriptor: &LayerDescriptor, - ) -> Result { - let path = self.blob_of(index, descriptor)?; - - // The descriptor's claim doubles as the ceiling on decompression. A - // blob that expands past it is refused mid-stream instead of being - // buffered whole and rejected by the size check below — same verdict, - // bounded cost. - let (content, digests) = read_blob(fs::File::open(&path)?, descriptor.uncompressed_size)?; - - check_identity(&digests, descriptor)?; - - let header_len = match SeqReader::new(&content).next() { - Some(Ok(record)) => record.len(), - Some(Err(e)) => return Err(e), - None => { - return Err(Error::LayerMismatch { - kind: descriptor.kind.clone(), - reason: "layer is empty; every layer starts with a header record".to_owned(), - }) - } - }; - - let header = LayerHeader::decode(&content[..header_len])?; - - check_header(&header, profile, descriptor)?; - - // Counted by iterating rather than with `count()`: `SeqReader` reports a - // malformed record as one `Err` item and then ends, so counting items - // would tally the failure as a record and drop the error with it. A - // descriptor written to match that inflated number would then read back - // clean. - let mut records = 1u64; - - for record in SeqReader::new(&content[header_len..]) { - record?; - records += 1; - } - - check_record_count(records, descriptor)?; - - Ok(Layer { - header, - content, - header_len, - digests, - }) - } -} diff --git a/crates/stelae/src/frame.rs b/crates/stelae/src/frame.rs deleted file mode 100644 index de87929f5..000000000 --- a/crates/stelae/src/frame.rs +++ /dev/null @@ -1,1906 +0,0 @@ -//! Deterministic CBOR sequence framing. -//! -//! Every layer blob is, uncompressed, a CBOR sequence (RFC 8742): concatenated -//! CBOR data items with no outer container. The first item is the -//! protocol-owned [`LayerHeader`]; the rest are the profile's content records, -//! which this crate never interprets. -//! -//! The encoding profile is RFC 8949 §4.2.1 ("core deterministic encoding"), -//! narrowed by the spec to the closed set the format needs: -//! -//! - integers in shortest form, -//! - definite lengths only, -//! - map keys sorted bytewise by their encoded form, no duplicates, -//! - no floats, no tags, no `undefined`, no simple values beyond -//! `false`/`true`/`null`, -//! - text strings valid UTF-8. -//! -//! Both directions enforce it. [`CanonicalCbor`] cannot be constructed from -//! bytes that violate the profile, so a record is validated before it is ever -//! written; [`SeqReader`] validates every record it yields. The read-side check -//! is not belt-and-braces: a layer's identity is the sha256 of these bytes, so -//! a producer that emits a non-canonical encoding would publish a diffId that -//! no independent re-encoding can reproduce. Rejecting it at the door is what -//! keeps "reproduce the digest" a decidable claim. -//! -//! ## Two readers, one validator -//! -//! A sequence is read either from bytes already in hand ([`SeqReader`], which -//! borrows out of the slice it was given) or from a stream ([`RecordReader`], -//! which refills a bounded window). They are two entry points, not two -//! implementations: both walk items with the same scanner, so a rule can -//! never hold on one path and lapse on the other. [`scan_item`] and -//! [`measure_item`] are that scanner's two exits — the first for callers that -//! hold the whole item, the second for callers that must decide whether an item -//! is worth holding at all. - -use std::{ - io::{Read, Write}, - ops::Range, -}; - -use crate::{digest::read_uninterrupted, Error, LAYER_FORMAT_VERSION}; - -/// Maximum nesting depth accepted by the canonical-form scanner. -/// -/// Layer records are flat by construction; the bound exists so that a hostile -/// blob cannot drive the scanner into a stack overflow. -pub const MAX_NESTING_DEPTH: usize = 64; - -/// A CBOR data item known to be in the protocol's deterministic encoding. -/// -/// The only way to obtain one is [`CanonicalCbor::new`] or [`encode`], both of -/// which validate. Everything downstream — records, layer headers, a profile's -/// opaque `scope` — carries the invariant in its type rather than by -/// convention. -#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct CanonicalCbor(Vec); - -impl CanonicalCbor { - /// Validate `bytes` as exactly one canonical CBOR data item. - /// - /// Trailing bytes are an error: a record is one item, and silently ignoring - /// a tail would let two different blobs claim the same logical content. - pub fn new(bytes: impl Into>) -> Result { - let bytes = bytes.into(); - let len = scan_item(&bytes)?; - - if len != bytes.len() { - return Err(Error::TrailingCbor { - trailing: bytes.len() - len, - }); - } - - Ok(Self(bytes)) - } - - pub fn as_bytes(&self) -> &[u8] { - &self.0 - } - - pub fn into_bytes(self) -> Vec { - self.0 - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// A decoder positioned at the start of the item. - /// - /// Profiles use this to read their own records back. The protocol only ever - /// uses it for the fields of the layer header. - pub fn decoder(&self) -> minicbor::Decoder<'_> { - minicbor::Decoder::new(&self.0) - } -} - -impl std::fmt::Debug for CanonicalCbor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "CanonicalCbor(0x{})", hex::encode(&self.0)) - } -} - -impl AsRef<[u8]> for CanonicalCbor { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -/// Encode one canonical CBOR item with `minicbor`, validating the result. -/// -/// `minicbor` already emits shortest-form integers and definite lengths, so the -/// validation is a guard rather than a fixer: it catches the two things the -/// encoder cannot know are wrong — an indefinite-length container opened -/// explicitly, and map keys written out of order. -/// -/// ``` -/// let record = stelae::frame::encode(|e| { -/// e.array(2)?.u64(42)?.str("hello")?; -/// Ok(()) -/// }) -/// .unwrap(); -/// assert_eq!(record.as_bytes(), &[0x82, 0x18, 0x2a, 0x65, b'h', b'e', b'l', b'l', b'o']); -/// ``` -pub fn encode(f: F) -> Result -where - F: FnOnce( - &mut minicbor::Encoder>, - ) -> Result<(), minicbor::encode::Error>, -{ - let mut encoder = minicbor::Encoder::new(Vec::new()); - f(&mut encoder).map_err(|e| Error::CborEncode(e.to_string()))?; - CanonicalCbor::new(encoder.into_writer()) -} - -/// Validate the canonical CBOR data item at the start of `bytes` and return its -/// length in bytes. Trailing bytes are left for the caller — this is what makes -/// a CBOR *sequence* walkable. -pub fn scan_item(bytes: &[u8]) -> Result { - let mut scanner = Scanner::new(bytes); - scanner.item(0)?; - Ok(scanner.pos) -} - -/// What scanning the start of `bytes` established about the item there. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Measure { - /// One complete, canonical item occupying the first `len` bytes. - Complete { len: usize }, - /// A canonical *prefix* of an item: nothing seen so far violates the - /// profile, but the item does not end within `bytes`. - Incomplete { - /// Lower bound, in bytes, on the whole item. - /// - /// Not an estimate and not a guess about what follows: it is what the - /// length prefixes already read oblige the encoder to deliver — the - /// bytes a string header claimed, plus one byte for each element of an - /// enclosing array or map that has not been reached. It is therefore - /// always greater than `bytes.len()`, which is what lets a refill loop - /// make progress. - required: u64, - }, -} - -/// Validate the start of `bytes` as a canonical CBOR item, tolerating an item -/// that runs off the end. -/// -/// The reason this exists rather than [`scan_item`] plus a derivation at the -/// call site: a streaming reader has to decide *whether to hold* a record -/// before it holds it, and [`Error::TruncatedCbor`] reports a local need — one -/// byte for a missing head, that chunk's `n` for a string body, a `usize::MAX` -/// sentinel for a length prefix beyond the platform's reach. Reconstructing the -/// record's total size from those would put length arithmetic in a second -/// place, which is exactly how two readers start disagreeing about what is -/// canonical. -/// -/// A violation of the deterministic profile is still an error, not an -/// `Incomplete`: no quantity of further bytes rehabilitates a float, a tag or a -/// non-shortest integer. -/// -/// ``` -/// use stelae::frame::{measure_item, Measure}; -/// -/// // A byte string of four bytes, of which two arrived. -/// assert_eq!( -/// measure_item(&[0x44, 0x01, 0x02]).unwrap(), -/// Measure::Incomplete { required: 5 }, -/// ); -/// // The same item, complete, with the next record's first byte behind it. -/// assert_eq!( -/// measure_item(&[0x44, 0x01, 0x02, 0x03, 0x04, 0x00]).unwrap(), -/// Measure::Complete { len: 5 }, -/// ); -/// ``` -pub fn measure_item(bytes: &[u8]) -> Result { - let mut scanner = Scanner::new(bytes); - - match scanner.item(0) { - Ok(()) => Ok(Measure::Complete { len: scanner.pos }), - // Every `TruncatedCbor` this scanner raises means "the item needs bytes - // that are not here", and `required` is what the raising site recorded. - Err(Error::TruncatedCbor { .. }) => Ok(Measure::Incomplete { - required: scanner.required, - }), - Err(e) => Err(e), - } -} - -struct Scanner<'a> { - bytes: &'a [u8], - pos: usize, - /// Lower bound on the size of the item under scan. Only meaningful once a - /// [`Error::TruncatedCbor`] has been raised; see [`Measure::Incomplete`]. - required: u64, -} - -impl<'a> Scanner<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { - bytes, - pos: 0, - required: 0, - } - } - - fn non_canonical(offset: usize, reason: impl Into) -> Error { - Error::NonCanonicalCbor { - offset, - reason: reason.into(), - } - } - - /// Report an item that runs off the end, recording what it still needs. - /// - /// The need is absolute (`offset` is measured from the start of the item), - /// so the recorded bound survives the unwind through enclosing containers, - /// which only add to it. - fn truncated(&mut self, offset: usize, expected: usize) -> Error { - self.required = (offset as u64).saturating_add(expected as u64); - - Error::TruncatedCbor { offset, expected } - } - - /// Add what an enclosing container still owes to the bound, on the way out - /// of a truncation. Every unread element is at least one byte. - fn note_pending(&mut self, e: &Error, items: u64) { - if matches!(e, Error::TruncatedCbor { .. }) { - self.required = self.required.saturating_add(items); - } - } - - fn byte(&mut self) -> Result { - let Some(b) = self.bytes.get(self.pos).copied() else { - return Err(self.truncated(self.pos, 1)); - }; - - self.pos += 1; - - Ok(b) - } - - fn take(&mut self, n: usize) -> Result<&'a [u8], Error> { - let Some(end) = self.pos.checked_add(n) else { - return Err(self.truncated(self.pos, n)); - }; - - let Some(slice) = self.bytes.get(self.pos..end) else { - return Err(self.truncated(self.pos, n)); - }; - - self.pos = end; - - Ok(slice) - } - - /// Read the argument of a head byte, rejecting every non-shortest encoding. - fn argument(&mut self, offset: usize, ai: u8) -> Result { - match ai { - 0..=23 => Ok(u64::from(ai)), - 24 => { - let v = u64::from(self.byte()?); - if v < 24 { - return Err(Self::non_canonical( - offset, - format!("value {v} must be encoded in the head byte, not as uint8"), - )); - } - Ok(v) - } - 25 => { - let v = u64::from(u16::from_be_bytes(self.take(2)?.try_into().unwrap())); - if v <= u64::from(u8::MAX) { - return Err(Self::non_canonical( - offset, - format!("value {v} must be encoded as uint8, not uint16"), - )); - } - Ok(v) - } - 26 => { - let v = u64::from(u32::from_be_bytes(self.take(4)?.try_into().unwrap())); - if v <= u64::from(u16::MAX) { - return Err(Self::non_canonical( - offset, - format!("value {v} must be encoded as uint16, not uint32"), - )); - } - Ok(v) - } - 27 => { - let v = u64::from_be_bytes(self.take(8)?.try_into().unwrap()); - if v <= u64::from(u32::MAX) { - return Err(Self::non_canonical( - offset, - format!("value {v} must be encoded as uint32, not uint64"), - )); - } - Ok(v) - } - 28..=30 => Err(Self::non_canonical( - offset, - format!("reserved additional information {ai}"), - )), - 31 => Err(Self::non_canonical( - offset, - "indefinite lengths are excluded by the deterministic profile", - )), - _ => unreachable!("additional information is 5 bits"), - } - } - - fn item(&mut self, depth: usize) -> Result<(), Error> { - if depth >= MAX_NESTING_DEPTH { - return Err(Error::CborTooDeep { - limit: MAX_NESTING_DEPTH, - }); - } - - let offset = self.pos; - let head = self.byte()?; - let major = head >> 5; - let ai = head & 0x1f; - - // Major type 7 carries simple values and floats; its additional - // information is not a length, so it never goes through `argument`. - if major == 7 { - return match ai { - 20..=22 => Ok(()), // false, true, null - 23 => Err(Self::non_canonical( - offset, - "`undefined` is excluded by the deterministic profile", - )), - 24 => Err(Self::non_canonical( - offset, - "simple values other than false/true/null are excluded", - )), - 25..=27 => Err(Self::non_canonical( - offset, - "floating-point values are excluded by the deterministic profile", - )), - 31 => Err(Self::non_canonical( - offset, - "`break` outside an indefinite-length item", - )), - _ => Err(Self::non_canonical( - offset, - format!("reserved simple value {ai}"), - )), - }; - } - - if major == 6 { - return Err(Self::non_canonical( - offset, - "tags are excluded by the deterministic profile", - )); - } - - let arg = self.argument(offset, ai)?; - - match major { - 0 | 1 => Ok(()), - 2 => { - let len = self.string_len(offset, arg)?; - self.take(len)?; - Ok(()) - } - 3 => { - let len = self.string_len(offset, arg)?; - let raw = self.take(len)?; - std::str::from_utf8(raw) - .map_err(|e| Self::non_canonical(offset, format!("invalid utf-8: {e}")))?; - Ok(()) - } - 4 => { - // `remaining` counts the elements *after* the one being scanned, - // so a truncation deep inside can be charged for the ones that - // never got their turn. - for remaining in (0..arg).rev() { - if let Err(e) = self.item(depth + 1) { - self.note_pending(&e, remaining); - return Err(e); - } - } - Ok(()) - } - 5 => self.map(arg, depth), - _ => unreachable!("major types 6 and 7 handled above"), - } - } - - /// Length of a byte or text string, as a `usize` this platform can address. - /// - /// A prefix beyond `usize` is reported as truncation with a sentinel - /// `expected`: the item is unreachable on this platform whatever follows. - /// The recorded bound stays exact, so a caller enforcing a ceiling refuses - /// it for its real size rather than for the sentinel. - fn string_len(&mut self, offset: usize, arg: u64) -> Result { - usize::try_from(arg).map_err(|_| { - self.required = (self.pos as u64).saturating_add(arg); - - Error::TruncatedCbor { - offset, - expected: usize::MAX, - } - }) - } - - fn map(&mut self, entries: u64, depth: usize) -> Result<(), Error> { - let all = self.bytes; - let mut previous: Option<&'a [u8]> = None; - - for remaining in (0..entries).rev() { - let key_start = self.pos; - - if let Err(e) = self.item(depth + 1) { - // Every entry left owes a key and a value; this one still owes - // its value. - self.note_pending(&e, remaining.saturating_mul(2).saturating_add(1)); - return Err(e); - } - - let key = &all[key_start..self.pos]; - - if let Some(previous) = previous { - match key.cmp(previous) { - std::cmp::Ordering::Less => { - return Err(Self::non_canonical( - key_start, - "map keys must be sorted bytewise by their encoded form", - )) - } - std::cmp::Ordering::Equal => { - return Err(Self::non_canonical(key_start, "duplicate map key")) - } - std::cmp::Ordering::Greater => {} - } - } - - previous = Some(key); - - if let Err(e) = self.item(depth + 1) { - self.note_pending(&e, remaining.saturating_mul(2)); - return Err(e); - } - } - - Ok(()) - } -} - -/// Writes a CBOR sequence, counting the records it emits and holding each to -/// the ceiling its reader will apply. -pub struct SeqWriter { - inner: W, - count: u64, - written: u64, - max_record: usize, -} - -impl SeqWriter { - /// A writer holding records to [`DEFAULT_MAX_RECORD`]. - pub fn new(inner: W) -> Self { - Self::with_max_record(inner, DEFAULT_MAX_RECORD) - } - - /// A writer holding records to `max_record`. - /// - /// `max_record` must be the ceiling the eventual reader is given, which is - /// why the profile owns the number rather than each end picking its own — - /// see [`crate::profile::Profile::max_record`]. - /// - /// Floored at one byte to agree with [`Limits::normalized`], which floors a - /// reader's ceiling the same way. Zero is not a meaningful ceiling for - /// either end — the smallest CBOR item is one byte — so what matters about - /// it is not which behaviour it selects but that both ends select the same - /// one. A writer keeping a literal zero would refuse every record a reader - /// at that ceiling goes on to accept, which is the disagreement this type - /// exists to prevent. - pub fn with_max_record(inner: W, max_record: usize) -> Self { - Self { - inner, - count: 0, - written: 0, - max_record: max_record.max(1), - } - } - - /// Append one record. The [`CanonicalCbor`] type is the proof that it is in - /// deterministic form, so nothing is re-checked here. - /// - /// Its *size* is checked, and that check is the writer's whole reason for - /// knowing a ceiling. A record past it is refused here rather than written - /// and discovered by whoever tries to read the layer back: a stele whose - /// records no reader will accept is not a stele, and the publisher is the - /// only party positioned to say so while the fix is still cheap. Reported - /// in the record's offset within the layer, the same coordinate - /// [`RecordReader`] fails in, so the two ends name the same byte. - pub fn write_record(&mut self, record: &CanonicalCbor) -> Result<(), Error> { - let bytes = record.as_bytes(); - - if bytes.len() > self.max_record { - return Err(Error::RecordTooLarge { - offset: self.written as usize, - required: bytes.len() as u64, - limit: self.max_record, - }); - } - - self.inner.write_all(bytes)?; - self.count += 1; - self.written += bytes.len() as u64; - - Ok(()) - } - - /// Number of records written so far, header included. - pub fn count(&self) -> u64 { - self.count - } - - pub fn into_inner(self) -> W { - self.inner - } -} - -/// Walks a CBOR sequence, validating each item's canonical form. -/// -/// Yields borrowed record bytes. Once an item fails validation the iterator is -/// exhausted — a sequence is not resynchronizable, and pretending otherwise -/// would hand the caller records from an arbitrary offset. -pub struct SeqReader<'a> { - bytes: &'a [u8], - offset: usize, - failed: bool, -} - -impl<'a> SeqReader<'a> { - pub fn new(bytes: &'a [u8]) -> Self { - Self { - bytes, - offset: 0, - failed: false, - } - } - - /// Byte offset of the next record. - pub fn offset(&self) -> usize { - self.offset - } -} - -impl<'a> Iterator for SeqReader<'a> { - type Item = Result<&'a [u8], Error>; - - fn next(&mut self) -> Option { - if self.failed || self.offset >= self.bytes.len() { - return None; - } - - let rest = &self.bytes[self.offset..]; - - match scan_item(rest) { - Ok(len) => { - let record = &rest[..len]; - self.offset += len; - Some(Ok(record)) - } - Err(e) => { - self.failed = true; - Some(Err(at_sequence_offset(e, self.offset))) - } - } - } -} - -impl std::iter::FusedIterator for SeqReader<'_> {} - -/// Restate a record-local error in the coordinates of the whole sequence. -/// -/// Both readers report positions the same way — an offset a reader hands back -/// is an offset into the layer, which is what a `diffId` covers and therefore -/// the only frame of reference two implementations can agree on. -fn at_sequence_offset(e: Error, base: usize) -> Error { - match e { - Error::NonCanonicalCbor { offset, reason } => Error::NonCanonicalCbor { - offset: base.saturating_add(offset), - reason, - }, - Error::TruncatedCbor { offset, expected } => Error::TruncatedCbor { - offset: base.saturating_add(offset), - expected, - }, - other => other, - } -} - -/// Largest single record [`RecordReader`] accepts by default: 16 MiB. -/// -/// Two orders of magnitude above the largest record any profile plans to write -/// (a Cardano block is order 100 KB), and small enough that refusing a hostile -/// length prefix costs a bounded allocation. A profile whose records genuinely -/// approach this is telling its publisher something, not this crate: raise the -/// limit deliberately through [`Limits`], with the profile's own reason. -pub const DEFAULT_MAX_RECORD: usize = 16 * 1024 * 1024; - -/// Refill window [`RecordReader`] starts with: 64 KiB, matching the buffers the -/// digest pipeline reads through. -pub const DEFAULT_WINDOW: usize = 64 * 1024; - -/// What a streaming read is allowed to hold. -/// -/// The bound the format actually needs is *one record fits in memory; a layer -/// does not*. Both fields exist to keep that promise checkable rather than -/// hoped for. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Limits { - /// Largest single record accepted. Checked against what a record's length - /// prefixes claim *before* a buffer grows to hold it, so a corrupt or - /// hostile prefix costs a comparison rather than an allocation. - pub max_record: usize, - /// Size the refill window starts at. It grows — never past `max_record` — - /// only for a record that does not fit, and never shrinks back, so peak - /// memory is `max(window, the largest record actually read)`. - pub window: usize, -} - -impl Default for Limits { - fn default() -> Self { - Self { - max_record: DEFAULT_MAX_RECORD, - window: DEFAULT_WINDOW, - } - } -} - -impl Limits { - /// A window larger than the record ceiling would buy nothing — nothing that - /// large is ever yielded — so it is clamped rather than refused. - fn normalized(self) -> Self { - Self { - max_record: self.max_record.max(1), - window: self.window.clamp(1, self.max_record.max(1)), - } - } -} - -/// Walks a CBOR sequence arriving over a stream, validating each record's -/// canonical form and holding no more than one record at a time. -/// -/// This is [`SeqReader`]'s guarantee list on a source too large to hold: every -/// record is validated against the deterministic profile before it is yielded, -/// and a bad record ends the walk rather than being skipped — a CBOR sequence -/// has no frame markers, so continuing past one would hand the caller records -/// from an arbitrary offset. -/// -/// It adds the guarantee a stream needs and a slice does not: a record's size -/// is checked against [`Limits::max_record`] as soon as its length prefixes -/// claim it, and before the window grows. Nothing here trusts a length prefix -/// far enough to allocate for it. -/// -/// Records are borrowed out of the window, so this is not an [`Iterator`]: the -/// borrow has to end before the window can be refilled. Callers loop on -/// [`RecordReader::next_record`]. -/// -/// ``` -/// use stelae::frame::{encode, RecordReader, SeqWriter}; -/// -/// let mut writer = SeqWriter::new(Vec::new()); -/// for i in 0..3u64 { -/// writer.write_record(&encode(|e| { e.u64(i)?; Ok(()) }).unwrap()).unwrap(); -/// } -/// let sequence = writer.into_inner(); -/// -/// let mut reader = RecordReader::new(std::io::Cursor::new(&sequence)); -/// let mut seen = 0; -/// while let Some(record) = reader.next_record() { -/// record.unwrap(); -/// seen += 1; -/// } -/// assert_eq!(seen, 3); -/// ``` -pub struct RecordReader { - source: R, - /// The refill window. Its length *is* its capacity: everything from - /// `filled` on is scratch space for the next read. - window: Vec, - /// Bytes of `window` that hold data read from the source. - filled: usize, - /// Where the next record starts within `window`. - cursor: usize, - limits: Limits, - /// Offset of the next record within the sequence, for error reporting. - offset: usize, - count: u64, - eof: bool, - failed: bool, -} - -impl RecordReader { - /// A reader with [`Limits::default`]. - pub fn new(source: R) -> Self { - Self::with_limits(source, Limits::default()) - } - - pub fn with_limits(source: R, limits: Limits) -> Self { - let limits = limits.normalized(); - - Self { - source, - window: vec![0u8; limits.window], - filled: 0, - cursor: 0, - limits, - offset: 0, - count: 0, - eof: false, - failed: false, - } - } - - /// The next record, or `None` at the end of the sequence. - /// - /// Once a record fails validation the reader is done: every later call - /// returns `None`, the same way [`SeqReader`] stops. - pub fn next_record(&mut self) -> Option> { - if self.failed { - return None; - } - - match self.advance() { - Ok(Some(record)) => { - self.count += 1; - Some(Ok(&self.window[record])) - } - Ok(None) => None, - Err(e) => { - self.failed = true; - Some(Err(e)) - } - } - } - - /// Number of records yielded so far. - /// - /// A record that failed validation is not counted — it is not a record. - /// Counting the failure instead is how a corrupt layer gets waved through - /// by a descriptor written to match the inflated number. - pub fn count(&self) -> u64 { - self.count - } - - /// Byte offset of the next record within the sequence. - pub fn offset(&self) -> usize { - self.offset - } - - /// Whether the walk ended because a record failed validation. - pub fn failed(&self) -> bool { - self.failed - } - - /// The source, once the walk is over. [`crate::layer`] uses it to reach the - /// digests the pipeline underneath accumulated. - pub fn into_inner(self) -> R { - self.source - } - - /// Locate the next record in the window, refilling until it is whole. - fn advance(&mut self) -> Result>, Error> { - loop { - let pending = &self.window[self.cursor..self.filled]; - - if pending.is_empty() { - if self.eof { - return Ok(None); - } - - self.fill(0)?; - continue; - } - - match measure_item(pending) { - Ok(Measure::Complete { len }) => { - // The window can hold more than one record, so a complete - // record is still checked: the ceiling is a promise about - // what a caller is handed, not only about what was - // allocated to get there. - self.check_ceiling(len as u64)?; - - let record = self.cursor..self.cursor + len; - self.cursor += len; - self.offset += len; - - return Ok(Some(record)); - } - Ok(Measure::Incomplete { required }) => { - // Before the window grows, not after: this is the whole - // point of `measure_item` reporting a size. - self.check_ceiling(required)?; - - if self.eof { - // Out of bytes with an unfinished record. Re-scan - // strictly so the error is the one the slice reader - // would have raised on the same bytes. - let e = scan_item(pending).expect_err("the item is incomplete"); - return Err(at_sequence_offset(e, self.offset)); - } - - let required = usize::try_from(required) - .expect("checked against the ceiling, which is a usize"); - - self.fill(required)?; - } - Err(e) => return Err(at_sequence_offset(e, self.offset)), - } - } - } - - fn check_ceiling(&self, required: u64) -> Result<(), Error> { - if required > self.limits.max_record as u64 { - return Err(Error::RecordTooLarge { - offset: self.offset, - required, - limit: self.limits.max_record, - }); - } - - Ok(()) - } - - /// Make room for a record of at least `required` bytes and read into it. - /// - /// Compacting first is what keeps the promise in [`Limits::window`]: the - /// bytes of the record under construction move to the front, so the window - /// holds one record plus whatever of the next one came along for the ride, - /// never a growing tail of records already handed out. - fn fill(&mut self, required: usize) -> Result<(), Error> { - if self.cursor > 0 { - self.window.copy_within(self.cursor..self.filled, 0); - self.filled -= self.cursor; - self.cursor = 0; - } - - if self.filled == self.window.len() { - // A full window and still no record: the record is at least one - // byte larger than everything held, which is the bound to check. - self.check_ceiling(self.filled as u64 + 1)?; - - let grown = self - .window - .len() - .saturating_mul(2) - .clamp(required.max(self.filled + 1), self.limits.max_record); - - self.window.resize(grown, 0); - } else if required > self.window.len() { - self.window.resize(required.min(self.limits.max_record), 0); - } - - let read = read_uninterrupted(&mut self.source, &mut self.window[self.filled..])?; - - if read == 0 { - self.eof = true; - } else { - self.filled += read; - } - - Ok(()) - } -} - -/// The first record of every layer, defined by the protocol so a blob stays -/// interpretable when detached from the registry that served it. -/// -/// `[format_version, profile: tstr, kind: tstr, scope: any]` -/// -/// `scope` is the profile's, and stays opaque: the protocol validates that it -/// is canonical CBOR and copies it, never reading inside. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LayerHeader { - pub format_version: u64, - pub profile: String, - pub kind: String, - pub scope: CanonicalCbor, -} - -impl LayerHeader { - /// A header at the format version this implementation writes. - pub fn new(profile: impl Into, kind: impl Into, scope: CanonicalCbor) -> Self { - Self { - format_version: LAYER_FORMAT_VERSION, - profile: profile.into(), - kind: kind.into(), - scope, - } - } - - pub fn encode(&self) -> Result { - let mut out = Vec::new(); - - { - let mut encoder = minicbor::Encoder::new(&mut out); - encoder - .array(4) - .and_then(|e| e.u64(self.format_version)) - .and_then(|e| e.str(&self.profile)) - .and_then(|e| e.str(&self.kind)) - .map_err(|e| Error::CborEncode(e.to_string()))?; - } - - out.extend_from_slice(self.scope.as_bytes()); - - CanonicalCbor::new(out) - } - - /// Parse a header record, failing closed on a format version this - /// implementation does not implement. - pub fn decode(record: &[u8]) -> Result { - let record = CanonicalCbor::new(record.to_vec())?; - let bytes = record.as_bytes(); - let mut decoder = record.decoder(); - - let fields = decoder - .array() - .map_err(|e| Error::MalformedHeader(e.to_string()))? - .ok_or_else(|| Error::MalformedHeader("indefinite-length array".into()))?; - - if fields != 4 { - return Err(Error::MalformedHeader(format!( - "expected 4 fields, found {fields}" - ))); - } - - let format_version = decoder - .u64() - .map_err(|e| Error::MalformedHeader(format!("format_version: {e}")))?; - - if format_version != LAYER_FORMAT_VERSION { - return Err(Error::MalformedHeader(format!( - "unsupported layer format version {format_version}; \ - this implementation implements {LAYER_FORMAT_VERSION}" - ))); - } - - let profile = decoder - .str() - .map_err(|e| Error::MalformedHeader(format!("profile: {e}")))? - .to_owned(); - - let kind = decoder - .str() - .map_err(|e| Error::MalformedHeader(format!("kind: {e}")))? - .to_owned(); - - let scope = CanonicalCbor::new(bytes[decoder.position()..].to_vec())?; - - Ok(Self { - format_version, - profile, - kind, - scope, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `minicbor` is the encoder the protocol hands to profiles. Unknown #1 of - /// the implementation plan asks whether it emits RFC 8949 §4.2.1 canonical - /// output by default; these are the boundary values where a non-canonical - /// encoder would differ, checked against bytes taken from RFC 8949 §3 and - /// Appendix A. - #[test] - fn minicbor_emits_shortest_form_integers() { - let cases: &[(u64, &[u8])] = &[ - (0, &[0x00]), - (1, &[0x01]), - (10, &[0x0a]), - (23, &[0x17]), - (24, &[0x18, 0x18]), - (25, &[0x18, 0x19]), - (100, &[0x18, 0x64]), - (255, &[0x18, 0xff]), - (256, &[0x19, 0x01, 0x00]), - (1000, &[0x19, 0x03, 0xe8]), - (65535, &[0x19, 0xff, 0xff]), - (65536, &[0x1a, 0x00, 0x01, 0x00, 0x00]), - (1_000_000, &[0x1a, 0x00, 0x0f, 0x42, 0x40]), - (4_294_967_295, &[0x1a, 0xff, 0xff, 0xff, 0xff]), - ( - 4_294_967_296, - &[0x1b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00], - ), - ( - u64::MAX, - &[0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], - ), - ]; - - for (value, expected) in cases { - let encoded = encode(|e| { - e.u64(*value)?; - Ok(()) - }) - .unwrap(); - assert_eq!(encoded.as_bytes(), *expected, "u64 {value}"); - } - - let negatives: &[(i64, &[u8])] = &[ - (-1, &[0x20]), - (-10, &[0x29]), - (-24, &[0x37]), - (-25, &[0x38, 0x18]), - (-100, &[0x38, 0x63]), - (-1000, &[0x39, 0x03, 0xe7]), - ]; - - for (value, expected) in negatives { - let encoded = encode(|e| { - e.i64(*value)?; - Ok(()) - }) - .unwrap(); - assert_eq!(encoded.as_bytes(), *expected, "i64 {value}"); - } - } - - #[test] - fn minicbor_emits_definite_lengths() { - let encoded = encode(|e| { - e.array(3)?.u64(1)?.u64(2)?.u64(3)?; - Ok(()) - }) - .unwrap(); - assert_eq!(encoded.as_bytes(), &[0x83, 0x01, 0x02, 0x03]); - - let encoded = encode(|e| { - e.bytes(&[0x01, 0x02, 0x03, 0x04])?; - Ok(()) - }) - .unwrap(); - assert_eq!(encoded.as_bytes(), &[0x44, 0x01, 0x02, 0x03, 0x04]); - - let encoded = encode(|e| { - e.map(2)?.str("a")?.u64(1)?.str("b")?.u64(2)?; - Ok(()) - }) - .unwrap(); - assert_eq!( - encoded.as_bytes(), - &[0xa2, 0x61, b'a', 0x01, 0x61, b'b', 0x02] - ); - } - - /// The encoder cannot know that an explicitly opened indefinite-length - /// container is wrong; the validation in [`encode`] is what catches it. - #[test] - fn encode_rejects_indefinite_lengths_from_minicbor() { - let err = encode(|e| { - e.begin_array()?.u64(1)?.end()?; - Ok(()) - }) - .unwrap_err(); - - assert!( - matches!(err, Error::NonCanonicalCbor { .. }), - "expected non-canonical, got {err:?}" - ); - } - - /// Likewise for map keys written out of order. - #[test] - fn encode_rejects_unsorted_map_keys() { - let err = encode(|e| { - e.map(2)?.str("b")?.u64(1)?.str("a")?.u64(2)?; - Ok(()) - }) - .unwrap_err(); - - assert!( - matches!(&err, Error::NonCanonicalCbor { reason, .. } if reason.contains("sorted")), - "expected sort violation, got {err:?}" - ); - } - - #[test] - fn rejects_non_shortest_integers() { - // 0x18 0x00: value 0 written as uint8. - let err = CanonicalCbor::new(vec![0x18, 0x00]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // 0x19 0x00 0x18: value 24 written as uint16. - let err = CanonicalCbor::new(vec![0x19, 0x00, 0x18]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // 0x1a 0x00 0x00 0x01 0x00: value 256 written as uint32. - let err = CanonicalCbor::new(vec![0x1a, 0x00, 0x00, 0x01, 0x00]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // 0x1b ... : value 65536 written as uint64. - let err = CanonicalCbor::new(vec![0x1b, 0, 0, 0, 0, 0, 1, 0, 0]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // Non-shortest length prefix on a byte string. - let err = CanonicalCbor::new(vec![0x58, 0x02, 0xaa, 0xbb]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - } - - #[test] - fn rejects_indefinite_lengths() { - // Indefinite array. - let err = CanonicalCbor::new(vec![0x9f, 0x01, 0xff]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // Indefinite map. - let err = CanonicalCbor::new(vec![0xbf, 0x61, b'a', 0x01, 0xff]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // Indefinite byte string. - let err = CanonicalCbor::new(vec![0x5f, 0x41, 0xaa, 0xff]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // Indefinite text string. - let err = CanonicalCbor::new(vec![0x7f, 0x61, b'a', 0xff]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - } - - #[test] - fn rejects_floats_tags_and_undefined() { - // f16 1.0 - let err = CanonicalCbor::new(vec![0xf9, 0x3c, 0x00]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // f32 - let err = CanonicalCbor::new(vec![0xfa, 0x47, 0xc3, 0x50, 0x00]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // f64 - let err = CanonicalCbor::new(vec![0xfb, 0x3f, 0xf1, 0, 0, 0, 0, 0, 0]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // tag 0 over a text string - let err = CanonicalCbor::new(vec![0xc0, 0x61, b'a']).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // undefined - let err = CanonicalCbor::new(vec![0xf7]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - } - - #[test] - fn rejects_unsorted_and_duplicate_map_keys() { - // {"b": 1, "a": 2} — out of order. - let err = CanonicalCbor::new(vec![0xa2, 0x61, b'b', 0x01, 0x61, b'a', 0x02]).unwrap_err(); - assert!( - matches!(&err, Error::NonCanonicalCbor { reason, .. } if reason.contains("sorted")), - "{err:?}" - ); - - // {"a": 1, "a": 2} — duplicate. - let err = CanonicalCbor::new(vec![0xa2, 0x61, b'a', 0x01, 0x61, b'a', 0x02]).unwrap_err(); - assert!( - matches!(&err, Error::NonCanonicalCbor { reason, .. } if reason.contains("duplicate")), - "{err:?}" - ); - - // Shorter keys sort first bytewise: {"a": 1, "aa": 2} is canonical, - // {"aa": 1, "a": 2} is not. - CanonicalCbor::new(vec![0xa2, 0x61, b'a', 0x01, 0x62, b'a', b'a', 0x02]).unwrap(); - let err = - CanonicalCbor::new(vec![0xa2, 0x62, b'a', b'a', 0x01, 0x61, b'a', 0x02]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - } - - #[test] - fn rejects_invalid_utf8_and_truncation() { - // Text string claiming 1 byte of invalid UTF-8. - let err = CanonicalCbor::new(vec![0x61, 0xff]).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - - // Byte string claiming 4 bytes but carrying 2. - let err = CanonicalCbor::new(vec![0x44, 0x01, 0x02]).unwrap_err(); - assert!(matches!(err, Error::TruncatedCbor { .. }), "{err:?}"); - - // Array claiming 3 items but carrying 2. - let err = CanonicalCbor::new(vec![0x83, 0x01, 0x02]).unwrap_err(); - assert!(matches!(err, Error::TruncatedCbor { .. }), "{err:?}"); - } - - #[test] - fn rejects_trailing_bytes_in_a_single_item() { - let err = CanonicalCbor::new(vec![0x01, 0x02]).unwrap_err(); - assert!( - matches!(err, Error::TrailingCbor { trailing: 1 }), - "{err:?}" - ); - } - - #[test] - fn rejects_excessive_nesting() { - // MAX_NESTING_DEPTH + 1 nested single-element arrays. - let mut bytes = vec![0x81; MAX_NESTING_DEPTH + 1]; - bytes.push(0x00); - - let err = CanonicalCbor::new(bytes).unwrap_err(); - assert!(matches!(err, Error::CborTooDeep { .. }), "{err:?}"); - } - - #[test] - fn sequence_roundtrip_is_byte_identical() { - let records: Vec = (0..8u64) - .map(|i| { - encode(|e| { - e.array(3)?.u64(i)?.bytes(&[i as u8; 4])?.str("record")?; - Ok(()) - }) - .unwrap() - }) - .collect(); - - let mut writer = SeqWriter::new(Vec::new()); - for record in &records { - writer.write_record(record).unwrap(); - } - assert_eq!(writer.count(), 8); - let written = writer.into_inner(); - - let read: Vec<&[u8]> = SeqReader::new(&written) - .collect::>() - .expect("every record is canonical"); - assert_eq!(read.len(), 8); - - // write -> read -> write is byte-identical: nothing about a record's - // encoding is lost or normalized on the way through. - let mut rewriter = SeqWriter::new(Vec::new()); - for record in &read { - rewriter - .write_record(&CanonicalCbor::new(record.to_vec()).unwrap()) - .unwrap(); - } - assert_eq!(rewriter.into_inner(), written); - } - - /// The writer refuses what the reader would refuse, at the same ceiling. - /// - /// Regression for a stele that published cleanly and restored nowhere: the - /// reader held records to 16 MiB, nothing held the writer to anything, and - /// a profile with a 24 MiB record produced 928 layers, a valid manifest and - /// an artifact whose first oversized record ended every restore of it. - #[test] - fn writer_refuses_a_record_past_its_ceiling() { - let big = encode(|e| { - e.bytes(&[0u8; 4096])?; - Ok(()) - }) - .unwrap(); - - let mut writer = SeqWriter::with_max_record(Vec::new(), 1024); - let err = writer.write_record(&big).unwrap_err(); - - assert!( - matches!(err, Error::RecordTooLarge { limit: 1024, .. }), - "{err:?}" - ); - - // Refused before the write, so nothing partial reached the sink. - assert_eq!(writer.count(), 0); - assert!(writer.into_inner().is_empty()); - } - - /// The offset a refusal names is the record's offset in the layer, so a - /// publisher and a reader failing on the same record report the same byte. - #[test] - fn writer_refusal_names_the_offset_the_reader_would() { - let small = encode(|e| { - e.bytes(&[0u8; 8])?; - Ok(()) - }) - .unwrap(); - let big = encode(|e| { - e.bytes(&[0u8; 4096])?; - Ok(()) - }) - .unwrap(); - - let mut writer = SeqWriter::with_max_record(Vec::new(), 1024); - writer.write_record(&small).unwrap(); - writer.write_record(&small).unwrap(); - - let err = writer.write_record(&big).unwrap_err(); - let Error::RecordTooLarge { offset, .. } = err else { - panic!("{err:?}"); - }; - - assert_eq!(offset, small.as_bytes().len() * 2); - } - - /// At any ceiling, the writer accepts exactly what a reader at that same - /// ceiling will. - /// - /// The guarantee the profile's number buys is that one value binds both - /// ends; a ceiling where the two disagree is that guarantee with a hole in - /// it, and the hole does not have to be a reachable value to be worth - /// closing. Zero is the only such value, because [`Limits::normalized`] - /// floors a reader at one byte: a writer keeping a literal zero refuses the - /// one-byte records that reader accepts. - #[test] - fn the_writer_accepts_exactly_what_the_reader_will() { - let one_byte = encode(|e| { - e.u64(0)?; - Ok(()) - }) - .unwrap(); - assert_eq!(one_byte.as_bytes().len(), 1, "smallest possible record"); - - let larger = encode(|e| { - e.bytes(&[0xab; 1000])?; - Ok(()) - }) - .unwrap(); - - let ceilings = [0, 1, 2, larger.as_bytes().len(), DEFAULT_MAX_RECORD]; - - for ceiling in ceilings { - for record in [&one_byte, &larger] { - let mut writer = SeqWriter::with_max_record(Vec::new(), ceiling); - let writer_took = writer.write_record(record).is_ok(); - - // The reader needs a sequence to walk, so the record is laid - // down by a writer with a ceiling that refuses nothing. - let mut permissive = SeqWriter::with_max_record(Vec::new(), usize::MAX); - permissive.write_record(record).unwrap(); - let sequence = permissive.into_inner(); - - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&sequence), - Limits { - max_record: ceiling, - window: 64, - }, - ); - let reader_took = matches!(reader.next_record(), Some(Ok(_))); - - assert_eq!( - writer_took, - reader_took, - "ceiling {ceiling} disagrees on a {}-byte record", - record.as_bytes().len() - ); - } - } - } - - /// A profile that raises its ceiling can write what the default refuses, - /// which is the whole point of the number being the profile's. - #[test] - fn a_raised_ceiling_admits_a_record_the_default_would_refuse() { - let record = encode(|e| { - e.bytes(&[0u8; DEFAULT_MAX_RECORD + 1])?; - Ok(()) - }) - .unwrap(); - - assert!(SeqWriter::new(Vec::new()).write_record(&record).is_err()); - - let mut writer = SeqWriter::with_max_record(Vec::new(), DEFAULT_MAX_RECORD * 4); - writer - .write_record(&record) - .expect("within the raised ceiling"); - - assert_eq!(writer.count(), 1); - } - - #[test] - fn sequence_reader_stops_at_the_first_bad_record() { - let good = encode(|e| { - e.u64(1)?; - Ok(()) - }) - .unwrap(); - - let mut bytes = good.as_bytes().to_vec(); - bytes.extend_from_slice(&[0x18, 0x00]); // non-shortest uint - bytes.extend_from_slice(good.as_bytes()); - - let mut reader = SeqReader::new(&bytes); - assert_eq!(reader.next().unwrap().unwrap(), good.as_bytes()); - assert!(matches!( - reader.next().unwrap(), - Err(Error::NonCanonicalCbor { offset: 1, .. }) - )); - assert!(reader.next().is_none(), "iterator must not resynchronize"); - } - - /// Every input the deterministic profile refuses, in the encodings a - /// hostile or buggy producer would actually emit. - /// - /// The list is the crate's rejection corpus: both readers are run over it - /// below, and the point is not that each one refuses — it is that they - /// refuse *identically*. Two readers that disagree about what is canonical - /// are a determinism bug wearing a compatibility costume: the same bytes - /// would restore on one path and fail on the other, and whichever produced - /// the layer would have published a `diffId` nobody else reproduces. - const NON_CANONICAL: &[(&str, &[u8])] = &[ - ("value 0 as uint8", &[0x18, 0x00]), - ("value 24 as uint16", &[0x19, 0x00, 0x18]), - ("value 256 as uint32", &[0x1a, 0x00, 0x00, 0x01, 0x00]), - ("value 65536 as uint64", &[0x1b, 0, 0, 0, 0, 0, 1, 0, 0]), - ("non-shortest byte-string length", &[0x58, 0x02, 0xaa, 0xbb]), - ("indefinite array", &[0x9f, 0x01, 0xff]), - ("indefinite map", &[0xbf, 0x61, b'a', 0x01, 0xff]), - ("indefinite byte string", &[0x5f, 0x41, 0xaa, 0xff]), - ("indefinite text string", &[0x7f, 0x61, b'a', 0xff]), - ("half-precision float", &[0xf9, 0x3c, 0x00]), - ("single-precision float", &[0xfa, 0x47, 0xc3, 0x50, 0x00]), - ( - "double-precision float", - &[0xfb, 0x3f, 0xf1, 0, 0, 0, 0, 0, 0], - ), - ("tag 0", &[0xc0, 0x61, b'a']), - ("undefined", &[0xf7]), - ("simple value 255", &[0xf8, 0xff]), - ("break outside an indefinite item", &[0xff]), - ("reserved additional information", &[0x1c]), - ( - "unsorted map keys", - &[0xa2, 0x61, b'b', 0x01, 0x61, b'a', 0x02], - ), - ( - "duplicate map key", - &[0xa2, 0x61, b'a', 0x01, 0x61, b'a', 0x02], - ), - ( - "map keys unsorted by length", - &[0xa2, 0x62, b'a', b'a', 0x01, 0x61, b'a', 0x02], - ), - ("invalid utf-8", &[0x61, 0xff]), - ("byte string cut short", &[0x44, 0x01, 0x02]), - ("array cut short", &[0x83, 0x01, 0x02]), - ]; - - fn good_record() -> CanonicalCbor { - encode(|e| { - e.array(2)?.u64(7)?.str("good")?; - Ok(()) - }) - .unwrap() - } - - /// Drain a [`RecordReader`], copying records out so the result can be - /// compared against the slice reader's borrowed ones. - fn drain(reader: &mut RecordReader) -> (Vec>, Option) { - let mut records = Vec::new(); - - while let Some(next) = reader.next_record() { - match next { - Ok(record) => records.push(record.to_vec()), - Err(e) => return (records, Some(e)), - } - } - - (records, None) - } - - /// The bound reported for an unfinished item is a real lower bound on the - /// whole item, derived from what the encoder has already committed to — - /// never an estimate, and never less than what is already in hand, or a - /// refill loop would spin. - #[test] - fn measure_reports_what_an_unfinished_item_still_needs() { - let cases: &[(&[u8], u64)] = &[ - // Nothing at all: a head byte, at least. - (&[], 1), - // bytes(4) with two of them: head + 4. - (&[0x44, 0x01, 0x02], 5), - // Three-element array, nothing inside: head + one byte each. - (&[0x83], 4), - // ... and with two elements present, the third is still owed. - (&[0x83, 0x01, 0x02], 4), - // A map with one entry present owes both halves of the next. - (&[0xa2, 0x61, b'a', 0x01], 6), - // Nested: the outer array owes its second element, the inner byte - // string owes its body. - (&[0x82, 0x43, 0xaa], 6), - // A text string whose length prefix is larger than any layer: - // reported at full size, which is what a ceiling check needs. - ( - &[0x7b, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00], - 1_099_511_627_785, - ), - ]; - - for (bytes, required) in cases { - assert_eq!( - measure_item(bytes).unwrap(), - Measure::Incomplete { - required: *required - }, - "0x{}", - hex::encode(bytes) - ); - - assert!( - *required > bytes.len() as u64, - "the bound must exceed what is in hand, or a refill loop stalls" - ); - } - } - - /// A violation is a verdict, not a request for more bytes: no quantity of - /// further input rehabilitates a float or a tag, so `measure_item` reports - /// those as errors rather than as an unfinished item. - #[test] - fn measure_refuses_rather_than_waits_on_non_canonical_input() { - for (name, bytes) in NON_CANONICAL { - let measured = measure_item(bytes); - - match measured { - // Truncation is the one honest "incomplete" in the corpus. - Ok(Measure::Incomplete { .. }) => assert!( - name.contains("cut short"), - "{name} must not be reported as merely unfinished" - ), - Ok(Measure::Complete { .. }) => panic!("{name} must not measure as complete"), - Err(_) => {} - } - } - } - - /// The guarantee the whole rejection corpus exists for: one scanner, two - /// entry points, one verdict. - #[test] - fn both_readers_refuse_the_same_bytes_the_same_way() { - let good = good_record(); - - for (name, bad) in NON_CANONICAL { - let mut bytes = good.as_bytes().to_vec(); - bytes.extend_from_slice(bad); - - let mut slice = SeqReader::new(&bytes); - assert_eq!(slice.next().unwrap().unwrap(), good.as_bytes(), "{name}"); - let slice_error = slice.next().unwrap().unwrap_err(); - assert!(slice.next().is_none(), "{name}: must not resynchronize"); - - // Window sizes across the interesting boundaries: one byte at a - // time, mid-record, and comfortably larger than the whole input. - for window in [1, 3, 4096] { - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&bytes), - Limits { - window, - ..Limits::default() - }, - ); - - let (records, error) = drain(&mut reader); - - assert_eq!(records, vec![good.as_bytes().to_vec()], "{name} @ {window}"); - assert_eq!( - reader.count(), - 1, - "{name} @ {window}: a failure is not a record" - ); - - let error = error.unwrap_or_else(|| panic!("{name} @ {window}: expected an error")); - - // Same variant, same offset, same reason — compared through the - // rendered message so a divergence in any of the three shows up - // as a diff rather than as a silently weaker assertion. - assert_eq!( - error.to_string(), - slice_error.to_string(), - "{name} @ {window}" - ); - - assert!( - reader.next_record().is_none(), - "{name} @ {window}: must not resynchronize" - ); - } - } - } - - /// Nesting is bounded for both readers by the same constant — built here - /// rather than in the corpus because the input is generated. - #[test] - fn both_readers_bound_nesting() { - let mut bytes = vec![0x81; MAX_NESTING_DEPTH + 1]; - bytes.push(0x00); - - let slice_error = SeqReader::new(&bytes).next().unwrap().unwrap_err(); - assert!( - matches!(slice_error, Error::CborTooDeep { .. }), - "{slice_error:?}" - ); - - let mut reader = RecordReader::new(std::io::Cursor::new(&bytes)); - let error = reader.next_record().unwrap().unwrap_err(); - assert_eq!(error.to_string(), slice_error.to_string()); - } - - /// Records that straddle every refill boundary still come back whole and in - /// order, whatever the window size — including a window far smaller than a - /// single record, which is the case the ceiling has to distinguish from a - /// record that is genuinely too big. - #[test] - fn records_survive_every_refill_boundary() { - let records: Vec = (0..64u64) - .map(|i| { - encode(|e| { - // Sizes crossing the 24/256 encoding boundaries, so records - // of several byte lengths land at every window offset. - e.array(2)?.u64(i)?.bytes(&vec![i as u8; i as usize * 7])?; - Ok(()) - }) - .unwrap() - }) - .collect(); - - let mut writer = SeqWriter::new(Vec::new()); - for record in &records { - writer.write_record(record).unwrap(); - } - let sequence = writer.into_inner(); - - let expected: Vec> = records.iter().map(|r| r.as_bytes().to_vec()).collect(); - - for window in [1, 2, 3, 17, 64, 129, 4096, 1 << 20] { - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&sequence), - Limits { - window, - ..Limits::default() - }, - ); - - let (read, error) = drain(&mut reader); - - assert!(error.is_none(), "window {window}: {error:?}"); - assert_eq!(read, expected, "window {window}"); - assert_eq!(reader.count(), 64); - assert_eq!(reader.offset(), sequence.len()); - } - } - - /// A length prefix is a claim, not an instruction. The reader checks it - /// against the ceiling before the window grows, so a record claiming a - /// terabyte costs a comparison — the attack surface the buffered path never - /// had, because there the bytes had to exist before they could be claimed. - #[test] - fn an_oversized_length_prefix_is_refused_before_it_is_allocated_for() { - // bytes(2^40): a header that arrives in nine bytes and asks for a - // terabyte. - let bytes: &[u8] = &[0x5b, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00]; - - let mut reader = RecordReader::new(std::io::Cursor::new(bytes)); - let error = reader.next_record().unwrap().unwrap_err(); - - assert!( - matches!( - error, - Error::RecordTooLarge { - offset: 0, - required: 1_099_511_627_785, - limit: DEFAULT_MAX_RECORD, - } - ), - "{error:?}" - ); - - // The window is still the window: nothing was allocated on the strength - // of the prefix. - assert!(reader.next_record().is_none()); - } - - /// The ceiling is checked against the record, not against the window, and - /// it is exact at the boundary — one byte either side decides it. - #[test] - fn the_record_ceiling_is_exact() { - let record = encode(|e| { - e.bytes(&[0xab; 1000])?; - Ok(()) - }) - .unwrap(); - - let len = record.len(); - assert_eq!(len, 1003, "3-byte head over a 1000-byte body"); - - let mut writer = SeqWriter::new(Vec::new()); - writer.write_record(&record).unwrap(); - let sequence = writer.into_inner(); - - // Exactly at the ceiling, from a window a fraction of the size. - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&sequence), - Limits { - max_record: len, - window: 16, - }, - ); - assert_eq!(reader.next_record().unwrap().unwrap(), record.as_bytes()); - assert!(reader.next_record().is_none()); - - // One byte under it, and the same record is refused — from the length - // prefix, before the window ever grew to 1000 bytes. - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&sequence), - Limits { - max_record: len - 1, - window: 16, - }, - ); - let error = reader.next_record().unwrap().unwrap_err(); - assert!( - matches!( - error, - Error::RecordTooLarge { - offset: 0, - required: 1003, - limit: 1002 - } - ), - "{error:?}" - ); - - // And a record that fits is not refused for the company it keeps: a - // window large enough to hold two of them still yields them one at a - // time. - let mut writer = SeqWriter::new(Vec::new()); - writer.write_record(&record).unwrap(); - writer.write_record(&record).unwrap(); - let pair = writer.into_inner(); - - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(&pair), - Limits { - max_record: len, - window: 4096, - }, - ); - let (read, error) = drain(&mut reader); - assert!(error.is_none(), "{error:?}"); - assert_eq!(read.len(), 2); - } - - /// A sequence that ends mid-record is truncation, not an end: the reader - /// says so rather than quietly reporting one record fewer. - #[test] - fn a_sequence_that_ends_mid_record_is_reported() { - let record = good_record(); - let mut bytes = record.as_bytes().to_vec(); - bytes.extend_from_slice(&record.as_bytes()[..2]); - - let mut reader = RecordReader::new(std::io::Cursor::new(&bytes)); - - assert_eq!(reader.next_record().unwrap().unwrap(), record.as_bytes()); - - let error = reader.next_record().unwrap().unwrap_err(); - assert!(matches!(error, Error::TruncatedCbor { .. }), "{error:?}"); - - // Reported in the coordinates of the sequence, as the slice reader - // would have. - assert_eq!( - error.to_string(), - SeqReader::new(&bytes) - .nth(1) - .unwrap() - .unwrap_err() - .to_string() - ); - } - - /// An empty sequence is an empty walk, not an error. - #[test] - fn an_empty_sequence_yields_nothing() { - let mut reader = RecordReader::new(std::io::Cursor::new(Vec::new())); - - assert!(reader.next_record().is_none()); - assert_eq!(reader.count(), 0); - assert_eq!(reader.offset(), 0); - } - - #[test] - fn layer_header_roundtrip() { - let scope = encode(|e| { - e.array(2)?.u64(7)?.u64(42)?; - Ok(()) - }) - .unwrap(); - - let header = LayerHeader::new("dev.example.toy", "notes", scope.clone()); - let encoded = header.encode().unwrap(); - let decoded = LayerHeader::decode(encoded.as_bytes()).unwrap(); - - assert_eq!(decoded, header); - assert_eq!(decoded.scope, scope); - assert_eq!(decoded.encode().unwrap(), encoded); - } - - /// The header's `scope` slot takes any canonical CBOR item, and the - /// protocol carries it through untouched. A profile using a map, an - /// array or a bare integer must all survive. - #[test] - fn layer_header_scope_stays_opaque() { - let scopes = [ - encode(|e| { - e.u64(3)?; - Ok(()) - }) - .unwrap(), - encode(|e| { - e.map(2)?.str("epoch")?.u64(550)?.str("shard")?.u64(0)?; - Ok(()) - }) - .unwrap(), - encode(|e| { - e.array(0)?; - Ok(()) - }) - .unwrap(), - encode(|e| { - e.null()?; - Ok(()) - }) - .unwrap(), - ]; - - for scope in scopes { - let header = LayerHeader::new("dev.example.toy", "notes", scope.clone()); - let decoded = LayerHeader::decode(header.encode().unwrap().as_bytes()).unwrap(); - assert_eq!(decoded.scope, scope); - } - } - - #[test] - fn layer_header_rejects_a_future_format_version() { - let scope = encode(|e| { - e.u64(0)?; - Ok(()) - }) - .unwrap(); - - let record = encode(|e| { - e.array(4)? - .u64(2)? - .str("dev.example.toy")? - .str("notes")? - .u64(0)?; - Ok(()) - }) - .unwrap(); - - let err = LayerHeader::decode(record.as_bytes()).unwrap_err(); - assert!( - matches!(&err, Error::MalformedHeader(m) if m.contains("format version")), - "{err:?}" - ); - - // Sanity: the same shape at the implemented version parses. - let ok = LayerHeader::new("dev.example.toy", "notes", scope) - .encode() - .unwrap(); - LayerHeader::decode(ok.as_bytes()).unwrap(); - } - - #[test] - fn layer_header_rejects_a_non_canonical_record() { - // A header whose `scope` is an indefinite-length array. - let mut bytes = vec![0x84, 0x01]; - bytes.extend_from_slice(&[0x6f]); // tstr(15) - bytes.extend_from_slice(b"dev.example.toy"); - bytes.extend_from_slice(&[0x65]); // tstr(5) - bytes.extend_from_slice(b"notes"); - bytes.extend_from_slice(&[0x9f, 0x01, 0xff]); // indefinite array - - let err = LayerHeader::decode(&bytes).unwrap_err(); - assert!(matches!(err, Error::NonCanonicalCbor { .. }), "{err:?}"); - } -} diff --git a/crates/stelae/src/inscription.rs b/crates/stelae/src/inscription.rs deleted file mode 100644 index 914863e44..000000000 --- a/crates/stelae/src/inscription.rs +++ /dev/null @@ -1,1059 +0,0 @@ -//! The inscription: a stele's canonical, signable document. -//! -//! The inscription is the OCI config blob of a stele and the anchor of the -//! whole protocol. Its sha256, taken over its RFC 8785 canonical JSON encoding, -//! is the stele's identity: what independent publishers reproduce, what -//! signatures cover, and what `history` chains together so the newest signed -//! inscription transitively attests every earlier one. -//! -//! Three of its fields — `position`, `parameters` and each layer's `scope` — -//! are the profile's, and this module never looks inside them. They are held as -//! [`serde_json::Value`], canonicalized like every other key, and hashed. That -//! is what lets determinism hold without the protocol knowing a profile's -//! vocabulary. -//! -//! ## Two rules that make the digest reproducible -//! -//! **Every number is an integer within ±(2^53 − 1).** RFC 8785 serializes -//! numbers per ECMAScript, so past that magnitude a `u64` renders as a rounded -//! double and two implementations diverge *silently*. The protocol refuses the -//! value instead ([`check_safe_numbers`]). Nothing in the schema needs the -//! range: an `uncompressedSize` for a mainnet epoch is around 4×10^10. -//! -//! **An invalid inscription has no digest.** [`Inscription::canonicalize`] -//! validates first, so it is not possible to obtain canonical bytes — and -//! therefore an identity — for a document that violates the schema, the naming -//! rules or the history invariant. - -use serde::{Deserialize, Serialize}; - -use crate::{ - profile::{checked_layer_media_type, validate_profile_name, MediaType, Profile}, - Digest, Error, MAX_SAFE_INTEGER, SCHEMA_VERSION, -}; - -/// The profile a stele belongs to. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct ProfileRef { - /// Reverse-DNS, vendor-owned, e.g. `io.txpipe.dolos.cardano`. - pub name: String, - /// Profile major version. A client refuses a value above the one it - /// implements. - pub version: u64, -} - -/// Compression parameters. Transport policy, pinned so that blobs dedupe across -/// publishers in practice; identity never depends on it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct Compression { - pub algo: String, - pub level: i64, -} - -/// One prior publication, as attested by this inscription. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct HistoryEntry { - pub sequence: u64, - pub inscription_digest: Digest, -} - -/// One layer of a stele, as the inscription describes it. -/// -/// Note what is absent: the compressed digest and the compressed size. Those -/// are transport facts and live in the OCI manifest. The inscription carries -/// only what is identity — the uncompressed digest — so it stays reproducible -/// by a party that compressed differently, or not at all. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct LayerDescriptor { - /// Profile-defined layer kind. - pub kind: String, - /// Profile-owned payload media type. - pub media_type: String, - /// sha256 over the uncompressed CBOR sequence — the layer's identity. - pub diff_id: Digest, - /// Number of records, header record included. - pub records: u64, - /// Uncompressed byte length. Summed across planned layers, this is the - /// restore-time disk preflight. - pub uncompressed_size: u64, - /// Profile-owned, opaque to the protocol. - pub scope: serde_json::Value, -} - -/// The canonical, signable document of a stele. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields, rename_all = "camelCase")] -pub struct Inscription { - /// Inscription schema version. Verifiers fail closed on anything but - /// [`SCHEMA_VERSION`]. - pub schema: u64, - pub profile: ProfileRef, - /// The protocol's monotonic ordering key. A profile decides what it counts - /// (the Dolos profile sets it to the Cardano epoch). - pub sequence: u64, - /// Profile-owned, opaque: where in its source domain this stele stands. - pub position: serde_json::Value, - /// Profile-owned, opaque: the parameters a reader needs to interpret the - /// layers. - pub parameters: serde_json::Value, - pub compression: Compression, - /// Every prior publication, contiguous and ascending, ending at - /// `sequence - 1`. - pub history: Vec, - pub layers: Vec, -} - -impl Inscription { - /// An inscription at the current schema version, with no history and no - /// layers yet. - pub fn new( - profile: &dyn Profile, - sequence: u64, - position: serde_json::Value, - parameters: serde_json::Value, - compression: Compression, - ) -> Self { - Self { - schema: SCHEMA_VERSION, - profile: ProfileRef { - name: profile.name().to_owned(), - version: profile.version(), - }, - sequence, - position, - parameters, - compression, - history: Vec::new(), - layers: Vec::new(), - } - } - - /// The RFC 8785 canonical JSON encoding of this inscription. - /// - /// Validates first: canonical bytes, and therefore an identity, exist only - /// for a well-formed inscription. - pub fn canonicalize(&self) -> Result, Error> { - let value = serde_json::to_value(self)?; - self.validate_structure()?; - check_safe_numbers(&value)?; - canonical_json(&value) - } - - /// The stele's identity: sha256 of [`Inscription::canonicalize`]. - pub fn digest(&self) -> Result { - Ok(Digest::compute(self.canonicalize()?)) - } - - /// Parse untrusted inscription bytes, fail-closed. - /// - /// Rejects an unknown generic key anywhere in the document (extension - /// happens only inside `position`, `parameters` and `scope`), a schema - /// version this implementation does not implement, a number outside the - /// JCS-safe integer range, a malformed profile name or payload media type, - /// and any violation of the history invariant. - /// - /// It deliberately does *not* check which profile the inscription belongs - /// to — that needs the profile implementation, and is - /// [`Inscription::check_profile`]. - /// - /// Nor does it reject a document with a repeated JSON key, which JSON - /// itself leaves undefined and `serde_json` resolves last-wins. That is not - /// a hole in practice: canonicalizing the result yields different bytes - /// from the input, so any check that compares an inscription against its - /// digest — which is every use of one — refuses it. `SteleDir` makes the - /// comparison explicit. - pub fn parse(bytes: &[u8]) -> Result { - let value: serde_json::Value = serde_json::from_slice(bytes)?; - - // Checked on the raw document, so that numbers inside the profile's - // opaque objects are covered too. - check_safe_numbers(&value)?; - - let inscription: Self = serde_json::from_value(value)?; - inscription.validate_structure()?; - - Ok(inscription) - } - - /// Full validation of a locally built inscription. - pub fn validate(&self) -> Result<(), Error> { - let value = serde_json::to_value(self)?; - self.validate_structure()?; - check_safe_numbers(&value) - } - - /// Refuse an inscription this profile implementation cannot read. - /// - /// Two ways it fails, both clean refusals rather than a partial restore: - /// the inscription belongs to a different profile, or it needs a profile - /// major version above the one implemented here. A layer of a kind this - /// implementation does not define is *not* one of them — see - /// [`Inscription::unknown_layers`] for why, and - /// [`Inscription::check_profile_strict`] for the side that still refuses - /// it. - pub fn check_profile(&self, profile: &dyn Profile) -> Result<(), Error> { - if self.profile.name != profile.name() { - return Err(Error::UnknownProfile { - found: self.profile.name.clone(), - expected: profile.name().to_owned(), - }); - } - - if self.profile.version > profile.version() { - return Err(Error::UnsupportedProfileVersion { - name: self.profile.name.clone(), - found: self.profile.version, - supported: profile.version(), - }); - } - - let kinds = profile.kinds(); - - for layer in &self.layers { - // Left to `unknown_layers`. Its media type is still checked: - // `validate_structure` has already established that it parses and - // does not squat the reserved `stelae` vendor. - if !kinds.contains(&layer.kind.as_str()) { - continue; - } - - // Pins the profile's own answer to the naming rules before it is - // used as the yardstick. - let defined = checked_layer_media_type(profile, &layer.kind)?; - let expected = MediaType::parse(&defined)?; - let found = MediaType::parse(&layer.media_type)?; - - // Compared on vendor and kind, not on the whole string. Those two - // are what make a descriptor unambiguously this profile's layer; - // the version and codec are transport detail a profile may move - // within one major, and freezing them here would refuse an - // inscription this implementation can otherwise read. - // - // `validate_structure` already established that `layer.media_type` - // parses and does not squat the reserved vendor. What it cannot - // know, without a profile in hand, is whether the name belongs to - // the profile the inscription claims — that is this check. - if found.vendor != expected.vendor || found.kind != expected.kind { - return Err(Error::InvalidMediaType { - value: layer.media_type.clone(), - reason: format!( - "profile {:?} names layer kind {:?} {defined:?}", - profile.name(), - layer.kind, - ), - }); - } - } - - Ok(()) - } - - /// The layers whose kind `profile` does not define, in inscription order. - /// - /// The client half of an additive change. A profile that gains a kind - /// publishes it as a new media type on a new layer, and an older reader - /// cannot store what it does not model — but refusing the whole stele over - /// it would make every additive change break every deployed reader. So the - /// protocol reports the layers rather than refusing them, and the profile - /// decides during planning what skipping each one costs. - /// - /// **Nothing here reads a scope.** Whether a layer is one a reader may skip - /// is a profile question, answered from the profile-owned `scope` the - /// descriptor carries, and the protocol interprets no field of it. - pub fn unknown_layers<'a>(&'a self, profile: &dyn Profile) -> Vec<&'a LayerDescriptor> { - let kinds = profile.kinds(); - - self.layers - .iter() - .filter(|layer| !kinds.contains(&layer.kind.as_str())) - .collect() - } - - /// [`Inscription::check_profile`], plus a refusal of any unknown kind. - /// - /// What a publisher checks, and the asymmetry is the point: a reader - /// consumes the layers it understands, while a publisher *attests* every - /// layer it lists — its inscription is the document independent parties - /// reproduce and, later, sign. Chaining onto a stele carrying a kind this - /// binary cannot build means either dropping that layer from the new stele - /// or attesting bytes it never read, and both are worse than stopping. - pub fn check_profile_strict(&self, profile: &dyn Profile) -> Result<(), Error> { - self.check_profile(profile)?; - - if let Some(layer) = self.unknown_layers(profile).first() { - return Err(Error::UnknownLayerKind { - profile: profile.name().to_owned(), - kind: layer.kind.clone(), - }); - } - - Ok(()) - } - - /// Layers of a given kind, in inscription order. - pub fn layers_of_kind<'a>( - &'a self, - kind: &'a str, - ) -> impl Iterator + 'a { - self.layers.iter().filter(move |l| l.kind == kind) - } - - /// Total uncompressed bytes across every layer — the restore-time disk - /// preflight, in the one place that knows the sizes. - pub fn uncompressed_size(&self) -> u64 { - self.layers.iter().map(|l| l.uncompressed_size).sum() - } - - fn validate_structure(&self) -> Result<(), Error> { - if self.schema != SCHEMA_VERSION { - return Err(Error::UnsupportedSchema { found: self.schema }); - } - - validate_profile_name(&self.profile.name)?; - - if self.profile.version == 0 { - return Err(Error::InvalidProfileName { - value: self.profile.name.clone(), - reason: "profile major version must be at least 1".to_owned(), - }); - } - - if self.compression.algo.is_empty() { - return Err(Error::Canonicalization( - "compression.algo must not be empty".to_owned(), - )); - } - - for layer in &self.layers { - if layer.kind.is_empty() { - return Err(Error::Canonicalization( - "a layer descriptor has an empty kind".to_owned(), - )); - } - - // Enforces the vendor-namespacing rules on whatever the publisher - // wrote, including that `vnd.stelae.*` is never a payload type. - MediaType::parse(&layer.media_type)?; - } - - self.validate_history() - } - - /// `history` holds exactly one entry per published sequence, strictly - /// ascending, contiguous, ending at `sequence - 1`. - /// - /// The protocol cannot know where a repository's history *starts* — the - /// first published sequence is a deployment parameter pinned alongside the - /// repository — so the invariant checked here is the part that is - /// universal: no gaps, no duplicates, no reordering, and nothing - /// missing between the last entry and this stele. - fn validate_history(&self) -> Result<(), Error> { - for (index, entry) in self.history.iter().enumerate() { - if entry.sequence >= self.sequence { - return Err(Error::HistoryInvariant(format!( - "entry {index} has sequence {} which is not below this stele's sequence {}", - entry.sequence, self.sequence - ))); - } - - if index == 0 { - continue; - } - - let previous = self.history[index - 1].sequence; - - match entry.sequence.cmp(&previous) { - std::cmp::Ordering::Less => { - return Err(Error::HistoryInvariant(format!( - "entry {index} has sequence {} after sequence {previous}: \ - history must be strictly ascending", - entry.sequence - ))) - } - std::cmp::Ordering::Equal => { - return Err(Error::HistoryInvariant(format!( - "sequence {} appears more than once", - entry.sequence - ))) - } - std::cmp::Ordering::Greater if entry.sequence != previous + 1 => { - return Err(Error::HistoryInvariant(format!( - "gap between sequence {previous} and sequence {}", - entry.sequence - ))) - } - std::cmp::Ordering::Greater => {} - } - } - - if let Some(last) = self.history.last() { - if last.sequence + 1 != self.sequence { - return Err(Error::HistoryInvariant(format!( - "history ends at sequence {} but this stele is sequence {}", - last.sequence, self.sequence - ))); - } - } - - Ok(()) - } -} - -/// The history a stele at `sequence` carries when it follows `previous`. -/// -/// The constructive half of the invariant [`Inscription::validate`] checks: one -/// builds the chain, the other refuses a document whose chain is broken. -/// -/// The three legal readings of what came before, and the one refusal: -/// -/// - **nothing there** — an empty history, which the protocol permits at any -/// sequence. The first stele of a repository carries no history, and so does -/// a publisher deliberately starting a new one at sequence 500; -/// - **the stele before this one** — the old history plus an entry naming it. -/// Contiguous by construction, so the protocol's invariant passes rather than -/// being relied upon; -/// - **anything else** — refused, naming both sequences and, for a gap, the -/// distance between them. A gap means a publisher skipped sequences, an equal -/// sequence means it is republishing one, and a higher one means the -/// repository is ahead of this node. All three are operational faults with -/// different fixes, so the message says which. -/// -/// Whether a deliberate gap ever gets a policy is not this function's to -/// invent; there is no flag here that overrides the refusal. -/// -/// It lives beside the invariant rather than beside a transport because a -/// verifier reaches it without one, and a rule this load-bearing should not be -/// compiled out of a build that still has to reproduce a chained digest. -pub fn history_for( - previous: Option<&Inscription>, - sequence: u64, -) -> Result, Error> { - let Some(previous) = previous else { - return Ok(Vec::new()); - }; - - let latest = previous.sequence; - - let reason = match latest.checked_add(1) { - Some(next) if next == sequence => { - let mut history = previous.history.clone(); - - history.push(HistoryEntry { - sequence: latest, - inscription_digest: previous.digest()?, - }); - - return Ok(history); - } - _ if latest >= sequence => { - "this stele is at or behind the repository's latest; a republish would restart the \ - chain rather than extend it" - .to_owned() - } - _ => format!( - "this node is {} sequences ahead, and a publish must follow the repository's latest \ - stele: this one would leave a gap no later stele could close", - sequence - latest, - ), - }; - - Err(Error::HistoryBreak { - latest, - publishing: sequence, - reason, - }) -} - -/// RFC 8785 canonical JSON encoding of an arbitrary value. -/// -/// Exposed so the conformance vectors exercise the same code path the -/// inscription digest depends on, rather than a parallel one. -pub fn canonical_json(value: &serde_json::Value) -> Result, Error> { - serde_jcs::to_vec(value).map_err(|e| Error::Canonicalization(e.to_string())) -} - -/// Assert that every number in `value` is an integer RFC 8785 renders exactly. -/// -/// Walks into arrays and objects, so a profile's opaque `position`, -/// `parameters` and `scope` are held to the same rule as the generic keys. See -/// the module documentation for why the range matters. -pub fn check_safe_numbers(value: &serde_json::Value) -> Result<(), Error> { - check_numbers_at(value, "$") -} - -fn check_numbers_at(value: &serde_json::Value, path: &str) -> Result<(), Error> { - match value { - serde_json::Value::Number(number) => { - if let Some(unsigned) = number.as_u64() { - if unsigned > MAX_SAFE_INTEGER as u64 { - return Err(Error::UnsafeInteger { - path: path.to_owned(), - value: number.to_string(), - }); - } - Ok(()) - } else if let Some(signed) = number.as_i64() { - if signed < -MAX_SAFE_INTEGER { - return Err(Error::UnsafeInteger { - path: path.to_owned(), - value: number.to_string(), - }); - } - Ok(()) - } else { - Err(Error::NonIntegerNumber { - path: path.to_owned(), - value: number.to_string(), - }) - } - } - serde_json::Value::Array(items) => items - .iter() - .enumerate() - .try_for_each(|(index, item)| check_numbers_at(item, &format!("{path}[{index}]"))), - serde_json::Value::Object(entries) => entries - .iter() - .try_for_each(|(key, item)| check_numbers_at(item, &format!("{path}.{key}"))), - _ => Ok(()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - struct Toy; - - impl Profile for Toy { - fn name(&self) -> &str { - "dev.example.toy" - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["notes"] - } - - fn layer_media_type(&self, kind: &str) -> Result { - Ok(format!("application/vnd.example.stele.{kind}.v1+zstd")) - } - - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("note-{sequence}")) - } - } - - fn digest_of(byte: u8) -> Digest { - Digest::from_bytes([byte; 32]) - } - - fn sample() -> Inscription { - let mut inscription = Inscription::new( - &Toy, - 3, - json!({"chapter": 3, "label": "third"}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: 9, - }, - ); - - inscription.history = vec![ - HistoryEntry { - sequence: 1, - inscription_digest: digest_of(1), - }, - HistoryEntry { - sequence: 2, - inscription_digest: digest_of(2), - }, - ]; - - inscription.layers = vec![LayerDescriptor { - kind: "notes".to_owned(), - media_type: "application/vnd.example.stele.notes.v1+zstd".to_owned(), - diff_id: digest_of(3), - records: 5, - uncompressed_size: 128, - scope: json!({"chapter": 3}), - }]; - - inscription - } - - /// An inscription at `sequence` carrying `history`, for the chain rules - /// below: the fields `history_for` reads and nothing else. - fn at(sequence: u64, history: Vec) -> Inscription { - let mut inscription = Inscription::new( - &Toy, - sequence, - json!({"chapter": sequence}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: 9, - }, - ); - - inscription.history = history; - inscription - } - - fn entry(sequence: u64) -> HistoryEntry { - HistoryEntry { - sequence, - inscription_digest: Digest::compute(sequence.to_be_bytes()), - } - } - - /// The first stele of a repository carries no history, at any sequence. - #[test] - fn an_empty_repository_starts_a_history() { - assert!(history_for(None, 0).unwrap().is_empty()); - assert!(history_for(None, 500).unwrap().is_empty()); - } - - #[test] - fn a_publish_that_follows_latest_extends_the_chain() { - let previous = at(3, vec![entry(1), entry(2)]); - - let history = history_for(Some(&previous), 4).unwrap(); - - assert_eq!( - history.iter().map(|e| e.sequence).collect::>(), - vec![1, 2, 3], - "the old history plus an entry naming the stele it came from" - ); - - assert_eq!(history[2].inscription_digest, previous.digest().unwrap()); - - // The invariant holds by construction rather than by inspection: a - // document built on this history validates. - at(4, history).validate().unwrap(); - } - - /// All three refusals name both sequences, because which of the three it is - /// decides what the publisher does about it. - #[test] - fn a_publish_that_does_not_follow_latest_is_refused() { - let previous = at(497, vec![]); - - for publishing in [500, 497, 496] { - let err = history_for(Some(&previous), publishing).unwrap_err(); - let message = err.to_string(); - - assert!( - matches!(err, Error::HistoryBreak { .. }), - "{publishing}: {err:?}" - ); - - assert!(message.contains("497"), "{publishing}: {message}"); - assert!( - message.contains(&publishing.to_string()), - "{publishing}: {message}" - ); - } - } - - #[test] - fn a_gap_and_a_republish_are_told_apart() { - let previous = at(497, vec![]); - - assert!(history_for(Some(&previous), 500) - .unwrap_err() - .to_string() - .contains("gap")); - - assert!(history_for(Some(&previous), 497) - .unwrap_err() - .to_string() - .contains("republish")); - - assert!(history_for(Some(&previous), 496) - .unwrap_err() - .to_string() - .contains("republish")); - } - - /// A gap says how far. "The repository is at 497 and you are at 500" is a - /// different incident from being one epoch out, and the operator reading - /// the message should not have to subtract to find out which they have. - #[test] - fn a_gap_names_the_distance_alongside_both_sequences() { - let previous = at(497, vec![]); - - let message = history_for(Some(&previous), 500).unwrap_err().to_string(); - - assert!(message.contains("497"), "{message}"); - assert!(message.contains("500"), "{message}"); - assert!(message.contains("3 sequences ahead"), "{message}"); - } - - #[test] - fn canonical_form_is_stable_and_parses_back() { - let inscription = sample(); - let canonical = inscription.canonicalize().unwrap(); - - // Canonical JSON is sorted, minimal and free of insignificant - // whitespace, so the bytes are the same on any run. - assert_eq!(canonical, inscription.canonicalize().unwrap()); - let text = String::from_utf8(canonical.clone()).unwrap(); - assert!(text.starts_with(r#"{"compression":{"algo":"zstd","level":9},"history":"#)); - assert!(!text.contains(' ')); - - let parsed = Inscription::parse(&canonical).unwrap(); - assert_eq!(parsed, inscription); - assert_eq!(parsed.digest().unwrap(), inscription.digest().unwrap()); - } - - #[test] - fn digest_is_sha256_of_the_canonical_bytes() { - let inscription = sample(); - assert_eq!( - inscription.digest().unwrap(), - Digest::compute(inscription.canonicalize().unwrap()) - ); - } - - /// Key order in the source document must not reach the digest — that is the - /// whole point of canonicalization. - #[test] - fn digest_ignores_source_key_order_and_whitespace() { - let inscription = sample(); - let canonical = inscription.canonicalize().unwrap(); - - let pretty = serde_json::to_vec_pretty(&inscription).unwrap(); - let reparsed = Inscription::parse(&pretty).unwrap(); - - assert_ne!(pretty, canonical); - assert_eq!(reparsed.canonicalize().unwrap(), canonical); - } - - #[test] - fn rejects_an_unknown_generic_key() { - let inscription = sample(); - let mut value = serde_json::to_value(&inscription).unwrap(); - value - .as_object_mut() - .unwrap() - .insert("extra".to_owned(), json!(1)); - - let err = Inscription::parse(&serde_json::to_vec(&value).unwrap()).unwrap_err(); - assert!( - matches!(&err, Error::Json(e) if e.to_string().contains("unknown field")), - "{err:?}" - ); - } - - /// Unknown keys are refused inside the generic *nested* objects too; - /// extension happens only inside the three opaque ones. - #[test] - fn rejects_an_unknown_key_in_a_nested_generic_object() { - for pointer in ["/profile", "/compression", "/layers/0", "/history/0"] { - let inscription = sample(); - let mut value = serde_json::to_value(&inscription).unwrap(); - value - .pointer_mut(pointer) - .unwrap() - .as_object_mut() - .unwrap() - .insert("extra".to_owned(), json!(1)); - - let err = Inscription::parse(&serde_json::to_vec(&value).unwrap()).unwrap_err(); - assert!( - matches!(&err, Error::Json(e) if e.to_string().contains("unknown field")), - "{pointer}: {err:?}" - ); - } - } - - /// The opaque objects, by contrast, take anything — that is the extension - /// point. - #[test] - fn opaque_objects_accept_arbitrary_shapes() { - for value in [ - json!({"deeply": {"nested": [1, 2, {"three": true}]}}), - json!([1, 2, 3]), - json!("a string"), - json!(null), - json!(0), - ] { - let mut inscription = sample(); - inscription.position = value.clone(); - inscription.parameters = value.clone(); - inscription.layers[0].scope = value.clone(); - - let canonical = inscription.canonicalize().unwrap(); - let parsed = Inscription::parse(&canonical).unwrap(); - assert_eq!(parsed.position, value); - assert_eq!(parsed.layers[0].scope, value); - } - } - - #[test] - fn rejects_an_unsupported_schema_version() { - let mut inscription = sample(); - inscription.schema = 2; - - let err = inscription.validate().unwrap_err(); - assert!( - matches!(err, Error::UnsupportedSchema { found: 2 }), - "{err:?}" - ); - - // And it must not be possible to obtain a digest for it. - assert!(inscription.canonicalize().is_err()); - } - - #[test] - fn rejects_an_unknown_profile_and_a_higher_major_version() { - let mut inscription = sample(); - inscription.profile.name = "com.acme.other".to_owned(); - let err = inscription.check_profile(&Toy).unwrap_err(); - assert!(matches!(err, Error::UnknownProfile { .. }), "{err:?}"); - - let mut inscription = sample(); - inscription.profile.version = 2; - let err = inscription.check_profile(&Toy).unwrap_err(); - assert!( - matches!( - err, - Error::UnsupportedProfileVersion { - found: 2, - supported: 1, - .. - } - ), - "{err:?}" - ); - - // An older major version is readable by a newer implementation. - let inscription = sample(); - inscription.check_profile(&Toy).unwrap(); - - // A layer kind the profile does not define is readable — skippable, and - // reported — but not publishable. - let mut inscription = sample(); - inscription.layers[0].kind = "blocks".to_owned(); - inscription.check_profile(&Toy).unwrap(); - - assert_eq!( - inscription - .unknown_layers(&Toy) - .iter() - .map(|l| l.kind.as_str()) - .collect::>(), - vec!["blocks"], - ); - - let err = inscription.check_profile_strict(&Toy).unwrap_err(); - assert!(matches!(err, Error::UnknownLayerKind { .. }), "{err:?}"); - } - - #[test] - fn rejects_a_malformed_profile_name() { - let mut inscription = sample(); - inscription.profile.name = "toy".to_owned(); - assert!(matches!( - inscription.validate().unwrap_err(), - Error::InvalidProfileName { .. } - )); - } - - #[test] - fn rejects_a_payload_media_type_in_the_reserved_namespace() { - let mut inscription = sample(); - inscription.layers[0].media_type = "application/vnd.stelae.stele.notes.v1+zstd".to_owned(); - - let err = inscription.validate().unwrap_err(); - assert!(matches!(err, Error::InvalidMediaType { .. }), "{err:?}"); - } - - #[test] - fn history_accepts_a_contiguous_ascending_chain() { - sample().validate().unwrap(); - - // The first stele of a repository carries no history. - let mut first = sample(); - first.sequence = 0; - first.history.clear(); - first.validate().unwrap(); - } - - #[test] - fn history_rejects_a_gap() { - let mut inscription = sample(); - inscription.sequence = 4; - inscription.history = vec![ - HistoryEntry { - sequence: 1, - inscription_digest: digest_of(1), - }, - HistoryEntry { - sequence: 3, - inscription_digest: digest_of(3), - }, - ]; - - let err = inscription.validate().unwrap_err(); - assert!( - matches!(&err, Error::HistoryInvariant(m) if m.contains("gap")), - "{err:?}" - ); - } - - #[test] - fn history_rejects_a_duplicate() { - let mut inscription = sample(); - inscription.history = vec![ - HistoryEntry { - sequence: 2, - inscription_digest: digest_of(1), - }, - HistoryEntry { - sequence: 2, - inscription_digest: digest_of(2), - }, - ]; - - let err = inscription.validate().unwrap_err(); - assert!( - matches!(&err, Error::HistoryInvariant(m) if m.contains("more than once")), - "{err:?}" - ); - } - - #[test] - fn history_rejects_reordering() { - let mut inscription = sample(); - inscription.history = vec![ - HistoryEntry { - sequence: 2, - inscription_digest: digest_of(2), - }, - HistoryEntry { - sequence: 1, - inscription_digest: digest_of(1), - }, - ]; - - let err = inscription.validate().unwrap_err(); - assert!( - matches!(&err, Error::HistoryInvariant(m) if m.contains("ascending")), - "{err:?}" - ); - } - - #[test] - fn history_must_reach_the_current_sequence() { - let mut inscription = sample(); - inscription.sequence = 9; - - let err = inscription.validate().unwrap_err(); - assert!( - matches!(&err, Error::HistoryInvariant(m) if m.contains("ends at sequence")), - "{err:?}" - ); - - let mut inscription = sample(); - inscription.history.push(HistoryEntry { - sequence: 3, - inscription_digest: digest_of(3), - }); - let err = inscription.validate().unwrap_err(); - assert!( - matches!(&err, Error::HistoryInvariant(m) if m.contains("not below")), - "{err:?}" - ); - } - - #[test] - fn rejects_numbers_outside_the_jcs_safe_integer_range() { - let safe = MAX_SAFE_INTEGER as u64; - - let mut inscription = sample(); - inscription.layers[0].uncompressed_size = safe; - inscription.canonicalize().unwrap(); - - let mut inscription = sample(); - inscription.layers[0].uncompressed_size = safe + 1; - let err = inscription.canonicalize().unwrap_err(); - assert!( - matches!(&err, Error::UnsafeInteger { path, .. } if path == "$.layers[0].uncompressedSize"), - "{err:?}" - ); - - // Including inside the profile's opaque objects, which is where a - // vendor would most plausibly put a raw u64. - let mut inscription = sample(); - inscription.position = json!({"raw": u64::MAX}); - let err = inscription.canonicalize().unwrap_err(); - assert!( - matches!(&err, Error::UnsafeInteger { path, .. } if path == "$.position.raw"), - "{err:?}" - ); - - let mut inscription = sample(); - inscription.parameters = json!({"nested": [0, -9_007_199_254_740_992i64]}); - let err = inscription.canonicalize().unwrap_err(); - assert!( - matches!(&err, Error::UnsafeInteger { path, .. } if path == "$.parameters.nested[1]"), - "{err:?}" - ); - } - - #[test] - fn rejects_non_integer_numbers() { - let mut inscription = sample(); - inscription.parameters = json!({"ratio": 0.5}); - - let err = inscription.canonicalize().unwrap_err(); - assert!( - matches!(&err, Error::NonIntegerNumber { path, .. } if path == "$.parameters.ratio"), - "{err:?}" - ); - - // `56.0` is an integral value but a JSON float, and RFC 8785 renders it - // through the float path. The rule is about the encoding, not the value. - let mut inscription = sample(); - inscription.parameters = json!({"whole": 56.0}); - assert!(matches!( - inscription.canonicalize().unwrap_err(), - Error::NonIntegerNumber { .. } - )); - } - - #[test] - fn parse_rejects_unsafe_numbers_in_raw_bytes() { - let raw = br#"{"schema":1,"profile":{"name":"dev.example.toy","version":1},"sequence":1,"position":{"n":18446744073709551615},"parameters":{},"compression":{"algo":"zstd","level":9},"history":[],"layers":[]}"#; - - let err = Inscription::parse(raw).unwrap_err(); - assert!(matches!(err, Error::UnsafeInteger { .. }), "{err:?}"); - } - - #[test] - fn uncompressed_size_sums_the_layers() { - let mut inscription = sample(); - inscription.layers.push(LayerDescriptor { - kind: "notes".to_owned(), - media_type: "application/vnd.example.stele.notes.v1+zstd".to_owned(), - diff_id: digest_of(4), - records: 2, - uncompressed_size: 72, - scope: json!({"chapter": 4}), - }); - - assert_eq!(inscription.uncompressed_size(), 200); - assert_eq!(inscription.layers_of_kind("notes").count(), 2); - assert_eq!(inscription.layers_of_kind("blocks").count(), 0); - } -} diff --git a/crates/stelae/src/layer.rs b/crates/stelae/src/layer.rs deleted file mode 100644 index 303c0273a..000000000 --- a/crates/stelae/src/layer.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! Reading a layer without holding it. -//! -//! [`crate::digest::read_blob`] decompresses a whole layer into a `Vec` and -//! hands it over. That is the right shape for a fixture and the wrong one for -//! the sizes a profile publishes: ADR-004's worked example gives a state shard -//! of 402,653,184 uncompressed bytes, and one mainnet epoch of blocks runs to -//! 0.5–1.5 GB. The write path never had this problem — -//! [`crate::digest::LayerWriter`] hashes, compresses and hashes again in one -//! pass with nothing buffered — so the asymmetry was one-sided and easy to -//! miss. -//! -//! [`LayerReader`] closes it: one pass over the compressed blob, records -//! yielded out of a bounded window, both digests established on the way past. -//! -//! ## Two bounds, and why one does not subsume the other -//! -//! - **Per record** ([`crate::frame::Limits`]) — a record is held whole, so its -//! size is checked against a ceiling before the window grows for it. -//! - **Per layer** (the descriptor's `uncompressedSize`) — the record bound -//! says nothing about how *many* records arrive. A blob whose descriptor -//! claims 400 MB can stream 400 GB one small record at a time and never trip -//! a per-record ceiling. `read_blob` refuses that blob with -//! [`Error::DecompressedTooLarge`]; so does this, at the same threshold, for -//! the same reason. -//! -//! ## Records arrive before the layer is proven -//! -//! A layer's `diffId` covers its whole byte string, so it cannot be confirmed -//! until the last record has gone past. Buying earlier verification would mean -//! a second pass over those 400 MB. This crate keeps the single pass and states -//! the consequence instead: **records are consumable before the layer is -//! proven, and only [`LayerReader::finish`] proves it.** A consumer must not -//! commit a checkpoint over records it has read until `finish` returns `Ok` — -//! which is what the Dolos restore pipeline already does, recording *completed* -//! layer diffIds in its progress file. - -use std::io::{self, Read}; - -use sha2::{Digest as _, Sha256}; - -use crate::{ - digest::{Digest, LayerDigests, Tap}, - frame::{LayerHeader, Limits, RecordReader}, - inscription::LayerDescriptor, - profile::Profile, - Error, -}; - -/// The compressed-blob pipeline, read end to end: -/// file → [`Tap`] (blob digest) → zstd → [`Meter`] (diffId, ceiling) → records. -type Pipeline = Meter>>>; - -/// A layer being read from a compressed blob, one record at a time. -/// -/// Every claim its descriptor makes is checked, and each one as early as the -/// bytes allow: the header record's profile and kind on construction, the -/// decompression ceiling as the stream advances, the identity digest, the -/// uncompressed size and the record count in [`LayerReader::finish`]. -pub struct LayerReader { - records: RecordReader>, - header: LayerHeader, - descriptor: LayerDescriptor, -} - -/// The descriptor and header a reader is verifying against, and nothing about -/// the stream itself — the same shape [`crate::dir::Layer`] reports, and for -/// the same reason: a caller that has to say *which* layer failed needs the -/// claims, not the pipeline. -impl std::fmt::Debug for LayerReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LayerReader") - .field("header", &self.header) - .field("descriptor", &self.descriptor) - .finish_non_exhaustive() - } -} - -impl LayerReader { - /// Open `source` — the compressed bytes of the layer `descriptor` - /// describes — and read its header record. - /// - /// Fails immediately if the blob's own header names a different profile or - /// kind than the document pointing at it: a layer that disagrees with its - /// descriptor is refused before any of its content is handed out. - pub fn new( - source: R, - profile: &dyn Profile, - descriptor: &LayerDescriptor, - limits: Limits, - ) -> Result { - let tap = Tap::new(source); - let decoder = zstd::stream::read::Decoder::new(tap)?; - let meter = Meter::new(decoder, descriptor.uncompressed_size); - - let mut records = RecordReader::with_limits(meter, limits); - - let header = match records.next_record() { - Some(Ok(record)) => LayerHeader::decode(record)?, - Some(Err(e)) => return Err(lift(e)), - None => { - return Err(Error::LayerMismatch { - kind: descriptor.kind.clone(), - reason: "layer is empty; every layer starts with a header record".to_owned(), - }) - } - }; - - check_header(&header, profile, descriptor)?; - - Ok(Self { - records, - header, - descriptor: descriptor.clone(), - }) - } - - /// The layer's header record, parsed and checked against the descriptor. - pub fn header(&self) -> &LayerHeader { - &self.header - } - - /// The next of the profile's content records, header excluded. - /// - /// Borrowed out of the reader's window, which is why this is not an - /// [`Iterator`]: the borrow ends before the window is refilled. The record - /// is canonical — that much is proven — but the *layer* is not proven until - /// [`LayerReader::finish`] says so. - pub fn next_record(&mut self) -> Option> { - match self.records.next_record() { - Some(Err(e)) => Some(Err(lift(e))), - other => other, - } - } - - /// Finish the read: drain whatever is left, then confirm the layer. - /// - /// This is the point at which the layer becomes trustworthy. It reads to - /// the end of the blob — the identity digest covers every byte, so there is - /// no confirming it early — and checks the record count, the uncompressed - /// size and the `diffId` against the descriptor. Anything a caller did with - /// records before this returned `Ok` was done on faith. - pub fn finish(mut self) -> Result { - loop { - match self.records.next_record() { - Some(Ok(_)) => {} - Some(Err(e)) => return Err(lift(e)), - None => break, - } - } - - // A caller that read a bad record and carried on anyway must not be - // handed a confirmation. Most such layers fail the digest check below - // — the bytes after the bad record were never read, so they never - // reached the hasher — but a layer whose *last* record is malformed - // hashes and sizes exactly as its descriptor claims. Refusing here is - // what makes "`finish` confirms the layer" true rather than usually - // true. - if self.records.failed() { - return Err(Error::LayerMismatch { - kind: self.descriptor.kind.clone(), - reason: "a record failed validation; the layer cannot be confirmed".to_owned(), - }); - } - - let count = self.records.count(); - let meter = self.records.into_inner(); - - let (decoder, diff_id, uncompressed_size) = meter.finish(); - - // The decoder ran to EOF, so the tap has already seen the whole blob; - // the drain is a safety net for a decoder that stops at a frame - // boundary instead. `BufReader` may hold bytes it read ahead, and those - // passed through the tap on the way in, so nothing is missed by - // dropping it. - let mut tap = decoder.finish().into_inner(); - io::copy(&mut tap, &mut io::sink())?; - - let (_, blob_digest, compressed_size) = tap.finish(); - - let digests = LayerDigests { - diff_id, - blob_digest, - uncompressed_size, - compressed_size, - }; - - check_identity(&digests, &self.descriptor)?; - check_record_count(count, &self.descriptor)?; - - Ok(digests) - } -} - -/// The layer header a blob carries has to agree with the document that points -/// at it, and with the profile the caller implements. -/// -/// Shared by both read paths so a header cannot be acceptable to one and not -/// the other. -pub(crate) fn check_header( - header: &LayerHeader, - profile: &dyn Profile, - descriptor: &LayerDescriptor, -) -> Result<(), Error> { - if header.profile != profile.name() { - return Err(Error::UnknownProfile { - found: header.profile.clone(), - expected: profile.name().to_owned(), - }); - } - - if header.kind != descriptor.kind { - return Err(Error::LayerMismatch { - kind: descriptor.kind.clone(), - reason: format!("header record names kind {:?}", header.kind), - }); - } - - Ok(()) -} - -/// The identity digest and the size the descriptor claims, against what the -/// bytes turned out to be. -pub(crate) fn check_identity( - digests: &LayerDigests, - descriptor: &LayerDescriptor, -) -> Result<(), Error> { - if digests.diff_id != descriptor.diff_id { - return Err(Error::DigestMismatch { - subject: format!("layer {:?}", descriptor.kind), - expected: descriptor.diff_id.to_string(), - actual: digests.diff_id.to_string(), - }); - } - - if digests.uncompressed_size != descriptor.uncompressed_size { - return Err(Error::LayerMismatch { - kind: descriptor.kind.clone(), - reason: format!( - "descriptor claims {} uncompressed bytes, blob holds {}", - descriptor.uncompressed_size, digests.uncompressed_size - ), - }); - } - - Ok(()) -} - -/// The record count, header record included. -pub(crate) fn check_record_count(records: u64, descriptor: &LayerDescriptor) -> Result<(), Error> { - if records != descriptor.records { - return Err(Error::LayerMismatch { - kind: descriptor.kind.clone(), - reason: format!( - "descriptor claims {} records, blob holds {records}", - descriptor.records - ), - }); - } - - Ok(()) -} - -/// Recover a protocol error that a pipeline stage had to smuggle through -/// [`io::Error`]. -/// -/// [`Read`] can only fail with an `io::Error`, but a decompression ceiling is a -/// protocol verdict, not an I/O fault. [`Meter`] boxes the real error inside -/// one; this takes it back out, so a caller matches on -/// [`Error::DecompressedTooLarge`] whichever path produced it. -fn lift(e: Error) -> Error { - match e { - Error::Io(io) if io.get_ref().is_some_and(|inner| inner.is::()) => *io - .into_inner() - .expect("checked by the guard") - .downcast::() - .expect("checked by the guard"), - other => other, - } -} - -/// Hashes, counts and bounds the uncompressed side of a layer stream. -/// -/// The ceiling is checked before the bytes are handed on, so it bounds what the -/// reader above is ever asked to hold — the same discipline -/// [`crate::digest::read_blob`] applies to its buffer. -struct Meter { - inner: R, - hasher: Sha256, - total: u64, - limit: u64, -} - -impl Meter { - fn new(inner: R, limit: u64) -> Self { - Self { - inner, - hasher: Sha256::new(), - total: 0, - limit, - } - } - - fn finish(self) -> (R, Digest, u64) { - ( - self.inner, - Digest::from_bytes(self.hasher.finalize().into()), - self.total, - ) - } -} - -impl Read for Meter { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let read = self.inner.read(buf)?; - - self.total += read as u64; - - if self.total > self.limit { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - Error::DecompressedTooLarge { limit: self.limit }, - )); - } - - self.hasher.update(&buf[..read]); - - Ok(read) - } -} diff --git a/crates/stelae/src/lib.rs b/crates/stelae/src/lib.rs deleted file mode 100644 index 46a4aa205..000000000 --- a/crates/stelae/src/lib.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! # Stelae -//! -//! A deterministic, content-addressed snapshot protocol. -//! -//! A *stele* is one published artifact set at one sequence point: an -//! **inscription** (the canonical-JSON document whose sha256 is the stele's -//! identity) plus its **layers** (content-addressed blobs of deterministic CBOR -//! records). Independent publishers that hold the same source data reproduce -//! the same inscription digest byte-for-byte, which is what makes k-of-n -//! attestation possible without a central authority. -//! -//! This crate is the *protocol*. It knows nothing about any particular dataset: -//! layer kinds, record shapes, tag strings and the contents of `position`, -//! `parameters` and `scope` all belong to a [`Profile`], which a vendor -//! supplies. The protocol handles framing, canonicalization, digests and the -//! naming rules that let distinct vendors coexist in one registry. -//! -//! The normative specification is `adrs/004_stelae_snapshots.md`. -//! -//! ## Module map -//! -//! - [`frame`] — deterministic CBOR sequences (RFC 8742 under the RFC 8949 -//! §4.2.1 profile) and the protocol-owned layer header record. -//! - [`digest`] — the streaming sha256 + zstd pipeline that yields a layer's -//! identity digest (`diffId`) and its transport digest in one pass. -//! - [`layer`] — reading a layer as a stream: records out of a bounded window, -//! both digests on the way past, confirmation at the end. -//! - [`inscription`] — the inscription schema, its RFC 8785 canonicalization, -//! its digest, and the `history` attestation invariant. -//! - [`profile`] — the [`Profile`] trait plus the protocol's media-type, -//! profile name and tag naming rules. -//! - [`transport`] — the seam a stele is written through and read back from: -//! the two halves a profile uses, the `diffId`→blob map both transports -//! answer differently, and the discarding writer that computes a stele's -//! identity without storing it. -//! - [`plan`] — what a restore has already done and what it has left to fetch: -//! the progress file, the resume rule, and remaining-bytes accounting. -//! - [`progress`] — the one seam a publish and a restore report through while -//! they run: an observer a caller passes in, callbacks only, silent by -//! default. -//! - [`dir`] — a minimal on-disk stele: the first implementation of that seam, -//! and the one a stele is inspectable by hand through. -//! - [`oci`] — (feature `oci`) an OCI registry as the other implementation: -//! push with blob-skip, pull by manifest. -//! -//! ## Boundaries this crate keeps -//! -//! - It never *constructs* a payload media type or a tag string. It asks the -//! profile and validates the answer against the normative naming rules. -//! - It never deserializes `position`, `parameters` or a layer's `scope` into a -//! typed value. They are opaque: canonicalized and hashed, never interpreted. -//! - It has no `dolos-*` dependency, so extracting it later is a directory move -//! rather than a refactor. - -pub mod codec; -pub mod digest; -pub mod dir; -pub mod frame; -pub mod inscription; -pub mod layer; -#[cfg(feature = "oci")] -pub mod oci; -pub mod plan; -pub mod profile; -pub mod progress; -pub mod transport; - -pub use digest::{Digest, LayerDigests, LayerWriter}; -pub use frame::{CanonicalCbor, LayerHeader, Limits, Measure, RecordReader, SeqReader, SeqWriter}; -pub use inscription::{ - canonical_json, Compression, HistoryEntry, Inscription, LayerDescriptor, ProfileRef, -}; -pub use layer::LayerReader; -pub use plan::{Remaining, RestoreProgress, Resume}; -pub use profile::{MediaType, Profile}; -pub use progress::{Event, Observer, Outcome, Progress}; -pub use transport::{ - BlobIndex, Discarding, DiscardingSink, LayerSpec, RecordSink, SteleReader, SteleWriter, - WrittenLayer, -}; - -/// Artifact type of a stele manifest. Generic tooling discovers stelae of every -/// profile by filtering on this; the inscription's `profile` field -/// discriminates. -pub const ARTIFACT_TYPE: &str = "application/vnd.stelae.stele.v1"; - -/// Media type of the inscription (the OCI config blob). -pub const INSCRIPTION_MEDIA_TYPE: &str = "application/vnd.stelae.inscription.v1+json"; - -/// Media type of a detached signature over an inscription digest, attached as -/// an OCI referrer artifact. Signing itself is not implemented in this crate -/// yet. -pub const SIGNATURE_MEDIA_TYPE: &str = "application/vnd.stelae.signature.v1"; - -/// Vendor token reserved by the protocol. `application/vnd.stelae.*` names -/// envelope types only and is never a payload media type — no profile may claim -/// it. See `profile::MediaType`. -pub const RESERVED_VENDOR: &str = "stelae"; - -/// Inscription schema version this implementation writes and accepts. A -/// verifier fails closed on any other value rather than guessing at a newer -/// layout. -pub const SCHEMA_VERSION: u64 = 1; - -/// Version of the layer header record this implementation writes and accepts. -pub const LAYER_FORMAT_VERSION: u64 = 1; - -/// The moving tag every profile is required to maintain alongside its immutable -/// per-sequence tags. -pub const MOVING_TAG: &str = "latest"; - -/// Ceiling on the size of a stele's OCI manifest. -/// -/// Not a spec limit — the OCI distribution specification sets none — but the -/// figure registries converge on, and the one ADR-004 sized the format against -/// ("~1,700 manifest descriptors, well under the 4 MiB manifest guidance"). A -/// stele that exceeds it is refused before the push rather than after a -/// registry answers `413`, because the failure is a property of the document -/// and the same everywhere. See [`oci`]. -pub const MANIFEST_SIZE_LIMIT: usize = 4 * 1024 * 1024; - -/// Largest integer RFC 8785 (via ECMAScript number serialization) renders -/// exactly: `2^53 - 1`. -/// -/// Every number in an inscription — including numbers inside the profile's -/// opaque objects — must be an integer within `±MAX_SAFE_INTEGER`. Past that -/// point two conformant JCS implementations still agree with each other but no -/// longer agree with the value the producer meant, and the divergence is -/// silent. The protocol therefore refuses the value rather than canonicalizing -/// it. See [`inscription::check_safe_numbers`]. -pub const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; - -/// Errors raised by the protocol. -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("json error: {0}")] - Json(#[from] serde_json::Error), - - #[error("cbor encoding error: {0}")] - CborEncode(String), - - /// The bytes are valid CBOR but violate the deterministic-encoding profile, - /// or use a construct the format excludes (tags, floats, indefinite - /// lengths). - #[error("non-canonical cbor at offset {offset}: {reason}")] - NonCanonicalCbor { offset: usize, reason: String }, - - #[error("truncated cbor: expected {expected} more byte(s) at offset {offset}")] - TruncatedCbor { offset: usize, expected: usize }, - - #[error("cbor nesting deeper than the protocol limit of {limit}")] - CborTooDeep { limit: usize }, - - /// A record whose length prefixes claim more than the ceiling its reader - /// was given. Raised from the prefix, before anything is allocated to hold - /// the record, so a corrupt or hostile length costs a comparison. - #[error( - "record at offset {offset} needs at least {required} bytes, past the \ - {limit}-byte ceiling set for a single record" - )] - RecordTooLarge { - offset: usize, - required: u64, - limit: usize, - }, - - #[error("expected exactly one cbor data item, found {trailing} trailing byte(s)")] - TrailingCbor { trailing: usize }, - - #[error("malformed layer header record: {0}")] - MalformedHeader(String), - - #[error("canonicalization failed: {0}")] - Canonicalization(String), - - /// A number outside the range RFC 8785 renders exactly. Refused rather than - /// silently rounded. - #[error( - "number at {path} is outside the JCS-safe integer range (±{MAX_SAFE_INTEGER}): {value}" - )] - UnsafeInteger { path: String, value: String }, - - /// A non-integer number. The inscription schema admits integers only, so - /// that canonicalization never depends on floating-point rendering. - #[error("number at {path} is not an integer: {value}")] - NonIntegerNumber { path: String, value: String }, - - #[error("unsupported inscription schema {found}; this implementation writes and accepts {SCHEMA_VERSION}")] - UnsupportedSchema { found: u64 }, - - #[error("inscription is for profile {found:?}; this implementation implements {expected:?}")] - UnknownProfile { found: String, expected: String }, - - #[error( - "inscription requires profile {name:?} major version {found}; this implementation implements {supported}" - )] - UnsupportedProfileVersion { - name: String, - found: u64, - supported: u64, - }, - - #[error("history invariant violated: {0}")] - HistoryInvariant(String), - - /// A publish that would not extend the repository's chain. - /// - /// Both sequences are in the message because the fix depends on which of - /// them is wrong: a gap means a publisher skipped epochs, an equal or lower - /// sequence means it is republishing one. There is deliberately no flag - /// that overrides this — see [`inscription::history_for`]. - /// - /// `reason` is owned rather than static so a gap can state its *distance*. - /// "The repository is at 500 and you are at 540" is a different incident - /// from being one epoch out, and an operator reading the message should not - /// have to subtract. - #[error( - "this repository's latest stele is sequence {latest} and this publish is sequence \ - {publishing}: {reason}" - )] - HistoryBreak { - latest: u64, - publishing: u64, - reason: String, - }, - - /// A profile's content record whose shape is not the one its kind declares - /// — the field count, a field's type or width, or bytes trailing the - /// record. Raised by [`codec`], which is where every profile checks it. - #[error("malformed {kind} record: {reason}")] - MalformedRecord { kind: &'static str, reason: String }, - - #[error("invalid digest {value:?}: {reason}")] - InvalidDigest { value: String, reason: String }, - - #[error("invalid media type {value:?}: {reason}")] - InvalidMediaType { value: String, reason: String }, - - #[error("invalid profile name {value:?}: {reason}")] - InvalidProfileName { value: String, reason: String }, - - #[error("invalid tag {value:?}: {reason}")] - InvalidTag { value: String, reason: String }, - - #[error("profile {profile:?} does not define layer kind {kind:?}")] - UnknownLayerKind { profile: String, kind: String }, - - #[error("digest mismatch for {subject}: expected {expected}, computed {actual}")] - DigestMismatch { - subject: String, - expected: String, - actual: String, - }, - - #[error("layer {kind:?} ({diff_id}) is not present in this stele")] - LayerNotFound { kind: String, diff_id: String }, - - #[error("layer {kind:?} disagrees with its descriptor: {reason}")] - LayerMismatch { kind: String, reason: String }, - - /// A blob expanded past the ceiling its caller set. Raised while - /// decompressing, before the bytes are held, so a hostile compression ratio - /// costs a bounded allocation rather than the process. - #[error("blob decompresses past the {limit}-byte ceiling set for it")] - DecompressedTooLarge { limit: u64 }, - - #[error("inscription.json is not in canonical form; its bytes must be exactly the RFC 8785 encoding of its content")] - NonCanonicalInscription, - - /// The manifest and the inscription describe different sets of layers. - /// - /// They are two views of one stele — the inscription holds identity, the - /// manifest holds transport — and the whole reason a registry can hand over - /// a `diffId`→blob map for free is that the two agree. A disagreement is a - /// refusal, in either direction: a layer the document describes and the - /// manifest does not carry cannot be fetched, and a layer the manifest - /// carries and the document does not describe is a blob nothing attests. - #[error("manifest disagrees with the inscription: {0}")] - ManifestMismatch(String), - - /// A manifest past [`MANIFEST_SIZE_LIMIT`]. - #[error( - "the manifest for this stele is {size} bytes, past the {MANIFEST_SIZE_LIMIT}-byte ceiling \ - registries converge on; it describes {layers} layer(s)" - )] - ManifestTooLarge { size: usize, layers: usize }, - - /// A layer a publisher asked to carry forward, whose blob the repository no - /// longer holds. - /// - /// Distinct from [`Error::LayerNotFound`], which is about a stele that does - /// not describe a layer. This one is the opposite shape: a stele describes - /// it, and the bytes are gone — a descriptor pointing at a blob a registry - /// has reclaimed. Publishing it would produce a well-formed stele nobody - /// can restore, so it is refused where it is discovered rather than where - /// it would eventually be noticed. - #[cfg(feature = "oci")] - #[error( - "layer {kind:?} ({diff_id}) cannot be carried forward: this repository no longer holds \ - its blob {blob}" - )] - BlobMissing { - kind: String, - diff_id: String, - blob: String, - }, - - /// A seal asked again of a transport whose layers did not all land. - /// - /// The publish path moves its layers concurrently and joins them at the - /// seal, so a layer's failure is reported *there* — once, as itself, with - /// whatever the registry or the network actually said. This is what every - /// seal after that one answers, and it is a refusal rather than a retry - /// because the failure is not one the transport can undo: a layer's staging - /// went with the round trip that lost it, and the bytes only exist in the - /// store the publisher built them from. - /// - /// The retry that *could* have undone it already happened, inside the round - /// trip and while the staging was still in hand — see - /// [`oci::Options::attempts`]. So a failure that gets this far is one the - /// registry repeated. - /// - /// So the recovery is a new publish, not another seal. Carrying the string - /// rather than the error keeps the original readable across the many later - /// calls that report it, which is the whole reason this variant exists - /// instead of a second copy of the cause. - #[cfg(feature = "oci")] - #[error("this publish cannot be sealed: a layer never reached the repository ({0})")] - LayerNotWritten(String), - - /// A repository name an operator gave that cannot address a repository. - /// - /// Raised while *reading* a name, never while using one, so a caller can - /// refuse a typo before it does anything the typo would have cost. - #[cfg(feature = "oci")] - #[error("{value:?} is not an OCI repository: {reason}")] - InvalidRepository { value: String, reason: String }, - - /// A staging directory the transport could not use. - /// - /// Distinct from [`Error::Io`], which is the catch-all every other bare - /// `?` in this crate falls through, because this is the one path an - /// operator types on a command line — `dolos snapshot publish - /// --scratch-dir`, `dolos bootstrap stelae --scratch-dir`, or the default - /// the binary derives from `storage.path`. A typo, an unmounted volume, a - /// directory owned by somebody else and a path that is already a regular - /// file all arrive here, and `std::io::Error` carries none of them: it - /// knows the errno and not the path that produced it. So the path is - /// captured where it is still in scope, which is the only place it can be. - /// - /// The message deliberately does *not* repeat what the operating system - /// said — that stays on the source, one line further down the chain, so a - /// rendered report says each thing once. - #[cfg(feature = "oci")] - #[error("cannot use {} as the staging directory", dir.display())] - Scratch { - dir: std::path::PathBuf, - #[source] - source: std::io::Error, - }, - - /// Anything the registry client reported. - #[cfg(feature = "oci")] - #[error("registry error: {0}")] - Registry(#[from] oci_client::errors::OciDistributionError), -} - -impl Error { - pub(crate) fn malformed(kind: &'static str, reason: impl Into) -> Self { - Self::MalformedRecord { - kind, - reason: reason.into(), - } - } -} diff --git a/crates/stelae/src/oci.rs b/crates/stelae/src/oci.rs deleted file mode 100644 index 379ec72bb..000000000 --- a/crates/stelae/src/oci.rs +++ /dev/null @@ -1,3092 +0,0 @@ -//! A stele in an OCI registry. -//! -//! This is the transport the format was designed for. Registries are -//! content-addressed, so a push asks whether each blob is already there and -//! sends only the ones that are not, and a pull reads a manifest that says -//! exactly which blob holds which layer. Everything before this module produced -//! a format *capable* of delta transfer; this is the half that performs any. -//! -//! It is the second implementation of [`crate::transport`] and adds no -//! vocabulary of its own: a registry is another place a -//! [`crate::dir::SteleDir`] could have been, and a profile driving one writes -//! the same code. -//! -//! ## Two documents, one stele -//! -//! A stele in a registry is an OCI image manifest whose config blob is the -//! inscription: -//! -//! ```text -//! manifest artifactType application/vnd.stelae.stele.v1 -//! config application/vnd.stelae.inscription.v1+json -> the inscription -//! layers[] the profile's media types, in inscription order, -//! each annotated with its kind, its diffId and its scope -//! ``` -//! -//! The two documents describe one thing from two sides. The inscription holds -//! **identity** — `diffId`s, over uncompressed bytes, reproducible by a -//! publisher who compressed differently. The manifest holds **transport** — -//! compressed digests and sizes, which are what a registry addresses a blob by -//! and are not stable across zstd versions. Neither is derivable from the -//! other, which is why [`crate::BlobIndex`] exists at all, and why a directory -//! has to reconstruct by brute force what a manifest states. -//! -//! Because they are two views of one stele, **a disagreement between them is a -//! refusal**, in either direction: a layer the inscription describes and the -//! manifest does not carry cannot be fetched, and a layer the manifest carries -//! and the inscription does not describe is a blob nothing attests. Positional -//! correspondence is checked as well as the `diffId` annotations, so the -//! ordering the canonical document fixes is the ordering on the wire. -//! -//! ## Bounded by one layer, in both directions -//! -//! Both push paths need the blob's digest *up front*, and a layer's digest is -//! only known once its last record has been written. So a layer is staged into -//! a temporary file exactly as a directory stages one, and then sent up from -//! it. A pulled layer is streamed to a temporary file and read back -//! synchronously. -//! -//! Neither direction ever holds a whole stele. The staging files are unlinked -//! at creation, so an abandoned push or a failed pull leaves nothing behind and -//! needs no cleanup path of its own. -//! -//! What the *upload* holds is decided by [`Options::monolithic_max`]. Above it, -//! a layer is streamed and one [`UPLOAD_CHUNK`] is resident at a time. At or -//! below it, the layer goes up as one request — a `POST` and a `PUT` carrying -//! the whole body — and is resident in full while it does, because the client -//! takes the body as bytes and there is no ordering in which it does not. -//! -//! That is bought deliberately: a `PATCH` costs the registry about three -//! seconds whatever it carries, so a publish's wall clock is its request count, -//! and 79 of mainnet's 81 layers fit under the threshold. What keeps the price -//! bounded is [`Options::upload_memory`] — a budget in *bytes*, spent by the -//! layers actually in flight rather than inferred from how many of them there -//! are. See [`Shared::resident`]. -//! -//! ## The async boundary, and the one rule it comes with -//! -//! `oci-client` is async and this crate is not: `export` and `restore` are -//! synchronous iterator code driving fallible store iterators, and threading a -//! runtime through them would change every profile's shape for the benefit of -//! one transport. So the transport owns **one runtime** and enters it with -//! `block_on` at each call — the idiom `dolos bootstrap mithril` already uses. -//! -//! **A [`Registry`] must never be used from inside an async context, and must -//! never be dropped inside one.** `Runtime::block_on` panics when called from a -//! runtime thread, and dropping a runtime from inside one panics too. Every -//! caller today is a synchronous CLI path, which is what makes this safe; a -//! caller that is not is a design question, and the answer is not a second -//! runtime. -//! -//! Several synchronous caller threads at once are fine, and the publish path -//! uses them: a profile driver may open and finish sinks from a pool of its -//! own producer threads. Everything those calls share — the push state, the -//! in-flight list, the permits — is behind its own lock, and `block_on` from -//! many non-runtime threads is exactly what a runtime is for. -//! -//! ## Why the publish path is concurrent, and where it joins again -//! -//! A publish is not one transfer; it is a few hundred small ones. A stele's -//! layers are cut at record-type and epoch granularity, so a mainnet publish -//! moves tens of new blobs of half a megabyte each and carries hundreds of -//! older ones forward — and every one of those, new or carried, costs at least -//! one round trip to a registry that may be an ocean away. Run in sequence, the -//! path spends nearly all of its wall clock waiting on a socket with the CPU -//! and the link both idle, and the carried-forward half makes it *worse every -//! epoch*: the stele gains layers, so the publish gains round trips, so the -//! cycle time grows linearly with the history behind it. -//! -//! Nothing about that is a bandwidth problem, so the answer is not bigger -//! layers — the cut geometry is the profile's, and it is deliberate. The answer -//! is to stop doing one round trip at a time. Every layer's round trips are -//! independent of every other layer's: a blob is addressed by its own content, -//! and no blob's upload observes another's. So they are **deferred onto the -//! runtime and run concurrently**, bounded by [`Options::concurrency`], and the -//! caller's thread goes back to reading the store rather than waiting on a -//! `PATCH`. -//! -//! What is *not* independent is the manifest, and that is the whole of the -//! safety argument this concurrency has to preserve: -//! -//! > **A manifest must never name a blob the registry has not committed.** -//! -//! So [`SteleWriter::seal`] is the join. Every deferred round trip is awaited -//! there — before the manifest is built, before the config blob goes up, before -//! either tag is written — and the first failure among them fails the seal, in -//! the state a failed seal has always left behind: layers unspent, nothing -//! tagged, and the caller free to seal again. A publish that dies mid-flight -//! leaves untagged blobs the registry reclaims, exactly as a serial one did. -//! -//! The bound is a permit taken *before* the staged layer is handed over rather -//! than inside the task, so it is also what keeps the scratch directory from -//! filling: at most [`Options::concurrency`] staged layers exist at once, -//! whatever order the sinks finish in. -//! -//! It is not the bound on memory, and reading it as one is the mistake this -//! paragraph exists to prevent. A layer count says nothing about bytes when the -//! layers differ in size by three orders of magnitude — mainnet's median layer -//! is 0.41 MB and its largest is 231 MB — and the single-request path spends -//! bytes. So there is a second permit, taken in the task where the size is -//! finally known, one per resident byte, against [`Options::upload_memory`]. -//! -//! ## A round trip nobody answered is made again -//! -//! Concurrency raised the number of round trips in flight; it did nothing about -//! the ones that fail for no reason and would have worked a second later. Over -//! eleven hours of the mainnet backfill the registry answered a create-session -//! `POST` with a bare `500` eight times, each lasting the milliseconds it took -//! to ask again — and each one cost the whole epoch, because a publish that -//! could not seal took the driver down with it, and the driver's recovery is to -//! restore its stores and replay the epoch it lost. -//! -//! That is the most expensive recovery in the system, bought for the cheapest -//! failure there is. So a round trip is **attempted [`Options::attempts`] -//! times**, with a doubling wait between them, and the failure the caller keeps -//! is the last attempt's: -//! -//! - **a `5xx`** — the registry answered, and what it said was about itself -//! rather than about the request. Asking again is the whole remedy; -//! - **no answer at all** — a connection refused, a request that timed out, a -//! socket that went away mid-body. -//! -//! Nothing else. A `4xx` is the registry saying something true about *this* -//! request — the credential, the digest, the name — and repeating it four times -//! only makes the diagnosis take longer to arrive. `429` is excluded on purpose -//! and not by omission: a registry rationing this publisher is a fact its -//! operator has to see, and a client that absorbed it would report the ration -//! as slowness. -//! -//! This is safe to do at every seam because every one of them is idempotent by -//! construction. A blob is addressed by the digest of its own content, so an -//! upload that half-happened and an upload that fully happened both converge on -//! the same blob when it is sent again; a `HEAD` and a manifest `GET` are -//! reads; and the manifest `PUT` writes bytes that are a pure function of the -//! stele. The one thing a retry needs and the serial path did not is the -//! staging file back at its first byte, which is why it is rewound per attempt -//! rather than consumed once. -//! -//! Every retry is announced through [`Event::Retry`], because a transport that -//! silently absorbed the failure class would have hidden the measurement that -//! motivated absorbing it. -//! -//! ## TLS, and the second rule it comes with -//! -//! The client speaks TLS through rustls, built with **no crypto provider wired -//! in** (`reqwest/rustls-no-provider`). The alternative is the backend -//! `oci-client`'s own `rustls-tls` feature selects, `aws-lc-rs`, whose -//! `aws-lc-sys` needs `cmake` on the build machine — a build tool this -//! protocol will not make a contributor install to compile a snapshot format. -//! `crates/stelae/Cargo.toml` records which dependency each half of that -//! choice lands on. -//! -//! The trade is the same one the async boundary makes: a guarantee moves from -//! build time to run time. -//! -//! **A process that opens a [`Registry`] must have installed a process-default -//! [`rustls`] `CryptoProvider` before it does so.** Nothing here can do it — -//! the choice of provider belongs to the program, not to one of its -//! transports, and a library that installed one would silently win a race -//! against whatever its host had chosen. Omitting it panics inside -//! [`Registry::open`], where `oci-client` builds its HTTP client: `reqwest` -//! resolves its TLS backend there, before any request and before any URL -//! scheme, so [`Options::insecure`] does not spare a plaintext registry. -//! That is the worse failure mode being bought — a runtime abort rather than a -//! link error — though the panic does name the missing feature. -//! -//! In Dolos this is `main()`, which installs `ring` for `mithril-client`'s -//! sake and covers this transport by the same line. In this crate's own tests -//! it is an explicit install in the fixture, so the suite proves the -//! precondition rather than inheriting a provider by luck. -//! -//! [`rustls`]: https://docs.rs/rustls -//! -//! ## Authentication -//! -//! Anonymous, a bearer token, or a Basic credential pair — whichever the caller -//! puts in [`Options::auth`]. That is the whole of it: [`Auth`] is a value the -//! caller constructs and hands over. -//! -//! **Where those credentials came from is not this crate's business, and it has -//! no way to ask.** A protocol library that read an environment variable would -//! be deciding its host's credential policy for it, and naming the variable -//! would freeze that decision into a published API — a program embedding this -//! transport gets no say in either. So a host reads its own environment, its -//! own configuration file, its own secret manager, or all three in whatever -//! order it has decided, and the answer arrives here as an [`Auth`]. -//! -//! In Dolos that host is the `dolos` binary; `dolos::common` holds the -//! variables and the precedence between them. - -use std::{ - collections::BTreeMap, - fs::File, - io::{Read, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, - pin::Pin, - sync::{Arc, Mutex}, - task::{Context, Poll}, - time::Duration, -}; - -use futures_util::Stream; -use oci_client::{ - client::{ClientConfig, ClientProtocol}, - manifest::{OciDescriptor, OciImageManifest, OCI_IMAGE_MEDIA_TYPE}, - secrets::RegistryAuth, - Client, -}; - -pub use oci_client::Reference; - -use crate::{ - digest::{read_uninterrupted, LayerDigests, LayerWriter}, - frame::{CanonicalCbor, Limits, SeqWriter}, - inscription::{canonical_json, Inscription, LayerDescriptor}, - layer::LayerReader, - profile::{checked_tag_for_sequence, validate_tag, Profile}, - progress::{Event, Observer}, - transport::{ - open_layer, BlobIndex, LayerSpec, RecordSink, SteleReader, SteleWriter, WrittenLayer, - }, - Digest, Error, ARTIFACT_TYPE, INSCRIPTION_MEDIA_TYPE, MANIFEST_SIZE_LIMIT, -}; - -/// How a [`Registry`] authenticates. -/// -/// The three shapes `oci-client` implements, named here rather than re-exported -/// so that a caller assembling credentials does not have to depend on the -/// registry client this transport happens to be built on. Constructing one is -/// the caller's whole side of the arrangement: this crate never sources -/// credentials, so there is no `from_env` here and no variable name for a host -/// to inherit. -#[derive(Clone, Default, PartialEq, Eq)] -pub enum Auth { - /// No credentials. What a genuinely public repository wants, and what a - /// registry that authenticates every request will answer with a 401. - #[default] - Anonymous, - /// A bearer token, as GHCR and the token-exchange registries issue. - Bearer(String), - /// A user and password, sent as HTTP Basic. What a registry fronted by - /// htpasswd — or by a Worker checking a credential table — expects. - Basic { user: String, password: String }, -} - -/// Says which shape it is and never what is in it. -/// -/// A transport is held in structures that get logged and printed in error -/// context; a derived `Debug` would put a publisher's password in the first -/// backtrace anybody pastes into an issue. -impl std::fmt::Debug for Auth { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Anonymous => f.write_str("Anonymous"), - Self::Bearer(_) => f.write_str("Bearer()"), - Self::Basic { user, .. } => f - .debug_struct("Basic") - .field("user", user) - .field("password", &"") - .finish(), - } - } -} - -impl Auth { - /// Whether these credentials name anybody. - /// - /// The question a host layering credential sources asks — "did that one say - /// anything, or do I fall through to the next?" — so it is answered here - /// rather than by every host matching on the variant. - pub fn is_anonymous(&self) -> bool { - matches!(self, Self::Anonymous) - } - - fn to_registry_auth(&self) -> RegistryAuth { - match self { - Self::Anonymous => RegistryAuth::Anonymous, - Self::Bearer(token) => RegistryAuth::Bearer(token.clone()), - Self::Basic { user, password } => RegistryAuth::Basic(user.clone(), password.clone()), - } - } -} - -/// Annotation naming a layer's profile-defined kind. -/// -/// The three annotation keys below are the specification's: ADR-004's "OCI -/// layout and the inscription" section names them, reverse-DNS under -/// `stelae.store`, a domain TxPipe owns. They are transport metadata and sit -/// outside the inscription, so they are outside a stele's identity — but only -/// two of them are informational. [`DIFF_ID_ANNOTATION`] is normative, because -/// it is load-bearing on the way back: it *is* the identity→blob map a -/// directory has to rebuild by decompressing everything. The golden freezes -/// all three. -pub const KIND_ANNOTATION: &str = "store.stelae.layer.kind"; - -/// Annotation carrying a layer's `diffId` — its identity, and the key of the -/// map a pull reads off the manifest. -pub const DIFF_ID_ANNOTATION: &str = "store.stelae.layer.diffId"; - -/// Annotation carrying the canonical JSON of a layer's profile-owned scope. -/// -/// Informational: a human or a generic tool reading the manifest can see which -/// epoch or shard a blob covers without fetching the config blob. -pub const SCOPE_ANNOTATION: &str = "store.stelae.layer.scope"; - -/// How much of a *streamed* layer is held in memory on the way up. -/// -/// The layers too large for [`DEFAULT_MONOLITHIC_MAX`], and nothing else: a -/// layer that fits goes up as one request and is resident in full while it -/// does. What is left here is one chunk at a time, allocated and handed to the -/// client, which sends it as one `PATCH` — and a `PATCH` costs the registry -/// about three seconds *whatever it carries*, flat across an eightfold change -/// in chunk size, because what it is spent on is the upload state the worker -/// round-trips through its object store rather than the bytes. So the chunk -/// count is the publish's wall clock, and this constant is what sets it: -/// mainnet's largest layer is 231 MB, which at 1 MiB was 221 round trips and -/// eleven minutes of a fifteen-minute publish. -/// -/// Concurrency is not the alternative. A `PATCH` answers with a `Location` -/// carrying the session's state hash and the next one is refused unless it -/// presents the current value, so a blob is one serial chain — -/// [`Options::concurrency`] bounds layers in flight, never chunks within a -/// layer. -/// -/// 4 MiB and not more because `oci-client` re-splits whatever this stream hands -/// it at its own `PUSH_CHUNK_MAX_SIZE`, which is 4 MiB and has no setter. -/// Anything larger here would go out as 4 MiB anyway, having cost the memory. -const UPLOAD_CHUNK: usize = 4 * 1024 * 1024; - -/// The largest layer this transport will push as one request. -/// -/// A `POST` followed by a `PUT` carrying the whole body skips the chunked -/// session entirely — no `PATCH` chain, no upload state round-tripped through -/// the registry's object store, no recombination — and measured against the -/// live registry it moves **3.29 MB/s against 0.27**. It also covers most of a -/// publish: 79 of mainnet's 81 layers are under this number, and the median -/// layer is 0.41 MB. -/// -/// 100 MB because that is what this registry advertises as -/// `OCI-Chunk-Max-Length`, decimal as the header is -/// (`registry/vendor/src/chunk.ts`, `MAXIMUM_CHUNK_UPLOAD_SIZE`). It is a -/// property of *that* registry and not of registries in general, which is why -/// it is [`Options::monolithic_max`] rather than a literal in the push path — -/// but it cannot be read from the wire: the header rides on the upload -/// session's response and `oci-client` extracts only the `Location` from it -/// (`client.rs`, `extract_location_header`), so nothing this crate calls ever -/// sees it. A default a caller can override is the whole of the honesty -/// available here. -pub const DEFAULT_MONOLITHIC_MAX: u64 = 1000 * 1000 * 100; - -/// How many bytes of layer a publish may hold in memory at once. -/// -/// One gibibyte, and it exists because a monolithic push turns -/// [`Options::concurrency`] into a claim on *memory* and not just on the -/// scratch directory. A layer count is the wrong unit for that: thirty-two -/// permits against a 100 MB threshold is 3.2 GB worst case, on a publisher pod -/// requesting 12 GiB and already sitting at seven. -/// -/// So the resident bytes are bounded directly rather than inferred from a -/// layer count — see [`Shared::resident`]. At the default threshold this is ten -/// large layers in flight at once, whatever the concurrency is set to, while -/// the median 0.41 MB layer costs a permit it will never wait for. -pub const DEFAULT_UPLOAD_MEMORY: u64 = 1024 * 1024 * 1024; - -/// What a push moved, and what it did not have to. -/// -/// The blob-skip is the whole point of a content-addressed registry, so its -/// outcome is a number the caller gets back rather than a line in a log: a -/// publisher that believes it is transferring a delta can check. -/// -/// Counts layer blobs only. The config blob — the inscription — is small, is -/// different for every stele by construction, and would only blur the number -/// that matters. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct Transfer { - /// Layer blobs the registry did not have, and that were uploaded. - pub layers_uploaded: u64, - /// Layer blobs the registry already had, and that were not. - pub layers_skipped: u64, - /// Layer blobs that were never built, because [`Registry::adopt_layer`] - /// took them from a stele already in this repository — or because - /// [`Registry::adopt_carried`] took them from an earlier attempt of this - /// same publish. - /// - /// Deliberately not folded into `layers_skipped`. A skipped layer was - /// built, hashed and then found to be present already, so the publisher - /// paid for it and saved only the upload; an adopted one was never read out - /// of a store at all. They are different costs and a publisher comparing - /// two publishes wants to tell them apart. - pub layers_reused: u64, - /// Compressed bytes uploaded. - pub bytes_uploaded: u64, - /// Compressed bytes the skip saved. - pub bytes_skipped: u64, - /// Compressed bytes an adopted layer did not move, as the manifest or the - /// record it came from reports them. - pub bytes_reused: u64, -} - -/// How many of a publish's layer round trips run at once. -/// -/// Eight, and the number is a floor on the registry's side of the trade rather -/// than a ceiling on this one's: the round trips are latency, not bandwidth, so -/// the transport would happily run more, and what stops it is that a stele's -/// blobs go to *one* repository behind one origin. A publisher that opens -/// thirty-two upload sessions at once against a registry sized for a container -/// image is a publisher that finds the registry's limits rather than its own. -/// -/// Eight moves the mainnet publish path off its serial floor by most of an -/// order of magnitude while staying inside what a modest origin answers without -/// complaint. [`Options::concurrency`] is there for an operator who has -/// measured their own. -pub const DEFAULT_CONCURRENCY: usize = 8; - -/// How many times one of a publish's round trips is made before the failure is -/// the caller's. -/// -/// Four, and the number is read off the failure it absorbs rather than chosen: -/// the registry's transient `500`s arrive alone and clear in the time it takes -/// to ask again, so the first retry is the one that does the work and the rest -/// are there for the case where it is a second longer than that. Bounded for -/// the same reason [`crate::Error::LayerNotWritten`] exists — a registry that -/// is *actually* refusing has to keep refusing, out loud, while whoever -/// launched the publish is still watching. -pub const DEFAULT_ATTEMPTS: u32 = 4; - -/// How long the transport waits after the first failed attempt; each later wait -/// doubles it. -/// -/// Three waits of 500ms, 1s and 2s put the ceiling at three and a half seconds -/// of patience per round trip — under the cost of one lost epoch by four orders -/// of magnitude, and small enough that a publish absorbing a handful of them a -/// night does not show up as a slower publish. -const RETRY_DELAY: Duration = Duration::from_millis(500); - -/// How to reach a registry. -#[derive(Debug, Clone)] -pub struct Options { - /// Talk to the registry over plaintext HTTP rather than HTTPS. - /// - /// For a registry on a loopback address — a test fixture, or a mirror - /// inside a cluster. Never for anything reachable from outside one. - pub insecure: bool, - - /// Where layers are staged on the way up and pulled blobs land on the way - /// down. Defaults to the platform temporary directory. - /// - /// Worth setting: a mainnet state shard is hundreds of megabytes - /// compressed, and the platform temporary directory is not always on the - /// volume with room for sixteen of them. - pub scratch_dir: Option, - - /// How to authenticate, decided entirely by the caller. - /// - /// Defaults to [`Auth::Anonymous`]. Nothing in this crate sources - /// credentials — see the module documentation for why that is a boundary - /// rather than an omission. - pub auth: Auth, - - /// How many layer round trips a publish runs at once. - /// - /// Defaults to [`DEFAULT_CONCURRENCY`]. `1` restores the strictly serial - /// path — an escape hatch for a registry that answers concurrency badly, - /// not a mode anything should want — and `0` is read as `1` rather than - /// refused, because a transport that could move nothing is not a - /// configuration anybody means. - /// - /// It bounds the staging directory as well as the wire: see the module - /// documentation. It does not bound memory — [`Options::upload_memory`] - /// does, and in the unit that one is spent in. - pub concurrency: usize, - - /// Re-prove that the registry still holds a blob being adopted out of a - /// manifest this transport pulled. - /// - /// Off, and that is the plain reading of the distribution specification - /// rather than an optimism: a blob referenced by a manifest under a live - /// tag is not garbage, and a registry that reclaims one has broken the - /// contract that makes *the stele the manifest came from* restorable — - /// which the `HEAD` would not have saved either. Paying a round trip per - /// carried layer to re-establish that is what made the publish path's cost - /// grow with the history behind it, for a check whose failure means the - /// repository is already unusable. - /// - /// On, [`Registry::adopt_layer`] proves each blob before the manifest names - /// it, concurrently with everything else the publish is doing — so the - /// check costs latency it can hide rather than latency it serializes. For - /// an operator publishing into a registry whose retention they do not - /// trust. - pub verify_adopted: bool, - - /// How many times a round trip is made before its failure is the caller's. - /// - /// Defaults to [`DEFAULT_ATTEMPTS`]. `0` and `1` both mean one attempt and - /// no retry — `0` is read as `1` rather than refused, for the reason - /// [`Options::concurrency`] reads it that way: a transport that would make - /// no attempt at all is not a configuration anybody means. - /// - /// Only the failures the module documentation lists are retried; a - /// registry's refusal of *this* request is never one of them, so raising - /// this does not slow down a publish that was going to fail anyway. - pub attempts: u32, - - /// The largest layer to push as one request rather than as a `PATCH` chain. - /// - /// Defaults to [`DEFAULT_MONOLITHIC_MAX`], which is what the registry this - /// was measured against advertises. A registry that accepts less is a - /// registry an operator has to say so about, because the advertisement is - /// not reachable from here — see the constant. - /// - /// Clamped down to [`Options::upload_memory`] when it is larger, so that a - /// layer at the threshold always fits the budget that admits it. `0` is - /// read as "never", not as "layers of no bytes": it streams everything, - /// which is the escape hatch for a registry that answers a monolithic - /// `PUT` badly. - pub monolithic_max: u64, - - /// How many bytes of layer this transport may hold in memory at once. - /// - /// Defaults to [`DEFAULT_UPLOAD_MEMORY`]. It bounds the single-request path - /// and nothing else — a streamed layer holds one [`UPLOAD_CHUNK`] whatever - /// this says — and it is a budget rather than a limit on any one layer: - /// several small layers share it, and a layer larger than the whole budget - /// cannot exist because the threshold is clamped to it. - pub upload_memory: u64, -} - -impl Default for Options { - fn default() -> Self { - Self { - insecure: false, - scratch_dir: None, - auth: Auth::default(), - concurrency: DEFAULT_CONCURRENCY, - verify_adopted: false, - attempts: DEFAULT_ATTEMPTS, - monolithic_max: DEFAULT_MONOLITHIC_MAX, - upload_memory: DEFAULT_UPLOAD_MEMORY, - } - } -} - -/// A repository an operator named, as `oci://HOST/PATH`. -/// -/// The `oci://` scheme is not this project's invention — it is how Helm, ORAS -/// and the rest of the ecosystem spell "this URL names an OCI registry -/// reference" — so parsing it belongs here, beside the client, rather than in -/// every command that takes one from a human. -/// -/// **Everything about the name is decided here, once.** That is the whole point -/// of the type: [`Registry::open`] used to take the host and the repository -/// path as two already-split strings, which meant every caller split the URL -/// itself and then handed back the pieces this module immediately glued -/// together again — while the only crate holding the grammar to split it -/// *correctly* was this one. -/// -/// Three things are refused, and the third is the one a hand-written splitter -/// gets wrong: -/// -/// - **A tag or a digest.** `oci://…/dolos:v1` names a stele, and which stele -/// is not part of naming the repository — a profile renders the tags, and a -/// caller that wants a particular one says so separately. -/// - **An empty host or path**, so the two halves a client needs both exist. -/// - **A host the distribution grammar would have inferred rather than read.** -/// [`Reference`]'s own parser applies registry defaults: a first component -/// with no dot and no colon is not a host at all, and `dolos/mainnet` -/// silently becomes `docker.io/dolos/mainnet`. Parsing and then checking that -/// the registry it reports is the text the operator actually wrote is what -/// turns that rewrite into a refusal. It also buys the rest of the grammar — -/// lowercase components, `.`/`_`/`-` separators, no empty segments — from the -/// parser that defines it rather than from a second copy. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Repository { - registry: String, - repository: String, -} - -impl Repository { - /// The registry host, with its port if it has one. - pub fn registry(&self) -> &str { - &self.registry - } - - /// The repository path within that registry. - pub fn repository(&self) -> &str { - &self.repository - } -} - -impl std::str::FromStr for Repository { - type Err = Error; - - fn from_str(raw: &str) -> Result { - let bad = |why: &str| Error::InvalidRepository { - value: raw.to_owned(), - reason: why.to_owned(), - }; - - let rest = raw - .strip_prefix(SCHEME) - .ok_or_else(|| bad(&format!("it does not start with `{SCHEME}`")))?; - - let (registry, repository) = rest - .split_once('/') - .ok_or_else(|| bad("it names a registry but no repository path"))?; - - if registry.is_empty() { - return Err(bad("it names no registry host")); - } - - if repository.is_empty() || repository.ends_with('/') { - return Err(bad("it names no repository path")); - } - - // A host may carry a port, so only the path is asked about a reference. - if repository.contains(':') || repository.contains('@') { - return Err(bad( - "it names a tag or a digest, and which stele to read is not part of naming \ - the repository", - )); - } - - let reference: Reference = rest - .parse() - .map_err(|_| bad("its repository path is not a valid OCI name"))?; - - // The parser applies registry defaults, so a first component it did not - // recognise as a host became part of the repository under `docker.io`. - // Publishing to a registry the operator did not name is worse than - // refusing, and this comparison is the only thing standing between the - // two. - if reference.registry() != registry { - return Err(bad(&format!( - "{registry:?} is not a registry host, so this would address \ - {:?} instead", - reference.registry(), - ))); - } - - Ok(Self { - registry: registry.to_owned(), - repository: repository.to_owned(), - }) - } -} - -/// Back in the spelling it was read from, so an error message names what the -/// operator typed. -impl std::fmt::Display for Repository { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{SCHEME}{}/{}", self.registry, self.repository) - } -} - -/// The URL scheme an OCI registry reference is named by. -/// -/// The ecosystem's, not this protocol's: Helm, ORAS and others already use it -/// for exactly this. -pub const SCHEME: &str = "oci://"; - -/// A stele repository in an OCI registry. -/// -/// Implements [`SteleWriter`], so a profile publishes into it exactly as it -/// would into a directory. Reading is [`Registry::pull`], which resolves a tag -/// into a [`Stele`] — the read handle, and the thing that implements -/// [`SteleReader`]. -/// -/// See the module documentation before calling any of this from async code. -pub struct Registry { - shared: Arc, -} - -struct Shared { - runtime: tokio::runtime::Runtime, - client: Client, - /// Registry and repository. The tag is a placeholder — blob operations do - /// not use one, and manifest operations build their own. - repository: Reference, - /// The same repository in the spelling it was opened with, for a caller - /// that has to write down where this transport publishes. - name: Repository, - auth: RegistryAuth, - scratch_dir: Option, - /// Behind its own [`Arc`] rather than inside this one, because a deferred - /// layer round trip has to reach it and must **not** reach the runtime: the - /// task would then hold the thing that is driving it, and the last handle - /// dropping inside a worker thread would drop a runtime from inside itself, - /// which panics. Every deferred task below is built out of cheap clones — - /// the client, the reference, this handle, the observer — and never out of - /// a `Shared`. - state: Arc>, - /// How many layer round trips may be outstanding at once, and with them how - /// many staged layers may exist at once. A permit is taken on the caller's - /// thread before the staging file is handed over and released when the - /// round trip is done. See [`Options::concurrency`]. - /// - /// Layers, not bytes: what this bounds is the scratch directory. Memory is - /// `resident`, below. - permits: Arc, - /// The deferred layer round trips, waiting to be joined by - /// [`SteleWriter::seal`]. - /// - /// Not in [`PushState`], for the reason `state` is not in `Shared`: the - /// tasks reach the state and must never reach the handles that own them. - inflight: Mutex>>>, - /// Who is watching this connection, in either direction. - /// - /// Beside the push state rather than inside it, because a [`Stele`] shares - /// this value and only ever reads: attaching an observer to the - /// [`Registry`] is what makes the pull it resolves report too, which is the - /// property a restore depends on — the reader is created inside the driver - /// and there is no other moment to wire it. - observer: Mutex, - /// [`Options::verify_adopted`], as it was given. - verify_adopted: bool, - /// [`Options::concurrency`], as it was resolved — the permit count, kept - /// beside the semaphore because the semaphore's own count is whatever is - /// free at the moment it is asked. - concurrency: usize, - /// [`Options::attempts`], as it was resolved. - attempts: u32, - /// The resident-byte budget, one permit per byte. - /// - /// A second bound beside `permits`, and a different unit on purpose. That - /// one counts layers and bounds the scratch directory; this one counts - /// bytes and bounds *memory*, which is what the single-request path spends - /// and what a layer count cannot express — the layers differ in size by - /// three orders of magnitude. - /// - /// Taken inside the deferred task rather than on the caller's thread, - /// because that is where a layer's size is known: the permit in - /// [`RecordSink::finish`] is taken before the layer is closed, and its - /// size does not exist yet. Held for the upload and released before the - /// backoff between attempts, so a publish waiting on a failing registry - /// holds no bytes at all. - /// - /// Cannot deadlock: `monolithic_max` is clamped to the budget, so any - /// single acquisition can be satisfied by an empty semaphore, and - /// `tokio`'s is fair — a large waiter is not overtaken by the small ones - /// behind it. - resident: Arc, - /// [`Options::monolithic_max`], as it was resolved against - /// [`Options::upload_memory`]. - monolithic_max: u64, -} - -#[derive(Default)] -struct PushState { - /// Layers finished since the last [`SteleWriter::seal`], in finish order. - pending: Vec, - transfer: Transfer, - /// The first deferred round trip that failed, rendered. - /// - /// Sticky, and that is the point. A failed [`Shared::join_layers`] empties - /// the handles it awaited, so without this a second seal would find nothing - /// in flight, agree that every layer was up, and publish a manifest naming - /// a blob that never landed — the one document this transport must never - /// write. See [`Error::LayerNotWritten`]. - failed: Option, -} - -/// One layer's share of [`Options::concurrency`], held for as long as its round -/// trips are outstanding. -/// -/// Named because it is passed hand to hand — taken on the caller's thread by -/// whoever is about to defer, moved into the task, and dropped when the task -/// ends — and a bare `OwnedSemaphorePermit` in three signatures says nothing -/// about which of those it is. -type Permit = tokio::sync::OwnedSemaphorePermit; - -/// The push state, for a deferred task that holds the state and not the -/// transport. -/// -/// A poisoned lock means a push panicked while holding it. The counters are -/// plain integers and the pending list is append-only, so what is behind the -/// lock is still coherent; refusing to look at it would turn one failed push -/// into a transport nobody can use. -fn lock(state: &Mutex) -> std::sync::MutexGuard<'_, PushState> { - state - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) -} - -impl Registry { - /// Open a repository — e.g. `oci://ghcr.io/txpipe/dolos-snapshots/mainnet`, - /// already parsed into a [`Repository`]. - /// - /// Takes the name as one value rather than as a pre-split pair, because - /// splitting it correctly needs the distribution grammar and this is the - /// only crate that has it. A caller holding a string an operator typed - /// parses it into a [`Repository`] and hands that over; nothing outside - /// this module needs to know where the host ends. - /// - /// Builds the runtime the whole transport runs on, and stores the - /// credentials [`Options::auth`] carries so they never have to be threaded - /// through a profile's call stack. - /// - /// # Panics - /// - /// If no process-default [`rustls`] `CryptoProvider` has been installed — - /// see the module documentation. The panic comes from `reqwest`, which - /// resolves its TLS backend when `oci-client` builds the HTTP client here, - /// so [`Options::insecure`] does not avoid it. - /// - /// [`rustls`]: https://docs.rs/rustls - pub fn open(repository: &Repository, options: Options) -> Result { - let protocol = if options.insecure { - ClientProtocol::Http - } else { - ClientProtocol::Https - }; - - // `use_monolithic_push` is what makes `push_blob` send a `POST` and - // then one `PUT` carrying the whole body instead of opening a chunked - // session. It reaches two paths: the layers under `monolithic_max` - // below, which is the point, and `Shared::put_bytes`, which was already - // calling `push_blob` for the inscription — a document of a few - // kilobytes that now costs one round trip fewer. - // - // The flag also removes `push_blob`'s fallback: without it a chunked - // push that trips a spec violation retries monolithically, and with it - // there is nothing to fall back *from*. That is the trade this whole - // change is, and the single-request path is the one measured against - // the registry this publishes to. - let client = Client::new(ClientConfig { - protocol, - use_monolithic_push: true, - ..Default::default() - }); - - // The tag is never read: `Reference` is the client's way of naming a - // repository, and every manifest operation below builds its own. - let reference = Reference::with_tag( - repository.registry.clone(), - repository.repository.clone(), - crate::MOVING_TAG.to_owned(), - ); - - let auth = options.auth.to_registry_auth(); - - // One worker per permit, and the two numbers are one decision. A - // deferred upload reads its staged layer with a *blocking* file read — - // see `blob_stream`, where that is deliberate — so a worker driving one - // upload is unavailable to another for the length of a read. Sizing the - // pool to the bound is what keeps that from turning a concurrency of - // eight into a concurrency of one on a slow volume. - let concurrency = options.concurrency.max(1); - - // The budget first, then the threshold against it: a layer at the - // threshold has to be admissible on an empty semaphore, or the task - // holding a layer larger than the whole budget would wait forever. - let upload_memory = options - .upload_memory - .min(tokio::sync::Semaphore::MAX_PERMITS as u64); - // `u32` because that is what one `acquire_many` can ask for; no layer - // this format cuts comes near it, and a threshold that did would be - // asking for four gibibytes of one blob in memory. - let monolithic_max = options - .monolithic_max - .min(upload_memory) - .min(u32::MAX as u64); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(concurrency) - .thread_name("stelae-oci") - .enable_all() - .build()?; - - // Stored once rather than passed per call: the client's own - // authenticated operations take credentials as an argument, but - // `blob_exists`, `push_blob_stream` and `push_manifest_raw` do not — - // they look them up here. - runtime.block_on(client.store_auth_if_needed(reference.resolve_registry(), &auth)); - - Ok(Self { - shared: Arc::new(Shared { - runtime, - client, - repository: reference, - name: repository.clone(), - auth, - scratch_dir: options.scratch_dir, - state: Arc::new(Mutex::new(PushState::default())), - permits: Arc::new(tokio::sync::Semaphore::new(concurrency)), - inflight: Mutex::new(Vec::new()), - observer: Mutex::new(Observer::silent()), - verify_adopted: options.verify_adopted, - concurrency, - attempts: options.attempts.max(1), - resident: Arc::new(tokio::sync::Semaphore::new(upload_memory as usize)), - monolithic_max, - }), - }) - } - - /// Where this transport stages the layers it moves, in either direction. - /// - /// [`Options::scratch_dir`] as it was given, so a caller that wants to - /// size the staging volume asks the transport that will use it rather than - /// re-deriving the path it handed over — two derivations of one directory - /// is one more than can be kept in step. `None` is the platform temporary - /// directory, which is not a path this can name because - /// [`Shared::scratch`] never names one either. - pub fn scratch_dir(&self) -> Option<&Path> { - self.shared.scratch_dir.as_deref() - } - - /// The repository this transport was opened on. - /// - /// Here for the reason [`Registry::scratch_dir`] is: a caller that has to - /// write down where its layers went asks the transport that sent them, - /// rather than carrying the name alongside the handle. Two spellings of one - /// destination is one more than can be kept in step, and the one that - /// drifts is the one a later run compares against. - pub fn repository(&self) -> &Repository { - &self.shared.name - } - - /// [`Options::concurrency`], as it was resolved. - /// - /// Asked by a caller sizing the volume this transport stages on, for the - /// reason [`Registry::scratch_dir`] is asked for the directory: the number - /// that bounds how many layers are staged at once is the transport's, and a - /// caller re-deriving it from its own configuration is a second copy of a - /// number that has to stay in step. - pub fn concurrency(&self) -> usize { - self.shared.concurrency - } - - /// What has been pushed through this transport since it was opened, or - /// since the last [`Registry::take_transfer`]. - /// - /// **What has *finished*.** Layer round trips are deferred and joined at - /// the seal, so a publish still in flight is a publish still counting, and - /// the moment these numbers are the whole story is after - /// [`SteleWriter::seal`] returns. Asked earlier they are a progress - /// reading; asked there they are the cost of the stele. A caller that wants - /// the second one does not have to do anything to get it — a publish ends - /// at a seal — and this does not block to manufacture it, because a - /// counter that waited for the network would be a strange thing for a - /// progress renderer to call. - pub fn transfer(&self) -> Transfer { - self.shared.locked().transfer - } - - /// The same numbers, and reset — so a publisher pushing several steles - /// through one transport reads each one's cost rather than a running total. - /// - /// Read it after the seal, for the reason [`Registry::transfer`] gives, and - /// with one more of its own: this one clears what it read, so a call made - /// while round trips are still in flight does not merely see a partial - /// figure, it takes the figure away from the seal that was going to - /// complete it. - pub fn take_transfer(&self) -> Transfer { - std::mem::take(&mut self.shared.locked().transfer) - } - - /// Resolve `tag` into a readable stele. - /// - /// The order is the specification, and every step is what makes the next - /// one safe: - /// - /// 1. tag → manifest, which names the config blob; - /// 2. config blob → its bytes, bounded by the size the manifest claims and - /// verified against the digest it is addressed by; - /// 3. those bytes → the inscription, whose *own* digest must equal that - /// same config digest. That is the one place identity and transport are - /// held against each other, and it is what stops a manifest from - /// pointing at a document nobody signed; - /// 4. [`Inscription::check_profile`] — **before any layer is fetched**, so - /// a stele of another profile costs one small GET and not a partial - /// restore. The read-side check, deliberately: a pull serves a reader, - /// and a caller that is about to *publish* on top of what it pulled owes - /// the stricter [`Inscription::check_profile_strict`] of its own; - /// 5. manifest ↔ inscription cross-check, which yields the [`BlobIndex`]. - pub fn pull(&self, profile: &dyn Profile, tag: &str) -> Result { - validate_tag(tag)?; - - let reference = self.shared.tagged(tag); - - let (manifest, _digest) = self.shared.retrying(|| { - Ok(self.shared.runtime.block_on( - self.shared - .client - .pull_image_manifest(&reference, &self.shared.auth), - )?) - })?; - - check_envelope(&manifest)?; - - let raw = self.shared.pull_blob_bytes(&reference, &manifest.config)?; - let inscription = Inscription::parse(&raw)?; - - if inscription.canonicalize()? != raw { - return Err(Error::NonCanonicalInscription); - } - - let config_digest = manifest.config.digest.parse::()?; - let identity = inscription.digest()?; - - if identity != config_digest { - return Err(Error::DigestMismatch { - subject: "inscription".to_owned(), - expected: config_digest.to_string(), - actual: identity.to_string(), - }); - } - - inscription.check_profile(profile)?; - - let blobs = read_manifest(&manifest, &inscription)?; - - Ok(Stele { - shared: Arc::clone(&self.shared), - reference, - manifest, - inscription, - blobs, - }) - } - - /// Resolve the immutable tag `profile` renders for `sequence`. - pub fn pull_sequence(&self, profile: &dyn Profile, sequence: u64) -> Result { - let tag = checked_tag_for_sequence(profile, sequence)?; - self.pull(profile, &tag) - } - - /// Resolve the profile's moving tag — the most recent stele. - pub fn pull_latest(&self, profile: &dyn Profile) -> Result { - let tag = profile.moving_tag().to_owned(); - self.pull(profile, &tag) - } - - /// The most recent stele, or `None` if this repository has never held one. - /// - /// The whole value of this over [`Registry::pull_latest`] is the - /// distinction it draws, and the distinction is load-bearing rather than - /// convenient. A publisher chains each stele to the one before it, so - /// "there is nothing to chain to" starts a history and *anything else* - /// must not: a timeout, a 500 or an expired token read as absence would - /// silently restart the chain, which is the exact outcome an inscription's - /// `history` exists to prevent. So only the shapes a registry uses to say - /// "no such manifest" become `None`, and every other failure propagates. - /// - /// Those shapes are three, because `oci-client` reports a 404 in three - /// ways depending on which layer of the client noticed it. Matching them - /// here rather than at a caller is the point: this is the only module in - /// the crate that has any business naming an `oci_client` error type. - pub fn latest(&self, profile: &dyn Profile) -> Result, Error> { - match self.pull_latest(profile) { - Ok(stele) => Ok(Some(stele)), - Err(Error::Registry(e)) if is_absent(&e) => Ok(None), - Err(e) => Err(e), - } - } - - /// Carry a layer this repository already holds into the stele being - /// written, without building it. - /// - /// This is the operation a content-addressed registry makes possible and a - /// directory does not: the caller has established — by whatever rule its - /// profile owns — that a layer it *would* write is the layer a previous - /// stele already published, so the bytes need neither be produced nor sent, - /// and the new manifest simply points at the blob the old one pointed at. - /// - /// **The new stele attests a layer it did not reproduce.** That is the - /// trade, and it is the caller's to make: nothing here can check that the - /// descriptor describes those bytes, because checking would mean reading - /// them, which is the cost being avoided. - /// - /// ## What is not checked, and why that is the default - /// - /// `source` is a stele this transport *pulled*: its manifest is live in - /// this repository under a tag, and it names this blob. A registry is not - /// permitted to reclaim a blob in that position, and one that does has - /// already broken the stele the descriptor came from — a `HEAD` here would - /// find the damage a publish too late and could not have prevented it. - /// - /// Re-establishing it per carried layer per publish is what made this path - /// cost a round trip for every layer of history behind the stele, and made - /// each publish slower than the one before it for a reason that had nothing - /// to do with what it was publishing. So it is not paid by default. - /// [`Options::verify_adopted`] restores the proof for an operator who wants - /// it, and pays for it concurrently rather than in sequence: the `HEAD` - /// still lands before the manifest names the blob, which is the only - /// ordering that was ever load-bearing. - /// - /// The caller names the layer by its `descriptor` — identity, out of an - /// inscription — and the stele it came from. The blob digest and the - /// compressed size are read off *that stele's manifest*, by exactly the - /// lookup [`SteleReader::stream_layer`] uses, rather than passed in beside - /// the descriptor: they are transport facts, they belong to the manifest, - /// and a caller assembling the pair by hand is a caller that can mismatch - /// them. - pub fn adopt_layer(&self, source: &Stele, descriptor: LayerDescriptor) -> Result<(), Error> { - let missing = || Error::LayerNotFound { - kind: descriptor.kind.clone(), - diff_id: descriptor.diff_id.to_string(), - }; - - let blob = source - .blobs - .blob_for(&descriptor.diff_id) - .ok_or_else(missing)?; - let named = blob.to_string(); - - let oci = source - .manifest - .layers - .iter() - .find(|layer| layer.digest == named) - .ok_or_else(missing)?; - - // Refused rather than clamped. A descriptor's size is an `i64` and a - // negative one is a manifest saying something impossible; clamping it - // to zero would carry that zero into the new manifest, where it becomes - // the ceiling a later reader holds the download to — so the stele would - // publish looking well-formed and refuse to restore. Every other - // malformed-manifest shape here is a refusal, and this is one too. - let compressed_size = u64::try_from(oci.size).map_err(|_| { - Error::ManifestMismatch(format!( - "layer {:?} ({}) claims a compressed size of {}", - descriptor.kind, descriptor.diff_id, oci.size, - )) - })?; - - let adopted = WrittenLayer { - digests: LayerDigests { - diff_id: descriptor.diff_id, - blob_digest: blob, - uncompressed_size: descriptor.uncompressed_size, - compressed_size, - }, - descriptor, - }; - - // A manifest naming a blob the registry no longer has is a refusal and - // not a miss: the stele that named it is published, and something has - // reclaimed underneath it. See [`Registry::adopt_carried`] for the case - // where the same absence is merely a rebuild — and `Options` for why - // this transport does not go looking for it by default. - if self.shared.verify_adopted { - let permit = self.shared.permit(); - - self.shared.prove_blob(&adopted, permit); - } - - let mut state = self.shared.locked(); - state.transfer.layers_reused += 1; - state.transfer.bytes_reused += adopted.digests.compressed_size; - state.pending.push(adopted); - - Ok(()) - } - - /// Carry a layer that is already in this repository, named in full. - /// - /// [`Registry::adopt_layer`] with the lookup already done — for a caller - /// holding a [`WrittenLayer`] this transport produced earlier rather than a - /// stele to read one out of. The pair is still not assembled by hand: it is - /// the measurement [`RecordSink::finish`] returned when the blob went up, - /// carried across whatever interruption the caller survived. - /// - /// **The blob check is a verdict, not an assertion.** `Ok(false)` means the - /// registry does not hold it and nothing was carried, which is a caller's - /// cue to build the layer after all. That is the difference from - /// [`Registry::adopt_layer`], where the same answer is an error: a - /// published manifest that names a reclaimed blob is a stele nobody can - /// restore, while a caller's own note that has gone stale costs a rebuild - /// and nothing else. - /// - /// Nothing here checks that the descriptor describes those bytes, for the - /// reason [`Registry::adopt_layer`] gives: checking would mean reading - /// them, which is the cost being avoided. - pub fn adopt_carried(&self, layer: WrittenLayer) -> Result { - if !self.shared.blob_exists(&layer.digests.blob_digest)? { - return Ok(false); - } - - let mut state = self.shared.locked(); - state.transfer.layers_reused += 1; - state.transfer.bytes_reused += layer.digests.compressed_size; - state.pending.push(layer); - - Ok(true) - } - - /// The layer this transport is carrying for the next seal under `diff_id`. - /// - /// What [`SteleWriter::seal`] would put in the manifest, asked for one - /// layer at a time — so a caller recording what it has finished records - /// the transport's own measurement rather than a reconstruction of it. - /// `None` once the layers have been spent by a seal, and for a `diffId` - /// this transport never wrote. - pub fn carried(&self, diff_id: &Digest) -> Option { - self.shared - .locked() - .pending - .iter() - .find(|layer| layer.digests.diff_id == *diff_id) - .cloned() - } -} - -/// Whether a registry error means "no such manifest" rather than "something -/// went wrong". -/// -/// `oci-client` does not normalize this, and the three shapes are not -/// interchangeable in practice: `distribution` answers a missing tag with a -/// `MANIFEST_UNKNOWN` envelope, a repository that has never existed with -/// `NAME_UNKNOWN`, and some registries answer with a bare 404 that the client -/// turns into `ImageManifestNotFoundError` or a `ServerError`. The client's own -/// referrers fallback matches the same set, for the same reason. -fn is_absent(error: &oci_client::errors::OciDistributionError) -> bool { - use oci_client::errors::{OciDistributionError as E, OciErrorCode}; - - match error { - E::ImageManifestNotFoundError(_) => true, - E::ServerError { code: 404, .. } => true, - E::RegistryError { envelope, .. } => envelope.errors.iter().any(|e| { - matches!( - e.code, - OciErrorCode::ManifestUnknown | OciErrorCode::NameUnknown - ) - }), - _ => false, - } -} - -/// Whether a failure is one that asking again can fix. -/// -/// The counterpart of [`is_absent`], and narrow for the same reason that one -/// is: a classification that guesses wide turns a registry's considered refusal -/// into four of them spread over three and a half seconds, and the caller reads -/// the last copy. Two shapes qualify. -/// -/// **A `5xx`.** The registry answered, and what it said was about itself. This -/// is the measured class — the create-session `POST` that returns -/// `500 INTERNAL_ERROR` and succeeds on the next ask. -/// -/// **No answer at all.** A connection refused, a request that timed out, a -/// socket closed mid-body. `oci-client` reports these through `reqwest` -/// untouched, and [`oci_client::Client::blob_exists`] reports a `5xx` that way -/// too — it asks `reqwest` for the status rather than mapping it — so both -/// shapes have to be read out of the same variant. -/// -/// Everything else propagates on the first attempt: -/// -/// - **a `4xx`** is the registry saying something true about *this* request. -/// The credential is wrong, the digest does not match what arrived, the -/// repository is not there. Repetition does not change any of those, it only -/// delays the report; -/// - **`429` in particular**, and that is a decision rather than an oversight. -/// A registry rationing this publisher is a fact its operator needs, and a -/// client that waited it out would deliver the ration as unexplained -/// slowness; -/// - **anything local** — a staging file that would not read, a manifest that -/// would not parse. Nothing on the far side is involved and nothing about -/// waiting helps. -fn is_transient(error: &Error) -> bool { - use oci_client::errors::OciDistributionError as E; - - let Error::Registry(error) = error else { - return false; - }; - - match error { - E::ServerError { code, .. } => (500..600).contains(code), - E::RequestError(source) => match source.status() { - Some(status) => status.is_server_error(), - None => source.is_timeout() || source.is_connect() || source.is_request(), - }, - _ => false, - } -} - -/// What a bounded retry keeps between attempts: how many are left, and how long -/// the next wait is. -/// -/// A value rather than a loop because there are two loops — one on the caller's -/// thread around a `block_on`, one inside a deferred task around an `await` — -/// and the decision they share is this and not the sleeping. Splitting it here -/// is what keeps the policy in one place while each loop waits the way its own -/// thread has to. -struct Backoff { - attempted: u32, - attempts: u32, - delay: Duration, -} - -impl Backoff { - fn new(attempts: u32, delay: Duration) -> Self { - Self { - attempted: 0, - attempts: attempts.max(1), - delay, - } - } - - /// How long to wait before making the round trip again, or `None` if this - /// failure is the caller's. - /// - /// Announces the retry it is about to allow, before the wait rather than - /// after it, so a watcher hears about a registry misbehaving while it is - /// still misbehaving. - fn wait_after(&mut self, error: &Error, observer: &Observer) -> Option { - self.attempted += 1; - - let remaining = self.attempts - self.attempted; - - if remaining == 0 || !is_transient(error) { - return None; - } - - observer.emit(Event::Retry { - attempt: self.attempted, - remaining, - reason: &error.to_string(), - }); - - let waiting = self.delay; - self.delay = self.delay.saturating_mul(2); - - Some(waiting) - } -} - -/// Run a round trip on the caller's thread, making it again while -/// [`is_transient`] says it is worth it. -/// -/// `op` does its own `block_on` and is run from a thread that is not the -/// transport's runtime — the rule the module documentation states — so the wait -/// is a plain thread sleep. `op` is called again from scratch, which is what -/// puts any resetting a second attempt needs inside it rather than around it. -fn retrying( - attempts: u32, - observer: &Observer, - mut op: impl FnMut() -> Result, -) -> Result { - let mut backoff = Backoff::new(attempts, RETRY_DELAY); - - loop { - match op() { - Ok(value) => return Ok(value), - Err(error) => match backoff.wait_after(&error, observer) { - Some(waiting) => std::thread::sleep(waiting), - None => return Err(error), - }, - } - } -} - -/// [`retrying`], for a round trip already inside the runtime. -/// -/// The deferred layer tasks, where the wait must yield the worker rather than -/// hold it: a thread sleeping here is one of [`Options::concurrency`] threads, -/// and parking it would stall an upload that has nothing wrong with it. -/// -/// `op` returns a future that owns everything it touches, for the reason -/// [`Shared::defer`] gives — and here for a second one: a future that borrowed -/// the closure could not be built twice. -async fn retrying_async( - attempts: u32, - observer: &Observer, - mut op: F, -) -> Result -where - F: FnMut() -> Fut, - Fut: std::future::Future>, -{ - let mut backoff = Backoff::new(attempts, RETRY_DELAY); - - loop { - match op().await { - Ok(value) => return Ok(value), - Err(error) => match backoff.wait_after(&error, observer) { - Some(waiting) => tokio::time::sleep(waiting).await, - None => return Err(error), - }, - } - } -} - -/// A staging file, back at its first byte and ready to be streamed. -/// -/// A `dup` rather than the file itself, because a stream consumes what it is -/// given and a second attempt has to read the same bytes again. The two -/// descriptors share an offset, so the seek here is what rewinds *the* file -/// however many handles are outstanding — which is sound because only one -/// attempt of one layer ever reads it at a time. -fn rewound(staged: &File) -> Result { - let mut again = staged.try_clone()?; - - again.seek(SeekFrom::Start(0))?; - - Ok(again) -} - -/// A staged layer, whole, for the single-request path. -/// -/// Read on the runtime's own worker thread and synchronously, for the reason -/// [`blob_stream`] reads that way: the file is local, the runtime is this -/// transport's own, and it is sized one worker per permit so a thread inside a -/// read cannot starve another upload. -/// -/// Read fresh per attempt rather than held across the backoff — the caller has -/// already taken the resident-byte permit that admits it, and holding the bytes -/// through a wait would multiply the budget by the retry window at exactly the -/// moment the registry is failing. -/// -/// `size` is the compressed size the digest pipeline reported as it wrote this -/// file, and the read is exact rather than to the end. One allocation of -/// exactly the layer, so what the resident-byte permit admitted is what the -/// upload actually holds — `read_to_end` would probe past the end and can grow -/// the buffer past the permit that paid for it. A file that does not hold -/// `size` bytes is a bug in this process and fails here rather than as a digest -/// the registry rejects. -fn staged_bytes(mut staged: File, size: u64) -> Result { - let mut body = vec![0u8; size as usize]; - - staged.read_exact(&mut body)?; - - Ok(bytes::Bytes::from(body)) -} - -/// A staged file, created where [`Options::scratch_dir`] said. -/// -/// Both calls that can fail against a *named* directory raise -/// [`Error::Scratch`], whose docstring carries the reason. The unnamed case -/// keeps the catch-all [`Error::Io`], because the platform temporary -/// directory is not a path anybody chose. -/// -/// Creating the directory lazily, here, is load-bearing elsewhere: it is what -/// makes a staging directory that exists after a run evidence that the run -/// staged in it. Nothing else creates it. -/// -/// A free function rather than a method so it can be tested against an -/// unusable directory without standing up a registry client — see -/// `an_unusable_staging_directory_names_itself`. -fn scratch_in(dir: Option<&Path>) -> Result { - let Some(dir) = dir else { - return Ok(tempfile::tempfile()?); - }; - - let staged = |source| Error::Scratch { - dir: dir.to_path_buf(), - source, - }; - - std::fs::create_dir_all(dir).map_err(staged)?; - - tempfile::tempfile_in(dir).map_err(staged) -} - -impl Shared { - fn locked(&self) -> std::sync::MutexGuard<'_, PushState> { - lock(&self.state) - } - - /// A handle on whoever is watching, taken once per operation. - /// - /// Cloned out from under the lock rather than emitted through it: a blob - /// download reports a delta per write, and holding a mutex across a - /// renderer's call would put this transport's byte loop behind whatever the - /// binary does with the event. - fn observer(&self) -> Observer { - self.observer - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone() - } - - fn watch(&self, observer: Observer) { - *self - .observer - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = observer; - } - - fn tagged(&self, tag: &str) -> Reference { - Reference::with_tag( - self.repository.registry().to_owned(), - self.repository.repository().to_owned(), - tag.to_owned(), - ) - } - - fn scratch(&self) -> Result { - scratch_in(self.scratch_dir.as_deref()) - } - - /// [`retrying`], with this transport's own bound and observer. - /// - /// Every round trip made on a caller's thread goes through here, so the - /// policy is stated once and no seam is left out by having been written - /// before the policy existed. - fn retrying(&self, op: impl FnMut() -> Result) -> Result { - retrying(self.attempts, &self.observer(), op) - } - - fn blob_exists(&self, digest: &Digest) -> Result { - let named = digest.to_string(); - - self.retrying(|| { - Ok(self - .runtime - .block_on(self.client.blob_exists(&self.repository, &named))?) - }) - } - - /// Take a permit, on the caller's thread. - /// - /// This is the back pressure, and taking it *here* rather than inside the - /// task is what makes it back pressure at all: the caller does not get to - /// stage a ninth layer while eight are still moving, so the scratch - /// directory is bounded by the same number as the wire. - /// - /// The resident-byte permit cannot be taken here — the layer is not closed - /// yet and has no size — which is why there are two of them and why they - /// are taken in different places. See [`Shared::resident`]. - fn permit(&self) -> tokio::sync::OwnedSemaphorePermit { - // The semaphore is never closed — nothing here closes one — so the only - // error this call has is unreachable. - self.runtime - .block_on(Arc::clone(&self.permits).acquire_owned()) - .expect("the transport's semaphore is never closed") - } - - /// Run one layer's round trips off the caller's thread. - /// - /// The future is built by the caller out of clones and owns everything it - /// touches, which is the invariant that keeps a task from holding the - /// runtime driving it — see [`Shared::state`]. - fn defer(&self, task: impl std::future::Future> + Send + 'static) { - let handle = self.runtime.spawn(task); - - self.inflight - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .push(handle); - } - - /// Wait for every deferred round trip, and report the first failure. - /// - /// **Every** one, including after a failure has already been seen: a task - /// left running is a task still writing to a registry the caller is about - /// to be told nothing was written to, and — on the error path — one that - /// would be cancelled by the runtime shutting down under it. The first - /// error is the one reported because the others are usually the same - /// network saying the same thing twice. - /// - /// A panicking task is re-raised on this thread rather than folded into an - /// error: a panic in an upload is a bug in this module, and turning it into - /// a returned `Err` would file it under "the registry refused". - /// - /// **A failure here is remembered.** The first call reports the cause - /// itself; every call after it reports [`Error::LayerNotWritten`], because - /// the handles are gone and a join that found nothing outstanding would - /// otherwise read as "everything landed". - /// - /// A failure that arrives here has already been retried — each round trip - /// was made [`Options::attempts`] times while its staged bytes were still - /// in hand, which is the only moment at which the transport can do anything - /// about it. So what reaches this point is a registry that means it, and - /// permanence is the right answer to it rather than a harsh one. - fn join_layers(&self) -> Result<(), Error> { - let handles: Vec<_> = std::mem::take( - &mut *self - .inflight - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()), - ); - - let (first, panicked) = self.runtime.block_on(async { - let mut first: Option = None; - let mut panicked: Option> = None; - - for handle in handles { - match handle.await { - Ok(Ok(())) => {} - Ok(Err(error)) => first = first.or(Some(error)), - Err(join) if join.is_panic() => panicked = panicked.or(Some(join.into_panic())), - // Nothing here cancels a task, so this arm is the runtime - // shutting down mid-join, which cannot happen while this - // call holds it. - Err(_) => {} - } - } - - (first, panicked) - }); - - if let Some(payload) = panicked { - std::panic::resume_unwind(payload); - } - - let mut state = self.locked(); - - if let Some(error) = first { - state.failed.get_or_insert_with(|| error.to_string()); - - return Err(error); - } - - match &state.failed { - Some(why) => Err(Error::LayerNotWritten(why.clone())), - None => Ok(()), - } - } - - /// Upload a staged layer, unless the registry already has it — later. - /// - /// The existence check *is* the blob-skip — the whole delta-transfer claim - /// reduces to this one `HEAD` per layer — and both outcomes are counted. - /// - /// The layer is added to the pending list here, on the caller's thread and - /// in finish order, so [`Registry::carried`] answers for it the moment its - /// sink returns and the manifest is built out of the order the inscription - /// was. What is deferred is the two round trips, and their failure, which - /// [`Shared::join_layers`] collects at the seal. - fn put_layer(&self, layer: &WrittenLayer, staged: File, permit: Permit) { - let digest = layer.digests.blob_digest; - let size = layer.digests.compressed_size; - let observer = self.observer(); - let attempts = self.attempts; - - self.locked().pending.push(layer.clone()); - - let client = self.client.clone(); - let repository = self.repository.clone(); - let state = Arc::clone(&self.state); - let resident = Arc::clone(&self.resident); - let monolithic_max = self.monolithic_max; - - self.defer(async move { - let _permit = permit; - let named = digest.to_string(); - - let present = retrying_async(attempts, &observer, || { - let client = client.clone(); - let repository = repository.clone(); - let named = named.clone(); - - async move { Ok(client.blob_exists(&repository, &named).await?) } - }) - .await?; - - if present { - // Announced even though nothing moves: "the registry already - // had this one" is the blob-skip working, and a watcher that - // only heard about uploads would read the whole point of a - // content-addressed registry as a stall. - observer.emit(Event::Blob { - moved: false, - bytes: size, - }); - - let mut state = lock(&state); - state.transfer.layers_skipped += 1; - state.transfer.bytes_skipped += size; - - return Ok(()); - } - - // Before the upload rather than after it, so a watcher knows how - // big the transfer it is about to see is while it is still - // happening. - // - // Once, however many attempts the upload takes. What a lost attempt - // moved is bytes that crossed the wire and are not coming back, so - // the deltas can outrun this announcement — which is a fact about - // the link, and [`Event::Bytes`] says so where a renderer will read - // it. - observer.emit(Event::Blob { - moved: true, - bytes: size, - }); - - // The staged bytes are still in hand here, which is the whole - // reason this is the right place for the retry: past the join the - // layer's only copy is the store it was built from, and the - // recovery costs a publish. Rewound per attempt, because both paths - // consume the handle they are given. - if monolithic_max > 0 && size <= monolithic_max { - retrying_async(attempts, &observer, || { - let client = client.clone(); - let repository = repository.clone(); - let named = named.clone(); - let resident = Arc::clone(&resident); - let staged = rewound(&staged); - - async move { - // Taken per attempt and released with the attempt, so - // the doubling wait between two of them holds no bytes: - // a registry answering `500` is exactly when this - // transport should be at its smallest. - let _bytes = resident - .acquire_many_owned(size as u32) - .await - .expect("the transport's semaphore is never closed"); - - client - .push_blob(&repository, staged_bytes(staged?, size)?, &named) - .await?; - - Ok(()) - } - }) - .await?; - - // Once, at the end, because there is nothing finer to say: the - // request either landed or it did not. A layer that took two - // attempts reports what it *is* rather than what crossed the - // wire, which is the opposite of the streamed path's answer and - // the honest one for a transfer with no intermediate states. - observer.emit(Event::Bytes(size)); - } else { - retrying_async(attempts, &observer, || { - let client = client.clone(); - let repository = repository.clone(); - let named = named.clone(); - let observer = observer.clone(); - let staged = rewound(&staged); - - async move { - client - .push_blob_stream(&repository, blob_stream(staged?, observer), &named) - .await?; - - Ok(()) - } - }) - .await?; - } - - let mut state = lock(&state); - state.transfer.layers_uploaded += 1; - state.transfer.bytes_uploaded += size; - - Ok(()) - }); - } - - /// Prove — later — that the registry still holds a blob being carried - /// forward. - /// - /// [`Options::verify_adopted`] only. Deferred for the reason an upload is, - /// and joined at the same point: what the check has to beat is the manifest - /// naming the blob, not the descriptor being handed back. - fn prove_blob(&self, layer: &WrittenLayer, permit: Permit) { - let kind = layer.descriptor.kind.clone(); - let diff_id = layer.descriptor.diff_id; - let blob = layer.digests.blob_digest; - - let client = self.client.clone(); - let repository = self.repository.clone(); - let observer = self.observer(); - let attempts = self.attempts; - - self.defer(async move { - let _permit = permit; - let named = blob.to_string(); - - // The retry is over the round trip and not over the verdict: a - // registry that answered "no" answered, and asking a second time is - // how a publisher talks itself into carrying a blob that is gone. - let present = retrying_async(attempts, &observer, || { - let client = client.clone(); - let repository = repository.clone(); - let named = named.clone(); - - async move { Ok(client.blob_exists(&repository, &named).await?) } - }) - .await?; - - match present { - true => Ok(()), - false => Err(Error::BlobMissing { - kind, - diff_id: diff_id.to_string(), - blob: named, - }), - } - }); - } - - /// Upload a small blob a caller already holds — the inscription, and - /// nothing else. - /// - /// Goes up in one request like a small layer does, and for the same reason: - /// `use_monolithic_push` is set on the client, so `push_blob` sends a - /// `POST` and a `PUT` rather than opening a chunked session for a - /// document of a few kilobytes. No resident-byte permit — the - /// inscription is bounded by [`MANIFEST_SIZE_LIMIT`]'s order of - /// magnitude and by being one document, not by a budget shared with the - /// layers. - fn put_bytes(&self, digest: &Digest, bytes: Vec) -> Result<(), Error> { - if self.blob_exists(digest)? { - return Ok(()); - } - - let named = digest.to_string(); - - // Into `Bytes` once, so an attempt after the first re-sends the - // document rather than re-allocating it: this is the inscription, and - // the clone a retry costs should be a refcount. - let bytes = bytes::Bytes::from(bytes); - - self.retrying(|| { - self.runtime.block_on(self.client.push_blob( - &self.repository, - bytes.clone(), - &named, - ))?; - - Ok(()) - }) - } - - /// Fetch a small blob into memory, bounded by the size its descriptor - /// claims. - /// - /// Only the config blob comes back this way. A layer never does — see - /// [`Shared::pull_blob_file`]. - fn pull_blob_bytes( - &self, - reference: &Reference, - descriptor: &OciDescriptor, - ) -> Result, Error> { - let mut buffer = Vec::with_capacity(descriptor.size.max(0) as usize); - - // No observer: the config blob is the inscription, not a layer, and a - // watcher summing byte deltas against a layer total would find them - // disagreeing by however large the document is. - self.retrying(|| { - // Emptied rather than reused, so a second attempt writes the - // document and not the document twice. `Blocking` counts from zero - // per attempt for the same reason, which it gets by being built - // here. - buffer.clear(); - - self.runtime.block_on(self.client.pull_blob( - reference, - descriptor, - Blocking::new( - &mut buffer, - descriptor.size, - &descriptor.digest, - Observer::silent(), - ), - ))?; - - Ok(()) - })?; - - Ok(buffer) - } - - /// Fetch a layer blob into a temporary file, ready to be read back. - /// - /// `pull_blob` verifies the blob digest as the bytes go past, which is the - /// transport half of the check; the identity half is the `diffId`, and - /// belongs to [`LayerReader::finish`]. - fn pull_blob_file( - &self, - reference: &Reference, - descriptor: &OciDescriptor, - ) -> Result { - let mut file = self.scratch()?; - let observer = self.observer(); - - // The whole layer lands here before `stream_layer` yields one record, - // so this loop is where a restore spends nearly all of its time and the - // only place it can report from. - observer.emit(Event::Blob { - moved: true, - bytes: descriptor.size.max(0) as u64, - }); - - self.retrying(|| { - // Back to empty before each attempt, for the reason a staged layer - // is rewound before each of its own: what a half-finished download - // left behind is not a prefix of what the next one writes, it is - // bytes in front of it. - file.set_len(0)?; - file.seek(SeekFrom::Start(0))?; - - self.runtime.block_on(self.client.pull_blob( - reference, - descriptor, - Blocking::new( - &mut file, - descriptor.size, - &descriptor.digest, - observer.clone(), - ), - ))?; - - Ok(()) - })?; - - file.seek(SeekFrom::Start(0))?; - - Ok(file) - } -} - -impl SteleWriter for Registry { - type Sink = RegistrySink; - - fn layer_sink( - &self, - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - ) -> Result { - let (sequence, media_type) = open_layer(profile, spec, level, || self.shared.scratch())?; - - Ok(RegistrySink { - shared: Arc::clone(&self.shared), - sequence, - kind: spec.kind.clone(), - media_type, - scope: spec.scope.clone(), - }) - } - - /// Put the second descriptor in the list [`SteleWriter::seal`] builds the - /// manifest from. - /// - /// The override the default's documentation asks for: this transport pairs - /// every layer the inscription describes against a layer it wrote, and a - /// descriptor with nothing beside it fails the seal. Nothing is uploaded - /// and nothing is checked — the blob was handed to the upload pool when the - /// first descriptor's sink finished, in this same publish, and the seal - /// joins that upload before it names either descriptor. So there is no - /// state of the registry under which it is there for one name and absent - /// for the other. - /// - /// Counted as **skipped** rather than reused, by the distinction - /// [`Transfer::layers_reused`] draws: these bytes were built out of a - /// store, hashed, and then not uploaded again — which is a blob-skip - /// exactly, and not a layer that was never read. - /// - /// ## And silent, unlike the blob-skip on the upload path - /// - /// That path emits [`Event::Blob`] with `moved: false` because a watcher - /// hearing only about uploads would read the skip as a stall. Nothing - /// stalls here: the caller closes the second descriptor's layer as - /// `Transferred`, so the layer cursor advances on its own, and the blob - /// this describes finished uploading moments ago in this same publish. - /// The event drives a per-blob bar, so emitting one would reset it to - /// "already in the registry" for bytes that had just crossed the wire — - /// which reads as a redundant upload rather than as one blob acquiring a - /// second name. - /// - /// The counters above do record it, and there the double count is the - /// reading that is wanted: a publisher comparing two publishes wants the - /// dump's bytes to appear as bytes it did not pay to move again. - fn carry_again( - &self, - written: &WrittenLayer, - scope: serde_json::Value, - ) -> Result { - let again = crate::transport::again(written, scope); - - let mut state = self.shared.locked(); - state.transfer.layers_skipped += 1; - state.transfer.bytes_skipped += again.digests.compressed_size; - state.pending.push(again.clone()); - - Ok(again) - } - - /// Publish the manifest, and with it the stele. - /// - /// The order is the whole of the safety argument: - /// - /// 1. **every layer blob is up.** The round trips were deferred by - /// [`RecordSink::finish`] and [`Registry::adopt_layer`] and ran - /// concurrently; this is where they are joined, and the first failure - /// among them is this call's failure; - /// 2. the inscription goes up as the config blob; - /// 3. the manifest is tagged with the immutable tag the profile renders for - /// this sequence; - /// 4. and **only then** the moving tag moves. - /// - /// Step 1 is the serialization point the concurrency is arranged around: - /// the manifest is built *after* it, so no document this transport writes - /// can name a blob the registry has not committed. Step 4 is last so that a - /// reader following `latest` never resolves to a stele whose blobs are - /// still uploading. A push that dies in the middle leaves untagged blobs - /// the registry will reclaim, and a `latest` that still points at the - /// previous stele — which is a stele, and restores. - /// - /// **A seal that succeeds consumes the layers finished since the last - /// one.** One transport can therefore publish several steles in turn — - /// which is what a publisher chaining a `history` does — and a second seal - /// of the same inscription is refused rather than republishing a manifest - /// over layers that are no longer accounted for. - /// - /// **A seal that fails consumes nothing.** Every fallible step runs before - /// the layers are taken, so a registry that answers a manifest push with a - /// 500 leaves a transport the caller can seal again — the blobs are - /// already up, and re-exporting a stele to recover from a transient error - /// is not a price this owes anyone. - /// - /// **A seal that fails at step 1 is the exception, and it is permanent.** A - /// layer that did not reach the repository cannot be sent again from here: - /// its staging went with the round trip that lost it, and the bytes are - /// only in the store the publisher built them from. So the failure is - /// remembered, every later seal answers [`Error::LayerNotWritten`], and the - /// recovery is another publish rather than another seal. Retrying the seal - /// is what would produce the one document this transport must never - /// write — see [`Shared::join_layers`]. - /// - /// Which is why the retry that *can* help is not here but inside the round - /// trip, where the staging file has not been spent yet: by the time a - /// failure reaches this join it has already been asked - /// [`Options::attempts`] times. - fn seal(&self, profile: &dyn Profile, inscription: &Inscription) -> Result { - // Both tags before either push. Validating the moving tag after the - // sequence manifest is already public would make a bad tag something - // the registry finds out about half way through. - let sequence_tag = checked_tag_for_sequence(profile, inscription.sequence)?; - let moving_tag = profile.moving_tag().to_owned(); - validate_tag(&moving_tag)?; - - // The join, and it is before the manifest is even built rather than - // merely before it is pushed: a document assembled out of a pending - // list whose blobs are still in flight is a document that must not - // exist, not one that must not be sent. Fallible, like every other step - // here, and — like every other step here — it runs before the layers - // are spent, so a transport whose upload the network broke can be - // sealed again once the caller has decided what to do about it. - self.shared.join_layers()?; - - // Scoped so the guard is gone before anything touches the network. - let (body, config) = { - let state = self.shared.locked(); - let (manifest, config) = build_manifest(inscription, &state.pending)?; - (manifest_bytes(&manifest)?, config) - }; - - let identity = Digest::compute(&config); - self.shared.put_bytes(&identity, config)?; - - self.shared.push_manifest(&sequence_tag, body.clone())?; - self.shared.push_manifest(&moving_tag, body)?; - - // Only here, with nothing fallible left, are the layers spent. - self.shared.locked().pending.clear(); - - Ok(identity) - } - - /// Report every blob this connection uploads. - /// - /// One of the two implementations that override the default — the other is - /// [`Stele`], and it shares this connection's state, so an observer - /// attached here is also attached to whatever this registry pulls. - fn observe(&self, observer: Observer) { - self.shared.watch(observer); - } -} - -impl Shared { - fn push_manifest(&self, tag: &str, body: Vec) -> Result<(), Error> { - let reference = self.tagged(tag); - - self.retrying(|| { - self.runtime.block_on(self.client.push_manifest_raw( - &reference, - body.clone(), - http::HeaderValue::from_static(OCI_IMAGE_MEDIA_TYPE), - ))?; - - Ok(()) - }) - } -} - -/// A layer being written into a registry, one record at a time. -/// -/// Staged into a temporary file for a reason that is not a limitation of this -/// implementation: both push paths take the digest up front, and a layer's -/// digest is the digest of its own compressed bytes. There is no ordering of -/// the operations in which an upload learns the name first. -/// -/// The staging file is unlinked at creation, so a sink dropped without -/// [`RecordSink::finish`] — an export that fails halfway with sixteen shards -/// open — leaves nothing to clean up and nothing to mistake for a blob. -pub struct RegistrySink { - shared: Arc, - sequence: SeqWriter>, - kind: String, - media_type: String, - scope: serde_json::Value, -} - -impl RecordSink for RegistrySink { - fn write_record(&mut self, record: &CanonicalCbor) -> Result<(), Error> { - self.sequence.write_record(record) - } - - fn records(&self) -> u64 { - self.sequence.count() - } - - /// Close the layer and hand it to the upload pool. - /// - /// The descriptor comes back as soon as the last record is framed: it is a - /// fact about bytes this sink already has, and nothing the registry says - /// can change it. The upload — and the `HEAD` that may make it - /// unnecessary — runs concurrently with whatever the caller does next, and - /// is joined by [`SteleWriter::seal`] before the manifest can name it. - /// - /// So an error here is a failure to *close the layer*; a failure to move it - /// surfaces at the seal. That is a change in when a registry's refusal is - /// reported and not in what it costs: a publish that cannot upload is a - /// publish that does not seal, in either arrangement, having tagged - /// nothing. - fn finish(self) -> Result { - let Self { - sequence, - shared, - kind, - media_type, - scope, - } = self; - - // Before the layer is closed, so the bound on outstanding round trips - // is also the bound on staged layers: a caller with sixteen sinks open - // waits here rather than filling the scratch directory. Taken before - // the fallible steps below for the same reason it is released by the - // task — a permit's lifetime is the layer's, and a layer that never - // closes has none. - let permit = shared.permit(); - - let count = sequence.count(); - let (mut staged, digests) = sequence.into_inner().finish()?; - - staged.flush()?; - staged.seek(SeekFrom::Start(0))?; - - let written = WrittenLayer { - descriptor: LayerDescriptor { - kind, - media_type, - diff_id: digests.diff_id, - records: count, - uncompressed_size: digests.uncompressed_size, - scope, - }, - digests, - }; - - shared.put_layer(&written, staged, permit); - - Ok(written) - } -} - -/// A stele pulled from a registry, and the read handle over it. -/// -/// Everything cheap has already happened by the time this exists: the manifest -/// and the inscription are in hand, verified against each other, and the -/// `diffId`→blob map came off the manifest rather than out of a scan. What is -/// left is the layers, and those are fetched one at a time as -/// [`SteleReader::stream_layer`] is called. -pub struct Stele { - shared: Arc, - reference: Reference, - manifest: OciImageManifest, - inscription: Inscription, - blobs: BlobIndex, -} - -/// What was resolved, and nothing about the connection that resolved it. -impl std::fmt::Debug for Stele { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Stele") - .field("reference", &self.reference.whole()) - .field("sequence", &self.inscription.sequence) - .field("layers", &self.manifest.layers.len()) - .finish_non_exhaustive() - } -} - -impl Stele { - /// The OCI manifest this stele was read from. - pub fn manifest(&self) -> &OciImageManifest { - &self.manifest - } - - /// Compressed bytes across every layer, as the manifest reports them. - /// - /// The whole-stele case of [`SteleReader::compressed_size`], for a caller - /// that wants the total without walking the inscription — a publisher - /// reporting what a repository holds, above all. A restore wants the - /// per-layer figure, because what it is going to fetch is a subset. - pub fn total_compressed_size(&self) -> u64 { - self.manifest - .layers - .iter() - .map(|layer| layer.size.max(0) as u64) - .sum() - } - - /// The manifest's own descriptor for a layer, by identity. - /// - /// One lookup, shared by the read path and the size estimate, so the two - /// cannot come to disagree about which blob holds a layer. Two steps and - /// both are needed: the [`BlobIndex`] maps identity to a blob digest, and - /// the manifest maps that digest to the descriptor carrying the compressed - /// size the download is held to. - fn layer_of(&self, index: &BlobIndex, descriptor: &LayerDescriptor) -> Option<&OciDescriptor> { - let blob = index.blob_for(&descriptor.diff_id)?.to_string(); - - self.manifest - .layers - .iter() - .find(|layer| layer.digest == blob) - } -} - -impl SteleReader for Stele { - type Blob = File; - - fn read_inscription(&self) -> Result { - Ok(self.inscription.clone()) - } - - fn blob_index(&self) -> Result { - Ok(self.blobs.clone()) - } - - /// Read the layer's compressed size off the manifest. - /// - /// Free, and the reason a registry restore can report a correct total - /// before it fetches anything: the manifest is already in hand by the time - /// a [`Stele`] exists. A negative size — a manifest claiming something - /// impossible — reads as `None` rather than as a number, so it widens the - /// estimate's stated uncertainty instead of shrinking its total. - fn compressed_size( - &self, - index: &BlobIndex, - descriptor: &LayerDescriptor, - ) -> Result, Error> { - Ok(self - .layer_of(index, descriptor) - .and_then(|oci| u64::try_from(oci.size).ok())) - } - - fn stream_layer( - &self, - index: &BlobIndex, - profile: &dyn Profile, - descriptor: &LayerDescriptor, - limits: Limits, - ) -> Result, Error> { - // The manifest's own descriptor, not one built here: it carries the - // compressed size, which is the ceiling the download is held to. - let oci = self - .layer_of(index, descriptor) - .ok_or_else(|| Error::LayerNotFound { - kind: descriptor.kind.clone(), - diff_id: descriptor.diff_id.to_string(), - })?; - - let file = self.shared.pull_blob_file(&self.reference, oci)?; - - LayerReader::new(file, profile, descriptor, limits) - } - - /// Report every blob this connection pulls. - /// - /// A restore resolves its [`Stele`] inside the driver, so this is the - /// spelling a caller reaches when it holds the reader; attaching to the - /// [`Registry`] the stele came from does the same thing, because both write - /// the same connection state. - fn observe(&self, observer: Observer) { - self.shared.watch(observer); - } -} - -/// Build the manifest for a stele whose layers are already written. -/// -/// Returns it together with the canonical inscription bytes, so the config -/// descriptor and the blob that is pushed under it cannot be computed from two -/// different encodings of the same document. -/// -/// Layers are listed in **inscription order**, matched by `diffId`: the -/// canonical document fixes the order, and the manifest follows it rather than -/// the order the sinks happened to finish in. Anything that does not match both -/// ways is a refusal — see the module documentation. -/// -/// Pure, so the shape of the artifact is frozen by a golden that needs no -/// network. -pub fn build_manifest( - inscription: &Inscription, - layers: &[WrittenLayer], -) -> Result<(OciImageManifest, Vec), Error> { - let config = inscription.canonicalize()?; - - let mut taken = vec![false; layers.len()]; - let mut descriptors = Vec::with_capacity(inscription.layers.len()); - - for described in &inscription.layers { - // Matched by identity, and the first unclaimed one wins: two layers - // with the same `diffId` are the same bytes, so which of them a - // descriptor points at cannot be observed. - let found = layers - .iter() - .enumerate() - .find(|(index, layer)| !taken[*index] && layer.descriptor.diff_id == described.diff_id); - - let Some((index, layer)) = found else { - return Err(Error::ManifestMismatch(format!( - "the inscription describes a {:?} layer ({}) that was never written", - described.kind, described.diff_id, - ))); - }; - - taken[index] = true; - descriptors.push(layer_descriptor(described, layer)?); - } - - if let Some(orphan) = taken.iter().position(|used| !used) { - let layer = &layers[orphan].descriptor; - - return Err(Error::ManifestMismatch(format!( - "a {:?} layer ({}) was written but the inscription does not describe it; \ - a blob nothing attests would be published", - layer.kind, layer.diff_id, - ))); - } - - let manifest = OciImageManifest { - schema_version: 2, - media_type: Some(OCI_IMAGE_MEDIA_TYPE.to_owned()), - artifact_type: Some(ARTIFACT_TYPE.to_owned()), - config: OciDescriptor { - media_type: INSCRIPTION_MEDIA_TYPE.to_owned(), - digest: Digest::compute(&config).to_string(), - size: config.len() as i64, - ..Default::default() - }, - layers: descriptors, - subject: None, - annotations: None, - }; - - Ok((manifest, config)) -} - -fn layer_descriptor( - described: &LayerDescriptor, - written: &WrittenLayer, -) -> Result { - let scope = String::from_utf8(canonical_json(&described.scope)?) - .map_err(|e| Error::Canonicalization(e.to_string()))?; - - let annotations = BTreeMap::from([ - (KIND_ANNOTATION.to_owned(), described.kind.clone()), - (DIFF_ID_ANNOTATION.to_owned(), described.diff_id.to_string()), - (SCOPE_ANNOTATION.to_owned(), scope), - ]); - - Ok(OciDescriptor { - media_type: described.media_type.clone(), - digest: written.digests.blob_digest.to_string(), - size: written.digests.compressed_size as i64, - annotations: Some(annotations), - ..Default::default() - }) -} - -/// The exact bytes of a manifest, canonicalized and held to the size ceiling. -/// -/// RFC 8785 through the same canonicalizer the inscription uses, so this crate -/// has one answer to "what are the bytes of this JSON document" rather than two -/// that agree until they do not. -/// -/// The ceiling is [`MANIFEST_SIZE_LIMIT`]. What it refuses is a stele with too -/// many layers: at roughly 350 bytes of descriptor and annotations apiece, a -/// manifest reaches 4 MiB somewhere around twelve thousand of them — nearly -/// seven times a mainnet stele's ~1,816. The comparison is in layers because -/// layers are what the ceiling counts; ADR-004's ~600 is a count of *epochs*, -/// and a mainnet stele carries three layers per epoch plus sixteen state -/// shards. It is not a limit anything is expected to reach; it is the limit -/// that turns "the registry answered 413" into a refusal that names the -/// document and the number of layers in it. -/// -/// `a_manifest_past_the_size_ceiling_is_refused` in `tests/oci.rs` measures -/// those figures rather than asserting them; keep the two in step. -pub fn manifest_bytes(manifest: &OciImageManifest) -> Result, Error> { - let body = canonical_json(&serde_json::to_value(manifest)?)?; - - if body.len() > MANIFEST_SIZE_LIMIT { - return Err(Error::ManifestTooLarge { - size: body.len(), - layers: manifest.layers.len(), - }); - } - - Ok(body) -} - -/// Check that a manifest is a stele's before anything inside it is trusted. -fn check_envelope(manifest: &OciImageManifest) -> Result<(), Error> { - match manifest.artifact_type.as_deref() { - Some(ARTIFACT_TYPE) => {} - Some(other) => { - return Err(Error::ManifestMismatch(format!( - "artifactType is {other:?}, not {ARTIFACT_TYPE:?}" - ))) - } - // Fail closed. A registry that strips `artifactType` — the OCI 1.1 - // field this artifact is discovered by — has published something this - // client cannot recognise as a stele, and reading it anyway would make - // the discovery contract advisory. - None => { - return Err(Error::ManifestMismatch(format!( - "no artifactType; a stele's manifest carries {ARTIFACT_TYPE:?}" - ))) - } - } - - if manifest.config.media_type != INSCRIPTION_MEDIA_TYPE { - return Err(Error::ManifestMismatch(format!( - "config blob is {:?}, not the inscription's {INSCRIPTION_MEDIA_TYPE:?}", - manifest.config.media_type, - ))); - } - - Ok(()) -} - -/// Read the identity→blob map off a manifest, holding it against the -/// inscription. -/// -/// This is the function that replaces a directory's `blob_index` scan, and the -/// reason a registry restore reads every blob once instead of twice. -/// -/// Both correspondences are checked, and they are not the same check: the -/// `diffId` annotation is what the map is *built* from, and positional -/// correspondence with `inscription.layers` is what proves the manifest -/// describes this document's layers and not some other stele's. A manifest that -/// carried the right blobs in the wrong order would pass the first and fail the -/// second. -/// -/// Pure, so the parsing half of the artifact is frozen by the same golden as -/// the building half. -pub fn read_manifest( - manifest: &OciImageManifest, - inscription: &Inscription, -) -> Result { - check_envelope(manifest)?; - - if manifest.layers.len() != inscription.layers.len() { - return Err(Error::ManifestMismatch(format!( - "the manifest carries {} layer(s) and the inscription describes {}", - manifest.layers.len(), - inscription.layers.len(), - ))); - } - - let mut blobs = BlobIndex::default(); - - for (position, (oci, described)) in manifest - .layers - .iter() - .zip(inscription.layers.iter()) - .enumerate() - { - let annotation = oci - .annotations - .as_ref() - .and_then(|annotations| annotations.get(DIFF_ID_ANNOTATION)) - .ok_or_else(|| { - Error::ManifestMismatch(format!( - "layer {position} carries no {DIFF_ID_ANNOTATION} annotation, \ - so nothing says which layer it holds" - )) - })?; - - let diff_id = annotation.parse::()?; - - if diff_id != described.diff_id { - return Err(Error::ManifestMismatch(format!( - "layer {position} is annotated {diff_id} and the inscription describes \ - {} there", - described.diff_id, - ))); - } - - if oci.media_type != described.media_type { - return Err(Error::ManifestMismatch(format!( - "layer {position} is {:?} in the manifest and {:?} in the inscription", - oci.media_type, described.media_type, - ))); - } - - blobs.insert(diff_id, oci.digest.parse::()?); - } - - Ok(blobs) -} - -/// A staged layer, as a stream of chunks. -/// -/// The path for the layers [`Options::monolithic_max`] excludes — on mainnet, -/// `blocks` and one `state-accounts` shard. Everything smaller goes up whole -/// through [`staged_bytes`], in one request rather than a `PATCH` chain. -/// -/// Reads from the staging file synchronously. The runtime under it is this -/// transport's own — the read is not waiting on anything it is responsible for -/// driving — and it is sized so that a read blocking a worker cannot starve the -/// other uploads: one worker per permit, decided in [`Registry::open`]. -/// -/// One chunk is allocated at a time and handed over, so what a *streamed* -/// upload holds is [`UPLOAD_CHUNK`] and not the layer. -/// Each chunk is reported as it is handed over, which is the only resolution -/// this loop has: a `PATCH` either went out or it did not, and the client does -/// not say how much of one has reached the wire. A single-request layer has no -/// such resolution to report and announces itself once, at the end. -fn blob_stream( - file: File, - observer: Observer, -) -> impl Stream> { - futures_util::stream::unfold(Some(file), move |state| { - let observer = observer.clone(); - - async move { - let mut file = state?; - let mut chunk = vec![0u8; UPLOAD_CHUNK]; - let mut filled = 0usize; - - while filled < chunk.len() { - match read_uninterrupted(&mut file, &mut chunk[filled..]) { - Ok(0) => break, - Ok(read) => filled += read, - Err(e) => return Some((Err(e.into()), None)), - } - } - - if filled == 0 { - return None; - } - - chunk.truncate(filled); - observer.emit(Event::Bytes(filled as u64)); - - Some((Ok(bytes::Bytes::from(chunk)), Some(file))) - } - }) -} - -/// A synchronous writer dressed as an asynchronous one, with a ceiling. -/// -/// [`oci_client::Client::pull_blob`] writes into a [`tokio::io::AsyncWrite`], -/// and everything this transport writes to is a file or a buffer. Rather than -/// take a dependency on `tokio`'s filesystem layer to get an async file that -/// would immediately be handed back to a blocking pool, the write happens where -/// it is: on the runtime's only thread, which is doing nothing else. -/// -/// The ceiling is not redundant with `pull_blob`'s digest check. That check -/// fails at the *end*, after every byte has been written; the ceiling fails as -/// soon as the stream exceeds what its descriptor claims, so a blob that lies -/// about its size costs its size and not the disk. -/// -/// A negative size clamps to a ceiling of zero here, and that is deliberate -/// rather than an oversight — unlike [`Registry::adopt_layer`], which refuses -/// one. The directions differ: a zero ceiling refuses every non-empty blob, -/// which is the safe answer to a manifest claiming something impossible, while -/// clamping on the way *into* a manifest would publish that impossible claim -/// forward as a number a later reader trusts. -struct Blocking<'a, W: Write> { - inner: &'a mut W, - written: u64, - limit: u64, - digest: String, - observer: Observer, -} - -impl<'a, W: Write> Blocking<'a, W> { - fn new(inner: &'a mut W, limit: i64, digest: &str, observer: Observer) -> Self { - Self { - inner, - written: 0, - limit: limit.max(0) as u64, - digest: digest.to_owned(), - observer, - } - } -} - -impl tokio::io::AsyncWrite for Blocking<'_, W> { - fn poll_write( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - let this = self.get_mut(); - - // Counted on what was *written*, never on what was offered. A `Write` - // may accept less than it was given, and the caller then offers the - // remainder — so counting the offer would tally those bytes twice and - // trip a ceiling the blob never reached. - let room = usize::try_from(this.limit - this.written).unwrap_or(usize::MAX); - - if room == 0 && !buf.is_empty() { - return Poll::Ready(Err(std::io::Error::other(format!( - "blob {} is larger than the {} bytes its descriptor claims", - this.digest, this.limit, - )))); - } - - // Truncated to the room left, so the ceiling is exact rather than "one - // buffer past": the byte that exceeds it is refused on the next call, - // with nothing over-written in between. - match this.inner.write(&buf[..buf.len().min(room)]) { - Ok(written) => { - this.written += written as u64; - // On what was written, for the same reason the ceiling is: a - // partial write's remainder is offered again, and reporting the - // offer would count those bytes twice. - this.observer.emit(Event::Bytes(written as u64)); - Poll::Ready(Ok(written)) - } - Err(e) => Poll::Ready(Err(e)), - } - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().inner.flush()) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().inner.flush()) - } -} - -#[cfg(test)] -mod tests { - use oci_client::errors::{OciDistributionError, OciEnvelope, OciError, OciErrorCode}; - - use super::*; - - fn envelope(code: OciErrorCode) -> OciDistributionError { - OciDistributionError::RegistryError { - envelope: OciEnvelope { - errors: vec![OciError { - code, - message: String::new(), - detail: serde_json::Value::Null, - }], - }, - url: "https://registry.invalid/v2/x/manifests/latest".to_owned(), - } - } - - fn server_error(code: u16) -> OciDistributionError { - OciDistributionError::ServerError { - code, - url: "https://registry.invalid/v2/x/manifests/latest".to_owned(), - message: String::new(), - } - } - - /// The three shapes a registry uses to say "no such manifest". - #[test] - fn absence_is_the_three_shapes_of_a_missing_manifest() { - assert!(is_absent( - &OciDistributionError::ImageManifestNotFoundError("latest".to_owned()) - )); - assert!(is_absent(&server_error(404))); - assert!(is_absent(&envelope(OciErrorCode::ManifestUnknown))); - assert!(is_absent(&envelope(OciErrorCode::NameUnknown))); - } - - /// A `reqwest` error carrying a status, which is how - /// [`oci_client::Client::blob_exists`] reports one. - /// - /// There is no constructor for one, so it is provoked: a response with the - /// status, asked to fail on it. Built on the transport's own runtime kind - /// rather than in an async test, because this module's rule is that its - /// types are used from synchronous code. - fn request_error(code: u16) -> OciDistributionError { - let response = http::Response::builder() - .status(code) - .body(Vec::new()) - .unwrap(); - - let refused = reqwest::Response::from(response) - .error_for_status() - .expect_err("a status the builder was given was not an error"); - - OciDistributionError::RequestError(refused) - } - - /// The two shapes worth asking again about, and the several that are not. - #[test] - fn a_transient_failure_is_the_registry_talking_about_itself() { - // The measured class: the create-session `POST` that answers 500 and - // works on the next ask. - assert!(is_transient(&Error::Registry(server_error(500)))); - assert!(is_transient(&Error::Registry(server_error(502)))); - assert!(is_transient(&Error::Registry(server_error(503)))); - - // And the same thing seen through `blob_exists`, which asks `reqwest` - // for the status instead of mapping it. - assert!(is_transient(&Error::Registry(request_error(500)))); - - // A refusal of *this* request is not. Repeating it only delays the - // report, and `429` is deliberately among them: a registry rationing - // this publisher is something its operator has to be told. - assert!(!is_transient(&Error::Registry(server_error(400)))); - assert!(!is_transient(&Error::Registry(server_error(404)))); - assert!(!is_transient(&Error::Registry(server_error(429)))); - assert!(!is_transient(&Error::Registry(request_error(429)))); - assert!(!is_transient(&Error::Registry(envelope( - OciErrorCode::DigestInvalid - )))); - assert!(!is_transient(&Error::Registry( - OciDistributionError::UnauthorizedError { - url: "https://registry.invalid/v2/".to_owned(), - } - ))); - - // Nor is anything that never involved the far side. A staging file that - // would not read reads the same the second time. - assert!(!is_transient(&Error::Io(std::io::Error::other("staging")))); - assert!(!is_transient(&Error::LayerNotWritten("earlier".to_owned()))); - } - - /// A recorder for what the retry loop announced. - #[derive(Default)] - struct Retries(Mutex>); - - impl crate::progress::Progress for Retries { - fn on(&self, event: Event<'_>) { - if let Event::Retry { - attempt, remaining, .. - } = event - { - self.0.lock().unwrap().push((attempt, remaining)); - } - } - } - - /// The loop with the waits taken out, so the bound can be exercised without - /// waiting out a real backoff. - fn retried( - attempts: u32, - observer: &Observer, - mut op: impl FnMut() -> Result, - ) -> Result { - let mut backoff = Backoff::new(attempts, Duration::ZERO); - - loop { - match op() { - Ok(value) => return Ok(value), - Err(error) => match backoff.wait_after(&error, observer) { - Some(_) => continue, - None => return Err(error), - }, - } - } - } - - #[test] - fn a_call_that_succeeds_is_made_once() { - let mut calls = 0; - - let value = retried(4, &Observer::silent(), || { - calls += 1; - Ok(7u8) - }) - .unwrap(); - - assert_eq!(value, 7); - assert_eq!(calls, 1, "a success must not be retried"); - } - - /// The whole point: the registry's bad half-second costs a half-second and - /// not an epoch. - #[test] - fn a_transient_failure_is_absorbed() { - let watcher = Arc::new(Retries::default()); - let observer = Observer::new(watcher.clone()); - - let mut calls = 0; - - let value = retried(4, &observer, || { - calls += 1; - - match calls < 3 { - true => Err(Error::Registry(server_error(500))), - false => Ok(calls), - } - }) - .unwrap(); - - assert_eq!(value, 3); - assert_eq!(calls, 3); - - // And it said so both times, counting the failures up and the patience - // down, so a watcher can tell a hiccup from a transport about to give - // up. - assert_eq!(*watcher.0.lock().unwrap(), vec![(1, 3), (2, 2)]); - } - - /// Bounded, so a registry that is wrong rather than flaky still fails — - /// and fails as itself, with what it actually said. - #[test] - fn patience_runs_out_and_the_last_failure_is_the_one_reported() { - let watcher = Arc::new(Retries::default()); - let observer = Observer::new(watcher.clone()); - - let mut calls = 0; - - let refused = retried(4, &observer, || { - calls += 1; - Err::<(), _>(Error::Registry(server_error(500))) - }) - .expect_err("a registry that never answered was treated as having answered"); - - assert_eq!(calls, 4, "the bound is attempts, not retries"); - assert!(matches!(refused, Error::Registry(_))); - - // Three retries for four attempts, and nothing announced for the last - // failure — there was no next attempt to announce. - assert_eq!(*watcher.0.lock().unwrap(), vec![(1, 3), (2, 2), (3, 1)]); - } - - /// A refusal of this request is reported the first time it is made. - #[test] - fn a_refusal_of_this_request_is_not_retried() { - let watcher = Arc::new(Retries::default()); - let observer = Observer::new(watcher.clone()); - - let mut calls = 0; - - let refused = retried(4, &observer, || { - calls += 1; - Err::<(), _>(Error::Registry(envelope(OciErrorCode::DigestInvalid))) - }) - .expect_err("a digest the registry rejected was retried"); - - assert_eq!(calls, 1, "a refusal must cost one round trip"); - assert!(matches!(refused, Error::Registry(_))); - assert!(watcher.0.lock().unwrap().is_empty()); - } - - /// `0` attempts is one attempt, for the reason `0` concurrency is one - /// permit: a transport that would make no attempt at all is not a - /// configuration anybody means. - #[test] - fn no_attempts_at_all_is_still_one_attempt() { - let mut calls = 0; - - let refused = retried(0, &Observer::silent(), || { - calls += 1; - Err::<(), _>(Error::Registry(server_error(500))) - }); - - assert_eq!(calls, 1); - assert!(refused.is_err()); - } - - fn repository(raw: &str) -> Result { - raw.parse() - } - - #[test] - fn a_repository_splits_into_a_registry_and_a_path() { - let parsed = repository("oci://ghcr.io/txpipe/dolos-snapshots/mainnet").unwrap(); - - assert_eq!(parsed.registry(), "ghcr.io"); - assert_eq!(parsed.repository(), "txpipe/dolos-snapshots/mainnet"); - - // A port belongs to the host, which is what makes the tag check safe to - // run on the path alone. - let local = repository("oci://127.0.0.1:5000/dolos").unwrap(); - - assert_eq!(local.registry(), "127.0.0.1:5000"); - assert_eq!(local.repository(), "dolos"); - - // And what it prints is what it parsed, so a message naming a - // repository names the one the operator typed. - assert_eq!(local.to_string(), "oci://127.0.0.1:5000/dolos"); - } - - #[test] - fn ordinary_repositories_parse() { - for raw in [ - "oci://ghcr.io/txpipe/dolos-snapshots/mainnet", - "oci://ghcr.io/txpipe/dolos_snapshots", - "oci://ghcr.io/txpipe/dolos.snapshots", - "oci://localhost:5000/dolos/mainnet", - "oci://127.0.0.1:5000/dolos", - ] { - assert!(repository(raw).is_ok(), "{raw}"); - } - } - - #[test] - fn a_name_that_cannot_address_a_repository_is_refused() { - for raw in [ - "ghcr.io/txpipe/dolos", // no scheme - "https://ghcr.io/txpipe/dolos", // the wrong scheme - "oci://ghcr.io", // no repository path - "oci://ghcr.io/", // still no repository path - "oci:///txpipe/dolos", // no host - "oci://ghcr.io/txpipe/dolos/", // a trailing slash - "oci://ghcr.io/txpipe/dolos:v1", // a tag names a stele - "oci://ghcr.io/txpipe/dolos@sha256:abc", // and so does a digest - "", - ] { - assert!(repository(raw).is_err(), "{raw:?}"); - } - } - - /// Names the distribution grammar refuses, which a split on `/` alone - /// cannot see. - /// - /// Each of these reaches the registry as part of the request path, so - /// accepting them buys an opaque error from someone else's server at the - /// end of a publish rather than a sentence at the start of one. - #[test] - fn a_path_outside_the_grammar_is_refused() { - for raw in [ - "oci://ghcr.io//txpipe/dolos", // an empty component - "oci://ghcr.io/txpipe//dolos", // an empty component, inside - "oci://ghcr.io/TxPipe/dolos", // uppercase; names are lowercase - "oci://ghcr.io/txpipe/dolos?x=1", // a query - "oci://ghcr.io/txpipe/dolos#frag", // a fragment - "oci://ghcr.io/txpipe/dolos snaps", // whitespace - "oci://ghcr.io/txpipe/-dolos", // a component opening on a separator - ] { - assert!(repository(raw).is_err(), "{raw:?}"); - } - } - - /// The refusal a hand-written splitter cannot make. - /// - /// `Reference`'s parser treats a first component with no dot and no colon - /// as part of the repository rather than as a host, so `dolos/mainnet` - /// resolves to `docker.io/dolos/mainnet`. An operator who wrote - /// `oci://dolos/mainnet` meant a registry called `dolos`, and publishing to - /// Docker Hub instead is the one outcome worse than refusing. - #[test] - fn a_host_the_parser_would_have_invented_is_refused() { - let err = repository("oci://dolos/mainnet").unwrap_err(); - - let message = err.to_string(); - assert!(message.contains("docker.io"), "{message}"); - assert!(message.contains("dolos"), "{message}"); - - // `localhost` is the one bare name the grammar does treat as a host, so - // it must still work — the check is against inference, not against - // hosts that happen to have no dot. - assert_eq!( - repository("oci://localhost:5000/dolos").unwrap().registry(), - "localhost:5000" - ); - } - - /// The half that carries the weight: a registry that failed is not a - /// registry that is empty. - /// - /// [`Registry::latest`] turns absence into `None`, and a publisher reads - /// `None` as "nothing to chain to" and starts a fresh history. So a - /// timeout, a 500 or an expired token widening into absence would silently - /// restart the attestation chain — which is the outcome an inscription's - /// `history` exists to prevent, arrived at without anything looking wrong. - #[test] - fn a_failed_request_is_never_absence() { - assert!(!is_absent(&server_error(500))); - assert!(!is_absent(&server_error(503))); - assert!(!is_absent(&envelope(OciErrorCode::Unauthorized))); - assert!(!is_absent(&envelope(OciErrorCode::Denied))); - assert!(!is_absent(&OciDistributionError::UnauthorizedError { - url: "https://registry.invalid/v2/x/manifests/latest".to_owned(), - })); - assert!(!is_absent(&OciDistributionError::GenericError(None))); - } - - /// A password never reaches a log through this type. - /// - /// [`Options`] derives `Debug` and error context is printed freely, so this - /// redaction is what stands between a publisher's credentials and the first - /// backtrace anybody pastes into an issue. - #[test] - fn credentials_are_redacted_in_debug_output() { - let basic = Auth::Basic { - user: "reader".to_owned(), - password: "hunter2".to_owned(), - }; - - let printed = format!("{basic:?}"); - assert!(printed.contains("reader"), "{printed}"); - assert!(!printed.contains("hunter2"), "{printed}"); - - let printed = format!("{:?}", Auth::Bearer("ghp_x".to_owned())); - assert!(!printed.contains("ghp_x"), "{printed}"); - - // And through the structure a caller actually holds, which is where it - // would leak from. - let printed = format!( - "{:?}", - Options { - auth: basic, - ..Default::default() - } - ); - assert!(!printed.contains("hunter2"), "{printed}"); - } - - /// A staging directory that cannot be used says which one, and why. - /// - /// The registry suite covers the same ground through a real publish, but - /// it needs a container and is `#[ignore]`d for it; this is the claim - /// under plain `cargo test`. The path names an existing regular file, - /// which `create_dir_all` cannot turn into a directory for anybody, `root` - /// included — so it is the one unusable directory that reproduces on every - /// platform and under every user the suite might run as. - #[test] - fn an_unusable_staging_directory_names_itself() { - let root = tempfile::tempdir().unwrap(); - let occupied = root.path().join("not-a-directory"); - std::fs::write(&occupied, b"").unwrap(); - - let error = scratch_in(Some(&occupied)).expect_err("staged in a regular file"); - - assert!( - matches!(&error, Error::Scratch { dir, .. } if dir == &occupied), - "fell through to the catch-all: {error:?}", - ); - - // The two halves the old `io error: File exists (os error 17)` had - // neither of: which directory, and that it was the staging one. - let message = error.to_string(); - assert!( - message.contains(&occupied.display().to_string()), - "{message}", - ); - assert!(message.contains("staging directory"), "{message}"); - - // And what the operating system said, exactly once, one line down the - // chain rather than repeated into the message above it. - let source = std::error::Error::source(&error).expect("no cause to render"); - assert!(!message.contains(&source.to_string()), "{message}"); - } - - /// The unnamed case keeps the catch-all, and still works. - #[test] - fn an_unnamed_staging_directory_still_stages() { - scratch_in(None).expect("the platform temporary directory is unusable"); - } -} diff --git a/crates/stelae/src/plan.rs b/crates/stelae/src/plan.rs deleted file mode 100644 index b34b1bed2..000000000 --- a/crates/stelae/src/plan.rs +++ /dev/null @@ -1,396 +0,0 @@ -//! What a restore has already done, and what it has left to fetch. -//! -//! A restore of a stele at profile sizes is hours of download and ingestion, -//! and the interruptions it has to survive are the ordinary ones: a reboot, a -//! dropped connection, a ctrl-C. Without something on disk saying what landed, -//! the only answer is "start again". This module is that something, plus the -//! arithmetic a caller needs to say how much is left. -//! -//! ## The resume rule is content addressing, not a policy -//! -//! A layer is done when its **`diffId`** is in the progress file. That is the -//! whole rule, and it is a consequence of what a `diffId` *is*: sha256 over the -//! layer's uncompressed bytes. A match means the same bytes, so a layer -//! completed under an *older* inscription is still the layer this restore would -//! fetch — the stele it was published in has nothing to do with it. -//! -//! [`Resume`] therefore exposes exactly one question, [`Resume::is_done`], and -//! it takes a `Digest` and nothing else. There is deliberately no way to ask it -//! about a kind, a scope, a sequence or an inscription: comparing any of those -//! to decide a layer is done would be the wrong rule, and the cheapest place to -//! rule it out is the signature. -//! -//! ## What is protocol here, and what is not -//! -//! The file's **shape** and its invariants are protocol: an inscription digest -//! and a set of `diffId`s. What counts as a layer being **complete** is not — -//! that is the profile's commit boundary, and only a profile knows where its -//! writes become durable. So nothing in this module decides when to call -//! [`RestoreProgress::record`]; it only guarantees that what was recorded -//! survives. -//! -//! The file's **name and location** are the profile's too. ADR-004 spells them -//! `/.snapshot-restore.json` for the Dolos profile, and -//! "snapshot" is that profile's word for a stele rather than the protocol's — -//! so every entry point here takes a path a caller chose. -//! -//! ## What this module is not -//! -//! Not layer selection, and not the preflight. ADR-004's code-layout sketch -//! puts both here, and both belong to the profile for the reason the Dolos -//! restore driver states: a layer's `scope` is opaque to the protocol, so -//! nothing but a profile can read an epoch out of one, and nothing but a -//! profile knows which epochs a node wants. -//! -//! Not a renderer either. [`Remaining`] is a pair of numbers. Turning them into -//! a progress bar is a caller's business, and an observer here would be a -//! second one beside [`crate::progress`], which is the seam the export and -//! restore commands share and the only one either of them reports through. - -use std::{ - collections::BTreeSet, - io::Write as _, - path::{Path, PathBuf}, -}; - -use serde::{Deserialize, Serialize}; - -use crate::{inscription::LayerDescriptor, transport::BlobIndex, Digest, Error, SteleReader}; - -/// A restore's progress, as it survives the process making it. -/// -/// Exactly what ADR-004 asks the file to record: the inscription digest, and -/// the `diffId`s of the layers that are done. The digest is not what decides a -/// resume — see the module documentation — but it is what lets an operator, or -/// a later diagnostic, tell which stele a half-finished restore was aimed at. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RestoreProgress { - /// The stele this restore is reading, by identity. - pub inscription_digest: Digest, - - /// The layers whose records are committed, by identity. - /// - /// A set rather than a list: the order layers complete in carries no - /// meaning, and an ordered set gives the file the same bytes for the same - /// progress, which is one less thing to wonder about when reading it by - /// hand. - pub completed: BTreeSet, -} - -impl RestoreProgress { - /// A restore of `inscription_digest` that has completed nothing. - pub fn new(inscription_digest: Digest) -> Self { - Self { - inscription_digest, - completed: BTreeSet::new(), - } - } - - /// Read the progress file at `path`, or `None` if there is none. - /// - /// **Only absence is `None`.** A file that exists and does not parse is an - /// error, not an empty resume: treating it as absence would silently - /// restart a restore from zero, which is the multi-hour outcome this - /// file exists to prevent, arrived at without anything looking wrong. - /// An operator who wants that outcome has `--force`, which says so. - pub fn load(path: &Path) -> Result, Error> { - let raw = match std::fs::read(path) { - Ok(raw) => raw, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(e.into()), - }; - - Ok(Some(serde_json::from_slice(&raw)?)) - } - - /// Record that the layer identified by `diff_id` is committed. - /// - /// In memory only. [`RestoreProgress::save`] is what makes it survive, and - /// the two are separate calls because a caller may want to record several - /// layers against one write — though the Dolos driver does not, since one - /// layer is one commit. - pub fn record(&mut self, diff_id: Digest) { - self.completed.insert(diff_id); - } - - /// Write the progress file at `path`, atomically. - /// - /// Through a temporary sibling and a rename, because the failure this file - /// exists to survive is a process that stops mid-write. A progress file - /// truncated half way through its own `completed` array would be refused by - /// [`RestoreProgress::load`] — correctly, and uselessly: the restore it - /// described would have to start over, which is the thing being avoided. A - /// rename is atomic on every filesystem this runs on, so a reader sees the - /// previous complete file or the next one. - pub fn save(&self, path: &Path) -> Result<(), Error> { - let staging = staging_path(path); - - // Scoped so the handle is closed before the rename: Windows refuses to - // rename a file that is still open. - { - let mut file = std::fs::File::create(&staging)?; - - file.write_all(&serde_json::to_vec(self)?)?; - - // Before the rename, not after: a rename that lands pointing at - // bytes the page cache has not written yet is the same truncated - // file by another route. - file.sync_all()?; - } - - std::fs::rename(&staging, path)?; - - Ok(()) - } - - /// Delete the progress file at `path`. - /// - /// A file that is not there is not an error: this is called after a restore - /// finishes, and a restore that never had to checkpoint anything is a - /// restore that finished too. - pub fn remove(path: &Path) -> Result<(), Error> { - match std::fs::remove_file(path) { - Ok(()) => Ok(()), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e.into()), - } - } - - /// The resume this progress permits. - pub fn resume(&self) -> Resume { - Resume { - completed: self.completed.clone(), - } - } -} - -fn staging_path(path: &Path) -> PathBuf { - let name = path - .file_name() - .map(|name| name.to_string_lossy().into_owned()) - .unwrap_or_default(); - - path.with_file_name(format!(".{name}.{}.tmp", std::process::id())) -} - -/// Which layers a restore may skip because they are already done. -/// -/// One question, taking one kind of argument. See the module documentation for -/// why the shape is the point: a `diffId` match means the same bytes, and -/// nothing else about a layer is evidence that it landed. -/// -/// **A caller must only ask about layers that are immutable.** The protocol -/// cannot tell which those are — a layer's `scope` is the profile's — so this -/// type answers whatever it is asked, and it is the profile's job never to ask -/// about a layer that describes a moving tip. In the Dolos profile that means -/// the epoch layers may be skipped and the state shards never are. -#[derive(Debug, Clone, Default)] -pub struct Resume { - completed: BTreeSet, -} - -impl Resume { - /// A restore that starts from nothing — the ordinary case, and what a - /// caller without `--continue` passes. - pub fn none() -> Self { - Self::default() - } - - /// The resume a progress file permits, or a fresh start if there is none. - /// - /// Takes the completed set whatever inscription it was recorded under. That - /// is the resume rule, not an oversight: an epoch layer is named by its - /// content, so a newer stele describing the same layer describes the same - /// bytes. - pub fn from_progress(progress: Option<&RestoreProgress>) -> Self { - match progress { - Some(progress) => progress.resume(), - None => Self::none(), - } - } - - /// Whether the layer identified by `diff_id` is already committed. - pub fn is_done(&self, diff_id: &Digest) -> bool { - self.completed.contains(diff_id) - } - - /// How many layers this resume carries. For a caller reporting what it - /// inherited from an earlier attempt. - pub fn len(&self) -> usize { - self.completed.len() - } - - pub fn is_empty(&self) -> bool { - self.completed.is_empty() - } -} - -/// What a restore still has to move over the wire. -/// -/// Compressed bytes, because that is what a download costs and what a time -/// estimate divides by a rate. The inscription carries only *uncompressed* -/// sizes — identity does not depend on a compressor — so these come from the -/// transport, through [`SteleReader::compressed_size`]. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct Remaining { - /// Layers still to fetch. - pub layers: usize, - - /// Compressed bytes across those layers, as the transport states them. - pub compressed_bytes: u64, - - /// Layers the transport could not state a compressed size for, and which - /// `compressed_bytes` therefore does not include. - /// - /// Carried rather than folded in, so an estimate that is missing something - /// says so. A total that silently under-reports is worse than one a caller - /// can see the shape of. - pub unsized_layers: usize, - - /// The largest of those layers, compressed. - /// - /// A restore stages one layer at a time — pulled, staged, drained and - /// dropped in sequence — so what its scratch volume has to hold at once is - /// this and never [`Remaining::compressed_bytes`]. Summed here rather than - /// in a second pass because the walk that sums the total is already - /// visiting every size there is. - /// - /// `None` when nothing is left to fetch, and when no remaining layer could - /// be sized at all. - pub largest_compressed: Option, -} - -impl Remaining { - /// Sum what `layers` will cost to fetch from `stele`. - /// - /// The caller has already decided *which* layers those are — dropped what a - /// resume says is done, and what its own selection never wanted. That split - /// is the same one the rest of this module keeps: which layers a restore - /// needs is the profile's question, and how big they are is the - /// transport's. - /// - /// One pass answers both questions a caller has about these bytes: what - /// they cost to move, and — through [`Remaining::largest_compressed`] — - /// what has to fit on disk while one of them is being moved. - pub fn of<'a, R, I>(stele: &R, index: &BlobIndex, layers: I) -> Result - where - R: SteleReader, - I: IntoIterator, - { - let mut remaining = Self::default(); - - for descriptor in layers { - remaining.layers += 1; - - match stele.compressed_size(index, descriptor)? { - Some(bytes) => { - remaining.compressed_bytes += bytes; - remaining.largest_compressed = remaining.largest_compressed.max(Some(bytes)); - } - None => remaining.unsized_layers += 1, - } - } - - Ok(remaining) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn digest(byte: u8) -> Digest { - Digest::from_bytes([byte; 32]) - } - - #[test] - fn a_progress_file_round_trips_through_the_filesystem() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join(".snapshot-restore.json"); - - assert_eq!(RestoreProgress::load(&path).unwrap(), None); - - let mut progress = RestoreProgress::new(digest(0xaa)); - progress.record(digest(1)); - progress.record(digest(2)); - progress.save(&path).unwrap(); - - assert_eq!(RestoreProgress::load(&path).unwrap(), Some(progress)); - - RestoreProgress::remove(&path).unwrap(); - assert_eq!(RestoreProgress::load(&path).unwrap(), None); - - // Removing what is not there is what the end of a restore that never - // checkpointed does. - RestoreProgress::remove(&path).unwrap(); - } - - /// A save leaves the file and nothing beside it: the staging sibling is - /// renamed, not left for the next reader to trip over. - #[test] - fn a_save_leaves_one_file_behind() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join(".snapshot-restore.json"); - - let progress = RestoreProgress::new(digest(0xaa)); - - progress.save(&path).unwrap(); - progress.save(&path).unwrap(); - - let found: Vec = std::fs::read_dir(temp.path()) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - - assert_eq!(found, vec![".snapshot-restore.json".to_owned()]); - } - - /// Absence is a fresh start; a file that does not parse is not. - /// - /// The two are a sentence apart in the code and hours apart for an - /// operator: reading a corrupt file as "nothing has been done" restarts - /// a restore from zero without anything looking wrong. - #[test] - fn a_corrupt_progress_file_is_not_an_empty_one() { - let temp = tempfile::tempdir().unwrap(); - let path = temp.path().join(".snapshot-restore.json"); - - std::fs::write(&path, b"{\"inscriptionDigest\": \"not a digest\"").unwrap(); - - assert!(RestoreProgress::load(&path).is_err()); - } - - /// The resume rule, stated as the test that would fail if anything but a - /// `diffId` were being compared: the progress was recorded under one - /// inscription and is read while restoring another. - #[test] - fn a_layer_stays_done_across_an_inscription_change() { - let mut progress = RestoreProgress::new(digest(0xaa)); - progress.record(digest(1)); - - let resume = Resume::from_progress(Some(&progress)); - - assert!(resume.is_done(&digest(1))); - assert!(!resume.is_done(&digest(2))); - assert_eq!(resume.len(), 1); - - // The same set, now consulted while restoring a different stele. The - // digest the file records is not an input to the question, which is the - // whole of the rule. - let newer = RestoreProgress { - inscription_digest: digest(0xbb), - completed: progress.completed.clone(), - }; - - assert!(Resume::from_progress(Some(&newer)).is_done(&digest(1))); - } - - #[test] - fn no_progress_is_a_resume_that_skips_nothing() { - let resume = Resume::from_progress(None); - - assert!(resume.is_empty()); - assert!(!resume.is_done(&digest(1))); - } -} diff --git a/crates/stelae/src/profile.rs b/crates/stelae/src/profile.rs deleted file mode 100644 index 03f603046..000000000 --- a/crates/stelae/src/profile.rs +++ /dev/null @@ -1,502 +0,0 @@ -//! The profile extension point and the naming rules that keep profiles apart. -//! -//! A [`Profile`] is a vendor's definition of what a stele of theirs contains: -//! which layer kinds exist, what a record looks like, what goes in `position`, -//! `parameters` and `scope`, and how a sequence renders as a tag. Dolos ships -//! the first one (`io.txpipe.dolos.cardano`); nothing in this crate knows that. -//! -//! ## Where the boundary runs -//! -//! The protocol **asks** for every vendor-owned string and **validates** the -//! answer; it never builds one. `checked_layer_media_type` and -//! `checked_tag_for_sequence` are that rule made mechanical — the core calls -//! those, not [`Profile::layer_media_type`] and [`Profile::tag_for_sequence`] -//! directly, so a profile that returns a colliding or malformed name is refused -//! at the boundary instead of publishing something a third party cannot coexist -//! with. -//! -//! The rules themselves are normative (ADR-004, "Naming, profiles and media -//! types"): -//! -//! 1. Payload media types are -//! `application/vnd.{vendor}.stele.{kind}.v{n}+{codec}` and carry a vendor -//! slot the publisher controls. `vnd.stelae.*` is reserved for the -//! protocol's envelope types and is never a payload type. -//! 2. Profile names are reverse-DNS and vendor-owned. -//! 3. The protocol never parses layer bodies or a profile's opaque objects. An -//! unknown profile is a clean refusal, never a partial restore. - -use crate::{frame::DEFAULT_MAX_RECORD, Error, MOVING_TAG, RESERVED_VENDOR}; - -/// A vendor's definition of what its stelae contain. -/// -/// Implementations are expected to be cheap, stateless descriptions — the trait -/// answers questions about naming and kinds, and deliberately has no hook for -/// anything dataset-shaped. If an implementation of the protocol ever needs to -/// ask a profile about chain points, epochs, stores or blocks, the boundary has -/// failed and the fix belongs on the protocol side, not in another trait -/// method. -pub trait Profile { - /// Reverse-DNS profile name, e.g. `io.txpipe.dolos.cardano`. - fn name(&self) -> &str; - - /// Major version of the profile this implementation implements. - /// - /// A client refuses an inscription whose profile major version is above - /// this: a newer major may have changed record shapes it would otherwise - /// misread. - fn version(&self) -> u64; - - /// The layer kinds this profile defines. Used to refuse an inscription that - /// names a kind the profile does not know before any blob is fetched. - fn kinds(&self) -> &[&str]; - - /// Media type for a layer of `kind`. Vendor-owned; the protocol validates - /// the shape but never composes the string. - fn layer_media_type(&self, kind: &str) -> Result; - - /// The immutable tag under which a stele at `sequence` is published. - fn tag_for_sequence(&self, sequence: u64) -> Result; - - /// The moving tag pointing at the most recent stele. Profiles may override - /// the spelling; the protocol only requires that one exists. - fn moving_tag(&self) -> &str { - MOVING_TAG - } - - /// Largest single record this profile's layers may contain. - /// - /// One number, asked of one party, and it bounds **both** ends: what a - /// publisher is allowed to write and what a reader will accept. That is the - /// point of putting it here rather than letting each end carry a constant. - /// A writer with the looser of two ceilings publishes a stele that restores - /// nowhere, and it publishes it *successfully* — the failure surfaces on - /// whoever tries to read it back, which is the party least able to fix it. - /// - /// The default is [`DEFAULT_MAX_RECORD`]. A profile whose records genuinely - /// exceed it raises this deliberately and says why, which is the - /// conversation `DEFAULT_MAX_RECORD`'s own documentation asks for. - fn max_record(&self) -> usize { - DEFAULT_MAX_RECORD - } -} - -/// Ask `profile` for the media type of `kind` and enforce the protocol's naming -/// rules on the answer. -pub fn checked_layer_media_type(profile: &dyn Profile, kind: &str) -> Result { - if !profile.kinds().contains(&kind) { - return Err(Error::UnknownLayerKind { - profile: profile.name().to_owned(), - kind: kind.to_owned(), - }); - } - - let media_type = profile.layer_media_type(kind)?; - let parsed = MediaType::parse(&media_type)?; - - // Parsing establishes that the profile's answer is well formed; this - // establishes that it is an answer to the question asked. Without it a - // descriptor can carry a `kind` and a `mediaType` whose embedded kind - // disagree — the exact ambiguity the naming rules exist to remove. - if parsed.kind != kind { - return Err(Error::InvalidMediaType { - value: media_type, - reason: format!( - "names layer kind {:?}, but was asked for {kind:?}", - parsed.kind - ), - }); - } - - Ok(media_type) -} - -/// Ask `profile` for the immutable tag of `sequence` and enforce OCI tag syntax -/// on the answer. -pub fn checked_tag_for_sequence(profile: &dyn Profile, sequence: u64) -> Result { - let tag = profile.tag_for_sequence(sequence)?; - validate_tag(&tag)?; - Ok(tag) -} - -/// Validate a profile name: reverse-DNS, lowercase, at least two labels. -pub fn validate_profile_name(name: &str) -> Result<(), Error> { - let invalid = |reason: &str| Error::InvalidProfileName { - value: name.to_owned(), - reason: reason.to_owned(), - }; - - if name.is_empty() { - return Err(invalid("empty")); - } - - let labels: Vec<&str> = name.split('.').collect(); - - if labels.len() < 2 { - return Err(invalid("expected reverse-DNS with at least two labels")); - } - - for label in labels { - if label.is_empty() { - return Err(invalid("empty label")); - } - - if !label - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - { - return Err(invalid( - "labels may contain lowercase ascii letters, digits and hyphens only", - )); - } - - if label.starts_with('-') || label.ends_with('-') { - return Err(invalid("labels must not start or end with a hyphen")); - } - } - - Ok(()) -} - -/// Validate an OCI tag: `[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}`. -pub fn validate_tag(tag: &str) -> Result<(), Error> { - let invalid = |reason: &str| Error::InvalidTag { - value: tag.to_owned(), - reason: reason.to_owned(), - }; - - if tag.is_empty() { - return Err(invalid("empty")); - } - - if tag.len() > 128 { - return Err(invalid("longer than 128 characters")); - } - - let mut bytes = tag.bytes(); - let first = bytes.next().expect("non-empty"); - - if !(first.is_ascii_alphanumeric() || first == b'_') { - return Err(invalid("must start with an alphanumeric or underscore")); - } - - if !bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'.' || b == b'-') { - return Err(invalid( - "may contain alphanumerics, underscore, period and hyphen only", - )); - } - - Ok(()) -} - -/// A parsed payload media type: -/// `application/vnd.{vendor}.stele.{kind}.v{n}+{codec}`. -/// -/// The protocol parses these to enforce the coexistence rules. It does not -/// build them and does not attach meaning to `kind` or `codec`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MediaType { - pub vendor: String, - pub kind: String, - pub version: u64, - pub codec: String, -} - -const MEDIA_TYPE_PREFIX: &str = "application/vnd."; -const STELE_INFIX: &str = ".stele."; - -impl MediaType { - pub fn parse(value: &str) -> Result { - let invalid = |reason: &str| Error::InvalidMediaType { - value: value.to_owned(), - reason: reason.to_owned(), - }; - - let rest = value - .strip_prefix(MEDIA_TYPE_PREFIX) - .ok_or_else(|| invalid("expected the `application/vnd.` prefix"))?; - - let (body, codec) = rest - .split_once('+') - .ok_or_else(|| invalid("expected a `+{codec}` suffix"))?; - - if codec.is_empty() { - return Err(invalid("empty codec")); - } - - let (vendor, tail) = body - .split_once(STELE_INFIX) - .ok_or_else(|| invalid("expected `.stele.` between the vendor and the kind"))?; - - if vendor.is_empty() { - return Err(invalid("empty vendor")); - } - - // Rule 1: `vnd.stelae.*` names envelope types only. A payload type that - // claimed it would make a vendor's blobs indistinguishable from the - // protocol's own. - if vendor == RESERVED_VENDOR || vendor.starts_with(&format!("{RESERVED_VENDOR}.")) { - return Err(invalid( - "`vnd.stelae.*` is reserved for the protocol's envelope types", - )); - } - - if !vendor - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'.') - { - return Err(invalid( - "vendor may contain lowercase ascii letters, digits, hyphens and periods only", - )); - } - - let (kind, version) = tail - .rsplit_once(".v") - .ok_or_else(|| invalid("expected a `.v{n}` version segment after the kind"))?; - - if kind.is_empty() { - return Err(invalid("empty kind")); - } - - if !kind - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') - { - return Err(invalid( - "kind may contain lowercase ascii letters, digits and hyphens only", - )); - } - - if version.is_empty() || !version.bytes().all(|b| b.is_ascii_digit()) { - return Err(invalid("version must be a decimal integer")); - } - - let version = version - .parse::() - .map_err(|_| invalid("version is out of range"))?; - - Ok(Self { - vendor: vendor.to_owned(), - kind: kind.to_owned(), - version, - codec: codec.to_owned(), - }) - } -} - -impl std::fmt::Display for MediaType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{MEDIA_TYPE_PREFIX}{}{STELE_INFIX}{}.v{}+{}", - self.vendor, self.kind, self.version, self.codec - ) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_the_reference_profile_media_types() { - for (raw, vendor, kind) in [ - ( - "application/vnd.dolos.stele.blocks.v1+zstd", - "dolos", - "blocks", - ), - ( - "application/vnd.dolos.stele.state.v1+zstd", - "dolos", - "state", - ), - ( - "application/vnd.acme.stele.receipts.v1+zstd", - "acme", - "receipts", - ), - ( - "application/vnd.example.stele.notes.v1+zstd", - "example", - "notes", - ), - ] { - let parsed = MediaType::parse(raw).unwrap(); - assert_eq!(parsed.vendor, vendor); - assert_eq!(parsed.kind, kind); - assert_eq!(parsed.version, 1); - assert_eq!(parsed.codec, "zstd"); - assert_eq!(parsed.to_string(), raw); - } - } - - /// The reserved-vendor rule is the one that makes third-party coexistence - /// safe, so it is checked in both spellings. - #[test] - fn rejects_the_reserved_vendor_slot() { - for raw in [ - "application/vnd.stelae.stele.blocks.v1+zstd", - "application/vnd.stelae.inscription.stele.x.v1+zstd", - ] { - let err = MediaType::parse(raw).unwrap_err(); - assert!( - matches!(&err, Error::InvalidMediaType { reason, .. } if reason.contains("reserved")), - "{raw}: {err:?}" - ); - } - } - - /// The protocol's own envelope types are not payload types and must not - /// parse as one. - #[test] - fn envelope_types_are_not_payload_types() { - for raw in [ - crate::ARTIFACT_TYPE, - crate::INSCRIPTION_MEDIA_TYPE, - crate::SIGNATURE_MEDIA_TYPE, - ] { - assert!(MediaType::parse(raw).is_err(), "{raw} must not parse"); - } - } - - #[test] - fn rejects_malformed_media_types() { - for raw in [ - "application/json", - "application/vnd.dolos.stele.blocks.v1", - "application/vnd.dolos.blocks.v1+zstd", - "application/vnd..stele.blocks.v1+zstd", - "application/vnd.dolos.stele..v1+zstd", - "application/vnd.dolos.stele.blocks.v+zstd", - "application/vnd.dolos.stele.blocks.vx+zstd", - "application/vnd.dolos.stele.blocks.v1+", - "application/vnd.Dolos.stele.blocks.v1+zstd", - "application/vnd.dolos.stele.Blocks.v1+zstd", - ] { - assert!(MediaType::parse(raw).is_err(), "{raw} must not parse"); - } - } - - #[test] - fn validates_profile_names() { - for good in [ - "io.txpipe.dolos.cardano", - "dev.example.toy", - "com.acme.receipts", - "a.b", - ] { - validate_profile_name(good).unwrap(); - } - - for bad in [ - "", - "dolos", - "io..dolos", - "IO.txpipe.dolos", - "io.txpipe.dolos_cardano", - "io.-txpipe.dolos", - "io.txpipe-.dolos", - ] { - assert!(validate_profile_name(bad).is_err(), "{bad} must not pass"); - } - } - - #[test] - fn validates_tags() { - for good in ["latest", "epoch-550", "note-0", "v1.6.0", "_x"] { - validate_tag(good).unwrap(); - } - - for bad in ["", ".hidden", "-leading", "with space", "with/slash"] { - assert!(validate_tag(bad).is_err(), "{bad:?} must not pass"); - } - - assert!(validate_tag(&"a".repeat(128)).is_ok()); - assert!(validate_tag(&"a".repeat(129)).is_err()); - } - - struct Fake { - media_type: &'static str, - tag: &'static str, - } - - impl Profile for Fake { - fn name(&self) -> &str { - "dev.example.fake" - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["notes"] - } - - fn layer_media_type(&self, _kind: &str) -> Result { - Ok(self.media_type.to_owned()) - } - - fn tag_for_sequence(&self, _sequence: u64) -> Result { - Ok(self.tag.to_owned()) - } - } - - /// A profile that hands back a colliding or malformed name is refused at - /// the boundary — the protocol validates what it is given rather than - /// trusting it, which is the other half of "never construct the string - /// yourself". - #[test] - fn checked_accessors_refuse_bad_profile_answers() { - let good = Fake { - media_type: "application/vnd.example.stele.notes.v1+zstd", - tag: "note-1", - }; - assert_eq!( - checked_layer_media_type(&good, "notes").unwrap(), - "application/vnd.example.stele.notes.v1+zstd" - ); - assert_eq!(checked_tag_for_sequence(&good, 1).unwrap(), "note-1"); - - let colliding = Fake { - media_type: "application/vnd.stelae.stele.notes.v1+zstd", - tag: "note-1", - }; - assert!(checked_layer_media_type(&colliding, "notes").is_err()); - - let bad_tag = Fake { - media_type: "application/vnd.example.stele.notes.v1+zstd", - tag: "note 1", - }; - assert!(checked_tag_for_sequence(&bad_tag, 1).is_err()); - - // A kind the profile does not declare never reaches the profile at all. - let err = checked_layer_media_type(&good, "blocks").unwrap_err(); - assert!(matches!(err, Error::UnknownLayerKind { .. }), "{err:?}"); - } - - /// A well-formed name for the *wrong* kind is still a wrong answer: the - /// descriptor it would produce carries a `kind` and a `mediaType` that - /// disagree, which is the ambiguity the naming rules exist to remove. - #[test] - fn checked_media_type_refuses_a_name_for_another_kind() { - let mislabelled = Fake { - media_type: "application/vnd.example.stele.blocks.v1+zstd", - tag: "note-1", - }; - - let err = checked_layer_media_type(&mislabelled, "notes").unwrap_err(); - assert!(matches!(err, Error::InvalidMediaType { .. }), "{err:?}"); - } - - #[test] - fn moving_tag_defaults_to_latest() { - let profile = Fake { - media_type: "application/vnd.example.stele.notes.v1+zstd", - tag: "note-1", - }; - assert_eq!(profile.moving_tag(), MOVING_TAG); - validate_tag(profile.moving_tag()).unwrap(); - } -} diff --git a/crates/stelae/src/progress.rs b/crates/stelae/src/progress.rs deleted file mode 100644 index 082f13c20..000000000 --- a/crates/stelae/src/progress.rs +++ /dev/null @@ -1,283 +0,0 @@ -//! What a transfer says about itself while it is still running. -//! -//! A publish and a restore are the two operations in this protocol that take -//! hours, and until this module existed both were silent for every one of them: -//! a caller learned what happened when it was over, from an [`Inscription`] or -//! a summary. This is the seam that makes the middle visible, and it is -//! deliberately the *only* one — there is no second observer on the profile -//! side and no `tracing` subscriber standing in for one. -//! -//! [`Inscription`]: crate::Inscription -//! -//! ## It carries callbacks, never counters -//! -//! Nothing here accumulates. [`Observer`] holds a handle and forwards; an -//! [`Event`] carries values the code emitting it already had in hand — the -//! layer it is on, the size the manifest states, the bytes a chunk moved. A -//! number this module would have to compute for itself is a number that does -//! not belong in it, because a transport that keeps a tally the caller does not -//! ask for is a transport whose cost nobody chose. -//! -//! ## Two emitters, because the numbers live in two places -//! -//! Neither half is honest alone: -//! -//! - the **profile driver** owns the layer loops, so it is the only code that -//! knows *n* of *m*, a layer's kind and scope, and whether it was produced, -//! inherited or skipped; -//! - the **transport** owns the bytes. A publish stages a layer and then -//! uploads it in chunks, and a restore pulls a whole blob to scratch before -//! the first record comes back out of it — so an observer wired only to the -//! driver reports nothing for the entire duration of a download, which on the -//! restore side is the entire operation. -//! -//! The two halves are not in step, and on the publish side they are -//! deliberately not: the transport moves a layer's blob concurrently with the -//! driver building the next one, so a blob's bytes arrive after the driver has -//! already closed the layer they belong to. Everything is reported before the -//! operation returns — that is what the seal's join buys — and nothing about -//! the order in between is promised. -//! -//! The transports answer through [`SteleWriter::observe`] and -//! [`SteleReader::observe`], which have default no-op bodies: reporting is -//! something a transport *may* do, not a tax every implementation pays. -//! -//! [`SteleWriter::observe`]: crate::SteleWriter::observe -//! [`SteleReader::observe`]: crate::SteleReader::observe -//! -//! ## Rendering is nobody's business here -//! -//! An [`Event`] is a fact, not a line of output. Which bar moves, whether -//! anything is drawn at all, and what a human reads is the binary's, which is -//! why the default is silence and why the handle is passed as an argument -//! rather than installed globally. - -use std::sync::Arc; - -/// Somewhere to report a transfer's progress to. -/// -/// One method, taking one enum, for a reason worth stating: a trait with a -/// method per event kind would make every new fact a breaking change for every -/// implementor, and the implementors are renderers — the least interesting code -/// to have to revisit. Matching an enum, a renderer that does not care about a -/// variant writes one arm. -/// -/// `Send + Sync` because a transport holds it behind an [`Arc`] and calls it -/// from wherever the bytes happen to be moving. -pub trait Progress: Send + Sync { - fn on(&self, event: Event<'_>); -} - -/// A handle on whatever is watching, and silence by default. -/// -/// Cheap to clone — an [`Arc`] bump — because both a driver and a transport -/// hold one for the same run. [`Observer::silent`] is the whole of "a caller -/// that passes nothing": every emission becomes a branch on `None`, and the -/// output is byte-for-byte what it was before this seam existed. -#[derive(Clone, Default)] -pub struct Observer(Option>); - -impl Observer { - /// An observer nobody is listening to. - pub fn silent() -> Self { - Self(None) - } - - /// Report to `progress`. - pub fn new(progress: Arc) -> Self { - Self(Some(progress)) - } - - /// Whether anything is listening. - /// - /// For an emitter deciding whether a *costly* event is worth assembling — - /// never for deciding whether to do the work, which must not depend on who - /// is watching. - pub fn is_silent(&self) -> bool { - self.0.is_none() - } - - pub fn emit(&self, event: Event<'_>) { - if let Some(progress) = &self.0 { - progress.on(event); - } - } -} - -/// What was resolved, and nothing about who is listening. -impl std::fmt::Debug for Observer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self.is_silent() { - true => f.write_str("Observer(silent)"), - false => f.write_str("Observer(watching)"), - } - } -} - -/// How a driver was done with a layer. -/// -/// Three outcomes rather than a boolean because the two ways of *not* moving a -/// layer cost different things and mean different things to whoever is -/// watching: an inherited layer was never read out of a store at all, while a -/// skipped one is work an earlier attempt already paid for. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Outcome { - /// The driver read the layer end to end: built out of the stores on a - /// publish, streamed into them on a restore. - /// - /// Whether its blob then crossed the wire is a separate question, and the - /// transport answers it with [`Event::Blob`] — a layer a publisher builds - /// and a registry turns out to already hold is `Transferred` here and - /// `Blob { moved: false, .. }` there. - Transferred, - - /// Nothing was done for it, because it was already done: a layer an - /// interrupted restore had committed before it died. - Skipped, - - /// Adopted whole from the stele before it, and never read out of a store. - Inherited, -} - -/// One thing that happened, as the code that did it saw it. -/// -/// Deltas rather than running totals throughout ([`Event::Records`], -/// [`Event::Bytes`]): a total is state, and state is what this seam does not -/// keep. A renderer that wants one adds them up, which is what a renderer is -/// for. -#[derive(Debug, Clone, Copy)] -pub enum Event<'a> { - /// A layer is now in flight: the `index`-th of `total`, counting from zero. - /// - /// `scope` is the profile's own opaque description of what the layer covers - /// — the epoch, the shard — carried so a watcher can name the layer the way - /// the inscription does rather than by its position alone. - LayerStarted { - index: usize, - total: usize, - kind: &'a str, - scope: &'a serde_json::Value, - }, - - /// The layer at `index` is done, one way or another. - /// - /// Carries the index rather than relying on the last - /// [`Event::LayerStarted`] because a profile may hold several layers - /// open at once — this one's sixteen state shards are written in a - /// single pass over the store — so "the layer in flight" is not always - /// a single thing. - LayerFinished { - index: usize, - total: usize, - kind: &'a str, - outcome: Outcome, - }, - - /// Records that went past since the last one of these. - /// - /// Batched at the emitter's own cadence rather than one per record: the - /// point is a bar that moves inside a layer that takes minutes, and a - /// virtual call per record on a mainnet store would be paying for - /// resolution nobody can see. - Records(u64), - - /// One layer's blob, announced before the transport handles it. - /// - /// `bytes` is its compressed size, which the transport knows up front in - /// both directions — from the digest pipeline on the way up, from the - /// manifest on the way down — so a watcher can size the transfer it is - /// about to see. `moved` is false when nothing will cross the wire because - /// the far side already holds it, and no [`Event::Bytes`] follows for it. - /// - /// **Several of these can be outstanding at once, and one is not finished - /// when the next arrives.** A publish runs its layer round trips - /// concurrently, so what this announces is one more blob the transfer has - /// taken on rather than the blob the transfer is now on. A renderer that - /// reset a per-blob bar here would show eight uploads fighting over one - /// bar; the shape that reads correctly is a running total, and the totals - /// are exact because every announced blob is accounted for before the - /// operation that announced it returns. - Blob { moved: bool, bytes: u64 }, - - /// Compressed bytes that crossed the wire since the last one of these. - /// - /// Across every blob the transport currently has in flight, and not for the - /// blob the most recent [`Event::Blob`] announced — see there. A publish - /// that is uploading eight layers at once reports one stream of deltas, - /// because the thing an operator is watching is the link and not one of the - /// eight. - /// - /// **These can total more than the blobs announced.** A round trip the - /// registry failed is made again from the blob's first byte, and the bytes - /// the lost attempt moved were still moved — see [`Event::Retry`]. So a - /// renderer keeping a running total holds the total to at least the - /// position rather than assuming the announcements bound it; what it is - /// reporting is the link, and the link carried them. - Bytes(u64), - - /// A round trip that failed in a way the transport answered by making it - /// again. - /// - /// Emitted before the wait, once per attempt that was thrown away, and it - /// is the only trace a retry leaves: a transport that quietly absorbed a - /// registry's `5xx` would turn "this registry is unwell" into "publishes - /// got slower", which is the diagnosis nobody can act on. A publisher's - /// business is to make progress anyway; an operator's is to know it had - /// to. - /// - /// `attempt` is the one that just failed, counting from one; `remaining` is - /// how many are left after it, so a watcher can tell a hiccup from a - /// transport about to give up. `reason` is the failure as it rendered - /// itself, because what an operator reads is the binary's business and not - /// this crate's. - Retry { - attempt: u32, - remaining: u32, - reason: &'a str, - }, -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Mutex; - - #[derive(Default)] - struct Recorder(Mutex>); - - impl Progress for Recorder { - fn on(&self, event: Event<'_>) { - if let Event::Bytes(n) = event { - self.0.lock().unwrap().push(n); - } - } - } - - /// The property every silent call site depends on: emitting into silence - /// does nothing and costs a branch. - #[test] - fn a_silent_observer_swallows_everything() { - let observer = Observer::silent(); - - assert!(observer.is_silent()); - observer.emit(Event::Bytes(1)); - observer.emit(Event::Records(1)); - - assert!(Observer::default().is_silent()); - } - - /// And a clone reports to the same place, which is what lets a driver and a - /// transport share one run's observer. - #[test] - fn a_clone_reports_to_the_same_place() { - let recorder = Arc::new(Recorder::default()); - let observer = Observer::new(recorder.clone()); - - assert!(!observer.is_silent()); - - observer.emit(Event::Bytes(1)); - observer.clone().emit(Event::Bytes(2)); - - assert_eq!(*recorder.0.lock().unwrap(), vec![1, 2]); - } -} diff --git a/crates/stelae/src/transport.rs b/crates/stelae/src/transport.rs deleted file mode 100644 index d6d5934e1..000000000 --- a/crates/stelae/src/transport.rs +++ /dev/null @@ -1,547 +0,0 @@ -//! The seam a stele is written through and read back from. -//! -//! Until this module existed there was exactly one way to hold a stele — a -//! directory ([`crate::dir::SteleDir`]) — and a profile that wanted to publish -//! one named that type in its signatures. That is the wrong dependency: -//! *where* a stele lives is transport, and a profile has no opinion about it. -//! -//! What a profile actually uses is two small halves: -//! -//! - the **write** half ([`SteleWriter`]) opens a one-record-at-a-time sink for -//! a [`LayerSpec`] and, when every layer is written, accepts the finished -//! inscription; -//! - the **read** half ([`SteleReader`]) reads an inscription, hands over the -//! identity→blob map, and streams a layer named by a [`LayerDescriptor`] -//! under [`Limits`]. -//! -//! Both are exactly what a directory already did, so the directory keeps -//! working unchanged and a registry lands beside it. -//! -//! ## `BlobIndex` is the concept that generalizes -//! -//! An inscription lists `diffId`s — identity — and deliberately never the -//! compressed digests that address a blob, because those are transport and -//! vary with the compressor. So every reader needs a map from one to the other, -//! and *how it obtains that map* is the sharpest difference between the two -//! transports there is: -//! -//! - a directory has no manifest, so [`crate::dir::SteleDir::blob_index`] -//! rebuilds the map by decompressing every blob — a full verification pass -//! over the stele, paid before the restore reads any of it; -//! - a registry has a manifest, and reads the map straight off it. -//! -//! Same type, same meaning, one scan versus one HTTP GET. Keeping [`BlobIndex`] -//! rather than parameterizing the reader on "how to find a blob" is what makes -//! that difference a cost and not an interface. -//! -//! ## A third implementation that stores nothing -//! -//! [`Discarding`] is the write half with the storing taken out: every layer is -//! framed, hashed and compressed exactly as a publish would, and the bytes go -//! to [`std::io::sink`] instead of to a file or a registry. What comes back is -//! the identity — a [`WrittenLayer`] per layer and the inscription's digest -//! from [`SteleWriter::seal`] — which is the whole of what a reproduction needs -//! and none of what it would have to store. -//! -//! ## What the seam deliberately does not carry -//! -//! No notion of *listing* what a repository holds, no tags beyond the two the -//! inscription's sequence implies, and nothing about signatures. A profile that -//! needs to ask a repository what is in it is asking a transport-specific -//! question and should hold the transport-specific type. - -use serde::{Deserialize, Serialize}; - -use crate::{ - digest::{LayerDigests, LayerWriter}, - frame::{CanonicalCbor, LayerHeader, Limits, SeqWriter}, - inscription::{Inscription, LayerDescriptor}, - layer::LayerReader, - profile::{checked_layer_media_type, Profile}, - progress::Observer, - Digest, Error, -}; - -/// What a profile has to say about a layer it is asking the protocol to write. -/// -/// Both scopes are the profile's and stay opaque: `header_scope` rides in the -/// layer's own header record so a detached blob is still interpretable, and -/// `scope` rides in the inscription so a client can plan without fetching -/// anything. They are different encodings of the same profile-owned idea, and -/// the protocol carries both without reading either. -#[derive(Debug, Clone)] -pub struct LayerSpec { - pub kind: String, - pub header_scope: CanonicalCbor, - pub scope: serde_json::Value, -} - -impl LayerSpec { - pub fn new( - kind: impl Into, - header_scope: CanonicalCbor, - scope: serde_json::Value, - ) -> Self { - Self { - kind: kind.into(), - header_scope, - scope, - } - } -} - -/// A written layer: the descriptor to put in the inscription, plus the -/// transport facts that do not belong there. -/// -/// Serializable so that a host can *hold one across a process*. Nothing in the -/// protocol reads such a file — it is not a stele and never becomes one — but a -/// publisher that wants to carry a layer it already uploaded into a later -/// attempt has to write down what the transport told it, and writing down a -/// half of the pair is what makes a descriptor and a blob able to disagree. See -/// [`crate::oci::Registry::adopt_carried`]. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct WrittenLayer { - pub descriptor: LayerDescriptor, - pub digests: LayerDigests, -} - -/// Map from a layer's identity (`diffId`) to the blob that holds it. -/// -/// In a registry this comes from the manifest. In a directory it is recovered -/// by [`crate::dir::SteleDir::blob_index`], which is the same map at a much -/// higher price — see the module documentation. -#[derive(Debug, Clone, Default)] -pub struct BlobIndex(std::collections::BTreeMap); - -impl BlobIndex { - pub fn blob_for(&self, diff_id: &Digest) -> Option { - self.0.get(diff_id).copied() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Record that `blob` holds the layer whose identity is `diff_id`. - /// - /// A second entry for one `diffId` is not a conflict to resolve: two blobs - /// that decompress to the same bytes are the same layer, differently - /// compressed, and either serves. The last one wins because a caller - /// building the index in manifest order should end up with the manifest's - /// own answer. - pub fn insert(&mut self, diff_id: Digest, blob: Digest) { - self.0.insert(diff_id, blob); - } -} - -impl FromIterator<(Digest, Digest)> for BlobIndex { - fn from_iter>(entries: I) -> Self { - Self(entries.into_iter().collect()) - } -} - -/// A layer being written, one record at a time. -/// -/// Push-style for the same reason a reader is pull-style: the party that owns -/// the records owns the errors of producing them. A profile streaming out of a -/// fallible store iterator keeps its own error type on its own side of the -/// boundary, and the protocol only ever sees a [`CanonicalCbor`] it was handed. -/// -/// Nothing is buffered. Records are framed, hashed and compressed on the way -/// past, so a layer of any size costs the compressor's window and one record — -/// which is the property that makes a transport usable at profile sizes, and -/// the reason this is a trait rather than a convenience. -/// -/// ## Nothing exists until `finish` -/// -/// A layer's name is the digest of its own compressed bytes, so it cannot be -/// known before the last record is written. Until then the layer is staged, -/// invisible to any reader, and a sink dropped without [`RecordSink::finish`] -/// takes its staging with it — an export that fails halfway leaves no partial -/// layer behind. -pub trait RecordSink { - /// Append one of the profile's records. - /// - /// The header record is already written; everything a caller adds is - /// content. [`CanonicalCbor`] is the proof that the record is in - /// deterministic form, so nothing is re-checked. - fn write_record(&mut self, record: &CanonicalCbor) -> Result<(), Error>; - - /// Records written so far, header record included — the number that ends - /// up in the descriptor. - fn records(&self) -> u64; - - /// Close the layer and hand back the descriptor to put in the inscription. - fn finish(self) -> Result; -} - -/// The write half of a stele. -/// -/// Two operations, in this order: open a sink per layer, then seal the stele -/// with the inscription that describes them. Everything vendor-owned — the -/// media type of a layer, the tag a sequence publishes under — is asked of the -/// [`Profile`] and validated against the naming rules, never composed here. -pub trait SteleWriter { - type Sink: RecordSink; - - /// Open a layer and stream records into it. - /// - /// The media type comes from the profile and is validated against the - /// naming rules before anything is created, so a profile that claims a name - /// it does not own is refused before it can publish under it. - /// - /// A sink owns everything it needs rather than borrowing the stele, so a - /// producer can hold many open at once and route each record to one of - /// them. That is the shape the Dolos profile's sixteen state shards need: - /// one pass over the store, sixteen layers being written. - fn layer_sink( - &self, - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - ) -> Result; - - /// Accept the finished inscription and make the stele readable. - /// - /// The last thing a publish does, and the reason it is one call rather than - /// a write followed by a flush: for a registry this is where the config - /// blob, the manifest and both tags land, in the one order that never - /// leaves a reader following the moving tag pointed at a stele whose blobs - /// are still uploading. - /// - /// `profile` is unused by a transport that does not name things — a - /// directory has no tags — and load-bearing for one that does. - /// - /// Returns the stele's identity: the sha256 of the canonical inscription. - fn seal(&self, profile: &dyn Profile, inscription: &Inscription) -> Result; - - /// Frame, compress and store one layer a caller already holds. - /// - /// A convenience over [`SteleWriter::layer_sink`] and nothing more, so the - /// buffered and streaming write paths cannot drift apart the way two - /// implementations would. It is the right call for a layer that fits in - /// memory — every record has to be materialized somewhere that outlives the - /// call, which is exactly what a sink exists to avoid at profile sizes. - fn write_layer<'a, I>( - &self, - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - records: I, - ) -> Result - where - I: IntoIterator, - Self: Sized, - { - let mut sink = self.layer_sink(profile, spec, level)?; - - for record in records { - sink.write_record(record)?; - } - - sink.finish() - } - - /// Attest a layer this stele already carries under a second descriptor. - /// - /// One blob, two names. A profile may need the same byte string described - /// twice — the Dolos profile's retained state dump at the epoch it is cut - /// in *is* that epoch's tip, same header and same records — and the - /// protocol has no reason to compress or move those bytes a second time - /// to say so. - /// - /// It takes a `scope` rather than a whole [`LayerSpec`], and that is what - /// makes the identity structural instead of a property the caller has to - /// keep true: there is no second header record to disagree with the first, - /// because there is no second write. The two descriptors are the same - /// `diffId`, the same `records` and the same `uncompressedSize`, differing - /// in exactly the one field the profile owns. - /// - /// **The default body is right for a transport that stores blobs by - /// content and keeps no list of what it is about to seal** — a directory - /// has a file named by a digest and nothing to tell. A transport that - /// builds its manifest out of what it wrote must override this and record - /// the second descriptor, or its seal will refuse a document describing a - /// layer it has no record of. - fn carry_again( - &self, - written: &WrittenLayer, - scope: serde_json::Value, - ) -> Result { - Ok(again(written, scope)) - } - - /// Report what this transport moves to `observer`, for as long as it is - /// attached. - /// - /// **The default body does nothing, and that is the point.** A transport - /// that has no bytes of its own to report — [`crate::dir::SteleDir`] writes - /// straight to files a caller can watch, [`Discarding`] moves nothing — - /// implements nothing and loses nothing; only [`crate::oci::Registry`], - /// whose uploads are the hours a publisher waits through, overrides it. A - /// method every implementor must answer would be a tax on every future - /// transport for a capability most of them do not have. - /// - /// The profile driver is what calls this — it is handed the observer as an - /// argument and passes it on — so a caller wires one observer once and both - /// halves of the stream come back through it. - fn observe(&self, _observer: Observer) {} -} - -/// The second descriptor [`SteleWriter::carry_again`] hands back. -/// -/// Shared rather than written twice, so an overriding transport and the -/// default cannot drift about which fields a re-attested layer keeps. -pub fn again(written: &WrittenLayer, scope: serde_json::Value) -> WrittenLayer { - WrittenLayer { - descriptor: LayerDescriptor { - scope, - ..written.descriptor.clone() - }, - digests: written.digests, - } -} - -/// Open a layer: everything every [`SteleWriter::layer_sink`] does before its -/// own sink struct exists. -/// -/// The media type is resolved through the profile and validated against the -/// naming rules, the framing is wrapped at the profile's record ceiling, and -/// the encoded [`LayerHeader`] goes in as the layer's first record. All three -/// are inside the layer's identity, so a transport that skipped any of them -/// would produce blobs that hash cleanly and are refused on read — a failure -/// that would be that transport's alone, invisible to every test exercising -/// the others. It lives here, once, so the next transport cannot open a layer -/// wrongly by omission. -/// -/// The byte destination arrives as a closure rather than a value because the -/// order matters: a profile claiming a media type it does not own is refused -/// *before* `sink` runs, so nothing is created — no staging file, no scratch -/// file — for a layer that was never going to be written. -pub(crate) fn open_layer( - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - sink: F, -) -> Result<(SeqWriter>, String), Error> -where - W: std::io::Write, - F: FnOnce() -> Result, -{ - let media_type = checked_layer_media_type(profile, &spec.kind)?; - let header = LayerHeader::new(profile.name(), &spec.kind, spec.header_scope.clone()); - - let mut sequence = - SeqWriter::with_max_record(LayerWriter::new(sink()?, level)?, profile.max_record()); - - sequence.write_record(&header.encode()?)?; - - Ok((sequence, media_type)) -} - -/// A stele that computes its identity and stores nothing. -/// -/// The write half of the seam with the storing taken out. Every field of a -/// [`WrittenLayer`] is a function of the record stream — [`RecordSink::finish`] -/// reads the descriptor off the digests the pipeline computed, and only *then* -/// does a directory rename onto the content-addressed name — so a writer whose -/// sinks discard their bytes hands back the same descriptors as one that keeps -/// them. Sealing is the same identity a directory returns after writing -/// `inscription.json`: the sha256 of the canonical document. -/// -/// That is the whole of `dolos snapshot digest`. A verifier reproduces a -/// published stele's layers from its own stores and compares descriptors, -/// without provisioning the disk the stele would occupy — hundreds of gigabytes -/// on mainnet — and without touching a registry. -/// -/// ## It compresses -/// -/// The one shortcut this type must not take. `diffId`, `records` and -/// `uncompressedSize` are all fixed before zstd sees a byte, so a writer that -/// skipped compression would reproduce every field the inscription carries and -/// still not be doing what a publish does. The bug class this exists to catch -/// is the one that only appears when the same bytes go through the same -/// pipeline twice, and the pipeline is [`LayerWriter`] — hash in, compress, -/// hash out. What is dropped is the last step, the write to a file, and nothing -/// upstream of it. -/// -/// The cost of that honesty is real: a reproduction pays the compressor in full -/// and saves only the I/O. It buys the blob digest and the compressed size, -/// which a comparison against a registry manifest needs and a document cannot -/// supply. -/// -/// ## What it cannot answer -/// -/// Nothing that needs bytes back. There is no [`SteleReader`] half here and -/// there cannot be one: a stele that stored nothing has nothing to stream, and -/// a caller wanting both halves wants a real transport. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct Discarding; - -/// A layer being written into nothing, one record at a time. -/// -/// The mirror of [`crate::dir::LayerSink`] with the file removed. Nothing is -/// buffered here either — records are framed, hashed and compressed on the way -/// past — so reproducing a mainnet state shard costs the compressor's window -/// and one record, the same as publishing one. -pub struct DiscardingSink { - sequence: SeqWriter>, - kind: String, - media_type: String, - scope: serde_json::Value, -} - -impl RecordSink for DiscardingSink { - fn write_record(&mut self, record: &CanonicalCbor) -> Result<(), Error> { - self.sequence.write_record(record) - } - - fn records(&self) -> u64 { - self.sequence.count() - } - - /// Close the layer and hand back the descriptor it would have been - /// published under. - /// - /// Finishing the compressed frame is not skippable: zstd's epilogue is part - /// of the blob, so the blob digest and the compressed size are only correct - /// once the encoder has been closed. Everything else falls out of the same - /// [`LayerDigests`] a directory reads. - fn finish(self) -> Result { - let Self { - sequence, - kind, - media_type, - scope, - } = self; - - let count = sequence.count(); - let (_, digests) = sequence.into_inner().finish()?; - - Ok(WrittenLayer { - descriptor: LayerDescriptor { - kind, - media_type, - diff_id: digests.diff_id, - records: count, - uncompressed_size: digests.uncompressed_size, - scope, - }, - digests, - }) - } -} - -impl SteleWriter for Discarding { - type Sink = DiscardingSink; - - /// Open a layer that goes nowhere. - /// - /// The media type is still asked of the profile and still validated against - /// the naming rules, and the header record is still the first thing in the - /// sequence. Both are inside the layer's identity, so a reproduction that - /// skipped either would compute a `diffId` no publish ever produces. - fn layer_sink( - &self, - profile: &dyn Profile, - spec: &LayerSpec, - level: i32, - ) -> Result { - let (sequence, media_type) = open_layer(profile, spec, level, || Ok(std::io::sink()))?; - - Ok(DiscardingSink { - sequence, - kind: spec.kind.clone(), - media_type, - scope: spec.scope.clone(), - }) - } - - /// Return the stele's identity without writing it down. - /// - /// [`Inscription::digest`] canonicalizes, which validates — so a document - /// that could not be sealed into a directory is refused here too, and a - /// reproduction never reports a digest over an inscription no publish could - /// have written. - fn seal(&self, _profile: &dyn Profile, inscription: &Inscription) -> Result { - inscription.digest() - } -} - -/// The read half of a stele. -/// -/// Three operations, in this order: read the inscription, obtain the -/// identity→blob map, then stream the layers the inscription describes. A -/// caller checks the profile between the first and the third — the protocol -/// cannot, because it does not know which profile the caller implements — and -/// that check is what makes an unreadable stele a clean refusal rather than a -/// partial restore. -pub trait SteleReader { - /// The byte source a layer is read out of. A file for both transports - /// today, because a registry stages a pulled blob rather than decompressing - /// off the socket. - type Blob: std::io::Read; - - /// Read and verify the inscription. - /// - /// The bytes must *be* the canonical encoding, not merely parse to the same - /// content: they are what a verifier hashes, so a re-encoded copy carries a - /// digest nobody else computes and is rejected rather than silently - /// repaired. - fn read_inscription(&self) -> Result; - - /// The `diffId` → blob map for this stele. - fn blob_index(&self) -> Result; - - /// How many compressed bytes fetching this layer moves, if the transport - /// can say. - /// - /// The inscription carries only *uncompressed* sizes, because identity must - /// not depend on a compressor — so it is the one number a document cannot - /// answer, and the only place "how much is left to download" can come from. - /// Where a transport keeps it differs exactly as [`BlobIndex`] does: a - /// registry reads it off the manifest, a directory asks the filesystem how - /// big the blob file is. Neither reads the blob. - /// - /// `None` means the transport holds no size for this layer, not that the - /// layer is empty. A caller summing these should carry the count of - /// unanswered layers rather than treat them as zero — - /// [`crate::plan::Remaining`] does. - fn compressed_size( - &self, - index: &BlobIndex, - descriptor: &LayerDescriptor, - ) -> Result, Error>; - - /// Stream one layer's records without holding it. - /// - /// Every claim the descriptor makes is checked, each as early as the bytes - /// allow: the header record's profile and kind on construction, the - /// decompression ceiling as the stream advances, and the identity digest, - /// the uncompressed size and the record count in - /// [`LayerReader::finish`]. Records are therefore consumable *before* the - /// layer is proven; see the [`crate::layer`] module documentation for the - /// discipline that requires of a consumer. - fn stream_layer( - &self, - index: &BlobIndex, - profile: &dyn Profile, - descriptor: &LayerDescriptor, - limits: Limits, - ) -> Result, Error>; - - /// Report what this transport moves to `observer`, for as long as it is - /// attached. - /// - /// The read half of [`SteleWriter::observe`], with the same default and the - /// same reason for it. It matters more here than there: a registry reader - /// pulls a whole blob to scratch before it yields its first record, so a - /// restore that reported only what the profile driver sees would be silent - /// for the download — and on a restore the download *is* the work. - fn observe(&self, _observer: Observer) {} -} diff --git a/crates/stelae/tests/data/rfc8785/README.md b/crates/stelae/tests/data/rfc8785/README.md deleted file mode 100644 index 6b535abb9..000000000 --- a/crates/stelae/tests/data/rfc8785/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# RFC 8785 (JCS) conformance vectors - -Vendored, unmodified, from the reference implementation repository that -accompanies RFC 8785: - -- Source: , `testdata/input` - and `testdata/output`. -- Retrieved: 2026-07-31. -- Licence: Apache-2.0 — same licence as this repository. - -`input/{name}.json` is the document to canonicalize; `output/{name}.json` is the -expected canonical form, compared **byte for byte** (the outputs carry raw UTF-8 -and a raw `0x7f`, so they are read as bytes, never as text). - -What each one pins down: - -| Vector | What it would catch | -|---|---| -| `arrays` | array order preserved while object keys are sorted | -| `french` | sorting is by code unit, never by locale collation | -| `structures` | nested objects sorted independently; empty objects; `\n` escaping | -| `unicode` | no Unicode normalization — `Å` stays decomposed | -| `values` | ECMAScript number rendering, and the JSON string escape set | -| `weird` | sorting by **UTF-16** code units: `😂` (surrogate pair `D83D DE02`) sorts *before* `דּ`, which bytewise UTF-8 ordering would get backwards | - -The suite's third file, `es6testfile100m.txt` (100 million number samples), is -deliberately not vendored. The number-rendering edge cases it covers are checked -instead from the table in RFC 8785 Appendix B, which is reproduced in -`tests/rfc8785.rs`. diff --git a/crates/stelae/tests/data/rfc8785/input/arrays.json b/crates/stelae/tests/data/rfc8785/input/arrays.json deleted file mode 100644 index 20e62263c..000000000 --- a/crates/stelae/tests/data/rfc8785/input/arrays.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - 56, - { - "d": true, - "10": null, - "1": [ ] - } -] diff --git a/crates/stelae/tests/data/rfc8785/input/french.json b/crates/stelae/tests/data/rfc8785/input/french.json deleted file mode 100644 index 4ff6d3de2..000000000 --- a/crates/stelae/tests/data/rfc8785/input/french.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "peach": "This sorting order", - "péché": "is wrong according to French", - "pêche": "but canonicalization MUST", - "sin": "ignore locale" -} diff --git a/crates/stelae/tests/data/rfc8785/input/structures.json b/crates/stelae/tests/data/rfc8785/input/structures.json deleted file mode 100644 index eb71efb84..000000000 --- a/crates/stelae/tests/data/rfc8785/input/structures.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "1": {"f": {"f": "hi","F": 5} ,"\n": 56.0}, - "10": { }, - "": "empty", - "a": { }, - "111": [ {"e": "yes","E": "no" } ], - "A": { } -} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/input/unicode.json b/crates/stelae/tests/data/rfc8785/input/unicode.json deleted file mode 100644 index 4b5bc7699..000000000 --- a/crates/stelae/tests/data/rfc8785/input/unicode.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "Unnormalized Unicode":"A\u030a" -} diff --git a/crates/stelae/tests/data/rfc8785/input/values.json b/crates/stelae/tests/data/rfc8785/input/values.json deleted file mode 100644 index f7712c2fb..000000000 --- a/crates/stelae/tests/data/rfc8785/input/values.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "numbers": [333333333.33333329, 1E30, 4.50, 2e-3, 0.000000000000000000000000001], - "string": "\u20ac$\u000F\u000aA'\u0042\u0022\u005c\\\"\/", - "literals": [null, true, false] -} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/input/weird.json b/crates/stelae/tests/data/rfc8785/input/weird.json deleted file mode 100644 index 53fabe67b..000000000 --- a/crates/stelae/tests/data/rfc8785/input/weird.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "\u20ac": "Euro Sign", - "\r": "Carriage Return", - "\u000a": "Newline", - "1": "One", - "\u0080": "Control\u007f", - "\ud83d\ude02": "Smiley", - "\u00f6": "Latin Small Letter O With Diaeresis", - "\ufb33": "Hebrew Letter Dalet With Dagesh", - "": "Browser Challenge" -} diff --git a/crates/stelae/tests/data/rfc8785/output/arrays.json b/crates/stelae/tests/data/rfc8785/output/arrays.json deleted file mode 100644 index 5efb93db7..000000000 --- a/crates/stelae/tests/data/rfc8785/output/arrays.json +++ /dev/null @@ -1 +0,0 @@ -[56,{"1":[],"10":null,"d":true}] \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/output/french.json b/crates/stelae/tests/data/rfc8785/output/french.json deleted file mode 100644 index 2e15cd1b2..000000000 --- a/crates/stelae/tests/data/rfc8785/output/french.json +++ /dev/null @@ -1 +0,0 @@ -{"peach":"This sorting order","péché":"is wrong according to French","pêche":"but canonicalization MUST","sin":"ignore locale"} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/output/structures.json b/crates/stelae/tests/data/rfc8785/output/structures.json deleted file mode 100644 index dc21e2424..000000000 --- a/crates/stelae/tests/data/rfc8785/output/structures.json +++ /dev/null @@ -1 +0,0 @@ -{"":"empty","1":{"\n":56,"f":{"F":5,"f":"hi"}},"10":{},"111":[{"E":"no","e":"yes"}],"A":{},"a":{}} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/output/unicode.json b/crates/stelae/tests/data/rfc8785/output/unicode.json deleted file mode 100644 index ee60fd121..000000000 --- a/crates/stelae/tests/data/rfc8785/output/unicode.json +++ /dev/null @@ -1 +0,0 @@ -{"Unnormalized Unicode":"Å"} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/output/values.json b/crates/stelae/tests/data/rfc8785/output/values.json deleted file mode 100644 index 29b720b6e..000000000 --- a/crates/stelae/tests/data/rfc8785/output/values.json +++ /dev/null @@ -1 +0,0 @@ -{"literals":[null,true,false],"numbers":[333333333.3333333,1e+30,4.5,0.002,1e-27],"string":"€$\u000f\nA'B\"\\\\\"/"} \ No newline at end of file diff --git a/crates/stelae/tests/data/rfc8785/output/weird.json b/crates/stelae/tests/data/rfc8785/output/weird.json deleted file mode 100644 index 62c83a333..000000000 --- a/crates/stelae/tests/data/rfc8785/output/weird.json +++ /dev/null @@ -1 +0,0 @@ -{"\n":"Newline","\r":"Carriage Return","1":"One","":"Browser Challenge","€":"Control","ö":"Latin Small Letter O With Diaeresis","€":"Euro Sign","😂":"Smiley","דּ":"Hebrew Letter Dalet With Dagesh"} \ No newline at end of file diff --git a/crates/stelae/tests/interrupted.rs b/crates/stelae/tests/interrupted.rs deleted file mode 100644 index 842b115d3..000000000 --- a/crates/stelae/tests/interrupted.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! A read that was interrupted is not a read that failed. -//! -//! Why the crate retries `ErrorKind::Interrupted` at all is stated once, on -//! `digest::read_uninterrupted`; what these tests add is that the three loops -//! routed through it actually hold the property under interruption. -//! -//! Every test below reads through [`Interrupting`], which raises `Interrupted` -//! before *every* successful read and then hands over a few bytes. -//! Interruptions therefore land between records and inside them, and the -//! results — records, digests, sizes, content — must be indistinguishable from -//! a read of the same bytes that was never interrupted. - -use std::io::{self, Read, Write as _}; - -use stelae::{ - digest::{digest_reader, read_blob, scan_blob}, - frame::{encode, Limits, RecordReader, SeqWriter}, - LayerWriter, -}; - -/// A source that returns `ErrorKind::Interrupted` before every successful read, -/// and yields at most `chunk` bytes when it does succeed. -/// -/// `chunk` is deliberately not a multiple of the record size: a source that -/// only ever stopped on a record boundary would leave the harder half of the -/// property — an interruption in the middle of a record the reader is still -/// assembling — untested. -struct Interrupting<'a> { - remaining: &'a [u8], - chunk: usize, - interrupt_next: bool, - interruptions: usize, -} - -impl<'a> Interrupting<'a> { - fn new(bytes: &'a [u8], chunk: usize) -> Self { - Self { - remaining: bytes, - chunk, - interrupt_next: true, - interruptions: 0, - } - } -} - -impl Read for Interrupting<'_> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - if self.interrupt_next { - self.interrupt_next = false; - self.interruptions += 1; - return Err(io::Error::new(io::ErrorKind::Interrupted, "signal")); - } - - self.interrupt_next = true; - - let take = self.chunk.min(buf.len()).min(self.remaining.len()); - buf[..take].copy_from_slice(&self.remaining[..take]); - self.remaining = &self.remaining[take..]; - - Ok(take) - } -} - -const RECORDS: u64 = 256; - -/// Not a divisor of any record's encoded length, so the cut points walk across -/// record boundaries rather than lining up with them. -const CHUNK: usize = 7; - -fn record(i: u64) -> stelae::CanonicalCbor { - encode(|e| { - e.array(2)? - .u64(i)? - .str("a repeated payload that compresses")?; - Ok(()) - }) - .unwrap() -} - -fn sequence() -> Vec { - let mut writer = SeqWriter::new(Vec::new()); - - for i in 0..RECORDS { - writer.write_record(&record(i)).unwrap(); - } - - writer.into_inner() -} - -fn layer_blob(content: &[u8]) -> Vec { - let mut writer = LayerWriter::new(Vec::new(), 9).unwrap(); - writer.write_all(content).unwrap(); - writer.finish().unwrap().0 -} - -/// `RecordReader::fill` refills a bounded window from the stream. An -/// interruption there used to end the walk with an `Io` error partway through a -/// layer; the whole sequence has to come out instead, in order, once. -#[test] -fn a_record_reader_walks_an_interrupted_stream_to_the_end() { - let bytes = sequence(); - - let mut source = Interrupting::new(&bytes, CHUNK); - let mut reader = RecordReader::with_limits( - &mut source, - Limits { - // Small enough that the window refills many times over the - // sequence, so the retry is exercised on every kind of boundary. - window: 128, - ..Default::default() - }, - ); - - let mut seen = Vec::new(); - while let Some(next) = reader.next_record() { - seen.push(next.unwrap().to_vec()); - } - - assert!(!reader.failed()); - assert_eq!(reader.count(), RECORDS); - assert_eq!( - seen, - (0..RECORDS) - .map(|i| record(i).as_ref().to_vec()) - .collect::>() - ); - assert!( - source.interruptions > RECORDS as usize, - "the source should have interrupted far more often than once per record, \ - got {}", - source.interruptions - ); -} - -/// `digest_reader` is what checks a stored blob against the digest it is named -/// by. An interruption must not turn a good blob into a failed verification. -#[test] -fn digest_reader_hashes_every_byte_of_an_interrupted_stream() { - let blob = layer_blob(&sequence()); - let expected = digest_reader(blob.as_slice()).unwrap(); - - let interrupted = digest_reader(Interrupting::new(&blob, CHUNK)).unwrap(); - - assert_eq!(interrupted, expected); - assert_eq!(interrupted.1, blob.len() as u64); -} - -/// Both blob readers run the decompressor over the stream, so the interruption -/// surfaces underneath zstd rather than at the top of the loop. Digests, sizes -/// and the uncompressed content all have to match the uninterrupted read. -#[test] -fn a_blob_reads_back_identically_through_interruptions() { - let content = sequence(); - let blob = layer_blob(&content); - let expected = scan_blob(blob.as_slice()).unwrap(); - - let scanned = scan_blob(Interrupting::new(&blob, CHUNK)).unwrap(); - assert_eq!(scanned, expected); - - let (read, digests) = - read_blob(Interrupting::new(&blob, CHUNK), expected.uncompressed_size).unwrap(); - assert_eq!(read, content); - assert_eq!(digests, expected); -} - -/// The retry is for `Interrupted` and nothing else: a source that fails for a -/// real reason still fails, and it fails at the loop that read it. -#[test] -fn a_real_io_error_still_ends_the_read() { - struct Broken; - - impl Read for Broken { - fn read(&mut self, _buf: &mut [u8]) -> io::Result { - Err(io::Error::new(io::ErrorKind::ConnectionReset, "peer went")) - } - } - - assert!(digest_reader(Broken).is_err()); - assert!(scan_blob(Broken).is_err()); - - let mut reader = RecordReader::new(Broken); - assert!(reader.next_record().unwrap().is_err()); -} diff --git a/crates/stelae/tests/memory.rs b/crates/stelae/tests/memory.rs deleted file mode 100644 index 823a5d6cd..000000000 --- a/crates/stelae/tests/memory.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Peak memory during a layer read is a property, not an aspiration. -//! -//! The protocol's read path used to hold a whole uncompressed layer: fine for a -//! fixture, fatal at the sizes a profile publishes (ADR-004's worked example -//! gives a state shard of 402,653,184 bytes). These tests are what stops that -//! from coming back. They instrument the global allocator with `stats_alloc` — -//! the idiom the root package's `tests/memory.rs` uses for store iteration — -//! and read a layer far larger than the reader's window while watching what the -//! process asks for. -//! -//! `bytes_allocated` is cumulative over the region, which is a *stronger* -//! statement than peak: a run that never allocates more than N bytes in total -//! certainly never holds more than N at once. It also means a reader that -//! allocated and freed one record per iteration would be caught here, not -//! excused by its tidiness. -//! -//! ## The write path is measured differently, and has to be -//! -//! Cumulative is unavailable on the write side. A producer handing 32,768 -//! records to a layer has to *encode* 32,768 records, and those allocations are -//! its own — incurred identically whether it buffers them or streams them, so a -//! cumulative figure reports the same ~33 MB for both paths and distinguishes -//! nothing. What separates them is what is *held*: bytes allocated inside the -//! region and not yet given back, sampled at every record and maximized. That -//! is peak rather than a bound on peak, which is the weaker of the two claims — -//! stated here rather than quietly substituted. -//! -//! ## Why these tests take a lock -//! -//! A `Region` reads a *process-wide* counter, and the test harness runs tests -//! in parallel by default — so a second test's setup, which is tens of -//! megabytes here, lands inside the first test's measurement and swamps a -//! 64 KiB budget. It is a race, so it passes on the machine you tried it on and -//! fails on the one you did not (this one surfaced on Windows CI). Every test -//! in this file therefore holds [`SERIAL`] for its whole body, setup included. - -use std::{ - alloc::System, - sync::{Mutex, MutexGuard}, -}; - -use serde_json::json; -use stats_alloc::{Region, StatsAlloc, INSTRUMENTED_SYSTEM}; - -use stelae::{ - dir::{LayerSpec, SteleDir}, - frame::{encode, CanonicalCbor, Limits, RecordReader, SeqWriter}, - Compression, Error, Inscription, Profile, RecordSink, SteleReader, SteleWriter, -}; - -#[global_allocator] -static GLOBAL: &StatsAlloc = &INSTRUMENTED_SYSTEM; - -/// Held by every test in this file, so that only one is allocating at a time. -/// A new test that forgets to take it will not fail here — it will make some -/// *other* test flaky, which is the failure worth spending a comment on. -static SERIAL: Mutex<()> = Mutex::new(()); - -fn exclusive() -> MutexGuard<'static, ()> { - // A poisoned lock means another test in this file panicked. That is its - // failure to report; this one still wants an uncontended allocator. - SERIAL - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) -} - -const PROFILE_NAME: &str = "dev.example.bulk"; -const COMPRESSION_LEVEL: i32 = 3; - -/// Records of roughly a kilobyte, which is the order of a Dolos `indexes` or -/// `state` record. Blocks are larger, but the point of the bound is that the -/// layer's size does not enter it. -const RECORD_BODY: usize = 1000; - -/// ~33 MB of layer against a 64 KiB window: 500 windows, so a reader that -/// buffered even a percent of the layer would show up. -const RECORDS: u64 = 32 * 1024; - -/// How many windows the layer must span for these tests to be worth running. -const MINIMUM_WINDOWS: usize = 400; - -/// What the streaming path is allowed to allocate, cumulatively, to read all of -/// it. Two orders of magnitude below the layer, and it does not move when the -/// layer does — which is the property under test. -/// -/// Above the 64 KiB window because the zstd bindings buffer on either side of -/// the decoder. What this does *not* cover is libzstd's own context: that is -/// `malloc`ed inside the C library and never reaches a Rust global allocator, -/// so `stats_alloc` cannot see it. It is bounded by the frame's window size — -/// a property of the compression parameters, not of the layer — so it does not -/// weaken the claim, but the number below is the Rust side only and saying so -/// is cheaper than someone rediscovering it. -const STREAMING_BUDGET: usize = 1024 * 1024; - -/// What the streaming *write* path is allowed to hold at any one moment: one -/// record, the compressor's buffers, and the sink's own handful of fields. -/// -/// The same figure as the read budget, for the same reasons — the zstd bindings -/// buffer on either side of the encoder, and libzstd's own context is -/// `malloc`ed inside the C library where `stats_alloc` cannot see it. The -/// observed peak is around 34 KB against a 33 MB layer, so this is a ceiling -/// with room under it rather than a number tuned until the test passed; what -/// matters is that it does not move when the layer does. -const SINK_BUDGET: usize = 1024 * 1024; - -/// Bytes allocated inside `region` and not yet returned. -/// -/// Saturating because a region can also *free* memory that was allocated before -/// it began, which is a negative change and not a measurement of anything. -fn in_flight(region: &Region<'_, System>) -> usize { - let change = region.change(); - change - .bytes_allocated - .saturating_sub(change.bytes_deallocated) -} - -struct BulkProfile; - -impl Profile for BulkProfile { - fn name(&self) -> &str { - PROFILE_NAME - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["bulk"] - } - - fn layer_media_type(&self, kind: &str) -> Result { - Ok(format!("application/vnd.example.stele.{kind}.v1+zstd")) - } - - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("bulk-{sequence}")) - } -} - -fn bulk_record(i: u64) -> CanonicalCbor { - // Not a constant body: identical records would compress to nothing and let - // a decoder cheat its way through the test. - let body: Vec = (0..RECORD_BODY).map(|b| (b as u64 ^ i) as u8).collect(); - - encode(|e| { - e.array(2)?.u64(i)?.bytes(&body)?; - Ok(()) - }) - .unwrap() -} - -fn scope() -> CanonicalCbor { - encode(|e| { - e.u64(0)?; - Ok(()) - }) - .unwrap() -} - -/// The raw uncompressed sequence, as it appears inside a layer blob. -fn sequence() -> Vec { - let mut writer = SeqWriter::new(Vec::new()); - - for i in 0..RECORDS { - writer.write_record(&bulk_record(i)).unwrap(); - } - - writer.into_inner() -} - -/// The framing reader on its own: no compression, no digests, nothing but the -/// refill loop. The bound here is tight enough to name — one window, plus the -/// small change of the walk. -#[test] -fn framing_a_large_sequence_costs_one_window() { - let _serial = exclusive(); - - let sequence = sequence(); - - let window = 64 * 1024; - let budget = 2 * window; - - assert!( - sequence.len() > MINIMUM_WINDOWS * window, - "the layer has to dwarf the window for this to prove anything" - ); - - let region = Region::new(GLOBAL); - - let mut reader = RecordReader::with_limits( - std::io::Cursor::new(sequence.as_slice()), - Limits { - window, - ..Limits::default() - }, - ); - - let mut count = 0u64; - while let Some(record) = reader.next_record() { - // Touch the record so nothing about this loop is optimized away, but - // never keep it: holding records is the caller's choice to make, and - // this caller declines. - assert!(!record.unwrap().is_empty()); - count += 1; - } - - let allocated = region.change().bytes_allocated; - - assert_eq!(count, RECORDS); - assert!( - allocated < budget, - "framing {} bytes should cost one {window}-byte window, not {allocated} bytes", - sequence.len(), - ); -} - -/// The whole path, as a restore would use it: a compressed blob on disk, -/// verified against its descriptor, records walked without ever holding the -/// layer. -#[test] -fn streaming_a_layer_does_not_scale_with_its_size() { - let _serial = exclusive(); - - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - - let records: Vec = (0..RECORDS).map(bulk_record).collect(); - - let written = stele - .write_layer( - &BulkProfile, - &LayerSpec::new("bulk", scope(), json!({})), - COMPRESSION_LEVEL, - &records, - ) - .unwrap(); - - let mut inscription = Inscription::new( - &BulkProfile, - 0, - json!({}), - json!({}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - inscription.layers = vec![written.descriptor.clone()]; - stele.seal(&BulkProfile, &inscription).unwrap(); - - let descriptor = &inscription.layers[0]; - assert!( - descriptor.uncompressed_size > (MINIMUM_WINDOWS * stelae::frame::DEFAULT_WINDOW) as u64, - "the layer has to dwarf the window for this to prove anything" - ); - - // Everything before the region is setup: building the index is a scan of - // the stele, and what it costs is not what is under test. - drop(records); - let index = stele.blob_index().unwrap(); - - let region = Region::new(GLOBAL); - - let mut reader = stele - .stream_layer(&index, &BulkProfile, descriptor, Limits::default()) - .unwrap(); - - let mut count = 1u64; // the header record, already consumed - while let Some(record) = reader.next_record() { - assert!(!record.unwrap().is_empty()); - count += 1; - } - - let digests = reader.finish().unwrap(); - - let allocated = region.change().bytes_allocated; - - assert_eq!(count, descriptor.records); - assert_eq!(digests.diff_id, descriptor.diff_id); - assert!( - allocated < STREAMING_BUDGET, - "streaming a {}-byte layer allocated {allocated} bytes; \ - the budget is {STREAMING_BUDGET}", - descriptor.uncompressed_size, - ); - - // The control. Without it this test proves only that some number is small: - // the buffered path reads the same blob and, by design, pays the layer's - // size for it. If that ever stops being true the budget above has stopped - // measuring the difference between the two paths. - let region = Region::new(GLOBAL); - let layer = stele.read_layer(&index, &BulkProfile, descriptor).unwrap(); - let buffered = region.change().bytes_allocated; - - assert_eq!(layer.as_bytes().len() as u64, descriptor.uncompressed_size); - assert!( - buffered as u64 > descriptor.uncompressed_size, - "the buffered path allocated {buffered} bytes for a {}-byte layer; \ - it is supposed to hold the whole thing", - descriptor.uncompressed_size, - ); -} - -/// The same property on the way in: a producer streams a layer it could not -/// hold. -/// -/// This is the bound the Dolos export needs. `write_layer` takes an iterator of -/// *references*, so every record has to be materialized somewhere that outlives -/// the call — fine for a chapter of notes, impossible for a mainnet epoch of -/// blocks at 0.5–1.5 GB. A sink takes each record by reference for the length -/// of one call and keeps nothing. -/// -/// Both paths write the same layer here, so the comparison is between two ways -/// of producing one artifact and not between two artifacts. They go into -/// separate steles: the blob is named by its own digest, so writing it twice -/// into one directory would be a rename onto a file that is already there. -#[test] -fn writing_a_layer_through_a_sink_does_not_scale_with_its_size() { - let _serial = exclusive(); - - let streamed_dir = tempfile::tempdir().unwrap(); - let buffered_dir = tempfile::tempdir().unwrap(); - - let streamed_stele = SteleDir::create(streamed_dir.path()).unwrap(); - let buffered_stele = SteleDir::create(buffered_dir.path()).unwrap(); - - let spec = LayerSpec::new("bulk", scope(), json!({})); - - // The sink. Each record is encoded, written and dropped; the peak is - // sampled with the record still in hand, so what it reports is one record - // plus whatever the protocol is holding on its behalf. - let region = Region::new(GLOBAL); - - let mut sink = streamed_stele - .layer_sink(&BulkProfile, &spec, COMPRESSION_LEVEL) - .unwrap(); - - let mut held = 0usize; - - for i in 0..RECORDS { - let record = bulk_record(i); - sink.write_record(&record).unwrap(); - held = held.max(in_flight(®ion)); - } - - let streamed = sink.finish().unwrap(); - let streamed_held = held.max(in_flight(®ion)); - - // The control. Without it this test proves only that some number is small: - // the buffered path writes the same records and, by construction, has to - // hold every one of them until the call returns. - let region = Region::new(GLOBAL); - - let records: Vec = (0..RECORDS).map(bulk_record).collect(); - let buffered = buffered_stele - .write_layer(&BulkProfile, &spec, COMPRESSION_LEVEL, &records) - .unwrap(); - - let buffered_held = in_flight(®ion); - drop(records); - - let size = streamed.descriptor.uncompressed_size; - - assert!( - size > (16 * SINK_BUDGET) as u64, - "the layer has to dwarf the budget for this to prove anything: \ - {size} bytes against {SINK_BUDGET}" - ); - - // One artifact, two ways of writing it. - assert_eq!(streamed.descriptor, buffered.descriptor); - assert_eq!(streamed.digests, buffered.digests); - - assert!( - streamed_held < SINK_BUDGET, - "streaming a {size}-byte layer held {streamed_held} bytes at peak; \ - the budget is {SINK_BUDGET}", - ); - - assert!( - buffered_held as u64 > size, - "the buffered path held {buffered_held} bytes for a {size}-byte layer; \ - it is supposed to hold the whole thing", - ); -} diff --git a/crates/stelae/tests/oci.rs b/crates/stelae/tests/oci.rs deleted file mode 100644 index fe766d507..000000000 --- a/crates/stelae/tests/oci.rs +++ /dev/null @@ -1,2604 +0,0 @@ -//! A stele in a registry, and the delta transfer that is the reason for it. -//! -//! Two kinds of test live here and they have opposite requirements, which is -//! why they share a file rather than a run: -//! -//! - [`the_manifest_shape_is_frozen`] and the refusals beside it need **no -//! network and no compressor**. They build a manifest from fixed digests and -//! compare the exact bytes, so a change to the manifest's shape is a -//! deliberate re-pin in the same commit and never a silent drift. They run in -//! CI, under `--all-features`, like any other test. -//! - everything marked `#[ignore]` needs a **registry**, and spawns one: -//! `docker run` of an OCI Distribution server, torn down on the way out. They -//! are the ones that prove a stele survives the round trip, that the second -//! push moves only what the registry lacks, and that neither direction holds -//! a layer. -//! -//! Run the second kind with: -//! -//! ```text -//! cargo test -p stelae --all-features --test oci -- --ignored --nocapture -//! ``` -//! -//! `STELAE_TEST_REGISTRY_IMAGE` chooses the server (default `registry:2`), so -//! the same suite can be pointed at another implementation — which is the only -//! way to find out whether a given registry accepts an OCI 1.1 `artifactType`. -//! -//! ## The fixture demands credentials -//! -//! Every registry this suite spawns is behind htpasswd, and every transport it -//! opens carries the pair. That is not incidental hardening: the registry this -//! transport is aimed at authenticates every request — access to a stele -//! repository is free and identity-less, and still credentialed — so a suite -//! that only ever spoke to an anonymous server would prove the round trip -//! against a server unlike the one it runs against. -//! -//! [`credentials_are_required`] is what keeps that honest. A server the fixture -//! does not know how to configure would run anonymous and every other test here -//! would pass regardless; that one fails instead, and says so. -//! -//! ## Running them over TLS -//! -//! The fixture speaks plaintext by default, which is enough for everything -//! about the *protocol* and evidence for nothing about the transport's crypto. -//! Set both of -//! -//! ```text -//! STELAE_TEST_REGISTRY_TLS_CERT=/abs/path/server.pem -//! STELAE_TEST_REGISTRY_TLS_KEY=/abs/path/server.key -//! ``` -//! -//! and the same suite runs against the same server terminating TLS, with the -//! client verifying the certificate for real. The certificate has to cover -//! `127.0.0.1` — that is where the fixture publishes — and its issuer has to be -//! trusted by the process, which on Linux means `SSL_CERT_FILE` pointing at the -//! CA. Nothing here weakens verification to make a self-signed certificate -//! work: a suite that accepted any certificate would pass just as happily with -//! the handshake broken, which is the one thing this mode exists to detect. -//! -//! ## Pointing them at a deployment -//! -//! Set -//! -//! ```text -//! STELAE_TEST_REGISTRY_URL=oci.example.com -//! STELAE_TEST_REGISTRY_USER=publisher -//! STELAE_TEST_REGISTRY_PASSWORD=… -//! ``` -//! -//! and the same suite runs against that registry instead of spawning one — -//! over TLS, no exceptions: a deployment is the one place plaintext has no -//! business. The docker knobs above are the container's and are ignored. -//! -//! A deployment persists between runs where a container never does, so every -//! fixture scopes its repositories under a fresh `staging/…` namespace — the -//! assertions written against an empty repository stay true, and a finished -//! run leaves its namespace to the deployment's own garbage collection. -//! -//! `STELAE_TEST_REGISTRY_PULL_USER` / `_PULL_PASSWORD` optionally name a -//! second pair, narrower by contract: expected to read a stele and be refused -//! a write, which is what [`the_read_only_pair_pulls_and_cannot_push`] proves. -//! The htpasswd fixture knows one pair, so that proof asks for a deployment. - -#![cfg(feature = "oci")] - -use std::{ - alloc::System, - collections::BTreeMap, - path::PathBuf, - sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Mutex, MutexGuard, - }, -}; - -use serde_json::json; -use stats_alloc::{StatsAlloc, INSTRUMENTED_SYSTEM}; - -use stelae::{ - dir::SteleDir, - frame::{encode, CanonicalCbor, Limits}, - inscription::LayerDescriptor, - oci::{ - build_manifest, manifest_bytes, read_manifest, Auth, Options, Registry, DIFF_ID_ANNOTATION, - KIND_ANNOTATION, SCOPE_ANNOTATION, - }, - progress::{Event, Observer, Progress}, - Compression, Digest, Error, HistoryEntry, Inscription, LayerDigests, LayerSpec, Profile, - RecordSink, SteleReader, SteleWriter, WrittenLayer, -}; - -#[global_allocator] -static GLOBAL: &StatsAlloc = &INSTRUMENTED_SYSTEM; - -/// Held by every test that spawns a registry. -/// -/// Two reasons, and the second is the one that would otherwise be found the -/// hard way: containers are expensive enough that starting five at once is -/// slower than starting them in turn, and the peak-allocation test reads a -/// *process-wide* counter, which cannot tell one test's allocations from -/// another's. -static SERIAL: Mutex<()> = Mutex::new(()); - -fn exclusive() -> MutexGuard<'static, ()> { - SERIAL - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) -} - -const PROFILE_NAME: &str = "dev.example.toy"; -const NOTES_MEDIA_TYPE: &str = "application/vnd.example.stele.notes.v1+zstd"; -const INDEX_MEDIA_TYPE: &str = "application/vnd.example.stele.index.v1+zstd"; -const COMPRESSION_LEVEL: i32 = 9; - -/// The same profile the directory tests use, and for the same reason: if the -/// transport had absorbed an assumption from Dolos, a vendor publishing -/// chapters of notes would not be able to use it. -struct ToyProfile; - -impl Profile for ToyProfile { - fn name(&self) -> &str { - PROFILE_NAME - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["notes", "index"] - } - - fn layer_media_type(&self, kind: &str) -> Result { - match kind { - "notes" => Ok(NOTES_MEDIA_TYPE.to_owned()), - "index" => Ok(INDEX_MEDIA_TYPE.to_owned()), - other => Err(Error::UnknownLayerKind { - profile: PROFILE_NAME.to_owned(), - kind: other.to_owned(), - }), - } - } - - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("chapter-{sequence}")) - } -} - -// --------------------------------------------------------------------------- -// The manifest, frozen without a network -// --------------------------------------------------------------------------- - -fn digest_of(byte: u8) -> Digest { - Digest::from_bytes([byte; 32]) -} - -fn note_record(id: u64) -> CanonicalCbor { - encode(|e| { - e.array(2)?.u64(id)?.str("a note")?; - Ok(()) - }) - .unwrap() -} - -fn notes_scope(chapter: u64) -> (CanonicalCbor, serde_json::Value) { - let header = encode(|e| { - e.array(2)?.u64(chapter)?.str("notes")?; - Ok(()) - }) - .unwrap(); - - (header, json!({"chapter": chapter})) -} - -fn index_scope(chapter: u64) -> (CanonicalCbor, serde_json::Value) { - let header = encode(|e| { - e.array(2)?.u64(chapter)?.str("index")?; - Ok(()) - }) - .unwrap(); - - (header, json!({"chapter": chapter, "sortedBy": "title"})) -} - -/// An inscription and the layers a publisher would have written for it, both -/// entirely synthetic. -/// -/// Nothing here is compressed. That is deliberate: a manifest carries the -/// *compressed* digest and size of every layer, and zstd's output moves between -/// library versions — which is the whole reason identity is anchored on -/// uncompressed bytes. A golden over real blobs would pin the compressor, fail -/// on an unrelated upgrade, and teach whoever hit it that the golden is noise. -fn fixture() -> (Inscription, Vec) { - let mut inscription = Inscription::new( - &ToyProfile, - 3, - json!({"chapter": 3, "shelf": "east"}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.history = vec![ - HistoryEntry { - sequence: 1, - inscription_digest: digest_of(0x11), - }, - HistoryEntry { - sequence: 2, - inscription_digest: digest_of(0x22), - }, - ]; - - inscription.layers = vec![ - LayerDescriptor { - kind: "notes".to_owned(), - media_type: NOTES_MEDIA_TYPE.to_owned(), - diff_id: digest_of(0xaa), - records: 4, - uncompressed_size: 155, - scope: notes_scope(3).1, - }, - LayerDescriptor { - kind: "index".to_owned(), - media_type: INDEX_MEDIA_TYPE.to_owned(), - diff_id: digest_of(0xbb), - records: 4, - uncompressed_size: 77, - scope: index_scope(3).1, - }, - ]; - - let layers = vec![ - written(&inscription.layers[0], digest_of(0xa1), 91, 155), - written(&inscription.layers[1], digest_of(0xb1), 60, 77), - ]; - - (inscription, layers) -} - -fn written( - descriptor: &LayerDescriptor, - blob_digest: Digest, - compressed_size: u64, - uncompressed_size: u64, -) -> WrittenLayer { - WrittenLayer { - descriptor: descriptor.clone(), - digests: LayerDigests { - diff_id: descriptor.diff_id, - blob_digest, - uncompressed_size, - compressed_size, - }, - } -} - -/// Done criterion 3: the manifest's shape is a re-pin, never a drift. -/// -/// Every byte below is determined by the specification — the artifact type, the -/// config media type, the profile's own layer media types, the annotation keys -/// and the canonical JSON encoding. If any of them moves, this string moves -/// with it, in the same commit. -#[test] -fn the_manifest_shape_is_frozen() { - let (inscription, layers) = fixture(); - - let (manifest, config) = build_manifest(&inscription, &layers).unwrap(); - - // The config blob is the canonical inscription, and the descriptor names it - // by its own digest — the one place identity and transport meet. - assert_eq!(config, inscription.canonicalize().unwrap()); - assert_eq!( - manifest.config.digest, - inscription.digest().unwrap().to_string() - ); - assert_eq!(manifest.config.size as usize, config.len()); - - let body = manifest_bytes(&manifest).unwrap(); - - assert_eq!( - String::from_utf8(body).unwrap(), - concat!( - r#"{"artifactType":"application/vnd.stelae.stele.v1","#, - r#""config":{"digest":"sha256:3eff2efc90e091c19097c2b7c33e6d5270bdcbc11306c7a0ce9265d0d5601cc4","#, - r#""mediaType":"application/vnd.stelae.inscription.v1+json","size":873},"#, - r#""layers":[{"annotations":{"#, - r#""store.stelae.layer.diffId":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","#, - r#""store.stelae.layer.kind":"notes","#, - r#""store.stelae.layer.scope":"{\"chapter\":3}"},"#, - r#""digest":"sha256:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","#, - r#""mediaType":"application/vnd.example.stele.notes.v1+zstd","size":91},"#, - r#"{"annotations":{"#, - r#""store.stelae.layer.diffId":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","#, - r#""store.stelae.layer.kind":"index","#, - r#""store.stelae.layer.scope":"{\"chapter\":3,\"sortedBy\":\"title\"}"},"#, - r#""digest":"sha256:b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1","#, - r#""mediaType":"application/vnd.example.stele.index.v1+zstd","size":60}],"#, - r#""mediaType":"application/vnd.oci.image.manifest.v1+json","schemaVersion":2}"#, - ), - "the manifest shape drifted" - ); -} - -/// The map a pull reads off the manifest is the map a directory rebuilds by -/// decompressing everything, and this is where the two are held to be the same -/// thing. -#[test] -fn a_manifest_yields_the_identity_to_blob_map() { - let (inscription, layers) = fixture(); - let (manifest, _) = build_manifest(&inscription, &layers).unwrap(); - - let blobs = read_manifest(&manifest, &inscription).unwrap(); - - assert_eq!(blobs.len(), 2); - assert_eq!(blobs.blob_for(&digest_of(0xaa)), Some(digest_of(0xa1))); - assert_eq!(blobs.blob_for(&digest_of(0xbb)), Some(digest_of(0xb1))); - assert_eq!(blobs.blob_for(&digest_of(0xcc)), None); -} - -/// The annotations ADR-004 asks for, checked as content rather than as bytes, -/// so a reader of this file can see what a generic OCI tool would find. -#[test] -fn every_layer_is_annotated_with_its_kind_diff_id_and_scope() { - let (inscription, layers) = fixture(); - let (manifest, _) = build_manifest(&inscription, &layers).unwrap(); - - for (oci, described) in manifest.layers.iter().zip(&inscription.layers) { - let annotations = oci.annotations.as_ref().unwrap(); - - assert_eq!(annotations[KIND_ANNOTATION], described.kind); - assert_eq!( - annotations[DIFF_ID_ANNOTATION], - described.diff_id.to_string() - ); - assert_eq!( - annotations[SCOPE_ANNOTATION], - serde_json::to_string(&described.scope).unwrap() - ); - } -} - -/// A disagreement between the two documents is a refusal, never a preference. -#[test] -fn a_manifest_that_disagrees_with_the_inscription_is_refused() { - let (inscription, layers) = fixture(); - - // A layer the inscription describes and nobody wrote. - let err = build_manifest(&inscription, &layers[..1]).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("never written")), - "{err:?}" - ); - - // A layer that was written and the inscription does not describe: a blob - // nothing attests. - let mut orphaned = layers.clone(); - orphaned.push(written(&layers[0].descriptor, digest_of(0xc1), 10, 20)); - let err = build_manifest(&inscription, &orphaned).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("does not describe")), - "{err:?}" - ); - - let (manifest, _) = build_manifest(&inscription, &layers).unwrap(); - - // Fewer layers than the document describes. - let mut short = manifest.clone(); - short.layers.pop(); - let err = read_manifest(&short, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("layer(s)")), - "{err:?}" - ); - - // The right blobs in the wrong order. The `diffId` annotations still map - // every layer to a real blob, so only the positional check catches this. - let mut reordered = manifest.clone(); - reordered.layers.reverse(); - let err = read_manifest(&reordered, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("annotated")), - "{err:?}" - ); - - // A layer with no annotation says nothing about which layer it holds. - let mut unannotated = manifest.clone(); - unannotated.layers[0].annotations = None; - let err = read_manifest(&unannotated, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains(DIFF_ID_ANNOTATION)), - "{err:?}" - ); - - // A media type the inscription does not claim. - let mut mistyped = manifest.clone(); - mistyped.layers[0].media_type = INDEX_MEDIA_TYPE.to_owned(); - let err = read_manifest(&mistyped, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("in the manifest")), - "{err:?}" - ); -} - -/// The envelope is checked before anything inside it is trusted, and a missing -/// `artifactType` fails closed. -#[test] -fn a_manifest_that_is_not_a_steles_is_refused() { - let (inscription, layers) = fixture(); - let (manifest, _) = build_manifest(&inscription, &layers).unwrap(); - - let mut stripped = manifest.clone(); - stripped.artifact_type = None; - let err = read_manifest(&stripped, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("no artifactType")), - "{err:?}" - ); - - let mut foreign = manifest.clone(); - foreign.artifact_type = Some("application/vnd.acme.thing.v1".to_owned()); - let err = read_manifest(&foreign, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("artifactType is")), - "{err:?}" - ); - - let mut wrong_config = manifest; - wrong_config.config.media_type = "application/vnd.oci.image.config.v1+json".to_owned(); - let err = read_manifest(&wrong_config, &inscription).unwrap_err(); - assert!( - matches!(&err, Error::ManifestMismatch(m) if m.contains("config blob is")), - "{err:?}" - ); -} - -/// What the 4 MiB ceiling refuses, stated as a test rather than as a comment. -/// -/// A descriptor and its three annotations run to roughly 350 bytes, so a -/// manifest reaches the ceiling somewhere around twelve thousand layers — -/// nearly seven times a mainnet stele's ~1,816 (ADR-004's ~600 epochs × three -/// per-epoch kinds, plus sixteen state shards). Nothing is expected to meet it. -/// The point is that when something does, the refusal names the document and -/// the layer count instead of arriving as a registry's `413`. -#[test] -fn a_manifest_past_the_size_ceiling_is_refused() { - /// Layers of a mainnet stele, by ADR-004's own sizing. - const MAINNET: usize = 600 * 3 + 16; - /// Comfortably past the ceiling: nothing here depends on where exactly it - /// falls, only that it is far above anything a profile would publish. - const TOO_MANY: usize = 16_000; - - let (template, written_template) = { - let (inscription, layers) = fixture(); - (inscription.layers[0].clone(), layers[0].clone()) - }; - - let build = |count: usize| { - let (mut inscription, _) = fixture(); - inscription.layers.clear(); - - let mut layers = Vec::with_capacity(count); - - for index in 0..count as u32 { - let mut described = template.clone(); - - // Distinct identities, so nothing collapses into one descriptor. - let mut bytes = [0u8; 32]; - bytes[..4].copy_from_slice(&index.to_be_bytes()); - described.diff_id = Digest::from_bytes(bytes); - described.scope = json!({"chapter": index}); - - let mut layer = written_template.clone(); - layer.descriptor = described.clone(); - layer.digests.diff_id = described.diff_id; - - inscription.layers.push(described); - layers.push(layer); - } - - build_manifest(&inscription, &layers).unwrap().0 - }; - - let err = manifest_bytes(&build(TOO_MANY)).unwrap_err(); - - assert!( - matches!(err, Error::ManifestTooLarge { layers, .. } if layers == TOO_MANY), - "{err:?}" - ); - - // A mainnet-sized stele passes it with room to spare, which is the claim - // ADR-004 sized the format against. - let body = manifest_bytes(&build(MAINNET)).unwrap(); - - println!( - "manifest: {} bytes for {MAINNET} layers ({} bytes per layer), \ - ceiling {} bytes", - body.len(), - body.len() / MAINNET, - stelae::MANIFEST_SIZE_LIMIT, - ); - - assert!( - body.len() < stelae::MANIFEST_SIZE_LIMIT / 2, - "a mainnet-sized manifest is {} bytes", - body.len(), - ); -} - -// --------------------------------------------------------------------------- -// A registry the test spawns -// --------------------------------------------------------------------------- - -/// The certificate the fixture hands the server, when the environment supplies -/// one. -/// -/// Read from the environment rather than generated here, because the half that -/// matters is the one this process cannot do to itself: the issuer has to be -/// trusted *before* the first client is built, and on Linux that is -/// `SSL_CERT_FILE`, which is read once. A test that minted a certificate would -/// have nowhere to put its CA. -struct Tls { - certificate: String, - key: String, -} - -impl Tls { - fn from_env() -> Option { - match ( - std::env::var("STELAE_TEST_REGISTRY_TLS_CERT"), - std::env::var("STELAE_TEST_REGISTRY_TLS_KEY"), - ) { - (Ok(certificate), Ok(key)) if !certificate.is_empty() && !key.is_empty() => { - Some(Self { certificate, key }) - } - (Ok(_), _) | (_, Ok(_)) => { - panic!("STELAE_TEST_REGISTRY_TLS_CERT and _KEY are set together or not at all") - } - _ => None, - } - } - - /// The `docker run` arguments that make the server terminate TLS. - /// - /// Single-file bind mounts, so the two may live in different directories - /// and neither directory is exposed whole. - fn docker_args(&self) -> Vec { - vec![ - "--volume".to_owned(), - format!("{}:/tls/certificate.pem:ro", self.certificate), - "--volume".to_owned(), - format!("{}:/tls/key.pem:ro", self.key), - "--env".to_owned(), - "REGISTRY_HTTP_TLS_CERTIFICATE=/tls/certificate.pem".to_owned(), - "--env".to_owned(), - "REGISTRY_HTTP_TLS_KEY=/tls/key.pem".to_owned(), - ] - } -} - -/// Install `ring` as the process-default crypto provider. -/// -/// `oci.rs` documents this as the caller's job — the transport is built on -/// rustls with no provider wired in — so the suite does it explicitly. Doing -/// it here rather than relying on whatever a dependency might have installed -/// is the point: if the precondition were ever dropped from the transport's -/// documentation, this line is what would still be true. -fn install_crypto_provider() { - static ONCE: std::sync::Once = std::sync::Once::new(); - - ONCE.call_once(|| { - rustls::crypto::ring::default_provider() - .install_default() - .expect("nothing else installed a provider first"); - }); -} - -/// The credentials the fixture's registry demands. -/// -/// A test credential, not a secret: it lives as long as one container. -/// [`HTPASSWD`] is the bcrypt encoding of this pair — `distribution` accepts no -/// other hash algorithm in an htpasswd file — so the two move together or not -/// at all. -const USER: &str = "stelae"; -const PASSWORD: &str = "stelae-fixture"; - -const HTPASSWD: &str = "stelae:$2y$05$1Hb22zONvzLAj4WaYl34/uDWF5rDgQkS9MoewgRvsTlsNrusMYTW6\n"; - -/// zot's whole configuration, which is a file or nothing: it has no environment -/// equivalent, and the image's own default carries no auth. -const ZOT_CONFIG: &str = r#"{ - "distSpecVersion": "1.1.1", - "storage": { "rootDirectory": "/var/lib/registry" }, - "http": { - "address": "0.0.0.0", - "port": "5000", - "auth": { "htpasswd": { "path": "/auth/htpasswd" } } - }, - "log": { "level": "warn" } -} -"#; - -/// The registry under test: a container this suite spawned, or a deployment -/// the environment pointed it at. -/// -/// `docker` rather than a library when it spawns one: the point of these tests -/// is that the client talks to a *real* registry, and a fake one written here -/// would only ever agree with this implementation's reading of the -/// specification. The remote arm is the same conviction carried further — the -/// registry the transport is actually aimed at, reached the way an operator -/// reaches it. -struct Fixture { - server: Server, - tls: bool, -} - -/// Where [`Fixture`]'s registry lives. -enum Server { - /// A container running an OCI Distribution server, removed when the - /// fixture is dropped. - Container { - container: String, - port: u16, - /// The htpasswd file and the configuration naming it, held so they - /// outlive the container that has them mounted. - _auth: tempfile::TempDir, - }, - /// A deployed registry, reached over TLS and never torn down. - Remote { - host: String, - push_user: String, - push_password: String, - pull: Option<(String, String)>, - /// This fixture's private corner of a registry that outlives it: a - /// repository prefix no other run writes to. - namespace: String, - }, -} - -impl Fixture { - fn spawn() -> Self { - install_crypto_provider(); - - if let Some(fixture) = Self::remote() { - fixture.wait_until_ready(); - - eprintln!( - "registry: deployment on {}, TLS, basic auth as {:?}", - fixture.address(), - fixture.push_user(), - ); - - return fixture; - } - - let image = - std::env::var("STELAE_TEST_REGISTRY_IMAGE").unwrap_or_else(|_| "registry:2".to_owned()); - - let tls = Tls::from_env(); - let auth = auth_dir(); - - let mut args: Vec = ["run", "--detach", "--rm", "--publish", "127.0.0.1::5000"] - .iter() - .map(|arg| (*arg).to_owned()) - .collect(); - - if let Some(tls) = &tls { - args.extend(tls.docker_args()); - } - - args.extend(auth_args(auth.path())); - args.push(image.clone()); - - let run = std::process::Command::new("docker") - .args(&args) - .output() - .expect("docker is required to run the registry tests"); - - assert!( - run.status.success(), - "docker run {image}: {}", - String::from_utf8_lossy(&run.stderr) - ); - - let container = String::from_utf8(run.stdout).unwrap().trim().to_owned(); - - let ports = std::process::Command::new("docker") - .args(["port", &container, "5000/tcp"]) - .output() - .expect("docker port"); - - let mapped = String::from_utf8(ports.stdout).unwrap(); - let port = mapped - .lines() - .find_map(|line| line.rsplit(':').next()) - .and_then(|port| port.trim().parse::().ok()) - .unwrap_or_else(|| panic!("no published port in {mapped:?}")); - - let fixture = Self { - server: Server::Container { - container, - port, - _auth: auth, - }, - tls: tls.is_some(), - }; - - fixture.wait_until_ready(); - - eprintln!( - "registry: {image} on {}, {}, basic auth as {USER:?}", - fixture.address(), - if fixture.tls { "TLS" } else { "plaintext" } - ); - - fixture - } - - /// The deployment the environment names, if it names one. - /// - /// A fresh namespace per fixture, because a deployment persists where a - /// container never does: two fixtures in one test are two namespaces — - /// which is what lets - /// [`a_layer_whose_blob_is_not_there_cannot_be_carried_forward`] keep - /// meaning "a place the blob is absent from" — and two runs never share - /// one. - fn remote() -> Option { - let host = std::env::var("STELAE_TEST_REGISTRY_URL").ok()?; - - let push_user = std::env::var("STELAE_TEST_REGISTRY_USER") - .expect("STELAE_TEST_REGISTRY_URL is set, so _USER is too"); - let push_password = std::env::var("STELAE_TEST_REGISTRY_PASSWORD") - .expect("STELAE_TEST_REGISTRY_URL is set, so _PASSWORD is too"); - - let pull = match ( - std::env::var("STELAE_TEST_REGISTRY_PULL_USER"), - std::env::var("STELAE_TEST_REGISTRY_PULL_PASSWORD"), - ) { - (Ok(user), Ok(password)) => Some((user, password)), - (Ok(_), _) | (_, Ok(_)) => panic!( - "STELAE_TEST_REGISTRY_PULL_USER and _PULL_PASSWORD are set \ - together or not at all" - ), - _ => None, - }; - - static FIXTURES: AtomicUsize = AtomicUsize::new(0); - - let run = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - let namespace = format!( - "staging/{run:x}-{}", - FIXTURES.fetch_add(1, Ordering::Relaxed) - ); - - Some(Self { - server: Server::Remote { - host, - push_user, - push_password, - pull, - namespace, - }, - tls: true, - }) - } - - fn address(&self) -> String { - match &self.server { - Server::Container { port, .. } => format!("127.0.0.1:{port}"), - Server::Remote { host, .. } => host.clone(), - } - } - - /// Wait until the server answers a real request. - /// - /// A connect is *not* the readiness signal, however much it looks like one: - /// Docker's port forwarder accepts on the published port from the moment - /// the container exists and only then tries to reach the process inside, so - /// a connect succeeds and the request that follows it dies as an incomplete - /// message. That failure looks exactly like a registry rejecting the - /// request, which is the wrong thing to conclude about a registry. - /// - /// So the probe is the client under test asking a repository nothing has - /// ever been written to for its latest stele. `Ok(None)` is the answer only - /// a registry that read the request can give — and under TLS it is - /// reachable only through a handshake that verified, which makes the same - /// call the readiness probe and the first assertion. - fn wait_until_ready(&self) { - for _ in 0..300 { - if self - .registry("stelae/readiness") - .latest(&ToyProfile) - .is_ok() - { - return; - } - - std::thread::sleep(std::time::Duration::from_millis(100)); - } - - let refusal = self - .registry("stelae/readiness") - .latest(&ToyProfile) - .err() - .map(|e| e.to_string()) - .unwrap_or_default(); - - panic!( - "the registry never answered on {}: {refusal}", - self.address() - ); - } - - fn registry(&self, repository: &str) -> Registry { - self.registry_staging_in(repository, None) - } - - /// The same transport, with the publish path's concurrency named. - /// - /// Only the tests whose subject is the concurrency itself set it; every - /// other test in this file runs at the default, which is the arrangement - /// the deployment uses. - fn registry_at(&self, repository: &str, concurrency: usize) -> Registry { - self.options(repository, |options| options.concurrency = concurrency) - } - - /// The same transport, re-proving every layer it carries forward. - fn verifying(&self, repository: &str) -> Registry { - self.options(repository, |options| options.verify_adopted = true) - } - - /// Every transport in this file is built here, so that whether the fixture - /// is speaking TLS — and which credentials it presents — is decided in - /// exactly one place. A test that assembled its own [`Options`] to set a - /// scratch directory would keep working against a plaintext fixture and - /// quietly send `http://` at a TLS one. - fn registry_staging_in(&self, repository: &str, scratch_dir: Option) -> Registry { - self.registry_as(repository, scratch_dir, self.credentials()) - } - - /// The pair the fixture's registry accepts writes under. - fn credentials(&self) -> Auth { - match &self.server { - Server::Container { .. } => Auth::Basic { - user: USER.to_owned(), - password: PASSWORD.to_owned(), - }, - Server::Remote { - push_user, - push_password, - .. - } => Auth::Basic { - user: push_user.clone(), - password: push_password.clone(), - }, - } - } - - /// The user [`credentials`](Self::credentials) authenticates as — what a - /// test that presents the right name with the wrong password asks for. - fn push_user(&self) -> &str { - match &self.server { - Server::Container { .. } => USER, - Server::Remote { push_user, .. } => push_user, - } - } - - /// A second pair with narrower rights, where the registry has one. - /// - /// `None` against a container: htpasswd grants every authenticated pair - /// the same thing, so there is no narrower pair to hand out. - fn pull_credentials(&self) -> Option { - match &self.server { - Server::Container { .. } => None, - Server::Remote { pull, .. } => pull.as_ref().map(|(user, password)| Auth::Basic { - user: user.clone(), - password: password.clone(), - }), - } - } - - fn registry_as(&self, repository: &str, scratch_dir: Option, auth: Auth) -> Registry { - self.open(repository, scratch_dir, auth, |_| {}) - } - - /// The fixture's own transport, with one thing about it changed. - /// - /// The knob the tests below reach for. Spelled as an edit rather than as - /// another argument so that a test naming the concurrency does not also - /// have to restate the credentials and the scheme this fixture decided. - fn options(&self, repository: &str, set: impl FnOnce(&mut Options)) -> Registry { - self.open(repository, None, self.credentials(), set) - } - - fn open( - &self, - repository: &str, - scratch_dir: Option, - auth: Auth, - set: impl FnOnce(&mut Options), - ) -> Registry { - let repository = match &self.server { - Server::Container { .. } => repository.to_owned(), - Server::Remote { namespace, .. } => format!("{namespace}/{repository}"), - }; - - let mut options = Options { - insecure: !self.tls, - scratch_dir, - auth, - ..Default::default() - }; - - set(&mut options); - - Registry::open( - &format!("oci://{}/{repository}", self.address()) - .parse() - .expect("the fixture named a usable repository"), - options, - ) - .unwrap() - } -} - -/// An htpasswd file, plus the configuration a registry that wants one in a file -/// rather than in the environment reads. -/// -/// Returned as a directory the caller holds: the container has both mounted, -/// and a `TempDir` dropped early would unlink them out from under it. -fn auth_dir() -> tempfile::TempDir { - let dir = tempfile::tempdir().expect("a temporary directory for the htpasswd file"); - - std::fs::write(dir.path().join("htpasswd"), HTPASSWD).expect("writing the htpasswd file"); - std::fs::write(dir.path().join("zot.json"), ZOT_CONFIG).expect("writing the zot config"); - - dir -} - -/// The `docker run` arguments that make a registry demand -/// [`USER`]/[`PASSWORD`]. -/// -/// **Both configurations, unconditionally, and no per-image branch.** The two -/// server families this suite is pointed at read their auth from different -/// places and each ignores the other's: `distribution` reads `REGISTRY_AUTH_*` -/// out of the environment and never opens `/etc/zot/config.json`, while `zot` -/// reads that file and knows nothing about `REGISTRY_*`. Applying both is -/// therefore not a guess about which image is running — it is the union of two -/// settings that cannot collide. -/// -/// A registry that reads neither would run anonymous, which every other test -/// here would be perfectly happy with. [`credentials_are_required`] is what -/// notices. -fn auth_args(dir: &std::path::Path) -> Vec { - let path = |name: &str| dir.join(name).display().to_string(); - - vec![ - "--volume".to_owned(), - format!("{}:/auth/htpasswd:ro", path("htpasswd")), - "--volume".to_owned(), - format!("{}:/etc/zot/config.json:ro", path("zot.json")), - "--env".to_owned(), - "REGISTRY_AUTH=htpasswd".to_owned(), - "--env".to_owned(), - "REGISTRY_AUTH_HTPASSWD_REALM=stelae".to_owned(), - "--env".to_owned(), - "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd".to_owned(), - ] -} - -impl Drop for Fixture { - fn drop(&mut self) { - if let Server::Container { container, .. } = &self.server { - let _ = std::process::Command::new("docker") - .args(["rm", "--force", container]) - .output(); - } - } -} - -/// Write a small stele of the toy profile through any transport. -/// -/// One layer handed over whole and one streamed into a sink, so both write -/// paths are exercised against whatever this is pointed at. `chapter` decides -/// what goes in the index layer, which is how the delta test makes two steles -/// that share a blob and differ in one. -fn write_stele(stele: &W, chapter: u64) -> Inscription { - write_stele_fallibly(stele, chapter).unwrap() -} - -/// The same publish, with every refusal handed back rather than unwrapped. -/// -/// For the tests whose subject *is* a refusal. Split out rather than made the -/// only spelling because the twenty callers that expect a stele would each -/// grow an `.unwrap()` that says nothing, and the one that does not would stop -/// standing out. -fn write_stele_fallibly(stele: &W, chapter: u64) -> Result { - let (notes_header, notes_scope) = notes_scope(3); - let (index_header, index_scope) = index_scope(chapter); - - let notes: Vec = (1..=3).map(note_record).collect(); - - let written_notes = stele.write_layer( - &ToyProfile, - &LayerSpec::new("notes", notes_header, notes_scope), - COMPRESSION_LEVEL, - ¬es, - )?; - - let mut sink = stele.layer_sink( - &ToyProfile, - &LayerSpec::new("index", index_header, index_scope), - COMPRESSION_LEVEL, - )?; - - for id in 1..=chapter { - sink.write_record(¬e_record(id))?; - } - - let written_index = sink.finish()?; - - let mut inscription = Inscription::new( - &ToyProfile, - chapter, - json!({"chapter": chapter, "shelf": "east"}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.layers = vec![written_notes.descriptor, written_index.descriptor]; - - stele.seal(&ToyProfile, &inscription)?; - - Ok(inscription) -} - -/// Every record of every layer, read back through the streaming reader. -fn records_of(stele: &R, inscription: &Inscription) -> Vec>> { - let index = stele.blob_index().unwrap(); - - inscription - .layers - .iter() - .map(|descriptor| { - let mut reader = stele - .stream_layer(&index, &ToyProfile, descriptor, Limits::default()) - .unwrap(); - - let mut records = vec![reader.header().encode().unwrap().as_bytes().to_vec()]; - - while let Some(record) = reader.next_record() { - records.push(record.unwrap().to_vec()); - } - - // Only now is the layer proven: the identity digest covers every - // byte, so nothing above was trustworthy until this returned. - let digests = reader.finish().unwrap(); - assert_eq!(digests.diff_id, descriptor.diff_id); - - records - }) - .collect() -} - -/// Done criterion 1: a stele pushed to a registry and pulled back is the same -/// stele. -/// -/// "The same" is checked against a directory rather than against itself: the -/// same records go into a `SteleDir` and into the registry, and the two are -/// compared on the inscription digest, on the identity of every layer, on the -/// compressed blob digest — the byte string the registry stores against the -/// byte string the directory names its file by — and on the records that come -/// back out. -#[test] -#[ignore = "spawns a registry"] -fn a_stele_survives_the_round_trip() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.registry("stelae/roundtrip"); - - let temp = tempfile::tempdir().unwrap(); - let directory = SteleDir::create(temp.path()).unwrap(); - - let published = write_stele(®istry, 3); - let on_disk = write_stele(&directory, 3); - - assert_eq!(published, on_disk, "one stele, two transports"); - println!( - "identity: {} ({} layers)", - published.digest().unwrap(), - published.layers.len() - ); - - // Both tags resolve, and to the same stele. - let latest = registry.pull_latest(&ToyProfile).unwrap(); - let by_sequence = registry - .pull_sequence(&ToyProfile, published.sequence) - .unwrap(); - - assert_eq!(latest.read_inscription().unwrap(), published); - assert_eq!(by_sequence.read_inscription().unwrap(), published); - assert_eq!( - latest.read_inscription().unwrap().digest().unwrap(), - on_disk.digest().unwrap(), - ); - - // The identity→blob map came off the manifest; the directory's came from - // decompressing everything. They agree blob for blob, which is what makes - // the manifest a shortcut rather than a second source of truth. - let pulled_blobs = latest.blob_index().unwrap(); - let disk_blobs = directory.blob_index().unwrap(); - - assert_eq!(pulled_blobs.len(), disk_blobs.len()); - - for descriptor in &published.layers { - assert_eq!( - pulled_blobs.blob_for(&descriptor.diff_id), - disk_blobs.blob_for(&descriptor.diff_id), - "layer {:?} is a different blob in the registry", - descriptor.kind, - ); - } - - // And the records themselves. - assert_eq!( - records_of(&latest, &published), - records_of(&directory, &on_disk), - "layers differ record for record", - ); - - println!( - "pulled {} layers, {} compressed bytes", - published.layers.len(), - latest.total_compressed_size(), - ); - - // The whole-stele figure is the per-layer one summed, which is what makes - // the per-layer answer usable for a restore's remaining-download estimate: - // a subset of the layers weighs a subset of the bytes, on the same scale. - let index = latest.blob_index().unwrap(); - - let summed: u64 = published - .layers - .iter() - .map(|layer| { - latest - .compressed_size(&index, layer) - .unwrap() - .expect("a pulled stele states every layer's compressed size") - }) - .sum(); - - assert_eq!(summed, latest.total_compressed_size()); - - // A stele of another profile is refused before a layer is fetched. - struct Other; - impl Profile for Other { - fn name(&self) -> &str { - "com.acme.receipts" - } - fn version(&self) -> u64 { - 1 - } - fn kinds(&self) -> &[&str] { - &["receipts"] - } - fn layer_media_type(&self, kind: &str) -> Result { - Ok(format!("application/vnd.acme.stele.{kind}.v1+zstd")) - } - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("r-{sequence}")) - } - } - - let err = registry.pull(&Other, "latest").unwrap_err(); - assert!(matches!(err, Error::UnknownProfile { .. }), "{err:?}"); -} - -/// Done criterion 2: the second push moves only what the registry lacks, and -/// the transport says so in a number. -/// -/// The two steles share their `notes` layer byte for byte — same records, same -/// scope, so the same `diffId` and, at a pinned compression level, the same -/// blob — and differ in their `index` layer. What the registry has to receive -/// is therefore exactly one blob, and what it must be spared is exactly one. -#[test] -#[ignore = "spawns a registry"] -fn a_second_push_uploads_only_what_is_missing() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.registry("stelae/delta"); - - let first = write_stele(®istry, 3); - let first_transfer = registry.take_transfer(); - - println!("first push: {first_transfer:?}"); - - assert_eq!(first_transfer.layers_uploaded, 2, "an empty repository"); - assert_eq!(first_transfer.layers_skipped, 0); - assert!(first_transfer.bytes_uploaded > 0); - assert_eq!(first_transfer.bytes_skipped, 0); - - let second = write_stele(®istry, 4); - let second_transfer = registry.take_transfer(); - - println!("second push: {second_transfer:?}"); - - assert_eq!( - second_transfer.layers_skipped, 1, - "the shared notes layer should not have moved", - ); - assert_eq!( - second_transfer.layers_uploaded, 1, - "only the new index layer should have moved", - ); - assert!(second_transfer.bytes_skipped > 0); - - // The skip is not a lie: both steles pull back whole, and the layer that - // was skipped is the one the first push put there. - let notes = &first.layers[0]; - assert_eq!(notes.diff_id, second.layers[0].diff_id); - - for sequence in [first.sequence, second.sequence] { - let stele = registry.pull_sequence(&ToyProfile, sequence).unwrap(); - let inscription = stele.read_inscription().unwrap(); - - assert_eq!(inscription.sequence, sequence); - records_of(&stele, &inscription); - } - - // And `latest` followed the second one. - let latest = registry.pull_latest(&ToyProfile).unwrap(); - assert_eq!(latest.read_inscription().unwrap(), second); -} - -// --------------------------------------------------------------------------- -// Peak allocation, in both directions -// --------------------------------------------------------------------------- - -/// Records of roughly a kilobyte, the order of a Dolos `indexes` or `state` -/// record. -const BULK_RECORD_BODY: usize = 1000; - -/// ~50 MB of layer. Ten times the budget below, so a transport that buffered -/// even a fifth of a layer would show up. -const BULK_RECORDS: u64 = 48 * 1024; - -/// [`BULK_RECORDS`], unless the run says otherwise. -/// -/// `STELAE_TEST_BULK_RECORDS` exists for the registry deployment's gates, -/// which ask the same question at a mainnet shard's scale — a gibibyte and up -/// — where a default that size would make every local run unbearable. Nothing -/// else moves: the budget stays fixed precisely because the layer does not. -fn bulk_records() -> u64 { - match std::env::var("STELAE_TEST_BULK_RECORDS") { - Ok(count) => count - .parse() - .expect("STELAE_TEST_BULK_RECORDS is a record count"), - Err(_) => BULK_RECORDS, - } -} - -/// What either direction may hold at any one moment, on the *streamed* path. -/// -/// The push peak is one upload chunk and a little change — 4 MiB and some -/// tens of kilobytes, measured, and stable to a fraction of a percent across -/// runs, because the client splits the chunk it is handed rather than copying -/// it. The pull side sits far below that. 5 MiB is the larger of the two with -/// room over it. -/// -/// What matters is not the number but that it does not move when the layer -/// does: the peak is bound by the chunk, and the layer here is ten times the -/// budget. `STELAE_TEST_BULK_RECORDS` below asks the same question at a -/// mainnet shard's scale and this constant does not follow it up. -/// -/// It is not the bound on the *single-request* path, which holds a whole layer -/// by construction and is bounded by [`Options::upload_memory`] instead — -/// [`the_single_request_path_is_bounded_by_its_byte_budget`]. Every push test -/// here therefore has to say which path it is measuring, and this one says so -/// by naming a threshold below the layer it sends. -const TRANSPORT_BUDGET: usize = 5 * 1024 * 1024; - -/// A record whose body zstd cannot shrink. -/// -/// This matters more here than it does in `tests/memory.rs`. That file measures -/// the framing and the codec, where a compressible body only makes the layer -/// cheaper to hold. Here the layer has to cross a socket, and a body that -/// compresses to nothing would leave the *transfer* — the part this test exists -/// to bound — moving a few kilobytes while the assertions talked about fifty -/// megabytes. So the body is a splitmix64 stream: cheap to generate, -/// deterministic, and incompressible. -fn bulk_record(i: u64) -> CanonicalCbor { - /// The splitmix64 finalizer. - fn mix(x: u64) -> u64 { - let z = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); - let z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); - z ^ (z >> 31) - } - - // Chained on the *state* rather than on a shared additive counter. The - // obvious version — seed by `i`, then step by a constant — makes record - // `i + 1` a one-word shift of record `i`, which zstd finds across its - // window and compresses seventy-fold. Every record here is its own orbit. - let mut state = mix(i); - let mut body = Vec::with_capacity(BULK_RECORD_BODY + 8); - - while body.len() < BULK_RECORD_BODY { - state = mix(state ^ 0x9e37_79b9_7f4a_7c15); - body.extend_from_slice(&state.to_le_bytes()); - } - - body.truncate(BULK_RECORD_BODY); - - encode(|e| { - e.array(2)?.u64(i)?.bytes(&body)?; - Ok(()) - }) - .unwrap() -} - -/// Bytes allocated and not yet returned, sampled from another thread for as -/// long as it is alive. -/// -/// A `Region` can only be read from the thread that owns it, and the peak that -/// matters here is *inside* a single call — the upload in `finish`, the -/// download in `stream_layer` — where there is nowhere to put a sample. So the -/// process-wide counters are polled instead, against a baseline taken when the -/// sampler starts. -struct Peak { - stop: std::sync::Arc, - peak: std::sync::Arc, - handle: Option>, -} - -impl Peak { - fn start() -> Self { - let base = GLOBAL.stats(); - let stop = std::sync::Arc::new(AtomicBool::new(false)); - let peak = std::sync::Arc::new(AtomicUsize::new(0)); - - let handle = { - let stop = std::sync::Arc::clone(&stop); - let peak = std::sync::Arc::clone(&peak); - - std::thread::spawn(move || { - while !stop.load(Ordering::Relaxed) { - let now = GLOBAL.stats(); - let allocated = now.bytes_allocated.saturating_sub(base.bytes_allocated); - let freed = now.bytes_deallocated.saturating_sub(base.bytes_deallocated); - - peak.fetch_max(allocated.saturating_sub(freed), Ordering::Relaxed); - - std::thread::sleep(std::time::Duration::from_micros(200)); - } - }) - }; - - Self { - stop, - peak, - handle: Some(handle), - } - } - - fn finish(mut self) -> usize { - self.stop.store(true, Ordering::Relaxed); - - if let Some(handle) = self.handle.take() { - let _ = handle.join(); - } - - self.peak.load(Ordering::Relaxed) - } -} - -/// Done criterion 4: neither direction scales with the layer, on the streamed -/// path. -/// -/// The layer is fifty times the budget and never fits in it, so a transport -/// that held one — buffering a blob before uploading it, or decompressing a -/// pulled one into memory — cannot pass. This is the `tests/memory.rs` -/// discipline extended to the transport; it lives here rather than there -/// because it needs a registry, and a bound measured against a mock would only -/// be a bound on the mock. -/// -/// The threshold is named rather than left at its default, and that is the -/// whole reason this test still means what it meant: at the default a fifty -/// megabyte layer is *under* [`Options::monolithic_max`] and goes up in one -/// request, holding itself while it does. Sending it as a chain is now a -/// decision a caller makes, so a test about the chain has to make it. -#[test] -#[ignore = "spawns a registry"] -fn neither_direction_holds_a_layer() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.options("stelae/memory", |options| options.monolithic_max = 0); - - let (header_scope, scope) = notes_scope(1); - let spec = LayerSpec::new("notes", header_scope, scope); - - // --- up --------------------------------------------------------------- - let sampler = Peak::start(); - let started = std::time::Instant::now(); - - let mut sink = registry - .layer_sink(&ToyProfile, &spec, COMPRESSION_LEVEL) - .unwrap(); - - for i in 0..bulk_records() { - sink.write_record(&bulk_record(i)).unwrap(); - } - - let layer = sink.finish().unwrap(); - - let mut inscription = Inscription::new( - &ToyProfile, - 1, - json!({"chapter": 1}), - json!({}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - inscription.layers = vec![layer.descriptor.clone()]; - - registry.seal(&ToyProfile, &inscription).unwrap(); - - let pushed = sampler.finish(); - - let size = layer.descriptor.uncompressed_size; - let compressed = layer.digests.compressed_size; - - println!( - "push: {size} uncompressed / {compressed} compressed bytes, \ - peak {pushed} bytes held, {:.1?} elapsed", - started.elapsed(), - ); - - assert!( - size > 8 * TRANSPORT_BUDGET as u64, - "the layer has to dwarf the budget for this to prove anything: \ - {size} against {TRANSPORT_BUDGET}", - ); - - // And so does what actually crossed the socket. A compressible fixture - // would leave every assertion here true and none of them about the - // transfer. - assert!( - compressed > 8 * TRANSPORT_BUDGET as u64, - "the blob has to dwarf the budget too, or the upload proved nothing: \ - {compressed} against {TRANSPORT_BUDGET}", - ); - - assert!( - pushed < TRANSPORT_BUDGET, - "pushing a {size}-byte layer held {pushed} bytes at peak; \ - the budget is {TRANSPORT_BUDGET}", - ); - - // --- down ------------------------------------------------------------- - let sampler = Peak::start(); - let started = std::time::Instant::now(); - - let stele = registry.pull_latest(&ToyProfile).unwrap(); - let read = stele.read_inscription().unwrap(); - let blobs = stele.blob_index().unwrap(); - - let mut reader = stele - .stream_layer(&blobs, &ToyProfile, &read.layers[0], Limits::default()) - .unwrap(); - - let mut count = 1u64; // the header record, already consumed - while let Some(record) = reader.next_record() { - assert!(!record.unwrap().is_empty()); - count += 1; - } - - let digests = reader.finish().unwrap(); - let pulled = sampler.finish(); - - println!( - "pull: {count} records, peak {pulled} bytes held, {:.1?} elapsed", - started.elapsed(), - ); - - assert_eq!(count, layer.descriptor.records); - assert_eq!(digests.diff_id, layer.descriptor.diff_id); - - assert!( - pulled < TRANSPORT_BUDGET, - "pulling a {size}-byte layer held {pulled} bytes at peak; \ - the budget is {TRANSPORT_BUDGET}", - ); -} - -/// Every [`Event::Bytes`] the transport emitted, in order. -/// -/// The only in-process view of *how many requests* an upload took. The -/// streamed path emits one of these per chunk handed to the client and a chunk -/// is one `PATCH` — its own documentation says so — while the single-request -/// path has nothing finer to report and emits once. So a sequence of deltas is -/// the shape of the upload, and comparing two of them compares two paths. -/// -/// Not a request count read off the wire, and it should not be mistaken for -/// one: a registry's own log would say more, and none of the three servers this -/// fixture runs against says it the same way. -#[derive(Default)] -struct Chunks(Mutex>); - -impl Progress for Chunks { - fn on(&self, event: Event<'_>) { - if let Event::Bytes(n) = event { - self.0 - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .push(n); - } - } -} - -impl Chunks { - fn seen(&self) -> Vec { - self.0 - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .clone() - } -} - -/// A layer that fits goes up in one request; one that does not still streams; -/// and neither is a different layer for it. -/// -/// The second half is the one decision 0026 rests on. A stele carries its -/// predecessor's layers forward by identity, so a layer republished through a -/// different transport path has to *be* the same layer — same `diffId` over the -/// same uncompressed bytes, same blob under the same digest — or every stele -/// after this change would carry nothing forward and every publish would be a -/// full upload. -/// -/// Both layers go into one publish, past one threshold, so neither the -/// registry's blob-skip nor a second server can be what the difference is: the -/// small layer is under [`Options::monolithic_max`] and the bulk layer is over -/// it, and they are new blobs in an empty repository either way. -/// -/// Serial on purpose. The deltas carry no layer identity, so attributing them -/// needs the uploads not to overlap — which is what a concurrency of one buys, -/// and the only thing this test wants from it. -#[test] -#[ignore = "spawns a registry"] -fn a_layer_under_the_threshold_goes_up_in_one_request() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - // Between the two layers below: the small one under it, the bulk one over. - const THRESHOLD: u64 = 1024 * 1024; - - // The transport's `UPLOAD_CHUNK`, which the crate keeps private. It is what - // sets the streamed event count — one per chunk handed to the client — and - // it is not the threshold: what makes the bulk layer a *chain* is being - // over this, and a "chain" of one would prove nothing. - const CHUNK: u64 = 4 * 1024 * 1024; - - // Comfortably over one upload chunk once compressed — the bodies are - // incompressible, so this is close to what crosses the socket. - const BULK: u64 = 6 * 1024; - - let chunks = std::sync::Arc::new(Chunks::default()); - - let registry = fixture.options("stelae/one-request", |options| { - options.monolithic_max = THRESHOLD; - options.concurrency = 1; - }); - - registry.observe(Observer::new(chunks.clone())); - - let (notes_header, notes_scope) = notes_scope(1); - let notes: Vec = (1..=3).map(note_record).collect(); - - let small = registry - .write_layer( - &ToyProfile, - &LayerSpec::new("notes", notes_header, notes_scope), - COMPRESSION_LEVEL, - ¬es, - ) - .unwrap(); - - let (index_header, index_scope) = index_scope(1); - - let mut sink = registry - .layer_sink( - &ToyProfile, - &LayerSpec::new("index", index_header, index_scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - // Its own range of the record space. Two of the three registries this runs - // against address blobs across the whole registry rather than per - // repository, so a layer that happened to be another test's layer would be - // skipped rather than uploaded and there would be nothing to count. - for i in 0..BULK { - sink.write_record(&bulk_record(2_000_000 + i)).unwrap(); - } - - let bulk = sink.finish().unwrap(); - - let mut inscription = Inscription::new( - &ToyProfile, - 1, - json!({"chapter": 1}), - json!({}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.layers = vec![small.descriptor.clone(), bulk.descriptor.clone()]; - - registry.seal(&ToyProfile, &inscription).unwrap(); - - let small_size = small.digests.compressed_size; - let bulk_size = bulk.digests.compressed_size; - - assert!( - small_size <= THRESHOLD, - "the small layer has to be under the threshold: {small_size} against {THRESHOLD}", - ); - assert!( - bulk_size > CHUNK, - "the bulk layer has to be over one upload chunk, or it streams as a \ - single delta and the chain below is a chain of one: \ - {bulk_size} against {CHUNK}", - ); - - let seen = chunks.seen(); - - assert_eq!( - seen.first().copied(), - Some(small_size), - "the layer under the threshold reported {seen:?}; \ - one request is one delta, and it is the whole layer", - ); - assert!( - seen.len() > 2, - "the layer over the threshold reported {seen:?}; \ - a chain is more than one chunk", - ); - assert_eq!( - seen.iter().sum::(), - small_size + bulk_size, - "the deltas do not add up to what was published: {seen:?}", - ); - - let stele = registry.pull_latest(&ToyProfile).unwrap(); - let read = stele.read_inscription().unwrap(); - - assert_eq!(read, inscription, "the inscription came back different"); - - let blobs = stele.blob_index().unwrap(); - - for (described, written) in read.layers.iter().zip([&small, &bulk]) { - let mut reader = stele - .stream_layer(&blobs, &ToyProfile, described, Limits::default()) - .unwrap(); - - while let Some(record) = reader.next_record() { - record.unwrap(); - } - - // Recomputed over every byte the registry gave back, so this is the - // identity the carry-forward will look for and not a number copied out - // of the descriptor that claimed it. - assert_eq!( - reader.finish().unwrap().diff_id, - written.descriptor.diff_id, - "the {} layer is not the layer that was pushed", - described.kind, - ); - } -} - -/// The single-request path holds its budget, not its concurrency. -/// -/// The one thing this change can break in production. A monolithic push is -/// resident in full, so a bound counted in *layers* would let -/// [`Options::concurrency`] multiply a hundred megabytes by thirty-two and take -/// the publisher pod down mid-publish — which costs an epoch and looks like a -/// registry failure. The bound is counted in bytes instead, and this is the -/// assertion that it is. -/// -/// Eight layers, all under the threshold so all resident, against a budget of -/// one and a half of them. A transport bounded by the layer count would hold -/// eight; one bounded by the budget holds two and makes the rest wait. -#[test] -#[ignore = "spawns a registry"] -fn the_single_request_path_is_bounded_by_its_byte_budget() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - /// Layers, all in flight at once as far as the permits are concerned. - const LAYERS: u64 = 8; - - /// Records each, ~4 MB compressed — incompressible bodies, so the layer and - /// the blob are the same order of magnitude. - const BULK: u64 = 4 * 1024; - - /// What the transport may hold. Under two layers, so seven eighths of the - /// publish cannot be resident whatever the concurrency says. - const BUDGET: u64 = 6 * 1024 * 1024; - - let registry = fixture.options("stelae/budget", |options| { - options.concurrency = LAYERS as usize; - options.upload_memory = BUDGET; - }); - - let sampler = Peak::start(); - - let mut inscription = Inscription::new( - &ToyProfile, - 1, - json!({"chapter": 1}), - json!({}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - for layer in 0..LAYERS { - let (header, scope) = notes_scope(layer); - - let mut sink = registry - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header, scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - // Offset per layer so no two layers are the same blob, and offset again - // past every other test in this file so no *other* test's blob is one - // of these: two of the three registries this runs against address blobs - // across the whole registry, and a skipped layer would leave the budget - // untested. - for i in 0..BULK { - sink.write_record(&bulk_record(3_000_000 + layer * BULK + i)) - .unwrap(); - } - - let written = sink.finish().unwrap(); - - // The threshold here is `BUDGET` — `monolithic_max` is left at its - // default and clamped down to it — so a layer over this line would - // stream, hold one chunk, and sail under the ceiling below without the - // byte budget ever being asked for. The assertion is what stops a - // change in what `bulk_record` compresses to from quietly turning this - // test into one that measures nothing. - assert!( - written.digests.compressed_size <= BUDGET, - "layer {layer} has to be resident for this to measure the budget: \ - {} against {BUDGET}", - written.digests.compressed_size, - ); - - inscription.layers.push(written.descriptor); - } - - registry.seal(&ToyProfile, &inscription).unwrap(); - - let held = sampler.finish(); - - let published: u64 = inscription - .layers - .iter() - .map(|layer| layer.uncompressed_size) - .sum(); - - println!("budget: {published} bytes published, peak {held} bytes held"); - - assert_eq!( - registry.transfer().layers_uploaded, - LAYERS, - "every layer has to have been uploaded, or the budget was never asked for", - ); - - // The budget plus the streaming allowance: the staging, the compressor and - // the client's own buffers are not what this bounds, and `TRANSPORT_BUDGET` - // is already the measured size of them. - let ceiling = BUDGET as usize + TRANSPORT_BUDGET; - - assert!( - held < ceiling, - "publishing {LAYERS} layers held {held} bytes at peak against a \ - {BUDGET}-byte budget; a bound counted in layers would hold about \ - {}", - LAYERS as usize * (published as usize / LAYERS as usize), - ); -} - -/// A blob that is not a layer never reaches the reader as one. -/// -/// The registry is content-addressed, so tampering with a stored blob is not -/// possible without changing its name — which is exactly what makes the -/// interesting failure a *manifest* that points at the wrong blob. -/// -/// Both refusals here land in `LayerReader::new`, before a single record past -/// the header is read: one because the header names another kind, one because -/// the index has no blob under that identity at all. The check at the *other* -/// end of the layer — the identity digest over every byte — is out of reach -/// from here for the reason the first case shows, and is -/// [`a_same_kind_blob_is_refused_when_the_layer_ends`]. -#[test] -#[ignore = "spawns a registry"] -fn a_layer_that_is_not_the_one_described_is_refused() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.registry("stelae/tamper"); - - let inscription = write_stele(®istry, 3); - - let stele = registry.pull_latest(&ToyProfile).unwrap(); - let blobs = stele.blob_index().unwrap(); - - // The notes layer's blob, under the index layer's descriptor: a real blob, - // correctly named, holding the wrong layer. - let mut swapped = inscription.layers[1].clone(); - swapped.diff_id = inscription.layers[0].diff_id; - - let err = stele - .stream_layer(&blobs, &ToyProfile, &swapped, Limits::default()) - .unwrap_err(); - - assert!(matches!(err, Error::LayerMismatch { .. }), "{err:?}"); - - // And a descriptor naming a layer the stele does not carry. - let mut absent = inscription.layers[0].clone(); - absent.diff_id = digest_of(0xee); - - let err = stele - .stream_layer(&blobs, &ToyProfile, &absent, Limits::default()) - .unwrap_err(); - - assert!(matches!(err, Error::LayerNotFound { .. }), "{err:?}"); -} - -/// The identity check at the end of a layer, reached. -/// -/// A `diffId` annotation lives in the manifest, outside the inscription, so -/// nothing about a stele's *identity* covers it — which makes a manifest that -/// points a descriptor at the wrong blob the tamper this format has to survive -/// on its own. Point it at a blob of another kind and the header record settles -/// it immediately, which is what -/// [`a_layer_that_is_not_the_one_described_is_refused`] shows. Point it at a -/// blob of its own kind and the header has nothing to say: only the hash of -/// every byte, once the layer ends, can tell the two apart. -#[test] -#[ignore = "spawns a registry"] -fn a_same_kind_blob_is_refused_when_the_layer_ends() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.registry("stelae/tamper-same-kind"); - - // Two layers of one kind under one scope, differing only in how many - // records they hold, so their headers are byte-identical and their - // identities are not. - let write = |count: u64| { - let (header, scope) = notes_scope(3); - - let mut sink = registry - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header, scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - for id in 1..=count { - sink.write_record(¬e_record(id)).unwrap(); - } - - sink.finish().unwrap() - }; - - let short = write(3); - let long = write(6); - - let mut inscription = Inscription::new( - &ToyProfile, - 1, - json!({"chapter": 1, "shelf": "east"}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.layers = vec![short.descriptor.clone(), long.descriptor.clone()]; - registry.seal(&ToyProfile, &inscription).unwrap(); - - let stele = registry.pull_latest(&ToyProfile).unwrap(); - - // The long layer's identity, pointed at the short layer's blob. The short - // one is the target on purpose: reading it stays inside the size the long - // descriptor claims, so the meter cannot refuse this before the digest - // does, and it is the digest that is under test. - let mut tampered = stele.blob_index().unwrap(); - tampered.insert(long.descriptor.diff_id, short.digests.blob_digest); - - let mut reader = stele - .stream_layer(&tampered, &ToyProfile, &long.descriptor, Limits::default()) - .unwrap(); - - // Every record reads cleanly. Nothing up to here is wrong; the layer is - // simply not the one that was asked for. - while let Some(record) = reader.next_record() { - record.unwrap(); - } - - let err = reader.finish().unwrap_err(); - - // Named exactly, because a `DigestMismatch` is also what a blob that - // arrived corrupt would produce: this one has to be the identity check, - // reporting the layer that was asked for against the one that was read. - assert!( - matches!( - &err, - Error::DigestMismatch { subject, expected, actual } - if subject.contains("notes") - && *expected == long.descriptor.diff_id.to_string() - && *actual == short.descriptor.diff_id.to_string() - ), - "{err:?}" - ); -} - -/// The scratch directory is honoured, and nothing survives a push. -/// -/// A mainnet state shard is hundreds of megabytes compressed, and sixteen of -/// them staged in the platform temporary directory is how a publish fills a -/// volume nobody was watching. Staging files are unlinked at creation, so this -/// checks the directory is used and left empty rather than that files are -/// cleaned up afterwards. -#[test] -#[ignore = "spawns a registry"] -fn staging_stays_in_the_scratch_directory_and_leaves_nothing() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let scratch = tempfile::tempdir().unwrap(); - let registry = fixture.registry_staging_in("stelae/scratch", Some(scratch.path().to_owned())); - - let inscription = write_stele(®istry, 3); - let stele = registry.pull_latest(&ToyProfile).unwrap(); - - records_of(&stele, &inscription); - - let left: Vec<_> = std::fs::read_dir(scratch.path()) - .unwrap() - .map(|entry| entry.unwrap().path()) - .collect(); - - assert!(left.is_empty(), "{left:?}"); - - // A sink that is abandoned mid-layer takes its staging with it. - let (header_scope, scope) = notes_scope(9); - let mut sink = registry - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header_scope, scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - sink.write_record(¬e_record(1)).unwrap(); - drop(sink); - - let left: Vec<_> = std::fs::read_dir(scratch.path()) - .unwrap() - .map(|entry| entry.unwrap().path()) - .collect(); - - assert!(left.is_empty(), "{left:?}"); -} - -/// A staging directory that cannot be used says which one, in both directions. -/// -/// Both directions, because one [`Options::scratch_dir`] serves both — a sink -/// on the way up and a pulled blob on the way down — and fixing the direction -/// somebody happened to test first is how this defect would come back. -/// -/// `an_unusable_staging_directory_names_itself` in `src/oci.rs` makes the same -/// claim without a container, and says why the unusable directory is an -/// existing regular file; this one makes it through a real publish and a real -/// pull. -#[test] -#[ignore = "spawns a registry"] -fn a_staging_directory_that_cannot_be_used_names_itself() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let root = tempfile::tempdir().unwrap(); - let occupied = root.path().join("not-a-directory"); - std::fs::write(&occupied, b"").unwrap(); - - let names_it = |err: &Error| { - assert!( - matches!(err, Error::Scratch { dir, .. } if dir == &occupied), - "fell through to the catch-all: {err:?}", - ); - - let message = err.to_string(); - assert!( - message.contains(&occupied.display().to_string()), - "{message}", - ); - assert!(message.contains("staging directory"), "{message}"); - }; - - // Up: the sink stages the layer it is building. - let (header_scope, scope) = notes_scope(3); - let Err(err) = fixture - .registry_staging_in("stelae/occupied", Some(occupied.clone())) - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header_scope, scope), - COMPRESSION_LEVEL, - ) - else { - panic!("staged a layer in a regular file") - }; - - names_it(&err); - - // Down: the same directory, against a stele that is really there. Published - // through a staging directory that works, so what fails below is the pull. - let staged = tempfile::tempdir().unwrap(); - let published = write_stele( - &fixture.registry_staging_in("stelae/occupied", Some(staged.path().to_owned())), - 3, - ); - - let reader = fixture.registry_staging_in("stelae/occupied", Some(occupied.clone())); - let stele = reader.pull_latest(&ToyProfile).unwrap(); - let index = stele.blob_index().unwrap(); - - let err = stele - .stream_layer(&index, &ToyProfile, &published.layers[0], Limits::default()) - .expect_err("staged a pulled blob in a regular file"); - - names_it(&err); -} - -/// `latest` tells "this repository holds nothing" apart from "this repository -/// could not be read", which is the distinction a publisher's history chain -/// rests on. -/// -/// Both halves against a real server, because the shape a registry uses to say -/// "no such manifest" is exactly the thing that cannot be established by -/// reading a client's source. -#[test] -#[ignore = "spawns a registry"] -fn latest_is_absent_until_something_is_published() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - let registry = fixture.registry("stelae/eventually"); - - assert!( - registry.latest(&ToyProfile).unwrap().is_none(), - "an empty repository holds no stele, and that is not an error" - ); - - let published = write_stele(®istry, 3); - - let found = registry - .latest(&ToyProfile) - .unwrap() - .expect("the repository holds a stele now"); - - assert_eq!(found.read_inscription().unwrap(), published); - - // A repository that never existed is absent too, and by a different - // registry error code than a missing tag in one that does. - let empty = fixture.registry("stelae/never-written-to"); - assert!(empty.latest(&ToyProfile).unwrap().is_none()); -} - -/// The registry the fixture spawns actually demands credentials, and a refusal -/// is never read as absence. -/// -/// Two claims, and the second is the one with teeth. `Registry::latest` turns -/// "no such manifest" into `None`, and a publisher reads `None` as "nothing to -/// chain to" and starts a fresh history — so a 401 widening into absence would -/// silently restart the attestation chain against a registry that simply did -/// not recognise the caller. `is_absent` is written not to, and this is that -/// claim against a server that really answers 401 rather than against a -/// hand-built error value. -/// -/// The first claim is what keeps the rest of this file honest: the fixture -/// configures htpasswd for the two server families it knows, and a registry -/// that read neither would run anonymous with every other test here passing -/// exactly as before. This one fails instead — which, for an operator pointing -/// `STELAE_TEST_REGISTRY_IMAGE` at a fourth implementation, is the fixture -/// saying it does not know how to make that one ask for credentials. -#[test] -#[ignore = "spawns a registry"] -fn credentials_are_required() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - // The pair the fixture configured reads the repository, which is the - // baseline every other test in this file rests on. - let allowed = fixture.registry("stelae/credentials"); - assert!(allowed.latest(&ToyProfile).unwrap().is_none()); - - for (who, auth) in [ - ("anonymous", Auth::Anonymous), - ( - "the wrong password", - Auth::Basic { - user: fixture.push_user().to_owned(), - password: "not-the-password".to_owned(), - }, - ), - ] { - let refused = fixture.registry_as("stelae/credentials", None, auth); - - let err = refused - .latest(&ToyProfile) - .expect_err("the registry answered an unauthenticated request"); - - println!("{who}: {err}"); - - // And a publish through this transport is refused rather than starting - // a chain, which is the consequence that matters. - assert!(refused.pull_latest(&ToyProfile).is_err(), "{who}"); - } -} - -/// The published read-only pair reads a stele whole and cannot write one. -/// -/// The deployment this suite points at hands consumers a pull-only pair — -/// free, identity-less, and still credentialed — and its access policy rests -/// on the registry enforcing that narrowness: a pull-only pair that could -/// push would make the published credential a write credential. So both -/// halves run against the real enforcement: a stele published under the full -/// pair pulls back whole under the narrow one, and the same narrow pair is -/// refused an upload. -/// -/// Only a deployment names a second pair; htpasswd grants every pair the same -/// thing. Against a container this says so and proves nothing. -#[test] -#[ignore = "spawns a registry"] -fn the_read_only_pair_pulls_and_cannot_push() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let Some(pull) = fixture.pull_credentials() else { - eprintln!( - "no pull-only pair here: set STELAE_TEST_REGISTRY_PULL_USER and \ - _PULL_PASSWORD to run this against a deployment" - ); - return; - }; - - let published = write_stele(&fixture.registry("stelae/read-only"), 3); - - let reading = fixture.registry_as("stelae/read-only", None, pull); - let stele = reading.pull_latest(&ToyProfile).unwrap(); - let inscription = stele.read_inscription().unwrap(); - - assert_eq!(inscription, published); - records_of(&stele, &inscription); - - // Through the seal, because that is where an upload's refusal is now - // reported: `finish` closes the layer and hands the round trips to the - // pool, so the credential is not asked about them until they are joined. - // The claim is unchanged — this pair cannot put a stele in this - // repository — and the seal is the honest place to make it, since a - // publish that seals is a publish that happened. - let refused = write_stele_fallibly(&reading, 4) - .expect_err("a pull-only pair was allowed to publish a stele"); - - println!("write through the pull-only pair: {refused}"); -} - -/// Carrying a layer forward costs nothing and asks nothing. -/// -/// The default, and the reason the publish path stopped getting slower with -/// every epoch behind it: `source` is a stele this transport pulled, its -/// manifest is live under a tag, and a registry may not reclaim a blob in that -/// position. So the layer is carried on the manifest's word — no round trip, -/// no bytes — and only the counters move. -#[test] -#[ignore = "spawns a registry"] -fn a_layer_is_carried_forward_on_the_manifest_that_names_it() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let source = fixture.registry("stelae/source"); - let published = write_stele(&source, 3); - let stele = source.latest(&ToyProfile).unwrap().unwrap(); - - // Reset first, so what is read back is this call's cost and not the two - // layers `write_stele` pushed through the same transport. - source.take_transfer(); - - source - .adopt_layer(&stele, published.layers[0].clone()) - .unwrap(); - - let transfer = source.take_transfer(); - - assert_eq!(transfer.layers_reused, 1); - assert_eq!(transfer.layers_uploaded, 0); - assert!(transfer.bytes_reused > 0); -} - -/// An operator who does not trust the repository's retention gets the check -/// back, and it still lands before the manifest does. -/// -/// This is the guarantee `verify_adopted` exists for, and the failure it -/// prevents is invisible without it: a manifest pointing at a reclaimed blob is -/// a perfectly well-formed stele that nobody can restore. What has moved is -/// *when* the refusal is reported — the `HEAD` runs concurrently with the rest -/// of the publish and is joined at the seal — and what has not moved is that -/// the refusal comes before anything is tagged. -/// -/// **Provoked with a second registry, not a second repository**, and the -/// difference is a real one this test found. `zot` answers `HEAD` *and* `GET` -/// for a blob under a repository it was never pushed to — its storage is -/// content-addressed across the whole registry — while `distribution` 2.8 and -/// 3.0 answer 404. So a sibling repository is not reliably a place a blob is -/// absent from, and on `zot` it would not even be the wrong answer: a registry -/// that will serve the blob is a registry where the manifest works. Only a -/// separate server is absent everywhere. -#[test] -#[ignore = "spawns a registry"] -fn a_verified_carry_refuses_a_blob_the_repository_does_not_hold() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let source = fixture.registry("stelae/source"); - let published = write_stele(&source, 3); - let stele = source.latest(&ToyProfile).unwrap().unwrap(); - - let somewhere_else = Fixture::spawn(); - let elsewhere = somewhere_else.verifying("stelae/source"); - - // The descriptor is handed back: nothing has been asked yet, and what the - // caller holds is a fact about bytes rather than a promise about a - // registry. - elsewhere - .adopt_layer(&stele, published.layers[0].clone()) - .unwrap(); - - // The seal is where the promise is collected, and it is refused. - let mut inscription = published.clone(); - inscription.layers = vec![published.layers[0].clone()]; - - let err = elsewhere.seal(&ToyProfile, &inscription).unwrap_err(); - - assert!(matches!(err, Error::BlobMissing { .. }), "{err:?}"); - - // And nothing was published: the moving tag in a repository that never had - // a stele still resolves to nothing. - assert!( - elsewhere.latest(&ToyProfile).unwrap().is_none(), - "a refused seal tagged a manifest anyway", - ); -} - -/// Concurrency changes what a publish costs and nothing about what it -/// produces. -/// -/// The claim the whole change rests on. The same records, published through the -/// serial path and through eight-way concurrency, must give the same -/// inscription — the same identity — the same manifest bytes, and the same -/// transfer counters, because none of those is a function of the order the -/// blobs happened to land in. -/// -/// Two repositories in one registry rather than two registries, so the -/// comparison is not also comparing two servers — with the one consequence -/// that the *counters* cannot be compared to each other. `zot` addresses -/// blobs across the whole registry rather than per repository, as -/// [`a_verified_carry_refuses_a_blob_the_repository_does_not_hold`] documents -/// at more length, so on that registry the second publish skips what the first -/// one uploaded. What is asserted instead is the property that holds on either -/// kind and is the one worth having: every layer and every byte the stele -/// describes is accounted for, whichever way the registry answered. -#[test] -#[ignore = "spawns a registry"] -fn concurrency_changes_the_cost_of_a_publish_and_not_the_stele() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let serial = fixture.registry_at("stelae/serial", 1); - let concurrent = fixture.registry_at("stelae/concurrent", 8); - - let one = write_stele(&serial, 5); - let other = write_stele(&concurrent, 5); - - assert_eq!(one, other, "the inscriptions differ"); - assert_eq!( - one.digest().unwrap(), - other.digest().unwrap(), - "the identities differ", - ); - - let from_serial = serial.pull_latest(&ToyProfile).unwrap(); - let from_concurrent = concurrent.pull_latest(&ToyProfile).unwrap(); - - assert_eq!( - manifest_bytes(from_serial.manifest()).unwrap(), - manifest_bytes(from_concurrent.manifest()).unwrap(), - "the manifests differ", - ); - - // The serial publish is the first into this registry, so nothing can have - // been there before it: two layers, both uploaded. - let counted = serial.transfer(); - - assert_eq!(counted.layers_uploaded, one.layers.len() as u64); - assert_eq!(counted.layers_skipped, 0); - - // And the concurrent one accounts for exactly the same layers and the same - // bytes, however the registry split them between "uploaded" and "the far - // side already had it". - let against = concurrent.transfer(); - - assert_eq!( - against.layers_uploaded + against.layers_skipped, - counted.layers_uploaded + counted.layers_skipped, - "a layer went unaccounted for", - ); - assert_eq!( - against.bytes_uploaded + against.bytes_skipped, - counted.bytes_uploaded + counted.bytes_skipped, - "the bytes do not add up", - ); - assert_eq!(against.layers_reused, 0, "nothing was carried forward"); - - // And it reads back, which is the property the manifest exists to serve. - let inscription = from_concurrent.read_inscription().unwrap(); - - assert_eq!(inscription, one); - records_of(&from_concurrent, &inscription); -} - -/// A publish abandoned while its layers are still in flight leaves the stele -/// before it standing. -/// -/// The concurrent path's version of the ordering argument, and the reason the -/// join is at the seal rather than anywhere later: layers go up in parallel, -/// but nothing is tagged until all of them are up, so a publisher that dies in -/// the middle — here, a transport dropped without a seal — leaves untagged -/// blobs the registry reclaims and a moving tag still pointing at a stele that -/// restores. -#[test] -#[ignore = "spawns a registry"] -fn a_publish_dropped_mid_flight_leaves_the_previous_stele_standing() { - let _serial = exclusive(); - - let fixture = Fixture::spawn(); - - let standing = write_stele(&fixture.registry("stelae/abandoned"), 3); - - { - let abandoning = fixture.registry_at("stelae/abandoned", 8); - - let (header, scope) = notes_scope(4); - let mut sink = abandoning - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header, scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - for id in 1..=64 { - sink.write_record(¬e_record(id)).unwrap(); - } - - sink.finish().unwrap(); - - // And dropped here, with the upload deferred and no seal to join it. - } - - let reopened = fixture.registry("stelae/abandoned"); - let stele = reopened.pull_latest(&ToyProfile).unwrap(); - - assert_eq!(stele.read_inscription().unwrap(), standing); -} - -/// A deferred upload's failure is the seal's failure, and it stays the seal's -/// failure. -/// -/// No registry: the transport is pointed at a port nothing is listening on, so -/// every round trip it defers is refused. That is enough to hold the two -/// properties that matter about the deferral, and it holds them under plain -/// `cargo test` rather than only where a container can be spawned. -/// -/// 1. **`finish` succeeds.** Closing a layer is a fact about bytes the sink -/// already has; a transport that could not reach the registry still hands -/// back the descriptor, because the caller's next act is to read more of its -/// store and not to wait on a socket. -/// 2. **`seal` fails, and every seal after it fails too.** The join empties the -/// handles it awaited, so a transport that forgot would find nothing -/// outstanding the second time, agree that every layer was up, and publish a -/// manifest naming a blob that never landed. That is the one document this -/// transport must never write, and it is exactly the document a concurrent -/// publish makes reachable — so the refusal is remembered rather than -/// recomputed. -/// 3. **The failure was retried first, and said so.** A connection nobody -/// answers is the transient class, so the round trip is made again before it -/// is anybody's failure — and the retry is announced, because a transport -/// that absorbed a registry's bad minute in silence would have hidden the -/// measurement that motivated absorbing it. Two attempts here rather than -/// the default four, so the test proves the loop runs without waiting out -/// the whole of its patience. -#[test] -fn a_deferred_upload_that_fails_fails_every_seal() { - let _serial = exclusive(); - - install_crypto_provider(); - - // Bound and dropped: the kernel just told us a port nobody has, and - // refusing a connection is faster and more portable than any other way of - // failing one. - let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let address = closed.local_addr().unwrap(); - drop(closed); - - let registry = Registry::open( - &format!("oci://{address}/stelae/nowhere").parse().unwrap(), - Options { - insecure: true, - concurrency: 4, - attempts: 2, - ..Default::default() - }, - ) - .unwrap(); - - let retries = std::sync::Arc::new(Retries::default()); - registry.observe(Observer::new(retries.clone())); - - let (header, scope) = notes_scope(1); - let notes: Vec = (1..=3).map(note_record).collect(); - - let written = registry - .write_layer( - &ToyProfile, - &LayerSpec::new("notes", header, scope), - COMPRESSION_LEVEL, - ¬es, - ) - .expect("closing a layer waited on the registry"); - - let mut inscription = Inscription::new( - &ToyProfile, - 1, - json!({"chapter": 1}), - json!({"noteWidth": 40}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.layers = vec![written.descriptor]; - - let refused = registry - .seal(&ToyProfile, &inscription) - .expect_err("sealed against a port nothing is listening on"); - - // The *cause*, not the sticky refusal — and asserting which way round that - // is, is the point. `LayerNotWritten` is what every seal *after* this one - // answers; a first seal that reported it would have thrown away what the - // network actually said, leaving an operator to debug a connection failure - // from a message about a layer. - assert!( - matches!(refused, Error::Registry(_)), - "the first seal reported the refusal instead of its cause: {refused:?}", - ); - - println!("the seal collected the deferred failure: {refused}"); - - // And it is remembered: nothing is outstanding any more, so a transport - // that only asked what was in flight would seal this stele over a blob that - // never landed. - let again = registry - .seal(&ToyProfile, &inscription) - .expect_err("the second seal published a manifest over a blob that never landed"); - - assert!( - matches!(again, Error::LayerNotWritten(_)), - "the second seal did not remember the first: {again:?}", - ); - - // Carrying the cause, so the operator reading the second refusal is not - // told less than the one who read the first. - let Error::LayerNotWritten(why) = &again else { - unreachable!() - }; - - assert!(!why.is_empty(), "the refusal names no cause"); - - println!("and every seal after it: {again}"); - - // One round trip was deferred — the existence check for the one layer — and - // it was made twice. The second seal had nothing outstanding to retry, so - // this also says the sticky refusal is answered without touching the - // network again. - assert_eq!( - retries.seen(), - vec![(1, 1)], - "a refused connection was not retried before the layer was declared lost", - ); -} - -/// What a transport said about the round trips it made again. -#[derive(Default)] -struct Retries(Mutex>); - -impl Progress for Retries { - fn on(&self, event: Event<'_>) { - if let Event::Retry { - attempt, remaining, .. - } = event - { - self.0.lock().unwrap().push((attempt, remaining)); - } - } -} - -impl Retries { - fn seen(&self) -> Vec<(u32, u32)> { - self.0.lock().unwrap().clone() - } -} - -/// The annotation map is a `BTreeMap`, so the canonical JSON above is not -/// hostage to insertion order. Cheap to state, and the kind of thing that only -/// breaks in the diff of an unrelated change. -#[test] -fn annotation_keys_are_ordered() { - let (inscription, layers) = fixture(); - let (manifest, _) = build_manifest(&inscription, &layers).unwrap(); - - let annotations: &BTreeMap = manifest.layers[0].annotations.as_ref().unwrap(); - let keys: Vec<&str> = annotations.keys().map(String::as_str).collect(); - - assert_eq!( - keys, - vec![DIFF_ID_ANNOTATION, KIND_ANNOTATION, SCOPE_ANNOTATION] - ); -} diff --git a/crates/stelae/tests/rfc8785.rs b/crates/stelae/tests/rfc8785.rs deleted file mode 100644 index 0f91544b7..000000000 --- a/crates/stelae/tests/rfc8785.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! RFC 8785 (JSON Canonicalization Scheme) conformance. -//! -//! The inscription digest is the identity of a stele: independent publishers -//! reproduce it, signatures cover it, and `history` chains it. All of that -//! rests on canonicalization being *the same function* everywhere — so the JCS -//! implementation is not an implementation detail to be assumed correct, it is -//! a conformance surface with an official test suite. This file runs it. -//! -//! Everything here goes through `stelae::canonical_json`, the same entry point -//! `Inscription::canonicalize` uses, rather than the underlying crate directly: -//! a passing vendored dependency proves nothing if the protocol reaches it by a -//! different path. -//! -//! Vector provenance is in `tests/data/rfc8785/README.md`. - -use std::path::PathBuf; - -fn data_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/rfc8785") -} - -const VECTORS: &[&str] = &[ - "arrays", - "french", - "structures", - "unicode", - "values", - "weird", -]; - -/// The six official vectors, compared byte for byte. -#[test] -fn official_vectors() { - for name in VECTORS { - let input = std::fs::read(data_dir().join(format!("input/{name}.json"))) - .unwrap_or_else(|e| panic!("reading input/{name}.json: {e}")); - let expected = std::fs::read(data_dir().join(format!("output/{name}.json"))) - .unwrap_or_else(|e| panic!("reading output/{name}.json: {e}")); - - let value: serde_json::Value = serde_json::from_slice(&input).unwrap(); - let canonical = stelae::canonical_json(&value).unwrap(); - - assert_eq!( - canonical, - expected, - "vector {name}\n got: {}\n expected: {}", - String::from_utf8_lossy(&canonical), - String::from_utf8_lossy(&expected), - ); - } -} - -/// Canonicalization is idempotent: feeding the canonical form back in returns -/// it unchanged. A publisher that re-canonicalizes a document it received must -/// land on the same bytes, or `history` verification breaks. -#[test] -fn canonicalization_is_idempotent() { - for name in VECTORS { - let expected = std::fs::read(data_dir().join(format!("output/{name}.json"))).unwrap(); - let value: serde_json::Value = serde_json::from_slice(&expected).unwrap(); - - assert_eq!( - stelae::canonical_json(&value).unwrap(), - expected, - "vector {name} is not a fixed point" - ); - } -} - -/// RFC 8785 Appendix B, "Number Serialization Samples", verbatim: the IEEE 754 -/// bit pattern and the JSON text ECMAScript renders for it. -/// -/// The two NaN/Infinity rows of the table are omitted — they are not -/// representable in JSON, and `serde_json` has no way to hold them. -const APPENDIX_B: &[(u64, &str)] = &[ - (0x0000000000000000, "0"), - (0x8000000000000000, "0"), - (0x0000000000000001, "5e-324"), - (0x8000000000000001, "-5e-324"), - (0x7fefffffffffffff, "1.7976931348623157e+308"), - (0xffefffffffffffff, "-1.7976931348623157e+308"), - (0x4340000000000000, "9007199254740992"), - (0xc340000000000000, "-9007199254740992"), - (0x4430000000000000, "295147905179352830000"), - (0x44b52d02c7e14af5, "9.999999999999997e+22"), - (0x44b52d02c7e14af6, "1e+23"), - (0x44b52d02c7e14af7, "1.0000000000000001e+23"), - (0x444b1ae4d6e2ef4e, "999999999999999700000"), - (0x444b1ae4d6e2ef4f, "999999999999999900000"), - (0x444b1ae4d6e2ef50, "1e+21"), - (0x3eb0c6f7a0b5ed8c, "9.999999999999997e-7"), - (0x3eb0c6f7a0b5ed8d, "0.000001"), - (0x41b3de4355555553, "333333333.3333332"), - (0x41b3de4355555554, "333333333.33333325"), - (0x41b3de4355555555, "333333333.3333333"), - (0x41b3de4355555556, "333333333.3333334"), - (0x41b3de4355555557, "333333333.33333343"), - (0xbecbf647612f3696, "-0.0000033333333333333333"), - (0x43143ff3c1cb0959, "1424953923781206.2"), -]; - -/// Number rendering is where two "conformant" implementations most plausibly -/// disagree, because it is ECMAScript's algorithm rather than anything JSON -/// specifies. Appendix B is the arbiter. -#[test] -fn appendix_b_number_serialization() { - for (bits, expected) in APPENDIX_B { - let value = serde_json::Number::from_f64(f64::from_bits(*bits)) - .map(serde_json::Value::Number) - .unwrap_or_else(|| panic!("{bits:#018x} is not representable in JSON")); - - let canonical = String::from_utf8(stelae::canonical_json(&value).unwrap()).unwrap(); - - assert_eq!( - canonical, *expected, - "IEEE 754 {bits:#018x} rendered as {canonical}, RFC 8785 says {expected}" - ); - } -} - -/// How integers render is the half of the question the inscription actually -/// depends on: every number in an inscription is an integer, and it must come -/// out as plain digits with no exponent and no decimal point. -#[test] -fn integers_render_as_plain_digits() { - let cases: &[(i64, &str)] = &[ - (0, "0"), - (1, "1"), - (550, "550"), - (-1, "-1"), - (21600, "21600"), - (43_210_000, "43210000"), - (402_653_184, "402653184"), - // An epoch's worth of mainnet blocks, and the largest value the - // inscription rule admits. - (40_000_000_000, "40000000000"), - (stelae::MAX_SAFE_INTEGER, "9007199254740991"), - (-stelae::MAX_SAFE_INTEGER, "-9007199254740991"), - ]; - - for (value, expected) in cases { - let canonical = - String::from_utf8(stelae::canonical_json(&serde_json::json!(value)).unwrap()).unwrap(); - assert_eq!(canonical, *expected, "integer {value}"); - } -} - -/// Past 2^53 - 1 the JCS crate keeps working and starts lying: a `u64` renders -/// as the nearest double, silently. This test pins that behaviour so the -/// protocol's refusal to canonicalize such values is understood as load-bearing -/// rather than as belt-and-braces — see `inscription::check_safe_numbers`. -#[test] -fn beyond_the_safe_range_rendering_is_lossy() { - let unsafe_value = serde_json::json!(u64::MAX); - let canonical = String::from_utf8(stelae::canonical_json(&unsafe_value).unwrap()).unwrap(); - - assert_eq!(canonical, "18446744073709552000"); - assert_ne!(canonical, u64::MAX.to_string()); - - // Which is exactly why the inscription refuses it before it can reach the - // canonicalizer. - let err = stelae::inscription::check_safe_numbers(&unsafe_value).unwrap_err(); - assert!( - matches!(err, stelae::Error::UnsafeInteger { .. }), - "expected a refusal, got {err:?}" - ); - - // The boundary itself is fine on both sides of zero. - for ok in [ - serde_json::json!(stelae::MAX_SAFE_INTEGER), - serde_json::json!(-stelae::MAX_SAFE_INTEGER), - serde_json::json!(0), - ] { - stelae::inscription::check_safe_numbers(&ok).unwrap(); - } -} diff --git a/crates/stelae/tests/toy_profile.rs b/crates/stelae/tests/toy_profile.rs deleted file mode 100644 index 901b4719c..000000000 --- a/crates/stelae/tests/toy_profile.rs +++ /dev/null @@ -1,1517 +0,0 @@ -//! The boundary proof: a stele of a profile the protocol knows nothing about. -//! -//! `dev.example.toy` publishes chapters of notes. It has no chain, no epochs, -//! no blocks, no ledger — nothing Cardano-shaped and nothing Dolos-shaped. If -//! the protocol had absorbed an assumption from its first real profile, this -//! file would not compile or would not pass, which is why it is the only -//! profile in the tree while the core is being written. -//! -//! What it demonstrates, end to end: -//! -//! 1. A stele is written to a directory and read back, records intact. -//! 2. Two independent write runs produce the same inscription digest. -//! 3. Every vendor-owned string in the artifact came from the profile; the core -//! composed none of them. -//! 4. `position`, `parameters` and `scope` survive as arbitrary shapes — the -//! core canonicalizes and hashes them without ever typing them. -//! 5. A stele of a *different* profile, or a profile major version above the -//! one implemented, is refused cleanly. -//! 6. Both write paths — a layer handed over whole, and a layer streamed into a -//! sink — produce the same artifact, and this profile publishes one of each. -//! 7. A stele carrying a layer kind this build does not define is *readable* — -//! the layer is reported and skipped — and not publishable on top of. - -use std::{collections::BTreeSet, io::Write as _}; - -use serde_json::json; - -use stelae::{ - digest::read_blob, - dir::{BlobIndex, LayerSpec, SteleDir, WrittenLayer}, - frame::{encode, CanonicalCbor, Limits}, - Compression, Discarding, Error, Inscription, LayerDescriptor, LayerWriter, Profile, RecordSink, - SteleReader, SteleWriter, -}; - -const PROFILE_NAME: &str = "dev.example.toy"; -const NOTES_MEDIA_TYPE: &str = "application/vnd.example.stele.notes.v1+zstd"; -const INDEX_MEDIA_TYPE: &str = "application/vnd.example.stele.index.v1+zstd"; -const COVERS_MEDIA_TYPE: &str = "application/vnd.example.stele.covers.v1+zstd"; -const COMPRESSION_LEVEL: i32 = 9; - -/// A vendor's profile. Everything here is the vendor's business; none of it is -/// known to `stelae`. -struct ToyProfile; - -impl Profile for ToyProfile { - fn name(&self) -> &str { - PROFILE_NAME - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["notes", "index"] - } - - fn layer_media_type(&self, kind: &str) -> Result { - match kind { - "notes" => Ok(NOTES_MEDIA_TYPE.to_owned()), - "index" => Ok(INDEX_MEDIA_TYPE.to_owned()), - other => Err(Error::UnknownLayerKind { - profile: PROFILE_NAME.to_owned(), - kind: other.to_owned(), - }), - } - } - - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("chapter-{sequence}")) - } -} - -/// The profile's own record shapes. The protocol never sees inside these. -struct Note { - id: u64, - title: &'static str, - body: &'static [u8], -} - -const NOTES: &[Note] = &[ - Note { - id: 1, - title: "on stelae", - body: b"a standing inscribed slab", - }, - Note { - id: 2, - title: "on determinism", - body: b"two publishers, one digest", - }, - Note { - id: 3, - title: "on profiles", - body: b"the vendor owns the payload", - }, -]; - -fn note_record(note: &Note) -> CanonicalCbor { - encode(|e| { - e.array(3)? - .u64(note.id)? - .str(note.title)? - .bytes(note.body)?; - Ok(()) - }) - .unwrap() -} - -fn index_record(note: &Note) -> CanonicalCbor { - encode(|e| { - e.array(2)?.str(note.title)?.u64(note.id)?; - Ok(()) - }) - .unwrap() -} - -/// The profile's scope for the notes layer: a CBOR array in the header record, -/// the same idea as a JSON object in the inscription. Two encodings of one -/// vendor-owned concept; the protocol carries both without reading either. -fn notes_scopes() -> (CanonicalCbor, serde_json::Value) { - let header = encode(|e| { - e.array(3)?.u64(3)?.u64(1)?.u64(3)?; - Ok(()) - }) - .unwrap(); - - (header, json!({"chapter": 3, "firstId": 1, "lastId": 3})) -} - -fn index_scopes() -> (CanonicalCbor, serde_json::Value) { - let header = encode(|e| { - e.map(1)?.str("chapter")?.u64(3)?; - Ok(()) - }) - .unwrap(); - - (header, json!({"chapter": 3})) -} - -/// Read a layer both ways and insist the two paths agree. -/// -/// Everything a consumer can observe is compared: the header, the records, the -/// digests, and — where they fail — the failure. The two readers exist because -/// one of them can afford to hold a layer and the other cannot; nothing else -/// about them is allowed to differ, because a layer that restores through one -/// and not the other is a determinism bug in the format itself. -fn read_both_ways( - stele: &SteleDir, - index: &BlobIndex, - descriptor: &LayerDescriptor, -) -> Result>, Error> { - let buffered = stele.read_layer(index, &ToyProfile, descriptor); - - // A window far below one record, so the streaming path is exercised at its - // refill boundaries rather than swallowing the layer in one read. - let limits = Limits { - window: 8, - ..Limits::default() - }; - - let streamed = (|| { - let mut reader = stele.stream_layer(index, &ToyProfile, descriptor, limits)?; - - let mut records = Vec::new(); - while let Some(record) = reader.next_record() { - records.push(record?.to_vec()); - } - - let digests = reader.finish()?; - - Ok::<_, Error>((records, digests)) - })(); - - match (buffered, streamed) { - (Ok(layer), Ok((records, digests))) => { - assert_eq!(layer.digests(), &digests, "digests"); - - let buffered_records: Vec> = layer - .records() - .map(|r| r.unwrap().to_vec()) - .collect::>(); - - assert_eq!(buffered_records, records, "records"); - - Ok(records) - } - (Err(buffered), Err(streamed)) => { - assert_eq!( - std::mem::discriminant(&buffered), - std::mem::discriminant(&streamed), - "both paths must refuse for the same reason: \ - buffered {buffered:?}, streaming {streamed:?}" - ); - - Err(buffered) - } - (Ok(_), Err(streamed)) => panic!("the streaming path alone refused it: {streamed:?}"), - (Err(buffered), Ok(_)) => panic!("the buffered path alone refused it: {buffered:?}"), - } -} - -/// Write a complete stele of the toy profile into `root`. -/// -/// This is the publisher side of the protocol in miniature: frame the profile's -/// records into layers, collect the descriptors the layers yield, and put them -/// in an inscription whose digest is the stele's identity. -fn write_stele(root: &std::path::Path) -> (Inscription, stelae::Digest) { - let stele = SteleDir::create(root).unwrap(); - - let (notes_header_scope, notes_scope) = notes_scopes(); - let (index_header_scope, index_scope) = index_scopes(); - - let notes: Vec = NOTES.iter().map(note_record).collect(); - - let written_notes = stele - .write_layer( - &ToyProfile, - &LayerSpec::new("notes", notes_header_scope, notes_scope), - COMPRESSION_LEVEL, - ¬es, - ) - .unwrap(); - - // The index layer is streamed into a sink instead, so this profile — which - // the protocol knows nothing about — exercises both write paths in the same - // stele. The records are never collected: a profile publishing at Dolos - // sizes cannot hold a layer, and the shape it needs has to work for a - // three-note chapter too. That the goldens below do not move is the proof - // that the two paths produce one artifact. - let mut sorted: Vec<&Note> = NOTES.iter().collect(); - sorted.sort_by_key(|n| n.title); - - let mut index_sink = stele - .layer_sink( - &ToyProfile, - &LayerSpec::new("index", index_header_scope, index_scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - for note in sorted { - index_sink.write_record(&index_record(note)).unwrap(); - } - - let written_index = index_sink.finish().unwrap(); - - let mut inscription = Inscription::new( - &ToyProfile, - 3, - json!({"chapter": 3, "shelf": "east", "curator": {"name": "example", "since": 1998}}), - json!({"noteWidth": 40, "titleOrder": "byte"}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.history = vec![ - stelae::HistoryEntry { - sequence: 1, - inscription_digest: stelae::Digest::from_bytes([0x11; 32]), - }, - stelae::HistoryEntry { - sequence: 2, - inscription_digest: stelae::Digest::from_bytes([0x22; 32]), - }, - ]; - - inscription.layers = vec![written_notes.descriptor, written_index.descriptor]; - - let digest = stele.seal(&ToyProfile, &inscription).unwrap(); - - (inscription, digest) -} - -#[test] -fn writes_a_stele_and_reads_it_back() { - let temp = tempfile::tempdir().unwrap(); - let (written, digest) = write_stele(temp.path()); - - // On-disk shape: one canonical document plus content-addressed blobs. - assert!(temp.path().join("inscription.json").is_file()); - let blobs = temp.path().join("blobs").join("sha256"); - assert_eq!(std::fs::read_dir(&blobs).unwrap().count(), 2); - - let stele = SteleDir::open(temp.path()).unwrap(); - let read = stele.read_inscription().unwrap(); - - assert_eq!(read, written); - assert_eq!(read.digest().unwrap(), digest); - read.check_profile(&ToyProfile).unwrap(); - - // The layers come back through the same identity the inscription pins. - let index = stele.blob_index().unwrap(); - assert_eq!(index.len(), 2); - - let notes_descriptor = read.layers_of_kind("notes").next().unwrap(); - let layer = stele - .read_layer(&index, &ToyProfile, notes_descriptor) - .unwrap(); - - assert_eq!(layer.header().profile, PROFILE_NAME); - assert_eq!(layer.header().kind, "notes"); - assert_eq!(layer.header().scope, notes_scopes().0); - assert_eq!(layer.digests().diff_id, notes_descriptor.diff_id); - - let records: Vec<&[u8]> = layer.records().collect::>().unwrap(); - assert_eq!(records.len(), NOTES.len()); - - for (record, note) in records.iter().zip(NOTES) { - assert_eq!(*record, note_record(note).as_bytes()); - } - - // And the profile can decode its own records, which the protocol never does. - let mut decoder = minicbor::Decoder::new(records[1]); - assert_eq!(decoder.array().unwrap(), Some(3)); - assert_eq!(decoder.u64().unwrap(), 2); - assert_eq!(decoder.str().unwrap(), "on determinism"); - assert_eq!(decoder.bytes().unwrap(), b"two publishers, one digest"); - - // The second layer reads back the same way. - let index_descriptor = read.layers_of_kind("index").next().unwrap(); - let index_layer = stele - .read_layer(&index, &ToyProfile, index_descriptor) - .unwrap(); - assert_eq!(index_layer.header().kind, "index"); - assert_eq!(index_layer.records().count(), NOTES.len()); - - // And both layers read back identically without ever being held. - for descriptor in &read.layers { - read_both_ways(&stele, &index, descriptor).unwrap(); - } -} - -/// The streaming reader is the one a restore uses, so the profile's records -/// have to survive it unchanged — through a window smaller than any of them, -/// which is the case that has to work for a 400 MB layer to be readable at all. -#[test] -fn a_layer_streams_back_record_for_record() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let stele = SteleDir::open(temp.path()).unwrap(); - let index = stele.blob_index().unwrap(); - - let notes_descriptor = inscription.layers_of_kind("notes").next().unwrap(); - - let mut reader = stele - .stream_layer( - &index, - &ToyProfile, - notes_descriptor, - Limits { - window: 4, - ..Limits::default() - }, - ) - .unwrap(); - - assert_eq!(reader.header().profile, PROFILE_NAME); - assert_eq!(reader.header().kind, "notes"); - assert_eq!(reader.header().scope, notes_scopes().0); - - let mut notes = NOTES.iter(); - while let Some(record) = reader.next_record() { - let expected = notes.next().expect("no more records were written"); - assert_eq!(record.unwrap(), note_record(expected).as_bytes()); - } - assert!(notes.next().is_none(), "every record came back"); - - // Only now is the layer proven. Everything above was read on the strength - // of the descriptor, which is the contract this reader makes explicit. - let digests = reader.finish().unwrap(); - assert_eq!(digests.diff_id, notes_descriptor.diff_id); - assert_eq!( - digests.uncompressed_size, - notes_descriptor.uncompressed_size - ); -} - -/// `finish` is the confirmation, and it does not depend on the caller having -/// been diligent: a reader dropped after one record proves nothing, and a -/// reader finished without consuming anything still reads the whole layer, -/// because the identity digest covers every byte either way. -#[test] -fn finish_confirms_the_layer_whether_or_not_the_records_were_read() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let stele = SteleDir::open(temp.path()).unwrap(); - let index = stele.blob_index().unwrap(); - let descriptor = &inscription.layers[0]; - - // Not a single content record consumed. - let untouched = stele - .stream_layer(&index, &ToyProfile, descriptor, Limits::default()) - .unwrap() - .finish() - .unwrap(); - - assert_eq!(untouched.diff_id, descriptor.diff_id); - - // One record consumed, then finished. Same verdict, same digests. - let mut reader = stele - .stream_layer(&index, &ToyProfile, descriptor, Limits::default()) - .unwrap(); - reader.next_record().unwrap().unwrap(); - - assert_eq!(reader.finish().unwrap(), untouched); -} - -/// Done criterion 2, and the property the whole protocol rests on: the same -/// source data written twice, independently, yields the same identity. -#[test] -fn two_independent_writes_produce_the_same_inscription_digest() { - let first = tempfile::tempdir().unwrap(); - let second = tempfile::tempdir().unwrap(); - - let (left, left_digest) = write_stele(first.path()); - let (right, right_digest) = write_stele(second.path()); - - assert_eq!(left, right); - assert_eq!(left_digest, right_digest); - assert_eq!(left.canonicalize().unwrap(), right.canonicalize().unwrap()); - - // Byte-for-byte on disk, both the document and every blob. - assert_eq!( - std::fs::read(first.path().join("inscription.json")).unwrap(), - std::fs::read(second.path().join("inscription.json")).unwrap(), - ); - - for descriptor in &left.layers { - let left_blob = SteleDir::open(first.path()) - .unwrap() - .blob_index() - .unwrap() - .blob_for(&descriptor.diff_id) - .unwrap(); - let right_blob = SteleDir::open(second.path()) - .unwrap() - .blob_index() - .unwrap() - .blob_for(&descriptor.diff_id) - .unwrap(); - - assert_eq!(left_blob, right_blob, "layer {:?}", descriptor.kind); - } -} - -/// Layer bytes survive a write → read → write round trip unchanged, so a -/// republished layer keeps its identity. -#[test] -fn layers_round_trip_byte_identically() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let stele = SteleDir::open(temp.path()).unwrap(); - let index = stele.blob_index().unwrap(); - - for descriptor in &inscription.layers { - let layer = stele.read_layer(&index, &ToyProfile, descriptor).unwrap(); - - // Re-frame the records that came back and compare the whole sequence. - // The header is re-encoded from its parsed form, so a field that did not - // survive parsing would show up as a byte difference here. - let mut rewritten = Vec::new(); - let mut writer = stelae::SeqWriter::new(&mut rewritten); - - let reencoded_header = layer.header().encode().unwrap(); - assert_eq!(reencoded_header.as_bytes(), layer.header_bytes()); - writer.write_record(&reencoded_header).unwrap(); - - for record in layer.records() { - writer - .write_record(&CanonicalCbor::new(record.unwrap().to_vec()).unwrap()) - .unwrap(); - } - - assert_eq!(rewritten, layer.as_bytes(), "layer {:?}", descriptor.kind); - assert_eq!( - stelae::Digest::compute(&rewritten), - descriptor.diff_id, - "layer {:?} identity", - descriptor.kind - ); - } -} - -/// The two write paths are one write path. -/// -/// The same records, handed over whole and streamed a record at a time, yield -/// the same identity digest, the same blob digest, the same descriptor and the -/// same bytes on disk. `write_layer` is a wrapper over `layer_sink`, so what -/// this pins is that the wrapper stayed thin: staging, digest-naming, the -/// rename and the descriptor have one implementation, and a change that made -/// the buffered path special would have to move a digest here to land. -#[test] -fn both_write_paths_produce_the_same_layer() { - let buffered_dir = tempfile::tempdir().unwrap(); - let streamed_dir = tempfile::tempdir().unwrap(); - - let buffered_stele = SteleDir::create(buffered_dir.path()).unwrap(); - let streamed_stele = SteleDir::create(streamed_dir.path()).unwrap(); - - let (header_scope, scope) = notes_scopes(); - let spec = LayerSpec::new("notes", header_scope, scope); - let records: Vec = NOTES.iter().map(note_record).collect(); - - let buffered = buffered_stele - .write_layer(&ToyProfile, &spec, COMPRESSION_LEVEL, &records) - .unwrap(); - - let mut sink = streamed_stele - .layer_sink(&ToyProfile, &spec, COMPRESSION_LEVEL) - .unwrap(); - - // A sink is a layer already in progress: the protocol's header record is - // written before the handle is returned, so a producer only ever adds its - // own records and the count is never off by one. - assert_eq!(sink.records(), 1); - - for record in &records { - sink.write_record(record).unwrap(); - } - - assert_eq!(sink.records(), 1 + NOTES.len() as u64); - - let streamed = sink.finish().unwrap(); - - assert_eq!(buffered.descriptor, streamed.descriptor); - assert_eq!(buffered.digests, streamed.digests); - - // Same blob digest means the same file name; the bytes under it are the - // same too, which is what an OCI registry would be asked to deduplicate. - let blob = |stele: &SteleDir, written: &WrittenLayer| { - std::fs::read(stele.blob_path(&written.digests.blob_digest)).unwrap() - }; - - assert_eq!( - blob(&buffered_stele, &buffered), - blob(&streamed_stele, &streamed) - ); - - // And one layer in each stele, with no staging file beside it. - for root in [buffered_dir.path(), streamed_dir.path()] { - let blobs = root.join("blobs"); - assert_eq!( - std::fs::read_dir(&blobs).unwrap().count(), - 1, - "only sha256/" - ); - assert_eq!(std::fs::read_dir(blobs.join("sha256")).unwrap().count(), 1); - } -} - -/// A layer whose blob is already on disk is deduplicated, not rewritten. -/// -/// The same records under the same header scope hash to the same blob digest, -/// which is the same file name — so the second write finds its destination -/// occupied by the bytes it was about to write. Publishing it anyway would -/// rewrite hundreds of megabytes with their own contents, and on Windows would -/// collide with any reader holding the blob open. The second write is therefore -/// expected to keep the file that is there and drop its own staging copy, while -/// handing back the very same [`WrittenLayer`] the first write produced. -/// -/// The proof that no bytes moved is a doctored modification time. A rename -/// replaces the file behind the name, and the timestamp belongs to the file, so -/// a mark set on the first write's blob survives only if the second write left -/// it alone — evidence that neither an equality assertion on the descriptor nor -/// a count of the directory could give on its own. -#[test] -fn a_blob_that_already_exists_is_deduplicated() { - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - let blobs = temp.path().join("blobs"); - - let (header_scope, scope) = notes_scopes(); - let spec = LayerSpec::new("notes", header_scope, scope); - let records: Vec = NOTES.iter().map(note_record).collect(); - - // Byte-identical writes: same records, same header scope, same stele. The - // header scope matters — it is inside the layer, so two shards of one kind - // that differ only there are different blobs and never meet here. - let write = || { - let mut sink = stele - .layer_sink(&ToyProfile, &spec, COMPRESSION_LEVEL) - .unwrap(); - - for record in &records { - sink.write_record(record).unwrap(); - } - - sink.finish().unwrap() - }; - - let first = write(); - let blob = stele.blob_path(&first.digests.blob_digest); - assert!(blob.is_file()); - - let mark = std::time::UNIX_EPOCH + std::time::Duration::from_secs(1_000_000_000); - std::fs::File::options() - .write(true) - .open(&blob) - .unwrap() - .set_times(std::fs::FileTimes::new().set_modified(mark)) - .unwrap(); - - let second = write(); - - // Whatever the writer observes is what it would have observed first: the - // descriptor and the digests come from the record stream, not from the - // rename that did not happen. - assert_eq!(first.descriptor, second.descriptor); - assert_eq!(first.digests, second.digests); - - // One blob, and it is the first write's file: the mark is still on it, so - // nothing was written over it. - assert_eq!(std::fs::read_dir(blobs.join("sha256")).unwrap().count(), 1); - assert_eq!( - std::fs::metadata(&blob).unwrap().modified().unwrap(), - mark, - "the blob was rewritten" - ); - - // And the duplicate staging file went with the sink that made it. - assert_eq!( - std::fs::read_dir(&blobs).unwrap().count(), - 1, - "only sha256/" - ); - - // The layer the second writer describes reads back, through both readers, - // out of the blob the first writer published. - let index = stele.blob_index().unwrap(); - assert_eq!(index.len(), 1); - - let expected: Vec> = records.iter().map(|r| r.as_bytes().to_vec()).collect(); - assert_eq!( - read_both_ways(&stele, &index, &second.descriptor).unwrap(), - expected - ); -} - -/// Sixteen sinks open at once, which is the case the sink exists for. -/// -/// The Dolos profile shards its state into sixteen layers and cannot walk the -/// store sixteen times, so it walks once and routes each record to the shard it -/// belongs in. That works only if a sink is an ordinary independent value: no -/// borrow of the stele it will land in, no shared staging name, no ordering -/// between them. Here the records interleave across all sixteen and every layer -/// still reads back — through both readers — exactly what was routed to it. -#[test] -fn sixteen_sinks_are_written_in_one_pass() { - const SHARDS: u64 = 16; - const RECORDS: u64 = 8 * SHARDS; - - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - let blobs = temp.path().join("blobs"); - - let mut sinks: Vec<_> = (0..SHARDS) - .map(|shard| { - let header_scope = encode(|e| { - e.array(1)?.u64(shard)?; - Ok(()) - }) - .unwrap(); - - stele - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header_scope, json!({"shard": shard})), - COMPRESSION_LEVEL, - ) - .unwrap() - }) - .collect(); - - let mut routed: Vec>> = vec![Vec::new(); SHARDS as usize]; - - for id in 0..RECORDS { - let shard = (id % SHARDS) as usize; - let record = encode(|e| { - e.array(2)?.u64(id)?.str("routed")?; - Ok(()) - }) - .unwrap(); - - sinks[shard].write_record(&record).unwrap(); - routed[shard].push(record.as_bytes().to_vec()); - } - - // Sixteen staging files beside `sha256/`, and not one layer yet: nothing is - // published until its digest is known, which is not until `finish`. - assert_eq!( - std::fs::read_dir(&blobs).unwrap().count() as u64, - SHARDS + 1 - ); - assert!(stele.blob_index().unwrap().is_empty()); - - let written: Vec = sinks - .into_iter() - .map(|sink| sink.finish().unwrap()) - .collect(); - - // Sixteen distinct layers, sixteen distinct blobs, nothing staged left. - let diff_ids: BTreeSet<_> = written.iter().map(|w| w.descriptor.diff_id).collect(); - let blob_digests: BTreeSet<_> = written.iter().map(|w| w.digests.blob_digest).collect(); - assert_eq!(diff_ids.len() as u64, SHARDS); - assert_eq!(blob_digests.len() as u64, SHARDS); - assert_eq!( - std::fs::read_dir(&blobs).unwrap().count(), - 1, - "only sha256/" - ); - - let index = stele.blob_index().unwrap(); - assert_eq!(index.len() as u64, SHARDS); - - for (shard, written) in written.iter().enumerate() { - let records = read_both_ways(&stele, &index, &written.descriptor).unwrap(); - assert_eq!(records, routed[shard], "shard {shard}"); - assert_eq!( - written.descriptor.records, - routed[shard].len() as u64 + 1, - "shard {shard} record count, header included" - ); - } -} - -/// A sink that is never finished leaves nothing behind. -/// -/// The case is a mainnet export that fails partway: sixteen state shards open, -/// hundreds of megabytes written, and then a store iterator returns an error. -/// Nothing would ever *read* what is left — staging files sit beside `sha256/` -/// and `blob_index` only considers digest-named entries inside it — but leaving -/// them on the disk is its own incident, so the sink removes its file on the -/// way out. -#[test] -fn a_sink_dropped_without_finishing_leaves_nothing() { - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - let blobs = temp.path().join("blobs"); - - let (header_scope, scope) = notes_scopes(); - - { - let mut sink = stele - .layer_sink( - &ToyProfile, - &LayerSpec::new("notes", header_scope, scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - for note in NOTES { - sink.write_record(¬e_record(note)).unwrap(); - } - - // Open: one staging file, which is not a blob and never was. - let staged: Vec<_> = std::fs::read_dir(&blobs) - .unwrap() - .map(|entry| entry.unwrap().path()) - .filter(|path| path.is_file()) - .collect(); - - assert_eq!(staged.len(), 1, "{staged:?}"); - assert!(stele.blob_index().unwrap().is_empty()); - } - - // Dropped: nothing staged, nothing published. - assert_eq!( - std::fs::read_dir(&blobs).unwrap().count(), - 1, - "only sha256/" - ); - assert_eq!(std::fs::read_dir(blobs.join("sha256")).unwrap().count(), 0); - assert!(stele.blob_index().unwrap().is_empty()); -} - -/// Unknown #4: nothing vendor-owned in the artifact was composed by the core. -/// -/// Every media type and tag in the stele is character-for-character what -/// `ToyProfile` returned. There is no fallback, no default and no template in -/// `stelae` that could have produced them — remove the profile and the strings -/// have no other source. -#[test] -fn the_core_composes_no_vendor_owned_string() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - assert_eq!(inscription.profile.name, ToyProfile.name()); - - for descriptor in &inscription.layers { - let from_profile = ToyProfile.layer_media_type(&descriptor.kind).unwrap(); - assert_eq!(descriptor.media_type, from_profile); - } - - assert_eq!( - stelae::profile::checked_tag_for_sequence(&ToyProfile, inscription.sequence).unwrap(), - "chapter-3" - ); - assert_eq!(ToyProfile.moving_tag(), "latest"); - - // The canonical document mentions the vendor's names and never the - // protocol's reserved one as a payload type. - let canonical = String::from_utf8(inscription.canonicalize().unwrap()).unwrap(); - assert!(canonical.contains(NOTES_MEDIA_TYPE)); - assert!(canonical.contains(INDEX_MEDIA_TYPE)); - assert!(!canonical.contains("vnd.stelae.stele")); - assert!(!canonical.contains("dolos")); - assert!(!canonical.contains("cardano")); -} - -/// The three opaque fields keep whatever the profile put in them, however alien -/// — the core canonicalizes and hashes, and never types them. -#[test] -fn opaque_fields_are_carried_not_interpreted() { - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - - let alien = json!({ - "shelf": ["east", "west"], - "curator": {"name": "example", "since": 1998, "active": true}, - "tags": [], - "retired": null, - }); - - let header_scope = encode(|e| { - e.array(2)?.str("anything")?.bool(true)?; - Ok(()) - }) - .unwrap(); - - let written = stele - .write_layer( - &ToyProfile, - &LayerSpec::new("notes", header_scope.clone(), alien.clone()), - COMPRESSION_LEVEL, - &[note_record(&NOTES[0])], - ) - .unwrap(); - - let mut inscription = Inscription::new( - &ToyProfile, - 0, - alien.clone(), - alien.clone(), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - inscription.layers = vec![written.descriptor]; - stele.seal(&ToyProfile, &inscription).unwrap(); - - let read = SteleDir::open(temp.path()) - .unwrap() - .read_inscription() - .unwrap(); - - assert_eq!(read.position, alien); - assert_eq!(read.parameters, alien); - assert_eq!(read.layers[0].scope, alien); - - let index = SteleDir::open(temp.path()).unwrap().blob_index().unwrap(); - let layer = SteleDir::open(temp.path()) - .unwrap() - .read_layer(&index, &ToyProfile, &read.layers[0]) - .unwrap(); - assert_eq!(layer.header().scope, header_scope); -} - -/// A client fails closed on a stele it cannot read, rather than restoring part -/// of it. -#[test] -fn a_foreign_profile_is_refused() { - struct Other; - - impl Profile for Other { - fn name(&self) -> &str { - "com.acme.receipts" - } - fn version(&self) -> u64 { - 1 - } - fn kinds(&self) -> &[&str] { - &["receipts"] - } - fn layer_media_type(&self, kind: &str) -> Result { - Ok(format!("application/vnd.acme.stele.{kind}.v1+zstd")) - } - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("r-{sequence}")) - } - } - - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let err = inscription.check_profile(&Other).unwrap_err(); - assert!(matches!(err, Error::UnknownProfile { .. }), "{err:?}"); - - // A profile major version above the implemented one is refused too. - let mut future = inscription.clone(); - future.profile.version = 2; - let err = future.check_profile(&ToyProfile).unwrap_err(); - assert!( - matches!(err, Error::UnsupportedProfileVersion { .. }), - "{err:?}" - ); - - // A layer kind the profile does not define is *not* one of these: it is - // skippable at read and refused only on the publish side — see - // `an_unknown_layer_kind_is_skippable_at_read_and_refused_at_publish`. - let mut unknown_kind = inscription.clone(); - unknown_kind.layers[0].kind = "receipts".to_owned(); - unknown_kind.check_profile(&ToyProfile).unwrap(); - let err = unknown_kind.check_profile_strict(&ToyProfile).unwrap_err(); - assert!(matches!(err, Error::UnknownLayerKind { .. }), "{err:?}"); -} - -/// Tampering is caught on both halves of a stele: the document, whose bytes are -/// its digest, and the blobs, whose names are their digests. -#[test] -fn tampering_is_caught_on_read() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - // Re-indenting the inscription leaves the content intact but changes the - // bytes a verifier would hash. - let pretty = serde_json::to_vec_pretty(&inscription).unwrap(); - std::fs::write(temp.path().join("inscription.json"), &pretty).unwrap(); - - let err = SteleDir::open(temp.path()) - .unwrap() - .read_inscription() - .unwrap_err(); - assert!(matches!(err, Error::NonCanonicalInscription), "{err:?}"); - - // A blob whose content no longer matches its name is corruption. - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - let stele = SteleDir::open(temp.path()).unwrap(); - let blob = stele - .blob_index() - .unwrap() - .blob_for(&inscription.layers[0].diff_id) - .unwrap(); - - let path = stele.blob_path(&blob); - let mut bytes = std::fs::read(&path).unwrap(); - let middle = bytes.len() / 2; - bytes[middle] ^= 0xff; - std::fs::write(&path, &bytes).unwrap(); - - assert!( - stele.blob_index().is_err(), - "a blob that disagrees with its name must not index" - ); -} - -/// A descriptor that lies about its layer is refused even when the blob itself -/// is intact — and refused identically whether the layer is held or streamed. -#[test] -fn a_descriptor_that_disagrees_with_its_layer_is_refused() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let stele = SteleDir::open(temp.path()).unwrap(); - let index = stele.blob_index().unwrap(); - - let mut wrong_size = inscription.layers[0].clone(); - wrong_size.uncompressed_size += 1; - let err = read_both_ways(&stele, &index, &wrong_size).unwrap_err(); - assert!(matches!(err, Error::LayerMismatch { .. }), "{err:?}"); - - let mut wrong_count = inscription.layers[0].clone(); - wrong_count.records += 1; - let err = read_both_ways(&stele, &index, &wrong_count).unwrap_err(); - assert!(matches!(err, Error::LayerMismatch { .. }), "{err:?}"); - - let mut wrong_kind = inscription.layers[0].clone(); - wrong_kind.kind = "index".to_owned(); - let err = read_both_ways(&stele, &index, &wrong_kind).unwrap_err(); - assert!(matches!(err, Error::LayerMismatch { .. }), "{err:?}"); - - let mut absent = inscription.layers[0].clone(); - absent.diff_id = stelae::Digest::from_bytes([0xab; 32]); - let err = read_both_ways(&stele, &index, &absent).unwrap_err(); - assert!(matches!(err, Error::LayerNotFound { .. }), "{err:?}"); - - // A descriptor claiming *less* than the layer holds is refused during - // decompression rather than after it, on both paths: the claim is the - // ceiling, and a blob that expands past its own descriptor is not read to - // the end just to be told so. - let mut too_small = inscription.layers[0].clone(); - too_small.uncompressed_size -= 1; - let err = read_both_ways(&stele, &index, &too_small).unwrap_err(); - assert!(matches!(err, Error::DecompressedTooLarge { .. }), "{err:?}"); -} - -/// A malformed record is not a record. -/// -/// Both readers report the first bad one and then end, so *counting* their -/// items tallies the failure itself as a record and discards the error with it. -/// A publisher who sets `records` to match that inflated number would hand back -/// a corrupt layer as `Ok`, in the one place whose documented job is to check -/// everything the descriptor claims. This is the guarantee a refill loop is -/// most likely to lose, which is why it is checked on both paths. -#[test] -fn a_malformed_record_is_reported_not_counted() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let stele = SteleDir::open(temp.path()).unwrap(); - let descriptor = inscription.layers[0].clone(); - - // Take the layer's bytes back out and append the head of a CBOR text string - // that never arrives — a byte the framing must refuse. - let blob_digest = stele - .blob_index() - .unwrap() - .blob_for(&descriptor.diff_id) - .unwrap(); - - let (mut content, _) = read_blob( - std::fs::File::open(stele.blob_path(&blob_digest)).unwrap(), - descriptor.uncompressed_size, - ) - .unwrap(); - - content.push(0x62); - - // Store it as a blob in its own right, under a descriptor claiming exactly - // what `count()` would have said: the real records, plus the failure. - let mut writer = LayerWriter::new(Vec::new(), COMPRESSION_LEVEL).unwrap(); - writer.write_all(&content).unwrap(); - let (blob, digests) = writer.finish().unwrap(); - std::fs::write(stele.blob_path(&digests.blob_digest), &blob).unwrap(); - - let corrupt = LayerDescriptor { - diff_id: digests.diff_id, - uncompressed_size: digests.uncompressed_size, - records: descriptor.records + 1, - ..descriptor - }; - - let index = stele.blob_index().unwrap(); - let err = read_both_ways(&stele, &index, &corrupt).unwrap_err(); - - assert!(matches!(err, Error::TruncatedCbor { .. }), "{err:?}"); - - // And a caller that ignores the bad record does not get a confirmation out - // of `finish` instead. This layer is the awkward case: the malformed byte - // is the *last* one, so every byte still reached the hasher and the digest - // and size the descriptor claims both hold. Only the record count and the - // reader's own memory of having failed stand between a corrupt layer and an - // `Ok`. - let mut reader = stele - .stream_layer(&index, &ToyProfile, &corrupt, Limits::default()) - .unwrap(); - - while let Some(record) = reader.next_record() { - if record.is_err() { - break; - } - } - - let err = reader.finish().unwrap_err(); - assert!(matches!(err, Error::LayerMismatch { .. }), "{err:?}"); -} - -/// A descriptor's media type has to be the one *this* profile defines for that -/// kind. `validate_structure` can only establish that the name is well formed -/// and does not squat the reserved vendor; whether it belongs to the profile -/// the inscription claims needs the profile in hand, which is `check_profile`. -#[test] -fn a_layer_media_type_that_is_not_the_profiles_is_refused() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, _) = write_stele(temp.path()); - - let with_media_type = |media_type: &str| { - let mut tampered = inscription.clone(); - tampered.layers[0].media_type = media_type.to_owned(); - tampered - }; - - // Another vendor's name, and this vendor's name for a different kind. Both - // are well formed, so structural validation passes them. - for media_type in [ - "application/vnd.other.stele.notes.v1+zstd", - INDEX_MEDIA_TYPE, - ] { - let tampered = with_media_type(media_type); - tampered.validate().unwrap(); - - let err = tampered.check_profile(&ToyProfile).unwrap_err(); - assert!(matches!(err, Error::InvalidMediaType { .. }), "{err:?}"); - } - - // Version and codec are transport detail the profile may move within one - // major, so they are not frozen here — only the vendor and the kind are. - for media_type in [ - "application/vnd.example.stele.notes.v2+zstd", - "application/vnd.example.stele.notes.v1+cbor", - ] { - with_media_type(media_type) - .check_profile(&ToyProfile) - .unwrap(); - } -} - -/// A profile that hands back a name it does not own is stopped at the boundary, -/// before anything is written. -#[test] -fn a_profile_cannot_claim_the_protocols_namespace() { - struct Squatter; - - impl Profile for Squatter { - fn name(&self) -> &str { - "dev.example.squatter" - } - fn version(&self) -> u64 { - 1 - } - fn kinds(&self) -> &[&str] { - &["notes"] - } - fn layer_media_type(&self, kind: &str) -> Result { - Ok(format!("application/vnd.stelae.stele.{kind}.v1+zstd")) - } - fn tag_for_sequence(&self, sequence: u64) -> Result { - Ok(format!("c-{sequence}")) - } - } - - let temp = tempfile::tempdir().unwrap(); - let stele = SteleDir::create(temp.path()).unwrap(); - - let (header_scope, scope) = notes_scopes(); - let err = stele - .write_layer( - &Squatter, - &LayerSpec::new("notes", header_scope, scope), - COMPRESSION_LEVEL, - &[note_record(&NOTES[0])], - ) - .unwrap_err(); - - assert!(matches!(err, Error::InvalidMediaType { .. }), "{err:?}"); - - // Nothing was written. - assert_eq!( - std::fs::read_dir(temp.path().join("blobs").join("sha256")) - .unwrap() - .count(), - 0 - ); -} - -/// Golden digests. -/// -/// Every value below is a sha256 over bytes the spec fully determines — the -/// canonical JSON of the inscription, and the deterministic CBOR sequence of -/// each layer. Nothing platform-, timing- or compression-dependent enters them, -/// so they are stable across machines and across zstd versions. -/// -/// That makes this the drift alarm for the whole encoding stack. If a change to -/// the CBOR framing, the JCS canonicalization, the schema's field names or the -/// header record's shape alters a single byte, these values move — and because -/// they *are* published identity, moving one silently is the failure the -/// protocol exists to prevent. A deliberate format change updates them in the -/// same commit that changes the spec; an accidental one shows up here first. -#[test] -fn golden_digests_pin_the_encoding() { - let temp = tempfile::tempdir().unwrap(); - let (inscription, digest) = write_stele(temp.path()); - - assert_eq!( - digest.to_string(), - "sha256:127aa748abafed971fc7ef690a60f1c7d5d1ee49d2e25d043920545c2be2f274", - "inscription digest drifted" - ); - - let expected_layers = [ - ( - "notes", - "sha256:e4f2187aa877f927788b5b4d59241fa2c92de3077eae30022731b4cfba0614f8", - 4u64, - 155u64, - ), - ( - "index", - "sha256:c00d73e03ccfa604f1c7ed5294f2986a4987d3400286ebda8c233503b350e502", - 4, - 77, - ), - ]; - - for (descriptor, (kind, diff_id, records, size)) in - inscription.layers.iter().zip(expected_layers) - { - assert_eq!(descriptor.kind, kind); - assert_eq!(descriptor.diff_id.to_string(), diff_id, "{kind} diffId"); - assert_eq!(descriptor.records, records, "{kind} record count"); - assert_eq!(descriptor.uncompressed_size, size, "{kind} size"); - } - - // The canonical document itself, so a change to key naming or ordering is - // visible in the diff rather than only as a moved hash. - let canonical = String::from_utf8(inscription.canonicalize().unwrap()).unwrap(); - assert_eq!( - canonical, - concat!( - r#"{"compression":{"algo":"zstd","level":9},"#, - r#""history":[{"inscriptionDigest":"sha256:1111111111111111111111111111111111111111111111111111111111111111","sequence":1},"#, - r#"{"inscriptionDigest":"sha256:2222222222222222222222222222222222222222222222222222222222222222","sequence":2}],"#, - r#""layers":[{"diffId":"sha256:e4f2187aa877f927788b5b4d59241fa2c92de3077eae30022731b4cfba0614f8","kind":"notes","#, - r#""mediaType":"application/vnd.example.stele.notes.v1+zstd","records":4,"#, - r#""scope":{"chapter":3,"firstId":1,"lastId":3},"uncompressedSize":155},"#, - r#"{"diffId":"sha256:c00d73e03ccfa604f1c7ed5294f2986a4987d3400286ebda8c233503b350e502","kind":"index","#, - r#""mediaType":"application/vnd.example.stele.index.v1+zstd","records":4,"#, - r#""scope":{"chapter":3},"uncompressedSize":77}],"#, - r#""parameters":{"noteWidth":40,"titleOrder":"byte"},"#, - r#""position":{"chapter":3,"curator":{"name":"example","since":1998},"shelf":"east"},"#, - r#""profile":{"name":"dev.example.toy","version":1},"schema":1,"sequence":3}"#, - ) - ); -} - -/// The discarding writer is faithful, not merely fast. -/// -/// The same records through both write halves: one into a directory, one into -/// nothing. Every field of the descriptor and every one of the four digests and -/// sizes has to agree — including the *blob* digest and the compressed size, -/// which only exist if zstd actually ran. That is the assertion this test is -/// for: a discarding writer that skipped compression would still reproduce -/// `diffId`, `records` and `uncompressedSize`, and would be exactly as wrong as -/// one that never ran at all. -/// -/// The seal is compared too, since a reproduction reports an identity: a -/// directory's comes from the bytes it wrote to `inscription.json`, and this -/// one from the document in hand. -#[test] -fn a_discarding_writer_reproduces_what_a_directory_stores() { - let temp = tempfile::tempdir().unwrap(); - let (stored, stored_digest) = write_stele(temp.path()); - - let (notes_header_scope, notes_scope) = notes_scopes(); - let (index_header_scope, index_scope) = index_scopes(); - - let notes: Vec = NOTES.iter().map(note_record).collect(); - - let reproduced_notes = Discarding - .write_layer( - &ToyProfile, - &LayerSpec::new("notes", notes_header_scope, notes_scope), - COMPRESSION_LEVEL, - ¬es, - ) - .unwrap(); - - let mut sorted: Vec<&Note> = NOTES.iter().collect(); - sorted.sort_by_key(|n| n.title); - - let mut index_sink = Discarding - .layer_sink( - &ToyProfile, - &LayerSpec::new("index", index_header_scope, index_scope), - COMPRESSION_LEVEL, - ) - .unwrap(); - - for note in sorted { - index_sink.write_record(&index_record(note)).unwrap(); - } - - let reproduced_index = index_sink.finish().unwrap(); - - // The stored stele's own blob digests, recovered the way a directory has - // to: by hashing the files it holds. Nothing in an inscription carries - // them, which is the point — they are transport, and a reproduction that - // agreed on identity while disagreeing on the compressed bytes would still - // publish a different blob. - let stored_blobs: BTreeSet = - std::fs::read_dir(temp.path().join("blobs").join(stelae::Digest::ALGORITHM)) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - - for (stored, reproduced) in stored - .layers - .iter() - .zip([&reproduced_notes, &reproduced_index]) - { - assert_eq!( - *stored, reproduced.descriptor, - "{}: the descriptor a publish would have written", - stored.kind, - ); - - assert!( - stored_blobs.contains(&reproduced.digests.blob_digest.to_hex()), - "{}: the reproduction named a blob the directory does not hold ({})", - stored.kind, - reproduced.digests.blob_digest, - ); - - let on_disk = std::fs::metadata( - temp.path() - .join("blobs") - .join(stelae::Digest::ALGORITHM) - .join(reproduced.digests.blob_digest.to_hex()), - ) - .unwrap() - .len(); - - assert_eq!( - reproduced.digests.compressed_size, on_disk, - "{}: the compressed size only exists if zstd ran", - stored.kind, - ); - } - - // And the identity, over a document assembled exactly as `write_stele` - // assembles it. - let mut inscription = Inscription::new( - &ToyProfile, - 3, - json!({"chapter": 3, "shelf": "east", "curator": {"name": "example", "since": 1998}}), - json!({"noteWidth": 40, "titleOrder": "byte"}), - Compression { - algo: "zstd".to_owned(), - level: COMPRESSION_LEVEL as i64, - }, - ); - - inscription.history = stored.history.clone(); - inscription.layers = vec![ - reproduced_notes.descriptor.clone(), - reproduced_index.descriptor.clone(), - ]; - - assert_eq!( - Discarding.seal(&ToyProfile, &inscription).unwrap(), - stored_digest, - ); - - // Nothing was written anywhere on the way: the only stele on disk is the - // one the directory wrote, and it has exactly its own two blobs. - assert_eq!(stored_blobs.len(), 2); -} - -/// The same vendor, one kind ahead: `dev.example.toy` after it started -/// publishing cover art. Same profile name and same major version, because an -/// *additive* kind is exactly the change that does not break a reader — which -/// is the claim the tests below check rather than assume. -struct FutureToyProfile; - -impl Profile for FutureToyProfile { - fn name(&self) -> &str { - PROFILE_NAME - } - - fn version(&self) -> u64 { - 1 - } - - fn kinds(&self) -> &[&str] { - &["notes", "index", "covers"] - } - - fn layer_media_type(&self, kind: &str) -> Result { - match kind { - "covers" => Ok(COVERS_MEDIA_TYPE.to_owned()), - other => ToyProfile.layer_media_type(other), - } - } - - fn tag_for_sequence(&self, sequence: u64) -> Result { - ToyProfile.tag_for_sequence(sequence) - } -} - -/// Write the toy stele, then have the newer publisher add its `covers` layer -/// and re-seal. -/// -/// Through the ordinary writer and the ordinary seal, so what the assertions -/// read back is a stele somebody could have published — not an inscription with -/// a descriptor pasted into it, which would prove nothing about the layer being -/// real. -fn published_ahead(root: &std::path::Path, scope: serde_json::Value) -> Inscription { - let (mut inscription, _) = write_stele(root); - let stele = SteleDir::open(root).unwrap(); - - let header = encode(|e| { - e.map(1)?.str("chapter")?.u64(3)?; - Ok(()) - }) - .unwrap(); - - let cover = encode(|e| { - e.str("a woodcut of the east shelf")?; - Ok(()) - }) - .unwrap(); - - let written = stele - .write_layer( - &FutureToyProfile, - &LayerSpec::new("covers", header, scope), - COMPRESSION_LEVEL, - &[cover], - ) - .unwrap(); - - inscription.layers.push(written.descriptor); - stele.seal(&FutureToyProfile, &inscription).unwrap(); - - inscription -} - -/// The blast radius of an additive kind, in one test. -/// -/// A profile that gains a kind publishes it as a new media type on a new layer. -/// If an older reader refused the whole stele over it, every additive change -/// would brick every deployed reader; so the reader takes the document, keeps -/// the layers it models, and *reports* the one it does not. The publish side -/// keeps the old rule, because a publisher attests every layer it lists. -#[test] -fn an_unknown_layer_kind_is_skippable_at_read_and_refused_at_publish() { - let temp = tempfile::tempdir().unwrap(); - let ahead = published_ahead(temp.path(), json!({"chapter": 3})); - - // The older reader takes the document. - ahead.check_profile(&ToyProfile).unwrap(); - - // And the layer it cannot model comes back whole — kind and scope — which - // is what leaves the skip-or-refuse decision with the profile rather than - // with the protocol. - let unknown = ahead.unknown_layers(&ToyProfile); - assert_eq!(unknown.len(), 1); - assert_eq!(unknown[0].kind, "covers"); - assert_eq!(unknown[0].scope, json!({"chapter": 3})); - - // The publisher that wrote it skips nothing, and may chain onto it. - assert!(ahead.unknown_layers(&FutureToyProfile).is_empty()); - ahead.check_profile_strict(&FutureToyProfile).unwrap(); - - // The older binary may not. - let err = ahead.check_profile_strict(&ToyProfile).unwrap_err(); - assert!( - matches!(&err, Error::UnknownLayerKind { kind, .. } if kind == "covers"), - "{err:?}" - ); - - // Skipping is about consumption and nothing else: the layers the old reader - // does model are still reachable through the identities the inscription - // pins, and the skipped one is still a layer of the stele. - let stele = SteleDir::open(temp.path()).unwrap(); - let index = stele.blob_index().unwrap(); - assert_eq!(index.len(), 3); - - for kind in ["notes", "index"] { - let descriptor = ahead.layers_of_kind(kind).next().unwrap(); - stele.read_layer(&index, &ToyProfile, descriptor).unwrap(); - } -} - -/// `required: true` in a layer's scope is a publisher telling older readers -/// that this layer is not optional: refuse the stele rather than restore a -/// partial one. -/// -/// The protocol never reads the flag — a scope is profile-owned and opaque — so -/// the planner below is the whole of the profile side, written out to show how -/// little `unknown_layers` leaves it to do. -#[test] -fn a_required_unknown_layer_is_the_profiles_own_refusal() { - fn plan(inscription: &Inscription) -> Result, String> { - let unknown = inscription.unknown_layers(&ToyProfile); - - match unknown - .iter() - .find(|layer| layer.scope.get("required") == Some(&json!(true))) - { - Some(layer) => Err(format!("{} is required: {}", layer.kind, layer.scope)), - None => Ok(unknown.iter().map(|layer| layer.kind.clone()).collect()), - } - } - - let optional = tempfile::tempdir().unwrap(); - let skipped = plan(&published_ahead(optional.path(), json!({"chapter": 3}))).unwrap(); - assert_eq!(skipped, vec!["covers".to_owned()]); - - // The same stele, the same reader, one flag apart. - let required = tempfile::tempdir().unwrap(); - let refusal = plan(&published_ahead( - required.path(), - json!({"chapter": 3, "required": true}), - )) - .unwrap_err(); - - assert!(refusal.contains("covers"), "{refusal}"); - assert!(refusal.contains("chapter"), "{refusal}"); - - // `required` is a scope field like any other to the protocol: it neither - // makes the layer known nor stops the document being read. - let stele = SteleDir::open(required.path()).unwrap(); - stele - .read_inscription() - .unwrap() - .check_profile(&ToyProfile) - .unwrap(); -} diff --git a/src/bin/dolos/snapshot/mod.rs b/src/bin/dolos/snapshot/mod.rs index 76f50fa91..6330de3a3 100644 --- a/src/bin/dolos/snapshot/mod.rs +++ b/src/bin/dolos/snapshot/mod.rs @@ -1,7 +1,7 @@ //! Publishing this node's data as a Stelae snapshot. //! //! Dolos's own word is "snapshot"; the protocol's is "stele". The translation -//! happens here and nowhere else — see `solution/stelae` and +//! happens here and nowhere else — see `crates/snapshot/PROFILE.md` and //! `adrs/004_stelae_snapshots.md`. //! //! `publish` writes one; `digest` says what one *would* be; `verify` checks a