diff --git a/.github/workflows/sign-hashlist.yml b/.github/workflows/sign-hashlist.yml new file mode 100644 index 00000000..3d6a16f5 --- /dev/null +++ b/.github/workflows/sign-hashlist.yml @@ -0,0 +1,126 @@ +name: sign-hashlist + +# The single writer of everything src/hashes/ serves. It derives what it signs: a run fetches new +# flux commits from the official repository, hashes their trees itself, and publishes list, signed +# document and provenance in one commit. Requests carry pointers, never hash values -- flux CI's +# dispatch token has Actions permission only. That bounds the credential rather than vouching for +# the tree: a fork network shares one object store, so a dispatched commit may be any commit ever +# pushed to flux or to a public fork of it. scripts/sign-hashlist.js states the bound in full. +# +# Level-triggered: every run reconciles the full delta between the flux remote's refs and the +# snapshot in the provenance record, so a dispatch lost to the concurrency group's +# newest-pending-wins cancellation is repaired by whichever run survives, and the daily sweep +# bounds the tail when nothing follows. +# +# Deliberately NOT triggered by pull_request_target or pull_request: this repository is public and +# either would expose the secrets to a fork. The push trigger watches only the human-edited ledger, +# which this workflow never writes, so it cannot retrigger itself. Both secrets live in the +# environment below, whose deployment branch policy admits master only -- a branch run is refused +# before its first step. + +on: + workflow_dispatch: + inputs: + commit: + description: 'flux commit SHA to publish' + required: false + ref: + description: 'ref name the caller saw (label of last resort, never authority)' + required: false + ref_type: + description: 'branch or tag (label of last resort)' + required: false + claimed_hash: + description: 'tree hash the caller computed -- a tripwire, never an input to the list' + required: false + push: + branches: [master] + paths: ['src/hashes/ledger.json'] + schedule: + - cron: '43 3 * * *' + +# The push happens over SSH with the deploy key -- the ruleset's one bypass -- so the run token +# needs read only. +permissions: + contents: read + +# Two runs signing at once would both read the same sequence, and one would publish over the other +# under a sequence already used. +concurrency: + group: sign-hashlist + cancel-in-progress: false + +jobs: + sign: + runs-on: ubuntu-latest + environment: hashlist-signing + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '20' + + # Reconcile, sign, verify, publish -- as one step, because the push can race a human PR + # merge, and recovering means re-syncing to origin/master and reconciling again from scratch. + # + # The verification runs against the published public keys, not the signing key: a mangled + # secret produces a well-formed document that no consumer will accept, and verifying here + # makes that a red run rather than a document that looks published and satisfies nobody. + - name: Reconcile, sign and publish + env: + HASHLIST_SIGNING_SEED_B64: ${{ secrets.HASHLIST_SIGNING_SEED_B64 }} + HASHLIST_DEPLOY_KEY: ${{ secrets.HASHLIST_DEPLOY_KEY }} + DISPATCH_COMMIT: ${{ inputs.commit }} + DISPATCH_REF: ${{ inputs.ref }} + DISPATCH_REF_TYPE: ${{ inputs.ref_type }} + DISPATCH_CLAIMED_HASH: ${{ inputs.claimed_hash }} + run: | + umask 077 + printf '%s\n' "$HASHLIST_DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" + printf 'github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n' > "$RUNNER_TEMP/known_hosts" + export GIT_SSH_COMMAND="ssh -F /dev/null -i $RUNNER_TEMP/deploy_key -o IdentitiesOnly=yes -o UserKnownHostsFile=$RUNNER_TEMP/known_hosts" + + git config user.email 'runonfluxbot@gmail.com' + git config user.name 'hashlist-signer' + + LIST=src/hashes/hashes.js + SIGNED=src/hashes/hashlist-signed.json + PROVENANCE=src/hashes/provenance.json + + for attempt in 1 2 3; do + git fetch --quiet origin master + git reset --quiet --hard origin/master + + RESULT=$(node scripts/sign-hashlist.js) + case "$RESULT" in + changed=false) + echo 'nothing to publish' + exit 0 + ;; + changed=state) + node scripts/validate.js + git add "$PROVENANCE" + git commit --quiet -m 'Reconcile flux refs' -- "$PROVENANCE" + ;; + changed=signed) + node scripts/verify-hashlist.js + node scripts/validate.js + git add "$LIST" "$SIGNED" "$PROVENANCE" + SEQ=$(node -p "JSON.parse(Buffer.from(require('./$SIGNED').payload_b64,'base64')).seq") + git commit --quiet -m "Sign hash list seq $SEQ" -- "$LIST" "$SIGNED" "$PROVENANCE" + ;; + *) + echo "unexpected reconciler output: $RESULT" + exit 1 + ;; + esac + + if git push --quiet "git@github.com:${GITHUB_REPOSITORY}.git" HEAD:master; then + echo "published ($RESULT)" + exit 0 + fi + echo 'push raced a human merge, retrying' + done + + echo 'could not publish after 3 attempts' + exit 1 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..7fee1301 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,56 @@ +name: validate + +# The published list is served by requiring it, so a file that does not load takes the endpoint +# down. The signer runs validate.js itself before pushing; this run covers human PRs and stands as +# the live tripwire on master -- the signer's deploy-key pushes trigger it, so every signing commit +# gets a green check and a red one always means something real. + +on: + pull_request: + push: + branches: [master] + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + # The outputs guard below diffs the merge commit against its first parent. + fetch-depth: 2 + - uses: actions/setup-node@v7 + with: + node-version: '20' + + # The three outputs are generated by the signer; edit the ledger instead. This catches + # honest mistakes, not attacks: pull_request runs the PR head's copy of this workflow, so a + # hostile PR could weaken the check it is judged by. The control against a hostile PR is + # required review on the ruleset -- this step just makes the mistake loud before a human + # looks. + # + # HEAD is the PR merge commit and HEAD^1 is the base branch as it stands NOW, so this sees + # exactly what merging the PR would change. (The event payload's base.sha is the base at the + # last synchronize -- stale on any PR whose base has since moved, and the signer moves master + # daily, so diffing against it flags every hash published since the PR was opened.) + - name: Outputs are generated, not edited + if: github.event_name == 'pull_request' + run: | + CHANGED=$(git diff --name-only HEAD^1 HEAD -- \ + src/hashes/hashes.js src/hashes/hashlist-signed.json src/hashes/provenance.json) + if [ -n "$CHANGED" ]; then + echo 'these files are generated by the signing workflow; edit src/hashes/ledger.json instead:' + echo "$CHANGED" + exit 1 + fi + + - run: node scripts/validate.js + + # The reconciler's own logic, driven against a local fake flux remote: bootstrap, derivation, + # claimed-hash mismatch, culls, duplicates, corruption, a wiped document, v1 compatibility, + # and the empty-tree hash. No network and no secrets -- it builds its own git remote in a + # temp dir, so it runs on fork pull requests like everything else here. + - name: Reconciler tests + run: npm test diff --git a/SIGNING.md b/SIGNING.md new file mode 100644 index 00000000..e205cfe1 --- /dev/null +++ b/SIGNING.md @@ -0,0 +1,115 @@ +# Signing the hash list + +`src/hashes/hashlist-signed.json` is the list this repository publishes, signed with Ed25519 so consumers can verify +it came from us rather than trusting the transport or whatever relayed it. + +It is published **alongside** `src/hashes/hashes.js`, not instead of it. Both are served. + +## The document + +`{ payload_b64, sig_b64 }`, where the payload is the exact signed bytes — a JSON object +`{ seq, issued_at, hashes }`. The signature covers the transmitted bytes, so verification never +depends on signer and verifier agreeing about JSON key order or whitespace. + +`seq` increases by one per signing run and never restarts. Its high-water mark is recorded in the +provenance record beside the document, and the signer takes the next sequence from whichever of the +two is higher — so losing the document, however that happens, does not reset the sequence. +`validate.js` refuses a document whose sequence is not exactly the recorded high-water. + +## The provenance record + +`src/hashes/provenance.json`, outside the signed payload, with **one writer — the signing +workflow**, which derives every post-cutover entry itself from commits it fetches from +`RunOnFlux/flux`: + +- a **row per listed hash** — `published` date, `commit`, `branch`, `tag`, `derived`. This is what + makes an entry attributable later: the list itself is opaque md5s, and the commit that produced + an entry can stop existing (a force-push, a branch deleted after merge). First attribution wins; + a new tag on a known commit annotates the existing row. Rows with `derived: false` predate + derivation (grandfathered at cutover) and have no commit to point at. **Read the signature accordingly**: + it says these are the hashes this repository published at this sequence, and for `derived: true` + rows it additionally means the signer fetched that commit and computed that hash itself. It + attests no provenance for grandfathered rows — those were carried across at cutover from the + unsigned list, on the authority of whatever published them at the time. +- a **commits map** (`sha → hash`) so nothing is fetched or hashed twice, and a **refs snapshot** + of the flux remote, which is what the reconciler diffs against — a publication request that gets + lost is repaired by the next run reconciling the full delta. +- **`signed`** — the sequence high-water and `issued_at`, stamped in the same commit as the signed + document. + +`src/hashes/ledger.json` is the one human-edited input: cull marks, reviewed through PRs. The +signer reads it and never writes it. The three output files are generated — `validate` fails any +PR that edits them directly. + +## Keys + +Consumers pin a set of public keys and accept a document signed by any one of them, so a second key +can take over without those consumers needing an update. + +| key | public key (raw ed25519, hex) | custody | use | +|---|---|---|---| +| 1 | `14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa` | CI, secret `HASHLIST_SIGNING_SEED_B64` in the `hashlist-signing` environment | day to day | +| 2 | `fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8` | cold, offline | continuity only | + +### Key 1 + +Generated 2026-08-24 straight into the secret `HASHLIST_SIGNING_SEED_B64`, which lives in the +`hashlist-signing` GitHub Environment whose deployment branch policy admits `master` only — a +workflow run on any other ref is refused before its first step, so a branch push cannot read it. +(This replaced the 2026-08-14 key, which had only ever lived as a repository-level secret: a +secret's value cannot be moved into an environment, and nothing had ever consumed the old key, so +regenerating was free.) +**There is no copy of the private half anywhere else, on purpose** — key 2 covers its loss, and a +second copy would only widen where it can leak from. + +To replace it, generate a new one the same way: + +```sh +node -e ' +const crypto = require("crypto"); +const seed = crypto.randomBytes(32); +const key = crypto.createPrivateKey({ + key: Buffer.concat([Buffer.from("302e020100300506032b657004220420","hex"), seed]), + format: "der", type: "pkcs8", +}); +process.stderr.write("public_key_hex=" + crypto.createPublicKey(key) + .export({format:"der", type:"spki"}).subarray(12).toString("hex") + "\n"); +process.stdout.write(seed.toString("base64")); +' | gh secret set HASHLIST_SIGNING_SEED_B64 --repo RunOnFlux/fluxhashes +``` + +The seed goes down the pipe and is never printed or written to disk. Put the printed public key in +the table above and in `scripts/verify-hashlist.js`. + +### Key 2 + +Generated offline, private half never on a networked machine, stored with the release signing +material. Not used in normal operation. + +Its purpose is continuity: without a second key, losing key 1 would mean nothing new could be +published until consumers were updated with a replacement. + +It does not provide revocation — removing a key from the pinned set requires updating consumers. +Two keys held in the same place buy nothing; the separation is the point. + +### The deploy key + +The signer pushes to `master` over SSH with a write deploy key, `HASHLIST_DEPLOY_KEY` in the same +environment — the one bypass on the master ruleset, which otherwise admits only reviewed PRs with +`validate` green (repository admins included). The run's own `GITHUB_TOKEN` stays read-only. To +rotate: generate a fresh keypair, replace the repository deploy key and the environment secret; +the ruleset's `DeployKey` bypass covers whatever write keys the repository holds, so it needs no +change — which is also why the repository must hold exactly this one write deploy key. + +The workflow also **pins GitHub's SSH host key** — a single `ssh-ed25519` line written to a +`known_hosts` file for the push, rather than trusting whatever `ssh-keyscan` returns at run time. +It is a rotation touchpoint: if GitHub ever rotates that key the push fails host verification and +the signing run goes red, which looks like a credential fault and is not one. The fix is to update +the pinned line in `.github/workflows/sign-hashlist.yml` against GitHub's published fingerprints. + +## Trust + +Anyone who can land a workflow change on `master` can read the secrets; a GitHub secret is an +access-controlled environment variable, not a vault. Environment scoping means landing that change +requires a reviewed merge — a branch push is no longer enough. Secrets are not passed to workflows +triggered by a pull request from a fork, which matters because this repository is public. diff --git a/package.json b/package.json index 3b2499fa..b7e94925 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "start": "nodemon index.js", - "test": "echo 'No tests have been implemented' && exit 1", + "test": "bash test/reconciler/run-tests.sh", "lint": "eslint ./ --fix" }, "author": "Tadeas Kmenta", diff --git a/scripts/sign-hashlist.js b/scripts/sign-hashlist.js new file mode 100644 index 00000000..71d101e9 --- /dev/null +++ b/scripts/sign-hashlist.js @@ -0,0 +1,420 @@ +#!/usr/bin/env node + +// The single writer of everything src/hashes/ serves. +// +// Reconciles the published list against RunOnFlux/flux itself: diffs the remote's refs against the +// snapshot in the provenance record, fetches each new commit, computes its ZelBack tree hash from +// the bytes it fetched, and emits hashes.js, the signed document and the provenance record together. +// A dispatch carries pointers, never hash values, so the credential that sends one needs no write +// authority -- and the signature means "this commit's tree hashes to this value", not "this was in +// the repository when I ran". +// +// That bounds the credential; it does not vouch for the tree. GitHub shares one object store across +// a fork network, so every commit ever pushed to any public fork of flux stays anonymously fetchable +// by SHA from the official remote -- and a dispatched commit matching no ref is still derived, on +// purpose, because a branch can move past a commit before we look. So a dispatch is trusted to name +// a commit reachable in flux's object store, forks included, which is a much larger set than "a tree +// RunOnFlux authored". The credential is held only by principals who can already land a branch on +// flux and get a hash listed that way; this is the same trust boundary, not a defence against it. +// +// The payload is signed and transmitted as exact bytes in base64, so verification never depends on +// the signer and the verifier agreeing about JSON key order or whitespace. +// +// stdout carries exactly one line -- changed=false, changed=state or changed=signed -- read by the +// workflow to decide what to commit. Everything human goes to stderr. + +const { execFileSync } = require('child_process'); +const crypto = require('crypto'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const ROOT = path.join(__dirname, '..'); +const LIST = path.join(ROOT, 'src', 'hashes', 'hashes.js'); +const OUTPUT = path.join(ROOT, 'src', 'hashes', 'hashlist-signed.json'); +const PROVENANCE = path.join(ROOT, 'src', 'hashes', 'provenance.json'); +const LEDGER = path.join(ROOT, 'src', 'hashes', 'ledger.json'); + +const FLUX_REMOTE = process.env.FLUX_REMOTE || 'https://github.com/RunOnFlux/flux'; + +// fluxbench's pipeline, byte for byte -- flux CI's Check Hash step runs the same one. The awk +// strips filenames before the sort, so the hash depends only on the multiset of file contents; +// LC_ALL=C pins the sort. +const TREE_HASH_PIPELINE = "find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}'"; + +// md5 of an empty stream. The pipeline yields it whenever nothing was hashed -- ZelBack absent +// (find errors; without pipefail the status is awk's, so the failure is otherwise invisible) or +// present but holding no regular files (find succeeds and emits nothing, which pipefail cannot +// see). Both must be refused: the value is a well-formed 32-hex hash that means "a node whose +// ZelBack contains no files is genuine FluxOS", and membership is monotonic, so listing it once +// costs a cull PR to undo. +const EMPTY_TREE_HASH = 'd41d8cd98f00b204e9800998ecf8427e'; + +// A raw 32-byte Ed25519 seed is not directly importable; Node wants PKCS8. The prefix is fixed for +// the algorithm, so prepending it is enough. +const PKCS8_ED25519_PREFIX = Buffer.from('302e020100300506032b657004220420', 'hex'); +const SPKI_ED25519_PREFIX_LENGTH = 12; + +function privateKeyFromSeed(seedB64) { + const seed = Buffer.from(seedB64, 'base64'); + if (seed.length !== 32) { + throw new Error(`signing seed must be 32 bytes, got ${seed.length}`); + } + return crypto.createPrivateKey({ + key: Buffer.concat([PKCS8_ED25519_PREFIX, seed]), + format: 'der', + type: 'pkcs8', + }); +} + +// The raw 32 bytes consumers pin, rather than any DER wrapping around them. +function rawPublicKey(privateKey) { + const spki = crypto.createPublicKey(privateKey).export({ format: 'der', type: 'spki' }); + return spki.subarray(SPKI_ED25519_PREFIX_LENGTH); +} + +function buildSignedDocument(seq, issuedAt, hashes, privateKey) { + if (!Number.isInteger(seq) || seq < 1) { + throw new Error('seq must be a positive integer'); + } + if (typeof issuedAt !== 'string' || Number.isNaN(Date.parse(issuedAt))) { + throw new Error('issued_at must be a parseable date string'); + } + if (!Array.isArray(hashes) || hashes.length === 0) { + throw new Error('hashes must be a non-empty array'); + } + if (!hashes.every((h) => typeof h === 'string' && /^[0-9a-f]{32}$/.test(h))) { + throw new Error('every hash must be a lowercase 32-character md5'); + } + + const payload = Buffer.from(JSON.stringify({ seq, issued_at: issuedAt, hashes }), 'utf8'); + const signature = crypto.sign(null, payload, privateKey); + + return { + payload_b64: payload.toString('base64'), + sig_b64: signature.toString('base64'), + }; +} + +function previousDocument() { + if (!fs.existsSync(OUTPUT)) { + return null; + } + const document = JSON.parse(fs.readFileSync(OUTPUT, 'utf8')); + return JSON.parse(Buffer.from(document.payload_b64, 'base64').toString('utf8')); +} + +// The provenance record is this writer's own state: attribution rows, the commit-to-hash map, the +// refs snapshot the reconciler diffs against, and the highest sequence ever signed. Absence is the +// state before the first run and starts the cutover bootstrap. A record that exists but does not +// parse is a red run: treating corruption as absence is exactly the restart this exists to prevent. +function readProvenance() { + if (!fs.existsSync(PROVENANCE)) { + return null; + } + const record = JSON.parse(fs.readFileSync(PROVENANCE, 'utf8')); + if (record.signed !== undefined && record.signed !== null + && (!Number.isInteger(record.signed.seq) || record.signed.seq < 1)) { + throw new Error(`provenance signed.seq is not a positive integer: ${record.signed.seq}`); + } + return record; +} + +// The ledger is the human-edited input: cull marks that take entries out of the list through a +// reviewed PR. The signer reads it and never writes it, which is what lets the workflow trigger on +// pushes to it without ever retriggering itself. Malformed is a red run, not a skip. +function readLedger() { + if (!fs.existsSync(LEDGER)) { + return { culls: [] }; + } + const ledger = JSON.parse(fs.readFileSync(LEDGER, 'utf8')); + if (!Array.isArray(ledger.culls)) { + throw new Error('ledger.culls is not an array'); + } + ledger.culls.forEach((cull) => { + if (!cull || typeof cull.hash !== 'string' || !/^[0-9a-f]{32}$/.test(cull.hash)) { + throw new Error(`ledger cull without a valid hash: ${JSON.stringify(cull)}`); + } + }); + return ledger; +} + +function git(args, options = {}) { + return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...options }); +} + +// refs/heads/* as listed; refs/tags/* resolved to the commit they point at (the ^{} peel when the +// tag is annotated). One request regardless of ref count. +function lsRemoteRefs(remote) { + const refs = {}; + const peeled = {}; + git(['ls-remote', '--heads', '--tags', remote]).split('\n').filter(Boolean).forEach((line) => { + const [sha, ref] = line.split('\t'); + if (ref.endsWith('^{}')) { + peeled[ref.slice(0, -3)] = sha; + } else { + refs[ref] = sha; + } + }); + Object.entries(peeled).forEach(([ref, sha]) => { refs[ref] = sha; }); + return refs; +} + +function ensureCacheRepo() { + const dir = process.env.FLUX_CACHE_DIR || path.join(os.tmpdir(), 'flux-hash-cache'); + if (!fs.existsSync(path.join(dir, '.git'))) { + fs.mkdirSync(dir, { recursive: true }); + git(['init', '--quiet'], { cwd: dir }); + } + return dir; +} + +// Fetch the named commit from the official remote and hash the pristine tree in a throwaway +// worktree. Nothing from the fetched tree is ever executed; the pipeline only reads bytes. +function deriveTreeHash(cache, sha) { + git(['fetch', '--quiet', '--depth', '1', FLUX_REMOTE, sha], { cwd: cache }); + const parent = fs.mkdtempSync(path.join(os.tmpdir(), 'flux-tree-')); + const worktree = path.join(parent, 'wt'); + try { + git(['worktree', 'add', '--quiet', '--detach', worktree, sha], { cwd: cache }); + // -o pipefail rather than folding it into the constant, so TREE_HASH_PIPELINE stays byte-identical + // to the one flux CI runs -- that identity is what makes claimed_hash comparable at all. + const hash = execFileSync('bash', ['-o', 'pipefail', '-c', TREE_HASH_PIPELINE], { cwd: worktree, encoding: 'utf8' }).trim(); + if (!/^[0-9a-f]{32}$/.test(hash)) { + throw new Error(`tree hash pipeline produced "${hash}"`); + } + if (hash === EMPTY_TREE_HASH) { + throw new Error('nothing was hashed -- this commit has no ZelBack files. Refusing to list the empty-tree hash'); + } + return hash; + } finally { + try { + git(['worktree', 'remove', '--force', worktree], { cwd: cache }); + } catch (error) { /* the worktree was never created */ } + fs.rmSync(parent, { recursive: true, force: true }); + } +} + +function writeList(hashes) { + const lines = ['function getHashes() {', ' return [']; + hashes.forEach((hash) => lines.push(` '${hash}',`)); + lines.push(' ];', '}', '', 'module.exports = {', ' getHashes,', '};', ''); + fs.writeFileSync(LIST, lines.join('\n')); +} + +function refLabel(ref) { + return ref.startsWith('refs/heads/') + ? { branch: ref.slice('refs/heads/'.length), tag: null } + : { branch: null, tag: ref.slice('refs/tags/'.length) }; +} + +function dispatchLabel() { + const ref = process.env.DISPATCH_REF || null; + if (!ref) { + return { branch: null, tag: null }; + } + return process.env.DISPATCH_REF_TYPE === 'tag' ? { branch: null, tag: ref } : { branch: ref, tag: null }; +} + +function main() { + const seedB64 = process.env.HASHLIST_SIGNING_SEED_B64; + if (!seedB64) { + throw new Error('HASHLIST_SIGNING_SEED_B64 is not set'); + } + + // eslint-disable-next-line global-require + const currentList = require('../src/hashes/hashes').getHashes(); + const previous = previousDocument(); + const provenance = readProvenance() || {}; + const ledger = readLedger(); + const before = JSON.stringify(provenance); + + const today = new Date().toISOString().slice(0, 10); + const rows = provenance.hashes || {}; + const commits = provenance.commits || {}; + const snapshot = provenance.refs || null; + + const current = lsRemoteRefs(FLUX_REMOTE); + process.stderr.write(`flux remote: ${Object.keys(current).length} refs\n`); + + // Cutover bootstrap: no snapshot means nothing has been derived yet. Grandfather everything + // already listed and snapshot the remote as it stands, so only movement from here on is derived. + if (!snapshot) { + currentList.forEach((hash) => { + if (!rows[hash]) { + rows[hash] = { + published: today, commit: null, branch: null, tag: null, derived: false, + }; + } else if (rows[hash].derived === undefined) { + rows[hash].derived = false; + } + }); + process.stderr.write(`bootstrap: grandfathered ${currentList.length} entries, snapshotting ${Object.keys(current).length} refs\n`); + } + + // The work set: every ref that moved since the snapshot, keyed by commit. A tag riding a commit + // that is also a branch tip annotates rather than duplicates. + const work = new Map(); + if (snapshot) { + Object.entries(current).forEach(([ref, sha]) => { + if (snapshot[ref] === sha) return; + // A tag that MOVED keeps its original attribution; only genuinely new tags join the set. + if (ref.startsWith('refs/tags/') && snapshot[ref] !== undefined) return; + const label = refLabel(ref); + if (!work.has(sha)) { + work.set(sha, label); + } else { + // A commit arriving as branch tip and tag in the same run keeps both labels, whichever + // ref was seen first. + const existing = work.get(sha); + if (label.tag && !existing.tag) existing.tag = label.tag; + if (label.branch && !existing.branch) existing.branch = label.branch; + } + }); + } + + const dispatchCommit = (process.env.DISPATCH_COMMIT || '').toLowerCase() || null; + const claimed = (process.env.DISPATCH_CLAIMED_HASH || '').toLowerCase() || null; + if (dispatchCommit && !/^[0-9a-f]{40}$/.test(dispatchCommit)) { + throw new Error(`dispatched commit is not a 40-character sha: ${dispatchCommit}`); + } + if (dispatchCommit && !work.has(dispatchCommit)) { + // Label from our own view of the remote first; the dispatch's ref fields are a fallback label + // for a ref that moved past the commit before we looked, never authority. + const ownRef = Object.keys(current).find((ref) => current[ref] === dispatchCommit); + work.set(dispatchCommit, ownRef ? refLabel(ownRef) : dispatchLabel()); + } + if (dispatchCommit && claimed && commits[dispatchCommit] && commits[dispatchCommit] !== claimed) { + throw new Error(`claimed hash ${claimed} does not match ${commits[dispatchCommit]} already derived for ${dispatchCommit}`); + } + + // Derive. A failed fetch of a dispatched commit is a red run -- the caller named a commit the + // official repository will not serve. A failed fetch from the ref sweep keeps that ref's old + // snapshot entry, so the next run retries it. + const failedRefs = new Set(); + const cache = ensureCacheRepo(); + work.forEach((label, sha) => { + if (commits[sha]) { + const row = rows[commits[sha]]; + if (row && label.tag && !row.tag) { + row.tag = label.tag; + } + return; + } + let hash; + try { + hash = deriveTreeHash(cache, sha); + } catch (error) { + if (sha === dispatchCommit) { + throw new Error(`dispatched commit ${sha}: ${error.message}`); + } + process.stderr.write(`skipping ${sha}: ${error.message}\n`); + Object.entries(current).forEach(([ref, s]) => { if (s === sha) failedRefs.add(ref); }); + return; + } + if (sha === dispatchCommit && claimed && claimed !== hash) { + throw new Error(`claimed hash ${claimed} does not match derived ${hash} for ${sha} -- environment drift or a hostile dispatch, either must be loud`); + } + commits[sha] = hash; + if (!rows[hash]) { + // First attribution wins: a hash republished from another branch keeps its original row. + rows[hash] = { + published: today, commit: sha, branch: label.branch, tag: label.tag, derived: true, + }; + process.stderr.write(`derived ${hash} from ${sha.slice(0, 9)} (${label.tag || label.branch || 'unlabelled'})\n`); + } else if (label.tag && !rows[hash].tag) { + rows[hash].tag = label.tag; + } + }); + + const nextRefs = {}; + Object.entries(current).forEach(([ref, sha]) => { + if (failedRefs.has(ref)) { + if (snapshot && snapshot[ref] !== undefined) { + nextRefs[ref] = snapshot[ref]; + } + return; + } + nextRefs[ref] = sha; + }); + + // Membership: what is listed, minus culls, plus everything derived that is not yet listed. + const culled = new Set(ledger.culls.map((cull) => cull.hash)); + const listed = new Set(currentList); + const retained = currentList.filter((hash) => !culled.has(hash)); + const additions = Object.keys(rows).filter( + (hash) => rows[hash].derived === true && !listed.has(hash) && !culled.has(hash), + ); + const newList = retained.concat(additions); + + // Monotonicity: an entry leaves the list only when the ledger says so. The append-with-cull + // construction above cannot trip this today -- it exists so that a future change to true + // regeneration-from-rows turns a dropped entry into a red run, never a signed loss + // (mutation-tested: it catches exactly that). + const surviving = new Set(newList); + currentList.forEach((hash) => { + if (!surviving.has(hash) && !culled.has(hash)) { + throw new Error(`entry ${hash} would vanish without a ledger cull -- refusing to publish`); + } + }); + if (newList.length === 0) { + throw new Error('refusing to publish an empty list'); + } + if (surviving.size !== newList.length) { + throw new Error('the regenerated list contains duplicates -- refusing to publish'); + } + + const listChanged = newList.length !== currentList.length + || newList.some((hash, i) => hash !== currentList[i]); + // No document has ever been published: the bootstrap run signs even an unchanged list. This is + // also the first proof that the stored seed matches the pinned key. + const mustSign = !previous; + + provenance.hashes = rows; + provenance.commits = commits; + provenance.refs = nextRefs; + + if (!listChanged && !mustSign) { + if (JSON.stringify(provenance) === before) { + process.stderr.write(`nothing changed at seq ${previous.seq}\n`); + process.stdout.write('changed=false\n'); + return; + } + // Attribution, tag annotations or the snapshot advanced with the membership intact: worth a + // commit, not worth a sequence. + fs.writeFileSync(PROVENANCE, `${JSON.stringify(provenance, null, 2)}\n`); + process.stderr.write(`state advanced, membership unchanged at seq ${previous.seq}\n`); + process.stdout.write('changed=state\n'); + return; + } + + const highWater = Math.max( + previous ? previous.seq : 0, + provenance.signed ? provenance.signed.seq : 0, + ); + const seq = highWater + 1; + const issuedAt = new Date().toISOString(); + const privateKey = privateKeyFromSeed(seedB64); + const document = buildSignedDocument(seq, issuedAt, newList, privateKey); + provenance.signed = { seq, issued_at: issuedAt }; + + writeList(newList); + fs.writeFileSync(OUTPUT, `${JSON.stringify(document, null, 2)}\n`); + fs.writeFileSync(PROVENANCE, `${JSON.stringify(provenance, null, 2)}\n`); + process.stderr.write(`signed seq ${seq} over ${newList.length} hashes (${additions.length} added, ${currentList.length - retained.length} culled)\n`); + process.stderr.write(`public key (raw, hex): ${rawPublicKey(privateKey).toString('hex')}\n`); + process.stdout.write('changed=signed\n'); +} + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`sign-hashlist: ${error.message}\n`); + process.exit(1); + } +} + +module.exports = { + privateKeyFromSeed, rawPublicKey, buildSignedDocument, readProvenance, readLedger, +}; diff --git a/scripts/validate.js b/scripts/validate.js new file mode 100644 index 00000000..3d309a40 --- /dev/null +++ b/scripts/validate.js @@ -0,0 +1,211 @@ +#!/usr/bin/env node + +// Shape-checks what this repository publishes. +// +// The list is served by requiring it, so a file that does not load takes the endpoint down rather +// than merely publishing something odd. The three outputs have one writer -- the signer, which +// runs this as a self-check before pushing -- so no legitimate commit can fail here: a red +// validate on master always means the generator or the repository rules are broken, which is the +// point of running it everywhere. +// +// This checks shape only. Whether a particular hash *should* be listed is not knowable from here: +// removing one that is still in use looks identical to removing one that is obsolete. + +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.join(__dirname, '..'); +const SIGNED = path.join(ROOT, 'src', 'hashes', 'hashlist-signed.json'); +const PROVENANCE = path.join(ROOT, 'src', 'hashes', 'provenance.json'); +const LEDGER = path.join(ROOT, 'src', 'hashes', 'ledger.json'); + +const failures = []; + +function check(condition, message) { + if (!condition) failures.push(message); +} + +function validateHashes() { + // eslint-disable-next-line global-require + const hashes = require('../src/hashes/hashes').getHashes(); + + check(Array.isArray(hashes), 'hashes.js did not return an array'); + if (!Array.isArray(hashes)) return null; + + check(hashes.length > 0, 'the list is empty'); + + const malformed = hashes.filter((hash) => typeof hash !== 'string' || !/^[0-9a-f]{32}$/.test(hash)); + check(malformed.length === 0, `${malformed.length} entries are not lowercase md5s: ${malformed.slice(0, 3)}`); + + // Harmless to serve, but a sign that an edit went in twice, which is worth seeing. + const duplicates = hashes.filter((hash, i) => hashes.indexOf(hash) !== i); + check(duplicates.length === 0, `${duplicates.length} duplicate entries: ${[...new Set(duplicates)].slice(0, 3)}`); + + process.stderr.write(`hashes.js: ${hashes.length} entries\n`); + return hashes; +} + +// The provenance record has one writer -- the signer, which owns the attribution rows, the +// commit-to-hash map, the refs snapshot its reconciler diffs against, and the sequence high-water. +// A malformed record takes the signer down, so shape is enforced here and on every PR. A +// grandfathered row (derived false) predates derivation and has no commit to point at. +function validateProvenance(hashes) { + if (!fs.existsSync(PROVENANCE)) { + process.stderr.write('no provenance record yet, skipping\n'); + return null; + } + + let record; + try { + record = JSON.parse(fs.readFileSync(PROVENANCE, 'utf8')); + } catch (error) { + check(false, `provenance record does not parse: ${error.message}`); + return null; + } + + if (record.signed !== undefined && record.signed !== null) { + check( + Number.isInteger(record.signed.seq) && record.signed.seq >= 1, + `provenance signed.seq is not a positive integer: ${record.signed.seq}`, + ); + check( + typeof record.signed.issued_at === 'string' && !Number.isNaN(Date.parse(record.signed.issued_at)), + `provenance signed.issued_at is not a parseable date: ${record.signed.issued_at}`, + ); + } + + const rows = record.hashes || {}; + Object.entries(rows).forEach(([hash, row]) => { + check(/^[0-9a-f]{32}$/.test(hash), `provenance row key is not a lowercase md5: ${hash}`); + const commitOk = row && ( + (typeof row.commit === 'string' && /^[0-9a-f]{40}$/.test(row.commit)) + || (row.commit === null && row.derived !== true) + ); + check( + row && typeof row === 'object' + && typeof row.published === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(row.published) + && commitOk + && (row.branch === null || typeof row.branch === 'string') + && (row.tag === null || typeof row.tag === 'string') + && (row.derived === undefined || typeof row.derived === 'boolean'), + `provenance row is malformed: ${hash}`, + ); + }); + + Object.entries(record.commits || {}).forEach(([sha, hash]) => { + check(/^[0-9a-f]{40}$/.test(sha), `commits map key is not a 40-character sha: ${sha}`); + check(typeof hash === 'string' && /^[0-9a-f]{32}$/.test(hash), `commits map value is not a lowercase md5: ${hash}`); + }); + Object.entries(record.refs || {}).forEach(([ref, sha]) => { + check(ref.startsWith('refs/'), `refs snapshot key is not a ref: ${ref}`); + check(typeof sha === 'string' && /^[0-9a-f]{40}$/.test(sha), `refs snapshot value is not a 40-character sha: ${ref}`); + }); + + // Post-cutover (the snapshot exists), membership is generated from the rows: a listed hash + // without a row means the generator and its record have diverged. + if (record.refs && hashes) { + const unattributed = hashes.filter((hash) => !rows[hash]); + check(unattributed.length === 0, `${unattributed.length} listed hashes have no provenance row: ${unattributed.slice(0, 3)}`); + } + + process.stderr.write(`provenance: ${Object.keys(rows).length} rows, ${Object.keys(record.commits || {}).length} commits, ${Object.keys(record.refs || {}).length} refs, signed seq ${record.signed ? record.signed.seq : 'none'}\n`); + return record; +} + +// The ledger is the human-edited input: cull marks reviewed through PRs. A cull may still be +// listed here -- the signer applies it on its next run -- so consistency with the list is not +// checkable; shape is. +function validateLedger() { + if (!fs.existsSync(LEDGER)) { + process.stderr.write('no ledger yet, skipping\n'); + return; + } + + let ledger; + try { + ledger = JSON.parse(fs.readFileSync(LEDGER, 'utf8')); + } catch (error) { + check(false, `ledger does not parse: ${error.message}`); + return; + } + + check(Array.isArray(ledger.culls), 'ledger.culls is not an array'); + (Array.isArray(ledger.culls) ? ledger.culls : []).forEach((cull, i) => { + check( + cull && typeof cull === 'object' + && typeof cull.hash === 'string' && /^[0-9a-f]{32}$/.test(cull.hash) + && typeof cull.reason === 'string' && cull.reason.length > 0 + && typeof cull.date === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(cull.date), + `ledger cull ${i} is malformed: ${JSON.stringify(cull)}`, + ); + }); + + process.stderr.write(`ledger: ${Array.isArray(ledger.culls) ? ledger.culls.length : 0} culls\n`); +} + +// The signed copy is written by CI and only exists once it has run, so its absence is not a failure +// -- unless the provenance record says a document was signed, in which case the document has gone +// missing and that must be a red run, not a skip. If it is there it must verify, it must describe +// the list beside it, and its sequence must be exactly the provenance high-water: below it is the +// restart the record exists to prevent, above it means the record missed a write. +function validateSigned(hashes, provenance) { + const recordedSeq = provenance && provenance.signed ? provenance.signed.seq : null; + + if (!fs.existsSync(SIGNED)) { + if (recordedSeq !== null) { + check(false, `provenance records signed seq ${recordedSeq} but there is no signed document`); + return; + } + process.stderr.write('no signed document yet, skipping\n'); + return; + } + + // eslint-disable-next-line global-require + const { verifyDocument, PINNED_PUBLIC_KEYS } = require('./verify-hashlist'); + + let payload; + try { + payload = verifyDocument(JSON.parse(fs.readFileSync(SIGNED, 'utf8')), PINNED_PUBLIC_KEYS); + } catch (error) { + check(false, `signed document does not verify: ${error.message}`); + return; + } + + check(Number.isInteger(payload.seq) && payload.seq >= 1, `signed sequence is not a positive integer: ${payload.seq}`); + check( + typeof payload.issued_at === 'string' && !Number.isNaN(Date.parse(payload.issued_at)), + `signed issued_at is not a parseable date: ${payload.issued_at}`, + ); + + if (recordedSeq === null) { + check(false, `signed document at seq ${payload.seq} but the provenance record has no signed seq`); + } else { + check( + payload.seq === recordedSeq, + `signed document is at seq ${payload.seq}, provenance records ${recordedSeq}`, + ); + } + + if (hashes) { + const matches = payload.hashes.length === hashes.length + && payload.hashes.every((hash, i) => hash === hashes[i]); + check(matches, `signed document lists ${payload.hashes.length} entries, hashes.js has ${hashes.length}`); + } + + process.stderr.write(`signed document: seq ${payload.seq}, ${payload.hashes.length} entries\n`); +} + +function main() { + const hashes = validateHashes(); + const provenance = validateProvenance(hashes); + validateLedger(); + validateSigned(hashes, provenance); + + if (failures.length) { + failures.forEach((failure) => process.stderr.write(` FAIL ${failure}\n`)); + process.exit(1); + } + process.stderr.write('ok\n'); +} + +if (require.main === module) main(); diff --git a/scripts/verify-hashlist.js b/scripts/verify-hashlist.js new file mode 100644 index 00000000..1459b38b --- /dev/null +++ b/scripts/verify-hashlist.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +// Verifies the signed hash list against the pinned public keys, the way a consumer will. +// +// Run immediately after signing, in the same job. A signing key that has been mangled -- pasted with +// a stray newline, truncated, replaced -- produces a document that is entirely well-formed and that +// no consumer will accept. Checking here makes that a failed workflow rather than a document that +// looks published and satisfies nobody. +// +// It deliberately does not use the signing key to check its own work. It uses the published public +// keys. + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const SPKI_ED25519_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); + +// Must match SIGNING.md and the set consumers pin. Any one of them verifying is enough, which is +// what lets a second key take over without updating consumers. +const PINNED_PUBLIC_KEYS = [ + '14837066068b258bfbd0749702056f7065361af44aed48761834744391cbbaaa', // 1, CI + 'fee7b0ccf2323954af68a249eaa61f957239eb222329e08a5b6a50ced649bae8', // 2, cold +]; + +function verifyDocument(document, publicKeysHex) { + const payload = Buffer.from(document.payload_b64, 'base64'); + const signature = Buffer.from(document.sig_b64, 'base64'); + + if (signature.length !== 64) { + throw new Error(`signature is ${signature.length} bytes, expected 64`); + } + + const accepted = publicKeysHex.some((hex) => { + const key = crypto.createPublicKey({ + key: Buffer.concat([SPKI_ED25519_PREFIX, Buffer.from(hex, 'hex')]), + format: 'der', + type: 'spki', + }); + return crypto.verify(null, payload, key, signature); + }); + + if (!accepted) { + throw new Error('signature does not verify under any pinned public key'); + } + + return JSON.parse(payload.toString('utf8')); +} + +function main(argv) { + const root = path.join(__dirname, '..'); + const signedPath = argv[0] || path.join(root, 'src', 'hashes', 'hashlist-signed.json'); + const document = JSON.parse(fs.readFileSync(signedPath, 'utf8')); + // eslint-disable-next-line global-require + const hashes = require('../src/hashes/hashes').getHashes(); + + const payload = verifyDocument(document, PINNED_PUBLIC_KEYS); + + // The signed bytes must be what we meant to sign, not merely something validly signed. + if (payload.hashes.length !== hashes.length) { + throw new Error(`signed ${payload.hashes.length} hashes, hashes.js has ${hashes.length}`); + } + if (!payload.hashes.every((hash, i) => hash === hashes[i])) { + throw new Error('signed hashes do not match hashes.js'); + } + + process.stderr.write(`verified: seq ${payload.seq}, ${payload.hashes.length} hashes\n`); +} + +if (require.main === module) { + try { + main(process.argv.slice(2)); + } catch (error) { + process.stderr.write(`verify-hashlist: ${error.message}\n`); + process.exit(1); + } +} + +module.exports = { verifyDocument, PINNED_PUBLIC_KEYS }; diff --git a/src/hashes/ledger.json b/src/hashes/ledger.json new file mode 100644 index 00000000..a1d1499a --- /dev/null +++ b/src/hashes/ledger.json @@ -0,0 +1,3 @@ +{ + "culls": [] +} diff --git a/src/routes.js b/src/routes.js index 90ac0782..4249a154 100644 --- a/src/routes.js +++ b/src/routes.js @@ -1,10 +1,35 @@ const apicache = require('apicache'); +const fs = require('fs'); +const path = require('path'); const hashes = require('./hashes/hashes'); const cache = apicache.middleware; +// Only successful responses are cached. By default apicache stores whatever the handler returned, +// including the 404 below -- so a request arriving before the first document was published would +// pin that 404 for the full cache window, well after the document existed. +const cacheSuccess = apicache.newInstance({ statusCodes: { include: [200] } }).middleware; + +const HASHLIST = path.join(__dirname, 'hashes', 'hashlist-signed.json'); + module.exports = (app) => { + // The same hash list, signed, so a consumer can verify it came from us rather than trusting this + // server or whatever relayed the response. Served alongside the unsigned array, not instead of it. + // + // Must come before the catch-all below, which would otherwise answer this path with the array. + app.get('/hashlist', cacheSuccess('5 minutes'), (req, res) => { + fs.readFile(HASHLIST, 'utf8', (error, document) => { + if (error) { + // Nothing signed has been published yet. 404 rather than an empty or partial document, + // which a caller could mistake for a valid list that simply excludes their entry. + res.status(404).json({ error: 'no signed hash list published' }); + return; + } + res.type('application/json').send(document); + }); + }); + app.get('*', cache('5 minutes'), (req, res) => { res.json(hashes.getHashes()); }); diff --git a/test/reconciler/run-tests.sh b/test/reconciler/run-tests.sh new file mode 100755 index 00000000..c4683717 --- /dev/null +++ b/test/reconciler/run-tests.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +# Local logic tests for the reconciler. GitHub platform semantics are proven separately (the +# 2026-08-24 sandbox proofs); this drives scripts/sign-hashlist.js against a local fake flux +# remote through every state transition the design names. +set -u + +# Everything scratch lives in a temp dir: the repository itself is never written to, so this is +# safe to run from a clean checkout and leaves nothing behind. +HERE="$(cd "$(dirname "$0")" && pwd)" +SRC="$(cd "$HERE/../.." && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT +WORK="$TMP/work" +FLUX="$TMP/fake-flux" +export HASHLIST_SIGNING_SEED_B64="MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" # test only +PASS=0; FAIL=0 + +say() { printf '\n=== %s\n' "$*"; } +ok() { PASS=$((PASS+1)); printf 'PASS %s\n' "$*"; } +bad() { FAIL=$((FAIL+1)); printf 'FAIL %s\n' "$*"; } + +run_signer() { # -> stdout line in $RESULT, exit code in $RC + cd "$WORK" + RESULT=$(node scripts/sign-hashlist.js 2>"$TMP/last-stderr.txt"); RC=$? + cd "$HERE" +} + +jsonq() { node -e "const j=JSON.parse(require('fs').readFileSync('$1','utf8')); console.log($2);"; } +listq() { node -e "console.log(require('$WORK/src/hashes/hashes.js').getHashes()$1);"; } +tree_hash() { (cd "$1" && find ./ZelBack -type f -exec md5sum {} + | awk '{print $1}' | LC_ALL=C sort | md5sum | awk '{printf $1}'); } + +# ---------- setup: fake flux remote with two files, master + dev ---------- +mkdir -p "$FLUX/ZelBack/src" +git -C "$FLUX" init -q -b master +git -C "$FLUX" config user.email t@t; git -C "$FLUX" config user.name t +git -C "$FLUX" config uploadpack.allowAnySHA1InWant true +echo 'alpha' > "$FLUX/ZelBack/src/a.js"; echo 'beta' > "$FLUX/ZelBack/src/b.js"; echo 'root' > "$FLUX/readme.md" +git -C "$FLUX" add -A; git -C "$FLUX" commit -qm c1 +HASH1=$(tree_hash "$FLUX") + +mkdir -p "$WORK" +tar -c -C "$SRC" --exclude=.git --exclude=node_modules . | tar -x -C "$WORK" +rm -f "$WORK/src/hashes/hashlist-signed.json" "$WORK/src/hashes/provenance.json" +# Pin the test key in the work copy, the same substitution the sandbox rehearsals make. +node -e " +const fs=require('fs'); +const {privateKeyFromSeed, rawPublicKey}=require('$WORK/scripts/sign-hashlist.js'); +const pub=rawPublicKey(privateKeyFromSeed(process.env.HASHLIST_SIGNING_SEED_B64)).toString('hex'); +const p='$WORK/scripts/verify-hashlist.js'; +const src=fs.readFileSync(p,'utf8').replace(/const PINNED_PUBLIC_KEYS = \[[^\]]*\];/, \"const PINNED_PUBLIC_KEYS = [\n '\"+pub+\"',\n];\"); +fs.writeFileSync(p,src); +" +printf 'function getHashes() {\n return [\n '\''%s'\'',\n '\''ffffffffffffffffffffffffffffffff'\'',\n ];\n}\n\nmodule.exports = {\n getHashes,\n};\n' "$HASH1" > "$WORK/src/hashes/hashes.js" +export FLUX_REMOTE="$FLUX" +export FLUX_CACHE_DIR="$TMP/cache" + +# ---------- 1. bootstrap: signs the grandfathered list, derives nothing ---------- +say "1 bootstrap" +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=signed" ] && ok "bootstrap signs" || bad "bootstrap: rc=$RC result=$RESULT" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'j.signed.seq')" = 1 ] && ok "seq 1" || bad "seq" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH1'].derived")" = "false" ] && ok "grandfathered underived" || bad "grandfather" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH1'].commit")" = "null" ] && ok "grandfathered commit null" || bad "commit null" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'Object.keys(j.refs).length')" = 1 ] && ok "refs snapshotted" || bad "refs" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'Object.keys(j.commits).length')" = 0 ] && ok "nothing derived at bootstrap" || bad "derived at bootstrap" +node -e " +const {verifyDocument} = require('$WORK/scripts/verify-hashlist.js'); +const {privateKeyFromSeed, rawPublicKey} = require('$WORK/scripts/sign-hashlist.js'); +const pub = rawPublicKey(privateKeyFromSeed(process.env.HASHLIST_SIGNING_SEED_B64)).toString('hex'); +const doc = JSON.parse(require('fs').readFileSync('$WORK/src/hashes/hashlist-signed.json','utf8')); +const p = verifyDocument(doc, [pub]); +if (p.seq !== 1 || p.hashes.length !== 2) throw new Error('payload wrong'); +" && ok "document verifies under the test key" || bad "verify" + +# ---------- 2. unchanged rerun ---------- +say "2 unchanged rerun" +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=false" ] && ok "no-op" || bad "rerun: rc=$RC result=$RESULT" + +# ---------- 3. new ZelBack commit on master: derived, appended, seq 2 ---------- +say "3 new tree commit" +echo 'gamma' >> "$FLUX/ZelBack/src/a.js"; git -C "$FLUX" commit -qam c2 +HASH2=$(tree_hash "$FLUX"); SHA2=$(git -C "$FLUX" rev-parse HEAD) +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=signed" ] && ok "signed" || bad "rc=$RC result=$RESULT" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'j.signed.seq')" = 2 ] && ok "seq 2" || bad "seq" +[ "$(listq ".includes('$HASH2')")" = "true" ] && ok "new hash listed" || bad "not listed" +[ "$(listq '.length')" = 3 ] && ok "appended, nothing lost" || bad "length" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH2'].branch")" = "master" ] && ok "own-view branch label" || bad "label" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH2'].derived")" = "true" ] && ok "derived" || bad "derived flag" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.commits['$SHA2']")" = "$HASH2" ] && ok "commits map" || bad "commits map" + +# ---------- 4. non-ZelBack commit: same tree hash, state-only, no seq burn ---------- +say "4 non-tree commit" +echo 'docs' >> "$FLUX/readme.md"; git -C "$FLUX" commit -qam c3 +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=state" ] && ok "state only" || bad "rc=$RC result=$RESULT" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'j.signed.seq')" = 2 ] && ok "no sequence burned" || bad "seq burned" + +# ---------- 5. dispatch hint for an orphaned commit ---------- +say "5 orphaned dispatch" +git -C "$FLUX" checkout -qb doomed +echo 'delta' > "$FLUX/ZelBack/src/d.js"; git -C "$FLUX" add -A; git -C "$FLUX" commit -qm c4 +HASH4=$(tree_hash "$FLUX"); SHA4=$(git -C "$FLUX" rev-parse HEAD) +git -C "$FLUX" checkout -q master; git -C "$FLUX" branch -qD doomed +DISPATCH_COMMIT=$SHA4 DISPATCH_REF=doomed DISPATCH_REF_TYPE=branch bash -c 'cd '"$WORK"' && node scripts/sign-hashlist.js' >"$TMP/out5.txt" 2>"$TMP/last-stderr.txt"; RC=$? +[ "$RC" = 0 ] && [ "$(cat "$TMP/out5.txt")" = "changed=signed" ] && ok "orphan derived via hint" || bad "rc=$RC $(cat "$TMP/out5.txt")" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH4'].branch")" = "doomed" ] && ok "dispatch fallback label" || bad "label" + +# ---------- 6. claimed_hash mismatch is a red run that publishes nothing ---------- +say "6 claimed mismatch" +echo 'epsilon' > "$FLUX/ZelBack/src/e.js"; git -C "$FLUX" add -A; git -C "$FLUX" commit -qm c5 +SHA5=$(git -C "$FLUX" rev-parse HEAD) +BEFORE=$(cat "$WORK/src/hashes/provenance.json") +DISPATCH_COMMIT=$SHA5 DISPATCH_CLAIMED_HASH=00000000000000000000000000000000 bash -c 'cd '"$WORK"' && node scripts/sign-hashlist.js' >/dev/null 2>"$TMP/last-stderr.txt"; RC=$? +grep -q 'environment drift or a hostile dispatch' "$TMP/last-stderr.txt" && [ "$RC" != 0 ] && ok "red run" || bad "rc=$RC" +[ "$(cat "$WORK/src/hashes/provenance.json")" = "$BEFORE" ] && ok "published nothing" || bad "state leaked" + +# ---------- 7. correct claimed_hash passes; catches up c5 too ---------- +say "7 claimed match" +HASH5=$(tree_hash "$FLUX") +DISPATCH_COMMIT=$SHA5 DISPATCH_CLAIMED_HASH=$HASH5 bash -c 'cd '"$WORK"' && node scripts/sign-hashlist.js' >"$TMP/out7.txt" 2>"$TMP/last-stderr.txt"; RC=$? +[ "$RC" = 0 ] && [ "$(cat "$TMP/out7.txt")" = "changed=signed" ] && [ "$(listq ".includes('$HASH5')")" = "true" ] && ok "derived with matching claim" || bad "rc=$RC" + +# ---------- 8. cull through the ledger ---------- +say "8 cull" +printf '{\n "culls": [\n { "hash": "ffffffffffffffffffffffffffffffff", "reason": "test cull", "date": "2026-08-24" }\n ]\n}\n' > "$WORK/src/hashes/ledger.json" +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=signed" ] && ok "resigned" || bad "rc=$RC result=$RESULT" +[ "$(listq ".includes('ffffffffffffffffffffffffffffffff')")" = "false" ] && ok "culled entry gone" || bad "still listed" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['ffffffffffffffffffffffffffffffff'] !== undefined")" = "true" ] && ok "cull keeps its audit row" || bad "row lost" + +# ---------- 9. duplicate entry in the list is refused ---------- +say "9 duplicate refused" +node -e " +const fs=require('fs'); const p='$WORK/src/hashes/hashes.js'; +fs.writeFileSync(p, fs.readFileSync(p,'utf8').replace(\" '$HASH1',\", \" '$HASH1',\n '$HASH1',\")); +" +run_signer +[ "$RC" != 0 ] && grep -q 'duplicates' "$TMP/last-stderr.txt" && ok "red on duplicates" || bad "rc=$RC" +node -e " +const fs=require('fs'); const p='$WORK/src/hashes/hashes.js'; +fs.writeFileSync(p, fs.readFileSync(p,'utf8').replace(\" '$HASH1',\n '$HASH1',\", \" '$HASH1',\")); +" + +# ---------- 10. tag on a known commit annotates without a sequence ---------- +say "10 tag annotation" +git -C "$FLUX" tag v1-test "$SHA2" +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=state" ] && ok "state only" || bad "rc=$RC result=$RESULT" +[ "$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH2'].tag")" = "v1-test" ] && ok "row annotated" || bad "tag" + +# ---------- 11. annotated tag on a NEW commit derives with tag label ---------- +say "11 tagged new commit" +git -C "$FLUX" checkout -q master +echo 'zeta' > "$FLUX/ZelBack/src/z.js"; git -C "$FLUX" add -A; git -C "$FLUX" commit -qm c6 +git -C "$FLUX" tag -a v2-test -m 'release' +HASH6=$(tree_hash "$FLUX") +run_signer +[ "$RC" = 0 ] && [ "$RESULT" = "changed=signed" ] && ok "signed" || bad "rc=$RC result=$RESULT" +TAGLABEL=$(jsonq "$WORK/src/hashes/provenance.json" "j.hashes['$HASH6'].tag || j.hashes['$HASH6'].branch") +[ -n "$TAGLABEL" ] && ok "labelled ($TAGLABEL)" || bad "no label" + +# ---------- 12. corrupt provenance is a red run ---------- +say "12 corrupt provenance" +cp "$WORK/src/hashes/provenance.json" "$TMP/prov-backup.json" +echo 'not json' > "$WORK/src/hashes/provenance.json" +run_signer +[ "$RC" != 0 ] && ok "red on corrupt provenance" || bad "rc=$RC" +cp "$TMP/prov-backup.json" "$WORK/src/hashes/provenance.json" + +# ---------- 13. corrupt ledger is a red run ---------- +say "13 corrupt ledger" +cp "$WORK/src/hashes/ledger.json" "$TMP/ledger-backup.json" +echo '{"culls": [{"hash": "short"}]}' > "$WORK/src/hashes/ledger.json" +run_signer +[ "$RC" != 0 ] && grep -q 'ledger cull' "$TMP/last-stderr.txt" && ok "red on bad cull" || bad "rc=$RC" +cp "$TMP/ledger-backup.json" "$WORK/src/hashes/ledger.json" + +# ---------- 14. validate.js green on the final state ---------- +say "14 validate green" +(cd "$WORK" && node scripts/validate.js 2>"$TMP/validate-out.txt"); RC=$? +[ "$RC" = 0 ] && ok "validate green" || { bad "validate rc=$RC"; cat "$TMP/validate-out.txt"; } + +# ---------- 15. validate red when the signed document is wiped ---------- +say "15 wiped document" +mv "$WORK/src/hashes/hashlist-signed.json" "$HERE/signed-backup.json" +(cd "$WORK" && node scripts/validate.js 2>"$TMP/validate-out.txt"); RC=$? +[ "$RC" != 0 ] && grep -q 'no signed document' "$TMP/validate-out.txt" && ok "wipe is loud" || bad "rc=$RC" +mv "$HERE/signed-backup.json" "$WORK/src/hashes/hashlist-signed.json" + +# ---------- 16. v1 provenance (rows + signed, no refs): bootstrap preserves the sequence ---------- +say "16 v1 compatibility" +node -e " +const fs=require('fs'); const p='$WORK/src/hashes/provenance.json'; +const j=JSON.parse(fs.readFileSync(p,'utf8')); +delete j.refs; delete j.commits; +Object.values(j.hashes).forEach((row)=>delete row.derived); +fs.writeFileSync(p, JSON.stringify(j,null,2)+'\n'); +" +SEQ_BEFORE=$(jsonq "$WORK/src/hashes/provenance.json" 'j.signed.seq') +run_signer +[ "$RC" = 0 ] && ok "v1 record accepted (result=$RESULT)" || bad "rc=$RC" +[ "$(jsonq "$WORK/src/hashes/provenance.json" 'Object.keys(j.refs).length > 0')" = "true" ] && ok "snapshot rebuilt" || bad "no snapshot" +SEQ_AFTER=$(jsonq "$WORK/src/hashes/provenance.json" 'j.signed.seq') +[ "$SEQ_AFTER" -ge "$SEQ_BEFORE" ] && ok "sequence preserved ($SEQ_BEFORE -> $SEQ_AFTER)" || bad "sequence restarted" + +# ---------- 17-18. nothing was hashed: the empty-tree hash must never be listed ---------- +# The pipeline yields d41d8cd98f00b204e9800998ecf8427e -- the md5 of an empty stream -- whenever +# nothing was hashed. It is a well-formed 32-hex value, so nothing downstream tells it apart from +# a real tree hash, and it would mean "a node whose ZelBack holds no regular files is genuine +# FluxOS". Membership is monotonic, so listing it once costs a cull PR. +# +# The two ways in have DIFFERENT guards, so each case asserts the message it should die on -- +# asserting only "the run went red" passes for any reason at all and tests nothing. +EMPTY_HASH=d41d8cd98f00b204e9800998ecf8427e + +say "17 ZelBack absent (find errors; caught by pipefail)" +git -C "$FLUX" checkout -q -b nozelback master +git -C "$FLUX" rm -rq ZelBack +git -C "$FLUX" commit -qm "drop ZelBack entirely" +SHA_NOZB=$(git -C "$FLUX" rev-parse HEAD) +BEFORE=$(cat "$WORK/src/hashes/provenance.json") +DISPATCH_COMMIT=$SHA_NOZB run_signer +grep -q 'Command failed' "$TMP/last-stderr.txt" && [ "$RC" != 0 ] \ + && ok "red on the pipeline's own failure" || bad "rc=$RC -- expected a pipefail death, got: $(tail -1 "$TMP/last-stderr.txt")" +[ "$(listq ".includes('$EMPTY_HASH')")" = "false" ] && ok "empty hash not listed" || bad "EMPTY HASH REACHED THE LIST" +[ "$(cat "$WORK/src/hashes/provenance.json")" = "$BEFORE" ] && ok "published nothing" || bad "state leaked" + +say "18 ZelBack holds no regular files (find exits 0; pipefail cannot see it)" +# Only symlinks and directories: find -type f matches nothing and exits 0, so the pipeline +# succeeds and returns the empty hash. Git cannot store an empty directory, but it stores a +# symlink (mode 120000) -- this is the reachable shape of "present but nothing to hash". +git -C "$FLUX" checkout -q -b emptyzelback master +git -C "$FLUX" rm -rq ZelBack +mkdir -p "$FLUX/ZelBack/sub" +ln -s /dev/null "$FLUX/ZelBack/link.js" +ln -s /dev/null "$FLUX/ZelBack/sub/deep.js" +git -C "$FLUX" add -A +git -C "$FLUX" commit -qm "ZelBack with no regular files" +SHA_EMPTYZB=$(git -C "$FLUX" rev-parse HEAD) +BEFORE=$(cat "$WORK/src/hashes/provenance.json") +DISPATCH_COMMIT=$SHA_EMPTYZB run_signer +grep -q 'nothing was hashed' "$TMP/last-stderr.txt" && [ "$RC" != 0 ] \ + && ok "red on the empty-hash guard" || bad "rc=$RC -- expected the guard to fire, got: $(tail -1 "$TMP/last-stderr.txt")" +[ "$(listq ".includes('$EMPTY_HASH')")" = "false" ] && ok "empty hash not listed" || bad "EMPTY HASH REACHED THE LIST" +[ "$(cat "$WORK/src/hashes/provenance.json")" = "$BEFORE" ] && ok "published nothing" || bad "state leaked" +git -C "$FLUX" checkout -q master + +printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" +[ "$FAIL" = 0 ] diff --git a/test/vectors/hashlist-key.json b/test/vectors/hashlist-key.json new file mode 100644 index 00000000..bdc266a3 --- /dev/null +++ b/test/vectors/hashlist-key.json @@ -0,0 +1,6 @@ +{ + "note": "TEST KEY. Committed so the vector can be regenerated. Signs nothing the network trusts.", + "derivation": "sha256('fluxos-hashlist-interop-vector-test-key-not-for-production')", + "seed_b64": "9RxIUq+szQNlRtpnI7K9lm9Rui18LJxkfQ28p0m9K9E=", + "public_key_hex": "d4b4591e5109c6512820f60f22c1ae61d0d8e0f7df7c2f3e51daeb573fc79063" +} diff --git a/test/vectors/hashlist.json b/test/vectors/hashlist.json new file mode 100644 index 00000000..9b2bf15c --- /dev/null +++ b/test/vectors/hashlist.json @@ -0,0 +1,4 @@ +{ + "payload_b64": "eyJzZXEiOjEsImlzc3VlZF9hdCI6IjIwMjYtMDgtMTdUMDA6MDA6MDAuMDAwWiIsImhhc2hlcyI6WyI4YWQ5Mjc1MThjZTVmMzc0MDZhZWQzOTcwMDEzNDA4MiJdfQ==", + "sig_b64": "z5/iS5JWeSAlTVe67VFv1EGupQJbvDffJWaJ3EfabtxFnSrletdBe2vqLPsAYeKBktXpsbdB62Zu1K0dGn03CQ==" +}