From afaca1fda5f7f7dfc951ed81d461eb9987678083 Mon Sep 17 00:00:00 2001 From: Santiago Date: Thu, 10 Sep 2026 20:11:55 -0300 Subject: [PATCH 1/2] fix(release): keep prereleases off stable channels --- .github/scripts/docker-tags.sh | 47 ++++++++ .github/scripts/test-docker-tags.sh | 54 +++++++++ .github/workflows/docker.yml | 25 +++-- AGENTS.md | 2 +- crates/fjall/src/archive/README.md | 2 +- crates/fjall/src/state/README.md | 2 +- docs/content/architecture/data-layer.mdx | 4 +- docs/content/configuration/schema.mdx | 10 +- docs/content/installation/binaries.mdx | 34 +++++- docs/content/installation/docker.mdx | 20 +++- docs/content/migration/_meta.yml | 3 + docs/content/migration/dolos-v2.mdx | 137 +++++++++++++++++++++++ docs/release-notes/v2.0.0-alpha.0.md | 92 +++++++++++++++ src/adapters/storage.rs | 2 +- 14 files changed, 414 insertions(+), 20 deletions(-) create mode 100755 .github/scripts/docker-tags.sh create mode 100755 .github/scripts/test-docker-tags.sh create mode 100644 docs/content/migration/_meta.yml create mode 100644 docs/content/migration/dolos-v2.mdx create mode 100644 docs/release-notes/v2.0.0-alpha.0.md diff --git a/.github/scripts/docker-tags.sh b/.github/scripts/docker-tags.sh new file mode 100755 index 000000000..250dd36c6 --- /dev/null +++ b/.github/scripts/docker-tags.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 + exit 2 +} + +[[ $# -eq 3 ]] || usage + +ref=$1 +sha=$2 +is_default_branch=$3 + +[[ $sha =~ ^[0-9a-fA-F]{7,}$ ]] || { + echo "invalid Git SHA: $sha" >&2 + exit 2 +} + +case "$is_default_branch" in + true|false) ;; + *) + echo "is-default-branch must be true or false" >&2 + exit 2 + ;; +esac + +if [[ $is_default_branch == true ]]; then + echo latest +fi + +# A release ref must be valid enough to identify all three numeric SemVer +# components. Prerelease identifiers preserve the exact version tag, but only +# an unsuffixed release gets the stable, major and minor aliases. +if [[ $ref =~ ^refs/tags/v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z]+([.-][0-9A-Za-z]+)*))?$ ]]; then + version=${ref#refs/tags/} + echo "$version" + + if [[ -z ${BASH_REMATCH[4]} ]]; then + echo stable + echo "v${BASH_REMATCH[1]}" + echo "v${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" + fi +fi + +# Match docker/metadata-action's default short-SHA spelling. +echo "sha-${sha:0:7}" diff --git a/.github/scripts/test-docker-tags.sh b/.github/scripts/test-docker-tags.sh new file mode 100755 index 000000000..ad422f343 --- /dev/null +++ b/.github/scripts/test-docker-tags.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +policy="$root/.github/scripts/docker-tags.sh" +sha=0123456789abcdef0123456789abcdef01234567 + +assert_tags() { + local name=$1 + local ref=$2 + local is_default_branch=$3 + local expected=$4 + local actual + + actual=$($policy "$ref" "$sha" "$is_default_branch") + + if [[ $actual != "$expected" ]]; then + printf '%s: unexpected tags\nexpected:\n%s\nactual:\n%s\n' \ + "$name" "$expected" "$actual" >&2 + return 1 + fi + + printf '%s: %s\n' "$name" "$(printf '%s\n' "$actual" | tr '\n' ' ' | sed 's/ $//')" +} + +assert_tags \ + main \ + refs/heads/main \ + true \ + $'latest\nsha-0123456' + +assert_tags \ + alpha \ + refs/tags/v2.0.0-alpha.0 \ + false \ + $'v2.0.0-alpha.0\nsha-0123456' + +assert_tags \ + rc \ + refs/tags/v2.0.0-rc.0 \ + false \ + $'v2.0.0-rc.0\nsha-0123456' + +assert_tags \ + stable \ + refs/tags/v2.0.0 \ + false \ + $'v2.0.0\nstable\nv2\nv2.0\nsha-0123456' + +assert_tags \ + non-semver-v-tag \ + refs/tags/vnightly \ + false \ + 'sha-0123456' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 81ad820b1..25476e5aa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,18 +74,29 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 + # `latest` deliberately tracks main, not the most recent release. The + # script is also the executable specification for which release aliases + # a stable or prerelease tag receives. + - name: Compute Docker tags + id: docker_tags + env: + IS_DEFAULT_BRANCH: ${{ github.ref == 'refs/heads/main' }} + run: | + tags="$(.github/scripts/docker-tags.sh "$GITHUB_REF" "$GITHUB_SHA" "$IS_DEFAULT_BRANCH")" + { + echo 'metadata<> "$GITHUB_OUTPUT" + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v6 with: images: ghcr.io/txpipe/dolos - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern=v{{major}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern=v{{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=semver,pattern=v{{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=sha + tags: ${{ steps.docker_tags.outputs.metadata }} - name: Login to DockerHub uses: docker/login-action@v4 diff --git a/AGENTS.md b/AGENTS.md index bfe75a7e1..70e2d4196 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,7 @@ Dolos uses three distinct storage backends, each serving a specific purpose: - **Database**: `/wal` ### Where the indexes live -There is no standalone index store — it was removed in v1.7. Every index is a +There is no standalone index store — it was removed in v2. Every index is a projection, and lives in the store that holds what it projects: - the live-UTxO tags (by address, payment, stake, policy, asset, script ref) project the UTxO set and live in the `StateStore` (`StateStore::utxos_by_tag`, written through `StateWriter::apply_utxo_tags` in the same batch as the set) - the archive tags and the exact lookups (by block hash, block number, tx hash) project the block history and live in the `ArchiveStore` (`ArchiveStore::slots_by_tag` / `slot_by_*`, written through `ArchiveWriter::apply_index` in the same batch as the blocks) diff --git a/crates/fjall/src/archive/README.md b/crates/fjall/src/archive/README.md index 9e3ebe951..70087f83a 100644 --- a/crates/fjall/src/archive/README.md +++ b/crates/fjall/src/archive/README.md @@ -11,7 +11,7 @@ This module implements the `ArchiveStore` trait using [Fjall](https://github.com | 3 | `archive-tags` | Block tags, append-only | Prefix scans by dimension and key | | 4 | `index-exact` | Block hash / block number / tx hash → slot | Point lookups | -The last two are projections of the blocks. They were a separate database until v1.7 (`/index`), and moving them here is what lets them be written in the same batch as the block locations they point at, so the history and its lookups commit together. They keep the compaction settings the standalone store gave them — `l0_threshold = 8`, `memtable_size_mb = 128` — so their behavior did not change with the move. +The last two are projections of the blocks. They were a separate database until v2 (`/index`), and moving them here is what lets them be written in the same batch as the block locations they point at, so the history and its lookups commit together. They keep the compaction settings the standalone store gave them — `l0_threshold = 8`, `memtable_size_mb = 128` — so their behavior did not change with the move. ## Key Schemas diff --git a/crates/fjall/src/state/README.md b/crates/fjall/src/state/README.md index 503856060..9a53ba097 100644 --- a/crates/fjall/src/state/README.md +++ b/crates/fjall/src/state/README.md @@ -71,7 +71,7 @@ Value: (empty) - `lookup_key`: the logical key, stored verbatim so a prefix scan can find it - `txo_ref`: `[tx_hash:32][index:4]`, the UTxO the tag points at -They are a projection of the UTxO set, so they are written in the same batch as the set (`StateWriter::apply_utxo_tags` / `undo_utxo_tags`) and read through `StateStore::utxos_by_tag`. Before v1.7 they lived in a separate index database; nothing about the key encoding changed with the move, only which journal, cache and write batch they live under. +They are a projection of the UTxO set, so they are written in the same batch as the set (`StateWriter::apply_utxo_tags` / `undo_utxo_tags`) and read through `StateStore::utxos_by_tag`. Before v2 they lived in a separate index database; nothing about the key encoding changed with the move, only which journal, cache and write batch they live under. Note the asymmetry with the archive's tags, which hash their key: here the lookup key is stored whole, because a live-UTxO query knows the key it is asking about and wants the exact refs back. diff --git a/docs/content/architecture/data-layer.mdx b/docs/content/architecture/data-layer.mdx index bcce24803..a155c446c 100644 --- a/docs/content/architecture/data-layer.mdx +++ b/docs/content/architecture/data-layer.mdx @@ -24,7 +24,7 @@ Under the configured `storage.path`, a running node maintains three on-disk stor | **Archive** | Immutable history: full block bodies, time-indexed logs of entity changes, and the reverse lookups over them — slots by address, payment credential, stake credential, policy, asset, datum and more, plus slot by block hash, block number and tx hash. | History is append-only and read differently from current state, so it uses its own layout (see below). The lookups are a projection of the blocks, so they live beside them and commit with them. | | **Mempool** | Submitted-but-unconfirmed transactions and their lifecycle state. | Pending transactions are transient and overlaid on top of committed state during validation. | -Before v1.7 the reverse lookups lived in a fourth store of their own, configured under `[storage.index]`. That store is gone, and so is the table. +Before v2 the reverse lookups lived in a fourth store of their own, configured under `[storage.index]`. That store is gone, and so is the table. ## Storage traits and pluggable backends @@ -32,7 +32,7 @@ Each store is defined as a trait in `dolos-core` — `WalStore`, `StateStore`, ` Available backends: -- **redb** (`dolos-redb3`) — an embedded ACID B+tree store. It backs the WAL (the only WAL implementation) and the mempool; its state and archive config variants were removed in v1.7, so a configuration naming them fails to load. +- **redb** (`dolos-redb3`) — an embedded ACID B+tree store. It backs the WAL (the only WAL implementation) and the mempool; its state and archive config variants were removed in v2, so a configuration naming them fails to load. - **fjall** (`dolos-fjall`) — an LSM-tree engine tuned for write-heavy workloads with many hot keys. The only persistent backend for the state and archive stores, and the default for both. - **no-op** — a store that silently discards writes, used to *disable* the archive. - **in-memory** — non-persistent stores for testing and ephemeral nodes. State and archive have builtin implementations in `dolos-core` backed by ordered maps, which serve their traits in full; the WAL and mempool use redb's memory backend instead. diff --git a/docs/content/configuration/schema.mdx b/docs/content/configuration/schema.mdx index 773e51b54..3564abbe9 100644 --- a/docs/content/configuration/schema.mdx +++ b/docs/content/configuration/schema.mdx @@ -114,7 +114,7 @@ The `storage` section controls how Dolos stores data in the local file system. E bundled dictionary and folds the former index store into state and archive; there are no compression settings. Run `dolos init` to upgrade a readable older configuration, or follow the manual steps in the - [migration guide](https://docs.txpipe.io/dolos/migration/dolos-v1-7) when it + [migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) when it cannot be read. Relabelling an older data directory as `v4` is unsupported and is not detected. @@ -155,7 +155,7 @@ deletes the files it has consumed as it goes. `--download-dir` moves it. | worker_threads | integer | 4 | | memtable_size_mb | integer | 64 | -- `backend`: `fjall` or `in_memory` (defaults to `fjall`). `in_memory` keeps the whole state in process memory: it is ephemeral — everything is lost on restart — and it holds the UTxO set and all entities in RAM, so it suits devnets, tooling and tests rather than a node following a public network. The `redb` value was removed in v1.7: a configuration still naming it fails to load, and an existing redb state directory has to be re-bootstrapped (a stelae restore is the shortest path). +- `backend`: `fjall` or `in_memory` (defaults to `fjall`). `in_memory` keeps the whole state in process memory: it is ephemeral — everything is lost on restart — and it holds the UTxO set and all entities in RAM, so it suits devnets, tooling and tests rather than a node following a public network. The `redb` value was removed in v2: a configuration still naming it fails to load, and an existing redb state directory has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available). - `path`: optional override for the state path (defaults to `/state`). - `cache`: size (MB) of the state cache. - `max_history`: maximum number of slots to keep before pruning. @@ -175,7 +175,7 @@ deletes the files it has consumed as it goes. `--download-dir` moves it. | worker_threads | integer | 4 | | memtable_size_mb | integer | 64 | -- `backend`: `fjall`, `in_memory`, or `no_op` (defaults to `fjall`). `in_memory` keeps the whole archive in process memory, block bodies included: ephemeral — everything is lost on restart — so it suits devnets, tooling and tests rather than a node following a public network. `no_op` disables the archive entirely. The `redb` value was removed in v1.7: a configuration still naming it fails to load, and an existing redb archive has to be re-bootstrapped (a stelae restore is the shortest path). +- `backend`: `fjall`, `in_memory`, or `no_op` (defaults to `fjall`). `in_memory` keeps the whole archive in process memory, block bodies included: ephemeral — everything is lost on restart — so it suits devnets, tooling and tests rather than a node following a public network. `no_op` disables the archive entirely. The `redb` value was removed in v2: a configuration still naming it fails to load, and an existing redb archive has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available). - `path`: optional override for the archive path (defaults to `/archive`). - `blocks_path`: optional override for block segment files. - `cache`: size (MB) of the archive index cache. @@ -187,9 +187,9 @@ The dictionary is part of the storage format, not a setting: every build carries A data directory written by an earlier layout is not readable and is not converted: bring a `v4` instance up by a fresh bootstrap (`dolos bootstrap mithril` or `relay`) or by importing a logical snapshot (`dolos bootstrap stelae`, the shortest path), both of which write the compressed layout from the first block. The version check is the only enforcement and it is at the configuration level: a `v3` or older `dolos.toml` is refused with the remedy named, and `dolos init` rewrites it. There is no check of the data itself, so a data directory from an older layout whose configuration was relabelled `v4` by hand is unsupported and is not detected — its blocks are not frames and reads of them fail. -### `storage.index` section — removed in v1.7 +### `storage.index` section — removed in v2 -The standalone index store is gone. Its lookups moved into the stores that hold what they project: the live-UTxO tags into `storage.state`, and the historical tags and exact lookups into `storage.archive`. Drop the table from the configuration; v1.7 bumps the storage version, so an existing data directory has to be re-bootstrapped (a stelae restore is the shortest path) and its configuration revisited anyway. +The standalone index store is gone. Its lookups moved into the stores that hold what they project: the live-UTxO tags into `storage.state`, and the historical tags and exact lookups into `storage.archive`. Drop the table from the configuration; v2 bumps the storage version, so an existing data directory has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available) and its configuration revisited anyway. Any tuning the table carried (`cache`, `worker_threads`, `max_journal_size`, and the rest) should be re-applied to `storage.state` and `storage.archive`, which now carry the keyspaces. `index.backend = "no_op"` had no replacement and needs none: `archive.backend = "no_op"` is the ledger-only switch and drops the historical lookups with the archive, while the live-UTxO tags stay with the UTxO set they project. diff --git a/docs/content/installation/binaries.mdx b/docs/content/installation/binaries.mdx index eece0aefb..a17e51f96 100644 --- a/docs/content/installation/binaries.mdx +++ b/docs/content/installation/binaries.mdx @@ -45,4 +45,36 @@ npm install @txpipe/dolos | [dolos-aarch64-apple-darwin.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-aarch64-apple-darwin.tar.gz) | Apple Silicon macOS | | [dolos-x86_64-pc-windows-msvc.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-x86_64-pc-windows-msvc.tar.gz) | x64 Windows | | [dolos-x86_64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-x86_64-unknown-linux-gnu.tar.gz) | x64 Linux | -| [dolos-aarch64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-aarch64-unknown-linux-gnu.tar.gz) | ARM64 Linux | \ No newline at end of file +| [dolos-aarch64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-aarch64-unknown-linux-gnu.tar.gz) | ARM64 Linux | + +## Opt in to v2.0.0-alpha.0 + +The v2 alpha is a prerelease and must be selected explicitly. The +`releases/latest` URLs above, `brew install txpipe/tap/dolos`, and +`npm install @txpipe/dolos` continue to select stable releases; ordinary +Homebrew and npm installs do not select this alpha. + +After the prerelease is published, use its exact release URLs: + +```sh +# macOS / Linux +curl --proto '=https' --tlsv1.2 -LsSf \ + https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.sh | sh + +# Windows PowerShell +powershell -ExecutionPolicy Bypass -c \ + "irm https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.ps1 | iex" +``` + +Direct downloads use the same versioned release directory: + +| File | Platform | +| --- | --- | +| [dolos-aarch64-apple-darwin.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-aarch64-apple-darwin.tar.gz) | Apple Silicon macOS | +| [dolos-x86_64-pc-windows-msvc.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-x86_64-pc-windows-msvc.tar.gz) | x64 Windows | +| [dolos-x86_64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-x86_64-unknown-linux-gnu.tar.gz) | x64 Linux | +| [dolos-aarch64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-aarch64-unknown-linux-gnu.tar.gz) | ARM64 Linux | + +These links exist only after the founder publishes the GitHub prerelease. The +release configuration keeps package-manager prerelease publication disabled, +so there is intentionally no ordinary Homebrew or npm command for this alpha. diff --git a/docs/content/installation/docker.mdx b/docs/content/installation/docker.mdx index 5eb34b996..5f0967b9a 100644 --- a/docs/content/installation/docker.mdx +++ b/docs/content/installation/docker.mdx @@ -10,6 +10,11 @@ docker run ghcr.io/txpipe/dolos:latest The result of the above command should show _Dolos'_ command-line help message. +`latest` deliberately tracks the `main` branch. It is useful for testing the +current development build, but it is mutable and is not the stable-release +channel. Use `stable` for the most recent stable release or an exact version tag +for reproducible deployments. + ## Entry Point The entry-point of the image points to _Dolos_ executable. You can pass the same command-line arguments that you would pass to the binary release running bare-metal. For example: @@ -39,7 +44,20 @@ To use a versioned image, replace the `latest` tag by the desired version with t ghcr.io/txpipe/dolos:v1 ``` -The `v1` tag is a floating alias that tracks the latest `1.x` release. For production deployments, consider pinning to a specific version (e.g. `v1.0.0`). +The `v1` tag is a floating alias that tracks the latest stable `1.x` release. For production deployments, consider pinning to a specific version (e.g. `v1.0.0`). Prereleases do not update `stable`, major aliases such as `v2`, or minor aliases such as `v2.0`. + +## Opt in to v2.0.0-alpha.0 + +Pull the alpha by its exact tag after the prerelease is published: + +```sh +docker pull ghcr.io/txpipe/dolos:v2.0.0-alpha.0 +docker run ghcr.io/txpipe/dolos:v2.0.0-alpha.0 --help +``` + +This exact tag is immutable release identity. It does not move `stable`, `v2` +or `v2.0`. It is also different from `latest`, which continues to follow +`main` and may contain changes newer than the alpha. ## Multiple Architectures diff --git a/docs/content/migration/_meta.yml b/docs/content/migration/_meta.yml new file mode 100644 index 000000000..b878135b2 --- /dev/null +++ b/docs/content/migration/_meta.yml @@ -0,0 +1,3 @@ +label: Migration +order: 5 +collapsed: true diff --git a/docs/content/migration/dolos-v2.mdx b/docs/content/migration/dolos-v2.mdx new file mode 100644 index 000000000..0de821d56 --- /dev/null +++ b/docs/content/migration/dolos-v2.mdx @@ -0,0 +1,137 @@ +--- +title: Migrating to Dolos v2 +sidebar: + label: Dolos v2 + order: 1 +--- + +import { Aside } from '@astrojs/starlight/components'; + +Dolos v2 changes the on-disk storage format from `v3` to `v4`. There is no +in-place data conversion. Whether you are coming from v1.6, v1.7.0-alpha.0 or +v1.7.0-alpha.1, the supported upgrade is to preserve what you need for +rollback, initialize a v2 configuration, and bootstrap fresh data. + + + +## Before you start + +1. Stop Dolos and prevent supervisors from restarting it during the upgrade. +2. Record the Dolos binary version, configuration, genesis files, service + settings and the chain point clients currently see. +3. Copy or snapshot the complete v1 data directory and configuration if you + need a rollback path. Test that the copy can be restored before deleting the + working data. +4. Provision enough free disk for the new stores and the selected bootstrap + method. Full history costs substantially more than ledger-only or a sliding + history window. Registry restores stage compressed layers under + `/scratch` unless `--scratch-dir` points elsewhere; Mithril + uses its download directory as well as the destination stores. +5. Reserve a maintenance window for the bootstrap and tip catch-up. A Stelae + restore is dominated by transfer time, Mithril rebuilds state from verified + immutable history, and relay sync replays the chain from origin. Network, + history window, CPU and storage performance all affect the duration. + +Do not let a v1 binary write to the new v4 directory. A rollback restores the +old binary, old configuration and old data together. + +## Initialize the v2 configuration + +Install the v2 binary you intend to run, then execute: + +```sh +dolos init +``` + +If the existing configuration is readable and declares an older storage +version, `dolos init` offers to delete the managed data and bootstrap again. +Accepting that prompt clears the WAL, state, archive and other managed storage; +it is irreversible unless you made a backup. Declining leaves both the data and +configuration untouched, and v2 continues to refuse them. + +Some old configurations cannot be parsed because they name a removed backend +or a retired option. In that case `dolos init` refuses to guess the storage +path. Read the path from the old configuration, remove or move that data +yourself, preserve the old configuration for rollback, then remove the working +configuration and run `dolos init` again. + +Review the generated file instead of copying only the new version string: + +- `storage.state.backend` and `storage.archive.backend` no longer accept + `redb`. Persistent state and archive storage use `fjall`. +- The standalone `[storage.index]` table is gone. A leftover table is ignored; + it does not migrate the old index database. Reapply relevant cache, worker, + journal, flush, L0 and memtable tuning to `storage.state` and + `storage.archive`. +- Archive blocks are written as compressed frames with Dolos' bundled + dictionary. There are no operator compression or sealing settings. + +## Bootstrap fresh v4 data + +Choose a source appropriate to your trust and recovery requirements: + +- **Stelae** restores logical data without replaying every block and can resume + interrupted registry transfers. Follow the + [Stelae bootstrap guide](https://docs.txpipe.io/dolos/bootstrap/stelae) from + [PR #1217](https://github.com/txpipe/dolos/pull/1217). Use it only after that + guide is published and after the release process identifies a snapshot that + has been restored and checked for your network. +- **Mithril** downloads certificate-verified Cardano immutable history and + rebuilds Dolos state. Run `dolos bootstrap mithril`; this requires space for + the download and the resulting stores and can take hours on mainnet. +- **Relay or local node** syncs from origin with + `dolos bootstrap relay`, then `dolos daemon`. It avoids trusting derived + snapshot state but can take hours to days. +- **Legacy Dolos snapshots** package storage-engine files. Only use a snapshot + explicitly produced for storage `v4`. Snapshot paths that interpolate + `${VERSION}` use the storage version (`v4`), not the product release + (`v2.0.0-alpha.0`). + + + +## Update scripts and integrations + +The v2 storage reorganization also changes operator-facing commands and +summaries: + +| Before v2 | In v2 | +| --- | --- | +| `data export --include-indexes` | Removed; indexes travel with state or archive data | +| `data cardinality-stats` | Removed; update scripts that invoke it | +| A separate `indexes` entry in data summaries | Removed; inspect state and archive summaries | +| Index-store tuning under `[storage.index]` | Move the applicable tuning to state and archive | +| `redb` state or archive backend | Use `fjall` for persistent data | + +Custom Rust integrations must also remove `Domain::Indexes`, +`Domain::indexes()` and `WorkUnit::commit_indexes()` and adopt the state/archive +writer contracts that now own their index updates. + +## Verify before cutover + +Start Dolos, let it reach the expected point, and run the read-only consistency +audit against the new data: + +```sh +dolos data check +``` + +The full audit scans substantial state and history and can take minutes on +mainnet. Treat any reported inconsistency as a failed upgrade. Also compare the +node tip with an independent source and exercise the API endpoints your clients +depend on before directing production traffic to v2. + +Keep the v1 backup until the v2 node has completed its intended soak and your +clients have passed their checks. To roll back, stop v2, move the entire v4 +directory aside, and restore the v1 binary, configuration and data as one set. diff --git a/docs/release-notes/v2.0.0-alpha.0.md b/docs/release-notes/v2.0.0-alpha.0.md new file mode 100644 index 000000000..517b88ffc --- /dev/null +++ b/docs/release-notes/v2.0.0-alpha.0.md @@ -0,0 +1,92 @@ +# Draft release notes: Dolos v2.0.0-alpha.0 + +> Publishing draft prepared against `49fbfe2b`. Recheck the final tag and links +> before publishing; this file is not the generated changelog. + +Dolos v2.0.0-alpha.0 continues the preview that began with +v1.7.0-alpha.0 and v1.7.0-alpha.1 under its final major-version identity. It +includes the work from both published alphas plus the changes merged since +alpha.1. Existing v1.7 alpha tags and artifacts remain available for +reproducibility, but new testing should use the v2 alpha. + +## What v2 adds since v1.6 + +- **Broader application data.** Mini Blockfrost adds account activity, asset, + epoch, pool, governance, script and address-derivation routes, while fixes to + input resolution, epoch history and pool data improve existing responses. +- **Computed Conway governance.** Dolos records committee, constitution and + proposal state and computes voting stake, ratification and enactment instead + of relying on a maintained outcome table. +- **Simpler, compressed storage.** State and archive now own the indexes over + their data, removing the separate index store. Archived block bodies are + compressed during normal writes with a dictionary bundled in the binary. +- **Restorable logical snapshots.** Stelae publication and restoration use + deterministic, content-addressed layers, support interrupted transfers and + make restore completion explicit. +- **Better operations.** `dolos data check` audits cursor, archive, epoch-log, + accounting and delegation consistency. Driver failures reach the process exit + status, and bootstrap and snapshot operations provide clearer progress and + capacity diagnostics. + +## A fresh bootstrap is mandatory + +V2 reads storage `v4`; v1.6 and both v1.7 alphas cannot be upgraded in place. +Back up the old binary, configuration and data if you require rollback, run +`dolos init`, and create fresh v4 data. Do not hand-edit an old configuration +to claim `version = "v4"` over old files. + +Follow the [v2 migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) +for removed backends, `[storage.index]`, CLI changes, bootstrap choices and +post-restore verification. The separate +[Stelae bootstrap guide](https://docs.txpipe.io/dolos/bootstrap/stelae) is being +reviewed in [PR #1217](https://github.com/txpipe/dolos/pull/1217); do not announce +that page or an official snapshot until both have been published and verified. + +## Install this alpha explicitly + +Stable channels do not select this prerelease. After the GitHub release is +published, use one of these exact-version commands: + +```sh +# macOS / Linux +curl --proto '=https' --tlsv1.2 -LsSf \ + https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.sh | sh + +# Windows PowerShell +powershell -ExecutionPolicy Bypass -c \ + "irm https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.ps1 | iex" + +# Docker +docker pull ghcr.io/txpipe/dolos:v2.0.0-alpha.0 +``` + +The `releases/latest` installers and ordinary Homebrew and npm installs +continue to select stable releases. Docker `stable`, `v2`, and `v2.0` are +reserved for stable releases and are not updated by this alpha. +Docker `latest` deliberately tracks the `main` branch and is mutable; it is not +the stable-release channel. Pin `v2.0.0-alpha.0` for alpha testing. + +## Known limitations and validation still required + +- V2 is an alpha. Full per-network API conformance, sync-from-genesis, + governance parity, restore checks and an epoch-boundary soak remain release + gates before a stable v2 release. +- Snapshot support does not establish snapshot availability. Preview, preprod + and mainnet sources must each be restored and checked before being + recommended. +- Stelae content digests verify bytes, not publisher identity; snapshots are + unsigned in this release, so registry and publisher trust remains explicit. +- Historical correctness fixes may require replay or a snapshot produced from + corrected data. Continuing an old derived state from its tip cannot recreate + history an earlier binary failed to record. +- Mini Blockfrost has a broader surface but does not claim complete Blockfrost + compatibility. Validate the routes and response behavior your application + needs. + +## Review sources + +- [Changes from v1.6.0 to the reviewed candidate](https://github.com/txpipe/dolos/compare/v1.6.0...49fbfe2b) +- [v1.7.0-alpha.0 release](https://github.com/txpipe/dolos/releases/tag/v1.7.0-alpha.0) +- [v1.7.0-alpha.1 release](https://github.com/txpipe/dolos/releases/tag/v1.7.0-alpha.1) +- [Dolos v2 migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) +- [Stelae bootstrap documentation PR](https://github.com/txpipe/dolos/pull/1217) diff --git a/src/adapters/storage.rs b/src/adapters/storage.rs index f6614a45d..6a87e541e 100644 --- a/src/adapters/storage.rs +++ b/src/adapters/storage.rs @@ -142,7 +142,7 @@ pub fn inspect_existing_data( pub const CURRENT_STORAGE_VERSION: StorageVersion = StorageVersion::V4; /// The migration guide the refusal points an operator at. -pub const MIGRATION_GUIDE_URL: &str = "https://docs.txpipe.io/dolos/migration/dolos-v1-7"; +pub const MIGRATION_GUIDE_URL: &str = "https://docs.txpipe.io/dolos/migration/dolos-v2"; /// Refuse a configuration at any storage version but the current one. /// From 4f102371508f1e4adeb33ec7de467b01e8696af2 Mon Sep 17 00:00:00 2001 From: Santiago Date: Thu, 10 Sep 2026 20:44:11 -0300 Subject: [PATCH 2/2] fix(release): address review feedback --- .github/scripts/docker-tags.sh | 47 -------- .github/scripts/test-docker-tags.sh | 54 --------- .github/workflows/docker.yml | 25 ++--- AGENTS.md | 4 +- crates/fjall/src/archive/README.md | 9 +- crates/fjall/src/state/README.md | 10 +- docs/content/architecture/data-layer.mdx | 4 +- docs/content/configuration/schema.mdx | 31 +++-- docs/content/installation/binaries.mdx | 32 ------ docs/content/installation/docker.mdx | 12 +- docs/content/migration/_meta.yml | 3 - docs/content/migration/dolos-v2.mdx | 137 ----------------------- docs/release-notes/v2.0.0-alpha.0.md | 92 --------------- src/adapters/storage.rs | 19 ++-- src/bin/dolos/init.rs | 18 +-- 15 files changed, 57 insertions(+), 440 deletions(-) delete mode 100755 .github/scripts/docker-tags.sh delete mode 100755 .github/scripts/test-docker-tags.sh delete mode 100644 docs/content/migration/_meta.yml delete mode 100644 docs/content/migration/dolos-v2.mdx delete mode 100644 docs/release-notes/v2.0.0-alpha.0.md diff --git a/.github/scripts/docker-tags.sh b/.github/scripts/docker-tags.sh deleted file mode 100755 index 250dd36c6..000000000 --- a/.github/scripts/docker-tags.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - echo "usage: $0 " >&2 - exit 2 -} - -[[ $# -eq 3 ]] || usage - -ref=$1 -sha=$2 -is_default_branch=$3 - -[[ $sha =~ ^[0-9a-fA-F]{7,}$ ]] || { - echo "invalid Git SHA: $sha" >&2 - exit 2 -} - -case "$is_default_branch" in - true|false) ;; - *) - echo "is-default-branch must be true or false" >&2 - exit 2 - ;; -esac - -if [[ $is_default_branch == true ]]; then - echo latest -fi - -# A release ref must be valid enough to identify all three numeric SemVer -# components. Prerelease identifiers preserve the exact version tag, but only -# an unsuffixed release gets the stable, major and minor aliases. -if [[ $ref =~ ^refs/tags/v([0-9]+)\.([0-9]+)\.([0-9]+)(-([0-9A-Za-z]+([.-][0-9A-Za-z]+)*))?$ ]]; then - version=${ref#refs/tags/} - echo "$version" - - if [[ -z ${BASH_REMATCH[4]} ]]; then - echo stable - echo "v${BASH_REMATCH[1]}" - echo "v${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" - fi -fi - -# Match docker/metadata-action's default short-SHA spelling. -echo "sha-${sha:0:7}" diff --git a/.github/scripts/test-docker-tags.sh b/.github/scripts/test-docker-tags.sh deleted file mode 100755 index ad422f343..000000000 --- a/.github/scripts/test-docker-tags.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -policy="$root/.github/scripts/docker-tags.sh" -sha=0123456789abcdef0123456789abcdef01234567 - -assert_tags() { - local name=$1 - local ref=$2 - local is_default_branch=$3 - local expected=$4 - local actual - - actual=$($policy "$ref" "$sha" "$is_default_branch") - - if [[ $actual != "$expected" ]]; then - printf '%s: unexpected tags\nexpected:\n%s\nactual:\n%s\n' \ - "$name" "$expected" "$actual" >&2 - return 1 - fi - - printf '%s: %s\n' "$name" "$(printf '%s\n' "$actual" | tr '\n' ' ' | sed 's/ $//')" -} - -assert_tags \ - main \ - refs/heads/main \ - true \ - $'latest\nsha-0123456' - -assert_tags \ - alpha \ - refs/tags/v2.0.0-alpha.0 \ - false \ - $'v2.0.0-alpha.0\nsha-0123456' - -assert_tags \ - rc \ - refs/tags/v2.0.0-rc.0 \ - false \ - $'v2.0.0-rc.0\nsha-0123456' - -assert_tags \ - stable \ - refs/tags/v2.0.0 \ - false \ - $'v2.0.0\nstable\nv2\nv2.0\nsha-0123456' - -assert_tags \ - non-semver-v-tag \ - refs/tags/vnightly \ - false \ - 'sha-0123456' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 25476e5aa..4b8d54f0a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,29 +74,18 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - # `latest` deliberately tracks main, not the most recent release. The - # script is also the executable specification for which release aliases - # a stable or prerelease tag receives. - - name: Compute Docker tags - id: docker_tags - env: - IS_DEFAULT_BRANCH: ${{ github.ref == 'refs/heads/main' }} - run: | - tags="$(.github/scripts/docker-tags.sh "$GITHUB_REF" "$GITHUB_SHA" "$IS_DEFAULT_BRANCH")" - { - echo 'metadata<> "$GITHUB_OUTPUT" - - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v6 with: images: ghcr.io/txpipe/dolos - tags: ${{ steps.docker_tags.outputs.metadata }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') }} + type=semver,pattern=v{{major}},enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') }} + type=semver,pattern=v{{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-') }} + type=semver,pattern=v{{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=sha - name: Login to DockerHub uses: docker/login-action@v4 diff --git a/AGENTS.md b/AGENTS.md index 70e2d4196..13d449c48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,8 +27,8 @@ Dolos uses three distinct storage backends, each serving a specific purpose: - **Database**: `/wal` ### Where the indexes live -There is no standalone index store — it was removed in v2. Every index is a -projection, and lives in the store that holds what it projects: +There is no standalone index store. Every index is a projection and lives in +the store that holds what it projects: - the live-UTxO tags (by address, payment, stake, policy, asset, script ref) project the UTxO set and live in the `StateStore` (`StateStore::utxos_by_tag`, written through `StateWriter::apply_utxo_tags` in the same batch as the set) - the archive tags and the exact lookups (by block hash, block number, tx hash) project the block history and live in the `ArchiveStore` (`ArchiveStore::slots_by_tag` / `slot_by_*`, written through `ArchiveWriter::apply_index` in the same batch as the blocks) diff --git a/crates/fjall/src/archive/README.md b/crates/fjall/src/archive/README.md index 70087f83a..6ba486a06 100644 --- a/crates/fjall/src/archive/README.md +++ b/crates/fjall/src/archive/README.md @@ -11,7 +11,9 @@ This module implements the `ArchiveStore` trait using [Fjall](https://github.com | 3 | `archive-tags` | Block tags, append-only | Prefix scans by dimension and key | | 4 | `index-exact` | Block hash / block number / tx hash → slot | Point lookups | -The last two are projections of the blocks. They were a separate database until v2 (`/index`), and moving them here is what lets them be written in the same batch as the block locations they point at, so the history and its lookups commit together. They keep the compaction settings the standalone store gave them — `l0_threshold = 8`, `memtable_size_mb = 128` — so their behavior did not change with the move. +The last two are projections of the blocks. Keeping them here lets the history +and its lookups commit in the same batch. Both keyspaces use +`l0_threshold = 8` and `memtable_size_mb = 128`. ## Key Schemas @@ -75,7 +77,10 @@ Internal prefix constants: ## Stelae -The `indexes` stele layer is the sorted output of `ArchiveStore::iter_archive_tags` followed by `iter_exact_records`, and the input of `ArchiveWriter::append_prehashed`. It is byte-identical to what the standalone index store produced, so the move required no media-type bump and no backfill. See `scan.rs` for the prefix walk both traversals share. +The `indexes` stele layer is the sorted output of +`ArchiveStore::iter_archive_tags` followed by `iter_exact_records`, and the +input of `ArchiveWriter::append_prehashed`. See `scan.rs` for the prefix walk +both traversals share. ## Pruning diff --git a/crates/fjall/src/state/README.md b/crates/fjall/src/state/README.md index 9a53ba097..3471ebc33 100644 --- a/crates/fjall/src/state/README.md +++ b/crates/fjall/src/state/README.md @@ -71,7 +71,9 @@ Value: (empty) - `lookup_key`: the logical key, stored verbatim so a prefix scan can find it - `txo_ref`: `[tx_hash:32][index:4]`, the UTxO the tag points at -They are a projection of the UTxO set, so they are written in the same batch as the set (`StateWriter::apply_utxo_tags` / `undo_utxo_tags`) and read through `StateStore::utxos_by_tag`. Before v2 they lived in a separate index database; nothing about the key encoding changed with the move, only which journal, cache and write batch they live under. +They are a projection of the UTxO set, so they are written in the same batch as +the set (`StateWriter::apply_utxo_tags` / `undo_utxo_tags`) and read through +`StateStore::utxos_by_tag`. Note the asymmetry with the archive's tags, which hash their key: here the lookup key is stored whole, because a live-UTxO query knows the key it is asking about and wants the exact refs back. @@ -194,9 +196,3 @@ writer.commit()?; | Schema parameter | Not required | Required | | Entity keyspaces | Unified with hash prefix | Separate per namespace | | Multimap support | Not supported | Supported | - -## Migration Notes - -This 4-keyspace design is **not backward compatible** with previous versions that used separate keyspaces per entity type. Users must recreate their state databases when upgrading. - -The removal of the schema parameter from `StateStore::open()` is also a breaking API change. diff --git a/docs/content/architecture/data-layer.mdx b/docs/content/architecture/data-layer.mdx index a155c446c..3939a8d16 100644 --- a/docs/content/architecture/data-layer.mdx +++ b/docs/content/architecture/data-layer.mdx @@ -24,15 +24,13 @@ Under the configured `storage.path`, a running node maintains three on-disk stor | **Archive** | Immutable history: full block bodies, time-indexed logs of entity changes, and the reverse lookups over them — slots by address, payment credential, stake credential, policy, asset, datum and more, plus slot by block hash, block number and tx hash. | History is append-only and read differently from current state, so it uses its own layout (see below). The lookups are a projection of the blocks, so they live beside them and commit with them. | | **Mempool** | Submitted-but-unconfirmed transactions and their lifecycle state. | Pending transactions are transient and overlaid on top of committed state during validation. | -Before v2 the reverse lookups lived in a fourth store of their own, configured under `[storage.index]`. That store is gone, and so is the table. - ## Storage traits and pluggable backends Each store is defined as a trait in `dolos-core` — `WalStore`, `StateStore`, `ArchiveStore`, and `MempoolStore` — and the rest of Dolos only ever talks to those traits. Concrete implementations are selected at runtime in `src/adapters/storage.rs`, which wraps each backend in an enum so a node can mix engines per store. Available backends: -- **redb** (`dolos-redb3`) — an embedded ACID B+tree store. It backs the WAL (the only WAL implementation) and the mempool; its state and archive config variants were removed in v2, so a configuration naming them fails to load. +- **redb** (`dolos-redb3`) — an embedded ACID B+tree store. It backs the WAL (the only WAL implementation) and the mempool. It is not a state or archive backend. - **fjall** (`dolos-fjall`) — an LSM-tree engine tuned for write-heavy workloads with many hot keys. The only persistent backend for the state and archive stores, and the default for both. - **no-op** — a store that silently discards writes, used to *disable* the archive. - **in-memory** — non-persistent stores for testing and ephemeral nodes. State and archive have builtin implementations in `dolos-core` backed by ordered maps, which serve their traits in full; the WAL and mempool use redb's memory backend instead. diff --git a/docs/content/configuration/schema.mdx b/docs/content/configuration/schema.mdx index 3564abbe9..d67c77192 100644 --- a/docs/content/configuration/schema.mdx +++ b/docs/content/configuration/schema.mdx @@ -109,14 +109,12 @@ The `storage` section controls how Dolos stores data in the local file system. E | version | string | "v4" | - `path`: root directory where all data will be stored. -- `version`: storage schema version (`v0` - `v4`). This release only reads - `v4`, which stores independently compressed zstd-3 block frames using the - bundled dictionary and folds the former index store into state and archive; - there are no compression settings. Run `dolos init` to upgrade a readable - older configuration, or follow the manual steps in the - [migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) when it - cannot be read. Relabelling an older data directory as `v4` is unsupported - and is not detected. +- `version`: storage schema version (`v0` - `v4`). Dolos reads `v4`, which + stores independently compressed zstd-3 block frames using the bundled + dictionary and keeps indexes in the state and archive stores; there are no + compression settings. Run `dolos init` and + [bootstrap](../bootstrap) to replace data in another format. Relabelling a + data directory as `v4` is unsupported and is not detected. Besides the stores, this directory holds `scratch/`: where `dolos snapshot publish`, `dolos snapshot backfill`, `dolos bootstrap stelae` and `dolos @@ -155,7 +153,7 @@ deletes the files it has consumed as it goes. `--download-dir` moves it. | worker_threads | integer | 4 | | memtable_size_mb | integer | 64 | -- `backend`: `fjall` or `in_memory` (defaults to `fjall`). `in_memory` keeps the whole state in process memory: it is ephemeral — everything is lost on restart — and it holds the UTxO set and all entities in RAM, so it suits devnets, tooling and tests rather than a node following a public network. The `redb` value was removed in v2: a configuration still naming it fails to load, and an existing redb state directory has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available). +- `backend`: `fjall` or `in_memory` (defaults to `fjall`). `in_memory` keeps the whole state in process memory: it is ephemeral — everything is lost on restart — and it holds the UTxO set and all entities in RAM, so it suits devnets, tooling and tests rather than a node following a public network. - `path`: optional override for the state path (defaults to `/state`). - `cache`: size (MB) of the state cache. - `max_history`: maximum number of slots to keep before pruning. @@ -175,23 +173,20 @@ deletes the files it has consumed as it goes. `--download-dir` moves it. | worker_threads | integer | 4 | | memtable_size_mb | integer | 64 | -- `backend`: `fjall`, `in_memory`, or `no_op` (defaults to `fjall`). `in_memory` keeps the whole archive in process memory, block bodies included: ephemeral — everything is lost on restart — so it suits devnets, tooling and tests rather than a node following a public network. `no_op` disables the archive entirely. The `redb` value was removed in v2: a configuration still naming it fails to load, and an existing redb archive has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available). +- `backend`: `fjall`, `in_memory`, or `no_op` (defaults to `fjall`). `in_memory` keeps the whole archive in process memory, block bodies included: ephemeral — everything is lost on restart — so it suits devnets, tooling and tests rather than a node following a public network. `no_op` disables the archive entirely. - `path`: optional override for the archive path (defaults to `/archive`). - `blocks_path`: optional override for block segment files. - `cache`: size (MB) of the archive index cache. - `max_journal_size`, `flush_on_commit`, `l0_threshold`, `worker_threads`, `memtable_size_mb`: Fjall tuning options. -Block segment files are always compressed: every block body is written as one zstd frame (level 3, checksummed) with the dictionary bundled in the binary and addressed by its physical location, from the first block of a fresh instance on. Nothing about this is configurable — there is no profile, dictionary path, cache bound, training, sealing or maintenance command — and the layout is part of storage `v4`. A key the archive table does not know, such as the `block_compression` table of an earlier build, fails the configuration load rather than being ignored. +Block segment files are always compressed: every block body is written as one zstd frame (level 3, checksummed) with the dictionary bundled in the binary and addressed by its physical location, from the first block of a fresh instance on. Nothing about this is configurable — there is no profile, dictionary path, cache bound, training, sealing or maintenance command — and the layout is part of storage `v4`. Unknown archive settings fail configuration loading. The dictionary is part of the storage format, not a setting: every build carries the same 112,640-byte asset (SHA-256 `c47b2eb1f69a997bf01a720f26bcc1b668e0fddb95431e91c90cebf4f9b87139`, zstd dictionary id `1075630411`), every frame names it in its header, and a frame written for any other dictionary is refused on read. Its provenance and the evaluation behind it are in the repository under `crates/flatfiles/dictionary/`. -A data directory written by an earlier layout is not readable and is not converted: bring a `v4` instance up by a fresh bootstrap (`dolos bootstrap mithril` or `relay`) or by importing a logical snapshot (`dolos bootstrap stelae`, the shortest path), both of which write the compressed layout from the first block. The version check is the only enforcement and it is at the configuration level: a `v3` or older `dolos.toml` is refused with the remedy named, and `dolos init` rewrites it. There is no check of the data itself, so a data directory from an older layout whose configuration was relabelled `v4` by hand is unsupported and is not detected — its blocks are not frames and reads of them fail. - -### `storage.index` section — removed in v2 - -The standalone index store is gone. Its lookups moved into the stores that hold what they project: the live-UTxO tags into `storage.state`, and the historical tags and exact lookups into `storage.archive`. Drop the table from the configuration; v2 bumps the storage version, so an existing data directory has to be re-bootstrapped (a Stelae restore is the shortest path when a verified snapshot is available) and its configuration revisited anyway. - -Any tuning the table carried (`cache`, `worker_threads`, `max_journal_size`, and the rest) should be re-applied to `storage.state` and `storage.archive`, which now carry the keyspaces. `index.backend = "no_op"` had no replacement and needs none: `archive.backend = "no_op"` is the ledger-only switch and drops the historical lookups with the archive, while the live-UTxO tags stay with the UTxO set they project. +Dolos opens only storage declared as `v4`. Run `dolos init` and bootstrap with +Mithril, a relay, or a compatible Stelae snapshot to create the required +layout. The version check applies to the configuration; relabelling data in +another format as `v4` is unsupported and is not detected. ### `storage.mempool` section diff --git a/docs/content/installation/binaries.mdx b/docs/content/installation/binaries.mdx index a17e51f96..cee8c5fad 100644 --- a/docs/content/installation/binaries.mdx +++ b/docs/content/installation/binaries.mdx @@ -46,35 +46,3 @@ npm install @txpipe/dolos | [dolos-x86_64-pc-windows-msvc.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-x86_64-pc-windows-msvc.tar.gz) | x64 Windows | | [dolos-x86_64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-x86_64-unknown-linux-gnu.tar.gz) | x64 Linux | | [dolos-aarch64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/latest/download/dolos-aarch64-unknown-linux-gnu.tar.gz) | ARM64 Linux | - -## Opt in to v2.0.0-alpha.0 - -The v2 alpha is a prerelease and must be selected explicitly. The -`releases/latest` URLs above, `brew install txpipe/tap/dolos`, and -`npm install @txpipe/dolos` continue to select stable releases; ordinary -Homebrew and npm installs do not select this alpha. - -After the prerelease is published, use its exact release URLs: - -```sh -# macOS / Linux -curl --proto '=https' --tlsv1.2 -LsSf \ - https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.sh | sh - -# Windows PowerShell -powershell -ExecutionPolicy Bypass -c \ - "irm https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.ps1 | iex" -``` - -Direct downloads use the same versioned release directory: - -| File | Platform | -| --- | --- | -| [dolos-aarch64-apple-darwin.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-aarch64-apple-darwin.tar.gz) | Apple Silicon macOS | -| [dolos-x86_64-pc-windows-msvc.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-x86_64-pc-windows-msvc.tar.gz) | x64 Windows | -| [dolos-x86_64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-x86_64-unknown-linux-gnu.tar.gz) | x64 Linux | -| [dolos-aarch64-unknown-linux-gnu.tar.gz](https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-aarch64-unknown-linux-gnu.tar.gz) | ARM64 Linux | - -These links exist only after the founder publishes the GitHub prerelease. The -release configuration keeps package-manager prerelease publication disabled, -so there is intentionally no ordinary Homebrew or npm command for this alpha. diff --git a/docs/content/installation/docker.mdx b/docs/content/installation/docker.mdx index 5f0967b9a..172be48ab 100644 --- a/docs/content/installation/docker.mdx +++ b/docs/content/installation/docker.mdx @@ -44,20 +44,20 @@ To use a versioned image, replace the `latest` tag by the desired version with t ghcr.io/txpipe/dolos:v1 ``` -The `v1` tag is a floating alias that tracks the latest stable `1.x` release. For production deployments, consider pinning to a specific version (e.g. `v1.0.0`). Prereleases do not update `stable`, major aliases such as `v2`, or minor aliases such as `v2.0`. +The `v1` tag is a floating alias that tracks the latest stable `1.x` release. For production deployments, consider pinning to a specific version (e.g. `v1.0.0`). -## Opt in to v2.0.0-alpha.0 +## Prerelease Images -Pull the alpha by its exact tag after the prerelease is published: +Prerelease images are published under their exact version tag and an immutable +commit SHA tag. For example: ```sh docker pull ghcr.io/txpipe/dolos:v2.0.0-alpha.0 docker run ghcr.io/txpipe/dolos:v2.0.0-alpha.0 --help ``` -This exact tag is immutable release identity. It does not move `stable`, `v2` -or `v2.0`. It is also different from `latest`, which continues to follow -`main` and may contain changes newer than the alpha. +Prereleases do not update `stable`, major aliases such as `v2`, or minor +aliases such as `v2.0`. The `latest` tag follows `main` independently. ## Multiple Architectures diff --git a/docs/content/migration/_meta.yml b/docs/content/migration/_meta.yml deleted file mode 100644 index b878135b2..000000000 --- a/docs/content/migration/_meta.yml +++ /dev/null @@ -1,3 +0,0 @@ -label: Migration -order: 5 -collapsed: true diff --git a/docs/content/migration/dolos-v2.mdx b/docs/content/migration/dolos-v2.mdx deleted file mode 100644 index 0de821d56..000000000 --- a/docs/content/migration/dolos-v2.mdx +++ /dev/null @@ -1,137 +0,0 @@ ---- -title: Migrating to Dolos v2 -sidebar: - label: Dolos v2 - order: 1 ---- - -import { Aside } from '@astrojs/starlight/components'; - -Dolos v2 changes the on-disk storage format from `v3` to `v4`. There is no -in-place data conversion. Whether you are coming from v1.6, v1.7.0-alpha.0 or -v1.7.0-alpha.1, the supported upgrade is to preserve what you need for -rollback, initialize a v2 configuration, and bootstrap fresh data. - - - -## Before you start - -1. Stop Dolos and prevent supervisors from restarting it during the upgrade. -2. Record the Dolos binary version, configuration, genesis files, service - settings and the chain point clients currently see. -3. Copy or snapshot the complete v1 data directory and configuration if you - need a rollback path. Test that the copy can be restored before deleting the - working data. -4. Provision enough free disk for the new stores and the selected bootstrap - method. Full history costs substantially more than ledger-only or a sliding - history window. Registry restores stage compressed layers under - `/scratch` unless `--scratch-dir` points elsewhere; Mithril - uses its download directory as well as the destination stores. -5. Reserve a maintenance window for the bootstrap and tip catch-up. A Stelae - restore is dominated by transfer time, Mithril rebuilds state from verified - immutable history, and relay sync replays the chain from origin. Network, - history window, CPU and storage performance all affect the duration. - -Do not let a v1 binary write to the new v4 directory. A rollback restores the -old binary, old configuration and old data together. - -## Initialize the v2 configuration - -Install the v2 binary you intend to run, then execute: - -```sh -dolos init -``` - -If the existing configuration is readable and declares an older storage -version, `dolos init` offers to delete the managed data and bootstrap again. -Accepting that prompt clears the WAL, state, archive and other managed storage; -it is irreversible unless you made a backup. Declining leaves both the data and -configuration untouched, and v2 continues to refuse them. - -Some old configurations cannot be parsed because they name a removed backend -or a retired option. In that case `dolos init` refuses to guess the storage -path. Read the path from the old configuration, remove or move that data -yourself, preserve the old configuration for rollback, then remove the working -configuration and run `dolos init` again. - -Review the generated file instead of copying only the new version string: - -- `storage.state.backend` and `storage.archive.backend` no longer accept - `redb`. Persistent state and archive storage use `fjall`. -- The standalone `[storage.index]` table is gone. A leftover table is ignored; - it does not migrate the old index database. Reapply relevant cache, worker, - journal, flush, L0 and memtable tuning to `storage.state` and - `storage.archive`. -- Archive blocks are written as compressed frames with Dolos' bundled - dictionary. There are no operator compression or sealing settings. - -## Bootstrap fresh v4 data - -Choose a source appropriate to your trust and recovery requirements: - -- **Stelae** restores logical data without replaying every block and can resume - interrupted registry transfers. Follow the - [Stelae bootstrap guide](https://docs.txpipe.io/dolos/bootstrap/stelae) from - [PR #1217](https://github.com/txpipe/dolos/pull/1217). Use it only after that - guide is published and after the release process identifies a snapshot that - has been restored and checked for your network. -- **Mithril** downloads certificate-verified Cardano immutable history and - rebuilds Dolos state. Run `dolos bootstrap mithril`; this requires space for - the download and the resulting stores and can take hours on mainnet. -- **Relay or local node** syncs from origin with - `dolos bootstrap relay`, then `dolos daemon`. It avoids trusting derived - snapshot state but can take hours to days. -- **Legacy Dolos snapshots** package storage-engine files. Only use a snapshot - explicitly produced for storage `v4`. Snapshot paths that interpolate - `${VERSION}` use the storage version (`v4`), not the product release - (`v2.0.0-alpha.0`). - - - -## Update scripts and integrations - -The v2 storage reorganization also changes operator-facing commands and -summaries: - -| Before v2 | In v2 | -| --- | --- | -| `data export --include-indexes` | Removed; indexes travel with state or archive data | -| `data cardinality-stats` | Removed; update scripts that invoke it | -| A separate `indexes` entry in data summaries | Removed; inspect state and archive summaries | -| Index-store tuning under `[storage.index]` | Move the applicable tuning to state and archive | -| `redb` state or archive backend | Use `fjall` for persistent data | - -Custom Rust integrations must also remove `Domain::Indexes`, -`Domain::indexes()` and `WorkUnit::commit_indexes()` and adopt the state/archive -writer contracts that now own their index updates. - -## Verify before cutover - -Start Dolos, let it reach the expected point, and run the read-only consistency -audit against the new data: - -```sh -dolos data check -``` - -The full audit scans substantial state and history and can take minutes on -mainnet. Treat any reported inconsistency as a failed upgrade. Also compare the -node tip with an independent source and exercise the API endpoints your clients -depend on before directing production traffic to v2. - -Keep the v1 backup until the v2 node has completed its intended soak and your -clients have passed their checks. To roll back, stop v2, move the entire v4 -directory aside, and restore the v1 binary, configuration and data as one set. diff --git a/docs/release-notes/v2.0.0-alpha.0.md b/docs/release-notes/v2.0.0-alpha.0.md deleted file mode 100644 index 517b88ffc..000000000 --- a/docs/release-notes/v2.0.0-alpha.0.md +++ /dev/null @@ -1,92 +0,0 @@ -# Draft release notes: Dolos v2.0.0-alpha.0 - -> Publishing draft prepared against `49fbfe2b`. Recheck the final tag and links -> before publishing; this file is not the generated changelog. - -Dolos v2.0.0-alpha.0 continues the preview that began with -v1.7.0-alpha.0 and v1.7.0-alpha.1 under its final major-version identity. It -includes the work from both published alphas plus the changes merged since -alpha.1. Existing v1.7 alpha tags and artifacts remain available for -reproducibility, but new testing should use the v2 alpha. - -## What v2 adds since v1.6 - -- **Broader application data.** Mini Blockfrost adds account activity, asset, - epoch, pool, governance, script and address-derivation routes, while fixes to - input resolution, epoch history and pool data improve existing responses. -- **Computed Conway governance.** Dolos records committee, constitution and - proposal state and computes voting stake, ratification and enactment instead - of relying on a maintained outcome table. -- **Simpler, compressed storage.** State and archive now own the indexes over - their data, removing the separate index store. Archived block bodies are - compressed during normal writes with a dictionary bundled in the binary. -- **Restorable logical snapshots.** Stelae publication and restoration use - deterministic, content-addressed layers, support interrupted transfers and - make restore completion explicit. -- **Better operations.** `dolos data check` audits cursor, archive, epoch-log, - accounting and delegation consistency. Driver failures reach the process exit - status, and bootstrap and snapshot operations provide clearer progress and - capacity diagnostics. - -## A fresh bootstrap is mandatory - -V2 reads storage `v4`; v1.6 and both v1.7 alphas cannot be upgraded in place. -Back up the old binary, configuration and data if you require rollback, run -`dolos init`, and create fresh v4 data. Do not hand-edit an old configuration -to claim `version = "v4"` over old files. - -Follow the [v2 migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) -for removed backends, `[storage.index]`, CLI changes, bootstrap choices and -post-restore verification. The separate -[Stelae bootstrap guide](https://docs.txpipe.io/dolos/bootstrap/stelae) is being -reviewed in [PR #1217](https://github.com/txpipe/dolos/pull/1217); do not announce -that page or an official snapshot until both have been published and verified. - -## Install this alpha explicitly - -Stable channels do not select this prerelease. After the GitHub release is -published, use one of these exact-version commands: - -```sh -# macOS / Linux -curl --proto '=https' --tlsv1.2 -LsSf \ - https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.sh | sh - -# Windows PowerShell -powershell -ExecutionPolicy Bypass -c \ - "irm https://github.com/txpipe/dolos/releases/download/v2.0.0-alpha.0/dolos-installer.ps1 | iex" - -# Docker -docker pull ghcr.io/txpipe/dolos:v2.0.0-alpha.0 -``` - -The `releases/latest` installers and ordinary Homebrew and npm installs -continue to select stable releases. Docker `stable`, `v2`, and `v2.0` are -reserved for stable releases and are not updated by this alpha. -Docker `latest` deliberately tracks the `main` branch and is mutable; it is not -the stable-release channel. Pin `v2.0.0-alpha.0` for alpha testing. - -## Known limitations and validation still required - -- V2 is an alpha. Full per-network API conformance, sync-from-genesis, - governance parity, restore checks and an epoch-boundary soak remain release - gates before a stable v2 release. -- Snapshot support does not establish snapshot availability. Preview, preprod - and mainnet sources must each be restored and checked before being - recommended. -- Stelae content digests verify bytes, not publisher identity; snapshots are - unsigned in this release, so registry and publisher trust remains explicit. -- Historical correctness fixes may require replay or a snapshot produced from - corrected data. Continuing an old derived state from its tip cannot recreate - history an earlier binary failed to record. -- Mini Blockfrost has a broader surface but does not claim complete Blockfrost - compatibility. Validate the routes and response behavior your application - needs. - -## Review sources - -- [Changes from v1.6.0 to the reviewed candidate](https://github.com/txpipe/dolos/compare/v1.6.0...49fbfe2b) -- [v1.7.0-alpha.0 release](https://github.com/txpipe/dolos/releases/tag/v1.7.0-alpha.0) -- [v1.7.0-alpha.1 release](https://github.com/txpipe/dolos/releases/tag/v1.7.0-alpha.1) -- [Dolos v2 migration guide](https://docs.txpipe.io/dolos/migration/dolos-v2) -- [Stelae bootstrap documentation PR](https://github.com/txpipe/dolos/pull/1217) diff --git a/src/adapters/storage.rs b/src/adapters/storage.rs index 6a87e541e..3c65bb9a5 100644 --- a/src/adapters/storage.rs +++ b/src/adapters/storage.rs @@ -136,13 +136,12 @@ pub fn inspect_existing_data( )) } -/// The storage version this binary reads. A store built by an older dolos is -/// not migrated in place: the supported path off it is a fresh `dolos init` -/// followed by a restore or a re-sync. +/// The only storage version this binary reads. Data in another format must be +/// cleared and bootstrapped again. pub const CURRENT_STORAGE_VERSION: StorageVersion = StorageVersion::V4; -/// The migration guide the refusal points an operator at. -pub const MIGRATION_GUIDE_URL: &str = "https://docs.txpipe.io/dolos/migration/dolos-v2"; +/// The bootstrap guide the refusal points an operator at. +pub const BOOTSTRAP_GUIDE_URL: &str = "https://docs.txpipe.io/dolos/bootstrap"; /// Refuse a configuration at any storage version but the current one. /// @@ -156,7 +155,7 @@ fn check_storage_version(version: &StorageVersion) -> Result<(), Error> { return Err(Error::StorageError(format!( "unsupported storage version `{version}`, this dolos only supports \ `{CURRENT_STORAGE_VERSION}`; run `dolos init` to upgrade the configuration and \ - re-bootstrap the data — see the migration guide at {MIGRATION_GUIDE_URL}" + re-bootstrap the data — see the bootstrap guide at {BOOTSTRAP_GUIDE_URL}" ))); } Ok(()) @@ -1400,8 +1399,8 @@ mod tests { toml::from_str(&toml).unwrap() } - /// A v1.6-era configuration is refused, and the refusal names both the - /// tool that performs the migration and the guide that describes it. + /// An unsupported configuration is refused, and the refusal names both + /// the tool that prepares it and the bootstrap guide. #[test] fn older_storage_versions_are_refused_with_the_remedy() { for stale in [ @@ -1425,8 +1424,8 @@ mod tests { "refusal must name the remedy: {message}" ); assert!( - message.contains(MIGRATION_GUIDE_URL), - "refusal must point at the migration guide: {message}" + message.contains(BOOTSTRAP_GUIDE_URL), + "refusal must point at the bootstrap guide: {message}" ); } diff --git a/src/bin/dolos/init.rs b/src/bin/dolos/init.rs index 1552a91c8..04feb8592 100644 --- a/src/bin/dolos/init.rs +++ b/src/bin/dolos/init.rs @@ -16,7 +16,7 @@ use std::{ str::FromStr, }; -use dolos::storage::{CURRENT_STORAGE_VERSION, MIGRATION_GUIDE_URL}; +use dolos::storage::{BOOTSTRAP_GUIDE_URL, CURRENT_STORAGE_VERSION}; use crate::{data::cleanup, feedback::Feedback}; @@ -595,8 +595,8 @@ impl ConfigEditor { StorageUpgrade::Unreadable => Err(miette!( "found a `{CONFIG_FILE}` this dolos cannot read, so the storage it configures \ cannot be located. Remove the data directory your existing `{CONFIG_FILE}` \ - names, then delete the file and run `dolos init` again — see the migration \ - guide at {MIGRATION_GUIDE_URL}" + names, then delete the file and run `dolos init` again — see the bootstrap \ + guide at {BOOTSTRAP_GUIDE_URL}" )), StorageUpgrade::Offer => { let delete = Confirm::new("Your storage is incompatible with the current version. Do you want to delete data and bootstrap?") @@ -624,7 +624,7 @@ impl ConfigEditor { return Err(miette!( "storage upgrade declined, leaving the existing setup untouched. Dolos will \ keep refusing to start against it until the data is re-bootstrapped — see \ - the migration guide at {MIGRATION_GUIDE_URL}" + the bootstrap guide at {BOOTSTRAP_GUIDE_URL}" )); } @@ -941,8 +941,8 @@ mod tests { .expect("declining aborts the init"); assert!( - error.to_string().contains(MIGRATION_GUIDE_URL), - "the abort must point at the migration guide: {error}" + error.to_string().contains(BOOTSTRAP_GUIDE_URL), + "the abort must point at the bootstrap guide: {error}" ); assert!( all_present(&files), @@ -1057,8 +1057,8 @@ mod tests { .expect("an unreadable config aborts the init"); assert!( - error.to_string().contains(MIGRATION_GUIDE_URL), - "the abort must point at the migration guide: {error}" + error.to_string().contains(BOOTSTRAP_GUIDE_URL), + "the abort must point at the bootstrap guide: {error}" ); assert!( all_present(&files), @@ -1149,7 +1149,7 @@ is_testnet = true .err() .expect("an unreadable config aborts the init"); - assert!(error.to_string().contains(MIGRATION_GUIDE_URL)); + assert!(error.to_string().contains(BOOTSTRAP_GUIDE_URL)); assert_eq!(std::fs::read_to_string(&config_file).unwrap(), unreadable); assert!(all_present(&files), "the abort must delete nothing");