diff --git a/crates/snapshot/src/facade.rs b/crates/snapshot/src/facade.rs new file mode 100644 index 000000000..fb90755c1 --- /dev/null +++ b/crates/snapshot/src/facade.rs @@ -0,0 +1,13 @@ +//! Supported headless snapshot assembly surface. +//! +//! This module gathers the profile's host-facing lifecycle without hiding the +//! narrower modules that implement it. A host can plan, encode, inspect, +//! verify and restore without importing a Dolos binary module or constructing +//! terminal feedback. + +pub use crate::{ + publisher::{Next, Publisher, RepositoryPublish}, + registry::{Auth, Point, Repository, SnapshotRepository, Tuning}, + restore::{execute as restore, Input as RestoreInput, RestoreOutcome, Restoring, Target}, + source::{Selection, SnapshotSource, StoreSnapshot}, +}; diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index e8c7feb78..6dd4b1dc9 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -48,6 +48,8 @@ //! to the protocol in the order above. //! - [`restore`] — its inverse: the driver that reads a stele back into an //! empty store set, in the order ADR-004 specifies. +//! - [`facade`] — supported headless assembly for planning, encoding, +//! inspection, verification and directory/OCI restoration. //! - [`preflight`] — the free-space policy both drivers refuse under, so a run //! that cannot fit its volume says so at minute zero. //! - `registry` (feature `oci`) — publishing into an OCI repository: the @@ -66,6 +68,7 @@ #[cfg(feature = "backfill")] pub mod backfill; pub mod export; +pub mod facade; pub mod layers; pub mod namespaces; pub mod node; diff --git a/crates/snapshot/src/publisher.rs b/crates/snapshot/src/publisher.rs index d43b64f48..7729a34ac 100644 --- a/crates/snapshot/src/publisher.rs +++ b/crates/snapshot/src/publisher.rs @@ -24,7 +24,10 @@ use stelae_driver::Standing; use crate::{ export::Plan, node, - registry::{self, Auth, Preview, Published, Publishing, Registry, Repository, Tuning}, + registry::{ + self, Auth, Preview, Published, Publishing, Registry, Repository, SnapshotRepository, + Tuning, + }, DolosProfile, Error, }; @@ -140,22 +143,40 @@ impl Publisher { ) -> Result { let scratch = node::scratch_dir(&config.storage, publish.scratch_dir); - let registry = registry::open( + Self::open_explicit( publish.repo, publish.insecure, auth, scratch, + registry::record_path_in(&config.storage.path), + publish.rebuild, publish.tuning, - )?; + ) + } + + /// Open a publisher from resolved host policy without a Dolos root + /// configuration. + /// + /// The journal path is explicit because it is application state: an + /// external publisher may place it beside its own stores without adopting + /// the Dolos command's directory layout. + #[allow(clippy::too_many_arguments)] + pub fn open_explicit( + repository: &Repository, + insecure: bool, + auth: Auth, + scratch_dir: PathBuf, + record_path: PathBuf, + rebuild: bool, + tuning: Tuning, + ) -> Result { + let registry = + SnapshotRepository::open(repository, insecure, auth, scratch_dir, tuning)?.into_inner(); Ok(Self { registry, - // The resumption record sits beside the stores, so an interrupted - // publish restarted against this repository carries forward the - // epoch layers it already uploaded instead of rebuilding them. - // `--rebuild` starts it over along with everything else. - record_path: registry::record_path_in(&config.storage.path), - rebuild: publish.rebuild, + record_path, + rebuild, }) } diff --git a/crates/snapshot/src/registry.rs b/crates/snapshot/src/registry.rs index 3933a1e6b..9521f592d 100644 --- a/crates/snapshot/src/registry.rs +++ b/crates/snapshot/src/registry.rs @@ -290,6 +290,63 @@ pub fn open( )?) } +/// An opened snapshot repository for inspection, verification and restore. +/// +/// The host supplies transport policy explicitly: repository, plaintext opt-in, +/// credentials, staging directory and tuning. No configuration files, +/// environment variables or terminal components are consulted here. +pub struct SnapshotRepository { + registry: Registry, +} + +impl SnapshotRepository { + /// Open a repository with fully resolved host inputs. + /// + /// **Never call this from inside an async context.** The OCI transport owns + /// its runtime; see [`open`]. + pub fn open( + repository: &Repository, + insecure: bool, + auth: Auth, + scratch_dir: PathBuf, + tuning: Tuning, + ) -> Result { + Ok(Self { + registry: open(repository, insecure, auth, scratch_dir, tuning)?, + }) + } + + /// Read the canonical document and manifest metadata without pulling + /// layers. + pub fn inspect(&self, point: Point) -> Result { + inspect(&self.registry, point) + } + + /// Stream and verify every layer named by the published document. + pub fn verify(&self, point: Point) -> Result { + verify(&self.registry, point) + } + + /// Restore one published stele into the explicit target stores. + pub fn restore( + &self, + point: Point, + node: Restoring<'_>, + target: Target<'_, A, S>, + observer: &Observer, + ) -> Result<(crate::restore::Plan, Outlook, Summary), Error> + where + A: ArchiveStore, + S: StateStore, + { + restore_registry(&self.registry, point, node, target, observer) + } + + pub(crate) fn into_inner(self) -> Registry { + self.registry + } +} + pub use stelae_driver::publish::Tuning; /// Where a publish is going and what the host running it knows about itself, diff --git a/crates/snapshot/src/restore.rs b/crates/snapshot/src/restore.rs index f825c722e..22479b93c 100644 --- a/crates/snapshot/src/restore.rs +++ b/crates/snapshot/src/restore.rs @@ -117,6 +117,16 @@ //! and any earlier chunk that was already committed is harmless for the same //! reason `set_cursor` is last: a restore that fails leaves no cursor, so what //! it wrote is not a node. +//! +//! ## External-host initialization +//! +//! Restoration is deliberately not node initialization. An embedding +//! application should open isolated scratch stores, call [`execute`], close +//! those stores, recover the replay checkpoint/WAL through its engine facade, +//! and only then open a replay session. A failed restore never selects genesis, +//! writes an application initialization marker or removes a directory. Genesis +//! fallback, marker ownership and destructive cleanup are host policy and must +//! remain explicit at that boundary. use std::{ collections::BTreeMap, @@ -1136,6 +1146,59 @@ where restore_stele(&stele, node, None, target, observer) } +/// A fully resolved restore input. +/// +/// The repository arm holds an already-opened client, so credentials, TLS +/// policy, staging and transport tuning have been decided before destination +/// stores are touched. +pub enum Input<'a> { + Directory(&'a Path), + Repository { + repository: &'a crate::registry::SnapshotRepository, + point: crate::registry::Point, + }, +} + +/// The structured result of a completed restore. +#[derive(Debug)] +pub struct RestoreOutcome { + pub plan: Plan, + pub outlook: Outlook, + pub summary: Summary, +} + +/// Restore from either supported transport through one headless entry point. +/// +/// `node` contains the explicit resume and space-check policy. Passing no +/// observer selects the protocol's silent observer and never constructs a +/// terminal renderer. +pub fn execute( + input: Input<'_>, + node: Restoring<'_>, + target: Target<'_, A, S>, + observer: Option<&Observer>, +) -> Result +where + A: ArchiveStore, + S: StateStore, +{ + let silent = Observer::silent(); + let observer = observer.unwrap_or(&silent); + + let (plan, outlook, summary) = match input { + Input::Directory(path) => restore_dir(path, node, target, observer)?, + Input::Repository { repository, point } => { + repository.restore(point, node, target, observer)? + } + }; + + Ok(RestoreOutcome { + plan, + outlook, + summary, + }) +} + /// The stele a restore is reading, and the terms it reads under. /// /// Carried as one value because every layer is read the same way, and because diff --git a/crates/snapshot/src/source.rs b/crates/snapshot/src/source.rs index cb3e4da14..e3109d84a 100644 --- a/crates/snapshot/src/source.rs +++ b/crates/snapshot/src/source.rs @@ -1,20 +1,79 @@ -//! Read-only access to the Dolos snapshot profile. +//! Headless access to the Dolos snapshot profile. +//! +//! A host owns storage lifecycle and application policy. This module gives it +//! one read-only view for planning, directory publication, digest reproduction +//! and verification without exposing the underlying store handles. + +use std::{num::NonZeroUsize, path::Path}; use dolos_core::{ArchiveStore, ChainPoint, StateStore}; use stelae::progress::Observer; use crate::{ - export::{self, Plan}, + export::{self, Document, Plan, Predecessor}, + inscription::Inscription, + planning::{self, EpochRange}, publisher::Publisher, registry::{Preview, Published}, Error, RetainedEpochs, }; +/// The selection shared by publication, digest and reproduction. +/// +/// All fields are optional so [`Selection::default`] preserves the profile's +/// measured defaults and selects every epoch available at the current cursor. +#[derive(Debug, Clone, Copy, Default)] +pub struct Selection { + pub epochs: Option, + pub index_band: Option, + pub producers: Option, +} + +impl Selection { + /// Apply every selection knob through the profile's canonical planning + /// functions. The order is shared by all hosts and commands. + pub fn apply(self, plan: Plan) -> Plan { + let plan = planning::restrict(plan, self.epochs); + let plan = planning::banded(plan, self.index_band); + planning::produced(plan, self.producers) + } +} + /// Profile operations available without exposing mutable storage handles. pub trait SnapshotSource { fn committed_position(&self) -> Result, Error>; fn epoch(&self) -> Result, Error>; fn plan(&self, network_magic: u64, retained: RetainedEpochs) -> Result; + + /// Build and select a plan in the same way for every snapshot operation. + fn selected_plan( + &self, + network_magic: u64, + retained: RetainedEpochs, + selection: Selection, + ) -> Result { + Ok(selection.apply(self.plan(network_magic, retained)?)) + } + + /// Write a directory stele through the same profile encoder publication + /// and registry publication use. + fn publish_directory( + &self, + destination: &Path, + plan: &Plan, + observer: &Observer, + ) -> Result; + + /// Reproduce a canonical inscription without writing it. + fn digest_document(&self, plan: &Plan, previous: &dyn Predecessor) -> Result; + + /// Rebuild every layer and compare it with a published inscription. + fn verify_reproduction( + &self, + published: &Inscription, + plan: &Plan, + ) -> Result; + fn preview(&self, publisher: &Publisher, plan: &Plan) -> Result; fn publish( &self, @@ -59,6 +118,27 @@ impl SnapshotSource for StoreSnapshot<'_, A, S> export::plan(self.state, network_magic, retained) } + fn publish_directory( + &self, + destination: &Path, + plan: &Plan, + observer: &Observer, + ) -> Result { + export::publish(destination, plan, self.archive, self.state, None, observer) + } + + fn digest_document(&self, plan: &Plan, previous: &dyn Predecessor) -> Result { + export::digest_document(plan, self.archive, self.state, previous) + } + + fn verify_reproduction( + &self, + published: &Inscription, + plan: &Plan, + ) -> Result { + export::verify_reproduction(published, plan, self.archive, self.state, None) + } + fn preview(&self, publisher: &Publisher, plan: &Plan) -> Result { publisher.preview(plan, self.archive) } diff --git a/crates/snapshot/tests/export.rs b/crates/snapshot/tests/export.rs index fc772c2c1..391b131ce 100644 --- a/crates/snapshot/tests/export.rs +++ b/crates/snapshot/tests/export.rs @@ -50,6 +50,7 @@ use dolos_core::{ }; use dolos_snapshot::{ export::{self, EpochWindow, Plan}, + facade::{Selection, SnapshotSource as _, StoreSnapshot}, layers::{blocks, indexes, logs, state}, state_layer_count, state_ns_for, DolosProfile, Network, RetainedEpochs, BLOCKS, INDEXES, LOG_KINDS, LOG_NAMESPACES, NAMESPACES, STATE_KINDS, UTXOS, @@ -1729,3 +1730,29 @@ fn banding_moves_no_bytes() { ); } } + +/// The supported facade keeps planning, directory encoding, digest and +/// reproduction on one store view and one plan. +#[test] +fn the_headless_facade_plans_encodes_and_reproduces_one_document() { + let domain: ToyDomain = harness(); + let source = StoreSnapshot::new(domain.archive(), domain.state()); + let plan = source + .selected_plan( + u64::from(domain.genesis().network_magic()), + RetainedEpochs::default(), + Selection::default(), + ) + .unwrap(); + + let destination = tempfile::tempdir().unwrap(); + let stele = destination.path().join("stele"); + let inscription = source + .publish_directory(&stele, &plan, &Observer::silent()) + .unwrap(); + let reproduced = source.verify_reproduction(&inscription, &plan).unwrap(); + let document = source.digest_document(&plan, &export::First).unwrap(); + + assert_eq!(reproduced.canonicalize().unwrap(), document.canonical); + assert_eq!(inscription.digest().unwrap(), document.identity); +} diff --git a/crates/snapshot/tests/registry_fixture/mod.rs b/crates/snapshot/tests/registry_fixture/mod.rs index 228647ae2..63cc5681f 100644 --- a/crates/snapshot/tests/registry_fixture/mod.rs +++ b/crates/snapshot/tests/registry_fixture/mod.rs @@ -227,6 +227,22 @@ impl Fixture { .unwrap() } + /// Open the same repository through the supported headless read facade. + pub fn snapshot_repository(&self, name: &str) -> registry::SnapshotRepository { + let repository = format!("oci://127.0.0.1:{}/{name}", self.port) + .parse() + .expect("the fixture named a usable repository"); + + registry::SnapshotRepository::open( + &repository, + true, + credentials(), + self.scratch.path().to_path_buf(), + registry::Tuning::default(), + ) + .unwrap() + } + /// Where the transports this fixture opens stage their layers. /// /// Exposed so a suite can hold a transport's own answer against what the diff --git a/crates/snapshot/tests/restore.rs b/crates/snapshot/tests/restore.rs index 098437347..d72200643 100644 --- a/crates/snapshot/tests/restore.rs +++ b/crates/snapshot/tests/restore.rs @@ -141,6 +141,45 @@ fn target(blank: &Blank) -> restore::Target<'_, impl ArchiveSto restore::Target::new(&blank.archive, blank.state()) } +/// The public assembly path accepts an explicit directory source and target +/// and can run without constructing a terminal observer. +#[test] +fn the_headless_facade_restores_a_directory_without_an_observer() { + let domain: ToyDomain = harness(); + let source = tempfile::tempdir().unwrap(); + let storage = tempfile::tempdir().unwrap(); + export_to(source.path(), &domain); + + let blank = Blank::::open(); + let outcome = restore::execute( + restore::Input::Directory(source.path()), + restore::Restoring { + network_magic: magic_of(&domain), + max_history: None, + storage_path: storage.path(), + resume: false, + skip_space_check: false, + }, + target(&blank), + None, + ) + .unwrap(); + + assert_eq!( + outcome.plan.position.point, + domain.state().read_cursor().unwrap().unwrap() + ); + assert_eq!( + blank.state().read_cursor().unwrap(), + domain.state().read_cursor().unwrap() + ); + assert_eq!(outcome.summary.layers_skipped, 0); + assert_eq!( + outcome.summary.layers_fetched, + outcome.plan.layers().count() + ); +} + // -------------------------------------------------------------------------- // 1. Refusals // -------------------------------------------------------------------------- @@ -1209,6 +1248,50 @@ fn restore_watched( } } +/// An interrupted low-level read can be resumed through the public assembly +/// facade, proving the facade honors the same checkpoint contract. +#[test] +fn the_headless_facade_resumes_an_interrupted_directory_restore() { + let domain: ToyDomain = harness(); + let magic = magic_of(&domain); + let stele = tempfile::tempdir().unwrap(); + export_to(stele.path(), &domain); + + let (epoch_layers, _) = layers_in_driver_order(stele.path(), magic); + let storage = tempfile::tempdir().unwrap(); + let blank = Blank::::open(); + + restore_checkpointed( + stele.path(), + storage.path(), + magic, + &blank, + false, + Some(epoch_layers[1]), + ) + .unwrap_err(); + + let resumed = restore::execute( + restore::Input::Directory(stele.path()), + restore::Restoring { + network_magic: magic, + max_history: None, + storage_path: storage.path(), + resume: true, + skip_space_check: false, + }, + target(&blank), + None, + ) + .unwrap(); + + assert_eq!(resumed.summary.layers_skipped, 1); + assert_eq!( + blank.state().read_cursor().unwrap(), + domain.state().read_cursor().unwrap() + ); +} + /// Done criterion 2: killed mid-way, resumed, and the same node — having /// refetched only what it had not finished. /// diff --git a/crates/snapshot/tests/restore_registry.rs b/crates/snapshot/tests/restore_registry.rs index 4468317fd..8c2441149 100644 --- a/crates/snapshot/tests/restore_registry.rs +++ b/crates/snapshot/tests/restore_registry.rs @@ -52,6 +52,7 @@ use dolos_core::{ }; use dolos_snapshot::{ export::Plan, + facade::RestoreInput, registry::{self, Point}, restore::{self, default_budget, progress_path_in, Checkpoint}, state_layer_count, Error, Network, NAMESPACES, UTXOS, @@ -225,7 +226,8 @@ fn a_registry_restore_is_a_directory_restore() { let fixture = Fixture::spawn(); let node = Node::build(); - let repository = fixture.repository("dolos/restore"); + let repository_name = "dolos/restore"; + let repository = fixture.repository(repository_name); node.publish(&repository, &node.first); // The same stele, written to a directory. @@ -261,15 +263,18 @@ fn a_registry_restore_is_a_directory_restore() { let from_registry = Blank::::open(); let registry_storage = tempfile::tempdir().unwrap(); - let by_registry = restore_from( - &repository, - Point::Latest, - registry_storage.path(), - node.magic, - &from_registry, - false, + let source = fixture.snapshot_repository(repository_name); + let by_registry = restore::execute( + RestoreInput::Repository { + repository: &source, + point: Point::Latest, + }, + restoring(registry_storage.path(), node.magic, false), + target(&from_registry), + None, ) - .unwrap(); + .unwrap() + .summary; assert_eq!( by_registry, by_dir, diff --git a/crates/snapshot/tests/snapshot_verify.rs b/crates/snapshot/tests/snapshot_verify.rs index 65845dca7..998ea9eed 100644 --- a/crates/snapshot/tests/snapshot_verify.rs +++ b/crates/snapshot/tests/snapshot_verify.rs @@ -67,7 +67,8 @@ const EPOCH_0: usize = 4; fn a_freshly_published_stele_verifies_clean() { let fixture = Fixture::spawn(); let node = Node::build(); - let repository = fixture.repository("dolos/verify-clean"); + let repository_name = "dolos/verify-clean"; + let repository = fixture.repository(repository_name); let first = node.publish(&repository, &node.first, false); let second = node.publish(&repository, &node.second, false); @@ -77,7 +78,8 @@ fn a_freshly_published_stele_verifies_clean() { "the interesting stele is one with inherited layers" ); - let verified = registry::verify(&repository, Point::Latest).unwrap(); + let reader = fixture.snapshot_repository(repository_name); + let verified = reader.verify(Point::Latest).unwrap(); assert_eq!(verified.identity, second.identity); assert_eq!( @@ -87,7 +89,7 @@ fn a_freshly_published_stele_verifies_clean() { assert!(verified.compressed_bytes > 0); // The immutable tag reads the predecessor back just as clean. - let predecessor = registry::verify(&repository, Point::Epoch(0)).unwrap(); + let predecessor = reader.verify(Point::Epoch(0)).unwrap(); assert_eq!(predecessor.identity, first.identity); @@ -307,12 +309,14 @@ fn a_reproduction_passes_at_the_published_epoch_and_fails_at_another() { fn an_inspection_reports_the_manifest_and_its_json_chains_a_digest() { let fixture = Fixture::spawn(); let node = Node::build(); - let repository = fixture.repository("dolos/inspect"); + let repository_name = "dolos/inspect"; + let repository = fixture.repository(repository_name); let first = node.publish(&repository, &node.first, false); let second = node.publish(&repository, &node.second, false); - let inspected = registry::inspect(&repository, Point::Latest).unwrap(); + let reader = fixture.snapshot_repository(repository_name); + let inspected = reader.inspect(Point::Latest).unwrap(); assert_eq!(inspected.identity, second.identity); assert_eq!( @@ -335,7 +339,7 @@ fn an_inspection_reports_the_manifest_and_its_json_chains_a_digest() { assert_eq!(total, inspected.total_compressed); // The `--json` output is the canonical document, verbatim. - let predecessor = registry::inspect(&repository, Point::Epoch(0)).unwrap(); + let predecessor = reader.inspect(Point::Epoch(0)).unwrap(); assert_eq!(predecessor.identity, first.identity); diff --git a/docs/headless-replay.md b/docs/headless-replay.md index 0cc5b7494..ab570af3f 100644 --- a/docs/headless-replay.md +++ b/docs/headless-replay.md @@ -14,8 +14,9 @@ cargo run --no-default-features --example headless_replay -- \ - `ReplayWorkspace::open(config, genesis)` opens a dataset without running ledger initialization, replaying pending work, or pruning history. - `workspace.snapshot()` lends read-only Dolos profile operations: - committed position, epoch, publication planning, preview and publication. - No domain or writable storage handle escapes the view. + committed position, selected planning, directory or registry publication, + digest reproduction and verification. No domain or writable storage handle + escapes the view. See `docs/headless-snapshots.md` for the restore boundary. - `workspace.start(stop_epoch)` consumes the inspection workspace and explicitly permits initialization and processing. Publish any pending boundary first. - `session.import_blocks(blocks)` processes a nonempty batch of trusted immutable diff --git a/docs/headless-snapshots.md b/docs/headless-snapshots.md new file mode 100644 index 000000000..3fecd265a --- /dev/null +++ b/docs/headless-snapshots.md @@ -0,0 +1,66 @@ +# Embedding Dolos snapshots + +The snapshot profile can be embedded without Dolos service features. The +`headless_snapshot` example opens committed stores through the replay facade, +builds one selected plan, writes a directory stele, and reproduces it through +the public API: + +```sh +cargo run --no-default-features --example headless_snapshot -- \ + ./dolos.toml ./stele +``` + +This facade was qualified from Dolos revision +`867b8596626b9d58246093a7d66f1f943af524eb`, the accepted headless replay +engine from PR #1327, against the pinned Stelae protocol/driver v0.2.0 source +revision `703cc34b`. + +## Planning and reproduction + +`ReplayWorkspace::snapshot()` returns a read-only `SnapshotSource`. The host +supplies the network magic, retained epochs and `facade::Selection`; the +library applies epoch restriction, index banding and producer sizing in the +same order for publication, digest and verification. The resulting plan can +be passed to: + +- `publish_directory` with an explicit destination and progress observer; +- `preview` or `publish` with an opened `publisher::Publisher`; +- `digest_document` with an explicit predecessor; or +- `verify_reproduction` with the published inscription. + +These operations return plans, documents, inscriptions or typed errors. They +do not print, exit, load configuration, or construct terminal components. + +## Registry reads and restore + +`facade::SnapshotRepository::open` accepts a repository, plaintext opt-in, +resolved `Auth`, scratch directory and transport tuning. The resulting client +supports `inspect`, `verify`, and registry restore. `facade::restore` is the +single restore entry point for both `RestoreInput::Directory` and +`RestoreInput::Repository`; it accepts explicit target stores, `Restoring` +resume/space policy, and an optional observer. `None` +selects a silent observer. + +The library deliberately does not infer credentials or a scratch directory. +An application may use Dolos's `node` helpers to resolve its existing config, +or provide its own policy. `publisher::Publisher::open_explicit` similarly +accepts the resolved repository, credentials, scratch and journal paths. + +## Safe cold-start sequence + +Treat restoration and initialization as separate state transitions: + +1. Resolve the source and credentials before changing the destination. +2. Create an isolated storage root and open its state and archive stores. +3. Call `restore::execute`. On failure, leave the application uninitialized; + do not fall back to genesis implicitly. +4. Close the restored stores, then open `ReplayWorkspace` on that storage root. +5. Call `workspace.start(...)`. This recovers the replay checkpoint/WAL and + performs normal Dolos domain initialization without advancing a pending + publication boundary. +6. Write any application initialization marker only after those steps succeed. + +Directory deletion, reuse of partial stores, genesis fallback, and marker +placement are application policy. Keep them explicit. In particular, never +restore over a live publisher dataset or run two publishers against the same +writable stores or repository head. diff --git a/examples/headless_snapshot.rs b/examples/headless_snapshot.rs new file mode 100644 index 000000000..431fdec8b --- /dev/null +++ b/examples/headless_snapshot.rs @@ -0,0 +1,76 @@ +//! Minimal external host for Dolos snapshot planning, encoding and +//! reproduction. +//! +//! Build without Dolos service features: +//! +//! ```text +//! cargo run --no-default-features --example headless_snapshot -- \ +//! ./dolos.toml ./stele +//! ``` +//! +//! The destination must not already contain a stele. Configuration precedence, +//! progress rendering and destination cleanup remain host responsibilities. + +use std::{error::Error, path::Path, sync::Arc}; + +use dolos::core::Genesis; +use dolos::engine::ReplayWorkspace; +use dolos_snapshot::{ + export::First, + facade::{Selection, SnapshotSource as _}, + progress::Observer, +}; + +type AnyError = Box; + +fn main() -> Result<(), AnyError> { + let mut args = std::env::args_os().skip(1); + let config_path = args.next().ok_or("missing path to dolos.toml")?; + let destination = args.next().ok_or("missing output directory")?; + + let config: dolos::core::config::RootConfig = config::Config::builder() + .add_source(config::File::from(Path::new(&config_path))) + .build()? + .try_deserialize()?; + + let genesis = Arc::new(Genesis::from_file_paths( + &config.genesis.byron_path, + &config.genesis.shelley_path, + &config.genesis.alonzo_path, + &config.genesis.conway_path, + config.genesis.force_protocol, + )?); + + // Opening a workspace does not initialize or advance the ledger. It gives + // the host an exclusive read-only snapshot view over the committed stores. + let workspace = ReplayWorkspace::open(&config, genesis.clone())?; + + let snapshot_result = (|| -> Result<(), AnyError> { + let snapshot = workspace.snapshot(); + let retained = dolos_snapshot::planning::retained_epochs(&config)?; + let plan = snapshot.selected_plan( + u64::from(genesis.network_magic()), + retained, + Selection::default(), + )?; + + let inscription = + snapshot.publish_directory(Path::new(&destination), &plan, &Observer::silent())?; + + // Reproduction uses the same plan and encoders and writes nothing. + snapshot.verify_reproduction(&inscription, &plan)?; + let document = snapshot.digest_document(&plan, &First)?; + + eprintln!("sequence: {}", plan.sequence); + eprintln!("layers: {}", document.layers); + eprintln!("identity: {}", document.identity); + Ok(()) + })(); + + let finish_result = workspace.finish(); + + snapshot_result?; + finish_result?; + + Ok(()) +} diff --git a/src/bin/dolos/bootstrap/stelae.rs b/src/bin/dolos/bootstrap/stelae.rs index 814bc4092..c9631b882 100644 --- a/src/bin/dolos/bootstrap/stelae.rs +++ b/src/bin/dolos/bootstrap/stelae.rs @@ -40,8 +40,9 @@ use dolos_core::config::RootConfig; use miette::{Context as _, IntoDiagnostic as _}; use dolos_snapshot::{ + facade::{self as snapshot, RestoreInput, SnapshotRepository}, node, - registry::{self, Point, Repository}, + registry::{self, Point}, restore::Source, }; @@ -160,71 +161,54 @@ impl Node { } } -fn restore_dir( - config: &RootConfig, - dir: &std::path::Path, - options: RestoreOptions<'_>, -) -> miette::Result<()> { - let node = Node::open(config)?; - let progress = SteleProgress::restoring(options.feedback); - - let (plan, outlook, summary) = dolos_snapshot::restore::restore_dir( - dir, - node.restoring(options.resume, options.skip_space_check), - node.target(), - &progress.observer(), - ) - .into_diagnostic() - .context("restoring the stele")?; - - progress.finish(); - - report(&plan, &outlook, &summary); - - Ok(()) -} - -fn restore_repo( - config: &RootConfig, - repo: &Repository, - point: Point, - insecure: bool, - scratch_dir: Option<&std::path::Path>, - options: RestoreOptions<'_>, -) -> miette::Result<()> { - // First: `Node::open` runs `ensure_storage_path`, so the default of - // `/scratch` needs no special case on a host where the - // storage directory does not exist yet. +fn restore(config: &RootConfig, args: &Args, options: RestoreOptions<'_>) -> miette::Result<()> { let node = Node::open(config)?; - - // Resolved here rather than inside the transport: which identity this node - // reads a registry as is the node's policy, and `dolos_snapshot::node` is - // where that policy lives. Where it stages comes from the same place. - let auth = node::registry_auth(&config.stelae).into_diagnostic()?; - - let scratch = node::scratch_dir(&config.storage, scratch_dir); - - let registry = registry::open(repo, insecure, auth, scratch, registry::Tuning::default()) - .into_diagnostic() - .context("opening the repository")?; - - println!("source: {repo} ({point})"); - let progress = SteleProgress::restoring(options.feedback); - - let (plan, outlook, summary) = registry::restore_registry( - ®istry, - point, - node.restoring(options.resume, options.skip_space_check), - node.target(), - &progress.observer(), - ) + let observer = progress.observer(); + let restoring = node.restoring(options.resume, options.skip_space_check); + + let outcome = match &args.source { + Source::Dir(dir) => snapshot::restore( + RestoreInput::Directory(dir), + restoring, + node.target(), + Some(&observer), + ), + Source::Repo(repo) => { + // Credential and staging resolution remain node policy. The + // profile facade receives the resolved inputs and never reads + // configuration or environment on its own. + let auth = node::registry_auth(&config.stelae).into_diagnostic()?; + let scratch = node::scratch_dir(&config.storage, args.scratch_dir.as_deref()); + let repository = SnapshotRepository::open( + repo, + args.insecure, + auth, + scratch, + registry::Tuning::default(), + ) + .into_diagnostic() + .context("opening the repository")?; + + println!("source: {repo} ({})", args.point); + + snapshot::restore( + RestoreInput::Repository { + repository: &repository, + point: args.point, + }, + restoring, + node.target(), + Some(&observer), + ) + } + } .into_diagnostic() .context("restoring the stele")?; progress.finish(); - report(&plan, &outlook, &summary); + report(&outcome.plan, &outcome.outlook, &outcome.summary); Ok(()) } @@ -336,17 +320,7 @@ pub fn run( skip_space_check: args.skip_space_check, }; - match &args.source { - Source::Dir(dir) => restore_dir(config, dir, options), - Source::Repo(repo) => restore_repo( - config, - repo, - args.point, - args.insecure, - args.scratch_dir.as_deref(), - options, - ), - } + restore(config, args, options) } #[cfg(test)] diff --git a/src/bin/dolos/snapshot/digest.rs b/src/bin/dolos/snapshot/digest.rs index 73f664ed5..bec232838 100644 --- a/src/bin/dolos/snapshot/digest.rs +++ b/src/bin/dolos/snapshot/digest.rs @@ -48,7 +48,10 @@ use std::path::PathBuf; use clap::Parser; use dolos_core::config::RootConfig; -use dolos_snapshot::export::{self, Following, Plan, Predecessor}; +use dolos_snapshot::{ + export::{Following, Plan, Predecessor}, + facade::{SnapshotSource as _, StoreSnapshot}, +}; use miette::{Context as _, IntoDiagnostic as _}; use super::EpochRange; @@ -98,6 +101,7 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { }; let plan = super::planned(config, &stores, &selection, "planning the reproduction")?; + let snapshot = StoreSnapshot::new(&stores.archive, &stores.state); super::report_plan(&plan)?; @@ -112,7 +116,7 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { let predecessor: &dyn Predecessor = match &previous { Some(following) => following, - None => &export::First, + None => &dolos_snapshot::export::First, }; match previous.as_ref().map(Predecessor::history) { @@ -120,7 +124,8 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { None => eprintln!("history: none; this reproduction starts a chain"), } - let document = export::digest_document(&plan, &stores.archive, &stores.state, predecessor) + let document = snapshot + .digest_document(&plan, predecessor) .into_diagnostic() .context("reproducing the stele")?; diff --git a/src/bin/dolos/snapshot/inspect.rs b/src/bin/dolos/snapshot/inspect.rs index 05ba76951..a7b5c0ca6 100644 --- a/src/bin/dolos/snapshot/inspect.rs +++ b/src/bin/dolos/snapshot/inspect.rs @@ -22,6 +22,7 @@ use clap::Parser; use dolos_core::config::RootConfig; use dolos_snapshot::{ + facade::SnapshotRepository, node, registry::{self, Point, Repository}, }; @@ -60,7 +61,7 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { // and this directory is never created here. let scratch = node::scratch_dir(&config.storage, None); - let registry = registry::open( + let repository = SnapshotRepository::open( &args.repo, args.insecure, auth, @@ -70,7 +71,8 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { .into_diagnostic() .context("opening the repository")?; - let inspected = registry::inspect(®istry, args.point) + let inspected = repository + .inspect(args.point) .into_diagnostic() .context("reading the stele")?; diff --git a/src/bin/dolos/snapshot/mod.rs b/src/bin/dolos/snapshot/mod.rs index 6330de3a3..399cd425a 100644 --- a/src/bin/dolos/snapshot/mod.rs +++ b/src/bin/dolos/snapshot/mod.rs @@ -24,7 +24,8 @@ use clap::{Parser, Subcommand}; use dolos_core::config::RootConfig; use dolos_snapshot::{ - export::{self, Plan}, + export::Plan, + facade::{Selection, SnapshotSource as _, StoreSnapshot}, planning::{self, PlanReport}, }; use miette::{Context as _, IntoDiagnostic as _}; @@ -82,18 +83,6 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res } } -/// The three knobs every command that walks these stores takes. -/// -/// Spelled per command rather than flattened into one clap group, because the -/// help text is not the same everywhere: `verify` takes all three only under -/// `--reproduce`, and says so. What is shared is what they mean, which is -/// [`dolos_snapshot::planning`]'s. -pub struct Selection { - pub epochs: Option, - pub index_band: Option, - pub producers: Option, -} - /// This node's plan, narrowed by the operator's selection. /// /// One sequence for `publish`, `digest` and `verify --reproduce`, because a @@ -112,14 +101,10 @@ pub fn planned( .into_diagnostic() .context("reading snapshot.state_epochs")?; - let plan = export::plan(&stores.state, u64::from(genesis.network_magic()), retained) + StoreSnapshot::new(&stores.archive, &stores.state) + .selected_plan(u64::from(genesis.network_magic()), retained, *selection) .into_diagnostic() - .context(what)?; - - let plan = planning::restrict(plan, selection.epochs); - let plan = planning::banded(plan, selection.index_band); - - Ok(planning::produced(plan, selection.producers)) + .context(what) } /// The report every command opens with: where the node stands and what the diff --git a/src/bin/dolos/snapshot/publish.rs b/src/bin/dolos/snapshot/publish.rs index 10d251cfd..8868200b0 100644 --- a/src/bin/dolos/snapshot/publish.rs +++ b/src/bin/dolos/snapshot/publish.rs @@ -2,11 +2,12 @@ use std::path::PathBuf; use clap::Parser; use dolos_core::config::RootConfig; -use dolos_snapshot::source::{SnapshotSource, StoreSnapshot}; use miette::{Context as _, IntoDiagnostic as _}; use dolos_snapshot::{ - export, node, + export, + facade::{SnapshotSource, StoreSnapshot}, + node, publisher::{Next, Publisher, RepositoryPublish}, registry::{self, Repository}, }; @@ -112,6 +113,7 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res let plan = super::planned(config, &stores, &selection, "planning the publish")?; super::report_plan(&plan)?; + let snapshot = StoreSnapshot::new(&stores.archive, &stores.state); match (&args.repo, &args.output_dir) { (Some(repo), _) => { @@ -128,15 +130,9 @@ pub fn run(config: &RootConfig, args: &Args, feedback: &Feedback) -> miette::Res }, }; - to_repository( - config, - &publish, - &plan, - &StoreSnapshot::new(&stores.archive, &stores.state), - feedback, - ) + to_repository(config, &publish, &plan, &snapshot, feedback) } - (None, Some(dir)) => to_directory(args, dir, &plan, &stores, feedback), + (None, Some(dir)) => to_directory(args, dir, &plan, &snapshot, feedback), // The required `destination` group already refuses this. (None, None) => unreachable!("one of --output-dir and --repo is required"), } @@ -146,7 +142,7 @@ fn to_directory( args: &Args, dir: &std::path::Path, plan: &export::Plan, - stores: &crate::common::Stores, + source: &dyn SnapshotSource, feedback: &Feedback, ) -> miette::Result<()> { if args.dry_run { @@ -159,16 +155,10 @@ fn to_directory( // walking a repository publish pays, and they are what this reports. let progress = SteleProgress::publishing(feedback); - let inscription = export::publish( - dir, - plan, - &stores.archive, - &stores.state, - None, - &progress.observer(), - ) - .into_diagnostic() - .context("exporting the stele")?; + let inscription = source + .publish_directory(dir, plan, &progress.observer()) + .into_diagnostic() + .context("exporting the stele")?; progress.finish(); diff --git a/src/bin/dolos/snapshot/verify.rs b/src/bin/dolos/snapshot/verify.rs index c1324ea1c..5ed7abb8b 100644 --- a/src/bin/dolos/snapshot/verify.rs +++ b/src/bin/dolos/snapshot/verify.rs @@ -34,6 +34,7 @@ use clap::Parser; use dolos_core::config::RootConfig; use dolos_snapshot::{ + facade::{SnapshotRepository, SnapshotSource as _, StoreSnapshot}, node, registry::{self, Point, Repository}, }; @@ -95,7 +96,7 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { // than moving a node's own data. let scratch = node::scratch_dir(&config.storage, None); - let registry = registry::open( + let repository = SnapshotRepository::open( &args.repo, args.insecure, auth, @@ -105,7 +106,8 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { .into_diagnostic() .context("opening the repository")?; - let verified = registry::verify(®istry, args.point) + let verified = repository + .verify(args.point) .into_diagnostic() .context("verifying the stele")?; @@ -151,15 +153,10 @@ pub fn run(config: &RootConfig, args: &Args) -> miette::Result<()> { super::report_plan(&plan)?; - let reproduced = dolos_snapshot::export::verify_reproduction( - &verified.inscription, - &plan, - &stores.archive, - &stores.state, - None, - ) - .into_diagnostic() - .context("reproducing the published stele from the local stores")?; + let reproduced = StoreSnapshot::new(&stores.archive, &stores.state) + .verify_reproduction(&verified.inscription, &plan) + .into_diagnostic() + .context("reproducing the published stele from the local stores")?; println!( "reproduced: {} layers rebuilt from the local stores; the documents are byte-identical \