feat(bench): add minibf performance regression harness

AGENTS.md

@@ -146,7 +146,7 @@ The project follows a modular workspace architecture with clear separation of co
 
 #### `xtask` (Development Automation)
 - **Purpose**: Development task automation following cargo-xtask pattern
-- **Role**: Build scripts and development utilities, including `cargo xtask archive-bench` — the archive segment benchmarks (store-level presets, node-level paired runs of two `dolos` binaries, dictionary tooling, report); see `xtask/archive-bench/README.md`
+- **Role**: Build scripts and development utilities, including `cargo xtask perf` — storage and minibf performance experiments with shared measurement and reporting; see `xtask/perf/README.md`
 
 ## Dependency Flow
 

benches/archive_backends.rs

@@ -1,10 +1,12 @@
-//! Regression guards for the archive backend's three read-shape families.
+//! Regression guards for logs, store-local indexes and compressed bodies.
 //!
 //! The store is populated with the synthetic content from
 //! `dolos_testing::archive` and measured against a fixed key sample: block
 //! location resolution and full block reads (`get_block_by_slot`), the
 //! per-account reward point reads (`read_logs` per epoch), and the per-epoch
 //! temporal-prefix scan (`iter_logs`).
+//! The queries module adds populated-domain index lookups, valid compressed
+//! bodies, a real Alonzo block and logs-only scale cases.
 //!
 //! These are *relative* regression guards on small tempdir populations.
 //! Absolute numbers say nothing about production behavior — the
@@ -16,6 +18,9 @@ use dolos_core::{
 };
 use dolos_testing::archive::{populate_archive, ArchiveShape};
 
+#[path = "archive_backends/queries.rs"]
+mod queries;
+
 /// The one namespace the log shapes exercise; the name is the real bulk
 /// namespace so the bench reads like the node.
 const NS: &str = "account-epochs";

benches/archive_backends/queries.rs

@@ -0,0 +1,182 @@
+use dolos_core::{ArchiveStore, Domain, StateStore};
+use dolos_testing::{
+    performance::{ApiFixture, FixtureShape},
+    toy_domain::{FjallStores, ToyStores},
+};
+use pallas::ledger::{addresses::Address, traverse::MultiEraBlock};
+
+fn fixture(blocks: usize) -> ApiFixture<FjallStores> {
+    ApiFixture::new(
+        FjallStores::open(),
+        FixtureShape {
+            blocks,
+            ..Default::default()
+        },
+    )
+    .unwrap()
+}
+
+#[divan::bench(args = [16, 64], sample_count = 30)]
+fn archive_address_tags(bencher: divan::Bencher, blocks: usize) {
+    let fixture = fixture(blocks);
+    let key = Address::from_bech32(&fixture.vectors.address)
+        .unwrap()
+        .to_vec();
+    let expected: Vec<_> = fixture
+        .vectors
+        .blocks
+        .iter()
+        .take(blocks)
+        .map(|block| block.slot)
+        .collect();
+    bencher.bench_local(|| {
+        let actual = fixture
+            .domain
+            .archive()
+            .slots_by_tag("address", &key, 0, u64::MAX)
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+        assert_eq!(divan::black_box(actual), expected);
+    });
+}
+
+#[divan::bench(args = [16, 64], sample_count = 30)]
+fn exact_transaction_lookup(bencher: divan::Bencher, blocks: usize) {
+    let fixture = fixture(blocks);
+    let keys: Vec<_> = fixture
+        .vectors
+        .blocks
+        .iter()
+        .take(blocks)
+        .map(|block| (hex::decode(&block.tx_hashes[0]).unwrap(), block.slot))
+        .collect();
+    bencher.bench_local(|| {
+        for (hash, slot) in &keys {
+            assert_eq!(
+                fixture
+                    .domain
+                    .archive()
+                    .slot_by_tx_hash(divan::black_box(hash))
+                    .unwrap(),
+                Some(*slot)
+            );
+        }
+        assert_eq!(
+            fixture
+                .domain
+                .archive()
+                .slot_by_tx_hash(&[0xff; 32])
+                .unwrap(),
+            None
+        );
+    });
+}
+
+#[divan::bench(args = [16, 64], sample_count = 30)]
+fn state_address_tags(bencher: divan::Bencher, blocks: usize) {
+    let fixture = fixture(blocks);
+    let key = Address::from_bech32(&fixture.vectors.address)
+        .unwrap()
+        .to_vec();
+    bencher.bench_local(|| {
+        let actual = fixture
+            .domain
+            .state()
+            .utxos_by_tag("address", divan::black_box(&key))
+            .unwrap();
+        assert_eq!(actual.len(), blocks);
+    });
+}
+
+#[divan::bench(args = [16, 64], sample_count = 30)]
+fn compressed_valid_block_reads(bencher: divan::Bencher, blocks: usize) {
+    let fixture = fixture(blocks);
+    bencher.bench_local(|| {
+        for (index, expected) in fixture.blocks.iter().enumerate().step_by(3) {
+            let slot = fixture.vectors.blocks[index].slot;
+            let body = fixture
+                .domain
+                .archive()
+                .get_block_by_slot(&slot)
+                .unwrap()
+                .unwrap();
+            assert_eq!(divan::black_box(body.as_slice()), expected.as_slice());
+        }
+    });
+}
+
+#[divan::bench(args = [16, 64], sample_count = 30)]
+fn compressed_reverse_block_scan(bencher: divan::Bencher, blocks: usize) {
+    let fixture = fixture(blocks);
+    bencher.bench_local(|| {
+        let mut count = 0;
+        for (slot, body) in fixture
+            .domain
+            .archive()
+            .get_range(Some(1), Some(blocks as u64 + 1))
+            .unwrap()
+            .rev()
+        {
+            let block = MultiEraBlock::decode(&body).unwrap();
+            assert_eq!(block.slot(), slot);
+            divan::black_box(block.header().hash());
+            count += 1;
+        }
+        assert_eq!(count, blocks);
+    });
+}
+
+#[divan::bench(args = [1000, 10000], sample_count = 30)]
+fn logs_only_scale(bencher: divan::Bencher, rows: u64) {
+    use dolos_testing::archive::{populate_archive, ArchiveShape};
+    let shape = ArchiveShape {
+        epochs: 4,
+        blocks_per_epoch: 0,
+        log_rows_per_epoch: rows,
+        slots_per_epoch: 432_000,
+        seed: 0,
+    };
+    let directory = tempfile::tempdir().unwrap();
+    let store = dolos_fjall::archive::ArchiveStore::open(
+        dolos_cardano::model::build_schema(),
+        directory.path(),
+        &Default::default(),
+    )
+    .unwrap();
+    populate_archive(&store, "account-epochs", &shape).unwrap();
+    assert!(store.get_tip().unwrap().is_none());
+    let range = shape.log_key(2, 0)..shape.log_key(3, 0);
+    bencher.bench_local(|| {
+        let mut count = 0;
+        for entry in store.iter_logs("account-epochs", range.clone()).unwrap() {
+            divan::black_box(entry.unwrap());
+            count += 1;
+        }
+        assert_eq!(count, rows);
+    });
+}
+
+#[divan::bench(sample_count = 50)]
+fn compressed_real_alonzo_block(bencher: divan::Bencher) {
+    use dolos_core::{ArchiveWriter, ChainPoint};
+    let body = std::sync::Arc::new(
+        hex::decode(include_str!("../../crates/cardano/test_data/alonzo27.block").trim()).unwrap(),
+    );
+    let block = MultiEraBlock::decode(&body).unwrap();
+    let point = ChainPoint::Specific(block.slot(), block.hash());
+    let directory = tempfile::tempdir().unwrap();
+    let store = dolos_fjall::archive::ArchiveStore::open(
+        dolos_cardano::model::build_schema(),
+        directory.path(),
+        &Default::default(),
+    )
+    .unwrap();
+    let writer = store.start_writer().unwrap();
+    writer.apply(&point, &body).unwrap();
+    writer.commit().unwrap();
+    bencher.bench_local(|| {
+        let actual = store.get_block_by_slot(&point.slot()).unwrap().unwrap();
+        assert_eq!(divan::black_box(actual.as_slice()), body.as_slice());
+    });
+}

crates/flatfiles/dictionary/README.md

@@ -24,7 +24,7 @@ command over the same files yields the same bytes.
 
 Three candidates were trained from the same corpus with the same seed and
 evaluated with the harness's `evaluate` command (then
-`dolos-archive-bench evaluate`, now `cargo xtask archive-bench evaluate`)
+`dolos-archive-bench evaluate`, now `cargo xtask perf storage evaluate`)
 on data disjoint from every
 training segment: mainnet segments 448–455 (156,621 Conway blocks, 877 MiB,
 the held-out window), one 20,000-block fixture per mainnet era, preprod

crates/testing/Cargo.toml

@@ -18,3 +18,4 @@ tokio-stream = { workspace = true }
 futures-core = { workspace = true }
 futures-util = { workspace = true }
 itertools = { workspace = true }
+serde = { workspace = true, features = ["derive"] }

crates/testing/src/archive.rs

@@ -53,7 +53,7 @@ impl ArchiveShape {
     /// [`Self::block_slots`] and [`populate_archive`] so the slots callers
     /// sample are exactly the slots the population wrote.
     pub fn stride(&self) -> u64 {
-        (self.slots_per_epoch / self.blocks_per_epoch).max(1)
+        (self.slots_per_epoch / self.blocks_per_epoch.max(1)).max(1)
     }
 
     /// Every block slot the population writes, in ascending order. Callers
@@ -97,33 +97,64 @@ fn filler(rng: &mut SplitMix64, len: usize) -> Vec<u8> {
     out
 }
 
-/// Write `shape` into `store`: one committed writer per epoch carrying that
-/// epoch's blocks (~1-2 KiB bodies) and its log rows (~30-60 B values) under
-/// `ns`, which must exist in the store's schema.
+/// Write `shape` into `store` in batches of at most 1,000 records. Blocks
+/// contain filler (~1-2 KiB) and log values contain filler (~30-60 B).
+/// Zero blocks per epoch produces a logs-only population.
 pub fn populate_archive<S: ArchiveStore>(
     store: &S,
     ns: Namespace,
     shape: &ArchiveShape,
 ) -> Result<(), ArchiveError> {
+    populate_archive_batched(store, ns, shape, 1000)
+}
+
+pub fn populate_archive_batched<S: ArchiveStore>(
+    store: &S,
+    ns: Namespace,
+    shape: &ArchiveShape,
+    batch_size: usize,
+) -> Result<(), ArchiveError> {
+    if batch_size == 0
+        || shape.slots_per_epoch == 0
+        || shape.blocks_per_epoch > shape.slots_per_epoch
+        || shape.epochs.checked_mul(shape.slots_per_epoch).is_none()
+    {
+        return Err(ArchiveError::InternalError(
+            "invalid archive population shape or batch size".into(),
+        ));
+    }
     let mut rng = SplitMix64(shape.seed);
     let stride = shape.stride();
 
     for epoch in 0..shape.epochs {
-        let writer = store.start_writer()?;
         let epoch_start = shape.epoch_start(epoch);
 
-        for i in 0..shape.blocks_per_epoch {
-            let slot = epoch_start + i * stride;
-            let len = 1024 + (rng.next_u64() % 1024) as usize;
-            writer.apply(&block_point(slot), &Arc::new(filler(&mut rng, len)))?;
+        for start in (0..shape.blocks_per_epoch).step_by(batch_size) {
+            let writer = store.start_writer()?;
+            for block in start
+                ..start
+                    .saturating_add(batch_size as u64)
+                    .min(shape.blocks_per_epoch)
+            {
+                let slot = epoch_start + block * stride;
+                let len = 1024 + (rng.next_u64() % 1024) as usize;
+                writer.apply(&block_point(slot), &Arc::new(filler(&mut rng, len)))?;
+            }
+            writer.commit()?;
         }
 
-        for i in 0..shape.log_rows_per_epoch {
-            let len = 30 + (rng.next_u64() % 31) as usize;
-            writer.write_log(ns, &shape.log_key(epoch, i), &filler(&mut rng, len))?;
+        for start in (0..shape.log_rows_per_epoch).step_by(batch_size) {
+            let writer = store.start_writer()?;
+            for row in start
+                ..start
+                    .saturating_add(batch_size as u64)
+                    .min(shape.log_rows_per_epoch)
+            {
+                let len = 30 + (rng.next_u64() % 31) as usize;
+                writer.write_log(ns, &shape.log_key(epoch, row), &filler(&mut rng, len))?;
+            }
+            writer.commit()?;
         }
-
-        writer.commit()?;
     }
 
     Ok(())

crates/testing/src/lib.rs

@@ -23,6 +23,8 @@ use dolos_core::*;
 pub mod archive;
 pub mod blocks;
 pub mod faults;
+pub mod measured;
+pub mod performance;
 pub mod synthetic;
 
 pub mod toy_domain;