diff --git a/bin/node/src/commands/lifecycle.rs b/bin/node/src/commands/lifecycle.rs index cc5660ecb7..ce3a306c96 100644 --- a/bin/node/src/commands/lifecycle.rs +++ b/bin/node/src/commands/lifecycle.rs @@ -57,7 +57,7 @@ impl BootstrapCommand { let genesis_block = read_bootstrap_genesis_block(self.genesis_block_file.as_deref(), self.network).await?; let genesis_commitment = genesis_block.inner().header().commitment(); - State::bootstrap(genesis_block, &self.data_directory)?; + State::bootstrap(genesis_block, &self.data_directory).await?; tracing::info!( target: crate::LOG_TARGET, { diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 073a96f85c..6761eaa79a 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -234,7 +234,9 @@ pub async fn seed_store_with_readers( ); let genesis_block = genesis_state.into_block().expect("genesis block should be created"); let genesis_header = genesis_block.inner().header().clone(); - State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); + State::bootstrap(genesis_block, &data_directory) + .await + .expect("store should bootstrap"); let (state, mut block_writer, writer_task) = load_state(data_directory.clone()).await; diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index db9efcad5b..7eb0120386 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -1,21 +1,13 @@ use std::num::NonZeroUsize; -use std::sync::Arc; use std::time::Duration; use miden_node_store::GenesisState; use miden_node_store::state::State; use miden_node_utils::fee::test_fee_params; -use miden_protocol::Word; -use miden_protocol::batch::ProvenBatch; -use miden_protocol::block::{BlockHeader, BlockNumber, ValidatorKeys}; +use miden_protocol::block::{BlockNumber, ValidatorKeys}; use miden_protocol::testing::random_secret_key::random_secret_key; use url::Url; -use crate::domain::transaction::AuthenticatedTransaction; -use crate::mempool::{Mempool, MempoolConfig}; -use crate::server::MempoolStats; -use crate::test_utils::MockProvenTxBuilder; -use crate::test_utils::batch::TransactionBatchConstructor; use crate::{ DEFAULT_BATCH_WORKERS, DEFAULT_MAX_BATCHES_PER_BLOCK, @@ -25,54 +17,10 @@ use crate::{ Sequencer, }; -#[test] -fn mempool_stats_track_uncommitted_work_and_the_canonical_tip() { - let shared = Mempool::shared(BlockNumber::GENESIS, MempoolConfig::default()); - let mut mempool = shared.lock().unwrap(); - let tx = Arc::new(AuthenticatedTransaction::from_inner( - MockProvenTxBuilder::with_account_index(100).build(), - )); - - mempool.add_transaction(Arc::clone(&tx)).unwrap(); - let stats = MempoolStats::from_mempool(&mempool); - assert_eq!(stats.chain_tip, BlockNumber::GENESIS); - assert_eq!(stats.uncommitted_transactions, 1); - assert_eq!(stats.unbatched_transactions, 1); - assert_eq!(stats.proposed_batches, 0); - assert_eq!(stats.proven_batches, 0); - - mempool.select_any_batch().unwrap(); - let stats = MempoolStats::from_mempool(&mempool); - assert_eq!(stats.uncommitted_transactions, 1); - assert_eq!(stats.unbatched_transactions, 0); - assert_eq!(stats.proposed_batches, 1); - assert_eq!(stats.proven_batches, 0); - - mempool.commit_batch(Arc::new(ProvenBatch::mocked_from_transactions([ - tx.raw_proven_transaction() - ]))); - let stats = MempoolStats::from_mempool(&mempool); - assert_eq!(stats.proposed_batches, 0); - assert_eq!(stats.proven_batches, 1); - - let block = mempool.select_block(); - let stats = MempoolStats::from_mempool(&mempool); - assert_eq!(stats.chain_tip, BlockNumber::GENESIS); - assert_eq!(stats.uncommitted_transactions, 1); - assert_eq!(stats.proven_batches, 0); - - let header = BlockHeader::mock(block.block_number, None, None, &[], Word::empty()); - mempool.commit_block(&header); - let stats = MempoolStats::from_mempool(&mempool); - assert_eq!(stats.chain_tip, BlockNumber::GENESIS.child()); - assert_eq!(stats.uncommitted_transactions, 0); - assert_eq!(stats.proven_batches, 0); -} - #[tokio::test] async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); - bootstrap_store(data_directory.path()); + bootstrap_store(data_directory.path()).await; let (state, block_writer, proof_writer) = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { @@ -99,7 +47,7 @@ async fn block_producer_starts_with_store_state() { assert_eq!(status.chain_tip, BlockNumber::GENESIS); } -fn bootstrap_store(path: &std::path::Path) { +async fn bootstrap_store(path: &std::path::Path) { let signer = random_secret_key(); let genesis_state = GenesisState::new( vec![], @@ -110,5 +58,5 @@ fn bootstrap_store(path: &std::path::Path) { ); let genesis_block = genesis_state.into_block().expect("genesis block should be created"); - State::bootstrap(genesis_block, path).expect("store should bootstrap"); + State::bootstrap(genesis_block, path).await.expect("store should bootstrap"); } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 2757232077..9bd30185ca 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -102,7 +102,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); - let genesis_commitment = Self::bootstrap(&data_directory); + let genesis_commitment = Self::bootstrap(&data_directory).await; let (state, ..) = State::for_tests(&data_directory).await; Self { state, @@ -111,7 +111,7 @@ impl TestStore { } } - fn bootstrap(path: &std::path::Path) -> Word { + async fn bootstrap(path: &std::path::Path) -> Word { let config = GenesisConfig::default(); let validator_key = miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey::read_from_bytes(&[7; 32]) @@ -124,7 +124,7 @@ impl TestStore { genesis_state.clone().into_block().expect("genesis block should be created"); let genesis_commitment = genesis_block.inner().header().commitment(); - State::bootstrap(genesis_block, path).expect("store should bootstrap"); + State::bootstrap(genesis_block, path).await.expect("store should bootstrap"); genesis_commitment } @@ -443,7 +443,8 @@ async fn rpc_rejects_post_deployment_network_account_tx() { miden_node_store::test_support::seed_network_account( &store.data_directory_path().join("miden-store.sqlite3"), network_account_id, - ); + ) + .await; // Build a non-deployment tx for that account. let (account, _) = build_test_account([0; 32]); @@ -577,7 +578,7 @@ async fn start_source_rpc( ) -> (RpcClient, TestStore, TestServerGuard) { let store = TestStore::start().await; let block_producer_dir = new_tempdir(); - TestStore::bootstrap(&block_producer_dir); + TestStore::bootstrap(&block_producer_dir).await; let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; let state = Arc::clone(&store.state); @@ -1071,7 +1072,7 @@ async fn start_rpc() -> (RpcClient, std::net::SocketAddr, TestStore, TestServerG let grpc_options = GrpcOptions::test(); let store = TestStore::start().await; let block_producer_dir = new_tempdir(); - TestStore::bootstrap(&block_producer_dir); + TestStore::bootstrap(&block_producer_dir).await; let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; let state = Arc::clone(&store.state); diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index b4680482fa..fbe3ccb4fb 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -42,8 +42,8 @@ use miden_protocol::{EMPTY_WORD, Word}; use thiserror::Error; use crate::COMPONENT; -pub use crate::db::models::queries::HISTORICAL_BLOCK_RETENTION; -use crate::db::models::queries::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; +pub use crate::db::HISTORICAL_BLOCK_RETENTION; +use crate::db::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; use crate::errors::AccountStateForestUpdateError; #[cfg(test)] diff --git a/crates/store/src/db/migrations.rs b/crates/store/src/db/migrations.rs index 213b6df53c..2ca8a0e1de 100644 --- a/crates/store/src/db/migrations.rs +++ b/crates/store/src/db/migrations.rs @@ -63,21 +63,5 @@ pub fn verify_latest_schema(database_filepath: &Path) -> std::result::Result<(), Ok(()) } -#[cfg(test)] -pub(crate) fn test_connection() -> diesel::SqliteConnection { - use diesel::{Connection, SqliteConnection}; - - let temp_dir = tempfile::tempdir().expect("failed to create temp directory"); - let database_filepath = temp_dir.path().join("test.sqlite3"); - bootstrap_database(&database_filepath).expect("database should bootstrap"); - - let conn = SqliteConnection::establish( - database_filepath.to_str().expect("temp database path should be valid UTF-8"), - ) - .expect("temp file sqlite should always work"); - let _kept_dir = temp_dir.keep(); - conn -} - #[cfg(test)] mod tests; diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 2155eeaf9c..7c61d85d4f 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::Context; -use diesel::{Connection, SqliteConnection}; +use miden_node_db::sqlite::{DbReader, DbWriter}; use miden_node_proto::domain::account::AccountInfo; use miden_node_utils::limiter::{ MAX_RESPONSE_PAYLOAD_BYTES, @@ -40,17 +40,18 @@ use tracing::info; use crate::db::migrations::{migrate_database, verify_latest_schema}; use crate::db::models::conv::SqlTypeConvert; -use crate::db::models::queries; +use crate::db::models::queries as diesel_queries; +use crate::db::models::queries::StorageMapValuesPage; pub use crate::db::models::queries::{ AccountCommitmentsPage, NullifiersPage, PublicAccountIdsPage, PublicAccountStateRootsPage, }; -use crate::db::models::queries::{ - BlockHeaderCommitment, +pub use crate::db::queries::{ + HISTORICAL_BLOCK_RETENTION, + PrecomputedPublicAccountState, PrecomputedPublicAccountStates, - StorageMapValuesPage, }; use crate::errors::{DatabaseError, NoteSyncError}; use crate::genesis::GenesisBlock; @@ -71,6 +72,18 @@ pub(crate) use migrations::bootstrap_database; #[cfg(test)] mod tests; +#[cfg(test)] +mod test_db; +#[cfg(test)] +pub(crate) use test_db::TestDb; + +/// Query functions on the `miden-node-db` SQLite framework. +/// +/// All writes run here; reads are migrated from [`models`] incrementally. +pub(crate) mod queries; + +mod utils; + pub(crate) mod models; /// [diesel](https://diesel.rs) generated schema @@ -99,21 +112,45 @@ impl Default for DatabaseOptions { /// The Store's database. /// /// Extends the underlying [`miden_node_db::Db`] type with functionality specific to the Store. +/// +/// The store is mid-migration to the `miden-node-db` SQLite framework: every write serializes on +/// the single framework writer connection, while most reads still run on the diesel pool. Reads +/// move to the framework reader pool one batch at a time until the diesel pool is removed. pub struct Db { - db: miden_node_db::Db, + diesel: miden_node_db::Db, + writer: DbWriter, + reader: DbReader, } impl Deref for Db { type Target = miden_node_db::Db; fn deref(&self) -> &Self::Target { - &self.db + &self.diesel } } impl DerefMut for Db { fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.db + &mut self.diesel + } +} + +/// The commitment of a [`BlockHeader`], stored alongside the header it belongs to. +/// +/// Keeping it in its own column lets the chain MMR be rebuilt at startup without deserializing +/// every header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct BlockHeaderCommitment(pub(crate) Word); + +impl BlockHeaderCommitment { + pub fn new(header: &BlockHeader) -> Self { + Self(header.commitment()) + } + + pub fn word(self) -> Word { + self.0 } } @@ -212,28 +249,29 @@ impl Db { fields(path=%database_filepath.display()) err, )] - pub fn bootstrap(database_filepath: PathBuf, genesis: GenesisBlock) -> anyhow::Result<()> { + pub async fn bootstrap( + database_filepath: PathBuf, + genesis: GenesisBlock, + ) -> anyhow::Result<()> { migrations::bootstrap_database(&database_filepath) .context("failed to bootstrap database schema")?; - let mut conn: SqliteConnection = diesel::sqlite::SqliteConnection::establish( - database_filepath.to_str().context("database filepath is invalid")?, - ) - .context("failed to open a database connection")?; - - miden_node_db::configure_connection_on_creation(&mut conn)?; + let (writer, _reader) = miden_node_db::sqlite::open(&database_filepath) + .context("failed to open a database connection")?; // Insert genesis block data. let genesis_block = genesis.into_inner(); - conn.transaction(move |conn| { - models::queries::apply_block( - conn, - &genesis_block, - &[], - &PrecomputedPublicAccountStates::new(), - ) - }) - .context("failed to insert genesis block")?; + writer + .write::<_, DatabaseError, _>("insert genesis block", move |tx| { + queries::apply_block( + tx, + &genesis_block, + &[], + &PrecomputedPublicAccountStates::new(), + ) + }) + .await + .context("failed to insert genesis block")?; Ok(()) } @@ -258,6 +296,8 @@ impl Db { verify_latest_schema(&database_filepath)?; let db = miden_node_db::Db::new_with_pool_size(&database_filepath, connection_pool_size)?; + let (writer, reader) = + miden_node_db::sqlite::open_with_pool_size(&database_filepath, connection_pool_size)?; info!( target: LOG_TARGET, sqlite= %database_filepath.display(), @@ -265,7 +305,13 @@ impl Db { "Connected to the database" ); - Ok(Self { db }) + Ok(Self { diesel: db, writer, reader }) + } + + /// The write handle, for tests that need to seed or corrupt rows no production method writes. + #[cfg(test)] + pub(crate) fn writer(&self) -> &DbWriter { + &self.writer } /// Applies all pending migrations to an existing DB. @@ -289,7 +335,7 @@ impl Db { after_nullifier: Option, ) -> Result { self.transact("read nullifiers paged", move |conn| { - queries::select_nullifiers_paged(conn, page_size, after_nullifier) + diesel_queries::select_nullifiers_paged(conn, page_size, after_nullifier) }) .await } @@ -316,7 +362,7 @@ impl Db { self.transact("nullifieres by prefix", move |conn| { let nullifier_prefixes = Vec::from_iter(nullifier_prefixes.into_iter().map(|prefix| prefix as u16)); - queries::select_nullifiers_by_prefix( + diesel_queries::select_nullifiers_by_prefix( conn, prefix_len as u8, &nullifier_prefixes[..], @@ -339,7 +385,7 @@ impl Db { maybe_block_number: Option, ) -> Result> { self.transact("block headers by block number", move |conn| { - let val = queries::select_block_header_by_block_num( + let val = diesel_queries::select_block_header_by_block_num( conn, maybe_block_number.map(|block_number| *block_number), )?; @@ -360,8 +406,10 @@ impl Db { block_number: ScopedBlockNum, ) -> Result> { self.transact("block headers and signatures by block number", move |conn| { - let val = - queries::select_block_header_and_signatures_by_block_num(conn, *block_number)?; + let val = diesel_queries::select_block_header_and_signatures_by_block_num( + conn, + *block_number, + )?; Ok(val) }) .await @@ -378,7 +426,7 @@ impl Db { blocks: impl Iterator + Send + 'static, ) -> Result> { self.transact("block headers from given block numbers", move |conn| { - let raw = queries::select_block_headers(conn, blocks.map(|block| *block))?; + let raw = diesel_queries::select_block_headers(conn, blocks.map(|block| *block))?; Ok(raw) }) .await @@ -392,7 +440,7 @@ impl Db { )] pub async fn select_all_block_header_commitments(&self) -> Result> { self.transact("all block headers", |conn| { - let raw = queries::select_all_block_header_commitments(conn)?; + let raw = diesel_queries::select_all_block_header_commitments(conn)?; Ok(raw) }) .await @@ -410,7 +458,7 @@ impl Db { after_account_id: Option, ) -> Result { self.transact("read account commitments paged", move |conn| { - queries::select_account_commitments_paged(conn, page_size, after_account_id) + diesel_queries::select_account_commitments_paged(conn, page_size, after_account_id) }) .await } @@ -427,7 +475,7 @@ impl Db { after_account_id: Option, ) -> Result { self.transact("read public account IDs paged", move |conn| { - queries::select_public_account_ids_paged(conn, page_size, after_account_id) + diesel_queries::select_public_account_ids_paged(conn, page_size, after_account_id) }) .await } @@ -444,7 +492,11 @@ impl Db { after_account_id: Option, ) -> Result { self.transact("read public account state roots paged", move |conn| { - queries::select_public_account_state_roots_paged(conn, page_size, after_account_id) + diesel_queries::select_public_account_state_roots_paged( + conn, + page_size, + after_account_id, + ) }) .await } @@ -456,7 +508,7 @@ impl Db { err, )] pub async fn select_account(&self, id: AccountId) -> Result { - self.transact("Get account details", move |conn| queries::select_account(conn, id)) + self.transact("Get account details", move |conn| diesel_queries::select_account(conn, id)) .await } @@ -466,14 +518,15 @@ impl Db { target = COMPONENT, err, )] - pub async fn select_network_accounts_subset( + pub async fn filter_network_accounts( &self, account_ids: Vec, ) -> Result> { - self.transact("Filter network accounts subset", move |conn| { - queries::select_network_accounts_subset(conn, &account_ids) - }) - .await + self.reader + .read("Filter network accounts", move |tx| { + queries::filter_network_accounts(tx, &account_ids) + }) + .await } /// Queries the account code by its commitment hash. @@ -487,7 +540,7 @@ impl Db { code_commitment: Word, ) -> Result>> { self.transact("Get account code by commitment", move |conn| { - queries::select_account_code_by_commitment(conn, code_commitment) + diesel_queries::select_account_code_by_commitment(conn, code_commitment) }) .await } @@ -504,12 +557,13 @@ impl Db { account_id: AccountId, block_num: ScopedBlockNum, ) -> Result> { - self.transact("Get account header with storage header at block", move |conn| { - queries::select_account_header_with_storage_header_at_block( - conn, account_id, *block_num, - ) - }) - .await + self.reader + .read("Get account header with storage header at block", move |tx| { + queries::select_account_header_with_storage_header_at_block( + tx, account_id, *block_num, + ) + }) + .await } #[miden_instrument( @@ -524,7 +578,12 @@ impl Db { ) -> Result, NoteSyncError> { let block_range = block_range.into_inner(); self.transact("notes sync task", move |conn| { - queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES) + diesel_queries::get_note_sync_multi( + conn, + ¬e_tags, + block_range, + MAX_RESPONSE_PAYLOAD_BYTES, + ) }) .await } @@ -538,7 +597,7 @@ impl Db { )] pub async fn select_notes_by_id(&self, note_ids: Vec) -> Result> { self.transact("note by id", move |conn| { - queries::select_notes_by_id(conn, note_ids.as_slice()) + diesel_queries::select_notes_by_id(conn, note_ids.as_slice()) }) .await } @@ -556,7 +615,7 @@ impl Db { up_to_block: ScopedBlockNum, ) -> Result> { self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments( + diesel_queries::select_existing_note_commitments( conn, note_commitments.as_slice(), *up_to_block, @@ -578,7 +637,7 @@ impl Db { up_to_block: ScopedBlockNum, ) -> Result> { self.transact("block note inclusion proofs by commitment", move |conn| { - models::queries::select_note_inclusion_proofs(conn, ¬e_commitments, *up_to_block) + diesel_queries::select_note_inclusion_proofs(conn, ¬e_commitments, *up_to_block) }) .await } @@ -610,29 +669,50 @@ impl Db { unresolved_note_nullifiers: Vec, prune_tip: BlockNumber, ) -> Result> { - self.transact("apply block", move |conn| { - models::queries::apply_block(conn, &signed_block, ¬es, &precomputed_public_states)?; - models::queries::prune_history(conn, prune_tip)?; - - let mut resolved_note_ids = BTreeMap::new(); - for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { - match queries::select_note_ids_by_nullifier(conn, chunk) { - Ok(note_ids) => resolved_note_ids.extend(note_ids), - Err(err) => { - tracing::warn!( - target: COMPONENT, - %err, - nullifiers.count = chunk.len(), - "Failed to resolve consumed note IDs for lifecycle events", - ); - break; - }, - } + self.writer + .write::<_, DatabaseError, _>("apply block", move |tx| { + queries::apply_block(tx, &signed_block, ¬es, &precomputed_public_states)?; + queries::prune_history(tx, prune_tip)?; + Ok(()) + }) + .await?; + + Ok(self.resolve_consumed_note_ids(unresolved_note_nullifiers).await) + } + + /// Maps consumed nullifiers back to their note IDs for lifecycle events, on a best-effort + /// basis. + /// + /// A failed lookup is logged and abandoned: the caller uses this only for reporting. + async fn resolve_consumed_note_ids( + &self, + nullifiers: Vec, + ) -> BTreeMap { + let mut resolved_note_ids = BTreeMap::new(); + for chunk in nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { + let chunk = chunk.to_vec(); + let count = chunk.len(); + let result = self + .transact("resolve consumed note ids", move |conn| { + diesel_queries::select_note_ids_by_nullifier(conn, &chunk) + }) + .await; + + match result { + Ok(note_ids) => resolved_note_ids.extend(note_ids), + Err(err) => { + tracing::warn!( + target: COMPONENT, + %err, + nullifiers.count = count, + "Failed to resolve consumed note IDs for lifecycle events", + ); + break; + }, } + } - Ok(resolved_note_ids) - }) - .await + resolved_note_ids } /// Selects storage map values for syncing storage maps for a specific account ID. @@ -649,7 +729,7 @@ impl Db { let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit); self.transact("select storage map sync values", move |conn| { - models::queries::select_account_storage_map_values_paged( + diesel_queries::select_account_storage_map_values_paged( conn, account_id, block_range, @@ -761,15 +841,16 @@ impl Db { #[miden_instrument( target = COMPONENT, )] - pub async fn select_account_vault_at_block( + pub async fn select_vault_at_block( &self, account_id: AccountId, block_num: ScopedBlockNum, ) -> Result, DatabaseError> { - self.transact("select account vault at block", move |conn| { - queries::select_account_vault_at_block(conn, account_id, *block_num) - }) - .await + self.reader + .read("select vault at block", move |tx| { + queries::select_vault_at_block(tx, account_id, *block_num) + }) + .await } pub async fn get_account_vault_sync( @@ -779,7 +860,7 @@ impl Db { ) -> Result<(BlockNumber, Vec)> { let block_range = block_range.into_inner(); self.transact("account vault sync", move |conn| { - queries::select_account_vault_assets(conn, account_id, block_range) + diesel_queries::select_account_vault_assets(conn, account_id, block_range) }) .await } @@ -787,7 +868,7 @@ impl Db { /// Returns the script for a note by its root. pub async fn select_note_script_by_root(&self, root: Word) -> Result> { self.transact("note script by root", move |conn| { - queries::select_note_script_by_root(conn, root) + diesel_queries::select_note_script_by_root(conn, root) }) .await } @@ -805,7 +886,7 @@ impl Db { ) -> Result<(BlockNumber, Vec)> { let block_range = block_range.into_inner(); self.transact("full transactions records", move |conn| { - queries::select_transactions_records(conn, &account_ids, block_range) + diesel_queries::select_transactions_records(conn, &account_ids, block_range) }) .await } diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs index abd26debca..c37cd3bbb7 100644 --- a/crates/store/src/db/models/conv.rs +++ b/crates/store/src/db/models/conv.rs @@ -39,7 +39,8 @@ use miden_protocol::account::{StorageSlotName, StorageSlotType}; use miden_protocol::block::{BlockHeader, BlockNumber}; use miden_protocol::note::NoteTag; -use crate::db::models::queries::{BlockHeaderCommitment, NetworkAccountType}; +use crate::db::BlockHeaderCommitment; +use crate::db::models::queries::NetworkAccountType; #[derive(Debug, thiserror::Error)] #[error("failed to convert from database type {from_type} into {into_type}")] diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs index 90a0401225..f62f6b081f 100644 --- a/crates/store/src/db/models/queries/accounts.rs +++ b/crates/store/src/db/models/queries/accounts.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use std::num::NonZeroUsize; use std::ops::RangeInclusive; @@ -6,10 +6,8 @@ use diesel::prelude::{Queryable, QueryableByName}; use diesel::query_dsl::methods::SelectDsl; use diesel::sqlite::Sqlite; use diesel::{ - AsChangeset, BoolExpressionMethods, ExpressionMethods, - Insertable, JoinOnDsl, NullableExpressionMethods, OptionalExtension, @@ -19,58 +17,31 @@ use diesel::{ SelectableHelper, SqliteConnection, }; -use miden_node_proto::domain::account::{AccountInfo, AccountSummary, AccountVaultDetails}; -use miden_node_utils::limiter::{ - MAX_RESPONSE_PAYLOAD_BYTES, - QueryParamAccountIdLimit, - QueryParamLimiter, -}; -use miden_node_utils::tracing::miden_instrument; +use miden_node_proto::domain::account::{AccountInfo, AccountSummary}; +use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; use miden_protocol::Word; use miden_protocol::account::{ Account, AccountCode, AccountId, - AccountPatch, AccountStorage, AccountStorageHeader, - AccountUpdateDetails, StorageMap, StorageMapKey, - StorageMapPatchEntries, StorageSlot, - StorageSlotContent, StorageSlotName, StorageSlotType, }; use miden_protocol::asset::{Asset, AssetId, AssetVault}; -use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; +use miden_protocol::block::BlockNumber; use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_standards::account::auth::NetworkAccount; -use crate::COMPONENT; -use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce}; +use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; #[cfg(test)] use crate::db::models::vec_raw_try_into; use crate::db::{AccountVaultValue, schema}; use crate::errors::DatabaseError; -mod at_block; -pub(crate) use at_block::select_account_header_with_storage_header_at_block; - -mod delta; -use delta::{ - AccountStateForInsert, - LatestAccountStateRow, - PartialAccountState, - PrecomputedFullAccountState, - apply_storage_patch_with_roots, - select_latest_account_state, -}; - -#[cfg(test)] -mod tests; - type StorageMapValueRow = (i64, String, Vec, Vec); type StorageHeaderWithEntries = (AccountStorageHeader, HashMap>); @@ -333,15 +304,6 @@ pub struct PublicAccountStateRootsPage { pub next_cursor: Option, } -/// Public account state commitments computed by the account state forest before SQLite writes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PrecomputedPublicAccountState { - pub(crate) vault_root: Word, - pub(crate) storage_map_roots: BTreeMap, -} - -pub(crate) type PrecomputedPublicAccountStates = BTreeMap; - /// Selects public account IDs with pagination. /// /// Returns up to `page_size` public account IDs, starting after `after_account_id` if provided. @@ -575,60 +537,6 @@ pub(crate) fn select_account_vault_assets( Ok((last_block_included, values)) } -/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. -/// -/// Selects, per vault key, the row whose validity interval covers `block_num`: -/// ```sql -/// SELECT asset FROM account_vault_assets -/// WHERE account_id = ?1 AND block_num <= ?2 AND valid_until > ?2 -/// LIMIT ?3 -/// ``` -/// -/// The read is bounded to [`AccountVaultDetails::MAX_RETURN_ENTRIES`] + 1 rows so an over-the-limit -/// vault can be detected without materializing the whole set. -pub(crate) fn select_account_vault_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use diesel::sql_types::{BigInt, Binary}; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let limit_sql = - i64::try_from(AccountVaultDetails::MAX_RETURN_ENTRIES + 1).expect("should fit within i64"); - - let entries: Vec>> = diesel::sql_query( - r" - SELECT asset FROM account_vault_assets - WHERE account_id = ?1 AND block_num <= ?2 AND valid_until > ?2 - LIMIT ?3 - ", - ) - .bind::(&account_id_bytes) - .bind::(block_num_sql) - .bind::(limit_sql) - .load::(conn)? - .into_iter() - .map(|row| row.asset) - .collect(); - - // Convert to assets, filtering out deletions (None values) - let mut assets = Vec::new(); - for asset_bytes in entries.into_iter().flatten() { - let asset = Asset::read_from_bytes(&asset_bytes)?; - assets.push(asset); - } - - Ok(assets) -} - -#[derive(QueryableByName)] -struct AssetRow { - #[diesel(sql_type = diesel::sql_types::Nullable)] - asset: Option>, -} - /// Select all accounts from the DB using the given [`SqliteConnection`]. /// /// # Returns @@ -937,900 +845,3 @@ impl TryInto for AccountSummaryRaw { }) } } - -/// Insert an account vault asset row into the DB using the given [`SqliteConnection`]. -/// -/// The new row is inserted open-ended (`valid_until = VALID_FOREVER`); any existing open row -/// with the same `(account_id, vault_key)` tuple has its validity interval closed at `block_num`. -/// -/// # Returns -/// -/// The number of affected rows. -pub(crate) fn insert_account_vault_asset( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - vault_key: AssetId, - asset: Option, -) -> Result { - let record = AccountAssetRowInsert::new(&account_id, &vault_key, block_num, asset); - - diesel::Connection::transaction(conn, |conn| { - // Close the previous version's validity interval at the new row's block. - let vault_key: Word = vault_key.into(); - let vault_key_bytes = vault_key.to_bytes(); - let account_id_bytes = account_id.to_bytes(); - let update_count = diesel::update(schema::account_vault_assets::table) - .filter( - schema::account_vault_assets::account_id - .eq(account_id_bytes) - .and(schema::account_vault_assets::vault_key.eq(vault_key_bytes)) - .and(schema::account_vault_assets::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::account_vault_assets::valid_until.eq(block_num.to_raw_sql())) - .execute(conn)?; - - // Insert the new open-ended row - let insert_count = diesel::insert_into(schema::account_vault_assets::table) - .values(record) - .execute(conn)?; - - Ok(update_count + insert_count) - }) -} - -/// Inserts a versioned account storage-map value using the given [`SqliteConnection`]. -/// -/// The new row is inserted open-ended, and any previous open row for the same -/// `(account_id, slot_name, key)` tuple has its validity interval closed at `block_num` first. -/// -/// # Returns -/// -/// The total number of inserted and invalidated rows. -/// -/// # Errors -/// -/// Returns an error if the previous row cannot be invalidated or the new row cannot be inserted. -pub(crate) fn insert_account_storage_map_value( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_name: StorageSlotName, - key: StorageMapKey, - value: Word, -) -> Result { - insert_account_storage_map_value_inner(conn, account_id, block_num, slot_name, key, value, true) -} - -/// Inserts a versioned account storage-map value with optional previous-row invalidation. -/// -/// `invalidate_previous` may be disabled when inserting state for a new account, for which no -/// previous open row can exist. The inserted row is always open-ended. -/// -/// # Returns -/// -/// The total number of inserted and invalidated rows. -/// -/// # Errors -/// -/// Returns an error if the requested invalidation or insertion fails. -fn insert_account_storage_map_value_inner( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_name: StorageSlotName, - key: StorageMapKey, - value: Word, - invalidate_previous: bool, -) -> Result { - let account_id = account_id.to_bytes(); - let key = key.to_bytes(); - let value = value.to_bytes(); - let slot_name = slot_name.to_raw_sql(); - let block_num = block_num.to_raw_sql(); - - let update_count = if invalidate_previous { - diesel::update(schema::account_storage_map_values::table) - .filter( - schema::account_storage_map_values::account_id - .eq(&account_id) - .and(schema::account_storage_map_values::slot_name.eq(&slot_name)) - .and(schema::account_storage_map_values::key.eq(&key)) - .and(schema::account_storage_map_values::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::account_storage_map_values::valid_until.eq(block_num)) - .execute(conn)? - } else { - 0 - }; - - let record = AccountStorageMapRowInsert { - account_id, - key, - value, - slot_name, - block_num, - valid_until: VALID_FOREVER, - }; - let insert_count = diesel::insert_into(schema::account_storage_map_values::table) - .values(record) - .execute(conn)?; - - Ok(update_count + insert_count) -} - -type PendingStorageInserts = Vec<(AccountId, StorageSlotName, StorageMapKey, Word)>; -type PendingAssetInserts = Vec<(AccountId, AssetId, Option)>; - -fn prepare_full_account_update( - update: &BlockAccountUpdate, - account: Account, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - let account_id = account.id(); - - // sanity check the commitment of account matches the final state commitment - if account.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - // collect storage-map inserts to apply after account upsert - let mut storage = Vec::new(); - for slot in account.storage().slots() { - if let StorageSlotContent::Map(storage_map) = slot.content() { - for (key, value) in storage_map.entries() { - storage.push((account_id, slot.name().clone(), *key, *value)); - } - } - } - - // collect vault-asset inserts to apply after account upsert - let mut assets = Vec::new(); - for asset in account.vault().assets() { - // Only insert assets with non-zero values for fungible assets - let should_insert = match asset { - Asset::Fungible(fungible) => fungible.amount().as_u64() > 0, - Asset::NonFungible(_) => true, - }; - if should_insert { - assets.push((account_id, asset.id(), Some(asset))); - } - } - - Ok((AccountStateForInsert::FullAccount(account), storage, assets)) -} - -/// Prepares a full public-account insertion using roots computed by the account-state forest. -/// -/// This avoids reconstructing the account's vault and storage maps in SQLite. The returned state -/// contains the account-row fields, while storage-map entries and vault assets are returned -/// separately for insertion after the account row has satisfied their foreign-key dependency. -/// Empty-word map entries and assets are omitted from the pending inserts. -/// -/// # Errors -/// -/// Returns an error if the full-state patch is missing its code or nonce, a required precomputed -/// storage root is absent, an asset is invalid, or the reconstructed account header does not match -/// the update's final state commitment. -fn prepare_precomputed_full_account_update( - update: &BlockAccountUpdate, - patch: &AccountPatch, - precomputed: &PrecomputedPublicAccountState, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - let account_id = patch.id(); - let code = patch.code().cloned().ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "full-state patch for account {account_id} is missing account code" - )) - })?; - let nonce = patch.final_nonce().ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "full-state patch for account {account_id} is missing final nonce" - )) - })?; - - let storage_header = apply_storage_patch_with_roots( - &AccountStorageHeader::new(Vec::new())?, - patch.storage(), - &precomputed.storage_map_roots, - )?; - let account_header = miden_protocol::account::AccountHeader::new( - account_id, - nonce, - precomputed.vault_root, - storage_header.to_commitment(), - code.commitment(), - ); - if account_header.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account_header.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - let storage = patch - .storage() - .maps() - .flat_map(|(slot_name, map_patch)| { - map_patch.entries().into_iter().flat_map(move |entries| { - entries - .as_map() - .iter() - .filter(|(_key, value)| **value != Word::empty()) - .map(move |(key, value)| (account_id, slot_name.clone(), *key, *value)) - }) - }) - .collect(); - let assets = patch - .vault() - .iter() - .filter(|(_asset_id, value)| **value != Word::empty()) - .map(|(asset_id, value)| { - Asset::from_id_and_value(*asset_id, *value) - .map(|asset| (account_id, *asset_id, Some(asset))) - }) - .collect::, _>>()?; - - // The patch carries full state, so it can be turned back into an account and classified with - // the canonical check. - let is_network_account = NetworkAccount::new(Account::try_from(patch)?).is_ok(); - let state = PrecomputedFullAccountState { - nonce, - code, - storage_header, - vault_root: precomputed.vault_root, - is_network_account, - }; - - Ok((AccountStateForInsert::PrecomputedFullState(state), storage, assets)) -} - -/// Prepares a partial public-account update using the latest row and precomputed forest roots. -/// -/// Unchanged header fields are carried forward from `existing`. The returned partial state is used -/// for the next account row, while storage-map values and vault asset updates are returned -/// separately for insertion after that row. Empty vault values are represented as removals. -/// -/// # Errors -/// -/// Returns an error if the existing row is invalid, a required precomputed storage root is absent, -/// a patched asset is invalid, or the reconstructed account header does not match the update's -/// final state commitment. -fn prepare_partial_account_update( - update: &BlockAccountUpdate, - account_id: AccountId, - patch: &AccountPatch, - precomputed: &PrecomputedPublicAccountState, - existing: &LatestAccountStateRow, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - // Build the minimal account state needed for partial patch application from the latest row that - // was loaded with the account's creation metadata. - let state_headers = existing.state_headers(account_id)?; - - // --- Process asset updates. --------------------------------- The patch carries absolute final - // values, so encode `Some` as update and `None` (an empty value word) as removal. - let mut assets = Vec::new(); - for (vault_key, value) in patch.vault().iter() { - let update_or_remove = if *value == Word::empty() { - None - } else { - Some(Asset::from_id_and_value(*vault_key, *value)?) - }; - assets.push((account_id, *vault_key, update_or_remove)); - } - - // --- Collect storage map updates. --------------------------- - - let mut storage = Vec::new(); - for (slot_name, map_patch) in patch.storage().maps() { - for (key, value) in map_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) - { - storage.push((account_id, slot_name.clone(), *key, *value)); - } - } - - // Apply the patch storage to the given storage header. - let new_storage_header = apply_storage_patch_with_roots( - &state_headers.storage_header, - patch.storage(), - &precomputed.storage_map_roots, - )?; - - let new_vault_root = precomputed.vault_root; - - // --- Compute updated account state for the accounts row. --- Use the absolute final nonce. - let new_nonce = patch.final_nonce().unwrap_or(state_headers.nonce); - - // Create minimal account state data for the row insert. - let account_state = PartialAccountState { - nonce: new_nonce, - code_commitment: state_headers.code_commitment, - storage_header: new_storage_header, - vault_root: new_vault_root, - }; - - let account_header = miden_protocol::account::AccountHeader::new( - account_id, - account_state.nonce, - account_state.vault_root, - account_state.storage_header.to_commitment(), - account_state.code_commitment, - ); - - if account_header.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account_header.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - Ok((AccountStateForInsert::PartialState(account_state), storage, assets)) -} - -/// Returns the subset of `account_ids` whose latest committed state is a network account. -/// -/// Unknown ids and non-network accounts are silently omitted. -pub(crate) fn select_network_accounts_subset( - conn: &mut SqliteConnection, - account_ids: &[AccountId], -) -> Result, DatabaseError> { - QueryParamAccountIdLimit::check(account_ids.len())?; - let id_bytes: Vec> = - account_ids.iter().map(miden_crypto::utils::Serializable::to_bytes).collect(); - - let rows: Vec> = - SelectDsl::select(schema::accounts::table, schema::accounts::account_id) - .filter( - schema::accounts::account_id - .eq_any(&id_bytes) - .and( - schema::accounts::network_account_type - .eq(NetworkAccountType::Network.to_raw_sql()), - ) - .and(schema::accounts::valid_until.eq(VALID_FOREVER)), - ) - .load::>(conn) - .map_err(DatabaseError::Diesel)?; - - rows.into_iter() - .map(|bytes| { - AccountId::read_from_bytes(&bytes).map_err(DatabaseError::DeserializationError) - }) - .collect() -} - -/// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn upsert_accounts( - conn: &mut SqliteConnection, - accounts: &[BlockAccountUpdate], - block_num: BlockNumber, - precomputed_public_states: &PrecomputedPublicAccountStates, -) -> Result { - let mut count = 0; - for update in accounts { - let account_id = update.account_id(); - let account_id_bytes = account_id.to_bytes(); - - // Pull the latest row once. Partial updates consume the state headers below, while every - // update carries forward creation metadata. - let existing = select_latest_account_state(conn, account_id)?; - let account_is_new = existing.is_none(); - - let created_at_block = match &existing { - Some(row) => row.created_at_block()?, - None => block_num, - }; - - // NOTE: we collect storage / asset inserts to apply them only after the account row is - // written. The storage and vault tables have FKs pointing to accounts `(account_id, - // block_num)`, so inserting them earlier would violate those constraints when inserting a - // brand-new account. - let (account_state, pending_storage_inserts, pending_asset_inserts) = match update.details() - { - AccountUpdateDetails::Private => (AccountStateForInsert::Private, vec![], vec![]), - - // New account is always a full account, but also comes as an update - AccountUpdateDetails::Public(patch) if patch.is_full_state() => { - if block_num == BlockNumber::GENESIS { - let account = Account::try_from(patch) - .expect("Patch to full account always works for full state patches"); - debug_assert_eq!(account_id, account.id()); - prepare_full_account_update(update, account)? - } else { - let precomputed = - precomputed_public_states.get(&account_id).ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "missing precomputed public account state for account {account_id}" - )) - })?; - prepare_precomputed_full_account_update(update, patch, precomputed)? - } - }, - - // Update of an existing account - AccountUpdateDetails::Public(patch) => { - let precomputed = precomputed_public_states.get(&account_id).ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "missing precomputed public account state for account {account_id}" - )) - })?; - let existing = - existing.as_ref().ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - prepare_partial_account_update(update, account_id, patch, precomputed, existing)? - }, - }; - - // Inherit the classification when the account already exists; otherwise classify it once at - // creation based on the new state. - let network_account_type = match &existing { - Some(row) => row.network_account_type()?, - None => match &account_state { - AccountStateForInsert::FullAccount(account) - if NetworkAccount::new(account.clone()).is_ok() => - { - NetworkAccountType::Network - }, - AccountStateForInsert::PrecomputedFullState(state) if state.is_network_account => { - NetworkAccountType::Network - }, - _ => NetworkAccountType::None, - }, - }; - - // Insert account _code_ for full accounts (new account creation) - if let AccountStateForInsert::FullAccount(ref account) = account_state { - let code = account.code(); - let code_value = AccountCodeRowInsert { - code_commitment: code.commitment().to_bytes(), - code: code.to_bytes(), - }; - diesel::insert_into(schema::account_codes::table) - .values(&code_value) - .on_conflict(schema::account_codes::code_commitment) - .do_nothing() - .execute(conn)?; - } - if let AccountStateForInsert::PrecomputedFullState(ref state) = account_state { - let code_value = AccountCodeRowInsert { - code_commitment: state.code.commitment().to_bytes(), - code: state.code.to_bytes(), - }; - diesel::insert_into(schema::account_codes::table) - .values(&code_value) - .on_conflict(schema::account_codes::code_commitment) - .do_nothing() - .execute(conn)?; - } - - // close the previous row's validity interval and insert NEW account row - diesel::update(schema::accounts::table) - .filter( - schema::accounts::account_id - .eq(&account_id_bytes) - .and(schema::accounts::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::accounts::valid_until.eq(block_num.to_raw_sql())) - .execute(conn)?; - - let account_value = match &account_state { - AccountStateForInsert::Private => AccountRowInsert::new_private( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - ), - AccountStateForInsert::FullAccount(account) => AccountRowInsert::new_from_account( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - account, - ), - AccountStateForInsert::PrecomputedFullState(state) => { - AccountRowInsert::new_from_precomputed_full_state( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - state, - ) - }, - AccountStateForInsert::PartialState(state) => AccountRowInsert::new_from_partial( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - state, - ), - }; - - diesel::insert_into(schema::accounts::table) - .values(&account_value) - .on_conflict((schema::accounts::account_id, schema::accounts::block_num)) - .do_update() - .set(&account_value) - .execute(conn)?; - - // insert pending storage map entries TODO consider batching - for (acc_id, slot_name, key, value) in pending_storage_inserts { - if account_is_new { - insert_account_storage_map_value_inner( - conn, acc_id, block_num, slot_name, key, value, false, - )?; - } else { - insert_account_storage_map_value(conn, acc_id, block_num, slot_name, key, value)?; - } - } - - for (acc_id, vault_key, update) in pending_asset_inserts { - insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; - } - - count += 1; - } - - Ok(count) -} - -#[derive(Insertable, Debug, Clone)] -#[diesel(table_name = schema::account_codes)] -pub(crate) struct AccountCodeRowInsert { - pub(crate) code_commitment: Vec, - pub(crate) code: Vec, -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::accounts)] -pub(crate) struct AccountRowInsert { - pub(crate) account_id: Vec, - pub(crate) network_account_type: i32, - pub(crate) block_num: i64, - pub(crate) account_commitment: Vec, - pub(crate) code_commitment: Option>, - pub(crate) nonce: Option, - pub(crate) storage_header: Option>, - pub(crate) vault_root: Option>, - pub(crate) created_at_block: i64, - pub(crate) valid_until: i64, -} - -impl AccountRowInsert { - /// Creates an insert row for a private account (no public state). - pub(crate) fn new_private( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: None, - code_commitment: None, - storage_header: None, - vault_root: None, - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - /// Creates an insert row from a full account (new account creation). - fn new_from_account( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - account: &Account, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: Some(nonce_to_raw_sql(account.nonce())), - code_commitment: Some(account.code().commitment().to_bytes()), - storage_header: Some(account.storage().to_header().to_bytes()), - vault_root: Some(account.vault().root().to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - fn new_from_precomputed_full_state( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - state: &PrecomputedFullAccountState, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - block_num: block_num.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - code_commitment: Some(state.code.commitment().to_bytes()), - nonce: Some(nonce_to_raw_sql(state.nonce)), - storage_header: Some(state.storage_header.to_bytes()), - vault_root: Some(state.vault_root.to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - /// Creates an insert row from a partial account state (patch update). - fn new_from_partial( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - state: &PartialAccountState, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: Some(nonce_to_raw_sql(state.nonce)), - code_commitment: Some(state.code_commitment.to_bytes()), - storage_header: Some(state.storage_header.to_bytes()), - vault_root: Some(state.vault_root.to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_vault_assets)] -pub(crate) struct AccountAssetRowInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) vault_key: Vec, - pub(crate) asset: Option>, - pub(crate) valid_until: i64, -} - -impl AccountAssetRowInsert { - pub(crate) fn new( - account_id: &AccountId, - vault_key: &AssetId, - block_num: BlockNumber, - asset: Option, - ) -> Self { - let account_id = account_id.to_bytes(); - let vault_key: Word = (*vault_key).into(); - let vault_key = vault_key.to_bytes(); - let block_num = block_num.to_raw_sql(); - let asset = asset.map(|asset| asset.to_bytes()); - Self { - account_id, - block_num, - vault_key, - asset, - valid_until: VALID_FOREVER, - } - } -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_storage_map_values)] -pub(crate) struct AccountStorageMapRowInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) slot_name: String, - pub(crate) key: Vec, - pub(crate) value: Vec, - pub(crate) valid_until: i64, -} - -// CLEANUP FUNCTIONS -// ================================================================================================ - -/// Number of historical blocks to retain for vault assets, storage map values, and account codes. -/// Rows whose validity interval ends at or below `prune_tip - HISTORICAL_BLOCK_RETENTION` will be -/// deleted; rows still valid anywhere inside the retention window (including all open-ended rows) -/// are retained. -pub const HISTORICAL_BLOCK_RETENTION: u32 = 50; - -/// Clean up old entries for all accounts, deleting entries that can no longer affect state -/// reconstruction at any block within the retention window. -/// -/// A row is applicable for blocks in `[block_num, valid_until)`, so it is deletable exactly when -/// its interval ends at or below the cutoff (`prune_tip - HISTORICAL_BLOCK_RETENTION`): it then -/// cannot cover any block inside the window. `prune_tip` is the effective tip for retention — it -/// lags the chain tip while old snapshot generations are still pinned by readers (see -/// [`crate::db::Db::apply_block`]). Account codes follow the same rule — a code is deleted only -/// when no account row whose interval reaches past the cutoff references it. -/// -/// # Returns -/// A tuple of `(vault_assets_deleted, storage_map_values_deleted, account_codes_deleted)` -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -pub(crate) fn prune_history( - conn: &mut SqliteConnection, - prune_tip: BlockNumber, -) -> Result<(usize, usize, usize), DatabaseError> { - let cutoff_block = i64::from(prune_tip.as_u32().saturating_sub(HISTORICAL_BLOCK_RETENTION)); - tracing::Span::current().record("cutoff_block", cutoff_block); - let vault_deleted = prune_account_vault_assets(conn, cutoff_block)?; - let storage_deleted = prune_account_storage_map_values(conn, cutoff_block)?; - let codes_deleted = prune_account_codes(conn, cutoff_block)?; - - Ok((vault_deleted, storage_deleted, codes_deleted)) -} - -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_vault_assets( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - // The literal `!= VALID_FOREVER` term (rather than a bound parameter) lets SQLite prove the - // predicate implies `idx_vault_cleanup`'s partial-index condition. - diesel::sql_query(format!( - "DELETE FROM account_vault_assets \ - WHERE valid_until != {VALID_FOREVER} \ - AND valid_until <= ?1" - )) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel) -} - -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_storage_map_values( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - // The literal `!= VALID_FOREVER` term (rather than a bound parameter) lets SQLite prove the - // predicate implies `idx_storage_cleanup`'s partial-index condition. - diesel::sql_query(format!( - "DELETE FROM account_storage_map_values \ - WHERE valid_until != {VALID_FOREVER} \ - AND valid_until <= ?1" - )) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel) -} - -/// Deletes account codes that are no longer referenced by any account row that can serve a read -/// within the retention window. -/// -/// An account code is safe to delete when no `accounts` row whose validity interval reaches past -/// the cutoff (`valid_until > cutoff_block`) references it. That single predicate covers rows -/// inside the window, all open-ended (current) rows, and each account's baseline row — the row -/// still valid at the cutoff even though it was written before it. -/// -/// Rather than re-checking every code on every prune, only codes whose deletability could have -/// changed since the previous prune are examined. A code survived the previous prune because at -/// least one `accounts` row with `valid_until > prev_cutoff` referenced it. For it to be -/// deletable now, all such rows must have expired by the new cutoff — including the longest-lived -/// one, whose `valid_until` therefore lands inside `(prev_cutoff, cutoff_block]`. Scanning the -/// rows that expired in that window thus finds every code that could have become deletable. The -/// scan is an `idx_accounts_code_validity` index range, so its cost scales with the number of -/// account updates since the previous prune, not with total history. Each candidate is deleted -/// only if the `idx_accounts_code_probe` existence probe finds no row still referencing it with -/// `valid_until > cutoff_block`. The previous cutoff is persisted in `prune_progress` within the -/// same transaction; when absent (first prune after migration, or a fresh database) a full pass -/// over all rows valid past the cutoff runs instead. -/// -/// Correctness of the windowed candidate set rests on two invariants: -/// - Rows are only ever closed to the `block_num` of the block currently being applied, which is -/// always above the cutoff, so every expiry crosses the window of some later prune. A write path -/// that back-dated `valid_until` below the current cutoff would leak the code forever. -/// - Every `account_codes` row is inserted alongside an `accounts` row referencing it (see -/// [`upsert_accounts`]); an orphan code with no referencing row would never become a candidate. -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_codes( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - let prev_cutoff: Option = - SelectDsl::select(schema::prune_progress::table, schema::prune_progress::codes_cutoff) - .first(conn) - .optional() - .map_err(DatabaseError::Diesel)?; - - let deleted = match prev_cutoff { - // Codes are already pruned through this cutoff and nothing can become collectable while the - // cutoff stands still. Equality is the common case: the cutoff is clamped to zero for the - // first `HISTORICAL_BLOCK_RETENTION` blocks, and a pinned snapshot freezes the prune tip - // across consecutive blocks. A strictly greater `prev_cutoff` is unreachable through - // `apply_block` (the prune tip never regresses) but is guarded against so an out-of-order - // caller cannot move the marker backwards or run the delete with an inverted window. - Some(prev_cutoff) if prev_cutoff >= cutoff_block => return Ok(0), - Some(prev_cutoff) => diesel::sql_query( - "DELETE FROM account_codes \ - WHERE code_commitment IN ( \ - SELECT DISTINCT code_commitment \ - FROM accounts INDEXED BY idx_accounts_code_validity \ - WHERE code_commitment IS NOT NULL \ - AND valid_until > ?1 \ - AND valid_until <= ?2 \ - ) \ - AND NOT EXISTS ( \ - SELECT 1 \ - FROM accounts INDEXED BY idx_accounts_code_probe \ - WHERE accounts.code_commitment = account_codes.code_commitment \ - AND accounts.valid_until > ?2 \ - )", - ) - .bind::(prev_cutoff) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel)?, - // No recorded cutoff: full pass. The forced `idx_accounts_code_validity` covering index - // keeps the subquery an index-only range scan, sized by rows valid at or after the cutoff - // rather than total history. - None => diesel::sql_query( - "DELETE FROM account_codes \ - WHERE code_commitment NOT IN ( \ - SELECT DISTINCT code_commitment \ - FROM accounts INDEXED BY idx_accounts_code_validity \ - WHERE code_commitment IS NOT NULL \ - AND valid_until > ?1 \ - )", - ) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel)?, - }; - - diesel::insert_into(schema::prune_progress::table) - .values(( - schema::prune_progress::id.eq(0), - schema::prune_progress::codes_cutoff.eq(cutoff_block), - )) - .on_conflict(schema::prune_progress::id) - .do_update() - .set(schema::prune_progress::codes_cutoff.eq(cutoff_block)) - .execute(conn) - .map_err(DatabaseError::Diesel)?; - - Ok(deleted) -} diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs deleted file mode 100644 index cfe91995cd..0000000000 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ /dev/null @@ -1,101 +0,0 @@ -use diesel::prelude::Queryable; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; -use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader}; -use miden_protocol::block::BlockNumber; -use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_protocol::{Felt, Word}; - -use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; -use crate::db::schema; -use crate::errors::DatabaseError; - -// ACCOUNT HEADER -// ================================================================================================ - -#[derive(Debug, Clone, Queryable)] -struct AccountHeaderDataRaw { - code_commitment: Option>, - nonce: Option, - storage_header: Option>, - vault_root: Option>, -} - -/// Queries the account header for a specific account at a specific block number. -/// -/// This reconstructs the `AccountHeader` by reading from the `accounts` table: -/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` -/// -/// Returns `None` if the account doesn't exist at that block. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number at which to query the account header -/// -/// # Returns -/// -/// * `Ok(Some((AccountHeader, AccountStorageHeader)))` - The headers if found -/// * `Ok(None)` - If account doesn't exist at that block -/// * `Err(DatabaseError)` - If there's a database error -pub(crate) fn select_account_header_with_storage_header_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::accounts; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - let account_data: Option = SelectDsl::select( - accounts::table - .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.le(block_num_sql)) - .order(accounts::block_num.desc()) - .limit(1), - ( - accounts::code_commitment, - accounts::nonce, - accounts::storage_header, - accounts::vault_root, - ), - ) - .first(conn) - .optional()?; - - let Some(AccountHeaderDataRaw { - code_commitment: code_commitment_bytes, - nonce: nonce_raw, - storage_header: storage_header_blob, - vault_root: vault_root_bytes, - }) = account_data - else { - return Ok(None); - }; - - let storage_header = match &storage_header_blob { - Some(blob) => AccountStorageHeader::read_from_bytes(blob)?, - None => AccountStorageHeader::new(Vec::new())?, - }; - - let storage_commitment = storage_header.to_commitment(); - - let code_commitment = code_commitment_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); - - let vault_root = vault_root_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - let account_header = - AccountHeader::new(account_id, nonce, vault_root, storage_commitment, code_commitment); - - Ok(Some((account_header, storage_header))) -} diff --git a/crates/store/src/db/models/queries/block_headers.rs b/crates/store/src/db/models/queries/block_headers.rs index e147a902ca..823f8474b4 100644 --- a/crates/store/src/db/models/queries/block_headers.rs +++ b/crates/store/src/db/models/queries/block_headers.rs @@ -1,4 +1,3 @@ -use diesel::prelude::Insertable; use diesel::query_dsl::methods::SelectDsl; use diesel::{ ExpressionMethods, @@ -11,17 +10,14 @@ use diesel::{ SelectableHelper, SqliteConnection, }; -use miden_crypto::Word; use miden_node_utils::limiter::{QueryParamBlockLimit, QueryParamLimiter}; -use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; -use miden_protocol::utils::serde::{Deserializable, Serializable}; +use miden_protocol::utils::serde::Deserializable; use super::DatabaseError; -use crate::COMPONENT; use crate::db::models::conv::SqlTypeConvert; use crate::db::models::vec_raw_try_into; -use crate::db::schema; +use crate::db::{BlockHeaderCommitment, schema}; /// Select a [`BlockHeader`] from the DB by its `block_num` using the given [`SqliteConnection`]. /// @@ -152,19 +148,6 @@ pub fn select_all_block_header_commitments( Ok(commitments) } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(transparent)] -pub struct BlockHeaderCommitment(pub(crate) Word); - -impl BlockHeaderCommitment { - pub fn new(header: &BlockHeader) -> Self { - Self(header.commitment()) - } - pub fn word(self) -> Word { - self.0 - } -} - #[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] #[diesel(table_name = schema::block_headers)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] @@ -198,42 +181,3 @@ impl TryInto<(BlockHeader, BlockSignatures)> for BlockHeaderRawRow { Ok((block_header, signatures)) } } - -#[derive(Debug, Clone, Insertable)] -#[diesel(table_name = schema::block_headers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct BlockHeaderInsert { - pub block_num: i64, - pub block_header: Vec, - pub signature: Vec, - pub commitment: Vec, -} - -/// Insert a [`BlockHeader`] to the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_block_header( - conn: &mut SqliteConnection, - block_header: &BlockHeader, - signatures: &BlockSignatures, -) -> Result { - let row = BlockHeaderInsert { - block_num: block_header.block_num().to_raw_sql(), - block_header: block_header.to_bytes(), - signature: signatures.to_bytes(), - commitment: BlockHeaderCommitment::new(block_header).to_raw_sql(), - }; - let count = diesel::insert_into(schema::block_headers::table).values(&[row]).execute(conn)?; - Ok(count) -} diff --git a/crates/store/src/db/models/queries/mod.rs b/crates/store/src/db/models/queries/mod.rs index 377a20e94d..709bb88140 100644 --- a/crates/store/src/db/models/queries/mod.rs +++ b/crates/store/src/db/models/queries/mod.rs @@ -25,12 +25,7 @@ //! transaction, any nesting of further `transaction(conn, || {})` has no effect and should be //! considered unnecessary boilerplate by default. -use diesel::SqliteConnection; -use miden_protocol::block::SignedBlock; -use miden_protocol::note::Nullifier; - use super::DatabaseError; -use crate::db::NoteRecord; mod transactions; pub use transactions::*; @@ -43,34 +38,3 @@ pub use nullifiers::NullifiersPage; pub(crate) use nullifiers::*; mod notes; pub(crate) use notes::*; - -/// Apply a new block to the state. -/// -/// # Returns -/// -/// Number of records inserted and/or updated. -pub(crate) fn apply_block( - conn: &mut SqliteConnection, - block: &SignedBlock, - notes: &[(NoteRecord, Option)], - precomputed_public_states: &PrecomputedPublicAccountStates, -) -> Result { - let mut count = 0; - // Note: ordering here is important as the relevant tables have FK dependencies. - count += insert_block_header(conn, block.header(), block.signatures())?; - count += upsert_accounts( - conn, - block.body().updated_accounts(), - block.header().block_num(), - precomputed_public_states, - )?; - count += insert_scripts(conn, notes.iter().map(|(note, _)| note))?; - count += insert_notes(conn, notes)?; - count += insert_transactions(conn, block.header().block_num(), block.body().transactions())?; - count += insert_nullifiers_for_block( - conn, - block.body().created_nullifiers(), - block.header().block_num(), - )?; - Ok(count) -} diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index f1fd1f2438..5e3bd8a045 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -6,14 +6,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::ops::RangeInclusive; -use diesel::prelude::{ - ExpressionMethods, - Insertable, - QueryDsl, - Queryable, - QueryableByName, - Selectable, -}; +use diesel::prelude::{ExpressionMethods, QueryDsl, Queryable, QueryableByName, Selectable}; use diesel::query_dsl::methods::SelectDsl; use diesel::sqlite::Sqlite; use diesel::{ @@ -29,7 +22,6 @@ use miden_node_utils::limiter::{ QueryParamNoteCommitmentLimit, QueryParamNoteTagLimit, }; -use miden_node_utils::tracing::miden_instrument; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::{BlockNoteIndex, BlockNumber}; @@ -50,15 +42,8 @@ use miden_protocol::note::{ PartialNoteMetadata, }; use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_standards::note::NetworkAccountTarget; - -use crate::COMPONENT; -use crate::db::models::conv::{ - SqlTypeConvert, - idx_to_raw_sql, - note_type_to_raw_sql, - raw_sql_to_idx, -}; + +use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_idx}; use crate::db::models::queries::select_block_header_by_block_num; use crate::db::models::{serialize_vec, vec_raw_try_into}; use crate::db::{DatabaseError, NoteRecord, NoteSyncRecord, NoteSyncUpdate, schema}; @@ -75,25 +60,6 @@ pub(crate) const NOTE_SYNC_BLOCK_OVERHEAD_BYTES: usize = 1600; /// sparse merkle path with 16 siblings (~608 bytes). pub(crate) const NOTE_SYNC_RECORD_BYTES: usize = 900; -// NETWORK NOTE TYPE -// ================================================================================================ - -/// Classifies network notes for database storage. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(i32)] -pub(crate) enum NetworkNoteType { - /// Not a network note. - None = 0, - /// Single account target network note (has `NetworkAccountTarget` attachment). - SingleTarget = 1, -} - -impl From for i32 { - fn from(value: NetworkNoteType) -> Self { - value as i32 - } -} - /// Select notes matching the given tags within a block range. /// /// # Parameters @@ -725,124 +691,3 @@ impl TryInto for BlockNoteIndexRawRow { Ok(index) } } - -/// Insert notes to the DB using the given [`SqliteConnection`]. Public notes should also have a -/// nullifier. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_notes( - conn: &mut SqliteConnection, - notes: &[(NoteRecord, Option)], -) -> Result { - let count = diesel::insert_into(schema::notes::table) - .values(Vec::from_iter( - notes - .iter() - .map(|(note, nullifier)| NoteInsertRow::from((note.clone(), *nullifier))), - )) - .execute(conn)?; - Ok(count) -} - -/// Insert scripts to the DB using the given [`SqliteConnection`]. It inserts the scripts held by -/// the notes passed as parameter. If the script root already exists in the DB, it will be ignored. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_scripts<'a>( - conn: &mut SqliteConnection, - notes: impl IntoIterator, -) -> Result { - let values = Vec::from_iter(notes.into_iter().filter_map(|note| { - let note_details = note.details.as_ref()?; - Some(( - schema::note_scripts::script_root.eq(note_details.script().root().to_bytes()), - schema::note_scripts::script.eq(note_details.script().to_bytes()), - )) - })); - let count = diesel::insert_or_ignore_into(schema::note_scripts::table) - .values(values) - .execute(conn)?; - - Ok(count) -} - -#[derive(Debug, Clone, PartialEq, Insertable)] -#[diesel(table_name = schema::notes)] -pub struct NoteInsertRow { - pub committed_at: i64, - - pub batch_index: i32, - pub note_index: i32, // index within batch - - pub note_id: Vec, - - pub note_type: i32, - pub sender: Vec, // AccountId - pub tag: i32, - - pub network_note_type: i32, - pub target_account_id: Option>, - pub attachment: Vec, - pub inclusion_path: Vec, - pub consumed_at: Option, - pub nullifier: Option>, - pub assets: Option>, - pub storage: Option>, - pub script_root: Option>, - pub serial_num: Option>, -} - -impl From<(NoteRecord, Option)> for NoteInsertRow { - fn from((note, nullifier): (NoteRecord, Option)) -> Self { - let target_account_id = NetworkAccountTarget::try_from(¬e.attachments).ok(); - let network_note_type = if target_account_id.is_some() && !note.metadata.is_private() { - NetworkNoteType::SingleTarget - } else { - NetworkNoteType::None - }; - - let attachment_bytes = note.attachments.to_bytes(); - - Self { - committed_at: note.block_num.to_raw_sql(), - batch_index: idx_to_raw_sql(note.note_index.batch_idx()), - note_index: idx_to_raw_sql(note.note_index.note_idx_in_batch()), - note_id: note.note_id.to_bytes(), - note_type: note_type_to_raw_sql(note.metadata.note_type() as u8), - sender: note.metadata.sender().to_bytes(), - tag: note.metadata.tag().to_raw_sql(), - network_note_type: network_note_type.into(), - target_account_id: target_account_id.map(|t| t.target_id().to_bytes()), - attachment: attachment_bytes, - inclusion_path: note.inclusion_path.to_bytes(), - consumed_at: None::, // New notes are always unconsumed. - nullifier: nullifier.as_ref().map(Nullifier::to_bytes), - assets: note.details.as_ref().map(|d| d.assets().to_bytes()), - storage: note.details.as_ref().map(|d| d.storage().to_bytes()), - script_root: note.details.as_ref().map(|d| d.script().root().to_bytes()), - serial_num: note.details.as_ref().map(|d| d.serial_num().to_bytes()), - } - } -} diff --git a/crates/store/src/db/models/queries/nullifiers.rs b/crates/store/src/db/models/queries/nullifiers.rs index 688b5c6f0b..9bc608c645 100644 --- a/crates/store/src/db/models/queries/nullifiers.rs +++ b/crates/store/src/db/models/queries/nullifiers.rs @@ -17,15 +17,13 @@ use miden_node_utils::limiter::{ QueryParamLimiter, QueryParamNullifierPrefixLimit, }; -use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::BlockNumber; use miden_protocol::note::Nullifier; use miden_protocol::utils::serde::{Deserializable, Serializable}; use super::DatabaseError; -use crate::COMPONENT; use crate::db::models::conv::{SqlTypeConvert, nullifier_prefix_to_raw_sql}; -use crate::db::models::utils::{get_nullifier_prefix, vec_raw_try_into}; +use crate::db::models::utils::vec_raw_try_into; use crate::db::{NullifierInfo, schema}; /// Returns nullifiers filtered by prefix within a block number range. @@ -210,65 +208,6 @@ pub(crate) fn select_nullifiers_paged( Ok(NullifiersPage { nullifiers, next_cursor }) } -/// Insert nullifiers for a block into the database. -/// -/// # Parameters -/// * `nullifiers`: List of nullifiers to insert -/// - Limit: 0 <= count <= 1000 -/// * `block_num`: Block number to associate with the nullifiers -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -/// -/// # Raw SQL -/// -/// ```sql -/// UPDATE notes -/// SET consumed_at = ?1 -/// WHERE nullifier IN (?2); -/// -/// INSERT INTO nullifiers (nullifier, nullifier_prefix, block_num) -/// VALUES (?1, ?2, ?3) -/// ``` -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_nullifiers_for_block( - conn: &mut SqliteConnection, - nullifiers: &[Nullifier], - block_num: BlockNumber, -) -> Result { - let serialized_nullifiers = - Vec::>::from_iter(nullifiers.iter().map(Nullifier::to_bytes)); - - let mut count = diesel::update(schema::notes::table) - .filter(schema::notes::nullifier.eq_any(&serialized_nullifiers)) - .set(schema::notes::consumed_at.eq(Some(block_num.to_raw_sql()))) - .execute(conn)?; - - count += diesel::insert_into(schema::nullifiers::table) - .values(Vec::from_iter(nullifiers.iter().zip(serialized_nullifiers.iter()).map( - |(nullifier, bytes)| { - ( - schema::nullifiers::nullifier.eq(bytes), - schema::nullifiers::nullifier_prefix - .eq(nullifier_prefix_to_raw_sql(get_nullifier_prefix(nullifier))), - schema::nullifiers::block_num.eq(block_num.to_raw_sql()), - ) - }, - ))) - .execute(conn)?; - - Ok(count) -} - #[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] #[diesel(table_name = schema::nullifiers)] #[diesel(check_for_backend(diesel::sqlite::Sqlite))] diff --git a/crates/store/src/db/models/queries/transactions.rs b/crates/store/src/db/models/queries/transactions.rs index 249cf75439..63166d80d0 100644 --- a/crates/store/src/db/models/queries/transactions.rs +++ b/crates/store/src/db/models/queries/transactions.rs @@ -1,6 +1,6 @@ use std::ops::RangeInclusive; -use diesel::prelude::{Insertable, Queryable}; +use diesel::prelude::Queryable; use diesel::query_dsl::methods::SelectDsl; use diesel::{ BoolExpressionMethods, @@ -18,21 +18,18 @@ use miden_node_utils::limiter::{ QueryParamLimiter, QueryParamNoteCommitmentLimit, }; -use miden_node_utils::tracing::miden_instrument; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; use miden_protocol::note::{NoteHeader, NoteId, Nullifier}; use miden_protocol::transaction::{ InputNoteCommitment, InputNotes, - OrderedTransactionHeaders, TransactionHeader, TransactionId, }; -use miden_protocol::utils::serde::{Deserializable, Serializable}; +use miden_protocol::utils::serde::Deserializable; use super::{DatabaseError, select_note_ids_by_nullifier, select_note_sync_records}; -use crate::COMPONENT; use crate::db::models::conv::SqlTypeConvert; use crate::db::models::serialize_vec; use crate::db::schema; @@ -51,102 +48,6 @@ pub struct TransactionRecordRaw { size_in_bytes: i64, } -/// Insert transactions to the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_transactions( - conn: &mut SqliteConnection, - block_num: BlockNumber, - transactions: &OrderedTransactionHeaders, -) -> Result { - let rows: Vec<_> = transactions - .as_slice() - .iter() - .map(|tx| TransactionSummaryRowInsert::new(tx, block_num)) - .collect(); - - let count = diesel::insert_into(schema::transactions::table).values(rows).execute(conn)?; - Ok(count) -} - -#[derive(Debug, Clone, PartialEq, Insertable)] -#[diesel(table_name = schema::transactions)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct TransactionSummaryRowInsert { - transaction_id: Vec, - account_id: Vec, - block_num: i64, - initial_state_commitment: Vec, - final_state_commitment: Vec, - input_notes: Vec, - output_notes: Vec, - size_in_bytes: i64, -} - -impl TransactionSummaryRowInsert { - #[expect( - clippy::cast_possible_wrap, - reason = "We will not approach the item count where i64 and usize cause issues" - )] - fn new( - transaction_header: &miden_protocol::transaction::TransactionHeader, - block_num: BlockNumber, - ) -> Self { - const HEADER_BASE_SIZE_BYTES: usize = 4 + 32 + 16 + 64; - const INPUT_NOTE_COMMITMENT_SIZE_BYTES: usize = 64; - const OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES: usize = 700; - // Worst case, every input note resolves to a consumed-note reference (nullifier + note id) - // in the sync response. Counting it per input keeps input-heavy transactions under the cap. - const CONSUMED_NOTE_REF_SIZE_BYTES: usize = 64; - - // Serialize input notes as full InputNoteCommitments (nullifier + optional NoteHeader). - let input_notes: Vec = - transaction_header.input_notes().iter().cloned().collect(); - let input_notes_binary = input_notes.to_bytes(); - - // Serialize output notes as full NoteHeaders (NoteId + NoteMetadata). - let output_notes: Vec = transaction_header.output_notes().to_vec(); - let output_notes_binary = output_notes.to_bytes(); - - // Manually calculate the estimated size of the transaction header to avoid - // the cost of serialization. The size estimation includes: - // - 4 bytes for block number - // - 32 bytes for transaction ID - // - 16 bytes for account ID - // - 64 bytes for initial + final state commitments (32 bytes each) - // - ~64 bytes per input note (nullifier + optional NoteHeader) - // - ~64 bytes per input note for its possible consumed-note reference - // - ~700 bytes per output note sync record (metadata header + inclusion proof) - let input_notes_size = (transaction_header.input_notes().num_notes() as usize) - * (INPUT_NOTE_COMMITMENT_SIZE_BYTES + CONSUMED_NOTE_REF_SIZE_BYTES); - let output_notes_size = - transaction_header.output_notes().len() * OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES; - let size_in_bytes = (HEADER_BASE_SIZE_BYTES + input_notes_size + output_notes_size) as i64; - - Self { - transaction_id: transaction_header.id().to_bytes(), - account_id: transaction_header.account_id().to_bytes(), - block_num: block_num.to_raw_sql(), - initial_state_commitment: transaction_header.initial_state_commitment().to_bytes(), - final_state_commitment: transaction_header.final_state_commitment().to_bytes(), - input_notes: input_notes_binary, - output_notes: output_notes_binary, - size_in_bytes, - } - } -} - /// Select complete transaction records for the given accounts and block range. /// /// # Parameters diff --git a/crates/store/src/db/models/utils.rs b/crates/store/src/db/models/utils.rs index 1415ee29eb..7ada1f580b 100644 --- a/crates/store/src/db/models/utils.rs +++ b/crates/store/src/db/models/utils.rs @@ -1,5 +1,4 @@ use diesel::{Connection, RunQueryDsl, SqliteConnection}; -use miden_protocol::note::Nullifier; use miden_protocol::utils::serde::Serializable; use crate::errors::DatabaseError; @@ -21,11 +20,6 @@ pub(crate) fn serialize_vec<'a, D: Serializable + 'a>( Vec::<_>::from_iter(raw.into_iter().map(::to_bytes)) } -/// Returns the high 16 bits of the provided nullifier. -pub fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { - (nullifier.most_significant_felt().as_canonical_u64() >> 48) as u16 -} - /// Converts a slice of length `N` to an array, returns `None` if invariant /// isn'crates/store/src/db/mod.rs upheld. pub fn slice_to_array(bytes: &[u8]) -> Option<[u8; N]> { diff --git a/crates/store/src/db/queries/apply_block.rs b/crates/store/src/db/queries/apply_block.rs new file mode 100644 index 0000000000..41a374b0f4 --- /dev/null +++ b/crates/store/src/db/queries/apply_block.rs @@ -0,0 +1,48 @@ +//! Writes every table a committed block touches. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::block::SignedBlock; +use miden_protocol::note::Nullifier; + +use crate::db::NoteRecord; +use crate::db::queries::{ + PrecomputedPublicAccountStates, + insert_block_header, + insert_note_scripts, + insert_notes, + insert_nullifiers_for_block, + insert_transactions, + upsert_accounts, +}; +use crate::errors::DatabaseError; + +/// Apply a new block to the state. +/// +/// # Returns +/// +/// Number of records inserted and/or updated. +pub(crate) fn apply_block( + tx: &WriteTx<'_>, + block: &SignedBlock, + notes: &[(NoteRecord, Option)], + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let mut count = 0; + // Note: ordering here is important as the relevant tables have FK dependencies. + count += insert_block_header(tx, block.header(), block.signatures())?; + count += upsert_accounts( + tx, + block.body().updated_accounts(), + block.header().block_num(), + precomputed_public_states, + )?; + count += insert_note_scripts(tx, notes.iter().map(|(note, _)| note))?; + count += insert_notes(tx, notes)?; + count += insert_transactions(tx, block.header().block_num(), block.body().transactions())?; + count += insert_nullifiers_for_block( + tx, + block.body().created_nullifiers(), + block.header().block_num(), + )?; + Ok(count) +} diff --git a/crates/store/src/db/queries/filter_network_accounts/filter_network_accounts.sql b/crates/store/src/db/queries/filter_network_accounts/filter_network_accounts.sql new file mode 100644 index 0000000000..60da11adaa --- /dev/null +++ b/crates/store/src/db/queries/filter_network_accounts/filter_network_accounts.sql @@ -0,0 +1,9 @@ +-- Returns which of the given accounts are network accounts in their latest committed state. +-- +-- Account ids are bound as a single array parameter so the statement text stays constant regardless +-- of how many are requested; see `miden_node_db::sqlite::InList`. +SELECT account_id +FROM accounts +WHERE account_id IN (SELECT value FROM rarray(?1)) + AND network_account_type = ?2 + AND valid_until = ?3; diff --git a/crates/store/src/db/queries/filter_network_accounts/mod.rs b/crates/store/src/db/queries/filter_network_accounts/mod.rs new file mode 100644 index 0000000000..8fc10a81b9 --- /dev/null +++ b/crates/store/src/db/queries/filter_network_accounts/mod.rs @@ -0,0 +1,33 @@ +//! Filters a set of accounts down to the network accounts among them. + +use std::collections::HashSet; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; +use miden_protocol::account::AccountId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::queries::{NetworkAccountType, VALID_FOREVER}; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("filter_network_accounts.sql"); + +/// Returns the subset of `account_ids` whose latest committed state is a network account. +/// +/// Unknown ids and non-network accounts are silently omitted. +pub(crate) fn filter_network_accounts( + tx: &ReadTx<'_>, + account_ids: &[AccountId], +) -> Result, DatabaseError> { + QueryParamAccountIdLimit::check(account_ids.len())?; + + let id_bytes = Vec::from_iter(account_ids.iter().map(Serializable::to_bytes)); + let ids = InList::from_blobs(id_bytes.iter().map(Vec::as_slice)); + + Ok(tx + .query(SQL, &[&ids, &NetworkAccountType::Network, &VALID_FOREVER], |row| { + row.get::(0) + })? + .into_iter() + .collect()) +} diff --git a/crates/store/src/db/queries/insert_block_header/insert_block_header.sql b/crates/store/src/db/queries/insert_block_header/insert_block_header.sql new file mode 100644 index 0000000000..07b1ba8c50 --- /dev/null +++ b/crates/store/src/db/queries/insert_block_header/insert_block_header.sql @@ -0,0 +1,3 @@ +-- Inserts a block header together with the signatures that committed it. +INSERT INTO block_headers (block_num, block_header, signature, commitment) +VALUES (?1, ?2, ?3, ?4) diff --git a/crates/store/src/db/queries/insert_block_header/mod.rs b/crates/store/src/db/queries/insert_block_header/mod.rs new file mode 100644 index 0000000000..e89208b6c3 --- /dev/null +++ b/crates/store/src/db/queries/insert_block_header/mod.rs @@ -0,0 +1,33 @@ +//! Inserts a block header and the signatures that committed it. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::{BlockHeader, BlockSignatures}; + +use crate::COMPONENT; +use crate::db::BlockHeaderCommitment; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_block_header.sql"); + +/// Inserts a [`BlockHeader`] and its [`BlockSignatures`]. +/// +/// The header's commitment is stored alongside it so the chain MMR can be rebuilt without +/// deserializing every header. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_block_header( + tx: &WriteTx<'_>, + block_header: &BlockHeader, + signatures: &BlockSignatures, +) -> Result { + let commitment = BlockHeaderCommitment::new(block_header).word(); + + Ok(tx.execute(SQL, &[&block_header.block_num(), block_header, signatures, &commitment])?) +} diff --git a/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql b/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql new file mode 100644 index 0000000000..c5933a00cc --- /dev/null +++ b/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql @@ -0,0 +1,4 @@ +-- Inserts a note script, keyed by its root. Scripts are shared across notes, so re-inserting a +-- known root is a no-op rather than a constraint violation. +INSERT OR IGNORE INTO note_scripts (script_root, script) +VALUES (?1, ?2) diff --git a/crates/store/src/db/queries/insert_note_scripts/mod.rs b/crates/store/src/db/queries/insert_note_scripts/mod.rs new file mode 100644 index 0000000000..46866e4e7c --- /dev/null +++ b/crates/store/src/db/queries/insert_note_scripts/mod.rs @@ -0,0 +1,38 @@ +//! Inserts the note scripts held by a block's notes. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; + +use crate::COMPONENT; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_note_script.sql"); + +/// Inserts the scripts held by the given notes. Notes without details (private notes) carry no +/// script, and a script root already in the table is left untouched. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_note_scripts<'a>( + tx: &WriteTx<'_>, + notes: impl IntoIterator, +) -> Result { + let mut count = 0; + for note in notes { + let Some(details) = note.details.as_ref() else { + continue; + }; + let script = details.script(); + // The column stores the root as its word representation. + let script_root = Word::from(script.root()); + count += tx.execute(SQL, &[&script_root, script])?; + } + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_notes/insert_note.sql b/crates/store/src/db/queries/insert_notes/insert_note.sql new file mode 100644 index 0000000000..36184c08e2 --- /dev/null +++ b/crates/store/src/db/queries/insert_notes/insert_note.sql @@ -0,0 +1,25 @@ +-- Inserts a note committed by a block. +-- +-- Public notes carry their nullifier and detail columns (assets, storage, script root, serial +-- number); private notes store NULL for all of them. `consumed_at` is always NULL here: a freshly +-- committed note is unconsumed until a later block's nullifiers mark it. +INSERT INTO notes ( + committed_at, + batch_index, + note_index, + note_id, + note_type, + sender, + tag, + network_note_type, + target_account_id, + attachment, + inclusion_path, + consumed_at, + nullifier, + assets, + storage, + script_root, + serial_num +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) diff --git a/crates/store/src/db/queries/insert_notes/mod.rs b/crates/store/src/db/queries/insert_notes/mod.rs new file mode 100644 index 0000000000..680cd70489 --- /dev/null +++ b/crates/store/src/db/queries/insert_notes/mod.rs @@ -0,0 +1,117 @@ +//! Inserts the notes created by a block. + +use miden_node_db::sqlite::{DbValue, ToSqlValue, WriteTx}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::{NoteAssets, NoteDetails, NoteStorage, Nullifier}; +use miden_standards::note::NetworkAccountTarget; + +use crate::COMPONENT; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_note.sql"); + +// NETWORK NOTE TYPE +// ================================================================================================ + +/// Classifies network notes for database storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub(crate) enum NetworkNoteType { + /// Not a network note. + None = 0, + /// Single account target network note (has `NetworkAccountTarget` attachment). + SingleTarget = 1, +} + +impl ToSqlValue for NetworkNoteType { + fn to_sql_value(&self) -> DbValue { + DbValue::integer(*self as i64) + } +} + +// QUERY +// ================================================================================================ + +/// Inserts the notes created by a block. Public notes are inserted with their nullifier. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_notes( + tx: &WriteTx<'_>, + notes: &[(NoteRecord, Option)], +) -> Result { + let mut count = 0; + for (note, nullifier) in notes { + count += insert_note(tx, note, *nullifier)?; + } + Ok(count) +} + +/// Inserts a single note, deriving its network-note classification from its attachments. +fn insert_note( + tx: &WriteTx<'_>, + note: &NoteRecord, + nullifier: Option, +) -> Result { + let target_account_id: Option = NetworkAccountTarget::try_from(¬e.attachments) + .ok() + .map(|target| target.target_id()); + // A private note is never routed to a network account, even when it carries the attachment. + let network_note_type = if target_account_id.is_some() && !note.metadata.is_private() { + NetworkNoteType::SingleTarget + } else { + NetworkNoteType::None + }; + + let batch_index = index_column(note.note_index.batch_idx()); + let note_index = index_column(note.note_index.note_idx_in_batch()); + let note_type = note.metadata.note_type() as u8; + + // Private notes carry no details, in which case every detail column is NULL. + let details = note.details.as_ref(); + let assets: Option<&NoteAssets> = details.map(NoteDetails::assets); + let storage: Option<&NoteStorage> = details.map(NoteDetails::storage); + // The column stores the script root as its word representation. + let script_root: Option = details.map(|d| Word::from(d.script().root())); + let serial_num: Option = details.map(NoteDetails::serial_num); + + Ok(tx.execute( + SQL, + &[ + ¬e.block_num, + &batch_index, + ¬e_index, + ¬e.note_id, + ¬e_type, + ¬e.metadata.sender(), + ¬e.metadata.tag(), + &network_note_type, + &target_account_id, + ¬e.attachments, + ¬e.inclusion_path, + // New notes are always unconsumed. + &None::, + &nullifier, + &assets, + &storage, + &script_root, + &serial_num, + ], + )?) +} + +/// Narrows a note index to the `u32` the column stores. +/// +/// Both indices are bounded by the block's batch and note limits, which are far below `u32::MAX`. +fn index_column(index: usize) -> u32 { + u32::try_from(index).expect("note indices are bounded well below u32::MAX") +} diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql b/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql new file mode 100644 index 0000000000..d734f365b3 --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql @@ -0,0 +1,4 @@ +-- Records a nullifier created by a block. The prefix column is indexed so nullifier lookups by +-- prefix never have to scan the full nullifier. +INSERT INTO nullifiers (nullifier, nullifier_prefix, block_num) +VALUES (?1, ?2, ?3) diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql b/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql new file mode 100644 index 0000000000..3bbca7a68c --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql @@ -0,0 +1,8 @@ +-- Marks the notes spent by a block's nullifiers as consumed at that block. +-- +-- Nullifiers are bound as a single array parameter so the statement text stays constant regardless +-- of how many the block created; see `miden_node_db::sqlite::InList`. Nullifiers whose note is not +-- stored here (a private note, or one committed before this node's history) match nothing. +UPDATE notes +SET consumed_at = ?1 +WHERE nullifier IN (SELECT value FROM rarray(?2)) diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs b/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs new file mode 100644 index 0000000000..8fde177c5f --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs @@ -0,0 +1,47 @@ +//! Records the nullifiers created by a block and marks the notes they consume. + +use miden_node_db::sqlite::{InList, WriteTx}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::Nullifier; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::db::utils::get_nullifier_prefix; +use crate::errors::DatabaseError; + +const SQL_MARK_NOTES_CONSUMED: &str = include_str!("mark_notes_consumed.sql"); +const SQL_INSERT_NULLIFIER: &str = include_str!("insert_nullifier.sql"); + +/// Inserts the nullifiers created by a block, and marks the notes they consume as consumed at that +/// block. +/// +/// # Parameters +/// * `nullifiers`: List of nullifiers to insert +/// - Limit: 0 <= count <= 1000 +/// * `block_num`: Block number to associate with the nullifiers +/// +/// # Returns +/// +/// The number of affected rows, counting both the consumed notes and the inserted nullifiers. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_nullifiers_for_block( + tx: &WriteTx<'_>, + nullifiers: &[Nullifier], + block_num: BlockNumber, +) -> Result { + let serialized = Vec::from_iter(nullifiers.iter().map(Serializable::to_bytes)); + let consumed = InList::from_blobs(serialized.iter().map(Vec::as_slice)); + + let mut count = tx.execute(SQL_MARK_NOTES_CONSUMED, &[&block_num, &consumed])?; + + for nullifier in nullifiers { + let prefix = get_nullifier_prefix(nullifier); + count += tx.execute(SQL_INSERT_NULLIFIER, &[nullifier, &prefix, &block_num])?; + } + + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_storage_map_value/close_storage_map_value_validity.sql b/crates/store/src/db/queries/insert_storage_map_value/close_storage_map_value_validity.sql new file mode 100644 index 0000000000..7454a2fe8e --- /dev/null +++ b/crates/store/src/db/queries/insert_storage_map_value/close_storage_map_value_validity.sql @@ -0,0 +1,10 @@ +-- Closes the previous version of a storage-map entry at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE account_storage_map_values +SET valid_until = ?1 +WHERE account_id = ?2 + AND slot_name = ?3 + AND key = ?4 + AND valid_until = ?5 diff --git a/crates/store/src/db/queries/insert_storage_map_value/insert_storage_map_value.sql b/crates/store/src/db/queries/insert_storage_map_value/insert_storage_map_value.sql new file mode 100644 index 0000000000..e659612419 --- /dev/null +++ b/crates/store/src/db/queries/insert_storage_map_value/insert_storage_map_value.sql @@ -0,0 +1,3 @@ +-- Inserts a storage-map entry as the current version of its key, valid from `block_num` onwards. +INSERT INTO account_storage_map_values (account_id, block_num, slot_name, key, value, valid_until) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) diff --git a/crates/store/src/db/queries/insert_storage_map_value/mod.rs b/crates/store/src/db/queries/insert_storage_map_value/mod.rs new file mode 100644 index 0000000000..1f4ae03c40 --- /dev/null +++ b/crates/store/src/db/queries/insert_storage_map_value/mod.rs @@ -0,0 +1,68 @@ +//! Writes a versioned account storage-map value. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::Word; +use miden_protocol::account::{AccountId, StorageMapKey, StorageSlotName}; +use miden_protocol::block::BlockNumber; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_CLOSE: &str = include_str!("close_storage_map_value_validity.sql"); +const SQL_INSERT: &str = include_str!("insert_storage_map_value.sql"); + +/// Inserts a versioned account storage-map value. +/// +/// The new row is inserted open-ended, and any previous open row for the same +/// `(account_id, slot_name, key)` tuple has its validity interval closed at `block_num` first. +/// +/// # Returns +/// +/// The total number of inserted and invalidated rows. +/// +/// # Errors +/// +/// Returns an error if the previous row cannot be invalidated or the new row cannot be inserted. +pub(crate) fn insert_storage_map_value( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + slot_name: &StorageSlotName, + key: StorageMapKey, + value: Word, +) -> Result { + insert_storage_map_value_inner(tx, account_id, block_num, slot_name, key, value, true) +} + +/// Inserts a versioned account storage-map value with optional previous-row invalidation. +/// +/// `invalidate_previous` may be disabled when inserting state for a new account, for which no +/// previous open row can exist. The inserted row is always open-ended. +/// +/// # Returns +/// +/// The total number of inserted and invalidated rows. +/// +/// # Errors +/// +/// Returns an error if the requested invalidation or insertion fails. +pub(super) fn insert_storage_map_value_inner( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + slot_name: &StorageSlotName, + key: StorageMapKey, + value: Word, + invalidate_previous: bool, +) -> Result { + let mut count = 0; + if invalidate_previous { + count += + tx.execute(SQL_CLOSE, &[&block_num, &account_id, slot_name, &key, &VALID_FOREVER])?; + } + + count += tx + .execute(SQL_INSERT, &[&account_id, &block_num, slot_name, &key, &value, &VALID_FOREVER])?; + + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_transactions/insert_transaction.sql b/crates/store/src/db/queries/insert_transactions/insert_transaction.sql new file mode 100644 index 0000000000..2e1ce61bbd --- /dev/null +++ b/crates/store/src/db/queries/insert_transactions/insert_transaction.sql @@ -0,0 +1,16 @@ +-- Records a transaction included in a block. +-- +-- `size_in_bytes` is the estimated size of the sync record this transaction produces; it lets the +-- transaction-record queries stop before they exceed the response payload limit without having to +-- deserialize each row. +INSERT INTO transactions ( + transaction_id, + account_id, + block_num, + initial_state_commitment, + final_state_commitment, + input_notes, + output_notes, + size_in_bytes +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) diff --git a/crates/store/src/db/queries/insert_transactions/mod.rs b/crates/store/src/db/queries/insert_transactions/mod.rs new file mode 100644 index 0000000000..122e66d63b --- /dev/null +++ b/crates/store/src/db/queries/insert_transactions/mod.rs @@ -0,0 +1,95 @@ +//! Inserts the transactions included in a block. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::NoteHeader; +use miden_protocol::transaction::{ + InputNoteCommitment, + OrderedTransactionHeaders, + TransactionHeader, +}; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_transaction.sql"); + +/// Inserts the transactions included in a block. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_transactions( + tx: &WriteTx<'_>, + block_num: BlockNumber, + transactions: &OrderedTransactionHeaders, +) -> Result { + let mut count = 0; + for header in transactions.as_slice() { + count += insert_transaction(tx, block_num, header)?; + } + Ok(count) +} + +/// Inserts a single transaction header. +fn insert_transaction( + tx: &WriteTx<'_>, + block_num: BlockNumber, + header: &TransactionHeader, +) -> Result { + // Serialize input notes as full InputNoteCommitments (nullifier + optional NoteHeader). + let input_notes: Vec = header.input_notes().iter().cloned().collect(); + let input_notes = input_notes.to_bytes(); + + // Serialize output notes as full NoteHeaders (NoteId + NoteMetadata). + let output_notes: Vec = header.output_notes().to_vec(); + let output_notes = output_notes.to_bytes(); + + Ok(tx.execute( + SQL, + &[ + &header.id(), + &header.account_id(), + &block_num, + &header.initial_state_commitment(), + &header.final_state_commitment(), + &input_notes, + &output_notes, + &estimated_sync_record_size(header), + ], + )?) +} + +/// Estimates the size of the sync record this transaction produces. +/// +/// The estimate is computed from note counts rather than by serializing the record, which would +/// cost far more than the estimate is worth. It is deliberately an over-estimate, so a response +/// assembled under the limit is always within it. +#[expect( + clippy::cast_possible_wrap, + reason = "We will not approach the item count where i64 and usize cause issues" +)] +fn estimated_sync_record_size(header: &TransactionHeader) -> i64 { + // - 4 bytes for block number + // - 32 bytes for transaction ID + // - 16 bytes for account ID + // - 64 bytes for initial + final state commitments (32 bytes each) + const HEADER_BASE_SIZE_BYTES: usize = 4 + 32 + 16 + 64; + const INPUT_NOTE_COMMITMENT_SIZE_BYTES: usize = 64; + const OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES: usize = 700; + // Worst case, every input note resolves to a consumed-note reference (nullifier + note id) in + // the sync response. Counting it per input keeps input-heavy transactions under the cap. + const CONSUMED_NOTE_REF_SIZE_BYTES: usize = 64; + + let input_notes_size = (header.input_notes().num_notes() as usize) + * (INPUT_NOTE_COMMITMENT_SIZE_BYTES + CONSUMED_NOTE_REF_SIZE_BYTES); + let output_notes_size = header.output_notes().len() * OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES; + + (HEADER_BASE_SIZE_BYTES + input_notes_size + output_notes_size) as i64 +} diff --git a/crates/store/src/db/queries/insert_vault_asset/close_vault_asset_validity.sql b/crates/store/src/db/queries/insert_vault_asset/close_vault_asset_validity.sql new file mode 100644 index 0000000000..0f3af6ad84 --- /dev/null +++ b/crates/store/src/db/queries/insert_vault_asset/close_vault_asset_validity.sql @@ -0,0 +1,9 @@ +-- Closes the previous version of a vault asset at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE account_vault_assets +SET valid_until = ?1 +WHERE account_id = ?2 + AND vault_key = ?3 + AND valid_until = ?4 diff --git a/crates/store/src/db/queries/insert_vault_asset/insert_vault_asset.sql b/crates/store/src/db/queries/insert_vault_asset/insert_vault_asset.sql new file mode 100644 index 0000000000..5952264c1c --- /dev/null +++ b/crates/store/src/db/queries/insert_vault_asset/insert_vault_asset.sql @@ -0,0 +1,4 @@ +-- Inserts a vault asset as the current version of its key, valid from `block_num` onwards. A NULL +-- asset records the removal of that key. +INSERT INTO account_vault_assets (account_id, block_num, vault_key, asset, valid_until) +VALUES (?1, ?2, ?3, ?4, ?5) diff --git a/crates/store/src/db/queries/insert_vault_asset/mod.rs b/crates/store/src/db/queries/insert_vault_asset/mod.rs new file mode 100644 index 0000000000..0abaa5dadb --- /dev/null +++ b/crates/store/src/db/queries/insert_vault_asset/mod.rs @@ -0,0 +1,42 @@ +//! Writes a versioned account vault asset. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::asset::{Asset, AssetId}; +use miden_protocol::block::BlockNumber; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_CLOSE: &str = include_str!("close_vault_asset_validity.sql"); +const SQL_INSERT: &str = include_str!("insert_vault_asset.sql"); + +/// Inserts an account vault asset row. +/// +/// The new row is inserted open-ended (`valid_until = VALID_FOREVER`); any existing open row with +/// the same `(account_id, vault_key)` tuple has its validity interval closed at `block_num`. A +/// `None` asset records the removal of that vault key. +/// +/// # Returns +/// +/// The number of affected rows. +pub(crate) fn insert_vault_asset( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + // The column stores the asset id as its word representation. + let vault_key = Word::from(vault_key); + + // Close the previous version's validity interval at the new row's block. + let mut count = + tx.execute(SQL_CLOSE, &[&block_num, &account_id, &vault_key, &VALID_FOREVER])?; + + count += + tx.execute(SQL_INSERT, &[&account_id, &block_num, &vault_key, &asset, &VALID_FOREVER])?; + + Ok(count) +} diff --git a/crates/store/src/db/queries/mod.rs b/crates/store/src/db/queries/mod.rs new file mode 100644 index 0000000000..525a4e41a0 --- /dev/null +++ b/crates/store/src/db/queries/mod.rs @@ -0,0 +1,128 @@ +//! Database query functions for the store, on the `miden-node-db` SQLite framework. +//! +//! Each function takes a [`ReadTx`](miden_node_db::sqlite::ReadTx) or +//! [`WriteTx`](miden_node_db::sqlite::WriteTx) and is driven from a [`Db`](crate::db::Db) method +//! through [`DbReader::read`](miden_node_db::sqlite::DbReader::read) / +//! [`DbWriter::write`](miden_node_db::sqlite::DbWriter::write). One module per query, holding the +//! function and the `.sql` file it executes. +//! +//! The store is being migrated to the framework incrementally: every write goes through the +//! modules here, while most reads still run on the diesel layer in [`crate::db::models`]. Read +//! queries move here one batch at a time until the diesel layer is removed. + +use miden_node_db::DatabaseError; +use miden_node_db::sqlite::{DbValue, DbValueRef, FromSqlValue, ToSqlValue}; + +// SHARED COLUMN TYPES +// ================================================================================================= + +/// Sentinel `valid_until` value marking a row as the current, open-ended version of its key. +/// +/// Versioned rows (`accounts`, `account_vault_assets`, `account_storage_map_values`) are +/// applicable for blocks in `[block_num, valid_until)`; updating a key closes the previous row's +/// interval by setting its `valid_until` to the new row's `block_num`. The open end is `i64::MAX` +/// rather than NULL so every validity predicate is a single range comparison that partial indexes +/// can serve. +pub(crate) const VALID_FOREVER: i64 = i64::MAX; + +/// Classifies accounts for database storage based on whether they are network accounts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub(crate) enum NetworkAccountType { + /// Not a network account. + None = 0, + /// A network account. + Network = 1, +} + +impl ToSqlValue for NetworkAccountType { + fn to_sql_value(&self) -> DbValue { + DbValue::integer(*self as i64) + } +} + +impl FromSqlValue for NetworkAccountType { + fn from_sql_value(value: DbValueRef<'_>) -> Result { + match value.as_i64()? { + 0 => Ok(Self::None), + 1 => Ok(Self::Network), + other => Err(DatabaseError::deserialization( + "NetworkAccountType", + InvalidNetworkAccountType(other), + )), + } + } +} + +#[derive(Debug, thiserror::Error)] +#[error("invalid network account type value {0}")] +struct InvalidNetworkAccountType(i64); + +// BLOCK QUERIES +// ================================================================================================= + +mod insert_block_header; +pub(crate) use insert_block_header::insert_block_header; + +// NOTE QUERIES +// ================================================================================================= + +mod insert_note_scripts; +pub(crate) use insert_note_scripts::insert_note_scripts; + +mod insert_notes; +pub(crate) use insert_notes::insert_notes; + +// NULLIFIER QUERIES +// ================================================================================================= + +mod insert_nullifiers_for_block; +pub(crate) use insert_nullifiers_for_block::insert_nullifiers_for_block; + +// TRANSACTION QUERIES +// ================================================================================================= + +mod insert_transactions; +pub(crate) use insert_transactions::insert_transactions; + +// ACCOUNT QUERIES +// ================================================================================================= + +mod insert_storage_map_value; +pub(crate) use insert_storage_map_value::insert_storage_map_value; + +mod insert_vault_asset; +pub(crate) use insert_vault_asset::insert_vault_asset; + +mod prune_history; +pub use prune_history::HISTORICAL_BLOCK_RETENTION; +pub(crate) use prune_history::prune_history; + +mod upsert_accounts; +pub(crate) use upsert_accounts::{AccountRow, upsert_accounts}; +pub use upsert_accounts::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; + +mod filter_network_accounts; +pub(crate) use filter_network_accounts::filter_network_accounts; + +mod select_account_header_with_storage_header_at_block; +pub(crate) use select_account_header_with_storage_header_at_block::select_account_header_with_storage_header_at_block; + +mod select_vault_at_block; +pub(crate) use select_vault_at_block::select_vault_at_block; + +#[cfg(test)] +mod select_full_account; +#[cfg(test)] +pub(crate) use select_full_account::select_full_account; + +#[cfg(test)] +mod select_latest_storage; +#[cfg(test)] +pub(crate) use select_latest_storage::select_latest_storage; + +// BLOCK APPLICATION +// ================================================================================================= + +mod apply_block; +pub(crate) use apply_block::apply_block; diff --git a/crates/store/src/db/queries/prune_history/mod.rs b/crates/store/src/db/queries/prune_history/mod.rs new file mode 100644 index 0000000000..5a453f7546 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/mod.rs @@ -0,0 +1,119 @@ +//! Deletes account history that can no longer serve a read inside the retention window. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; + +use crate::COMPONENT; +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_VAULT_ASSETS: &str = include_str!("prune_account_vault_assets.sql"); +const SQL_STORAGE_MAP_VALUES: &str = include_str!("prune_account_storage_map_values.sql"); +const SQL_ACCOUNT_CODES_FULL: &str = include_str!("prune_account_codes_full.sql"); +const SQL_ACCOUNT_CODES_WINDOWED: &str = include_str!("prune_account_codes_windowed.sql"); +const SQL_SELECT_PRUNE_PROGRESS: &str = include_str!("select_prune_progress.sql"); +const SQL_UPSERT_PRUNE_PROGRESS: &str = include_str!("upsert_prune_progress.sql"); + +/// The two pruning statements spell the open-ended sentinel out as a literal so SQLite can match it +/// against the partial cleanup indexes; that literal has to stay in step with [`VALID_FOREVER`]. +const _: () = assert!( + VALID_FOREVER == 9_223_372_036_854_775_807, + "the `valid_until != ` literal in the pruning statements is out of date" +); + +/// Number of historical blocks to retain for vault assets, storage map values, and account codes. +/// Rows whose validity interval ends at or below `prune_tip - HISTORICAL_BLOCK_RETENTION` will be +/// deleted; rows still valid anywhere inside the retention window (including all open-ended rows) +/// are retained. +pub const HISTORICAL_BLOCK_RETENTION: u32 = 50; + +/// Clean up old entries for all accounts, deleting entries that can no longer affect state +/// reconstruction at any block within the retention window. +/// +/// A row is applicable for blocks in `[block_num, valid_until)`, so it is deletable exactly when +/// its interval ends at or below the cutoff (`prune_tip - HISTORICAL_BLOCK_RETENTION`): it then +/// cannot cover any block inside the window. `prune_tip` is the effective tip for retention — it +/// lags the chain tip while old snapshot generations are still pinned by readers (see +/// [`crate::db::Db::apply_block`]). Account codes follow the same rule — a code is deleted only +/// when no account row whose interval reaches past the cutoff references it. +/// +/// # Returns +/// A tuple of `(vault_assets_deleted, storage_map_values_deleted, account_codes_deleted)` +#[miden_instrument( + target = COMPONENT, + err, + fields( + cutoff_block, + ), +)] +pub(crate) fn prune_history( + tx: &WriteTx<'_>, + prune_tip: BlockNumber, +) -> Result<(usize, usize, usize), DatabaseError> { + let cutoff_block = i64::from(prune_tip.as_u32().saturating_sub(HISTORICAL_BLOCK_RETENTION)); + tracing::Span::current().record("cutoff_block", cutoff_block); + + let vault_deleted = tx.execute(SQL_VAULT_ASSETS, &[&cutoff_block])?; + let storage_deleted = tx.execute(SQL_STORAGE_MAP_VALUES, &[&cutoff_block])?; + let codes_deleted = prune_account_codes(tx, cutoff_block)?; + + Ok((vault_deleted, storage_deleted, codes_deleted)) +} + +/// Deletes account codes that are no longer referenced by any account row that can serve a read +/// within the retention window. +/// +/// An account code is safe to delete when no `accounts` row whose validity interval reaches past +/// the cutoff (`valid_until > cutoff_block`) references it. That single predicate covers rows +/// inside the window, all open-ended (current) rows, and each account's baseline row — the row +/// still valid at the cutoff even though it was written before it. +/// +/// Rather than re-checking every code on every prune, only codes whose deletability could have +/// changed since the previous prune are examined. A code survived the previous prune because at +/// least one `accounts` row with `valid_until > prev_cutoff` referenced it. For it to be +/// deletable now, all such rows must have expired by the new cutoff — including the longest-lived +/// one, whose `valid_until` therefore lands inside `(prev_cutoff, cutoff_block]`. Scanning the +/// rows that expired in that window thus finds every code that could have become deletable. The +/// previous cutoff is persisted in `prune_progress` within the same transaction; when absent +/// (first prune after migration, or a fresh database) a full pass over all rows valid past the +/// cutoff runs instead. +/// +/// Correctness of the windowed candidate set rests on two invariants: +/// - Rows are only ever closed to the `block_num` of the block currently being applied, which is +/// always above the cutoff, so every expiry crosses the window of some later prune. A write path +/// that back-dated `valid_until` below the current cutoff would leak the code forever. +/// - Every `account_codes` row is inserted alongside an `accounts` row referencing it (see +/// [`upsert_accounts`](super::upsert_accounts)); an orphan code with no referencing row would +/// never become a candidate. +#[miden_instrument( + target = COMPONENT, + err, + fields( + cutoff_block, + ), +)] +fn prune_account_codes(tx: &WriteTx<'_>, cutoff_block: i64) -> Result { + let prev_cutoff = tx + .query(SQL_SELECT_PRUNE_PROGRESS, &[], |row| row.get::(0))? + .into_iter() + .next(); + + let deleted = match prev_cutoff { + // Codes are already pruned through this cutoff and nothing can become collectable while the + // cutoff stands still. Equality is the common case: the cutoff is clamped to zero for the + // first `HISTORICAL_BLOCK_RETENTION` blocks, and a pinned snapshot freezes the prune tip + // across consecutive blocks. A strictly greater `prev_cutoff` is unreachable through + // `apply_block` (the prune tip never regresses) but is guarded against so an out-of-order + // caller cannot move the marker backwards or run the delete with an inverted window. + Some(prev_cutoff) if prev_cutoff >= cutoff_block => return Ok(0), + Some(prev_cutoff) => { + tx.execute(SQL_ACCOUNT_CODES_WINDOWED, &[&prev_cutoff, &cutoff_block])? + }, + None => tx.execute(SQL_ACCOUNT_CODES_FULL, &[&cutoff_block])?, + }; + + tx.execute(SQL_UPSERT_PRUNE_PROGRESS, &[&cutoff_block])?; + + Ok(deleted) +} diff --git a/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql b/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql new file mode 100644 index 0000000000..b71bdf51fd --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql @@ -0,0 +1,16 @@ +-- Deletes account codes that no account row reaching past the retention cutoff still references. +-- +-- The full pass, used when no previous cutoff has been recorded (the first prune after migration, +-- or a fresh database). The single `valid_until > ?1` predicate covers rows inside the window, all +-- open-ended (current) rows, and each account's baseline row — the row still valid at the cutoff +-- even though it was written before it. +-- +-- The forced `idx_accounts_code_validity` covering index keeps the subquery an index-only range +-- scan, sized by rows valid at or after the cutoff rather than by total history. +DELETE FROM account_codes +WHERE code_commitment NOT IN ( + SELECT DISTINCT code_commitment + FROM accounts INDEXED BY idx_accounts_code_validity + WHERE code_commitment IS NOT NULL + AND valid_until > ?1 +) diff --git a/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql b/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql new file mode 100644 index 0000000000..78db2a4eef --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql @@ -0,0 +1,25 @@ +-- Deletes account codes that became collectable since the previous prune. +-- +-- Candidates are the codes referenced by rows whose validity interval ended inside +-- `(?1, ?2]` — the window between the previous cutoff and this one. A code that survived the +-- previous prune did so because some row with `valid_until > ?1` referenced it; for it to be +-- deletable now, the longest-lived such row must have expired by `?2`, which puts its +-- `valid_until` in exactly that window. The scan is an `idx_accounts_code_validity` index range, +-- so its cost scales with account updates since the previous prune, not with total history. +-- +-- Each candidate is then deleted only if the `idx_accounts_code_probe` existence probe finds no +-- row still referencing it past the new cutoff. +DELETE FROM account_codes +WHERE code_commitment IN ( + SELECT DISTINCT code_commitment + FROM accounts INDEXED BY idx_accounts_code_validity + WHERE code_commitment IS NOT NULL + AND valid_until > ?1 + AND valid_until <= ?2 +) +AND NOT EXISTS ( + SELECT 1 + FROM accounts INDEXED BY idx_accounts_code_probe + WHERE accounts.code_commitment = account_codes.code_commitment + AND accounts.valid_until > ?2 +) diff --git a/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql b/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql new file mode 100644 index 0000000000..4c67bcf433 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql @@ -0,0 +1,8 @@ +-- Deletes storage-map rows whose validity interval ends at or below the retention cutoff. +-- +-- The literal sentinel term (rather than a bound parameter) lets SQLite prove the predicate implies +-- `idx_storage_cleanup`'s partial-index condition. It is kept in sync with `VALID_FOREVER` by a +-- compile-time assertion in this module. +DELETE FROM account_storage_map_values +WHERE valid_until != 9223372036854775807 + AND valid_until <= ?1 diff --git a/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql b/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql new file mode 100644 index 0000000000..3e2d98827d --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql @@ -0,0 +1,8 @@ +-- Deletes vault-asset rows whose validity interval ends at or below the retention cutoff. +-- +-- The literal sentinel term (rather than a bound parameter) lets SQLite prove the predicate implies +-- `idx_vault_cleanup`'s partial-index condition. It is kept in sync with `VALID_FOREVER` by a +-- compile-time assertion in this module. +DELETE FROM account_vault_assets +WHERE valid_until != 9223372036854775807 + AND valid_until <= ?1 diff --git a/crates/store/src/db/queries/prune_history/select_prune_progress.sql b/crates/store/src/db/queries/prune_history/select_prune_progress.sql new file mode 100644 index 0000000000..edcb2acdb9 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/select_prune_progress.sql @@ -0,0 +1,3 @@ +-- Returns the cutoff through which account-code pruning has completed, if any prune has run under +-- this schema. The table holds at most one row, pinned to `id = 0`. +SELECT codes_cutoff FROM prune_progress diff --git a/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql b/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql new file mode 100644 index 0000000000..a111760097 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql @@ -0,0 +1,5 @@ +-- Records the cutoff through which account-code pruning has completed. Written in the same +-- transaction as the prune itself, so the marker is exact and crash-consistent. +INSERT INTO prune_progress (id, codes_cutoff) +VALUES (0, ?1) +ON CONFLICT(id) DO UPDATE SET codes_cutoff = excluded.codes_cutoff diff --git a/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs new file mode 100644 index 0000000000..9f14b491e2 --- /dev/null +++ b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs @@ -0,0 +1,59 @@ +//! Returns an account's header as of a block. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader}; +use miden_protocol::block::BlockNumber; +use miden_protocol::{Felt, Word}; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_header_at_block.sql"); + +/// The header columns as stored; every one of them is nullable for private accounts. +type AccountHeaderRow = (Option, Option, Option, Option); + +/// Queries the account header for a specific account at a specific block number. +/// +/// This reconstructs the [`AccountHeader`] by reading from the `accounts` table: +/// `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root`. +/// +/// # Returns +/// +/// * `Ok(Some((AccountHeader, AccountStorageHeader)))` - The headers if found +/// * `Ok(None)` - If account doesn't exist at that block +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_header_with_storage_header_at_block( + tx: &ReadTx<'_>, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + let row = tx + .query(SQL, &[&account_id, &block_num], |row| -> Result { + Ok(( + row.get::>(0)?, + row.get::>(1)?, + row.get::>(2)?, + row.get::>(3)?, + )) + })? + .into_iter() + .next(); + + let Some((code_commitment, nonce, storage_header, vault_root)) = row else { + return Ok(None); + }; + + // A private account stores none of these, in which case the header reads as empty/default. + let storage_header = storage_header.unwrap_or(AccountStorageHeader::new(Vec::new())?); + let storage_commitment = storage_header.to_commitment(); + + let account_header = AccountHeader::new( + account_id, + nonce.unwrap_or(Felt::ZERO), + vault_root.unwrap_or_default(), + storage_commitment, + code_commitment.unwrap_or_default(), + ); + + Ok(Some((account_header, storage_header))) +} diff --git a/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql new file mode 100644 index 0000000000..e43199b80e --- /dev/null +++ b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql @@ -0,0 +1,9 @@ +-- Returns the columns making up the given account's header as of a block. +-- +-- The most recent row at or before the block holds the state in force then. +SELECT code_commitment, nonce, storage_header, vault_root +FROM accounts +WHERE account_id = ?1 + AND block_num <= ?2 +ORDER BY block_num DESC +LIMIT 1; diff --git a/crates/store/src/db/queries/select_full_account/mod.rs b/crates/store/src/db/queries/select_full_account/mod.rs new file mode 100644 index 0000000000..ec641dcab5 --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/mod.rs @@ -0,0 +1,59 @@ +//! Reconstructs a full account from the tables holding its latest committed state. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Felt; +use miden_protocol::account::{Account, AccountCode, AccountId}; +use miden_protocol::asset::{Asset, AssetVault}; + +use crate::db::queries::{VALID_FOREVER, select_latest_storage}; +use crate::errors::DatabaseError; + +const SQL_NONCE_AND_CODE: &str = include_str!("select_account_nonce_and_code.sql"); +const SQL_VAULT: &str = include_str!("select_account_vault.sql"); + +/// Reconstruct full Account from database tables for the latest account state +/// +/// This function queries the database tables to reconstruct a complete Account object: +/// - Code from `account_codes` table +/// - Nonce and storage header from `accounts` table +/// - Storage map entries from `account_storage_map_values` table +/// - Vault from `account_vault_assets` table +/// +/// # Note +/// +/// A stop-gap solution to retain store API and construct `AccountInfo` types. +/// The function should ultimately be removed, and any queries be served from the +/// `State` which contains an `SmtForest` to serve the latest and most recent +/// historical data. +// TODO: remove eventually once refactoring is complete +pub(crate) fn select_full_account( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + // Get account metadata (nonce, code_commitment) and code in a single join query + let (nonce, code) = tx + .query(SQL_NONCE_AND_CODE, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::>(0)?, row.get::(1)?)) + })? + .into_iter() + .next() + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let nonce = nonce.ok_or_else(|| { + DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) + })?; + + // Reconstruct storage using existing helper function + let storage = select_latest_storage(tx, account_id)?; + + // Reconstruct vault from account_vault_assets table; a NULL asset marks a removal. + let assets = tx + .query(SQL_VAULT, &[&account_id, &VALID_FOREVER], |row| row.get::>(0))? + .into_iter() + .flatten() + .collect::>(); + + let vault = AssetVault::new(&assets)?; + + Ok(Account::new(account_id, vault, storage, code, nonce, None)?) +} diff --git a/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql b/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql new file mode 100644 index 0000000000..958360ddd0 --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql @@ -0,0 +1,6 @@ +-- Returns the nonce and code of the given account's latest committed state. +SELECT accounts.nonce, account_codes.code +FROM accounts +INNER JOIN account_codes ON accounts.code_commitment = account_codes.code_commitment +WHERE accounts.account_id = ?1 + AND accounts.valid_until = ?2; diff --git a/crates/store/src/db/queries/select_full_account/select_account_vault.sql b/crates/store/src/db/queries/select_full_account/select_account_vault.sql new file mode 100644 index 0000000000..2b71aec9cb --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/select_account_vault.sql @@ -0,0 +1,7 @@ +-- Returns the assets currently held in the given account's vault. +-- +-- A NULL asset marks a removal, and is skipped by the caller. +SELECT asset +FROM account_vault_assets +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_latest_storage/mod.rs b/crates/store/src/db/queries/select_latest_storage/mod.rs new file mode 100644 index 0000000000..b4fe59b71f --- /dev/null +++ b/crates/store/src/db/queries/select_latest_storage/mod.rs @@ -0,0 +1,104 @@ +//! Reconstructs an account's current storage from the header and its map entries. + +use std::collections::{BTreeMap, HashMap}; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::account::{ + AccountId, + AccountStorage, + AccountStorageHeader, + StorageMap, + StorageMapKey, + StorageSlot, + StorageSlotName, + StorageSlotType, +}; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_STORAGE_HEADER: &str = include_str!("select_account_storage_header.sql"); +const SQL_MAP_ENTRIES: &str = include_str!("select_account_storage_map_entries.sql"); + +/// An account's storage header together with its map entries, keyed by slot. +pub(crate) type StorageHeaderWithEntries = + (AccountStorageHeader, HashMap>); + +/// Reconstructs the account's current storage: value slots come from the header, map slots from the +/// stored map entries. +pub(crate) fn select_latest_storage( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + let (storage_header, map_entries_by_slot) = select_latest_storage_components(tx, account_id)?; + + // Reconstruct StorageSlots from header slots + map entries + let slots = storage_header + .slots() + .map(|slot_header| { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = + map_entries_by_slot.get(slot_header.name()).cloned().unwrap_or_default(); + StorageSlot::with_map( + slot_header.name().clone(), + StorageMap::with_entries(entries)?, + ) + }, + }; + Ok(slot) + }) + .collect::, DatabaseError>>()?; + + Ok(AccountStorage::new(slots)?) +} + +/// Fetch account storage header and all storage maps +pub(crate) fn select_latest_storage_components( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + // The column is nullable, and the account may have no row at all. + let storage_blob = tx + .query(SQL_STORAGE_HEADER, &[&account_id, &VALID_FOREVER], |row| { + row.get::>(0) + })? + .into_iter() + .next() + .flatten(); + + let header = match storage_blob { + Some(header) => header, + None => AccountStorageHeader::new(Vec::new())?, + }; + + Ok((header, select_latest_storage_map_entries_all(tx, account_id)?)) +} + +// TODO this is expensive and should only be called from tests +fn select_latest_storage_map_entries_all( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result>, DatabaseError> { + let map_values = tx.query(SQL_MAP_ENTRIES, &[&account_id, &VALID_FOREVER], |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + })?; + + let mut map_entries_by_slot: HashMap> = + HashMap::new(); + for (slot_name, key, value) in map_values { + map_entries_by_slot.entry(slot_name).or_default().insert(key, value); + } + + Ok(map_entries_by_slot) +} diff --git a/crates/store/src/db/queries/select_latest_storage/select_account_storage_header.sql b/crates/store/src/db/queries/select_latest_storage/select_account_storage_header.sql new file mode 100644 index 0000000000..3e9cc3e9d9 --- /dev/null +++ b/crates/store/src/db/queries/select_latest_storage/select_account_storage_header.sql @@ -0,0 +1,5 @@ +-- Returns the storage header of the given account's latest committed state. +SELECT storage_header +FROM accounts +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_latest_storage/select_account_storage_map_entries.sql b/crates/store/src/db/queries/select_latest_storage/select_account_storage_map_entries.sql new file mode 100644 index 0000000000..5482d51cd9 --- /dev/null +++ b/crates/store/src/db/queries/select_latest_storage/select_account_storage_map_entries.sql @@ -0,0 +1,5 @@ +-- Returns every current storage map entry of the given account. +SELECT slot_name, key, value +FROM account_storage_map_values +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_vault_at_block/mod.rs b/crates/store/src/db/queries/select_vault_at_block/mod.rs new file mode 100644 index 0000000000..d7007bd7b7 --- /dev/null +++ b/crates/store/src/db/queries/select_vault_at_block/mod.rs @@ -0,0 +1,31 @@ +//! Returns the assets in an account's vault as of a block. + +use miden_node_db::sqlite::ReadTx; +use miden_node_proto::domain::account::AccountVaultDetails; +use miden_protocol::account::AccountId; +use miden_protocol::asset::Asset; +use miden_protocol::block::BlockNumber; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_vault_at_block.sql"); + +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. +/// +/// The read is bounded to [`AccountVaultDetails::MAX_RETURN_ENTRIES`] + 1 rows so an over-the-limit +/// vault can be detected without materializing the whole set. +pub(crate) fn select_vault_at_block( + tx: &ReadTx<'_>, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + let limit = + i64::try_from(AccountVaultDetails::MAX_RETURN_ENTRIES + 1).expect("should fit within i64"); + + // A NULL asset marks a removal, and is filtered out here. + Ok(tx + .query(SQL, &[&account_id, &block_num, &limit], |row| row.get::>(0))? + .into_iter() + .flatten() + .collect()) +} diff --git a/crates/store/src/db/queries/select_vault_at_block/select_vault_at_block.sql b/crates/store/src/db/queries/select_vault_at_block/select_vault_at_block.sql new file mode 100644 index 0000000000..146a52f95f --- /dev/null +++ b/crates/store/src/db/queries/select_vault_at_block/select_vault_at_block.sql @@ -0,0 +1,10 @@ +-- Returns the assets in the given account's vault as of a block. +-- +-- Selects, per vault key, the row whose validity interval covers the block; a NULL asset marks a +-- removal and is skipped by the caller. +SELECT asset +FROM account_vault_assets +WHERE account_id = ?1 + AND block_num <= ?2 + AND valid_until > ?2 +LIMIT ?3; diff --git a/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql b/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql new file mode 100644 index 0000000000..9026cf5159 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql @@ -0,0 +1,8 @@ +-- Closes an account's current row at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE accounts +SET valid_until = ?1 +WHERE account_id = ?2 + AND valid_until = ?3 diff --git a/crates/store/src/db/models/queries/accounts/delta.rs b/crates/store/src/db/queries/upsert_accounts/delta.rs similarity index 70% rename from crates/store/src/db/models/queries/accounts/delta.rs rename to crates/store/src/db/queries/upsert_accounts/delta.rs index 15e2956b98..79e3ea3427 100644 --- a/crates/store/src/db/models/queries/accounts/delta.rs +++ b/crates/store/src/db/queries/upsert_accounts/delta.rs @@ -10,8 +10,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; +use miden_node_db::sqlite::ReadTx; #[cfg(test)] use miden_protocol::EMPTY_WORD; use miden_protocol::account::{ @@ -28,60 +27,53 @@ use miden_protocol::account::{ #[cfg(test)] use miden_protocol::account::{StorageMap, StorageMapKey}; use miden_protocol::block::BlockNumber; -use miden_protocol::utils::serde::{Deserializable, Serializable}; use miden_protocol::{Felt, Word}; -use super::{NetworkAccountType, VALID_FOREVER}; -use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; -use crate::db::schema; +use crate::db::queries::{NetworkAccountType, VALID_FOREVER}; use crate::errors::DatabaseError; #[cfg(test)] mod tests; +const SQL_LATEST_ACCOUNT_STATE: &str = include_str!("select_latest_account_state.sql"); + // TYPES // ================================================================================================ /// Latest account row fields needed by account update preparation. -#[derive(diesel::prelude::Queryable)] pub(super) struct LatestAccountStateRow { - created_at_block: i64, - network_account_type: i32, - nonce: Option, - code_commitment: Option>, - storage_header: Option>, + created_at_block: BlockNumber, + network_account_type: NetworkAccountType, + nonce: Option, + code_commitment: Option, + storage_header: Option, } impl LatestAccountStateRow { - pub(super) fn created_at_block(&self) -> Result { - Ok(BlockNumber::from_raw_sql(self.created_at_block)?) + pub(super) fn created_at_block(&self) -> BlockNumber { + self.created_at_block } - pub(super) fn network_account_type(&self) -> Result { - Ok(NetworkAccountType::from_raw_sql(self.network_account_type)?) + pub(super) fn network_account_type(&self) -> NetworkAccountType { + self.network_account_type } pub(super) fn state_headers( &self, account_id: AccountId, ) -> Result { - let nonce = raw_sql_to_nonce(self.nonce.ok_or_else(|| { + let nonce = self.nonce.ok_or_else(|| { DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) - })?); - - let code_commitment = self - .code_commitment - .as_deref() - .map(Word::read_from_bytes) - .transpose()? - .ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "No code_commitment found for account {account_id}" - )) - })?; - - let storage_header = match self.storage_header.as_deref() { - Some(bytes) => AccountStorageHeader::read_from_bytes(bytes)?, + })?; + + let code_commitment = self.code_commitment.ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "No code_commitment found for account {account_id}" + )) + })?; + + let storage_header = match self.storage_header.clone() { + Some(header) => header, None => AccountStorageHeader::new(Vec::new())?, }; @@ -135,40 +127,22 @@ pub(super) enum AccountStateForInsert { // ================================================================================================ /// Selects the latest account state needed to prepare any account update. -/// -/// The query fetches: -/// - `created_at_block` and `network_account_type` for every update -/// - `nonce` (preserved when a partial patch omits its final nonce) -/// - `code_commitment` (unchanged in partial deltas) -/// - `storage_header` (to apply storage delta) -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT created_at_block, network_account_type, nonce, code_commitment, storage_header -/// FROM accounts -/// WHERE account_id = ?1 AND valid_until = {VALID_FOREVER} -/// ``` pub(super) fn select_latest_account_state( - conn: &mut SqliteConnection, + tx: &ReadTx<'_>, account_id: AccountId, ) -> Result, DatabaseError> { - let row = SelectDsl::select( - schema::accounts::table, - ( - schema::accounts::created_at_block, - schema::accounts::network_account_type, - schema::accounts::nonce, - schema::accounts::code_commitment, - schema::accounts::storage_header, - ), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .get_result(conn) - .optional()?; - - Ok(row) + Ok(tx + .query(SQL_LATEST_ACCOUNT_STATE, &[&account_id, &VALID_FOREVER], |row| { + Ok(LatestAccountStateRow { + created_at_block: row.get::(0)?, + network_account_type: row.get::(1)?, + nonce: row.get::>(2)?, + code_commitment: row.get::>(3)?, + storage_header: row.get::>(4)?, + }) + })? + .into_iter() + .next()) } // HELPER FUNCTIONS @@ -226,31 +200,7 @@ pub(super) fn apply_storage_patch( map_updates.insert(slot_name, storage_map.root()); } - let mut slots = - Vec::from_iter(header.slots().filter(|slot| !removed.contains(slot.name())).map(|slot| { - let slot_name = slot.name(); - if let Some(new_value) = value_updates.remove(slot_name) { - StorageSlotHeader::new(slot_name.clone(), slot.slot_type(), new_value) - } else if let Some(new_root) = map_updates.remove(slot_name) { - StorageSlotHeader::new(slot_name.clone(), slot.slot_type(), new_root) - } else { - slot.clone() - } - })); - - // Any updates left over belong to slots created by the patch. - for (slot_name, value) in value_updates { - slots.push(StorageSlotHeader::new(slot_name.clone(), StorageSlotType::Value, value)); - } - for (slot_name, root) in map_updates { - slots.push(StorageSlotHeader::new(slot_name.clone(), StorageSlotType::Map, root)); - } - - slots.sort_by_key(StorageSlotHeader::id); - - AccountStorageHeader::new(slots).map_err(|e| { - DatabaseError::DataCorrupted(format!("Failed to create storage header: {e:?}")) - }) + build_patched_header(header, value_updates, map_updates, &removed) } /// Applies a storage patch to an existing storage header using precomputed map roots. @@ -297,6 +247,16 @@ pub(super) fn apply_storage_patch_with_roots( map_updates.insert(slot_name, root); } + build_patched_header(header, value_updates, map_updates, &removed) +} + +/// Rebuilds a storage header from the patch's value updates, map roots, and removals. +fn build_patched_header( + header: &AccountStorageHeader, + mut value_updates: HashMap<&StorageSlotName, Word>, + mut map_updates: HashMap<&StorageSlotName, Word>, + removed: &HashSet<&StorageSlotName>, +) -> Result { let mut slots = Vec::from_iter(header.slots().filter(|slot| !removed.contains(slot.name())).map(|slot| { let slot_name = slot.name(); diff --git a/crates/store/src/db/models/queries/accounts/delta/tests.rs b/crates/store/src/db/queries/upsert_accounts/delta/tests.rs similarity index 86% rename from crates/store/src/db/models/queries/accounts/delta/tests.rs rename to crates/store/src/db/queries/upsert_accounts/delta/tests.rs index 8eb735fa21..93d36e0fd0 100644 --- a/crates/store/src/db/models/queries/accounts/delta/tests.rs +++ b/crates/store/src/db/queries/upsert_accounts/delta/tests.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use assert_matches::assert_matches; -use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -12,9 +11,11 @@ use miden_protocol::account::{ Account, AccountBuilder, AccountComponent, + AccountHeader, AccountId, AccountIdVersion, AccountPatch, + AccountStorageHeader, AccountStoragePatch, AccountType, AccountUpdateDetails, @@ -29,36 +30,73 @@ use miden_protocol::account::{ StorageValuePatch, }; use miden_protocol::asset::{Asset, FungibleAsset}; -use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber, ValidatorKeys}; +use miden_protocol::block::{ + BlockAccountUpdate, + BlockHeader, + BlockNumber, + BlockSignatures, + ValidatorKeys, +}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::testing::account_id::{ ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1, }; -use miden_protocol::utils::serde::Serializable; use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::{Approver, AuthSingleSig}; use miden_standards::code_builder::CodeBuilder; -use crate::db::models::queries::accounts::{ +use crate::db::queries::{ + self, PrecomputedPublicAccountState, PrecomputedPublicAccountStates, VALID_FOREVER, - select_account_header_with_storage_header_at_block, - select_account_vault_at_block, - select_full_account, - upsert_accounts, }; -use crate::db::schema::accounts; +use crate::db::{Result, TestDb}; use crate::errors::DatabaseError; -fn setup_test_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) } -fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use crate::db::schema::block_headers; +fn select_full_account(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_full_account(tx, account_id)) +} +fn select_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_vault_at_block(tx, account_id, block_num)) +} + +fn select_account_header_with_storage_header_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| { + queries::select_account_header_with_storage_header_at_block(tx, account_id, block_num) + }) +} + +fn insert_block_header(db: &TestDb, block_num: BlockNumber) { let secret_key = SigningKey::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -74,17 +112,30 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { test_fee_params(), 0_u8.into(), ); - let signature = secret_key.sign(block_header.commitment()); - - diesel::insert_into(block_headers::table) - .values(( - block_headers::block_num.eq(i64::from(block_num.as_u32())), - block_headers::block_header.eq(block_header.to_bytes()), - block_headers::signature.eq(signature.to_bytes()), - block_headers::commitment.eq(block_header.commitment().to_bytes()), - )) - .execute(conn) - .expect("Failed to insert block header"); + let signatures = + BlockSignatures::new(vec![secret_key.sign(block_header.commitment())]).unwrap(); + + db.write::<_, DatabaseError, _>(move |tx| { + queries::insert_block_header(tx, &block_header, &signatures) + }) + .expect("Failed to insert block header"); +} + +/// Returns the current `accounts` row's commitment, nonce, and code commitment. +fn latest_account_row(db: &TestDb, account_id: AccountId) -> (Word, Option, Option) { + const SQL: &str = "SELECT account_commitment, nonce, code_commitment \ + FROM accounts WHERE account_id = ?1 AND valid_until = ?2"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::(0)?, row.get::>(1)?, row.get::>(2)?)) + })? + .into_iter() + .next()) + }) + .expect("query should succeed") + .expect("Account should exist in DB") } fn precomputed_state_from_account(account: &Account) -> PrecomputedPublicAccountState { @@ -144,10 +195,10 @@ fn callback_delta_test_account(seed: [u8; 32], slot_index: usize) -> Account { .unwrap() } -fn insert_public_account(conn: &mut SqliteConnection, block_num: BlockNumber, account: &Account) { +fn insert_public_account(db: &TestDb, block_num: BlockNumber, account: &Account) { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - conn, + db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -160,14 +211,14 @@ fn insert_public_account(conn: &mut SqliteConnection, block_num: BlockNumber, ac } fn apply_callback_delta( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, faucet_id: AccountId, block: BlockNumber, amount: u64, nonce_delta: u64, ) -> Account { - let prev = select_full_account(conn, account_id).expect("load account"); + let prev = select_full_account(db, account_id).expect("load account"); let callback_template = FungibleAsset::new(faucet_id, amount).unwrap(); let prev_amount = match prev.vault().get(callback_template.id()) { Some(Asset::Fungible(f)) => f.amount().as_u64(), @@ -193,7 +244,7 @@ fn apply_callback_delta( let precomputed_public_states = precomputed_states_from_account(&expected); upsert_accounts( - conn, + db, &[BlockAccountUpdate::new( account_id, expected.to_commitment(), @@ -204,7 +255,7 @@ fn apply_callback_delta( ) .expect("partial delta upsert failed"); - let after = select_full_account(conn, account_id).expect("load account after"); + let after = select_full_account(db, account_id).expect("load account after"); assert_eq!(after.vault().root(), expected.vault().root(), "vault root mismatch"); assert_eq!(after.to_commitment(), expected.to_commitment(), "commitment mismatch"); after @@ -241,7 +292,7 @@ fn optimized_delta_matches_full_account_method() { const NONCE_DELTA: u64 = 5; const VAULT_AMOUNT: u64 = 500; - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create an account with value slots only (no map slots to avoid SmtForest complexity) let slot_value_initial = Word::from([ @@ -279,8 +330,8 @@ fn optimized_delta_matches_full_account_method() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); // Insert the initial account at block 1 (full state) - no vault assets let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); @@ -290,7 +341,7 @@ fn optimized_delta_matches_full_account_method() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -299,7 +350,7 @@ fn optimized_delta_matches_full_account_method() { // Verify initial state let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); assert_eq!(full_account_before.nonce(), account.nonce()); assert!( full_account_before.vault().assets().next().is_none(), @@ -371,13 +422,13 @@ fn optimized_delta_matches_full_account_method() { final_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); // ----- VERIFY: Query the DB and check that optimized path produced correct results ----- let (header_after, storage_header_after) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_2) + select_account_header_with_storage_header_at_block(&db, account.id(), block_2) .expect("Query should succeed") .expect("Account should exist"); @@ -405,8 +456,8 @@ fn optimized_delta_matches_full_account_method() { ); // Verify vault assets - let vault_assets_after = select_account_vault_at_block(&mut conn, account.id(), block_2) - .expect("Query vault should succeed"); + let vault_assets_after = + select_vault_at_block(&db, account.id(), block_2).expect("Query vault should succeed"); assert_eq!(vault_assets_after.len(), 1, "Should have 1 vault asset"); assert_matches!(&vault_assets_after[0], Asset::Fungible(f) => { @@ -422,8 +473,8 @@ fn optimized_delta_matches_full_account_method() { ); // Also verify we can load the full account and it has correct state - let full_account_after = select_full_account(&mut conn, account.id()) - .expect("Failed to load full account after update"); + let full_account_after = + select_full_account(&db, account.id()).expect("Failed to load full account after update"); assert_eq!(full_account_after.nonce(), expected_nonce, "Full account nonce mismatch"); assert_eq!( @@ -454,7 +505,7 @@ fn optimized_delta_updates_non_empty_vault() { const ADDED_AMOUNT_BLOCK_3: u64 = 150; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let faucet_id_1 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1).unwrap(); @@ -488,9 +539,9 @@ fn optimized_delta_updates_non_empty_vault() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); let block_3 = BlockNumber::from(BLOCK_NUM_3); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Block 1: insert full-state patch (initial account with 700 tokens of faucet_id) let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); @@ -500,7 +551,7 @@ fn optimized_delta_updates_non_empty_vault() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -508,7 +559,7 @@ fn optimized_delta_updates_non_empty_vault() { .expect("Initial upsert failed"); let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); // Block 2: partial patch — remove faucet_id (700), add faucet_id_1 (250) let removed_asset = Asset::Fungible(FungibleAsset::new(faucet_id, INITIAL_AMOUNT).unwrap()); @@ -540,11 +591,11 @@ fn optimized_delta_updates_non_empty_vault() { expected_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); - let vault_assets_after = select_account_vault_at_block(&mut conn, account.id(), block_2) - .expect("Query vault should succeed"); + let vault_assets_after = + select_vault_at_block(&db, account.id(), block_2).expect("Query vault should succeed"); assert_eq!(vault_assets_after.len(), 1, "Should have 1 vault asset"); assert_matches!(&vault_assets_after[0], Asset::Fungible(f) => { @@ -552,8 +603,8 @@ fn optimized_delta_updates_non_empty_vault() { assert_eq!(f.amount().as_u64(), ADDED_AMOUNT_BLOCK_2, "Amount should match"); }); - let full_account_after = select_full_account(&mut conn, account.id()) - .expect("Failed to load full account after update"); + let full_account_after = + select_full_account(&db, account.id()).expect("Failed to load full account after update"); assert_eq!(full_account_after.vault().root(), expected_vault_root); assert_eq!(full_account_after.to_commitment(), expected_commitment); @@ -586,11 +637,11 @@ fn optimized_delta_updates_non_empty_vault() { commitment_3, AccountUpdateDetails::Public(partial_patch_3), ); - upsert_accounts(&mut conn, &[account_update_3], block_3, &precomputed_public_states_3) + upsert_accounts(&db, &[account_update_3], block_3, &precomputed_public_states_3) .expect("Block 3 upsert failed"); let full_account_final = - select_full_account(&mut conn, account.id()).expect("Failed to load after block 3"); + select_full_account(&db, account.id()).expect("Failed to load after block 3"); let final_assets: Vec = full_account_final.vault().assets().collect(); assert_eq!(final_assets.len(), 1, "Should have exactly 1 vault asset"); @@ -614,29 +665,22 @@ fn optimized_delta_updates_preserve_callback_flag() { const ADDED_AMOUNT_BLOCK_3: u64 = 150; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); let block_3 = BlockNumber::from(3u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); let faucet_id = callback_enabled_faucet_id(); let account = callback_delta_test_account(ACCOUNT_SEED, SLOT_INDEX); - insert_public_account(&mut conn, block_1, &account); + insert_public_account(&db, block_1, &account); - apply_callback_delta( - &mut conn, - account.id(), - faucet_id, - block_2, - ADDED_AMOUNT_BLOCK_2, - NONCE_DELTA, - ); + apply_callback_delta(&db, account.id(), faucet_id, block_2, ADDED_AMOUNT_BLOCK_2, NONCE_DELTA); let final_account = apply_callback_delta( - &mut conn, + &db, account.id(), faucet_id, block_3, @@ -674,7 +718,7 @@ fn optimized_delta_updates_storage_map_header() { // Use nonzero nonce delta (required when storage/vault changes). const NONCE_DELTA: u64 = 1; - let mut conn = setup_test_db(); + let db = TestDb::new(); let map_key = StorageMapKey::new(Word::from([ Felt::new_unchecked(MAP_KEY_VALUES[0]), @@ -722,8 +766,8 @@ fn optimized_delta_updates_storage_map_header() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); let account_update_initial = BlockAccountUpdate::new( @@ -732,7 +776,7 @@ fn optimized_delta_updates_storage_map_header() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -740,7 +784,7 @@ fn optimized_delta_updates_storage_map_header() { .expect("Initial upsert failed"); let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); let map_patch = StorageMapPatch::from_iters([], [(map_key, map_value_updated)]); let storage_patch = AccountStoragePatch::from_raw(BTreeMap::from_iter([( @@ -771,11 +815,11 @@ fn optimized_delta_updates_storage_map_header() { expected_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); let (header_after, storage_header_after) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_2) + select_account_header_with_storage_header_at_block(&db, account.id(), block_2) .expect("Query should succeed") .expect("Account should exist"); @@ -828,11 +872,11 @@ fn partial_public_upsert_requires_precomputed_state() { const ACCOUNT_SEED: [u8; 32] = [80u8; 32]; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let component_storage = vec![StorageSlot::with_value(StorageSlotName::mock(SLOT_INDEX), EMPTY_WORD)]; @@ -860,7 +904,7 @@ fn partial_public_upsert_requires_precomputed_state() { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -871,7 +915,7 @@ fn partial_public_upsert_requires_precomputed_state() { ) .expect("initial full-state upsert failed"); - let mut current_account = select_full_account(&mut conn, account.id()).unwrap(); + let mut current_account = select_full_account(&db, account.id()).unwrap(); let patch = AccountPatch::new( account.id(), AccountStoragePatch::new(), @@ -883,7 +927,7 @@ fn partial_public_upsert_requires_precomputed_state() { current_account.apply_patch(&patch).unwrap(); let err = upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), current_account.to_commitment(), @@ -902,11 +946,11 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { const ACCOUNT_SEED: [u8; 32] = [81u8; 32]; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let component_storage = vec![StorageSlot::with_value(StorageSlotName::mock(SLOT_INDEX), EMPTY_WORD)]; @@ -931,7 +975,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -942,7 +986,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { ) .expect("initial full-state upsert failed"); - let mut expected_account = select_full_account(&mut conn, account.id()).unwrap(); + let mut expected_account = select_full_account(&db, account.id()).unwrap(); let patch = AccountPatch::new( account.id(), AccountStoragePatch::new(), @@ -958,7 +1002,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { Word::from([Felt::new_unchecked(999); 4]); let err = upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), expected_account.to_commitment(), @@ -986,10 +1030,10 @@ fn upsert_private_account() { // Use fixed commitment values to validate storage behavior. const COMMITMENT_WORDS: [u64; 4] = [1, 2, 3, 4]; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(BLOCK_NUM); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Create a private account ID let account_id = AccountId::dummy( @@ -1010,29 +1054,14 @@ fn upsert_private_account() { let account_update = BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Private); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Private account upsert failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Private account upsert failed"); // Verify the account exists and commitment matches - let (stored_commitment, stored_nonce, stored_code): (Vec, Option, Option>) = - accounts::table - .filter(accounts::account_id.eq(account_id.to_bytes())) - .filter(accounts::valid_until.eq(VALID_FOREVER)) - .select((accounts::account_commitment, accounts::nonce, accounts::code_commitment)) - .first(&mut conn) - .expect("Account should exist in DB"); + let (stored_commitment, stored_nonce, stored_code) = latest_account_row(&db, account_id); - assert_eq!( - stored_commitment, - account_commitment.to_bytes(), - "Stored commitment should match" - ); + assert_eq!(stored_commitment, account_commitment, "Stored commitment should match"); // Private accounts have NULL for nonce, code_commitment, storage_header, vault_root assert!(stored_nonce.is_none(), "Private account should have NULL nonce"); @@ -1053,10 +1082,10 @@ fn upsert_full_state_delta() { // Use explicit slot index to avoid magic numbers. const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(BLOCK_NUM); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Create an account with storage let slot_value = Word::from([ @@ -1099,17 +1128,12 @@ fn upsert_full_state_delta() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &precomputed_states_from_account(&account), - ) - .expect("Full-state delta upsert failed"); + upsert_accounts(&db, &[account_update], block_num, &precomputed_states_from_account(&account)) + .expect("Full-state delta upsert failed"); // Verify the account state was stored correctly let (header, storage_header) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_num) + select_account_header_with_storage_header_at_block(&db, account.id(), block_num) .expect("Query should succeed") .expect("Account should exist"); @@ -1126,8 +1150,7 @@ fn upsert_full_state_delta() { ); // Verify we can load the full account back - let loaded_account = - select_full_account(&mut conn, account.id()).expect("Should load full account"); + let loaded_account = select_full_account(&db, account.id()).expect("Should load full account"); assert_eq!(loaded_account.nonce(), account.nonce()); assert_eq!(loaded_account.code().commitment(), account.code().commitment()); diff --git a/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql b/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql new file mode 100644 index 0000000000..6e4758dca8 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql @@ -0,0 +1,5 @@ +-- Stores an account's code, keyed by its commitment. Code is shared across accounts and across an +-- account's versions, so a commitment already present is left untouched. +INSERT INTO account_codes (code_commitment, code) +VALUES (?1, ?2) +ON CONFLICT(code_commitment) DO NOTHING diff --git a/crates/store/src/db/queries/upsert_accounts/mod.rs b/crates/store/src/db/queries/upsert_accounts/mod.rs new file mode 100644 index 0000000000..672c06d17c --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/mod.rs @@ -0,0 +1,539 @@ +//! Writes the account state produced by a block. +//! +//! Every account table is versioned: a row is applicable for blocks in `[block_num, valid_until)`, +//! and writing a new version closes the previous one. This module owns that bookkeeping for the +//! `accounts` row itself and drives the per-key writes in +//! [`insert_vault_asset`](super::insert_vault_asset) and +//! [`insert_storage_map_value`](super::insert_storage_map_value). + +use std::collections::BTreeMap; + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::account::{ + Account, + AccountCode, + AccountHeader, + AccountId, + AccountPatch, + AccountStorageHeader, + AccountUpdateDetails, + StorageMapKey, + StorageMapPatchEntries, + StorageSlotContent, + StorageSlotName, +}; +use miden_protocol::asset::{Asset, AssetId}; +use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; +use miden_protocol::{Felt, Word}; +use miden_standards::account::auth::NetworkAccount; + +use crate::COMPONENT; +use crate::db::queries::insert_storage_map_value::insert_storage_map_value_inner; +use crate::db::queries::{ + NetworkAccountType, + VALID_FOREVER, + insert_storage_map_value, + insert_vault_asset, +}; +use crate::errors::DatabaseError; + +mod delta; +use delta::{ + AccountStateForInsert, + LatestAccountStateRow, + PartialAccountState, + PrecomputedFullAccountState, + apply_storage_patch_with_roots, + select_latest_account_state, +}; + +#[cfg(test)] +mod tests; + +const SQL_INSERT_ACCOUNT_CODE: &str = include_str!("insert_account_code.sql"); +const SQL_CLOSE_ACCOUNT_VALIDITY: &str = include_str!("close_account_validity.sql"); +const SQL_UPSERT_ACCOUNT: &str = include_str!("upsert_account.sql"); + +// PRECOMPUTED PUBLIC ACCOUNT STATE +// ================================================================================================ + +/// Public account state commitments computed by the account state forest before SQLite writes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrecomputedPublicAccountState { + pub vault_root: Word, + pub storage_map_roots: BTreeMap, +} + +pub type PrecomputedPublicAccountStates = BTreeMap; + +// QUERY +// ================================================================================================ + +type PendingStorageInserts = Vec<(AccountId, StorageSlotName, StorageMapKey, Word)>; +type PendingAssetInserts = Vec<(AccountId, AssetId, Option)>; + +/// Writes the state of every account a block updated. +/// +/// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn upsert_accounts( + tx: &WriteTx<'_>, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let mut count = 0; + for update in accounts { + upsert_account(tx, update, block_num, precomputed_public_states)?; + count += 1; + } + + Ok(count) +} + +/// Writes a single account's new state, closing its previous version's validity interval. +fn upsert_account( + tx: &WriteTx<'_>, + update: &BlockAccountUpdate, + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result<(), DatabaseError> { + let account_id = update.account_id(); + + // Pull the latest row once. Partial updates consume the state headers below, while every update + // carries forward creation metadata. + let existing = select_latest_account_state(tx, account_id)?; + let account_is_new = existing.is_none(); + + let created_at_block = + existing.as_ref().map_or(block_num, LatestAccountStateRow::created_at_block); + + // NOTE: we collect storage / asset inserts to apply them only after the account row is written. + // The storage and vault tables have FKs pointing to accounts `(account_id, block_num)`, so + // inserting them earlier would violate those constraints when inserting a brand-new account. + let (account_state, pending_storage_inserts, pending_asset_inserts) = + prepare_account_update(update, block_num, precomputed_public_states, existing.as_ref())?; + + // Inherit the classification when the account already exists; otherwise classify it once at + // creation based on the new state. + let network_account_type = match &existing { + Some(row) => row.network_account_type(), + None => match &account_state { + AccountStateForInsert::FullAccount(account) + if NetworkAccount::new(account.clone()).is_ok() => + { + NetworkAccountType::Network + }, + AccountStateForInsert::PrecomputedFullState(state) if state.is_network_account => { + NetworkAccountType::Network + }, + _ => NetworkAccountType::None, + }, + }; + + // Insert account _code_ for full accounts (new account creation). + match &account_state { + AccountStateForInsert::FullAccount(account) => insert_account_code(tx, account.code())?, + AccountStateForInsert::PrecomputedFullState(state) => insert_account_code(tx, &state.code)?, + AccountStateForInsert::Private | AccountStateForInsert::PartialState(_) => {}, + } + + // Close the previous row's validity interval and insert the NEW account row. + tx.execute(SQL_CLOSE_ACCOUNT_VALIDITY, &[&block_num, &account_id, &VALID_FOREVER])?; + + let row = AccountRow::new( + account_id, + network_account_type, + update.final_state_commitment(), + block_num, + created_at_block, + &account_state, + ); + row.upsert(tx)?; + + // Insert pending storage map entries. TODO consider batching + for (acc_id, slot_name, key, value) in pending_storage_inserts { + if account_is_new { + // A brand-new account cannot have a previous open row to invalidate. + insert_storage_map_value_inner(tx, acc_id, block_num, &slot_name, key, value, false)?; + } else { + insert_storage_map_value(tx, acc_id, block_num, &slot_name, key, value)?; + } + } + + for (acc_id, vault_key, asset) in pending_asset_inserts { + insert_vault_asset(tx, acc_id, block_num, vault_key, asset)?; + } + + Ok(()) +} + +/// Stores an account's code, keyed by its commitment; a commitment already present is left as is. +fn insert_account_code(tx: &WriteTx<'_>, code: &AccountCode) -> Result<(), DatabaseError> { + tx.execute(SQL_INSERT_ACCOUNT_CODE, &[&code.commitment(), code])?; + Ok(()) +} + +// UPDATE PREPARATION +// ================================================================================================ + +/// Turns a block's account update into the row state to write, plus the storage-map and vault +/// writes that follow it. +fn prepare_account_update( + update: &BlockAccountUpdate, + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, + existing: Option<&LatestAccountStateRow>, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = update.account_id(); + + match update.details() { + AccountUpdateDetails::Private => Ok((AccountStateForInsert::Private, vec![], vec![])), + + // New account is always a full account, but also comes as an update + AccountUpdateDetails::Public(patch) if patch.is_full_state() => { + if block_num == BlockNumber::GENESIS { + let account = Account::try_from(patch) + .expect("Patch to full account always works for full state patches"); + debug_assert_eq!(account_id, account.id()); + prepare_full_account_update(update, account) + } else { + let precomputed = precomputed_state(precomputed_public_states, account_id)?; + prepare_precomputed_full_account_update(update, patch, precomputed) + } + }, + + // Update of an existing account + AccountUpdateDetails::Public(patch) => { + let precomputed = precomputed_state(precomputed_public_states, account_id)?; + let existing = existing.ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + prepare_partial_account_update(update, account_id, patch, precomputed, existing) + }, + } +} + +/// Looks up the forest-computed state for a public account, which every non-genesis public update +/// requires. +fn precomputed_state( + precomputed_public_states: &PrecomputedPublicAccountStates, + account_id: AccountId, +) -> Result<&PrecomputedPublicAccountState, DatabaseError> { + precomputed_public_states.get(&account_id).ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "missing precomputed public account state for account {account_id}" + )) + }) +} + +fn prepare_full_account_update( + update: &BlockAccountUpdate, + account: Account, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = account.id(); + + // sanity check the commitment of account matches the final state commitment + if account.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + // collect storage-map inserts to apply after account upsert + let mut storage = Vec::new(); + for slot in account.storage().slots() { + if let StorageSlotContent::Map(storage_map) = slot.content() { + for (key, value) in storage_map.entries() { + storage.push((account_id, slot.name().clone(), *key, *value)); + } + } + } + + // collect vault-asset inserts to apply after account upsert + let mut assets = Vec::new(); + for asset in account.vault().assets() { + // Only insert assets with non-zero values for fungible assets + let should_insert = match asset { + Asset::Fungible(fungible) => fungible.amount().as_u64() > 0, + Asset::NonFungible(_) => true, + }; + if should_insert { + assets.push((account_id, asset.id(), Some(asset))); + } + } + + Ok((AccountStateForInsert::FullAccount(account), storage, assets)) +} + +/// Prepares a full public-account insertion using roots computed by the account-state forest. +/// +/// This avoids reconstructing the account's vault and storage maps in SQLite. The returned state +/// contains the account-row fields, while storage-map entries and vault assets are returned +/// separately for insertion after the account row has satisfied their foreign-key dependency. +/// Empty-word map entries and assets are omitted from the pending inserts. +/// +/// # Errors +/// +/// Returns an error if the full-state patch is missing its code or nonce, a required precomputed +/// storage root is absent, an asset is invalid, or the reconstructed account header does not match +/// the update's final state commitment. +fn prepare_precomputed_full_account_update( + update: &BlockAccountUpdate, + patch: &AccountPatch, + precomputed: &PrecomputedPublicAccountState, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = patch.id(); + let code = patch.code().cloned().ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "full-state patch for account {account_id} is missing account code" + )) + })?; + let nonce = patch.final_nonce().ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "full-state patch for account {account_id} is missing final nonce" + )) + })?; + + let storage_header = apply_storage_patch_with_roots( + &AccountStorageHeader::new(Vec::new())?, + patch.storage(), + &precomputed.storage_map_roots, + )?; + let account_header = AccountHeader::new( + account_id, + nonce, + precomputed.vault_root, + storage_header.to_commitment(), + code.commitment(), + ); + if account_header.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account_header.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + let storage = patch + .storage() + .maps() + .flat_map(|(slot_name, map_patch)| { + map_patch.entries().into_iter().flat_map(move |entries| { + entries + .as_map() + .iter() + .filter(|(_key, value)| **value != Word::empty()) + .map(move |(key, value)| (account_id, slot_name.clone(), *key, *value)) + }) + }) + .collect(); + let assets = patch + .vault() + .iter() + .filter(|(_asset_id, value)| **value != Word::empty()) + .map(|(asset_id, value)| { + Asset::from_id_and_value(*asset_id, *value) + .map(|asset| (account_id, *asset_id, Some(asset))) + }) + .collect::, _>>()?; + + // The patch carries full state, so it can be turned back into an account and classified with + // the canonical check. + let is_network_account = NetworkAccount::new(Account::try_from(patch)?).is_ok(); + let state = PrecomputedFullAccountState { + nonce, + code, + storage_header, + vault_root: precomputed.vault_root, + is_network_account, + }; + + Ok((AccountStateForInsert::PrecomputedFullState(state), storage, assets)) +} + +/// Prepares a partial public-account update using the latest row and precomputed forest roots. +/// +/// Unchanged header fields are carried forward from `existing`. The returned partial state is used +/// for the next account row, while storage-map values and vault asset updates are returned +/// separately for insertion after that row. Empty vault values are represented as removals. +/// +/// # Errors +/// +/// Returns an error if the existing row is invalid, a required precomputed storage root is absent, +/// a patched asset is invalid, or the reconstructed account header does not match the update's +/// final state commitment. +fn prepare_partial_account_update( + update: &BlockAccountUpdate, + account_id: AccountId, + patch: &AccountPatch, + precomputed: &PrecomputedPublicAccountState, + existing: &LatestAccountStateRow, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + // Build the minimal account state needed for partial patch application from the latest row that + // was loaded with the account's creation metadata. + let state_headers = existing.state_headers(account_id)?; + + // --- Process asset updates. --------------------------------- The patch carries absolute final + // values, so encode `Some` as update and `None` (an empty value word) as removal. + let mut assets = Vec::new(); + for (vault_key, value) in patch.vault().iter() { + let update_or_remove = if *value == Word::empty() { + None + } else { + Some(Asset::from_id_and_value(*vault_key, *value)?) + }; + assets.push((account_id, *vault_key, update_or_remove)); + } + + // --- Collect storage map updates. --------------------------- + + let mut storage = Vec::new(); + for (slot_name, map_patch) in patch.storage().maps() { + for (key, value) in map_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) + { + storage.push((account_id, slot_name.clone(), *key, *value)); + } + } + + // Apply the patch storage to the given storage header. + let new_storage_header = apply_storage_patch_with_roots( + &state_headers.storage_header, + patch.storage(), + &precomputed.storage_map_roots, + )?; + + let new_vault_root = precomputed.vault_root; + + // --- Compute updated account state for the accounts row. --- Use the absolute final nonce. + let new_nonce = patch.final_nonce().unwrap_or(state_headers.nonce); + + // Create minimal account state data for the row insert. + let account_state = PartialAccountState { + nonce: new_nonce, + code_commitment: state_headers.code_commitment, + storage_header: new_storage_header, + vault_root: new_vault_root, + }; + + let account_header = AccountHeader::new( + account_id, + account_state.nonce, + account_state.vault_root, + account_state.storage_header.to_commitment(), + account_state.code_commitment, + ); + + if account_header.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account_header.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + Ok((AccountStateForInsert::PartialState(account_state), storage, assets)) +} + +// ACCOUNT ROW +// ================================================================================================ + +/// The `accounts` row written for an account's new state. +/// +/// Private accounts carry no public state, so every optional column is `None` for them. +pub(crate) struct AccountRow { + account_id: AccountId, + network_account_type: NetworkAccountType, + block_num: BlockNumber, + account_commitment: Word, + code_commitment: Option, + nonce: Option, + storage_header: Option, + vault_root: Option, + created_at_block: BlockNumber, +} + +impl AccountRow { + /// Builds the row for the given prepared account state. + fn new( + account_id: AccountId, + network_account_type: NetworkAccountType, + account_commitment: Word, + block_num: BlockNumber, + created_at_block: BlockNumber, + state: &AccountStateForInsert, + ) -> Self { + let mut row = Self::new_private( + account_id, + network_account_type, + account_commitment, + block_num, + created_at_block, + ); + + match state { + AccountStateForInsert::Private => {}, + AccountStateForInsert::FullAccount(account) => { + row.code_commitment = Some(account.code().commitment()); + row.nonce = Some(account.nonce()); + row.storage_header = Some(account.storage().to_header()); + row.vault_root = Some(account.vault().root()); + }, + AccountStateForInsert::PrecomputedFullState(state) => { + row.code_commitment = Some(state.code.commitment()); + row.nonce = Some(state.nonce); + row.storage_header = Some(state.storage_header.clone()); + row.vault_root = Some(state.vault_root); + }, + AccountStateForInsert::PartialState(state) => { + row.code_commitment = Some(state.code_commitment); + row.nonce = Some(state.nonce); + row.storage_header = Some(state.storage_header.clone()); + row.vault_root = Some(state.vault_root); + }, + } + + row + } + + /// Builds the row for a private account, which has no public state. + pub(crate) fn new_private( + account_id: AccountId, + network_account_type: NetworkAccountType, + account_commitment: Word, + block_num: BlockNumber, + created_at_block: BlockNumber, + ) -> Self { + Self { + account_id, + network_account_type, + block_num, + account_commitment, + code_commitment: None, + nonce: None, + storage_header: None, + vault_root: None, + created_at_block, + } + } + + /// Writes the row as the account's current, open-ended version. + pub(crate) fn upsert(&self, tx: &WriteTx<'_>) -> Result { + Ok(tx.execute( + SQL_UPSERT_ACCOUNT, + &[ + &self.account_id, + &self.network_account_type, + &self.block_num, + &self.account_commitment, + &self.code_commitment, + &self.nonce, + &self.storage_header, + &self.vault_root, + &self.created_at_block, + &VALID_FOREVER, + ], + )?) + } +} diff --git a/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql b/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql new file mode 100644 index 0000000000..f15e012e63 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql @@ -0,0 +1,10 @@ +-- Returns the fields of an account's current row that preparing its next update depends on: +-- +-- * `created_at_block` and `network_account_type` are carried forward by every update +-- * `nonce` is preserved when a partial patch omits its final nonce +-- * `code_commitment` is unchanged by partial patches +-- * `storage_header` is the header the storage patch is applied to +SELECT created_at_block, network_account_type, nonce, code_commitment, storage_header +FROM accounts +WHERE account_id = ?1 + AND valid_until = ?2 diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/queries/upsert_accounts/tests.rs similarity index 74% rename from crates/store/src/db/models/queries/accounts/tests.rs rename to crates/store/src/db/queries/upsert_accounts/tests.rs index cecdf0537f..cec7f82efe 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/queries/upsert_accounts/tests.rs @@ -2,8 +2,7 @@ use std::collections::BTreeMap; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use miden_node_proto::domain::account::AccountVaultDetails; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -31,69 +30,133 @@ use miden_protocol::account::{ StorageSlotType, }; use miden_protocol::asset::{NonFungibleAsset, NonFungibleAssetDetails}; -use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber, ValidatorKeys}; +use miden_protocol::block::{ + BlockAccountUpdate, + BlockHeader, + BlockNumber, + BlockSignatures, + ValidatorKeys, +}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::testing::account_id::AccountIdBuilder; -use miden_protocol::utils::serde::{Deserializable, Serializable}; use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::{Approver, AuthSingleSig}; use miden_standards::code_builder::CodeBuilder; use super::*; -use crate::db::models::conv::SqlTypeConvert; -use crate::db::schema; +use crate::db::queries::{self, HISTORICAL_BLOCK_RETENTION, VALID_FOREVER}; +use crate::db::{Result, TestDb}; use crate::errors::DatabaseError; -fn setup_test_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) +} + +fn insert_vault_asset( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + db.write(move |tx| queries::insert_vault_asset(tx, account_id, block_num, vault_key, asset)) +} + +fn prune_history(db: &TestDb, chain_tip: BlockNumber) -> Result<(usize, usize, usize)> { + db.write(move |tx| queries::prune_history(tx, chain_tip)) } +fn select_latest_storage(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_latest_storage(tx, account_id)) +} + +fn select_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_vault_at_block(tx, account_id, block_num)) +} + +fn select_account_header_with_storage_header_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| { + queries::select_account_header_with_storage_header_at_block(tx, account_id, block_num) + }) +} + +fn filter_network_accounts( + db: &TestDb, + account_ids: &[AccountId], +) -> Result> { + let account_ids = account_ids.to_vec(); + db.read(move |tx| queries::filter_network_accounts(tx, &account_ids)) +} + +// TEST HELPERS +// ================================================================================================ + /// Test helper: reconstructs account storage at a given block from DB. /// /// Reads `accounts.storage_header` and `account_storage_map_values` to reconstruct /// the full `AccountStorage` at the specified block. fn reconstruct_account_storage_at_block( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, block_num: BlockNumber, -) -> Result { - use schema::account_storage_map_values as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Query storage header blob for this account at or before this block - let storage_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter(schema::accounts::account_id.eq(&account_id_bytes)) - .filter(schema::accounts::block_num.le(block_num_sql)) - .order(schema::accounts::block_num.desc()) - .limit(1) - .first(conn) - .optional()? - .flatten(); - - let Some(blob) = storage_blob else { +) -> Result { + const SQL_STORAGE_HEADER: &str = "SELECT storage_header FROM accounts \ + WHERE account_id = ?1 AND block_num <= ?2 \ + ORDER BY block_num DESC LIMIT 1"; + const SQL_MAP_VALUES: &str = "SELECT slot_name, key, value FROM account_storage_map_values \ + WHERE account_id = ?1 AND block_num <= ?2 \ + ORDER BY slot_name ASC, key ASC, block_num DESC"; + + let header = db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL_STORAGE_HEADER, &[&account_id, &block_num], |row| { + row.get::>(0) + })? + .into_iter() + .next() + .flatten()) + })?; + + let Some(header) = header else { return Ok(AccountStorage::new(Vec::new())?); }; - let header = AccountStorageHeader::read_from_bytes(&blob)?; + // Rows arrive newest-first per key, so the first one seen for a key is the latest. + let map_values = db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx.query(SQL_MAP_VALUES, &[&account_id, &block_num], |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + })?) + })?; - // Query all map values for this account up to and including this block. - let map_values: Vec<(i64, String, Vec, Vec)> = - SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) - .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) - .load(conn)?; - - // For each (slot_name, key) pair, keep only the latest entry let mut latest_map_entries: BTreeMap<(StorageSlotName, StorageMapKey), Word> = BTreeMap::new(); - for (_, slot_name_str, key_bytes, value_bytes) in map_values { - let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { - DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) - })?; - let key = StorageMapKey::read_from_bytes(&key_bytes)?; - let value = Word::read_from_bytes(&value_bytes)?; + for (slot_name, key, value) in map_values { latest_map_entries.entry((slot_name, key)).or_insert(value); } @@ -164,9 +227,7 @@ fn create_test_account_with_storage() -> (Account, AccountId) { (account, account_id) } -fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use crate::db::schema::block_headers; - +fn insert_block_header(db: &TestDb, block_num: BlockNumber) { let secret_key = SigningKey::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -182,17 +243,44 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { test_fee_params(), 0_u8.into(), ); - let signature = secret_key.sign(block_header.commitment()); - - diesel::insert_into(block_headers::table) - .values(( - block_headers::block_num.eq(i64::from(block_num.as_u32())), - block_headers::block_header.eq(block_header.to_bytes()), - block_headers::signature.eq(signature.to_bytes()), - block_headers::commitment.eq(block_header.commitment().to_bytes()), - )) - .execute(conn) - .expect("Failed to insert block header"); + let signatures = + BlockSignatures::new(vec![secret_key.sign(block_header.commitment())]).unwrap(); + + db.write::<_, DatabaseError, _>(move |tx| { + queries::insert_block_header(tx, &block_header, &signatures) + }) + .expect("Failed to insert block header"); +} + +/// Counts the rows of `accounts` for the given account, and how many of them are current. +fn count_account_rows(db: &TestDb, account_id: AccountId) -> (i64, i64) { + const SQL: &str = "SELECT COUNT(*), COUNT(*) FILTER (WHERE valid_until = ?2) \ + FROM accounts WHERE account_id = ?1"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::(0)?, row.get::(1)?)) + })? + .into_iter() + .next() + .unwrap_or((0, 0))) + }) + .expect("Failed to count account rows") +} + +/// Returns whether the account's current row stores a storage header. +fn latest_account_has_storage_header(db: &TestDb, account_id: AccountId) -> Option { + const SQL: &str = "SELECT storage_header IS NOT NULL FROM accounts \ + WHERE account_id = ?1 AND valid_until = ?2"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| row.get::(0))? + .into_iter() + .next()) + }) + .expect("Failed to query storage header presence") } fn precomputed_state_from_account(account: &Account) -> PrecomputedPublicAccountState { @@ -270,9 +358,9 @@ fn assert_storage_map_slot_entries( #[test] fn select_account_header_at_block_returns_none_for_nonexistent() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let account_id = AccountId::dummy( [99u8; 15], @@ -282,21 +370,20 @@ fn select_account_header_at_block_returns_none_for_nonexistent() { ); // Query for a non-existent account - let result = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed"); + let result = select_account_header_with_storage_header_at_block(&db, account_id, block_num) + .expect("Query should succeed"); assert!(result.is_none(), "Should return None for non-existent account"); } #[test] fn select_account_header_at_block_returns_correct_header() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Insert the account let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -306,17 +393,12 @@ fn select_account_header_at_block_returns_correct_header() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); // Query the account header let (header, _storage_header) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) + select_account_header_with_storage_header_at_block(&db, account_id, block_num) .expect("Query should succeed") .expect("Header should exist"); @@ -331,14 +413,14 @@ fn select_account_header_at_block_returns_correct_header() { #[test] fn select_account_header_at_block_historical_query() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num_1 = BlockNumber::from_epoch(0); let block_num_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); + insert_block_header(&db, block_num_1); + insert_block_header(&db, block_num_2); // Insert the account at block 1 let nonce_1 = account.nonce(); @@ -349,17 +431,12 @@ fn select_account_header_at_block_historical_query() { AccountUpdateDetails::Public(patch_1), ); - upsert_accounts( - &mut conn, - &[account_update_1], - block_num_1, - &PrecomputedPublicAccountStates::new(), - ) - .expect("First upsert failed"); + upsert_accounts(&db, &[account_update_1], block_num_1, &PrecomputedPublicAccountStates::new()) + .expect("First upsert failed"); // Query at block 1 - should return the account let (header_1, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_1) + select_account_header_with_storage_header_at_block(&db, account_id, block_num_1) .expect("Query should succeed") .expect("Header should exist at block 1"); @@ -367,7 +444,7 @@ fn select_account_header_at_block_historical_query() { // Query at block 2 - should return the same account (most recent before block 2) let (header_2, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_2) + select_account_header_with_storage_header_at_block(&db, account_id, block_num_2) .expect("Query should succeed") .expect("Header should exist at block 2"); @@ -378,13 +455,13 @@ fn select_account_header_at_block_historical_query() { // ================================================================================================ #[test] -fn select_account_vault_at_block_empty() { - let mut conn = setup_test_db(); +fn select_vault_at_block_empty() { + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Insert account without vault assets let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -394,17 +471,11 @@ fn select_account_vault_at_block_empty() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); // Query vault - should return empty (the test account has no assets) - let assets = select_account_vault_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed"); + let assets = select_vault_at_block(&db, account_id, block_num).expect("Query should succeed"); assert!(assets.is_empty(), "Account should have no assets"); } @@ -414,12 +485,12 @@ fn select_account_vault_at_block_empty() { #[test] fn upsert_accounts_inserts_storage_header() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, account_id) = create_test_account_with_storage(); // Block 1 let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment_original = account.storage().to_commitment(); let storage_slots_len = account.storage().slots().len(); @@ -436,18 +507,14 @@ fn upsert_accounts_inserts_storage_header() { ); // Upsert account - let result = upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ); + let result = + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()); assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); // Query storage header back - let queried_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query storage header"); + let queried_storage = + select_latest_storage(&db, account_id).expect("Failed to query storage header"); // Verify storage commitment matches assert_eq!( @@ -460,28 +527,26 @@ fn upsert_accounts_inserts_storage_header() { assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); // Verify exactly 1 latest account with storage exists - let header_count: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .filter(schema::accounts::storage_header.is_not_null()) - .count() - .get_result(&mut conn) - .expect("Failed to count accounts with storage"); - - assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); + let (_, latest_accounts) = count_account_rows(&db, account_id); + assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); + assert_eq!( + latest_account_has_storage_header(&db, account_id), + Some(true), + "the latest account row must store a storage header" + ); } #[test] fn upsert_accounts_closes_previous_validity_interval() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, account_id) = create_test_account_with_storage(); // Block 1 and 2 let block_num_1 = BlockNumber::from_epoch(0); let block_num_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); + insert_block_header(&db, block_num_1); + insert_block_header(&db, block_num_2); // Save storage commitment before moving account let storage_commitment_1 = account.storage().to_commitment(); @@ -497,7 +562,7 @@ fn upsert_accounts_closes_previous_validity_interval() { AccountUpdateDetails::Public(patch_1), ); - upsert_accounts(&mut conn, &[account_update_1], block_num_1, &precomputed_1) + upsert_accounts(&db, &[account_update_1], block_num_1, &precomputed_1) .expect("First upsert failed"); // Create modified account with different storage value @@ -544,31 +609,17 @@ fn upsert_accounts_closes_previous_validity_interval() { AccountUpdateDetails::Public(patch_2), ); - upsert_accounts(&mut conn, &[account_update_2], block_num_2, &precomputed_2) + upsert_accounts(&db, &[account_update_2], block_num_2, &precomputed_2) .expect("Second upsert failed"); - // Verify 2 total account rows exist (both historical records) - let total_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .count() - .get_result(&mut conn) - .expect("Failed to count total accounts"); - + // Both historical records must exist, but only one of them is open-ended (the latest). + let (total_accounts, latest_accounts) = count_account_rows(&db, account_id); assert_eq!(total_accounts, 2, "Expected 2 total account records"); - - // Verify only 1 is open-ended (latest) - let latest_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .count() - .get_result(&mut conn) - .expect("Failed to count latest accounts"); - assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); // Verify latest storage matches second update - let latest_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query latest storage"); + let latest_storage = + select_latest_storage(&db, account_id).expect("Failed to query latest storage"); assert_eq!( latest_storage.to_commitment(), @@ -577,9 +628,8 @@ fn upsert_accounts_closes_previous_validity_interval() { ); // Verify historical query returns first update - let storage_at_block_1 = - reconstruct_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); + let storage_at_block_1 = reconstruct_account_storage_at_block(&db, account_id, block_num_1) + .expect("Failed to query storage at block 1"); assert_eq!( storage_at_block_1.to_commitment(), @@ -590,7 +640,7 @@ fn upsert_accounts_closes_previous_validity_interval() { #[test] fn upsert_accounts_with_multiple_storage_slots() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create account with 3 storage slots let account_id = AccountId::dummy( @@ -632,7 +682,7 @@ fn upsert_accounts_with_multiple_storage_slots() { .unwrap(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment = account.storage().to_commitment(); let account_commitment = account.to_commitment(); @@ -644,17 +694,11 @@ fn upsert_accounts_with_multiple_storage_slots() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Upsert with multiple storage slots failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Upsert with multiple storage slots failed"); // Query back and verify - let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let queried_storage = select_latest_storage(&db, account_id).expect("Failed to query storage"); assert_eq!( queried_storage.to_commitment(), @@ -676,7 +720,7 @@ fn upsert_accounts_with_multiple_storage_slots() { #[test] fn upsert_accounts_with_empty_storage() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create account with no component storage slots (only auth slot) let account_id = AccountId::dummy( @@ -708,7 +752,7 @@ fn upsert_accounts_with_empty_storage() { .unwrap(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment = account.storage().to_commitment(); let account_commitment = account.to_commitment(); @@ -720,17 +764,11 @@ fn upsert_accounts_with_empty_storage() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Upsert with empty storage failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Upsert with empty storage failed"); // Query back and verify - let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let queried_storage = select_latest_storage(&db, account_id).expect("Failed to query storage"); assert_eq!( queried_storage.to_commitment(), @@ -742,15 +780,7 @@ fn upsert_accounts_with_empty_storage() { assert_eq!(queried_storage.slots().len(), 2, "Expected 2 storage slots (auth component)"); // Verify the storage header blob exists in database - let storage_header_exists: Option = SelectDsl::select( - schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)), - schema::accounts::storage_header.is_not_null(), - ) - .first(&mut conn) - .optional() - .expect("Failed to check storage header existence"); + let storage_header_exists = latest_account_has_storage_header(&db, account_id); assert_eq!( storage_header_exists, @@ -763,10 +793,10 @@ fn upsert_accounts_with_empty_storage() { // ================================================================================================ #[test] -fn select_latest_account_storage_ordering_semantics() { - let mut conn = setup_test_db(); +fn select_latest_storage_ordering_semantics() { + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let slot_name = StorageSlotName::mock(0); let key_1 = StorageMapKey::from_index(1); @@ -800,26 +830,20 @@ fn select_latest_account_storage_ordering_semantics() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_storage(&db, account_id).expect("Failed to query storage"); let expected = BTreeMap::from_iter(entries); assert_storage_map_slot_entries(&storage, &slot_name, &expected); } #[test] -fn select_latest_account_storage_multiple_slots() { - let mut conn = setup_test_db(); +fn select_latest_storage_multiple_slots() { + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let slot_name_1 = StorageSlotName::mock(0); let slot_name_2 = StorageSlotName::mock(1); @@ -868,16 +892,10 @@ fn select_latest_account_storage_multiple_slots() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_storage(&db, account_id).expect("Failed to query storage"); let expected_slot_1 = BTreeMap::from_iter([(key_a, value_a)]); let expected_slot_2 = BTreeMap::from_iter([(key_b, value_b)]); @@ -887,12 +905,12 @@ fn select_latest_account_storage_multiple_slots() { } #[test] -fn select_latest_account_storage_slot_updates() { - let mut conn = setup_test_db(); +fn select_latest_storage_slot_updates() { + let db = TestDb::new(); let block_1 = BlockNumber::from_epoch(0); let block_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let slot_name = StorageSlotName::mock(0); let key_1 = StorageMapKey::from_index(1); @@ -913,7 +931,7 @@ fn select_latest_account_storage_slot_updates() { AccountUpdateDetails::Public(patch), ); - upsert_accounts(&mut conn, &[account_update], block_1, &PrecomputedPublicAccountStates::new()) + upsert_accounts(&db, &[account_update], block_1, &PrecomputedPublicAccountStates::new()) .expect("upsert_accounts failed"); let map_patch = StorageMapPatch::from_iters([], [(key_1, value_2), (key_2, value_3)]); @@ -944,11 +962,10 @@ fn select_latest_account_storage_slot_updates() { AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_storage(&db, account_id).expect("Failed to query storage"); let expected = BTreeMap::from_iter([(key_1, value_2), (key_2, value_3)]); assert_storage_map_slot_entries(&storage, &slot_name, &expected); @@ -963,7 +980,7 @@ fn select_latest_account_storage_slot_updates() { /// Focuses on deduplication logic that relies on ordering by (`vault_key` ASC and `block_num` /// DESC). #[test] -fn select_account_vault_at_block_historical_with_updates() { +fn select_vault_at_block_historical_with_updates() { use assert_matches::assert_matches; use miden_protocol::asset::FungibleAsset; use miden_protocol::testing::account_id::{ @@ -971,7 +988,7 @@ fn select_account_vault_at_block_historical_with_updates() { ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1, }; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -982,9 +999,9 @@ fn select_account_vault_at_block_historical_with_updates() { let block_2 = BlockNumber::from_epoch(1); let block_3 = BlockNumber::from_epoch(2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Insert account at block 1 let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -996,7 +1013,7 @@ fn select_account_vault_at_block_historical_with_updates() { for block in [block_1, block_2, block_3] { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block, &precomputed_states_from_account(&account), @@ -1008,36 +1025,36 @@ fn select_account_vault_at_block_historical_with_updates() { let asset_v1 = Asset::Fungible(FungibleAsset::new(faucet_id, 1000).unwrap()); let vault_key_1 = asset_v1.id(); - insert_account_vault_asset(&mut conn, account_id, block_1, vault_key_1, Some(asset_v1)) + insert_vault_asset(&db, account_id, block_1, vault_key_1, Some(asset_v1)) .expect("insert vault asset failed"); // Update vault asset at block 2: vault_key_1 = 2000 tokens (updated value) let asset_v2 = Asset::Fungible(FungibleAsset::new(faucet_id, 2000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key_1, Some(asset_v2)) + insert_vault_asset(&db, account_id, block_2, vault_key_1, Some(asset_v2)) .expect("insert vault asset update failed"); // Add a second vault_key at block 2 (different faucet for different vault key) let faucet_id_2 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1).unwrap(); let asset_key2 = Asset::Fungible(FungibleAsset::new(faucet_id_2, 500).unwrap()); let vault_key_2 = asset_key2.id(); - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key_2, Some(asset_key2)) + insert_vault_asset(&db, account_id, block_2, vault_key_2, Some(asset_key2)) .expect("insert second vault asset failed"); // Update vault_key_1 again at block 3: vault_key_1 = 3000 tokens let asset_v3 = Asset::Fungible(FungibleAsset::new(faucet_id, 3000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_3, vault_key_1, Some(asset_v3)) + insert_vault_asset(&db, account_id, block_3, vault_key_1, Some(asset_v3)) .expect("insert vault asset update 2 failed"); // Query at block 1: should only see vault_key_1 with 1000 tokens - let assets_at_block_1 = select_account_vault_at_block(&mut conn, account_id, block_1) - .expect("Query at block 1 should succeed"); + let assets_at_block_1 = + select_vault_at_block(&db, account_id, block_1).expect("Query at block 1 should succeed"); assert_eq!(assets_at_block_1.len(), 1, "Should have 1 asset at block 1"); assert_matches!(&assets_at_block_1[0], Asset::Fungible(f) if f.amount().as_u64() == 1000); // Query at block 2: should see vault_key_1 with 2000 tokens AND vault_key_2 with 500 tokens - let assets_at_block_2 = select_account_vault_at_block(&mut conn, account_id, block_2) - .expect("Query at block 2 should succeed"); + let assets_at_block_2 = + select_vault_at_block(&db, account_id, block_2).expect("Query at block 2 should succeed"); assert_eq!(assets_at_block_2.len(), 2, "Should have 2 assets at block 2"); @@ -1051,8 +1068,8 @@ fn select_account_vault_at_block_historical_with_updates() { assert!(amounts.contains(&500), "Block 2 should have vault_key_2 with 500 tokens"); // Query at block 3: should see vault_key_1 with 3000 tokens AND vault_key_2 with 500 tokens - let assets_at_block_3 = select_account_vault_at_block(&mut conn, account_id, block_3) - .expect("Query at block 3 should succeed"); + let assets_at_block_3 = + select_vault_at_block(&db, account_id, block_3).expect("Query at block 3 should succeed"); assert_eq!(assets_at_block_3.len(), 2, "Should have 2 assets at block 3"); @@ -1068,13 +1085,13 @@ fn select_account_vault_at_block_historical_with_updates() { /// Tests that the query bounds the number of rows it reads, so an over-the-limit vault is detected /// without materializing the whole set. #[test] -fn select_account_vault_at_block_bounds_read_to_limit() { - let mut conn = setup_test_db(); +fn select_vault_at_block_bounds_read_to_limit() { + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_1 = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_1); + insert_block_header(&db, block_1); let patch = AccountPatch::try_from(account.clone()).unwrap(); let account_update = BlockAccountUpdate::new( @@ -1083,7 +1100,7 @@ fn select_account_vault_at_block_bounds_read_to_limit() { AccountUpdateDetails::Public(patch), ); upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block_1, &PrecomputedPublicAccountStates::new(), @@ -1098,27 +1115,26 @@ fn select_account_vault_at_block_bounds_read_to_limit() { for i in 0..asset_count { let details = NonFungibleAssetDetails::new(faucet_id, vec![i as u8, (i >> 8) as u8]); let asset = Asset::NonFungible(NonFungibleAsset::new(&details)); - insert_account_vault_asset(&mut conn, account_id, block_1, asset.id(), Some(asset)) + insert_vault_asset(&db, account_id, block_1, asset.id(), Some(asset)) .expect("insert vault asset failed"); } // The query is capped at `MAX_RETURN_ENTRIES + 1` rows even though more assets exist, which is // enough for the caller to detect that the limit was exceeded. - let assets = select_account_vault_at_block(&mut conn, account_id, block_1) - .expect("query should succeed"); + let assets = select_vault_at_block(&db, account_id, block_1).expect("query should succeed"); assert_eq!(assets.len(), AccountVaultDetails::MAX_RETURN_ENTRIES + 1); } /// Tests that a 5-block history returns the correct asset per block. #[test] -fn select_account_vault_at_block_exponential_updates() { +fn select_vault_at_block_exponential_updates() { const BLOCK_COUNT: u32 = 5; use assert_matches::assert_matches; use miden_protocol::asset::{AssetId, FungibleAsset}; use miden_protocol::testing::account_id::ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -1127,7 +1143,7 @@ fn select_account_vault_at_block_exponential_updates() { let blocks: Vec = (0..BLOCK_COUNT).map(BlockNumber::from).collect(); for block in &blocks { - insert_block_header(&mut conn, *block); + insert_block_header(&db, *block); } let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -1139,7 +1155,7 @@ fn select_account_vault_at_block_exponential_updates() { for block in &blocks { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), *block, &precomputed_states_from_account(&account), @@ -1152,13 +1168,13 @@ fn select_account_vault_at_block_exponential_updates() { for (index, block) in blocks.iter().enumerate() { let amount = 1u64 << index; let asset = Asset::Fungible(FungibleAsset::new(faucet_id, amount).unwrap()); - insert_account_vault_asset(&mut conn, account_id, *block, vault_key, Some(asset)) + insert_vault_asset(&db, account_id, *block, vault_key, Some(asset)) .expect("insert vault asset failed"); } for (index, block) in blocks.iter().enumerate() { - let assets_at_block = select_account_vault_at_block(&mut conn, account_id, *block) - .expect("Query at block should succeed"); + let assets_at_block = + select_vault_at_block(&db, account_id, *block).expect("Query at block should succeed"); assert_eq!(assets_at_block.len(), 1, "Should have 1 asset at block"); let expected_amount = 1u64 << index; @@ -1172,12 +1188,12 @@ fn select_account_vault_at_block_exponential_updates() { /// Tests that deleted vault assets (asset = None) are correctly excluded from results, and that the /// deduplication handles deletion entries properly. #[test] -fn select_account_vault_at_block_with_deletion() { +fn select_vault_at_block_with_deletion() { use assert_matches::assert_matches; use miden_protocol::asset::FungibleAsset; use miden_protocol::testing::account_id::ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -1188,9 +1204,9 @@ fn select_account_vault_at_block_with_deletion() { let block_2 = BlockNumber::from_epoch(1); let block_3 = BlockNumber::from_epoch(2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Insert account at block 1 let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -1202,7 +1218,7 @@ fn select_account_vault_at_block_with_deletion() { for block in [block_1, block_2, block_3] { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block, &precomputed_states_from_account(&account), @@ -1214,31 +1230,31 @@ fn select_account_vault_at_block_with_deletion() { let asset = Asset::Fungible(FungibleAsset::new(faucet_id, 1000).unwrap()); let vault_key = asset.id(); - insert_account_vault_asset(&mut conn, account_id, block_1, vault_key, Some(asset)) + insert_vault_asset(&db, account_id, block_1, vault_key, Some(asset)) .expect("insert vault asset failed"); // Delete the vault asset at block 2 (insert with asset = None) - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key, None) + insert_vault_asset(&db, account_id, block_2, vault_key, None) .expect("delete vault asset failed"); // Re-add the vault asset at block 3 with different amount let asset_v3 = Asset::Fungible(FungibleAsset::new(faucet_id, 2000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_3, vault_key, Some(asset_v3)) + insert_vault_asset(&db, account_id, block_3, vault_key, Some(asset_v3)) .expect("re-add vault asset failed"); // Query at block 1: should see the asset - let assets_at_block_1 = select_account_vault_at_block(&mut conn, account_id, block_1) - .expect("Query at block 1 should succeed"); + let assets_at_block_1 = + select_vault_at_block(&db, account_id, block_1).expect("Query at block 1 should succeed"); assert_eq!(assets_at_block_1.len(), 1, "Should have 1 asset at block 1"); // Query at block 2: should NOT see the asset (it was deleted) - let assets_at_block_2 = select_account_vault_at_block(&mut conn, account_id, block_2) - .expect("Query at block 2 should succeed"); + let assets_at_block_2 = + select_vault_at_block(&db, account_id, block_2).expect("Query at block 2 should succeed"); assert!(assets_at_block_2.is_empty(), "Should have no assets at block 2 (deleted)"); // Query at block 3: should see the re-added asset with new amount - let assets_at_block_3 = select_account_vault_at_block(&mut conn, account_id, block_3) - .expect("Query at block 3 should succeed"); + let assets_at_block_3 = + select_vault_at_block(&db, account_id, block_3).expect("Query at block 3 should succeed"); assert_eq!(assets_at_block_3.len(), 1, "Should have 1 asset at block 3"); assert_matches!(&assets_at_block_3[0], Asset::Fungible(f) if f.amount().as_u64() == 2000); } @@ -1247,27 +1263,30 @@ fn select_account_vault_at_block_with_deletion() { // ================================================================================================ /// Counts the number of rows in `account_codes`. -fn count_account_codes(conn: &mut SqliteConnection) -> usize { - use schema::account_codes; - - let val = - SelectDsl::select(account_codes::table, diesel::dsl::count(account_codes::code_commitment)) - .get_result::(conn) - .expect("Failed to count account_codes"); - usize::try_from(u64::try_from(val).unwrap()).unwrap() -} +fn count_account_codes(db: &TestDb) -> usize { + const SQL: &str = "SELECT COUNT(*) FROM account_codes"; -/// Returns whether a specific code commitment exists in `account_codes`. -fn account_code_exists(conn: &mut SqliteConnection, code_commitment: Word) -> bool { - use schema::account_codes; + let count = db + .read::<_, DatabaseError, _>(|tx| { + Ok(tx.query(SQL, &[], |row| row.get::(0))?.into_iter().next().unwrap_or(0)) + }) + .expect("Failed to count account_codes"); - let n = - SelectDsl::select(account_codes::table, diesel::dsl::count(account_codes::code_commitment)) - .filter(account_codes::code_commitment.eq(code_commitment.to_bytes())) - .get_result::(conn) - .expect("Failed to query account_codes"); + usize::try_from(count).expect("row counts are non-negative") +} - n == 1 +/// Returns whether a specific code commitment exists in `account_codes`. +fn account_code_exists(db: &TestDb, code_commitment: Word) -> bool { + const SQL: &str = "SELECT EXISTS(SELECT 1 FROM account_codes WHERE code_commitment = ?1)"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&code_commitment], |row| row.get::(0))? + .into_iter() + .next() + .unwrap_or(false)) + }) + .expect("Failed to query account_codes") } /// Creates a full-state [`BlockAccountUpdate`] for the given account. @@ -1327,7 +1346,7 @@ fn build_account_with_code_seeded(push_value: u32, seed: [u8; 32]) -> Account { /// window, while the new (latest) code is retained. #[test] fn prune_account_code_retains_latest_after_code_change() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: account created with code A. // Block RETENTION+1 (=51): account updated to code B — within the retention window at prune @@ -1338,9 +1357,9 @@ fn prune_account_code_retains_latest_after_code_change() { let block_code_b = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 1); let block_prunable = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_prunable); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_prunable); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1359,7 +1378,7 @@ fn prune_account_code_retains_latest_after_code_change() { // Block 0: insert account with code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1368,31 +1387,27 @@ fn prune_account_code_retains_latest_after_code_change() { // Block RETENTION+1: update the same account ID to code B via a full-state delta. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // Advance past retention window and prune. cutoff = block_prunable - RETENTION = 2*RETENTION+1 // - RETENTION = RETENTION+1 = block_code_b - let (_, _, codes_deleted) = - prune_history(&mut conn, block_prunable).expect("prune_history failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prunable).expect("prune_history failed"); // Only code A was dropped; code B is still referenced by the latest accounts row. assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "old code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "old code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); // Confirm the latest account row still points to code B. let (latest_header, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_prunable) + select_account_header_with_storage_header_at_block(&db, account_id, block_prunable) .expect("query failed") .expect("account must still exist"); assert_eq!( @@ -1406,7 +1421,7 @@ fn prune_account_code_retains_latest_after_code_change() { /// code A must be retained because it is still the latest. #[test] fn prune_account_code_retains_revisited_code() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: code A. // Block RETENTION+1: code B (will be outside retention window at prune time). @@ -1420,10 +1435,10 @@ fn prune_account_code_retains_revisited_code() { let block_code_a_again = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 2); let block_prunable = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 2); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_code_a_again); - insert_block_header(&mut conn, block_prunable); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_code_a_again); + insert_block_header(&db, block_prunable); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1441,7 +1456,7 @@ fn prune_account_code_retains_revisited_code() { // Block 0: code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1449,7 +1464,7 @@ fn prune_account_code_retains_revisited_code() { .expect("block 0 upsert failed"); // Block RETENTION+1: code B. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), @@ -1457,7 +1472,7 @@ fn prune_account_code_retains_revisited_code() { .expect("block code_b upsert failed"); // Block RETENTION+2: back to code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_code_a_again, &precomputed_states_from_account(&account_a), @@ -1466,21 +1481,20 @@ fn prune_account_code_retains_revisited_code() { // Before pruning: both codes must be in account_codes (code A inserted once via ON CONFLICT DO // NOTHING, code B inserted once). - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // Advance past retention window and prune. - let (_, _, codes_deleted) = - prune_history(&mut conn, block_prunable).expect("prune_history failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prunable).expect("prune_history failed"); // Code B is no longer referenced by any account row within the retention window → pruned. Code // A is still referenced by the block_code_a_again accounts row (within cutoff) → retained. assert_eq!(codes_deleted, 1, "exactly one code (B) must be pruned"); - assert!(account_code_exists(&mut conn, code_commitment_a), "code A must be retained"); - assert!(!account_code_exists(&mut conn, code_commitment_b), "code B must be pruned"); + assert!(account_code_exists(&db, code_commitment_a), "code A must be retained"); + assert!(!account_code_exists(&db, code_commitment_b), "code B must be pruned"); // Confirm the latest account row still points to code A. let (latest_header, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_prunable) + select_account_header_with_storage_header_at_block(&db, account_id, block_prunable) .expect("query failed") .expect("account must still exist"); assert_eq!( @@ -1496,7 +1510,7 @@ fn prune_account_code_retains_revisited_code() { /// code becomes prunable. #[test] fn prune_account_code_retains_baseline_code() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: code A. // Block 2*RETENTION: code B. @@ -1510,10 +1524,10 @@ fn prune_account_code_retains_baseline_code() { let block_first_prune = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); let block_second_prune = BlockNumber::from(3 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_first_prune); - insert_block_header(&mut conn, block_second_prune); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_first_prune); + insert_block_header(&db, block_second_prune); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1525,7 +1539,7 @@ fn prune_account_code_retains_baseline_code() { // Block 0: code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1533,47 +1547,40 @@ fn prune_account_code_retains_baseline_code() { .expect("block 0 upsert failed"); // Block 2*RETENTION: code B. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // First prune: the block-0 row is the baseline (its successor is above the cutoff), so code A // must survive. let (_, _, codes_deleted) = - prune_history(&mut conn, block_first_prune).expect("prune_history failed"); + prune_history(&db, block_first_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "no code may be pruned while code A backs the baseline row"); - assert!( - account_code_exists(&mut conn, code_commitment_a), - "baseline code A must be retained" - ); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(account_code_exists(&db, code_commitment_a), "baseline code A must be retained"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); // Second prune: the code-B row is now at or below the cutoff and supersedes the block-0 row, so // code A is no longer reachable from any in-window read. let (_, _, codes_deleted) = - prune_history(&mut conn, block_second_prune).expect("prune_history failed"); + prune_history(&db, block_second_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "old code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "old code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); } /// Returns the cutoff recorded in `prune_progress`, if any. -fn codes_prune_cutoff(conn: &mut SqliteConnection) -> Option { - SelectDsl::select(schema::prune_progress::table, schema::prune_progress::codes_cutoff) - .first(conn) - .optional() - .expect("Failed to query prune_progress") +fn codes_prune_cutoff(db: &TestDb) -> Option { + const SQL: &str = "SELECT codes_cutoff FROM prune_progress"; + + db.read::<_, DatabaseError, _>(|tx| { + Ok(tx.query(SQL, &[], |row| row.get::(0))?.into_iter().next()) + }) + .expect("Failed to query prune_progress") } /// Prune test 5: the incremental (windowed) codes prune must not delete a code whose expiring @@ -1581,7 +1588,7 @@ fn codes_prune_cutoff(conn: &mut SqliteConnection) -> Option { /// it once the last reference expires in a later window. #[test] fn prune_account_code_incremental_cross_account_reference() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // The "switcher" account changes code first; the "holdout" account keeps code A pinned. // Both accounts are created with code A at block 0. @@ -1600,7 +1607,7 @@ fn prune_account_code_incremental_cross_account_reference() { let block_third_prune = BlockNumber::from(4 * HISTORICAL_BLOCK_RETENTION + 3); for block in [block_0, block_switcher_to_b, block_holdout_to_b] { - insert_block_header(&mut conn, block); + insert_block_header(&db, block); } let switcher_on_a = build_account_with_code(1); @@ -1619,7 +1626,7 @@ fn prune_account_code_incremental_cross_account_reference() { for account in [&switcher_on_a, &holdout_on_a] { upsert_accounts( - &mut conn, + &db, &[make_full_state_update(account)], block_0, &precomputed_states_from_account(account), @@ -1628,11 +1635,11 @@ fn prune_account_code_incremental_cross_account_reference() { } let (_, _, codes_deleted) = - prune_history(&mut conn, block_first_prune).expect("prune_history failed"); + prune_history(&db, block_first_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "no code is collectable while both accounts run code A"); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&switcher_on_b)], block_switcher_to_b, &precomputed_states_from_account(&switcher_on_b), @@ -1640,15 +1647,15 @@ fn prune_account_code_incremental_cross_account_reference() { .expect("switcher code-change upsert failed"); let (_, _, codes_deleted) = - prune_history(&mut conn, block_second_prune).expect("prune_history failed"); + prune_history(&db, block_second_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "code A must survive while the holdout still references it"); assert!( - account_code_exists(&mut conn, code_commitment_a), + account_code_exists(&db, code_commitment_a), "code A must be retained while the holdout references it" ); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&holdout_on_b)], block_holdout_to_b, &precomputed_states_from_account(&holdout_on_b), @@ -1656,20 +1663,17 @@ fn prune_account_code_incremental_cross_account_reference() { .expect("holdout code-change upsert failed"); let (_, _, codes_deleted) = - prune_history(&mut conn, block_third_prune).expect("prune_history failed"); + prune_history(&db, block_third_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); } /// Prune test 6: `prune_progress` records the cutoff of the last codes prune; re-pruning at the /// same or a lower cutoff deletes nothing and never moves the marker backwards. #[test] fn prune_account_codes_marker_never_regresses() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Same shape as prune test 2: code A at block 0 is superseded by code B at block R+1, so a // prune at tip 2R+1 (cutoff R+1) collects code A. @@ -1677,59 +1681,54 @@ fn prune_account_codes_marker_never_regresses() { let block_code_b = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 1); let block_prune = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), ) .expect("block 0 upsert failed"); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(codes_prune_cutoff(&mut conn), None, "no marker before the first prune"); + assert_eq!(codes_prune_cutoff(&db), None, "no marker before the first prune"); let cutoff = i64::from(HISTORICAL_BLOCK_RETENTION + 1); - let (_, _, codes_deleted) = prune_history(&mut conn, block_prune).expect("first prune failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prune).expect("first prune failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must record the cutoff"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must record the cutoff"); // Re-pruning at the same tip is a no-op. - let (_, _, codes_deleted) = prune_history(&mut conn, block_prune).expect("second prune failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prune).expect("second prune failed"); assert_eq!(codes_deleted, 0, "re-pruning at the same cutoff must delete nothing"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must be unchanged"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must be unchanged"); // Pruning at a lower tip (cutoff 0) must not move the marker backwards. - let (_, _, codes_deleted) = - prune_history(&mut conn, BlockNumber::from(HISTORICAL_BLOCK_RETENTION)) - .expect("stale prune failed"); + let (_, _, codes_deleted) = prune_history(&db, BlockNumber::from(HISTORICAL_BLOCK_RETENTION)) + .expect("stale prune failed"); assert_eq!(codes_deleted, 0, "pruning below the marker must delete nothing"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must never regress"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must never regress"); } #[test] #[miden_node_test_macro::enable_logging] fn network_accounts_subset_classifies_correctly() { - use crate::db::models::queries::accounts::{ - AccountRowInsert, - NetworkAccountType, - select_network_accounts_subset, - }; + use crate::db::queries::{AccountRow, NetworkAccountType}; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(1); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Three accounts with distinct classifications. AccountIds are dummies — the queries only care // about the (account_id, network_account_type, valid_until) tuple, not protocol-level validity. @@ -1763,22 +1762,20 @@ fn network_accounts_subset_classifies_correctly() { (public_id, NetworkAccountType::None), (private_id, NetworkAccountType::None), ] { - let row = AccountRowInsert::new_private(id, ty, Word::default(), block_num, block_num); - diesel::insert_into(crate::db::schema::accounts::table) - .values(&row) - .execute(&mut conn) - .unwrap(); + db.write::<_, DatabaseError, _>(move |tx| { + AccountRow::new_private(id, ty, Word::default(), block_num, block_num).upsert(tx) + }) + .unwrap(); } // Batched lookup returns only the network-classified id; public, private, and unknown ids are // all omitted. let subset = - select_network_accounts_subset(&mut conn, &[network_id, public_id, private_id, unknown_id]) - .unwrap(); + filter_network_accounts(&db, &[network_id, public_id, private_id, unknown_id]).unwrap(); assert_eq!(subset.len(), 1); assert!(subset.contains(&network_id)); // Empty input slice short-circuits to an empty result. - let empty = select_network_accounts_subset(&mut conn, &[]).unwrap(); + let empty = filter_network_accounts(&db, &[]).unwrap(); assert!(empty.is_empty()); } diff --git a/crates/store/src/db/queries/upsert_accounts/upsert_account.sql b/crates/store/src/db/queries/upsert_accounts/upsert_account.sql new file mode 100644 index 0000000000..8eb19adf22 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/upsert_account.sql @@ -0,0 +1,27 @@ +-- Writes an account's state at `block_num` as its current, open-ended version. +-- +-- Re-applying the same block overwrites that block's row rather than failing, so an interrupted +-- block application can be replayed. The key columns are excluded from the update: they are what +-- the conflict matched on. +INSERT INTO accounts ( + account_id, + network_account_type, + block_num, + account_commitment, + code_commitment, + nonce, + storage_header, + vault_root, + created_at_block, + valid_until +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) +ON CONFLICT(account_id, block_num) DO UPDATE SET + network_account_type = excluded.network_account_type, + account_commitment = excluded.account_commitment, + code_commitment = excluded.code_commitment, + nonce = excluded.nonce, + storage_header = excluded.storage_header, + vault_root = excluded.vault_root, + created_at_block = excluded.created_at_block, + valid_until = excluded.valid_until diff --git a/crates/store/src/db/test_db.rs b/crates/store/src/db/test_db.rs new file mode 100644 index 0000000000..a9ebda0c2e --- /dev/null +++ b/crates/store/src/db/test_db.rs @@ -0,0 +1,107 @@ +//! A blocking handle over the framework's connection pools, for testing query functions. +//! +//! Query functions take a [`ReadTx`]/[`WriteTx`], which the pools only ever hand out inside a +//! `read`/`write` closure on an async call. That is the right shape for production code, but it +//! would force every query-level test to be `async`. [`TestDb`] owns a current-thread runtime and +//! blocks on those calls, so the tests stay plain `#[test]` functions while still exercising the +//! same pools, PRAGMAs, and transaction behaviour the node uses. + +use std::path::{Path, PathBuf}; + +use diesel::Connection; +use miden_node_db::sqlite::{DbReader, DbWriter, ReadTx, WriteTx}; +use miden_node_db::{DatabaseError, default_connection_pool_size}; +use tokio::runtime::Runtime; + +use crate::db::migrations::bootstrap_database; + +/// A database with framework handles over it, driven synchronously. +/// +/// While the store is mid-migration off diesel, [`TestDb::diesel_conn`] also hands out diesel +/// connections over the same file so tests can drive the read queries still living in +/// [`crate::db::models`]. +pub(crate) struct TestDb { + // Held as `Option` so [`Drop`] can drop the pools inside the runtime's context: their pooled + // connections are closed on a blocking task, which panics without a runtime to spawn it on. + writer: Option, + reader: Option, + runtime: Runtime, + path: PathBuf, +} + +impl TestDb { + /// Bootstraps a throwaway database in the OS temp directory and opens handles over it. + /// + /// The temporary directory is intentionally leaked so the file outlives the handle; these are + /// test databases in the OS temp directory. + pub(crate) fn new() -> Self { + let temp_dir = tempfile::tempdir().expect("failed to create temp directory"); + let path = temp_dir.path().join("test.sqlite3"); + bootstrap_database(&path).expect("database should bootstrap"); + let _kept_dir = temp_dir.keep(); + + Self::open(&path) + } + + /// Opens handles over an existing, already migrated database file. + pub(crate) fn open(path: &Path) -> Self { + let (writer, reader) = + miden_node_db::sqlite::open_with_pool_size(path, default_connection_pool_size()) + .expect("temp file sqlite should always work"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime should build"); + + Self { + writer: Some(writer), + reader: Some(reader), + runtime, + path: path.to_path_buf(), + } + } + + /// Opens a fresh diesel connection over the same database file. + /// + /// Reads not yet migrated to the framework still run on diesel; WAL mode makes the extra + /// connection safe alongside the framework pools. + pub(crate) fn diesel_conn(&self) -> diesel::SqliteConnection { + let mut conn = diesel::SqliteConnection::establish( + self.path.to_str().expect("temp database path should be valid UTF-8"), + ) + .expect("temp file sqlite should always work"); + miden_node_db::configure_connection_on_creation(&mut conn) + .expect("connection PRAGMAs should apply"); + conn + } + + /// Runs `query` inside a read-only transaction. + pub(crate) fn read(&self, query: F) -> Result + where + F: FnOnce(&ReadTx<'_>) -> Result + Send + 'static, + R: Send + 'static, + E: From + Send + 'static, + { + let reader = self.reader.as_ref().expect("handles live until drop"); + self.runtime.block_on(reader.read("test read", query)) + } + + /// Runs `query` inside a read-write transaction, committing it if `query` returns `Ok`. + pub(crate) fn write(&self, query: F) -> Result + where + F: FnOnce(&WriteTx<'_>) -> Result + Send + 'static, + R: Send + 'static, + E: From + Send + 'static, + { + let writer = self.writer.as_ref().expect("handles live until drop"); + self.runtime.block_on(writer.write("test write", query)) + } +} + +impl Drop for TestDb { + fn drop(&mut self) { + let _guard = self.runtime.enter(); + self.writer.take(); + self.reader.take(); + } +} diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 9dad5cc135..a1da54a500 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,7 +1,8 @@ +use std::ops::RangeInclusive; use std::sync::{Arc, Mutex}; use assert_matches::assert_matches; -use diesel::{Connection, SqliteConnection}; +use miden_node_db::sqlite::WriteTx; use miden_node_proto::domain::account::{AccountSummary, StorageMapEntries}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; @@ -14,6 +15,7 @@ use miden_protocol::account::{ AccountId, AccountIdVersion, AccountPatch, + AccountStorage, AccountStoragePatch, AccountType, AccountUpdateDetails, @@ -26,7 +28,7 @@ use miden_protocol::account::{ StorageSlotName, StorageSlotPatch, }; -use miden_protocol::asset::{Asset, FungibleAsset}; +use miden_protocol::asset::{Asset, AssetId, FungibleAsset}; use miden_protocol::block::{ BlockAccountUpdate, BlockHeader, @@ -50,6 +52,7 @@ use miden_protocol::note::{ NoteHeader, NoteId, NoteMetadata, + NoteScript, NoteTag, NoteType, Nullifier, @@ -88,20 +91,250 @@ use crate::account_state_forest::{ HISTORICAL_BLOCK_RETENTION, TestAccountStateForestExt, }; +use crate::db::models::queries as diesel_queries; use crate::db::models::queries::{ - PrecomputedPublicAccountState, - PrecomputedPublicAccountStates, + NOTE_SYNC_BLOCK_OVERHEAD_BYTES, + NOTE_SYNC_RECORD_BYTES, StorageMapValue, - insert_account_storage_map_value, + StorageMapValuesPage, }; -use crate::db::models::{queries, utils}; -use crate::errors::DatabaseError; +use crate::db::queries::{self, PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; +use crate::db::{AccountVaultValue, BlockHeaderCommitment, NoteSyncUpdate, Result, TestDb, utils}; +use crate::errors::{DatabaseError, NoteSyncError}; -fn create_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn insert_block_header( + db: &TestDb, + header: &BlockHeader, + signatures: &BlockSignatures, +) -> Result { + let header = header.clone(); + let signatures = signatures.clone(); + db.write(move |tx| queries::insert_block_header(tx, &header, &signatures)) +} + +fn insert_notes(db: &TestDb, notes: &[(NoteRecord, Option)]) -> Result { + let notes = notes.to_vec(); + db.write(move |tx| queries::insert_notes(tx, ¬es)) +} + +fn insert_note_scripts(db: &TestDb, notes: &[NoteRecord]) -> Result { + let notes = notes.to_vec(); + db.write(move |tx| queries::insert_note_scripts(tx, notes.iter())) +} + +fn insert_nullifiers_for_block( + db: &TestDb, + nullifiers: &[Nullifier], + block_num: BlockNumber, +) -> Result { + let nullifiers = nullifiers.to_vec(); + db.write(move |tx| queries::insert_nullifiers_for_block(tx, &nullifiers, block_num)) +} + +fn insert_transactions( + db: &TestDb, + block_num: BlockNumber, + transactions: &OrderedTransactionHeaders, +) -> Result { + let transactions = transactions.clone(); + db.write(move |tx| queries::insert_transactions(tx, block_num, &transactions)) +} + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) +} + +fn insert_storage_map_value( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + slot_name: StorageSlotName, + key: StorageMapKey, + value: Word, +) -> Result { + db.write(move |tx| { + queries::insert_storage_map_value(tx, account_id, block_num, &slot_name, key, value) + }) +} + +fn insert_vault_asset( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + db.write(move |tx| queries::insert_vault_asset(tx, account_id, block_num, vault_key, asset)) +} + +fn prune_history(db: &TestDb, chain_tip: BlockNumber) -> Result<(usize, usize, usize)> { + db.write(move |tx| queries::prune_history(tx, chain_tip)) +} + +// Read drivers below run on the diesel layer until their queries migrate to the framework; each one +// opens a fresh diesel connection over the same database file. + +fn select_all_nullifiers(db: &TestDb) -> Result> { + diesel_queries::select_all_nullifiers(&mut db.diesel_conn()) +} + +fn select_nullifiers_by_prefix( + db: &TestDb, + prefix_len: u8, + nullifier_prefixes: &[u16], + block_range: RangeInclusive, +) -> Result<(Vec, BlockNumber)> { + diesel_queries::select_nullifiers_by_prefix( + &mut db.diesel_conn(), + prefix_len, + nullifier_prefixes, + block_range, + ) +} + +fn select_notes_since_block_by_tag( + db: &TestDb, + note_tags: &[u32], + block_range: RangeInclusive, +) -> Result> { + diesel_queries::select_notes_since_block_by_tag(&mut db.diesel_conn(), note_tags, block_range) +} + +fn select_notes_by_id(db: &TestDb, note_ids: &[NoteId]) -> Result> { + diesel_queries::select_notes_by_id(&mut db.diesel_conn(), note_ids) +} + +fn select_note_script_by_root(db: &TestDb, root: Word) -> Result> { + diesel_queries::select_note_script_by_root(&mut db.diesel_conn(), root) +} + +fn get_note_sync_multi( + db: &TestDb, + note_tags: &[u32], + block_range: RangeInclusive, + max_response_payload_bytes: usize, +) -> std::result::Result, NoteSyncError> { + diesel_queries::get_note_sync_multi( + &mut db.diesel_conn(), + note_tags, + block_range, + max_response_payload_bytes, + ) } -fn create_block(conn: &mut SqliteConnection, block_num: BlockNumber) { +fn select_block_header_by_block_num( + db: &TestDb, + maybe_block_num: Option, +) -> Result> { + diesel_queries::select_block_header_by_block_num(&mut db.diesel_conn(), maybe_block_num) +} + +fn select_block_header_and_signatures_by_block_num( + db: &TestDb, + block_num: BlockNumber, +) -> Result> { + diesel_queries::select_block_header_and_signatures_by_block_num( + &mut db.diesel_conn(), + block_num, + ) +} + +fn select_block_headers(db: &TestDb, blocks: Vec) -> Result> { + diesel_queries::select_block_headers(&mut db.diesel_conn(), blocks.into_iter()) +} + +fn select_all_block_header_commitments(db: &TestDb) -> Result> { + diesel_queries::select_all_block_header_commitments(&mut db.diesel_conn()) +} + +fn select_account(db: &TestDb, account_id: AccountId) -> Result { + diesel_queries::select_account(&mut db.diesel_conn(), account_id) +} + +fn select_all_accounts(db: &TestDb) -> Result> { + diesel_queries::select_all_accounts(&mut db.diesel_conn()) +} + +fn select_account_code_by_commitment( + db: &TestDb, + code_commitment: Word, +) -> Result>> { + diesel_queries::select_account_code_by_commitment(&mut db.diesel_conn(), code_commitment) +} + +fn select_latest_storage(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_latest_storage(tx, account_id)) +} + +fn select_account_storage_map_values_paged( + db: &TestDb, + account_id: AccountId, + block_range: RangeInclusive, + limit: usize, +) -> Result { + diesel_queries::select_account_storage_map_values_paged( + &mut db.diesel_conn(), + account_id, + block_range, + limit, + ) +} + +fn select_account_vault_assets( + db: &TestDb, + account_id: AccountId, + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec)> { + diesel_queries::select_account_vault_assets(&mut db.diesel_conn(), account_id, block_range) +} + +fn select_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_vault_at_block(tx, account_id, block_num)) +} + +fn select_transactions_records( + db: &TestDb, + account_ids: &[AccountId], + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec)> { + diesel_queries::select_transactions_records(&mut db.diesel_conn(), account_ids, block_range) +} + +// TEST HELPERS +// ================================================================================================ + +fn create_block(db: &TestDb, block_num: BlockNumber) { + let (block_header, signatures) = mock_block(block_num); + insert_block_header(db, &block_header, &signatures).unwrap(); +} + +/// [`create_block`] for tests that already hold a write transaction. +fn create_block_in(tx: &WriteTx<'_>, block_num: BlockNumber) -> Result<()> { + let (block_header, signatures) = mock_block(block_num); + queries::insert_block_header(tx, &block_header, &signatures)?; + Ok(()) +} + +fn mock_block(block_num: BlockNumber) -> (BlockHeader, BlockSignatures) { let block_header = BlockHeader::new( 1_u8.into(), num_to_word(2), @@ -120,11 +353,7 @@ fn create_block(conn: &mut SqliteConnection, block_num: BlockNumber) { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - conn.transaction(|conn| { - queries::insert_block_header(conn, &block_header, &dummy_signature)?; - Ok::<_, DatabaseError>(()) - }) - .unwrap(); + (block_header, dummy_signature) } fn precomputed_states_from_account(account: &Account) -> PrecomputedPublicAccountStates { @@ -147,32 +376,27 @@ fn precomputed_states_from_account(account: &Account) -> PrecomputedPublicAccoun #[test] #[miden_node_test_macro::enable_logging] fn sql_insert_nullifiers_for_block() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let nullifiers = [num_to_nullifier(1 << 48)]; let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // Insert a new nullifier succeeds { - conn.transaction(|conn| { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); - assert_eq!(res.unwrap(), nullifiers.len(), "There should be one entry"); - Ok::<_, DatabaseError>(()) - }) - .unwrap(); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); + assert_eq!(res.unwrap(), nullifiers.len(), "There should be one entry"); } // Inserting the nullifier twice is an error { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); assert!(res.is_err(), "Inserting the same nullifier twice is an error"); } // even if the block number is different { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num + 1); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num + 1); assert!( res.is_err(), @@ -185,7 +409,7 @@ fn sql_insert_nullifiers_for_block() { let nullifiers: Vec<_> = (0..10).map(num_to_nullifier).collect(); let block_num = 1.into(); - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); assert_eq!(res.unwrap(), nullifiers.len(), "There should be 10 entries"); } @@ -194,9 +418,8 @@ fn sql_insert_nullifiers_for_block() { #[test] #[miden_node_test_macro::enable_logging] fn sql_insert_transactions() { - let mut conn = create_db(); - let conn = &mut conn; - let count = insert_transactions(conn); + let db = &TestDb::new(); + let count = insert_mock_transactions(db); assert_eq!(count, 2, "Two elements must have been inserted"); } @@ -204,13 +427,12 @@ fn sql_insert_transactions() { #[test] #[miden_node_test_macro::enable_logging] fn sql_select_nullifiers() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // test querying empty table - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert!(nullifiers.is_empty()); // test multiple entries @@ -219,10 +441,10 @@ fn sql_select_nullifiers() { let nullifier = num_to_nullifier(i); state.push(NullifierInfo { nullifier, block_num }); - let res = queries::insert_nullifiers_for_block(conn, &[nullifier], block_num); + let res = insert_nullifiers_for_block(db, &[nullifier], block_num); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers, state); } } @@ -248,18 +470,17 @@ pub fn create_note(account_id: AccountId) -> Note { #[test] #[miden_node_test_macro::enable_logging] fn sql_select_note_script_by_root() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(conn, block_num); + create_block(db, block_num); let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -278,57 +499,51 @@ fn sql_select_note_script_by_root() { }; state.push(note.clone()); - let res = queries::insert_scripts(conn, [¬e]); + let res = insert_note_scripts(db, std::slice::from_ref(¬e)); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); // test querying the script by the root - let note_script = - queries::select_note_script_by_root(conn, Word::from(new_note.script().root())).unwrap(); + let note_script = select_note_script_by_root(db, Word::from(new_note.script().root())).unwrap(); assert_eq!(note_script, Some(new_note.script().clone())); // test querying the script by the root that is not in the database - let note_script = queries::select_note_script_by_root(conn, [0_u16; 4].into()).unwrap(); + let note_script = select_note_script_by_root(db, [0_u16; 4].into()).unwrap(); assert_eq!(note_script, None); } // Generates an account, inserts into the database, and creates a note for it. fn make_account_and_note( - conn: &mut SqliteConnection, + db: &TestDb, block_num: BlockNumber, init_seed: [u8; 32], account_type: AccountType, ) -> (AccountId, Note) { - conn.transaction(|conn| { - let account = mock_account_code_and_storage(account_type, [], Some(init_seed)); - let account_id = account.id(); - queries::upsert_accounts( - conn, - &[BlockAccountUpdate::new( - account_id, - account.to_commitment(), - AccountUpdateDetails::Public(AccountPatch::try_from(account.clone()).unwrap()), - )], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + let account = mock_account_code_and_storage(account_type, [], Some(init_seed)); + let account_id = account.id(); + upsert_accounts( + db, + &[BlockAccountUpdate::new( + account_id, + account.to_commitment(), + AccountUpdateDetails::Public(AccountPatch::try_from(account.clone()).unwrap()), + )], + block_num, + &precomputed_states_from_account(&account), + ) + .unwrap(); - let new_note = create_note(account_id); - Ok::<_, DatabaseError>((account_id, new_note)) - }) - .unwrap() + (account_id, create_note(account_id)) } #[test] #[miden_node_test_macro::enable_logging] fn sql_select_accounts() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // test querying empty table - let accounts = queries::select_all_accounts(conn).unwrap(); + let accounts = select_all_accounts(db).unwrap(); assert!(accounts.is_empty()); // test multiple entries let mut state = vec![]; @@ -349,19 +564,19 @@ fn sql_select_accounts() { details: None, }); - let res = queries::upsert_accounts( - conn, + let res = upsert_accounts( + db, &[BlockAccountUpdate::new( account_id, account_commitment, AccountUpdateDetails::Private, )], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); - let accounts = queries::select_all_accounts(conn).unwrap(); + let accounts = select_all_accounts(db).unwrap(); assert_eq!(accounts, state); } } @@ -369,8 +584,7 @@ fn sql_select_accounts() { #[test] #[miden_node_test_macro::enable_logging] fn sync_account_vault_basic_validation() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); // Create a public account for vault testing let public_account_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); @@ -380,16 +594,16 @@ fn sync_account_vault_basic_validation() { let invalid_block_from: BlockNumber = 10.into(); // Create blocks - create_block(conn, block_from); - create_block(conn, block_mid); - create_block(conn, block_to); + create_block(db, block_from); + create_block(db, block_mid); + create_block(db, block_to); for block in [block_from, block_mid, block_to] { - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(public_account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } @@ -402,28 +616,16 @@ fn sync_account_vault_basic_validation() { let vault_key_2 = fungible_asset_2.id(); // Insert vault assets for the public account at different blocks - queries::insert_account_vault_asset( - conn, - public_account_id, - block_from, - vault_key_1, - Some(fungible_asset_1), - ) - .unwrap(); - queries::insert_account_vault_asset( - conn, - public_account_id, - block_mid, - vault_key_2, - Some(fungible_asset_2), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_from, vault_key_1, Some(fungible_asset_1)) + .unwrap(); + insert_vault_asset(db, public_account_id, block_mid, vault_key_2, Some(fungible_asset_2)) + .unwrap(); // Update an existing vault asset (sets previous as not latest) let updated_fungible_asset_1 = Asset::Fungible(FungibleAsset::new(public_account_id, 1500).unwrap()); - queries::insert_account_vault_asset( - conn, + insert_vault_asset( + db, public_account_id, block_to, vault_key_1, @@ -432,11 +634,7 @@ fn sync_account_vault_basic_validation() { .unwrap(); // Test invalid block range - should return error - let result = queries::select_account_vault_assets( - conn, - public_account_id, - invalid_block_from..=block_to, - ); + let result = select_account_vault_assets(db, public_account_id, invalid_block_from..=block_to); assert!(result.is_err(), "expected error for invalid block range"); let Err(crate::errors::DatabaseError::InvalidBlockRange { .. }) = result else { @@ -445,8 +643,7 @@ fn sync_account_vault_basic_validation() { // Test with valid block range - should return vault assets let (last_block, values) = - queries::select_account_vault_assets(conn, public_account_id, block_from..=block_to) - .unwrap(); + select_account_vault_assets(db, public_account_id, block_from..=block_to).unwrap(); // Should return assets we inserted assert!(!values.is_empty(), "vault assets should have data"); @@ -463,25 +660,24 @@ fn sync_account_vault_basic_validation() { #[miden_node_test_macro::enable_logging] fn select_nullifiers_by_prefix_works() { const PREFIX_LEN: u8 = 16; - let mut conn = create_db(); - let conn = &mut conn; // test empty table + let db = &TestDb::new(); + // test empty table let block_number0 = 0.into(); let block_number10 = 10.into(); let (nullifiers, block_number_reached) = - queries::select_nullifiers_by_prefix(conn, PREFIX_LEN, &[], block_number0..=block_number10) - .unwrap(); + select_nullifiers_by_prefix(db, PREFIX_LEN, &[], block_number0..=block_number10).unwrap(); assert!(nullifiers.is_empty()); assert_eq!(block_number_reached, block_number10); // test single item let nullifier1 = num_to_nullifier(1 << 48); let block_number1 = 1.into(); - create_block(conn, block_number1); + create_block(db, block_number1); - queries::insert_nullifiers_for_block(conn, &[nullifier1], block_number1).unwrap(); + insert_nullifiers_for_block(db, &[nullifier1], block_number1).unwrap(); - let (nullifiers, block_number_reached) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, block_number_reached) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier1)], block_number0..=block_number10, @@ -500,16 +696,16 @@ fn select_nullifiers_by_prefix_works() { // test two elements let nullifier2 = num_to_nullifier(2 << 48); let block_number2 = 2.into(); - create_block(conn, block_number2); + create_block(db, block_number2); - queries::insert_nullifiers_for_block(conn, &[nullifier2], block_number2).unwrap(); + insert_nullifiers_for_block(db, &[nullifier2], block_number2).unwrap(); - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers, vec![(nullifier1, block_number1), (nullifier2, block_number2)]); // only the nullifiers matching the prefix are included - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier1)], block_number0..=block_number10, @@ -522,8 +718,8 @@ fn select_nullifiers_by_prefix_works() { block_num: block_number1 }] ); - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier2)], block_number0..=block_number10, @@ -538,8 +734,8 @@ fn select_nullifiers_by_prefix_works() { ); // All matching nullifiers are included - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -563,8 +759,8 @@ fn select_nullifiers_by_prefix_works() { ); // If a non-matching prefix is provided, no nullifiers are returned - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&num_to_nullifier(3 << 48))], block_number0..=block_number10, @@ -574,8 +770,8 @@ fn select_nullifiers_by_prefix_works() { // If a block number is provided, only matching nullifiers created at or after that block are // returned - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -595,12 +791,12 @@ fn select_nullifiers_by_prefix_works() { // Nullifiers are not returned if the block number is after the last nullifier let nullifier3 = num_to_nullifier(3 << 48); let block_number3 = 3.into(); - create_block(conn, block_number3); + create_block(db, block_number3); - queries::insert_nullifiers_for_block(conn, &[nullifier3], block_number3).unwrap(); + insert_nullifiers_for_block(db, &[nullifier3], block_number3).unwrap(); - let (nullifiers, block_number_reached) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, block_number_reached) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -629,13 +825,13 @@ fn select_nullifiers_by_prefix_works() { #[test] #[miden_node_test_macro::enable_logging] fn db_block_header() { - let mut conn = create_db(); - let conn = &mut conn; // test querying empty table + let db = &TestDb::new(); + // test querying empty table let block_number = 1; - let res = queries::select_block_header_by_block_num(conn, Some(block_number.into())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_number.into())).unwrap(); assert!(res.is_none()); - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert!(res.is_none()); let block_header = BlockHeader::new( @@ -656,20 +852,20 @@ fn db_block_header() { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - queries::insert_block_header(conn, &block_header, &dummy_signature).unwrap(); + insert_block_header(db, &block_header, &dummy_signature).unwrap(); + let first_signature = dummy_signature; // test fetch unknown block header let block_number = 1; - let res = queries::select_block_header_by_block_num(conn, Some(block_number.into())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_number.into())).unwrap(); assert!(res.is_none()); // test fetch block header by block number - let res = - queries::select_block_header_by_block_num(conn, Some(block_header.block_num())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_header.block_num())).unwrap(); assert_eq!(res.unwrap(), block_header); // test fetch latest block header - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert_eq!(res.unwrap(), block_header); let block_header2 = BlockHeader::new( @@ -689,46 +885,59 @@ fn db_block_header() { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header2.commitment())]).unwrap(); - queries::insert_block_header(conn, &block_header2, &dummy_signature).unwrap(); + insert_block_header(db, &block_header2, &dummy_signature).unwrap(); - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert_eq!(res.unwrap(), block_header2); - let res = queries::select_block_headers( - conn, - [block_header.block_num(), block_header2.block_num()].into_iter(), - ) - .unwrap(); - assert_eq!(res, [block_header, block_header2]); + let res = select_block_headers(db, vec![block_header.block_num(), block_header2.block_num()]) + .unwrap(); + assert_eq!(res, [block_header.clone(), block_header2.clone()]); + + // commitments come back in block number order + let commitments = select_all_block_header_commitments(db).unwrap(); + assert_eq!( + commitments, + [ + BlockHeaderCommitment::new(&block_header), + BlockHeaderCommitment::new(&block_header2), + ] + ); + + // test fetch block header with its signatures + let stored = + select_block_header_and_signatures_by_block_num(db, block_header.block_num()).unwrap(); + assert_eq!(stored, Some((block_header, first_signature))); + + let missing = select_block_header_and_signatures_by_block_num(db, 1.into()).unwrap(); + assert!(missing.is_none()); } #[test] #[miden_node_test_macro::enable_logging] fn notes() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num_1 = 1.into(); - create_block(conn, block_num_1); + create_block(db, block_num_1); let block_range = BlockNumber::GENESIS..=BlockNumber::from(1); // test empty table - let res = queries::select_notes_since_block_by_tag(conn, &[], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[], block_range.clone()).unwrap(); assert!(res.is_empty()); - let res = - queries::select_notes_since_block_by_tag(conn, &[1, 2, 3], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[1, 2, 3], block_range.clone()).unwrap(); assert!(res.is_empty()); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); // test insertion - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num_1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -755,24 +964,24 @@ fn notes() { inclusion_path: inclusion_path.clone(), }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // test empty tags - let res = queries::select_notes_since_block_by_tag(conn, &[], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[], block_range.clone()).unwrap(); assert!(res.is_empty()); let block_range_1 = 2.into()..=2.into(); // test no updates - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range_1).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range_1).unwrap(); assert!(res.is_empty()); // test match - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range.clone()).unwrap(); assert_eq!(res, vec![note.clone().into()]); let block_num_2 = note.block_num + 1; - create_block(conn, block_num_2); + create_block(db, block_num_2); // insertion second note with same tag, but on higher block let note2 = NoteRecord { @@ -785,19 +994,19 @@ fn notes() { inclusion_path: inclusion_path.clone(), }; - queries::insert_notes(conn, &[(note2.clone(), None)]).unwrap(); + insert_notes(db, &[(note2.clone(), None)]).unwrap(); let block_range = 0.into()..=2.into(); // only the first matching block is returned; `get_note_sync_multi` loops this inside a single // database transaction when multiple blocks are requested. - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range).unwrap(); assert_eq!(res, vec![note.clone().into()]); let block_range = 2.into()..=2.into(); // only the second note is returned when range is restricted to block 2 - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range).unwrap(); assert_eq!(res, vec![note2.clone().into()]); // test query notes by id @@ -805,7 +1014,7 @@ fn notes() { let note_ids = Vec::from_iter(notes.iter().map(|note| NoteId::from_raw(note.note_id))); - let res = queries::select_notes_by_id(conn, ¬e_ids).unwrap(); + let res = select_notes_by_id(db, ¬e_ids).unwrap(); assert_eq!(res, notes); // test notes have correct details @@ -820,8 +1029,7 @@ fn notes() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_across_multiple_blocks() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); @@ -831,12 +1039,12 @@ fn note_sync_across_multiple_blocks() { for block_num_raw in 1..=3u32 { let block_num = BlockNumber::from(block_num_raw); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, block_num_raw.into())], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -870,8 +1078,8 @@ fn note_sync_across_multiple_blocks() { attachments, inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); } // Build an MMR with enough leaves to cover all blocks (0..=3). @@ -884,8 +1092,8 @@ fn note_sync_across_multiple_blocks() { // A single call to get_note_sync_multi should return all 3 blocks. let block_range = BlockNumber::GENESIS..=BlockNumber::from(3); - let updates = queries::get_note_sync_multi( - conn, + let updates = get_note_sync_multi( + db, &[tag], block_range, miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES, @@ -915,8 +1123,7 @@ fn note_sync_across_multiple_blocks() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_multi_respects_payload_limit() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let tag = 43u32; @@ -924,12 +1131,12 @@ fn note_sync_multi_respects_payload_limit() { for block_num_raw in 1..=3u32 { let block_num = BlockNumber::from(block_num_raw); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, block_num_raw.into())], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -963,14 +1170,13 @@ fn note_sync_multi_respects_payload_limit() { attachments, inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); } - let one_block_budget = - queries::NOTE_SYNC_BLOCK_OVERHEAD_BYTES + queries::NOTE_SYNC_RECORD_BYTES; - let updates = queries::get_note_sync_multi( - conn, + let one_block_budget = NOTE_SYNC_BLOCK_OVERHEAD_BYTES + NOTE_SYNC_RECORD_BYTES; + let updates = get_note_sync_multi( + db, &[tag], BlockNumber::GENESIS..=BlockNumber::from(3), one_block_budget, @@ -991,17 +1197,16 @@ fn note_sync_multi_respects_payload_limit() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_no_matching_tags() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block_num = BlockNumber::from(1); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -1026,13 +1231,13 @@ fn note_sync_no_matching_tags() { attachments: NoteAttachments::default(), inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); // Query with a different tag should return empty vec. let range = BlockNumber::GENESIS..=BlockNumber::from(1); - let result = queries::get_note_sync_multi( - conn, + let result = get_note_sync_multi( + db, &[999], range, miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES, @@ -1042,22 +1247,15 @@ fn note_sync_no_matching_tags() { } fn insert_account_patch( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, block_number: BlockNumber, patch: &AccountPatch, ) { for (slot_name, slot_patch) in patch.storage().maps() { for (k, v) in slot_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) { - insert_account_storage_map_value( - conn, - account_id, - block_number, - slot_name.clone(), - *k, - *v, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block_number, slot_name.clone(), *k, *v) + .unwrap(); } } } @@ -1069,29 +1267,28 @@ fn sql_account_storage_map_values_insertion() { use miden_protocol::account::StorageMapPatch; - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block1: BlockNumber = 1.into(); let block2: BlockNumber = 2.into(); - create_block(conn, block1); - create_block(conn, block2); + create_block(db, block1); + create_block(db, block2); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE_2).unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -1114,10 +1311,10 @@ fn sql_account_storage_map_values_insertion() { Some(Felt::new_unchecked(2)), ) .unwrap(); - insert_account_patch(conn, account_id, block1, &patch1); + insert_account_patch(db, account_id, block1, &patch1); - let storage_map_page = queries::select_account_storage_map_values_paged( - conn, + let storage_map_page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block1, 1024, @@ -1137,10 +1334,10 @@ fn sql_account_storage_map_values_insertion() { Some(Felt::new_unchecked(3)), ) .unwrap(); - insert_account_patch(conn, account_id, block2, &patch2); + insert_account_patch(db, account_id, block2, &patch2); - let storage_map_values = queries::select_account_storage_map_values_paged( - conn, + let storage_map_values = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block2, 1024, @@ -1167,7 +1364,7 @@ fn sql_account_storage_map_values_insertion() { #[test] fn select_storage_map_sync_values() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(5); @@ -1183,69 +1380,29 @@ fn select_storage_map_sync_values() { let block3 = BlockNumber::from(3); for block in [block1, block2, block3] { - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } // Insert data across multiple blocks using individual inserts Block 1: key1 -> value1, key2 -> // value2 - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block1, - slot_name.clone(), - key1, - value1, - ) - .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block1, - slot_name.clone(), - key2, - value2, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block1, slot_name.clone(), key1, value1).unwrap(); + insert_storage_map_value(db, account_id, block1, slot_name.clone(), key2, value2).unwrap(); // Block 2: key2 -> value3 (update), key3 -> value3 (new) - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block2, - slot_name.clone(), - key2, - value3, - ) - .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block2, - slot_name.clone(), - key3, - value3, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block2, slot_name.clone(), key2, value3).unwrap(); + insert_storage_map_value(db, account_id, block2, slot_name.clone(), key3, value3).unwrap(); // Block 3: key1 -> value2 (update) - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block3, - slot_name.clone(), - key1, - value2, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block3, slot_name.clone(), key1, value2).unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::from(2)..=BlockNumber::from(3), 1024, @@ -1281,28 +1438,19 @@ fn select_storage_map_sync_values() { #[test] fn select_storage_map_sync_values_for_network_account() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); - let (account_id, _) = - make_account_and_note(&mut conn, block_num, [42u8; 32], AccountType::Public); + let (account_id, _) = make_account_and_note(db, block_num, [42u8; 32], AccountType::Public); let slot_name = StorageSlotName::mock(7); let key = StorageMapKey::from_index(1); let value = num_to_word(10); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - slot_name.clone(), - key, - value, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block_num, slot_name.clone(), key, value).unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -1318,7 +1466,7 @@ fn select_storage_map_sync_values_for_network_account() { #[test] fn select_storage_map_sync_values_paginates_until_last_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(7); @@ -1326,34 +1474,34 @@ fn select_storage_map_sync_values_paginates_until_last_block() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block1, slot_name.clone(), @@ -1361,8 +1509,8 @@ fn select_storage_map_sync_values_paginates_until_last_block() { num_to_word(11), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block2, slot_name.clone(), @@ -1370,8 +1518,8 @@ fn select_storage_map_sync_values_paginates_until_last_block() { num_to_word(22), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block3, slot_name.clone(), @@ -1380,13 +1528,9 @@ fn select_storage_map_sync_values_paginates_until_last_block() { ) .unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - BlockNumber::GENESIS..=block3, - 1, - ) - .unwrap(); + let page = + select_account_storage_map_values_paged(db, account_id, BlockNumber::GENESIS..=block3, 1) + .unwrap(); assert_eq!(page.last_block_included, block1, "should truncate at block 1"); assert_eq!(page.values.len(), 1, "should include block 1 only"); @@ -1397,25 +1541,25 @@ fn select_storage_map_sync_values_paginates_until_last_block() { /// `last_block_num.saturating_sub(1) = -1` which failed `BlockNumber::from_raw_sql`. #[test] fn select_storage_map_sync_values_all_entries_in_genesis_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(8); let genesis = BlockNumber::GENESIS; - create_block(&mut conn, genesis); + create_block(db, genesis); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], genesis, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // Insert 3 entries, all in genesis block for i in 0..3 { - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, genesis, slot_name.clone(), @@ -1428,12 +1572,7 @@ fn select_storage_map_sync_values_all_entries_in_genesis_block() { // Query with limit=1 so that raw.len() (3) > limit (1), triggering the pagination branch. All // entries are in block 0, so take_while produces nothing and last_block_num.saturating_sub(1) = // -1. - let result = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - genesis..=genesis, - 1, - ); + let result = select_account_storage_map_values_paged(db, account_id, genesis..=genesis, 1); // Should not error - should return a valid page (possibly with empty values indicating no // progress, which the caller interprets as limit_exceeded) @@ -1450,24 +1589,24 @@ fn select_storage_map_sync_values_all_entries_in_genesis_block() { /// data. #[test] fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(10); let block5 = BlockNumber::from(5); - create_block(&mut conn, block5); + create_block(db, block5); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block5, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); for i in 0..3 { - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block5, slot_name.clone(), @@ -1478,9 +1617,7 @@ fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { } // limit=1, so 3 rows > 1 triggers pagination. All in block 5. - let page = - queries::select_account_storage_map_values_paged(&mut conn, account_id, block5..=block5, 1) - .unwrap(); + let page = select_account_storage_map_values_paged(db, account_id, block5..=block5, 1).unwrap(); assert!(page.values.is_empty(), "should have no values when single block exceeds limit"); assert_eq!(page.last_block_included, block5, "should signal no progress at block 5"); @@ -1490,7 +1627,7 @@ fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { /// limit causing block 3 to be dropped. #[test] fn select_storage_map_sync_values_multi_block_pagination() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(11); @@ -1498,35 +1635,35 @@ fn select_storage_map_sync_values_multi_block_pagination() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // 1 entry in block 1, 1 in block 2, 1 in block 3 - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block1, slot_name.clone(), @@ -1534,8 +1671,8 @@ fn select_storage_map_sync_values_multi_block_pagination() { num_to_word(11), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block2, slot_name.clone(), @@ -1543,8 +1680,8 @@ fn select_storage_map_sync_values_multi_block_pagination() { num_to_word(22), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_storage_map_value( + db, account_id, block3, slot_name.clone(), @@ -1554,13 +1691,9 @@ fn select_storage_map_sync_values_multi_block_pagination() { .unwrap(); // limit=2: query fetches 3 rows (limit+1), drops block 3, keeps blocks 1-2 - let page = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - BlockNumber::GENESIS..=block3, - 2, - ) - .unwrap(); + let page = + select_account_storage_map_values_paged(db, account_id, BlockNumber::GENESIS..=block3, 2) + .unwrap(); assert_eq!(page.values.len(), 2, "should include entries from blocks 1 and 2"); assert_eq!(page.last_block_included, block2, "last included block should be 2"); @@ -1582,60 +1715,33 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { crate::db::migrations::bootstrap_database(&db_path).unwrap(); let db = crate::db::Db::load(db_path).await.unwrap(); let slot_name_for_db = slot_name.clone(); - db.query("insert paged values", move |db_conn| { - db_conn.transaction(|db_conn| { - create_block(db_conn, block1); - create_block(db_conn, block2); - create_block(db_conn, block3); - - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 0)], - block1, - &queries::PrecomputedPublicAccountStates::new(), - )?; - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 1)], - block2, - &queries::PrecomputedPublicAccountStates::new(), - )?; - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 2)], - block3, - &queries::PrecomputedPublicAccountStates::new(), - )?; + db.writer() + .write::<_, DatabaseError, _>("insert paged values", move |tx| { + for block in [block1, block2, block3] { + create_block_in(tx, block)?; + } - queries::insert_account_storage_map_value( - db_conn, - account_id, - block1, - slot_name_for_db.clone(), - num_to_storage_map_key(1), - num_to_word(10), - )?; - queries::insert_account_storage_map_value( - db_conn, - account_id, - block2, - slot_name_for_db.clone(), - num_to_storage_map_key(2), - num_to_word(20), - )?; - queries::insert_account_storage_map_value( - db_conn, - account_id, - block3, - slot_name_for_db.clone(), - num_to_storage_map_key(3), - num_to_word(30), - )?; - Ok::<_, DatabaseError>(()) + for (index, block) in [block1, block2, block3].into_iter().enumerate() { + queries::upsert_accounts( + tx, + &[mock_block_account_update(account_id, index as u64)], + block, + &PrecomputedPublicAccountStates::new(), + )?; + let entry = (index + 1) as u64; + queries::insert_storage_map_value( + tx, + account_id, + block, + &slot_name_for_db, + num_to_storage_map_key(entry), + num_to_word(entry * 10), + )?; + } + Ok(()) }) - }) - .await - .unwrap(); + .await + .unwrap(); let details = db .reconstruct_storage_map_from_db( @@ -1670,33 +1776,32 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block crate::db::migrations::bootstrap_database(&db_path).unwrap(); let db = crate::db::Db::load(db_path).await.unwrap(); let slot_name_for_db = slot_name.clone(); - db.query("insert entries in single block", move |db_conn| { - db_conn.transaction(|db_conn| { - create_block(db_conn, block5); + db.writer() + .write::<_, DatabaseError, _>("insert entries in single block", move |tx| { + create_block_in(tx, block5)?; queries::upsert_accounts( - db_conn, + tx, &[mock_block_account_update(account_id, 0)], block5, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), )?; // Insert 3 entries, all in the same block for i in 1..=3 { - queries::insert_account_storage_map_value( - db_conn, + queries::insert_storage_map_value( + tx, account_id, block5, - slot_name_for_db.clone(), + &slot_name_for_db, num_to_storage_map_key(i), num_to_word(i * 10), )?; } - Ok::<_, DatabaseError>(()) + Ok(()) }) - }) - .await - .unwrap(); + .await + .unwrap(); // Use limit=1 so that 3 entries in a single block exceed the limit. block_range_start is block5 // (the first block with data), and the target is also block5. @@ -1836,33 +1941,25 @@ fn mock_block_transaction_with_output_notes( ) } -fn insert_transactions(conn: &mut SqliteConnection) -> usize { +/// Inserts an account and two transactions against it at block 1, returning the rows written. +fn insert_mock_transactions(db: &TestDb) -> usize { let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); - conn.transaction(|conn| { - let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - let account_updates = vec![mock_block_account_update(account_id, 1)]; + let account_updates = vec![mock_block_account_update(account_id, 1)]; - let mock_tx1 = - mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 1); - let mock_tx2 = - mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 2); - let ordered_tx_headers = OrderedTransactionHeaders::new_unchecked(vec![mock_tx1, mock_tx2]); + let mock_tx1 = + mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 1); + let mock_tx2 = + mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 2); + let ordered_tx_headers = OrderedTransactionHeaders::new_unchecked(vec![mock_tx1, mock_tx2]); - queries::upsert_accounts( - conn, - &account_updates, - block_num, - &queries::PrecomputedPublicAccountStates::new(), - ) + upsert_accounts(db, &account_updates, block_num, &PrecomputedPublicAccountStates::new()) .unwrap(); - let count = queries::insert_transactions(conn, block_num, &ordered_tx_headers).unwrap(); - Ok::<_, DatabaseError>(count) - }) - .unwrap() + insert_transactions(db, block_num, &ordered_tx_headers).unwrap() } fn mock_account_code_and_storage( @@ -1914,12 +2011,12 @@ fn mock_account_code_and_storage( #[test] fn test_select_account_code_by_commitment() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num_1 = BlockNumber::from(1); // Create block 1 - create_block(&mut conn, block_num_1); + create_block(db, block_num_1); // Create an account with code at block 1 using the existing mock function let account = mock_account_code_and_storage(AccountType::Public, [], None); @@ -1929,8 +2026,8 @@ fn test_select_account_code_by_commitment() { let expected_code = account.code().to_bytes(); // Insert the account at block 1 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -1942,7 +2039,7 @@ fn test_select_account_code_by_commitment() { .unwrap(); // Query code by commitment - should return the code - let code = queries::select_account_code_by_commitment(&mut conn, code_commitment) + let code = select_account_code_by_commitment(db, code_commitment) .unwrap() .expect("Code should exist"); assert_eq!(code, expected_code); @@ -1950,21 +2047,20 @@ fn test_select_account_code_by_commitment() { // Query code for non-existent commitment - should return None let non_existent_commitment = [0u8; 32]; let non_existent_commitment = Word::read_from_bytes(&non_existent_commitment).unwrap(); - let code_other = - queries::select_account_code_by_commitment(&mut conn, non_existent_commitment).unwrap(); + let code_other = select_account_code_by_commitment(db, non_existent_commitment).unwrap(); assert!(code_other.is_none(), "Code should not exist for non-existent commitment"); } #[test] fn test_select_account_code_by_commitment_multiple_codes() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num_1 = BlockNumber::from(1); let block_num_2 = BlockNumber::from(2); // Create blocks - create_block(&mut conn, block_num_1); - create_block(&mut conn, block_num_2); + create_block(db, block_num_1); + create_block(db, block_num_2); // Create account with code v1 at block 1 let code_v1_str = "\ @@ -1979,8 +2075,8 @@ fn test_select_account_code_by_commitment_multiple_codes() { let code_v1 = account_v1.code().to_bytes(); // Insert the account at block 1 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account_v1.id(), account_v1.to_commitment(), @@ -2014,8 +2110,8 @@ fn test_select_account_code_by_commitment_multiple_codes() { ); // Insert the updated account at block 2 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account_v2.id(), account_v2.to_commitment(), @@ -2027,16 +2123,14 @@ fn test_select_account_code_by_commitment_multiple_codes() { .unwrap(); // Both codes should be retrievable by their respective commitments - let code_from_v1_commitment = - queries::select_account_code_by_commitment(&mut conn, code_v1_commitment) - .unwrap() - .expect("v1 code should exist"); + let code_from_v1_commitment = select_account_code_by_commitment(db, code_v1_commitment) + .unwrap() + .expect("v1 code should exist"); assert_eq!(code_from_v1_commitment, code_v1, "v1 commitment should return v1 code"); - let code_from_v2_commitment = - queries::select_account_code_by_commitment(&mut conn, code_v2_commitment) - .unwrap() - .expect("v2 code should exist"); + let code_from_v2_commitment = select_account_code_by_commitment(db, code_v2_commitment) + .unwrap() + .expect("v2 code should exist"); assert_eq!(code_from_v2_commitment, code_v2, "v2 commitment should return v2 code"); } @@ -2086,7 +2180,7 @@ async fn genesis_with_account_assets() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with account containing storage maps can be inserted. @@ -2158,7 +2252,7 @@ async fn genesis_with_account_storage_map() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with account containing both vault assets and storage maps. @@ -2223,7 +2317,7 @@ async fn genesis_with_account_assets_and_storage() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with multiple accounts of different types. Tests realistic genesis @@ -2324,15 +2418,15 @@ async fn genesis_with_multiple_accounts() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } #[test] #[miden_node_test_macro::enable_logging] fn regression_1461_full_state_delta_inserts_vault_assets() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num: BlockNumber = 1.into(); - create_block(&mut conn, block_num); + create_block(db, block_num); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let fungible_asset = FungibleAsset::new(faucet_id, 5000).unwrap(); @@ -2354,20 +2448,11 @@ fn regression_1461_full_state_delta_inserts_vault_assets() { AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); - let (_, vault_assets) = queries::select_account_vault_assets( - &mut conn, - account_id, - BlockNumber::GENESIS..=block_num, - ) - .unwrap(); + let (_, vault_assets) = + select_account_vault_assets(db, account_id, BlockNumber::GENESIS..=block_num).unwrap(); // Before the fix, vault_assets was empty let vault_asset = vault_assets.first().unwrap(); @@ -2504,7 +2589,7 @@ fn serialization_symmetry_note_id_vec() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_block_header() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -2524,13 +2609,12 @@ fn db_roundtrip_block_header() { // Insert let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - queries::insert_block_header(&mut conn, &block_header, &dummy_signature).unwrap(); + insert_block_header(db, &block_header, &dummy_signature).unwrap(); // Retrieve - let retrieved = - queries::select_block_header_by_block_num(&mut conn, Some(block_header.block_num())) - .unwrap() - .expect("Block header should exist"); + let retrieved = select_block_header_by_block_num(db, Some(block_header.block_num())) + .unwrap() + .expect("Block header should exist"); assert_eq!(block_header, retrieved, "BlockHeader DB roundtrip must be symmetric"); } @@ -2538,17 +2622,17 @@ fn db_roundtrip_block_header() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_nullifiers() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let nullifiers: Vec = (0..5).map(|i| num_to_nullifier(i << 48)).collect(); // Insert - queries::insert_nullifiers_for_block(&mut conn, &nullifiers, block_num).unwrap(); + insert_nullifiers_for_block(db, &nullifiers, block_num).unwrap(); // Retrieve - let retrieved = queries::select_all_nullifiers(&mut conn).unwrap(); + let retrieved = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers.len(), retrieved.len(), "Should retrieve same number of nullifiers"); for (orig, info) in nullifiers.iter().zip(retrieved.iter()) { @@ -2560,9 +2644,9 @@ fn db_roundtrip_nullifiers() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_account() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let account = mock_account_code_and_storage(AccountType::Public, [], Some([99u8; 32])); let account_id = account.id(); @@ -2575,16 +2659,11 @@ fn db_roundtrip_account() { account_commitment, AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); // Retrieve - let retrieved = queries::select_all_accounts(&mut conn).unwrap(); + let retrieved = select_all_accounts(db).unwrap(); assert_eq!(retrieved.len(), 1, "Should have one account"); let retrieved_info = &retrieved[0]; @@ -2602,16 +2681,16 @@ fn db_roundtrip_account() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_notes() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -2629,12 +2708,12 @@ fn db_roundtrip_notes() { }; // Insert - queries::insert_scripts(&mut conn, [¬e]).unwrap(); - queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // Retrieve let note_ids = vec![NoteId::from_raw(note.note_id)]; - let retrieved = queries::select_notes_by_id(&mut conn, ¬e_ids).unwrap(); + let retrieved = select_notes_by_id(db, ¬e_ids).unwrap(); assert_eq!(retrieved.len(), 1, "Should have one note"); let retrieved_note = &retrieved[0]; @@ -2657,19 +2736,19 @@ fn db_roundtrip_notes() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_vault_assets() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); // Create account first - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -2678,16 +2757,11 @@ fn db_roundtrip_vault_assets() { let vault_key = asset.id(); // Insert vault asset - queries::insert_account_vault_asset(&mut conn, account_id, block_num, vault_key, Some(asset)) - .unwrap(); + insert_vault_asset(db, account_id, block_num, vault_key, Some(asset)).unwrap(); // Retrieve - let (_, vault_assets) = queries::select_account_vault_assets( - &mut conn, - account_id, - BlockNumber::GENESIS..=block_num, - ) - .unwrap(); + let (_, vault_assets) = + select_account_vault_assets(db, account_id, BlockNumber::GENESIS..=block_num).unwrap(); assert_eq!(vault_assets.len(), 1, "Should have one vault asset"); let retrieved = &vault_assets[0]; @@ -2700,44 +2774,36 @@ fn db_roundtrip_vault_assets() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_storage_map_values() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); let slot_name = StorageSlotName::mock(5); let key = StorageMapKey::from_index(12345u32); let value = num_to_word(67890); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // Insert - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - slot_name.clone(), - key, - value, - ) - .unwrap(); + insert_storage_map_value(db, account_id, block_num, slot_name.clone(), key, value).unwrap(); // Retrieve - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -2758,9 +2824,9 @@ fn db_roundtrip_storage_map_values() { fn db_roundtrip_account_storage_with_maps() { use miden_protocol::account::StorageMap; - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); // Create storage with both value slots and map slots let storage_map = StorageMap::with_entries(vec![ @@ -2823,17 +2889,11 @@ fn db_roundtrip_account_storage_with_maps() { account.to_commitment(), AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); - // Retrieve the storage using select_latest_account_storage (reconstructs from header + map - // values) - let retrieved_storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + // Retrieve the storage using select_latest_storage (reconstructs from header + map values) + let retrieved_storage = select_latest_storage(db, account_id).unwrap(); let retrieved_commitment = retrieved_storage.to_commitment(); // Verify the commitment matches (this proves the reconstruction is correct) @@ -2873,7 +2933,7 @@ fn db_roundtrip_account_storage_with_maps() { } // Also verify full account reconstruction via select_account (which calls select_full_account) - let account_info = queries::select_account(&mut conn, account_id).unwrap(); + let account_info = select_account(db, account_id).unwrap(); assert!(account_info.details.is_some(), "Public account should have details"); let retrieved_account = account_info.details.unwrap(); assert_eq!( @@ -2886,12 +2946,11 @@ fn db_roundtrip_account_storage_with_maps() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_note_metadata_attachment() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); - let (account_id, _) = - make_account_and_note(&mut conn, block_num, [1u8; 32], AccountType::Public); + let (account_id, _) = make_account_and_note(db, block_num, [1u8; 32], AccountType::Public); let target = NetworkAccountTarget::new(account_id, NoteExecutionHint::Always) .expect("NetworkAccountTarget creation should succeed for network account"); @@ -2912,11 +2971,11 @@ fn db_roundtrip_note_metadata_attachment() { inclusion_path: SparseMerklePath::default(), }; - queries::insert_scripts(&mut conn, [¬e]).unwrap(); - queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // Fetch the note back and verify the attachment is preserved - let retrieved = queries::select_notes_by_id(&mut conn, &[NoteId::from_raw(note.note_id)]) + let retrieved = select_notes_by_id(db, &[NoteId::from_raw(note.note_id)]) .expect("select_notes_by_id should succeed"); assert_eq!(retrieved.len(), 1, "Should retrieve exactly one note"); @@ -2937,8 +2996,8 @@ fn db_roundtrip_note_metadata_attachment() { // Note sync uses a narrower record than `select_notes_by_id`, but it must retain attachments so // the RPC layer can expose single-word values. - let synced = queries::select_notes_since_block_by_tag( - &mut conn, + let synced = select_notes_since_block_by_tag( + db, &[metadata.tag().as_u32()], BlockNumber::GENESIS..=block_num, ) @@ -2950,8 +3009,7 @@ fn db_roundtrip_note_metadata_attachment() { #[test] #[miden_node_test_macro::enable_logging] fn test_prune_history() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let public_account_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); @@ -2968,16 +3026,16 @@ fn test_prune_history() { let block_tip: BlockNumber = (HISTORICAL_BLOCK_RETENTION + CUTOFF_BLOCK_OFFSET).into(); for block in [block_0, block_old, block_cutoff, block_update, block_tip] { - create_block(conn, block); + create_block(db, block); } // Create account for block in [block_0, block_old, block_cutoff, block_update, block_tip] { - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(public_account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } @@ -2995,56 +3053,23 @@ fn test_prune_history() { // Stale entry at block_0, superseded at block_old which is also below the cutoff — should be // deleted. let stale_asset = Asset::Fungible(FungibleAsset::new(public_account_id, 500).unwrap()); - queries::insert_account_vault_asset( - conn, - public_account_id, - block_0, - vault_key_old, - Some(stale_asset), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_0, vault_key_old, Some(stale_asset)).unwrap(); // Entry at block_old, superseded only at block_update which is above the cutoff — must be // retained as the key's baseline for reads at block_cutoff. - queries::insert_account_vault_asset( - conn, - public_account_id, - block_old, - vault_key_old, - Some(asset_1), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_old, vault_key_old, Some(asset_1)).unwrap(); // Entry exactly at cutoff (block_cutoff, should be retained) - queries::insert_account_vault_asset( - conn, - public_account_id, - block_cutoff, - vault_key_cutoff, - Some(asset_2), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_cutoff, vault_key_cutoff, Some(asset_2)) + .unwrap(); // Recent entry (should always be retained) - queries::insert_account_vault_asset( - conn, - public_account_id, - block_tip, - vault_key_recent, - Some(asset_3), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_tip, vault_key_recent, Some(asset_3)).unwrap(); // Update an entry to create a non-latest version let updated_asset = Asset::Fungible(FungibleAsset::new(public_account_id, 1500).unwrap()); - queries::insert_account_vault_asset( - conn, - public_account_id, - block_update, - vault_key_old, - Some(updated_asset), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_update, vault_key_old, Some(updated_asset)) + .unwrap(); // Insert storage map values at different blocks let slot_name = StorageSlotName::mock(5); @@ -3059,8 +3084,8 @@ fn test_prune_history() { // Stale entry at block_0, superseded at block_old which is also below the cutoff — should be // deleted. - insert_account_storage_map_value( - conn, + insert_storage_map_value( + db, public_account_id, block_0, slot_name.clone(), @@ -3071,8 +3096,8 @@ fn test_prune_history() { // Entry at block_old, superseded only at block_update which is above the cutoff — must be // retained as the key's baseline for reads at block_cutoff. - insert_account_storage_map_value( - conn, + insert_storage_map_value( + db, public_account_id, block_old, slot_name.clone(), @@ -3082,8 +3107,8 @@ fn test_prune_history() { .unwrap(); // Storage map entry at cutoff boundary (block_cutoff) - insert_account_storage_map_value( - conn, + insert_storage_map_value( + db, public_account_id, block_cutoff, slot_name.clone(), @@ -3093,8 +3118,8 @@ fn test_prune_history() { .unwrap(); // Recent storage map entry - insert_account_storage_map_value( - conn, + insert_storage_map_value( + db, public_account_id, block_tip, slot_name.clone(), @@ -3104,8 +3129,8 @@ fn test_prune_history() { .unwrap(); // Update map_key_old to create a non-latest entry at block_update - insert_account_storage_map_value( - conn, + insert_storage_map_value( + db, public_account_id, block_update, slot_name.clone(), @@ -3116,16 +3141,12 @@ fn test_prune_history() { // Verify initial state - should have 5 vault assets and 5 storage map values let (_, initial_vault_assets) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert_eq!(initial_vault_assets.len(), 5, "should have 5 vault assets before cleanup"); - let initial_storage_values = queries::select_account_storage_map_values_paged( - conn, - public_account_id, - block_0..=block_tip, - 1024, - ) - .unwrap(); + let initial_storage_values = + select_account_storage_map_values_paged(db, public_account_id, block_0..=block_tip, 1024) + .unwrap(); assert_eq!( initial_storage_values.values.len(), 5, @@ -3134,8 +3155,7 @@ fn test_prune_history() { // Run cleanup with chain_tip = block_tip, cutoff will be block_tip - HISTORICAL_BLOCK_RETENTION // = block_cutoff - let (vault_deleted, storage_deleted, _codes_deleted) = - queries::prune_history(conn, block_tip).unwrap(); + let (vault_deleted, storage_deleted, _codes_deleted) = prune_history(db, block_tip).unwrap(); // Only the block_0 rows are deletable: they are superseded at block_old, which is also below // the cutoff. The block_old rows are superseded only above the cutoff, so they remain the @@ -3145,7 +3165,7 @@ fn test_prune_history() { // Verify remaining vault assets - should have 4 (baseline at block_old, cutoff, update, tip) let (_, remaining_vault_assets) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert_eq!(remaining_vault_assets.len(), 4, "should have 4 vault assets after cleanup"); // Verify no vault asset at block_0 remains @@ -3174,13 +3194,9 @@ fn test_prune_history() { // Verify remaining storage map values - should have 4 (baseline at block_old, cutoff, update, // tip) - let remaining_storage_values = queries::select_account_storage_map_values_paged( - conn, - public_account_id, - block_0..=block_tip, - 1024, - ) - .unwrap(); + let remaining_storage_values = + select_account_storage_map_values_paged(db, public_account_id, block_0..=block_tip, 1024) + .unwrap(); assert_eq!( remaining_storage_values.values.len(), 4, @@ -3213,8 +3229,7 @@ fn test_prune_history() { // Regression check for baseline loss: reconstructing the vault at the cutoff block must still // see block_old's value, even though that row is older than the cutoff. - let assets_at_cutoff = - queries::select_account_vault_at_block(conn, public_account_id, block_cutoff).unwrap(); + let assets_at_cutoff = select_vault_at_block(db, public_account_id, block_cutoff).unwrap(); assert!( assets_at_cutoff.contains(&asset_1), "vault reconstruction at the cutoff must include the baseline written at block_old" @@ -3225,24 +3240,18 @@ fn test_prune_history() { let faucet_4 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_3).unwrap(); let asset_old = Asset::Fungible(FungibleAsset::new(faucet_4, 9999).unwrap()); let vault_key_old_latest = asset_old.id(); - queries::insert_account_vault_asset( - conn, - public_account_id, - block_0, - vault_key_old_latest, - Some(asset_old), - ) - .unwrap(); + insert_vault_asset(db, public_account_id, block_0, vault_key_old_latest, Some(asset_old)) + .unwrap(); // This entry at block 0 keeps an open validity interval. Run cleanup again - let (vault_deleted_2, ..) = queries::prune_history(conn, block_tip).unwrap(); + let (vault_deleted_2, ..) = prune_history(db, block_tip).unwrap(); // The old open-ended entry should not be deleted (vault_deleted_2 should be 0) assert_eq!(vault_deleted_2, 0, "should not delete any open-ended entries"); // Verify the old open-ended entry still exists let (_, vault_assets_with_latest) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert!( vault_assets_with_latest .iter() @@ -3263,13 +3272,13 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { /// Reconstructs storage map root from DB entries at a specific block. fn reconstruct_storage_map_root_from_db( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, slot_name: &StorageSlotName, block_num: BlockNumber, ) -> Option { - let storage_values = queries::select_account_storage_map_values_paged( - conn, + let storage_values = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -3315,7 +3324,7 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { Some(smt.root()) } - let mut conn = create_db(); + let db = &TestDb::new(); let mut forest = AccountStateForest::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); @@ -3323,29 +3332,29 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3378,12 +3387,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block1, &patch_1); + insert_account_patch(db, account_id, block1, &patch_1); forest.update_account(block1, &patch_1); // Verify forest matches DB for block 1 let forest_root_1 = forest.get_storage_map_root(account_id, &slot_map, block1).unwrap(); - let db_root_1 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block1) + let db_root_1 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block1) .expect("DB should have storage map root"); assert_eq!( @@ -3411,12 +3420,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block2, &patch_2); + insert_account_patch(db, account_id, block2, &patch_2); forest.update_account(block2, &patch_2); // Verify forest matches DB for block 2 let forest_root_2 = forest.get_storage_map_root(account_id, &slot_map, block2).unwrap(); - let db_root_2 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block2) + let db_root_2 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block2) .expect("DB should have storage map root"); assert_eq!( @@ -3444,12 +3453,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block3, &patch_3); + insert_account_patch(db, account_id, block3, &patch_3); forest.update_account(block3, &patch_3); // Verify forest matches DB for block 3 let forest_root_3 = forest.get_storage_map_root(account_id, &slot_map, block3).unwrap(); - let db_root_3 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block3) + let db_root_3 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block3) .expect("DB should have storage map root"); assert_eq!( @@ -3459,9 +3468,8 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { // Verify we can query historical roots let forest_root_1_check = forest.get_storage_map_root(account_id, &slot_map, block1).unwrap(); - let db_root_1_check = - reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block1) - .expect("DB should have storage map root"); + let db_root_1_check = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block1) + .expect("DB should have storage map root"); assert_eq!( forest_root_1_check, db_root_1_check, "Historical query for block 1 should match" @@ -3772,16 +3780,16 @@ fn account_state_forest_preserves_most_recent_vault_only() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_transactions() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3806,12 +3814,11 @@ fn db_roundtrip_transactions() { ) }) .collect(); - queries::insert_notes(&mut conn, &output_notes).unwrap(); - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_notes(db, &output_notes).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); let expected_sync_records: Vec<_> = tx @@ -3843,16 +3850,16 @@ fn db_roundtrip_transactions() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3861,11 +3868,10 @@ fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { // Notes erased within the same block are not inserted into the `notes` table, so transaction // sync should classify them as erased instead of failing the whole request. - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); let expected = TransactionRecord { @@ -3885,16 +3891,16 @@ fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { #[test] #[miden_node_test_macro::enable_logging] fn select_transactions_records_resolves_consumed_public_note_refs() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3917,12 +3923,11 @@ fn select_transactions_records_resolves_consumed_public_note_refs() { attachments: NoteAttachments::default(), inclusion_path: SparseMerklePath::default(), }; - queries::insert_notes(&mut conn, &[(note_record, Some(nullifier))]).unwrap(); - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_notes(db, &[(note_record, Some(nullifier))]).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); assert_eq!(record.consumed_note_refs, vec![(nullifier, note_id)]); @@ -3938,25 +3943,25 @@ const OUTPUT_NOTE_SIZE_BYTES: usize = 700; /// every transaction after the one that did not fit. #[test] fn select_transactions_records_reports_truncation_below_payload_cap() { - let mut conn = create_db(); + let db = &TestDb::new(); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block1 = BlockNumber::from(1); let block2 = BlockNumber::from(2); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - queries::upsert_accounts( - &mut conn, + create_block(db, block1); + create_block(db, block2); + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3969,22 +3974,12 @@ fn select_transactions_records_reports_truncation_below_payload_cap() { let tx1 = mock_block_transaction_with_output_notes(bob, 1, block1_notes); let tx2 = mock_block_transaction_with_output_notes(bob, 2, block2_notes); - queries::insert_transactions( - &mut conn, - block1, - &OrderedTransactionHeaders::new_unchecked(vec![tx1.clone()]), - ) - .unwrap(); - queries::insert_transactions( - &mut conn, - block2, - &OrderedTransactionHeaders::new_unchecked(vec![tx2]), - ) - .unwrap(); + insert_transactions(db, block1, &OrderedTransactionHeaders::new_unchecked(vec![tx1.clone()])) + .unwrap(); + insert_transactions(db, block2, &OrderedTransactionHeaders::new_unchecked(vec![tx2])).unwrap(); let (last_block_included, records) = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block2) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block2).unwrap(); assert_eq!(last_block_included, block1, "cursor must point at the last complete block"); assert_eq!(records.len(), 1, "only the complete block's transaction should be returned"); @@ -3996,31 +3991,25 @@ fn select_transactions_records_reports_truncation_below_payload_cap() { /// the query must surface an explicit error instead. #[test] fn select_transactions_records_errors_when_single_block_exceeds_payload_cap() { - let mut conn = create_db(); + let db = &TestDb::new(); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block1 = BlockNumber::from(1); - create_block(&mut conn, block1); - queries::upsert_accounts( - &mut conn, + create_block(db, block1); + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); let cap = miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; let oversized_notes = cap / OUTPUT_NOTE_SIZE_BYTES + 100; let tx = mock_block_transaction_with_output_notes(bob, 1, oversized_notes); - queries::insert_transactions( - &mut conn, - block1, - &OrderedTransactionHeaders::new_unchecked(vec![tx]), - ) - .unwrap(); + insert_transactions(db, block1, &OrderedTransactionHeaders::new_unchecked(vec![tx])).unwrap(); - let result = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block1); + let result = select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block1); assert_matches!( result, diff --git a/crates/store/src/db/utils.rs b/crates/store/src/db/utils.rs new file mode 100644 index 0000000000..0bb7d89887 --- /dev/null +++ b/crates/store/src/db/utils.rs @@ -0,0 +1,9 @@ +//! Small conversion helpers shared by the store's queries. + +use miden_protocol::note::Nullifier; + +/// Returns the high 16 bits of the provided nullifier. +pub fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { + // The shift leaves exactly the 16 bits the prefix is defined as. + (nullifier.most_significant_felt().as_canonical_u64() >> 48) as u16 +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ec40d3fe3b..c2c0c0cc29 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -65,18 +65,17 @@ pub fn default_sqlite_connection_pool_size() -> std::num::NonZeroUsize { /// This module is hidden from public docs and not part of the stable API. It exists so /// integration tests in sibling crates (e.g. `miden-node-rpc`) can seed network-account /// rows directly into the store's SQLite database without us widening the visibility of -/// internal diesel types. +/// the internal query layer. #[doc(hidden)] pub mod test_support { use std::path::Path; - use diesel::prelude::*; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; - use crate::db::models::queries::{AccountRowInsert, NetworkAccountType}; - use crate::db::schema; + use crate::db::queries::{AccountRow, NetworkAccountType}; + use crate::errors::DatabaseError; /// Opens a fresh connection to the store's SQLite database and inserts a private /// network-account row for `account_id`, marking it as a network account in the @@ -85,20 +84,22 @@ pub mod test_support { /// Intended for integration tests that need to exercise the network-account gate /// without running a transaction through the block producer. The store's WAL mode /// makes a secondary connection safe. - pub fn seed_network_account(db_path: &Path, account_id: AccountId) { - let mut conn = SqliteConnection::establish(db_path.to_str().expect("db path is utf-8")) - .expect("connect to store sqlite"); + pub async fn seed_network_account(db_path: &Path, account_id: AccountId) { + let (writer, _reader) = + miden_node_db::sqlite::open(db_path).expect("connect to store sqlite"); - let row = AccountRowInsert::new_private( - account_id, - NetworkAccountType::Network, - Word::default(), - BlockNumber::from(0), - BlockNumber::from(0), - ); - diesel::insert_into(schema::accounts::table) - .values(&row) - .execute(&mut conn) + writer + .write::<_, DatabaseError, _>("seed network account", move |tx| { + AccountRow::new_private( + account_id, + NetworkAccountType::Network, + Word::default(), + BlockNumber::from(0), + BlockNumber::from(0), + ) + .upsert(tx) + }) + .await .expect("insert network account row"); } } diff --git a/crates/store/src/state/bootstrap.rs b/crates/store/src/state/bootstrap.rs index ae6cc65e30..26fa47e4c0 100644 --- a/crates/store/src/state/bootstrap.rs +++ b/crates/store/src/state/bootstrap.rs @@ -17,7 +17,7 @@ impl State { name = "store.bootstrap", err, )] - pub fn bootstrap(genesis: GenesisBlock, data_directory: &Path) -> anyhow::Result<()> { + pub async fn bootstrap(genesis: GenesisBlock, data_directory: &Path) -> anyhow::Result<()> { let data_directory = DataDirectory::load(data_directory.to_path_buf()).with_context(|| { format!("failed to load data directory at {}", data_directory.display()) @@ -32,7 +32,7 @@ impl State { tracing::debug!(target: LOG_TARGET, path=%block_store.display(), "Block store created"); let database_filepath = data_directory.database_path(); - Db::bootstrap(database_filepath.clone(), genesis).with_context(|| { + Db::bootstrap(database_filepath.clone(), genesis).await.with_context(|| { format!("failed to bootstrap database at {}", database_filepath.display()) })?; tracing::debug!(target: LOG_TARGET, path=%database_filepath.display(), "Database created"); diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index 04df83d628..151a8c60cc 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -39,8 +39,7 @@ use crate::COMPONENT; #[cfg(feature = "rocksdb")] use crate::LOG_TARGET; use crate::account_state_forest::AccountStateForest; -use crate::db::Db; -use crate::db::models::queries::BlockHeaderCommitment; +use crate::db::{BlockHeaderCommitment, Db}; use crate::errors::{DatabaseError, StateInitializationError}; // CONSTANTS @@ -734,7 +733,6 @@ fn verify_account_state_forest_record( #[cfg(test)] mod tests { - use diesel::{ExpressionMethods, RunQueryDsl}; use miden_protocol::account::{ AccountId, AccountStorageHeader, @@ -746,7 +744,6 @@ mod tests { use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::crypto::merkle::mmr::Mmr; use miden_protocol::testing::account_id::ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE; - use miden_protocol::utils::serde::Serializable; use super::*; @@ -811,27 +808,27 @@ mod tests { let signing_key = SigningKey::new(); let db = crate::db::Db::load(db_path).await.expect("test database should load"); - db.query("insert corrupted block headers", move |conn| { - for header in &headers { - let signatures = miden_protocol::block::BlockSignatures::new(vec![ - signing_key.sign(header.commitment()), - ]) - .expect("one signature is within bounds"); - crate::db::models::queries::insert_block_header(conn, header, &signatures)?; - } - - diesel::update(crate::db::schema::block_headers::table) - .filter(crate::db::schema::block_headers::block_num.eq(2_i64)) - .set( - crate::db::schema::block_headers::commitment - .eq(Word::from([42, 0, 0, 0u32]).to_bytes()), - ) - .execute(conn)?; - - Ok::<_, DatabaseError>(()) - }) - .await - .expect("test block headers should be inserted"); + db.writer() + .write::<_, DatabaseError, _>("insert corrupted block headers", move |tx| { + for header in &headers { + let signatures = miden_protocol::block::BlockSignatures::new(vec![ + signing_key.sign(header.commitment()), + ]) + .expect("one signature is within bounds"); + crate::db::queries::insert_block_header(tx, header, &signatures)?; + } + + // Corrupt the stored commitment of one header so it disagrees with the header it + // was stored alongside. + tx.execute( + "UPDATE block_headers SET commitment = ?1 WHERE block_num = ?2", + &[&Word::from([42, 0, 0, 0u32]), &BlockNumber::from(2)], + )?; + + Ok(()) + }) + .await + .expect("test block headers should be inserted"); let error = load_mmr(&db) .await diff --git a/crates/store/src/state/view/account/mod.rs b/crates/store/src/state/view/account/mod.rs index 342cd6a46e..eddb280fac 100644 --- a/crates/store/src/state/view/account/mod.rs +++ b/crates/store/src/state/view/account/mod.rs @@ -136,7 +136,7 @@ impl StateView { account_id: AccountId, block_num: ScopedBlockNum, ) -> Result { - let assets = self.db.select_account_vault_at_block(account_id, block_num).await?; + let assets = self.db.select_vault_at_block(account_id, block_num).await?; if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { return Ok(AccountVaultDetails::LimitExceeded); @@ -407,6 +407,6 @@ impl StateView { &self, account_ids: &[AccountId], ) -> Result, DatabaseError> { - self.db.select_network_accounts_subset(account_ids.to_vec()).await + self.db.filter_network_accounts(account_ids.to_vec()).await } }