diff --git a/.github/workflows/run-nethtest.yml b/.github/workflows/run-nethtest.yml index 0f10f317c6fd..3720fd608c35 100644 --- a/.github/workflows/run-nethtest.yml +++ b/.github/workflows/run-nethtest.yml @@ -30,7 +30,7 @@ on: zkevm_version: description: zkEVM fixtures release tag (zkevmTest only), or latest / latest- required: false - default: tests-zkevm@v0.6.2 + default: tests-zkevm@v0.8.2 type: string filter: description: Regex filter for test names @@ -76,7 +76,7 @@ on: zkevm_version: description: zkEVM fixtures release tag (zkevmTest only), or latest / latest- required: false - default: tests-zkevm@v0.6.2 + default: tests-zkevm@v0.8.2 type: string filter: description: Regex filter for test names (optional) @@ -106,9 +106,9 @@ env: EEST_VERSION: ${{ inputs.eest_version || 'tests-glamsterdam-devnet@v8.1.0' }} EEST_ARCHIVE: fixtures_glamsterdam-devnet.tar.gz # zkEVM stateless-preview fixtures ship in their own release line of the same repo; - # zkevmTest jobs resolve this instead of EEST_VERSION. Pinned to v0.6.2, which this - # branch's witness roots match (dispatch with zkevm_version to test other releases). - ZKEVM_VERSION: ${{ inputs.zkevm_version || 'tests-zkevm@v0.6.2' }} + # zkevmTest jobs resolve this instead of EEST_VERSION. Pinned to the release whose stateless + # input/output schemas this branch implements. + ZKEVM_VERSION: ${{ inputs.zkevm_version || 'tests-zkevm@v0.8.2' }} # ubuntu-latest runners have 16 GB of RAM. The runner ships with server GC # (throughput-oriented, collects lazily), which previously ballooned past the # runner limit on the full fixture set. Workstation GC keeps the heap tight, diff --git a/.github/workflows/stateless-tests.yml b/.github/workflows/stateless-tests.yml index 2af64960f0a6..67ce5b62120d 100644 --- a/.github/workflows/stateless-tests.yml +++ b/.github/workflows/stateless-tests.yml @@ -47,32 +47,32 @@ jobs: matrix: include: - input: 25526356.ssz - hash: 5822695902784ea16c68f3becbf1fc53650cd2289a28d6da0e60a8ebddeba05e - output: 0c19e14cfaca03f19c7f7ffe3b12063ce1b18d933446038ebf244e26934221e5 + hash: 93dcf5b1b46c811371e26e99f975bb2e4e6294717abc2d050320839bd855233c + output: 018cc34e1eb14c42412dc26aeca8e7e1bf540f216753c7431568985b66ecca1f - input: 25532382.ssz - hash: b16c95ce92f518bbdbc62f331457d936f46442befd6aececc8cbe726e52b5935 - output: d07954628423c7185ea21c81b41ec4678c4dd51c86347397ddb3f8beb7be3905 + hash: 1706a8838f8cbb36f78bfdca3d83f8c9779a06dc0e50541172f099b9107c6a2a + output: d38ffa0643d60c76a25c4ab4ffeeb40d99f2dbedf240b3854b23a9d50233a833 - input: 25532442.ssz - hash: 26896435247a3e69c5917ea623aa8908951922e7acfb54b1e13f6628adcadc8b - output: 491ccd71fe841cc88a08d8ae99b6d088c3358b95abca37cfca6faeea3947b846 + hash: d226fd9a75388fa455e305da180cf6637f4663b9bb64f56a92800da97e8bcb0c + output: a88d5bd033c480258a436756fc8949313faf557deb9dc5dbe912ae7865d5d1f5 - input: 25532454.ssz - hash: f5012e7e8324709b4c415f78bcc2542cca6db045f1b9a7856c59a8cd4c422c43 - output: 43a91651e029b7f09f8f10a014db4afc497ebf09ad6b32756b66ff29cac46ef3 + hash: d75dc30142d089ccdcc226d61e1a4f496f329fc71dd188d87fec9857ef483fa7 + output: efbd3959d821214a3e6f8342548a0574e6eca68958c2ddae579921b2c6ccdc59 - input: 25532471.ssz - hash: a0225bbbdb2a7b12e68ac1d1e8bcaabce297c13ac0237fd8d67b69678bc3bff3 - output: 1b1e03db92e8ba35fdb356206548ceb124bf8e8b9d8d0bbd769c8eedd725ce64 + hash: 315dce051c8509c204c68d99c3ea9790e1369aa1e29838428c7cc9a86caff498 + output: 03a85934149babe2bb198c8f9aed24abbdad3eb1e959ff62249a20e0798aaef6 - input: 25532480.ssz - hash: ec1c24bdb687c95b5e4c53ca064995ee8be22467f59fe36d2dea052dd6412d57 - output: ab8b8c21e69e87f9dc7d766b6d04829e99e64f1e3f9b838208b1a7480b1b55c3 + hash: e80386d5dc016a4b63bc6c24bf89ab636995c335dfcfdbea4df3554251c341d4 + output: 0bdd918dd79095dd95e5c68657ed418dee6043e4e4a1d5d5974d4824e24be8f7 - input: 25532553.ssz - hash: 4fc6a2e66562523bb3b757d5d0691cafabd1b24d237a51b78988bb63c507f223 - output: 2fc28f0d034d94aeebaa168f3bb080726f64b4d9edf6864f379527145c5b1c21 + hash: 7c2ec0703bb1fba9760d9f9f9ff0c0409bf9bd52f3b3c0d166cf2f55f0f5ff45 + output: 9ffd288195959be5fc9bb414b8cd83968a1a653234e25811693950afcfcda880 - input: 25532642.ssz - hash: c4255a01122dd7ff5d4bcb3d466d4bf3b714088483505ed8e2390797e17ee62e - output: 495a565c9aef290d3fe1f4f9605105eeb67cd8b42941de82a40e4f02644380a0 + hash: c3d82df9b4a68c0348ffb003fd5e8d5e3d45de7fcda5b7f6e4e4fdc64a33c2db + output: 0a7fe08924c2e0feaeccfdf7541f80333cfe687f8be7c73bb4fbc88c27db6159 - input: 25532711.ssz - hash: 1bb5815b03b8d2b47dc4081a8c11fdf763c4ae6414f91c19f7a52d7eb0bf0f8f - output: 49f1cd340014da0d021f3a2d00853ff7591dc0cacd5ecab174e863d6fc60ef0c + hash: 19e76df2e2c4439e1dbec2ab075560d4e17817c2194deb119097ec1fae28331d + output: adf8830ca4fc0fc93d4e0968d84a6a4007081b67bbe6de68d3d6c177f2e812dd steps: - name: Check out repository uses: actions/checkout@v6 @@ -86,7 +86,7 @@ jobs: - name: Download input working-directory: src/Nethermind/Nethermind.Stateless.ZiskGuest/bin run: | - curl -fsSL "https://us-southeast-1.linodeobjects.com/zktests/20260714/${{ matrix.input }}" \ + curl -fsSL "https://us-southeast-1.linodeobjects.com/zktests/20260819/${{ matrix.input }}" \ -o "${{ matrix.input }}" echo "${{ matrix.hash }} ${{ matrix.input }}" | sha256sum --status -c - || \ { echo "::error::Checksum failed"; exit 1; } @@ -96,7 +96,7 @@ jobs: working-directory: src/Nethermind/Nethermind.Stateless.ZiskGuest run: | set -euo pipefail - suffix=012500000001000000000000000c00000004000000080000000800000057b05d6900000000 + suffix=0101000000000000000100 make run INPUT="${{ matrix.input }}" 2>&1 | tee "$RUNNER_TEMP/run.log" grep -q "${{ matrix.output }}$suffix" "$RUNNER_TEMP/run.log" || \ { echo "::error::Output mismatch"; exit 1; } diff --git a/Directory.Packages.props b/Directory.Packages.props index d16f0ab8f815..70fc83ad5fc5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -66,6 +66,7 @@ + @@ -84,7 +85,6 @@ - diff --git a/nuget.config b/nuget.config index 57eee4de79a2..005344647065 100644 --- a/nuget.config +++ b/nuget.config @@ -18,6 +18,7 @@ + diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Ethereum.Blockchain.Pyspec.Test.csproj b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Ethereum.Blockchain.Pyspec.Test.csproj index fb25c5779093..c2a28b1a499a 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Ethereum.Blockchain.Pyspec.Test.csproj +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Ethereum.Blockchain.Pyspec.Test.csproj @@ -4,6 +4,6 @@ - + diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/Constants.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/Constants.cs index df1ea5bb7adf..fb6a5807b9ca 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/Constants.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/Constants.cs @@ -7,6 +7,6 @@ namespace Ethereum.Blockchain.Pyspec.Test.ZkEvmFixtures; // ARCHIVE_URL_TEMPLATE already points there, so only the version/name need overriding. public static class Constants { - public const string ArchiveVersion = "tests-zkevm@v0.6.2"; + public const string ArchiveVersion = "tests-zkevm@v0.8.2"; public const string ArchiveName = "fixtures_zkevm.tar.gz"; } diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/ZkEvmBlockchainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/ZkEvmBlockchainTests.cs index 9bb37180dd73..6a4e0dfaf9b6 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/ZkEvmBlockchainTests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ZkEvmFixtures/ZkEvmBlockchainTests.cs @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -extern alias stateless; - using System; using System.Buffers.Binary; using System.Collections.Generic; @@ -15,7 +13,6 @@ using Nethermind.Core.ExecutionRequest; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; -using Nethermind.Int256; using Nethermind.Serialization.Ssz; using Nethermind.Specs; using Nethermind.Specs.Forks; @@ -23,9 +20,6 @@ using Nethermind.Stateless.Execution; using Nethermind.Stateless.Execution.IO; using NUnit.Framework; -using StatelessExecutionPayloadV1 = stateless::Nethermind.Merge.Plugin.SszRest.SszExecutionPayloadV1; -using StatelessExecutionPayloadV3 = stateless::Nethermind.Merge.Plugin.SszRest.SszExecutionPayloadV3; -using StatelessExecutionPayloadV4 = stateless::Nethermind.Merge.Plugin.SszRest.SszExecutionPayloadV4; namespace Ethereum.Blockchain.Pyspec.Test.ZkEvmFixtures; @@ -89,29 +83,73 @@ private static IEnumerable LoadStatelessTests() [TestFixture] public class StatelessSchemaTests { + private const ulong ChainId = BlockchainIds.Mainnet; + + // Past every Mainnet fork activation, so the current-fork schema resolves to the newest known rules + private const ulong BlockNumber = 30_000_000; + private const ulong Timestamp = 2_000_000_000; + + [TestCase(InputDecoder.CurrentForkSchemaId)] + [TestCase(InputDecoder.AmsterdamSchemaId)] + public void Revision_1_schema_roundtrips(ushort schemaId) + { + byte[] encoded = schemaId == InputDecoder.AmsterdamSchemaId + ? EncodeInput(new SszExecutionPayloadAmsterdam(), schemaId) + : EncodeInput(new SszExecutionPayload(), schemaId); + + StatelessPayload payload = InputDecoder.Decode(encoded); + + using (Assert.EnterMultipleScope()) + { + Assert.That(payload.SchemaId, Is.EqualTo(schemaId)); + Assert.That(payload.ChainId, Is.EqualTo(ChainId)); + Assert.That(payload.GetBlock().Header.RequestsHash, Is.EqualTo(ExecutionRequestExtensions.EmptyRequestsHash)); + } + } + + /// + /// Block reconstruction must stay out of : a throw before + /// is published reports the zero sentinel. + /// + [Test] + public void Decoding_defers_block_reconstruction() + { + byte[] encoded = EncodeInput(new SszExecutionPayload(), InputDecoder.CurrentForkSchemaId, MalformedTransaction); + + StatelessPayload payload = InputDecoder.Decode(encoded); + + Assert.That(payload.GetBlock, Throws.InvalidOperationException); + } + + [Test] + public void Malformed_transaction_rlp_reports_the_decoded_metadata() + { + byte[] encoded = EncodeInput(new SszExecutionPayload(), InputDecoder.CurrentForkSchemaId, MalformedTransaction); + byte[] expected = StatelessValidationResult.Encode(new StatelessValidationResult + { + NewPayloadRequestRoot = InputDecoder.Decode(encoded).NewPayloadRequestRoot, + IsSuccess = false, + ChainId = ChainId, + SchemaId = InputDecoder.CurrentForkSchemaId + }); + + Assert.That(StatelessExecutor.Execute(encoded), Is.EqualTo(expected)); + } + [TestCase(ProtocolFork.Cancun)] [TestCase(ProtocolFork.Prague)] [TestCase(ProtocolFork.Osaka)] [TestCase(ProtocolFork.BPO1)] [TestCase(ProtocolFork.BPO2)] [TestCase(ProtocolFork.Amsterdam)] - public void Revision_1_schema_roundtrips(ProtocolFork fork) + public void Fork_name_roundtrips(ProtocolFork fork) { - byte[] encoded = fork == ProtocolFork.Amsterdam - ? EncodeInput(fork) - : EncodeInput(fork); - - StatelessPayload payload = InputDecoder.Decode(encoded); bool foundByName = ProtocolForkExtensions.TryGetByName(fork.GetName(), out ProtocolFork forkByName); - Hash256 expectedRequestsHash = fork >= ProtocolFork.Prague ? ExecutionRequestExtensions.EmptyRequestsHash : null; using (Assert.EnterMultipleScope()) { - Assert.That(payload.ProtocolFork, Is.EqualTo(fork)); - Assert.That(payload.ChainConfig.ChainId, Is.EqualTo(1)); Assert.That(foundByName, Is.True); Assert.That(forkByName, Is.EqualTo(fork)); - Assert.That(payload.Block.Header.RequestsHash, Is.EqualTo(expectedRequestsHash)); } } @@ -154,8 +192,11 @@ public void Schema_prefix_must_be_two_bytes(int length) Assert.That(() => InputDecoder.Decode(encoded), Throws.TypeOf()); } - [TestCase(0x0f01)] - [TestCase(0x1002)] + [TestCase(0x0000)] + [TestCase(0x0002)] + [TestCase(0x1001)] + [TestCase(0x1401)] + [TestCase(0x1502)] [TestCase(0x1601)] public void Unsupported_schema_id_is_rejected(int schemaId) { @@ -167,62 +208,112 @@ public void Unsupported_schema_id_is_rejected(int schemaId) Throws.TypeOf().With.Message.Contains($"0x{schemaId:x4}")); } - [TestCase(10UL, 20UL, true)] - [TestCase(9UL, 20UL, false)] - [TestCase(10UL, 19UL, false)] - public void Every_fork_activation_bound_must_be_active(ulong blockNumber, ulong timestamp, bool expected) - { - SszForkActivation activation = new() { BlockNumber = [10], Timestamp = [20] }; - - Assert.That(activation.IsActive(CreateHeader(blockNumber, timestamp)), Is.EqualTo(expected)); - } - - [Test] - public void Fork_activation_requires_at_least_one_bound() - { - SszForkActivation activation = new() { BlockNumber = [], Timestamp = [] }; - - Assert.That(() => activation.IsActive(CreateHeader(10, 20)), Throws.TypeOf()); - } - [TestCase(BlockchainIds.Sepolia, false)] [TestCase(BlockchainIds.Gnosis, true)] [TestCase(BlockchainIds.Chiado, true)] public void Amsterdam_schema_uses_chain_appropriate_fork_catalog(ulong chainId, bool usesGnosisRules) + { + ForkActivation activation = new(1, 20); + ISpecProvider provider = StatelessSpecProvider.Create(chainId, ProtocolFork.Amsterdam, activation); + IReleaseSpec spec = provider.GetSpec(activation); + + using (Assert.EnterMultipleScope()) + { + Assert.That(provider.ChainId, Is.EqualTo(chainId)); + Assert.That(spec.Name, Is.EqualTo(Amsterdam.Instance.Name)); + Assert.That(spec, usesGnosisRules ? Is.SameAs(AmsterdamGnosis.Instance) : Is.SameAs(Amsterdam.Instance)); + } + } + + [TestCase(BlockchainIds.Mainnet)] + [TestCase(BlockchainIds.Sepolia)] + [TestCase(BlockchainIds.Gnosis)] + public void Current_fork_schema_takes_the_rules_from_the_chain_schedule(ulong chainId) { IForkAwareSpecProvider baseProvider = chainId switch { + BlockchainIds.Mainnet => MainnetSpecProvider.Instance, BlockchainIds.Sepolia => SepoliaSpecProvider.Instance, BlockchainIds.Gnosis => GnosisSpecProvider.Instance, - BlockchainIds.Chiado => ChiadoSpecProvider.Instance, _ => throw new AssertionException($"Unsupported test chain: {chainId}") }; - ForkConfig forkConfig = new() - { - Activation = new SszForkActivation { BlockNumber = [], Timestamp = [20] } - }; - - ISpecProvider provider = StatelessSpecProvider.Create(baseProvider, chainId, forkConfig, ProtocolFork.Amsterdam); - IReleaseSpec spec = provider.GetSpec(new ForkActivation(1, 20)); + ForkActivation activation = new(BlockNumber, Timestamp); + ISpecProvider provider = StatelessSpecProvider.Create(chainId, ProtocolFork.Current, activation); using (Assert.EnterMultipleScope()) { Assert.That(provider.ChainId, Is.EqualTo(chainId)); - Assert.That(spec.Name, Is.EqualTo(Amsterdam.Instance.Name)); - Assert.That(spec, usesGnosisRules ? Is.SameAs(AmsterdamGnosis.Instance) : Is.SameAs(Amsterdam.Instance)); + Assert.That(provider.GetSpec(activation).Name, Is.EqualTo(baseProvider.GetSpec(activation).Name)); } } - private static byte[] EncodeInput(ProtocolFork fork) - where TExecutionPayload : StatelessExecutionPayloadV1, - stateless::Nethermind.Merge.Plugin.SszRest.ISszExecutionPayloadFactory, - ISszCodec, new() + [TestCaseSource(nameof(BlobVersionedHashCases))] + public bool Blob_versioned_hashes_must_match_in_payload_order(Transaction[] transactions, Hash256[] expected) => + StatelessExecutor.BlobVersionedHashesMatch(transactions, expected); + + private static IEnumerable BlobVersionedHashCases() + { + yield return new TestCaseData(new[] { BlobTx(1, 2), BlobTx(3) }, Hashes(1, 2, 3)) + .Returns(true).SetName("Matching hashes in payload order"); + yield return new TestCaseData(new[] { new Transaction(), BlobTx(1), new Transaction() }, Hashes(1)) + .Returns(true).SetName("Non-blob transactions are skipped"); + yield return new TestCaseData(new[] { new Transaction() }, Hashes()) + .Returns(true).SetName("No blob transactions and no hashes"); + yield return new TestCaseData(new[] { BlobTx(1, 2), BlobTx(3) }, Hashes(1, 3, 2)) + .Returns(false).SetName("Hashes out of payload order"); + yield return new TestCaseData(new[] { BlobTx(1, 2) }, Hashes(1)) + .Returns(false).SetName("Fewer hashes than the payload commits to"); + yield return new TestCaseData(new[] { BlobTx(1) }, Hashes(1, 2)) + .Returns(false).SetName("More hashes than the payload commits to"); + } + + private static Transaction BlobTx(params byte[] ids) { + byte[][] hashes = new byte[ids.Length][]; + + for (int i = 0; i < ids.Length; i++) + hashes[i] = HashBytes(ids[i]); + + return new Transaction { BlobVersionedHashes = hashes }; + } + + private static Hash256[] Hashes(params byte[] ids) + { + Hash256[] hashes = new Hash256[ids.Length]; + + for (int i = 0; i < ids.Length; i++) + hashes[i] = new Hash256(HashBytes(ids[i])); + + return hashes; + } + + private static byte[] HashBytes(byte id) + { + byte[] bytes = new byte[Hash256.Size]; + + bytes[^1] = id; + + return bytes; + } + + /// Transaction payload that is well-formed SSZ but not decodable as transaction RLP. + private static SszProgressiveBytes[] MalformedTransaction => [new() { Bytes = [0xff, 0xff] }]; + + private static byte[] EncodeInput( + TExecutionPayload executionPayload, ushort schemaId, SszProgressiveBytes[] transactions = null) + where TExecutionPayload : SszExecutionPayload, ISszCodec, new() + { + executionPayload.BlockNumber = BlockNumber; + executionPayload.Timestamp = Timestamp; + + if (transactions is not null) + executionPayload.Transactions = transactions; + StatelessInput input = new() { NewPayloadRequest = new() { - ExecutionPayload = new TExecutionPayload(), + ExecutionPayload = executionPayload, VersionedHashes = [], ParentBeaconBlockRoot = Hash256.Zero, ExecutionRequests = new() @@ -240,36 +331,15 @@ private static byte[] EncodeInput(ProtocolFork fork) Codes = [], Headers = [] }, - ChainConfig = new() - { - ChainId = 1, - ActiveFork = new() - { - Activation = new() - { - BlockNumber = [0], - Timestamp = [] - } - } - }, + ChainId = ChainId, PublicKeys = [] }; byte[] payload = StatelessInput.Encode(input); byte[] encoded = new byte[sizeof(ushort) + payload.Length]; - BinaryPrimitives.WriteUInt16BigEndian(encoded, fork.ToRevision1SchemaId()); + BinaryPrimitives.WriteUInt16BigEndian(encoded, schemaId); payload.AsSpan().CopyTo(encoded.AsSpan(sizeof(ushort))); return encoded; } - - private static BlockHeader CreateHeader(ulong blockNumber, ulong timestamp) => new( - Hash256.Zero, - Hash256.Zero, - Address.Zero, - UInt256.Zero, - blockNumber, - 0, - timestamp, - []); } diff --git a/src/Nethermind/Ethereum.Ssz.Test/SszConsensusTestLoader.cs b/src/Nethermind/Ethereum.Ssz.Test/SszConsensusTestLoader.cs index cc58702146ba..2ecbbf1d8326 100644 --- a/src/Nethermind/Ethereum.Ssz.Test/SszConsensusTestLoader.cs +++ b/src/Nethermind/Ethereum.Ssz.Test/SszConsensusTestLoader.cs @@ -13,7 +13,8 @@ namespace Ethereum.Ssz.Test; public static class SszConsensusTestLoader { private const string ArchiveUrlTemplate = "https://github.com/ethereum/consensus-specs/releases/download/{0}/{1}"; - private const string DefaultVersion = "v1.6.1"; + // v1.6.1 predates the EIP-7916 change that made the base subtree the left child. + private const string DefaultVersion = "v1.7.0-alpha.13"; private const string DefaultArchive = "general.tar.gz"; private static string? s_testsRoot; diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index 088b496a7f4e..519071bfaac5 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -607,6 +607,10 @@ private static readonly (string ExpectedError, string Substring)[] ValidationErr ("BlockException.GAS_USED_OVERFLOW", "ExceededGasLimit:"), ("BlockException.RLP_BLOCK_LIMIT_EXCEEDED", "ExceededBlockSizeLimit: Exceeded block size limit"), ("BlockException.INVALID_DEPOSIT_EVENT_LAYOUT", "DepositsInvalid: Invalid deposit event layout:"), + // A layout that still decodes yields well-formed but wrong requests, so it surfaces as a + // hash mismatch: the requests hash pre-Amsterdam, the BAL hash once EIP-7928 covers the log. + ("BlockException.INVALID_DEPOSIT_EVENT_LAYOUT", "InvalidRequestsHash: Requests hash mismatch in block"), + ("BlockException.INVALID_DEPOSIT_EVENT_LAYOUT", "InvalidBlockLevelAccessListHash:"), ("BlockException.INVALID_BASEFEE_PER_GAS", "InvalidBaseFeePerGas: Does not match calculated"), ("BlockException.INVALID_BLOCK_TIMESTAMP_OLDER_THAN_PARENT", "InvalidTimestamp: Timestamp in header cannot be lower than ancestor"), ("BlockException.INVALID_BLOCK_NUMBER", "InvalidBlockNumber: Block number does not match the parent"), diff --git a/src/Nethermind/Ethereum.Test.Base/TransactionTestBase.cs b/src/Nethermind/Ethereum.Test.Base/TransactionTestBase.cs index 83f4a9c41a61..90045fd2d468 100644 --- a/src/Nethermind/Ethereum.Test.Base/TransactionTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/TransactionTestBase.cs @@ -125,6 +125,7 @@ private static bool ExceptionMatches(string expected, string observed) private static readonly System.Collections.Generic.Dictionary s_exceptionToErrorFragments = new() { + ["TransactionException.NONCE_OVERFLOW"] = ["NonceTooHigh"], ["TransactionException.TYPE_4_EMPTY_AUTHORIZATION_LIST"] = ["EIP-7702 transaction with empty auth list"], ["TransactionException.TYPE_4_INVALID_AUTHORIZATION_FORMAT"] = ["InvalidAuthorityList", .. s_rlpDecodeFragments], ["TransactionException.TYPE_4_INVALID_AUTHORITY_SIGNATURE"] = ["InvalidAuthoritySignature", .. s_rlpDecodeFragments], diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index 3cd8614d2cc6..c021575dabf7 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -8,7 +8,7 @@ using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using RocksDbSharp; +using Nethermind.RocksDbBindings; using IWriteBatch = Nethermind.Core.IWriteBatch; namespace Nethermind.Db.Rocks; @@ -17,7 +17,7 @@ public class ColumnDb : IDb, ISortedKeyValueStore, IMergeableKeyValueStore, IKey { private readonly RocksDb _rocksDb; internal readonly DbOnTheRocks _mainDb; - internal readonly ColumnFamilyHandle _columnFamily; + internal readonly IColumnFamilyHandle _columnFamily; private readonly DisposableLazy? _iteratorManager; private readonly DisposableLazy _seekIteratorManager; @@ -75,7 +75,7 @@ public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags w { get { - ColumnFamilyHandle[] columnFamilies = new ColumnFamilyHandle[keys.Length]; + IColumnFamilyHandle[] columnFamilies = new IColumnFamilyHandle[keys.Length]; Array.Fill(columnFamilies, _columnFamily); return _rocksDb.MultiGet(keys, columnFamilies); } @@ -145,10 +145,12 @@ public void Compact() => public void SetWriteBuffer(long sizeBytes) { - string[] keys = ["write_buffer_size", "max_bytes_for_level_base"]; - string[] values = [sizeBytes.ToString(), (sizeBytes * 4).ToString()]; - Native.Instance.rocksdb_set_options_cf( - _rocksDb.Handle, _columnFamily.Handle, keys.Length, keys, values); + KeyValuePair[] options = + [ + new("write_buffer_size", sizeBytes.ToString()), + new("max_bytes_for_level_base", (sizeBytes * 4).ToString()), + ]; + _rocksDb.SetOptions(_columnFamily, options); } public byte[]? FirstKey diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs index 423ff2f8fea0..ce453ffe1f4b 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnsDb.cs @@ -10,7 +10,7 @@ using Nethermind.Core; using Nethermind.Db.Rocks.Config; using Nethermind.Logging; -using RocksDbSharp; +using Nethermind.RocksDbBindings; using IWriteBatch = Nethermind.Core.IWriteBatch; namespace Nethermind.Db.Rocks; @@ -24,12 +24,12 @@ public class ColumnsDb : DbOnTheRocks, IColumnsDb where T : struct, Enum private volatile T[]? _cachedColumnKeys; private volatile int _cachedMaxOrdinal = -1; - public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IReadOnlyList keys, IntPtr? sharedCache = null) + public ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IReadOnlyList keys, nint? sharedCache = null) : this(basePath, settings, dbConfig, rocksDbConfigFactory, logManager, ResolveKeys(keys), sharedCache) { } - private ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, (IReadOnlyList Keys, IList ColumnNames) keyInfo, IntPtr? sharedCache) + private ColumnsDb(string basePath, DbSettings settings, IDbConfig dbConfig, IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, (IReadOnlyList Keys, IList ColumnNames) keyInfo, nint? sharedCache) : base(basePath, settings, dbConfig, rocksDbConfigFactory, logManager, keyInfo.ColumnNames, sharedCache: sharedCache) { foreach (T key in keyInfo.Keys) @@ -53,11 +53,7 @@ protected override long FetchTotalPropertyValue(string propertyName) long total = 0; foreach (KeyValuePair kv in _columnDbs) { - long value = long.TryParse(_db.GetProperty(propertyName, kv.Value._columnFamily), out long parsedValue) - ? parsedValue - : 0; - - total += value; + total += _db.TryGetIntProperty(propertyName, kv.Value._columnFamily, out ulong value) ? (long)value : 0; } return total; @@ -108,7 +104,7 @@ private static (IReadOnlyList Keys, IList ColumnNames) ResolveKeys(IR return (resolvedKeys, columnNames); } - protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) + protected override void BuildOptions(IRocksDbConfig dbConfig, Options options, nint? sharedCache, IMergeOperator? mergeOperator) { base.BuildOptions(dbConfig, options, sharedCache, mergeOperator); options.SetCreateMissingColumnFamilies(); @@ -124,11 +120,9 @@ protected override void BuildOptions(IRocksDbConfig dbConfig, Options< protected override void ApplyOptions(IDictionary options) { - string[] keys = options.Select, string>(static e => e.Key).ToArray(); - string[] values = options.Select, string>(static e => e.Value).ToArray(); foreach (KeyValuePair cols in _columnDbs) { - _rocksDbNative.rocksdb_set_options_cf(_db.Handle, cols.Value._columnFamily.Handle, keys.Length, keys, values); + _db.SetOptions(cols.Value._columnFamily, options); } base.ApplyOptions(options); } @@ -179,16 +173,15 @@ public ColumnDbSnapshot(ColumnsDb columnsDb, Snapshot snapshot) { _snapshot = snapshot; - // Create two shared ReadOptions for all column readers instead of 2 per reader. - // ReadOptions in RocksDbSharp has a finalizer but no IDisposable — creating many - // short-lived instances causes Gen1/Gen2 GC pressure from finalizer queue buildup. + // Create two shared ReadOptions for all column readers instead of 2 per reader — + // creating many short-lived instances costs a native handle each and, when one is left + // to its finalizer, Gen1/Gen2 GC pressure from finalizer queue buildup. _sharedReadOptions = CreateReadOptions(columnsDb, snapshot); _sharedCacheMissReadOptions = CreateReadOptions(columnsDb, snapshot); _sharedCacheMissReadOptions.SetFillCache(false); // Single shared delegate for GetViewBetween — avoids per-reader closure allocation. - // Note: each GetViewBetween call still creates a new ReadOptions with a finalizer; - // that is pre-existing behavior not addressed by this PR. + // Each call still creates its own ReadOptions, disposed by the returned view. Func readOptionsFactory = () => CreateReadOptions(columnsDb, snapshot); T[] keys = CreateKeyCache(columnsDb); GetCachedMaxOrdinal(columnsDb, keys); @@ -273,9 +266,8 @@ public void Dispose() if (Interlocked.Exchange(ref _disposed, 1) != 0) return; // Explicitly destroy native ReadOptions handles to prevent finalizer queue buildup. - // GC.SuppressFinalize prevents the finalizer from running on already-destroyed handles. - RocksDbReader.DestroyReadOptions(_sharedReadOptions); - RocksDbReader.DestroyReadOptions(_sharedCacheMissReadOptions); + _sharedReadOptions.Dispose(); + _sharedCacheMissReadOptions.Dispose(); _snapshot.Dispose(); } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs index a04c59ddd0b5..810cb748aa01 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/AdjustedRocksdbConfig.cs @@ -1,15 +1,13 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; - namespace Nethermind.Db.Rocks.Config; public class AdjustedRocksdbConfig( IRocksDbConfig baseConfig, string additionalRocksDbOptions, ulong writeBufferSize, - IntPtr? blockCache = null + nint? blockCache = null ) : IRocksDbConfig { public ulong? WriteBufferSize => writeBufferSize; @@ -40,5 +38,5 @@ public class AdjustedRocksdbConfig( public double CompressibilityHint => baseConfig.CompressibilityHint; public FlushOnExitMode FlushOnExit => baseConfig.FlushOnExit; - public IntPtr? BlockCache => blockCache ?? baseConfig.BlockCache; + public nint? BlockCache => blockCache ?? baseConfig.BlockCache; } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs index 3f0bc7db2247..d0fee48608fc 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IRocksDbConfig.cs @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; - namespace Nethermind.Db.Rocks.Config; public interface IRocksDbConfig @@ -21,5 +19,5 @@ public interface IRocksDbConfig bool EnableFileWarmer { get; } double CompressibilityHint { get; } FlushOnExitMode FlushOnExit { get; } - IntPtr? BlockCache { get; } + nint? BlockCache { get; } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index d8e331e8e9d2..7ede34136af5 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -64,7 +64,7 @@ private void EnsureConfigIsAvailable(string propertyName) public bool EnableFileWarmer => ReadConfig(nameof(EnableFileWarmer)); public double CompressibilityHint => ReadConfig(nameof(CompressibilityHint)); public FlushOnExitMode FlushOnExit => ReadConfig(nameof(FlushOnExit)) ?? FlushOnExitMode.WalOnly; - public IntPtr? BlockCache => null; + public nint? BlockCache => null; private T? ReadConfig(string propertyName) => ReadConfig(_dbConfig, propertyName, _reversedPrefixes); diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index adb528bfc1a7..7d2393cb0ea3 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -6,8 +6,6 @@ using System.Buffers.Binary; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Abstractions; using System.Linq; @@ -29,7 +27,7 @@ using Nethermind.Db.Rocks.Config; using Nethermind.Db.Rocks.Statistics; using Nethermind.Logging; -using RocksDbSharp; +using Nethermind.RocksDbBindings; using Testably.Abstractions; using IWriteBatch = Nethermind.Core.IWriteBatch; @@ -43,6 +41,8 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private static readonly ConcurrentDictionary _dbsByPath = new(); + private static readonly FlushOptions _defaultFlushOptions = new(); + private bool _isDisposing; private bool _isDisposed; @@ -69,7 +69,7 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private long _maxThisDbSize; - private IntPtr? _rowCache = null; + private Cache? _rowCache; private readonly DbSettings _settings; @@ -85,8 +85,6 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private readonly IFileSystem _fileSystem; - protected readonly RocksDbSharp.Native _rocksDbNative; - private ITunableDb.TuneType _currentTune = ITunableDb.TuneType.Default; private string CorruptMarkerPath => Path.Join(_fullPath, "corrupt.marker"); @@ -112,15 +110,13 @@ public DbOnTheRocks( IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IList? columnFamilies = null, - Native? rocksDbNative = null, IFileSystem? fileSystem = null, - IntPtr? sharedCache = null) + nint? sharedCache = null) { _logger = logManager.GetClassLogger(); _settings = dbSettings; Name = _settings.DbName; _fileSystem = fileSystem ?? new RealFileSystem(); - _rocksDbNative = rocksDbNative ?? Native.Instance; _rocksDbConfigFactory = rocksDbConfigFactory; _perTableDbConfig = rocksDbConfigFactory.GetForDatabase(Name, null); _db = Init(basePath, dbSettings.DbPath, dbConfig, logManager, columnFamilies, dbSettings.DeleteOnStart, sharedCache); @@ -145,7 +141,7 @@ private RocksDb Open(string path, (DbOptions Options, ColumnFamilies? Families) } private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogManager? logManager, - IList? columnNames = null, bool deleteOnStart = false, IntPtr? sharedCache = null) + IList? columnNames = null, bool deleteOnStart = false, nint? sharedCache = null) { _fullPath = GetFullDbPath(dbPath, basePath); _logger = logManager?.GetClassLogger() ?? default; @@ -186,7 +182,7 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan // ReSharper disable once VirtualMemberCallInConstructor if (_logger.IsDebug) _logger.Debug($"Loading DB {Name,-13} from {_fullPath} with max memory footprint of {_maxThisDbSize / 1000 / 1000,5} MB"); - RocksDb db = _dbsByPath.GetOrAdd(_fullPath, (s, tuple) => Open(s, tuple), (DbOptions, columnFamilies)); + RocksDb db = _dbsByPath.GetOrAdd(_fullPath, Open, (DbOptions, columnFamilies)); if (dbConfig.EnableMetricsUpdater) { @@ -199,7 +195,7 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan foreach (ColumnFamilies.Descriptor columnFamily in columnFamilies) { if (columnFamily.Name == "default") continue; - if (db.TryGetColumnFamily(columnFamily.Name, out ColumnFamilyHandle handle)) + if (db.TryGetColumnFamily(columnFamily.Name, out IColumnFamilyHandle? handle)) { DbMetricsUpdater columnMetricUpdater = new( Name + "_" + columnFamily.Name, columnFamily.Options, db, handle, dbConfig, _isUsingSharedBlockCache, _logger); @@ -226,7 +222,7 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan if (_logger.IsWarn) _logger.Warn("If your database did not close properly you need to call 'find -type f -name '*LOCK*' -delete' from the database folder"); throw; } - catch (RocksDbSharpException x) + catch (RocksDbException x) { HandleFatalDbError(x); throw; @@ -237,9 +233,16 @@ private void WarmupFile(string basePath, RocksDb db) { long availableMemory = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; _logger.Info($"Warming up database {Name} assuming {availableMemory} bytes of available memory"); + List? liveFiles = db.GetLiveFilesMetadata(); + if (liveFiles is null) + { + if (_logger.IsWarn) _logger.Warn($"Unable to read live files metadata of database {Name}. Skipping warmup."); + return; + } + List<(FileMetadata metadata, DateTime creationTime)> fileMetadataEntries = []; - foreach (LiveFileMetadata liveFileMetadata in db.GetLiveFilesMetadata()) + foreach (LiveFileMetadata liveFileMetadata in liveFiles) { string fullPath = Path.Join(basePath, liveFileMetadata.FileMetadata.FileName); try @@ -312,7 +315,7 @@ public int Compare( } } - private void HandleFatalDbError(RocksDbSharpException rocksDbException) + private void HandleFatalDbError(RocksDbException rocksDbException) { bool corruption = rocksDbException.Message.Contains("Corruption:", StringComparison.Ordinal); bool ioError = rocksDbException.Message.Contains("IO error", StringComparison.Ordinal); @@ -355,20 +358,20 @@ private void RepairIfCorrupted(DbOptions dbOptions) } if (_logger.IsWarn) _logger.Warn($"Corrupted DB marker detected for db {_fullPath}. Attempting repair..."); - _rocksDbNative.rocksdb_repair_db(dbOptions.Handle, _fullPath); + RepairDb(dbOptions, _fullPath!); if (_logger.IsWarn) _logger.Warn($"Repair completed. Some data may be lost. Consider a full resync."); _fileSystem.File.Delete(corruptMarker); } + protected virtual void RepairDb(DbOptions dbOptions, string path) => RocksDb.Repair(dbOptions, path); + protected internal void UpdateReadMetrics() => Interlocked.Increment(ref _totalReads.Value); protected internal void UpdateWriteMetrics() => Interlocked.Increment(ref _totalWrites.Value); protected virtual long FetchTotalPropertyValue(string propertyName) => - long.TryParse(_db.GetProperty(propertyName), out long parsedValue) - ? parsedValue - : 0; + _db.TryGetIntProperty(propertyName, out ulong value) ? (long)value : 0; public IDbMeta.DbMetric GatherMetric() { @@ -408,7 +411,7 @@ public long EstimatedCount { return FetchTotalPropertyValue("rocksdb.estimate-num-keys"); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { if (_logger.IsWarn) _logger.Warn($"Failed to read DB key count estimate {e.Message}"); @@ -426,7 +429,7 @@ private long GetSize() long blobSize = FetchTotalPropertyValue("rocksdb.total-blob-file-size"); return sstSize + blobSize; } - catch (RocksDbSharpException e) + catch (RocksDbException e) { if (_logger.IsWarn) _logger.Warn($"Failed to update DB size metrics {e.Message}"); @@ -446,7 +449,7 @@ private long GetCacheSize() } return FetchTotalPropertyValue("rocksdb.block-cache-usage"); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { if (_logger.IsWarn) _logger.Warn($"Failed to update DB size metrics {e.Message}"); @@ -461,7 +464,7 @@ private long GetIndexSize() { return FetchTotalPropertyValue("rocksdb.estimate-table-readers-mem"); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { if (_logger.IsWarn) _logger.Warn($"Failed to update DB size metrics {e.Message}"); @@ -476,7 +479,7 @@ private long GetMemtableSize() { return FetchTotalPropertyValue("rocksdb.cur-size-all-mem-tables"); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { if (_logger.IsWarn) _logger.Warn($"Failed to update DB size metrics {e.Message}"); @@ -533,7 +536,7 @@ public static string NormalizeRocksDbOptions(string dbOptions) return dbOptions; } - protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options options, IntPtr? sharedCache, IMergeOperator? mergeOperator) where T : Options + protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options options, nint? sharedCache, IMergeOperator? mergeOperator) where T : Options { // This section is about the table factory and block cache, apparently. // This affects the format of the SST files and usually requires resyncing to take effect. @@ -576,15 +579,7 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio // changes to the table options must be applied before setting to set. options.SetBlockBasedTableFactory(tableOptions); - IntPtr optsPtr = Marshal.StringToHGlobalAnsi(NormalizeRocksDbOptions(dbConfig.RocksDbOptions)); - try - { - _rocksDbNative.rocksdb_get_options_from_string(options.Handle, optsPtr, options.Handle); - } - finally - { - Marshal.FreeHGlobal(optsPtr); - } + options.ApplyFromString(NormalizeRocksDbOptions(dbConfig.RocksDbOptions)); if (dbConfig.WriteBufferSize > 0) { @@ -634,7 +629,7 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio options.IncreaseParallelism(Environment.ProcessorCount); // VERY important to reduce stalls. Allow L0->L1 compaction to happen with multiple thread. - _rocksDbNative.rocksdb_options_set_max_subcompactions(options.Handle, (uint)Environment.ProcessorCount); + options.SetMaxSubcompactions((uint)Environment.ProcessorCount); #endregion @@ -647,8 +642,8 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio // more CPU efficient. // Note: Memtable also acts like a per-key cache that does not get updated on read. So in some case // maybe it makes more sense to put more memory to memtable. - _rowCache = _rocksDbNative.rocksdb_cache_create_lru(new UIntPtr(dbConfig.RowCacheSize.Value)); - _rocksDbNative.rocksdb_options_set_row_cache(options.Handle, _rowCache.Value); + _rowCache = Cache.CreateLru(dbConfig.RowCacheSize.Value); + options.SetRowCache(_rowCache); } options.SetCreateIfMissing(); @@ -666,15 +661,7 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio if (dbConfig.AdditionalRocksDbOptions is not null) { - optsPtr = Marshal.StringToHGlobalAnsi(NormalizeRocksDbOptions(dbConfig.AdditionalRocksDbOptions)); - try - { - _rocksDbNative.rocksdb_get_options_from_string(options.Handle, optsPtr, options.Handle); - } - finally - { - Marshal.FreeHGlobal(optsPtr); - } + options.ApplyFromString(NormalizeRocksDbOptions(dbConfig.AdditionalRocksDbOptions)); } if (mergeOperator is not null) @@ -691,14 +678,14 @@ protected virtual void BuildOptions(IRocksDbConfig dbConfig, Options optio WriteOptions = CreateWriteOptions(dbConfig); _noWalWrite = CreateWriteOptions(dbConfig); - _noWalWrite.DisableWal(1); + _noWalWrite.SetDisableWal(true); _lowPriorityWriteOptions = CreateWriteOptions(dbConfig); - _rocksDbNative.rocksdb_writeoptions_set_low_pri(_lowPriorityWriteOptions.Handle, 1); + _lowPriorityWriteOptions.SetLowPriority(true); _lowPriorityAndNoWalWrite = CreateWriteOptions(dbConfig); - _lowPriorityAndNoWalWrite.DisableWal(1); - _rocksDbNative.rocksdb_writeoptions_set_low_pri(_lowPriorityAndNoWalWrite.Handle, 1); + _lowPriorityAndNoWalWrite.SetDisableWal(true); + _lowPriorityAndNoWalWrite.SetLowPriority(true); _defaultReadOptions = CreateReadOptions(); @@ -756,7 +743,7 @@ internal ReadOptions CreateReadOptions() void IReadOnlyKeyValueStore.DangerousReleaseMemory(in ReadOnlySpan span) => _reader.DangerousReleaseMemory(span); - internal byte[]? GetWithIterator(ReadOnlySpan key, ColumnFamilyHandle? cf, IteratorManager iteratorManager, ReadFlags flags, out bool success) + internal byte[]? GetWithIterator(ReadOnlySpan key, IteratorManager iteratorManager, ReadFlags flags, out bool success) { ThrowIfDisposing(); @@ -808,57 +795,18 @@ internal static bool TryGetCeilingWithIterator( } /// Pool for calls with - tailing iterators with large read steps. - internal DisposableLazy? CreateLazyReadAheadIteratorManager(ColumnFamilyHandle? cf) => + internal DisposableLazy? CreateLazyReadAheadIteratorManager(IColumnFamilyHandle? cf) => _readAheadReadOptions is null ? null : CreateLazyIteratorManager(cf, _readAheadReadOptions); /// Pool for ceiling seeks - tailing iterators. - internal DisposableLazy CreateLazySeekIteratorManager(ColumnFamilyHandle? cf) => + internal DisposableLazy CreateLazySeekIteratorManager(IColumnFamilyHandle? cf) => CreateLazyIteratorManager(cf, _seekReadOptions); - private DisposableLazy CreateLazyIteratorManager(ColumnFamilyHandle? cf, ReadOptions readOptions) => + private DisposableLazy CreateLazyIteratorManager(IColumnFamilyHandle? cf, ReadOptions readOptions) => new(() => new IteratorManager(_db, cf, readOptions)); - internal unsafe byte[]? Get(ReadOnlySpan key, ColumnFamilyHandle? cf, ReadOptions readOptions) - { - // TODO: update when merged upstream: https://github.com/curiosity-ai/rocksdb-sharp/pull/61 - // return _db.Get(key, cf, (flags & ReadFlags.HintCacheMiss) != 0 ? _hintCacheMissOptions : _defaultReadOptions); - - nint db = _db.Handle; - nint read_options = readOptions.Handle; - UIntPtr skLength = (UIntPtr)key.Length; - IntPtr handle; - IntPtr errPtr; - fixed (byte* ptr = &MemoryMarshal.GetReference(key)) - { - handle = cf is null - ? Native.Instance.rocksdb_get_pinned(db, read_options, ptr, skLength, out errPtr) - : Native.Instance.rocksdb_get_pinned_cf(db, read_options, cf.Handle, ptr, skLength, out errPtr); - } - - if (errPtr != IntPtr.Zero) ThrowRocksDbException(errPtr); - if (handle == IntPtr.Zero) return null; - - try - { - IntPtr valuePtr = Native.Instance.rocksdb_pinnableslice_value(handle, out UIntPtr valueLength); - if (valuePtr == IntPtr.Zero) - { - return null; - } - - int length = (int)valueLength; - byte[] result = new byte[length]; - new ReadOnlySpan((void*)valuePtr, length).CopyTo(new Span(result)); - return result; - } - finally - { - Native.Instance.rocksdb_pinnableslice_destroy(handle); - } - - [DoesNotReturn, StackTraceHidden] - static void ThrowRocksDbException(nint errPtr) => throw new RocksDbException(errPtr); - } + internal byte[]? Get(ReadOnlySpan key, IColumnFamilyHandle? cf, ReadOptions readOptions) => + _db.Get(key, cf, readOptions); /// /// iterator.Next() is about 10 to 20 times faster than iterator.Seek(). @@ -869,7 +817,7 @@ private DisposableLazy CreateLazyIteratorManager(ColumnFamilyHa /// /// /// - private bool TryCloseReadAhead(Iterator iterator, ReadOnlySpan key, out byte[]? result) + private static bool TryCloseReadAhead(Iterator iterator, ReadOnlySpan key, out byte[]? result) { // Probably hash db. Can't really do this with hashdb. Even with batched trie visitor, its going to skip a lot. if (key.Length <= 32) @@ -930,7 +878,7 @@ private bool TryCloseReadAhead(Iterator iterator, ReadOnlySpan key, out by public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) => SetWithColumnFamily(key, null, value, flags); - internal void SetWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? cf, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + internal void SetWithColumnFamily(ReadOnlySpan key, IColumnFamilyHandle? cf, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -947,7 +895,7 @@ internal void SetWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? cf _db.Put(key, value, cf, WriteFlagsToWriteOptions(flags)); } } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -971,7 +919,7 @@ _ when (flags & WriteFlags.LowPriority) == WriteFlags.LowPriority => _lowPriorit { return _db.MultiGet(keys); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -979,7 +927,7 @@ _ when (flags & WriteFlags.LowPriority) == WriteFlags.LowPriority => _lowPriorit } } - internal Span GetSpanWithColumnFamily(scoped ReadOnlySpan key, ColumnFamilyHandle? cf, ReadOptions readOptions) + internal Span GetSpanWithColumnFamily(scoped ReadOnlySpan key, IColumnFamilyHandle? cf, ReadOptions readOptions) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -996,58 +944,21 @@ internal Span GetSpanWithColumnFamily(scoped ReadOnlySpan key, Colum } return span; } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; } } - internal unsafe int GetCStyleWithColumnFamily(scoped ReadOnlySpan key, Span output, ColumnFamilyHandle? cf, ReadOptions readOptions) + internal int GetCStyleWithColumnFamily(scoped ReadOnlySpan key, Span output, IColumnFamilyHandle? cf, ReadOptions readOptions) { ObjectDisposedException.ThrowIf(_isDisposing, this); UpdateReadMetrics(); - nint db = _db.Handle; - nint readOptionsHandle = readOptions.Handle; - UIntPtr skLength = (UIntPtr)key.Length; - IntPtr errPtr; - IntPtr slice; - fixed (byte* ptr = &MemoryMarshal.GetReference(key)) - { - slice = cf is null - ? Native.Instance.rocksdb_get_pinned(db, readOptionsHandle, ptr, skLength, out errPtr) - : Native.Instance.rocksdb_get_pinned_cf(db, readOptionsHandle, cf.Handle, ptr, skLength, out errPtr); - } - - if (errPtr != IntPtr.Zero) ThrowRocksDbException(errPtr); - if (slice == IntPtr.Zero) return 0; - - IntPtr valuePtr = Native.Instance.rocksdb_pinnableslice_value(slice, out UIntPtr valueLength); - if (valuePtr == IntPtr.Zero) - { - Native.Instance.rocksdb_pinnableslice_destroy(slice); - return 0; - } - - int length = (int)valueLength; - if (output.Length < length) - { - Native.Instance.rocksdb_pinnableslice_destroy(slice); - ThrowNotEnoughMemory(length, output.Length); - } - - new ReadOnlySpan((void*)valuePtr, length).CopyTo(output); - Native.Instance.rocksdb_pinnableslice_destroy(slice); - return length; - - [DoesNotReturn, StackTraceHidden] - static void ThrowRocksDbException(nint errPtr) => throw new RocksDbException(errPtr); - - [DoesNotReturn, StackTraceHidden] - static void ThrowNotEnoughMemory(int length, int bufferLength) => - throw new ArgumentException($"Output buffer not large enough. Output size: {length}, Buffer size: {bufferLength}"); + int length = _db.Get(key, output, cf, readOptions); + return length < 0 ? 0 : length; } public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags writeFlags) => @@ -1063,14 +974,14 @@ public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags f { _db.Merge(key, value, null, WriteFlagsToWriteOptions(flags)); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; } } - internal void MergeWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? cf, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) + internal void MergeWithColumnFamily(ReadOnlySpan key, IColumnFamilyHandle? cf, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -1080,7 +991,7 @@ internal void MergeWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? { _db.Merge(key, value, cf, WriteFlagsToWriteOptions(flags)); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1097,58 +1008,25 @@ public void DangerousReleaseMemory(in ReadOnlySpan span) _db.DangerousReleaseMemory(span); } - public ReadOnlySpan GetNativeSlice(scoped ReadOnlySpan key, out IntPtr handle, ReadFlags flags) + public ReadOnlySpan GetNativeSlice(scoped ReadOnlySpan key, out nint handle, ReadFlags flags) => GetNativeSlice(key, null, out handle, flags); - public unsafe ReadOnlySpan GetNativeSlice(scoped ReadOnlySpan key, ColumnFamilyHandle? cf, out IntPtr handle, ReadFlags flags) + public ReadOnlySpan GetNativeSlice(scoped ReadOnlySpan key, IColumnFamilyHandle? cf, out nint handle, ReadFlags flags) { - // TODO: update when merged upstream: https://github.com/curiosity-ai/rocksdb-sharp/pull/61 - // return _db.Get(key, cf, (flags & ReadFlags.HintCacheMiss) != 0 ? _hintCacheMissOptions : _defaultReadOptions); - - handle = default; - nint db = _db.Handle; - nint read_options = ((flags & ReadFlags.HintCacheMiss) != 0 ? _hintCacheMissOptions : _defaultReadOptions).Handle; - UIntPtr skLength = (UIntPtr)key.Length; - IntPtr errPtr; - IntPtr slice; - fixed (byte* ptr = &MemoryMarshal.GetReference(key)) - { - slice = cf is null - ? Native.Instance.rocksdb_get_pinned(db, read_options, ptr, skLength, out errPtr) - : Native.Instance.rocksdb_get_pinned_cf(db, read_options, cf.Handle, ptr, skLength, out errPtr); - } - - if (errPtr != IntPtr.Zero) ThrowRocksDbException(errPtr); - if (slice == IntPtr.Zero) return null; - - try - { - IntPtr valuePtr = Native.Instance.rocksdb_pinnableslice_value(slice, out UIntPtr valueLength); - if (valuePtr == IntPtr.Zero) - { - Native.Instance.rocksdb_pinnableslice_destroy(slice); - return null; - } - - int length = (int)valueLength; - handle = slice; - return new ReadOnlySpan((void*)valuePtr, length); - } - catch + ReadOptions readOptions = (flags & ReadFlags.HintCacheMiss) != 0 ? _hintCacheMissOptions : _defaultReadOptions; + if (!_db.TryGetPinned(key, out PinnedSlice slice, cf, readOptions)) { - Native.Instance.rocksdb_pinnableslice_destroy(slice); - throw; + handle = default; + return null; } - [DoesNotReturn, StackTraceHidden] - static unsafe void ThrowRocksDbException(nint errPtr) => throw new RocksDbException(errPtr); + ReadOnlySpan value = slice.Value; + handle = slice.DangerousDetach(); + return value; } - public void DangerousReleaseHandle(IntPtr handle) - { - if (handle != default) - Native.Instance.rocksdb_pinnableslice_destroy(handle); - } + public void DangerousReleaseHandle(nint handle) => + PinnedSlice.DangerousDestroy(handle); public void Remove(ReadOnlySpan key) { @@ -1158,7 +1036,7 @@ public void Remove(ReadOnlySpan key) { _db.Remove(key, null, WriteOptions); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1173,13 +1051,13 @@ public void Remove(ReadOnlySpan key) return GetAllCore(ordered); } - protected internal Iterator CreateIterator(ReadOptions readOptions, ColumnFamilyHandle? ch = null) + protected internal Iterator CreateIterator(ReadOptions readOptions, IColumnFamilyHandle? ch = null) { try { return _db.NewIterator(ch, readOptions); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1208,7 +1086,7 @@ private void IteratorSeekToFirstWithErrorHandling(Iterator iterator) { iterator.SeekToFirst(); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1221,7 +1099,7 @@ private void IteratorSeekWithErrorHandling(Iterator iterator, byte[] key) { iterator.Seek(key); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1234,7 +1112,7 @@ private void IteratorNextWithErrorHandling(Iterator iterator) { iterator.Next(); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1247,23 +1125,23 @@ private void IteratorDisposeWithErrorHandling(Iterator iterator) { iterator.Dispose(); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; } } - internal IEnumerable> GetAllCore(bool ordered, ColumnFamilyHandle? ch = null) => - GetAllCore(ordered, ch, static iterator => new KeyValuePair(iterator.Key(), iterator.Value())); + internal IEnumerable> GetAllCore(bool ordered, IColumnFamilyHandle? ch = null) => + GetAllCore(ordered, ch, static iterator => new KeyValuePair(iterator.GetKeySpan().ToArray(), iterator.GetValueSpan().ToArray())); - internal IEnumerable GetAllKeysCore(bool ordered, ColumnFamilyHandle? ch = null) => - GetAllCore(ordered, ch, static iterator => iterator.Key()); + internal IEnumerable GetAllKeysCore(bool ordered, IColumnFamilyHandle? ch = null) => + GetAllCore(ordered, ch, static iterator => iterator.GetKeySpan().ToArray()); - internal IEnumerable GetAllValuesCore(bool ordered, ColumnFamilyHandle? ch = null) => - GetAllCore(ordered, ch, static iterator => iterator.Value()); + internal IEnumerable GetAllValuesCore(bool ordered, IColumnFamilyHandle? ch = null) => + GetAllCore(ordered, ch, static iterator => iterator.GetValueSpan().ToArray()); - private IEnumerable GetAllCore(bool ordered, ColumnFamilyHandle? ch, Func projection) + private IEnumerable GetAllCore(bool ordered, IColumnFamilyHandle? ch, Func projection) { byte[]? resumeKey = null; bool hasMore; @@ -1283,7 +1161,7 @@ private IEnumerable GetAllCore(bool ordered, ColumnFamilyHandle? ch, Func< private bool ReadFullEnumerationBatch( bool ordered, - ColumnFamilyHandle? ch, + IColumnFamilyHandle? ch, byte[]? resumeKey, Func projection, ArrayPoolList batch, @@ -1317,7 +1195,7 @@ private bool ReadFullEnumerationBatch( batch.Add(projection(iterator)); if (batch.Count == FullEnumerationBatchSize) { - byte[] boundaryKey = iterator.Key(); + byte[] boundaryKey = iterator.GetKeySpan().ToArray(); IteratorNextWithErrorHandling(iterator); if (iterator.Valid()) { @@ -1344,12 +1222,12 @@ private bool ReadFullEnumerationBatch( } finally { - RocksDbReader.DestroyReadOptions(readOptions); + readOptions.Dispose(); } } } - protected internal bool KeyExistsWithColumn(ReadOnlySpan key, ColumnFamilyHandle? cf) + protected internal bool KeyExistsWithColumn(ReadOnlySpan key, IColumnFamilyHandle? cf) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -1357,7 +1235,7 @@ protected internal bool KeyExistsWithColumn(ReadOnlySpan key, ColumnFamily { return _db.HasKey(key, cf, _defaultReadOptions); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; @@ -1408,7 +1286,7 @@ private static WriteBatch CreateWriteBatch() private static void ReturnWriteBatch(WriteBatch batch) { - Native.Instance.rocksdb_writebatch_data(batch.Handle, out UIntPtr size); + nuint size = batch.DataSize; if (size > (uint)16.KiB || _reusableWriteBatch is not null) { batch.Dispose(); @@ -1442,21 +1320,21 @@ public void Dispose() _dbOnTheRocks._currentBatches.TryRemove(this); ReturnWriteBatch(_rocksBatch); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { _dbOnTheRocks.HandleFatalDbError(e); throw; } } - public void Delete(ReadOnlySpan key, ColumnFamilyHandle? cf = null) + public void Delete(ReadOnlySpan key, IColumnFamilyHandle? cf = null) { ObjectDisposedException.ThrowIf(_isDisposed, this); _rocksBatch.Delete(key, cf); } - public void Set(ReadOnlySpan key, ReadOnlySpan value, ColumnFamilyHandle? cf = null, WriteFlags flags = WriteFlags.None) + public void Set(ReadOnlySpan key, ReadOnlySpan value, IColumnFamilyHandle? cf = null, WriteFlags flags = WriteFlags.None) { ObjectDisposedException.ThrowIf(_isDisposed, this); @@ -1482,7 +1360,7 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags public void Merge(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) => Merge(key, value, null, flags); - public void Merge(ReadOnlySpan key, ReadOnlySpan value, ColumnFamilyHandle? cf = null, WriteFlags flags = WriteFlags.None) + public void Merge(ReadOnlySpan key, ReadOnlySpan value, IColumnFamilyHandle? cf = null, WriteFlags flags = WriteFlags.None) { ObjectDisposedException.ThrowIf(_isDisposed, this); @@ -1503,7 +1381,7 @@ private void FlushOnTooManyWrites() _dbOnTheRocks._db.Write(currentBatch, _dbOnTheRocks.WriteFlagsToWriteOptions(_writeFlags)); ReturnWriteBatch(currentBatch); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { _dbOnTheRocks.HandleFatalDbError(e); throw; @@ -1518,7 +1396,7 @@ public virtual void Flush(bool onlyWal = false) InnerFlush(onlyWal); } - public void FlushWithColumnFamily(ColumnFamilyHandle familyHandle) + public void FlushWithColumnFamily(IColumnFamilyHandle familyHandle) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -1532,27 +1410,29 @@ public virtual void SyncWal() ObjectDisposedException.ThrowIf(_isDisposing, this); try { - _rocksDbNative.rocksdb_flush_wal(_db.Handle, 1); + FlushWal(); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); throw; } } + private void FlushWal() => _db.FlushWal(sync: true); + private void InnerFlush(bool onlyWal) { try { - _rocksDbNative.rocksdb_flush_wal(_db.Handle, 1); + FlushWal(); if (!onlyWal) { - _rocksDbNative.rocksdb_flush(_db.Handle, FlushOptions.DefaultFlushOptions.Handle); + _db.Flush(_defaultFlushOptions); } } - catch (RocksDbSharpException e) + catch (RocksDbException e) { // Fast-shuts down on corruption or IO error; anything else falls through, so log it // rather than swallowing the flush failure silently. @@ -1561,13 +1441,13 @@ private void InnerFlush(bool onlyWal) } } - private void InnerFlush(ColumnFamilyHandle columnFamilyHandle) + private void InnerFlush(IColumnFamilyHandle columnFamilyHandle) { try { - _rocksDbNative.rocksdb_flush_cf(_db.Handle, FlushOptions.DefaultFlushOptions.Handle, columnFamilyHandle.Handle); + _db.Flush(_defaultFlushOptions, columnFamilyHandle); } - catch (RocksDbSharpException e) + catch (RocksDbException e) { HandleFatalDbError(e); if (_logger.IsWarn) _logger.Warn($"Failed to flush {Name} DB: {e.Message}"); @@ -1607,22 +1487,6 @@ private void Delete() } } - private class FlushOptions - { - internal static FlushOptions DefaultFlushOptions { get; } = new(); - - public IntPtr Handle { get; private set; } = Native.Instance.rocksdb_flushoptions_create(); - - ~FlushOptions() - { - if (Handle != IntPtr.Zero) - { - Native.Instance.rocksdb_flushoptions_destroy(Handle); - Handle = IntPtr.Zero; - } - } - } - protected virtual void ReleaseUnmanagedResources() { // ReSharper disable once ConstantConditionalAccessQualifier @@ -1636,15 +1500,12 @@ protected virtual void ReleaseUnmanagedResources() _seekIteratorManager.Dispose(); _db.Dispose(); - RocksDbReader.DestroyReadOptions(_defaultReadOptions); - RocksDbReader.DestroyReadOptions(_hintCacheMissOptions); - RocksDbReader.DestroyReadOptions(_readAheadReadOptions); - RocksDbReader.DestroyReadOptions(_seekReadOptions); + _defaultReadOptions.Dispose(); + _hintCacheMissOptions.Dispose(); + _readAheadReadOptions?.Dispose(); + _seekReadOptions.Dispose(); - if (_rowCache.HasValue) - { - _rocksDbNative.rocksdb_cache_destroy(_rowCache.Value); - } + _rowCache?.Dispose(); } public void Dispose() @@ -1770,9 +1631,9 @@ public virtual void Tune(ITunableDb.TuneType type) protected virtual void ApplyOptions(IDictionary options) => _db.SetOptions(options); - private IDictionary GetStandardOptions() => + private Dictionary GetStandardOptions() => // Defaults are from rocksdb source code - new Dictionary() + new() { { "write_buffer_size", _writeBufferSize.ToString() }, { "max_write_buffer_number", _maxWriteBufferNumber.ToString() }, @@ -1795,8 +1656,8 @@ private IDictionary GetStandardOptions() => { "hard_pending_compaction_bytes_limit", 256.GiB.ToString() }, }; - private IDictionary GetHashDbOptions() => - new Dictionary() + private static Dictionary GetHashDbOptions() => + new() { // Some database config is slightly faster on a hash db database. These are applied when hash db is detected // to prevent unexpected regression. @@ -1818,7 +1679,7 @@ private IDictionary GetHashDbOptions() => /// This caps the maximum allowed number of l0 files, which is also the read response time amplification. /// /// - private IDictionary GetHeavyWriteOptions(ulong l0SizeTarget) + private Dictionary GetHeavyWriteOptions(ulong l0SizeTarget) { // Make buffer (probably) smaller so that it does not take too much memory to have many of them. // More buffer means more parallel flush, but each read has to go through all buffers one by one, much like l0 @@ -1837,7 +1698,7 @@ private IDictionary GetHeavyWriteOptions(ulong l0SizeTarget) ulong l0FileNumTarget = l0SizeTarget / l0FileSize; ulong l1SizeTarget = l0SizeTarget; - return new Dictionary() + return new() { { "write_buffer_size", bufferSize.ToString() }, { "max_write_buffer_number", maxBufferNumber.ToString() }, @@ -1856,9 +1717,9 @@ private IDictionary GetHeavyWriteOptions(ulong l0SizeTarget) }; } - private IDictionary GetDisableCompactionOptions() + private Dictionary GetDisableCompactionOptions() { - IDictionary heavyWriteOption = GetHeavyWriteOptions(32UL.GiB); + Dictionary heavyWriteOption = GetHeavyWriteOptions(32UL.GiB); heavyWriteOption["disable_auto_compactions"] = "true"; // Increase the size of the write buffer, which reduces the number of l0 files by 4x. This does slow down @@ -1871,7 +1732,7 @@ private IDictionary GetDisableCompactionOptions() } - private static IDictionary GetBlobFilesOptions() => + private static Dictionary GetBlobFilesOptions() => // Enable blob files, see: https://rocksdb.org/blog/2021/05/26/integrated-blob-db.html // This is very useful for blocks, as it almost eliminates 95% of the compaction as the main db no longer // store the actual data, but only points to blob files. This config reduces total blocks db writes from about @@ -1886,7 +1747,7 @@ private static IDictionary GetBlobFilesOptions() => // get a lot of compaction. So can't turn this on all the time. Turning this back off will just put back // new data to SST files. - new Dictionary() + new() { { "enable_blob_files", "true" }, { "blob_compression_type", "kSnappyCompression" }, @@ -1913,7 +1774,7 @@ public class IteratorManager : IDisposable private readonly ManagedIterators _readaheadIterators2 = new(); private readonly ManagedIterators _readaheadIterators3 = new(); private readonly RocksDb _rocksDb; - private readonly ColumnFamilyHandle? _cf; + private readonly IColumnFamilyHandle? _cf; private readonly ReadOptions _readOptions; private readonly Timer _timer; @@ -1924,7 +1785,7 @@ public class IteratorManager : IDisposable // This is about once every two second maybe at max throughput. private const int IteratorUsageLimit = 1000000; - public IteratorManager(RocksDb rocksDb, ColumnFamilyHandle? cf, ReadOptions readOptions) + public IteratorManager(RocksDb rocksDb, IColumnFamilyHandle? cf, ReadOptions readOptions) { _rocksDb = rocksDb; _cf = cf; @@ -2071,26 +1932,13 @@ public byte[]? LastKey public ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan lastKey) => GetViewBetween(firstKey, lastKey, null); - internal ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan lastKey, ColumnFamilyHandle? cf) + internal ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan lastKey, IColumnFamilyHandle? cf) { ReadOptions readOptions = CreateReadOptions(); - - IntPtr iterateLowerBound; - IntPtr iterateUpperBound; - - unsafe - { - iterateLowerBound = Marshal.AllocHGlobal(firstKey.Length); - firstKey.CopyTo(new Span(iterateLowerBound.ToPointer(), firstKey.Length)); - Native.Instance.rocksdb_readoptions_set_iterate_lower_bound(readOptions.Handle, iterateLowerBound, (UIntPtr)firstKey.Length); - - iterateUpperBound = Marshal.AllocHGlobal(lastKey.Length); - lastKey.CopyTo(new Span(iterateUpperBound.ToPointer(), lastKey.Length)); - Native.Instance.rocksdb_readoptions_set_iterate_upper_bound(readOptions.Handle, iterateUpperBound, (UIntPtr)lastKey.Length); - } + readOptions.SetIterateBounds(firstKey, lastKey); Iterator iterator = CreateIterator(readOptions, cf); - return new RocksdbSortedView(iterator, readOptions, iterateLowerBound, iterateUpperBound); + return new RocksdbSortedView(iterator, readOptions); } public bool TryGetCeiling( @@ -2120,7 +1968,7 @@ public IKeyValueStoreSnapshot CreateSnapshot() public sealed class RocksDbSnapshot( DbOnTheRocks mainDb, Func readOptionsFactory, - ColumnFamilyHandle? columnFamily, + IColumnFamilyHandle? columnFamily, Snapshot snapshot ) : RocksDbReader(mainDb, readOptionsFactory, null, columnFamily), IKeyValueStoreSnapshot { diff --git a/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs b/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs index 5511b010c374..fdcdf779bc11 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs @@ -3,44 +3,61 @@ using System; using System.Threading; -using Microsoft.Win32.SafeHandles; -using RocksDbSharp; +using Nethermind.Config; +using Nethermind.Core.Exceptions; +using Nethermind.Db.Rocks.Config; +using Nethermind.RocksDbBindings; namespace Nethermind.Db.Rocks; -public class HyperClockCacheWrapper : SafeHandleZeroOrMinusOneIsInvalid +public sealed class HyperClockCacheWrapper : IDisposable { - private static readonly Lock _nativeCacheLock = new(); - + private readonly Cache _cache; private readonly long _capacity; - public HyperClockCacheWrapper(ulong capacity = 32_000_000) : base(ownsHandle: true) + private int _disposed; + + /// The cache capacity in bytes. Must be greater than zero. + /// + /// is zero, which rocksdb cannot allocate a handle table for. + /// + public HyperClockCacheWrapper(ulong capacity = 32_000_000) { - lock (_nativeCacheLock) + // A zero capacity makes rocksdb request a zero-length anonymous mapping for the handle + // table and abort the process ("Anonymous mmap for RocksDB HyperClockCache failed"), + // so reject it here while it can still be reported as the configuration error it is. + if (capacity == 0) { - SetHandle(Native.Instance.rocksdb_cache_create_hyper_clock(new UIntPtr(capacity), 0)); + throw new InvalidConfigurationException( + $"Block cache capacity must be greater than zero. Check Db.{nameof(IDbConfig.SharedBlockCacheSize)} and FlatDb.{nameof(IFlatDbConfig.BlockCacheSizeBudget)}.", + ExitCodes.ForbiddenOptionValue); } - // If the native call returned a zero/null handle, SafeHandle won't call ReleaseHandle, - // so don't add pressure either — keep add/remove balanced. - _capacity = IsInvalid ? 0 : (long)capacity; - if (_capacity > 0) GC.AddMemoryPressure(_capacity); + + _cache = Cache.CreateHyperClock(capacity); + _capacity = (long)capacity; + GC.AddMemoryPressure(_capacity); } - public IntPtr Handle => DangerousGetHandle(); + public nint Handle => _cache.Handle; + + public long GetUsage() => (long)_cache.GetUsage(); + + /// Keeps the reported memory pressure balanced when an owner abandons the wrapper undisposed. + /// The native handle has its own critical finalizer, so only the pressure is released here. + ~HyperClockCacheWrapper() => Release(disposing: false); - protected override bool ReleaseHandle() + public void Dispose() { - lock (_nativeCacheLock) - { - Native.Instance.rocksdb_cache_destroy(handle); - } - if (_capacity > 0) GC.RemoveMemoryPressure(_capacity); - return true; + Release(disposing: true); + GC.SuppressFinalize(this); } - public long GetUsage() + private void Release(bool disposing) { - ObjectDisposedException.ThrowIf(IsClosed, this); - return (long)Native.Instance.rocksdb_cache_get_usage(DangerousGetHandle()); + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; + + // Zero when the constructor rejected the capacity before any pressure was added. + if (_capacity > 0) GC.RemoveMemoryPressure(_capacity); + if (disposing) _cache.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs b/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs index 263865ac4824..12bfe955e21f 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/MergeOperatorAdapter.cs @@ -2,15 +2,15 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Nethermind.Core.Collections; -using RocksDbSharp; + +using INativeMergeOperator = Nethermind.RocksDbBindings.IMergeOperator; namespace Nethermind.Db.Rocks; -// Also see RocksDbSharp.MergeOperatorImpl -internal class MergeOperatorAdapter(IMergeOperator inner) : MergeOperator +// Also see Nethermind.RocksDbBindings.MergeOperators.MergeOperatorImpl +internal class MergeOperatorAdapter(IMergeOperator inner) : INativeMergeOperator { public string Name => inner.Name; @@ -31,10 +31,7 @@ private static unsafe nint GetResult(ArrayPoolList? data, out nint resultL data.AsSpan().CopyTo(result); resultLength = result.Length; - - // Fixing RocksDbSharp invalid callback signature, TODO: submit an issue/PR - Unsafe.SkipInit(out success); - Unsafe.As(ref success) = 1; + success = 1; return (nint)resultPtr; } @@ -60,5 +57,5 @@ public unsafe nint FullMerge(nint key, nuint keyLength, nint existingValue, nuin return GetResult(result, out newValueLength, out success); } - unsafe void MergeOperator.DeleteValue(nint value, nuint valueLength) => NativeMemory.Free((void*)value); + unsafe void INativeMergeOperator.DeleteValue(nint value, nuint valueLength) => NativeMemory.Free((void*)value); } diff --git a/src/Nethermind/Nethermind.Db.Rocks/Nethermind.Db.Rocks.csproj b/src/Nethermind/Nethermind.Db.Rocks/Nethermind.Db.Rocks.csproj index e54aea53e966..925890abc894 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Nethermind.Db.Rocks.csproj +++ b/src/Nethermind/Nethermind.Db.Rocks/Nethermind.Db.Rocks.csproj @@ -16,7 +16,7 @@ + - diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksDbExtensions.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksDbExtensions.cs deleted file mode 100644 index 56a5b1b5e4f8..000000000000 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksDbExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using RocksDbSharp; -using RocksDbNative = RocksDbSharp.Native; - -namespace Nethermind.Db.Rocks; - -internal static class RocksDbExtensions -{ - private static readonly ReadOptions _defaultReadOptions = new(); - - internal static unsafe void DangerousReleaseMemory(this RocksDb _, in ReadOnlySpan span) - { - ref byte ptr = ref MemoryMarshal.GetReference(span); - nint intPtr = new(Unsafe.AsPointer(ref ptr)); - - RocksDbNative.Instance.rocksdb_free(intPtr); - } - - internal static unsafe Span GetSpan(this RocksDb db, scoped ReadOnlySpan key, ColumnFamilyHandle? cf = null, ReadOptions? readOptionObj = null) - { - nint readOptions = _defaultReadOptions.Handle; - if (readOptionObj is not null) readOptions = readOptionObj.Handle; - - long keyLength = (long)key.Length; - - nint result; - nint error; - UIntPtr valueLength; - - fixed (byte* ptr = key) - { - nuint keyLengthPtr = (UIntPtr)keyLength; - result = cf is null - ? RocksDbNative.Instance.rocksdb_get(db.Handle, readOptions, ptr, keyLengthPtr, out valueLength, out error) - : RocksDbNative.Instance.rocksdb_get_cf(db.Handle, readOptions, cf.Handle, ptr, keyLengthPtr, out valueLength, out error); - - } - - if (error != IntPtr.Zero) - throw new RocksDbException(error); - - if (result == IntPtr.Zero) - return default; - - Span span = new((void*)result, (int)valueLength); - - return span; - } -} diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksDbReader.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksDbReader.cs index 0bbe18cd8bf2..964495977b6a 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksDbReader.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/RocksDbReader.cs @@ -2,10 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Runtime.InteropServices; using System.Threading; using Nethermind.Core; -using RocksDbSharp; +using Nethermind.RocksDbBindings; namespace Nethermind.Db.Rocks; @@ -26,12 +25,12 @@ public class RocksDbReader(DbOnTheRocks mainDb, ReadOptions hintCacheMissOptions, Func readOptionsFactory, DisposableLazy? iteratorManager = null, - ColumnFamilyHandle? columnFamily = null) : ISortedKeyValueStore, IDisposable + IColumnFamilyHandle? columnFamily = null) : ISortedKeyValueStore, IDisposable { private readonly DbOnTheRocks _mainDb = mainDb; private readonly Func _readOptionsFactory = readOptionsFactory; private readonly DisposableLazy? _iteratorManager = iteratorManager; - private readonly ColumnFamilyHandle? _columnFamily = columnFamily; + private readonly IColumnFamilyHandle? _columnFamily = columnFamily; private readonly ReadOptions _options = options; private readonly ReadOptions _hintCacheMissOptions = hintCacheMissOptions; @@ -41,7 +40,7 @@ public class RocksDbReader(DbOnTheRocks mainDb, public RocksDbReader(DbOnTheRocks mainDb, Func readOptionsFactory, DisposableLazy? iteratorManager = null, - ColumnFamilyHandle? columnFamily = null) + IColumnFamilyHandle? columnFamily = null) : this(mainDb, readOptionsFactory(), readOptionsFactory(), readOptionsFactory, iteratorManager, columnFamily) { _ownsReadOptions = true; @@ -55,27 +54,15 @@ public virtual void Dispose() return; } - DestroyReadOptions(_options); - DestroyReadOptions(_hintCacheMissOptions); - } - - /// - /// Destroys a native ReadOptions handle and suppresses its finalizer to prevent - /// finalizer queue buildup from short-lived ReadOptions instances. - /// - internal static void DestroyReadOptions(ReadOptions? options) - { - if (options is null) return; - - Native.Instance.rocksdb_readoptions_destroy(options.Handle); - GC.SuppressFinalize(options); + _options.Dispose(); + _hintCacheMissOptions.Dispose(); } public byte[]? Get(scoped ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { if ((flags & ReadFlags.HintReadAhead) != 0 && _iteratorManager is not null) { - byte[]? result = _mainDb.GetWithIterator(key, _columnFamily, _iteratorManager.Value, flags, out bool success); + byte[]? result = _mainDb.GetWithIterator(key, _iteratorManager.Value, flags, out bool success); if (success) { return result; @@ -126,22 +113,9 @@ public byte[]? LastKey public ISortedView GetViewBetween(ReadOnlySpan firstKey, ReadOnlySpan lastKey) { ReadOptions readOptions = _readOptionsFactory(); - - IntPtr iterateLowerBound = IntPtr.Zero; - IntPtr iterateUpperBound = IntPtr.Zero; - - unsafe - { - iterateLowerBound = Marshal.AllocHGlobal(firstKey.Length); - firstKey.CopyTo(new Span(iterateLowerBound.ToPointer(), firstKey.Length)); - Native.Instance.rocksdb_readoptions_set_iterate_lower_bound(readOptions.Handle, iterateLowerBound, (UIntPtr)firstKey.Length); - - iterateUpperBound = Marshal.AllocHGlobal(lastKey.Length); - lastKey.CopyTo(new Span(iterateUpperBound.ToPointer(), lastKey.Length)); - Native.Instance.rocksdb_readoptions_set_iterate_upper_bound(readOptions.Handle, iterateUpperBound, (UIntPtr)lastKey.Length); - } + readOptions.SetIterateBounds(firstKey, lastKey); Iterator iterator = _mainDb.CreateIterator(readOptions, _columnFamily); - return new RocksdbSortedView(iterator, readOptions, iterateLowerBound, iterateUpperBound); + return new RocksdbSortedView(iterator, readOptions); } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs b/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs index 89e63c0ce814..1b68d3caec48 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/RocksdbSortedView.cs @@ -2,32 +2,22 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Runtime.InteropServices; using Nethermind.Core; -using RocksDbSharp; +using Nethermind.RocksDbBindings; namespace Nethermind.Db.Rocks; -internal class RocksdbSortedView(Iterator iterator, ReadOptions readOptions, IntPtr lowerBound = default, IntPtr upperBound = default) : ISortedView +internal class RocksdbSortedView(Iterator iterator, ReadOptions readOptions) : ISortedView { private readonly Iterator _iterator = iterator; private readonly ReadOptions _readOptions = readOptions; - private readonly IntPtr _lowerBound = lowerBound; - private readonly IntPtr _upperBound = upperBound; private bool _started = false; + // The read options own the iterate-bound buffers, so the iterator must go first. public void Dispose() { _iterator.Dispose(); - RocksDbReader.DestroyReadOptions(_readOptions); - if (_lowerBound != IntPtr.Zero) - { - Marshal.FreeHGlobal(_lowerBound); - } - if (_upperBound != IntPtr.Zero) - { - Marshal.FreeHGlobal(_upperBound); - } + _readOptions.Dispose(); } public bool StartBefore(ReadOnlySpan value) diff --git a/src/Nethermind/Nethermind.Db.Rocks/Statistics/DbMetricsUpdater.cs b/src/Nethermind/Nethermind.Db.Rocks/Statistics/DbMetricsUpdater.cs index 712fa0bd6d6d..00e51b8cf528 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Statistics/DbMetricsUpdater.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Statistics/DbMetricsUpdater.cs @@ -8,11 +8,11 @@ using Nethermind.Core.Extensions; using Nethermind.Db.Rocks.Config; using Nethermind.Logging; -using RocksDbSharp; +using Nethermind.RocksDbBindings; namespace Nethermind.Db.Rocks.Statistics; -public partial class DbMetricsUpdater(string dbName, Options dbOptions, RocksDb db, ColumnFamilyHandle? cf, IDbConfig dbConfig, bool isUsingSharedBlockCache, ILogger logger) +public partial class DbMetricsUpdater(string dbName, Options dbOptions, RocksDb db, IColumnFamilyHandle? cf, IDbConfig dbConfig, bool isUsingSharedBlockCache, ILogger logger) : IDisposable where T : Options { @@ -30,20 +30,26 @@ private void UpdateMetrics(object? state) try { // It seems that currently there is no other option with .NET api to extract the compaction statistics than through the dumped string - string compactionStatsString = ""; - compactionStatsString = cf is not null ? db.GetProperty("rocksdb.stats", cf) : db.GetProperty("rocksdb.stats"); + string? compactionStatsString = cf is not null ? db.GetProperty("rocksdb.stats", cf) : db.GetProperty("rocksdb.stats"); ProcessCompactionStats(compactionStatsString); LogMemoryProfile(); if (dbConfig.EnableDbStatistics) { - string dbStatsString = dbOptions.GetStatisticsString(); - ProcessStatisticsString(dbStatsString); + string? dbStatsString = dbOptions.GetStatisticsString(); + if (dbStatsString is not null) + { + ProcessStatisticsString(dbStatsString); + } + else if (logger.IsWarn) + { + logger.Warn($"No RocksDB statistics available for {dbName} database."); + } } } catch (Exception exc) { - logger.Error($"Error when updating metrics for {dbName} database.", exc); + if (logger.IsError) logger.Error($"Error when updating metrics for {dbName} database.", exc); // Maybe we would like to stop the _timer here to avoid logging the same error all over again? } } @@ -84,7 +90,7 @@ public void ProcessStatisticsString(string dbStatsString) } } - public void ProcessCompactionStats(string compactionStatsString) + public void ProcessCompactionStats(string? compactionStatsString) { if (!string.IsNullOrEmpty(compactionStatsString)) { @@ -93,7 +99,7 @@ public void ProcessCompactionStats(string compactionStatsString) } else { - logger.Warn($"No RocksDB compaction stats available for {dbName} database."); + if (logger.IsWarn) logger.Warn($"No RocksDB compaction stats available for {dbName} database."); } } @@ -154,7 +160,7 @@ private void ExtractIntervalCompaction(string compactionStatsDump) } else { - logger.Warn($"Cannot find 'Interval compaction' stats for {dbName} database in the compaction stats dump:{Environment.NewLine}{compactionStatsDump}"); + if (logger.IsWarn) logger.Warn($"Cannot find 'Interval compaction' stats for {dbName} database in the compaction stats dump:{Environment.NewLine}{compactionStatsDump}"); } } } @@ -188,7 +194,7 @@ long Prop(string name) long numKeys = Prop("rocksdb.estimate-num-keys"); long liveFiles = 0; // num_levels is configurable above the default 7; querying only levels 0..6 would undercount L7+. - int numLevels = Native.Instance.rocksdb_options_get_num_levels(dbOptions.Handle); + int numLevels = dbOptions.GetNumLevels(); for (int level = 0; level < numLevels; level++) { liveFiles += Math.Max(0, Prop($"rocksdb.num-files-at-level{level}")); @@ -217,7 +223,7 @@ public void Dispose() using ManualResetEvent waitHandle = new(false); if (timer.Dispose(waitHandle) && !waitHandle.WaitOne(TimeSpan.FromSeconds(1))) { - logger.Warn($"DbMetricsUpdater for {dbName} did not complete within the timeout during disposal."); + if (logger.IsWarn) logger.Warn($"DbMetricsUpdater for {dbName} did not complete within the timeout during disposal."); } } } diff --git a/src/Nethermind/Nethermind.Db.Test/DbMetricsUpdaterTests.cs b/src/Nethermind/Nethermind.Db.Test/DbMetricsUpdaterTests.cs index 0675bac8df88..293fab7863ed 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbMetricsUpdaterTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbMetricsUpdaterTests.cs @@ -8,7 +8,7 @@ using NUnit.Framework; using NSubstitute; using Nethermind.Logging; -using RocksDbSharp; +using Nethermind.RocksDbBindings; namespace Nethermind.Db.Test { diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 0d8c12e4e652..b29bbd3587da 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -12,14 +12,15 @@ using Nethermind.Core; using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; +using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; using Nethermind.Core.Test; using Nethermind.Db.Rocks; using Nethermind.Db.Rocks.Config; using Nethermind.Logging; +using Nethermind.RocksDbBindings; using NSubstitute; using NUnit.Framework; -using RocksDbSharp; using IWriteBatch = Nethermind.Core.IWriteBatch; namespace Nethermind.Db.Test @@ -51,17 +52,17 @@ public void WriteOptions_is_correct() IDbConfig config = new DbConfig(); using DbOnTheRocks db = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, _rocksdbConfigFactory, LimboLogs.Instance); - WriteOptions? options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority); - Assert.That(Native.Instance.rocksdb_writeoptions_get_low_pri(options.Handle), Is.EqualTo(1)); - Assert.That(Native.Instance.rocksdb_writeoptions_get_disable_WAL(options.Handle), Is.EqualTo(0)); + WriteOptions options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority)!; + Assert.That(options.GetLowPriority(), Is.True); + Assert.That(options.GetDisableWal(), Is.False); - options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority | WriteFlags.DisableWAL); - Assert.That(Native.Instance.rocksdb_writeoptions_get_low_pri(options.Handle), Is.EqualTo(1)); - Assert.That(Native.Instance.rocksdb_writeoptions_get_disable_WAL(options.Handle), Is.EqualTo(1)); + options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority | WriteFlags.DisableWAL)!; + Assert.That(options.GetLowPriority(), Is.True); + Assert.That(options.GetDisableWal(), Is.True); - options = db.WriteFlagsToWriteOptions(WriteFlags.DisableWAL); - Assert.That(Native.Instance.rocksdb_writeoptions_get_low_pri(options.Handle), Is.EqualTo(0)); - Assert.That(Native.Instance.rocksdb_writeoptions_get_disable_WAL(options.Handle), Is.EqualTo(1)); + options = db.WriteFlagsToWriteOptions(WriteFlags.DisableWAL)!; + Assert.That(options.GetLowPriority(), Is.False); + Assert.That(options.GetDisableWal(), Is.True); } [Test] @@ -154,10 +155,28 @@ public void CanOpenWithAdditionalConfig(string opts, bool success) } else { - Assert.That(act, Throws.TypeOf()); + Assert.That(act, Throws.InstanceOf()); } } + [Test] + public void SharedCacheCanBeCreatedAndDisposed() + { + HyperClockCacheWrapper cache = new((ulong)10.KiB); + + Assert.That(cache.Handle, Is.Not.Zero); + Assert.That(() => cache.GetUsage(), Throws.Nothing); + + cache.Dispose(); + // Disposal must stay exactly-once so the GC memory pressure accounting cannot go negative. + cache.Dispose(); + } + + [Test] + // rocksdb aborts the process on a zero capacity, so this must fail as a configuration error. + public void SharedCacheRejectsZeroCapacity() => + Assert.That(() => new HyperClockCacheWrapper(0), Throws.TypeOf()); + [TestCase(true)] [TestCase(false)] public void UseSharedCacheIfNoCacheIsSpecified(bool explicitCache) @@ -256,7 +275,7 @@ public void Corrupted_exception_on_open_writes_marker_and_shuts_down() fileSystem: fileSystem, onFatalShutdown: () => didShutDown = true); } - catch (RocksDbSharpException) + catch (RocksDbException) { exceptionThrown = true; } @@ -292,7 +311,7 @@ public void Io_error_on_open_shuts_down_without_writing_marker(string exceptionM openExceptionMessage: exceptionMessage, onFatalShutdown: () => didShutDown = true); } - catch (RocksDbSharpException) + catch (RocksDbException) { exceptionThrown = true; } @@ -314,20 +333,20 @@ public void If_marker_exists_on_open_then_repair_before_open() string markerFile = Path.Join(Path.GetTempPath(), "test", "test", "corrupt.marker"); file.Exists(markerFile).Returns(true); - RocksDbSharp.Native native = Substitute.For(); + bool didRepair = false; try { - _ = new DbOnTheRocks(Path.Join(Path.GetTempPath(), "test"), GetRocksDbSettings("test", "test"), config, _rocksdbConfigFactory, + _ = new RepairTrackingDbOnTheRocks(Path.Join(Path.GetTempPath(), "test"), GetRocksDbSettings("test", "test"), config, _rocksdbConfigFactory, LimboLogs.Instance, fileSystem: fileSystem, - rocksDbNative: native); + onRepair: () => didRepair = true); } catch (Exception) { } - native.Received().rocksdb_repair_db(Arg.Any(), Arg.Any(), out Arg.Any()); + Assert.That(didRepair, Is.True); file.Received().Delete(markerFile); } @@ -518,6 +537,40 @@ void AssertFindsNothing(ReadOnlySpan lowerBoundIncl, ReadOnlySpan up } } + [Test] + public void Can_read_back_empty_value() + { + byte[] key = [1, 2, 3]; + _db.Set(key, []); + + Assert.That(_db.KeyExists(key), Is.True); + Assert.That(_db.Get(key), Is.Empty); + Assert.That(_db.Get(key, []), Is.Zero); + + Span span = _db.GetSpan(key); + Assert.That(span.IsEmpty, Is.True); + _db.DangerousReleaseMemory(span); + + if (_db is IReadOnlyNativeKeyValueStore nativeStore) + { + ReadOnlySpan slice = nativeStore.GetNativeSlice(key, out nint handle); + Assert.That(slice.IsEmpty, Is.True); + nativeStore.DangerousReleaseHandle(handle); + } + + Assert.That(AllocatedSpan, Is.Zero); + } + + [Test] + public void Get_into_output_buffer_reports_missing_key_and_rejects_undersized_buffer() + { + byte[] key = [1, 2, 3]; + _db.Set(key, [4, 5, 6]); + + Assert.That(_db.Get([9, 9, 9], new byte[3]), Is.Zero); + Assert.That(() => _db.Get(key, new byte[2]), Throws.ArgumentException); + } + [Test] public void Snapshot_test() { @@ -859,16 +912,28 @@ class CorruptedDbOnTheRocks( IRocksDbConfigFactory rocksDbConfigFactory, ILogManager logManager, IList? columnFamilies = null, - RocksDbSharp.Native? rocksDbNative = null, IFileSystem? fileSystem = null, string openExceptionMessage = "Corruption: test corruption", Action? onFatalShutdown = null - ) : DbOnTheRocks(basePath, dbSettings, dbConfig, rocksDbConfigFactory, logManager, columnFamilies, rocksDbNative, fileSystem) + ) : DbOnTheRocks(basePath, dbSettings, dbConfig, rocksDbConfigFactory, logManager, columnFamilies, fileSystem) { - protected override RocksDb DoOpen(string path, (DbOptions Options, ColumnFamilies? Families) db) => throw new RocksDbSharpException(openExceptionMessage); + protected override RocksDb DoOpen(string path, (DbOptions Options, ColumnFamilies? Families) db) => throw new RocksDbException(openExceptionMessage); // The open path throws from the base constructor, so the caller never gets a reference to // observe FatalShutdown on; report it through the injected callback instead of exiting. protected override void FatalShutdown() => onFatalShutdown?.Invoke(); } + + class RepairTrackingDbOnTheRocks( + string basePath, + DbSettings dbSettings, + IDbConfig dbConfig, + IRocksDbConfigFactory rocksDbConfigFactory, + ILogManager logManager, + IFileSystem fileSystem, + Action onRepair + ) : DbOnTheRocks(basePath, dbSettings, dbConfig, rocksDbConfigFactory, logManager, fileSystem: fileSystem) + { + protected override void RepairDb(DbOptions dbOptions, string path) => onRepair(); + } } diff --git a/src/Nethermind/Nethermind.Runner.Test/Module/FlatRocksDbConfigAdjusterTests.cs b/src/Nethermind/Nethermind.Runner.Test/Module/FlatRocksDbConfigAdjusterTests.cs index 974f164ee8e0..56d19e32cb7c 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Module/FlatRocksDbConfigAdjusterTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Module/FlatRocksDbConfigAdjusterTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; +using Nethermind.Core.Exceptions; using Nethermind.Db; using Nethermind.Db.Rocks.Config; using Nethermind.Init.Modules; @@ -90,4 +91,17 @@ public void FlatDatabase_DelegatesToBaseFactoryWithCorrectParameters() _baseFactory.Received(1).GetForDatabase(nameof(DbNames.Flat), nameof(FlatDbColumns.Account)); } + + [TestCase(0UL, TestName = "FlatDatabase_WithZeroCacheBudget_ReportsConfigurationError")] + [TestCase(2UL, TestName = "FlatDatabase_WithCacheBudgetTooSmallToSplit_ReportsConfigurationError")] + public void FlatDatabase_WithUnusableCacheBudget_ReportsConfigurationError(ulong budget) + { + _flatDbConfig.Layout.Returns(FlatLayout.Flat); + _flatDbConfig.BlockCacheSizeBudget.Returns(budget); + + FlatRocksDbConfigAdjuster adjuster = new(_baseFactory, _flatDbConfig, _disposeStack, LimboLogs.Instance); + + Assert.That(() => adjuster.GetForDatabase(nameof(DbNames.Flat), nameof(FlatDbColumns.Account)), + Throws.TypeOf()); + } } diff --git a/src/Nethermind/Nethermind.Runner/packages.lock.json b/src/Nethermind/Nethermind.Runner/packages.lock.json index 241e6225d342..fa3b623d120e 100644 --- a/src/Nethermind/Nethermind.Runner/packages.lock.json +++ b/src/Nethermind/Nethermind.Runner/packages.lock.json @@ -752,8 +752,8 @@ "ConcurrentHashSet": "[1.3.0, )", "Nethermind.Api": "[1.40.0-unstable, )", "Nethermind.Db": "[1.40.0-unstable, )", - "NonBlocking": "[2.1.2, )", - "RocksDB": "[10.10.1.1747, 10.10.1.1747]" + "Nethermind.RocksDbBindings": "[11.8.1-preview.75, )", + "NonBlocking": "[2.1.2, )" } }, "nethermind.db.rpc": { @@ -1633,6 +1633,12 @@ "System.IO.Hashing": "10.0.10" } }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", @@ -1681,15 +1687,6 @@ "prometheus-net": "8.2.1" } }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } - }, "SCrypt": { "type": "CentralTransitive", "requested": "[2.0.0.2, )", @@ -1840,20 +1837,17 @@ "resolved": "1.0.5", "contentHash": "GVzsykooi0PAydDuhdWB7ugFyHPYYkwiBFGy5sVMlKeXsqBqUdvOgqnT/29bf74b3Uhm76gfC6fiW9nEW0xA3Q==" }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", "resolved": "1.0.0", "contentHash": "G5L8iaJusi7h4zkFtiSEhA8VRX7l6SsupdKRZ7QiHIzYR/860i/HS4wZ+/Uxn2fqkU4WRBuj+aGLFyX1x0VGCw==" - }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } } }, "net10.0/linux-x64": { @@ -1941,20 +1935,17 @@ "resolved": "1.0.5", "contentHash": "GVzsykooi0PAydDuhdWB7ugFyHPYYkwiBFGy5sVMlKeXsqBqUdvOgqnT/29bf74b3Uhm76gfC6fiW9nEW0xA3Q==" }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", "resolved": "1.0.0", "contentHash": "G5L8iaJusi7h4zkFtiSEhA8VRX7l6SsupdKRZ7QiHIzYR/860i/HS4wZ+/Uxn2fqkU4WRBuj+aGLFyX1x0VGCw==" - }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } } }, "net10.0/osx-arm64": { @@ -2042,20 +2033,17 @@ "resolved": "1.0.5", "contentHash": "GVzsykooi0PAydDuhdWB7ugFyHPYYkwiBFGy5sVMlKeXsqBqUdvOgqnT/29bf74b3Uhm76gfC6fiW9nEW0xA3Q==" }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", "resolved": "1.0.0", "contentHash": "G5L8iaJusi7h4zkFtiSEhA8VRX7l6SsupdKRZ7QiHIzYR/860i/HS4wZ+/Uxn2fqkU4WRBuj+aGLFyX1x0VGCw==" - }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } } }, "net10.0/osx-x64": { @@ -2143,20 +2131,17 @@ "resolved": "1.0.5", "contentHash": "GVzsykooi0PAydDuhdWB7ugFyHPYYkwiBFGy5sVMlKeXsqBqUdvOgqnT/29bf74b3Uhm76gfC6fiW9nEW0xA3Q==" }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", "resolved": "1.0.0", "contentHash": "G5L8iaJusi7h4zkFtiSEhA8VRX7l6SsupdKRZ7QiHIzYR/860i/HS4wZ+/Uxn2fqkU4WRBuj+aGLFyX1x0VGCw==" - }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } } }, "net10.0/win-x64": { @@ -2244,21 +2229,18 @@ "resolved": "1.0.5", "contentHash": "GVzsykooi0PAydDuhdWB7ugFyHPYYkwiBFGy5sVMlKeXsqBqUdvOgqnT/29bf74b3Uhm76gfC6fiW9nEW0xA3Q==" }, + "Nethermind.RocksDbBindings": { + "type": "CentralTransitive", + "requested": "[11.8.1-preview.75, )", + "resolved": "11.8.1-preview.75", + "contentHash": "vRS1g7aC72B5TNLRh3fBSE9YycL0aAB2EO+iix+IBMWvbOvNNQlW/w2cJ3OnXtoyp4Wc1UbggHgTgeZtMfunkQ==" + }, "Nethermind.TurboPForBindings": { "type": "CentralTransitive", "requested": "[1.0.0, )", "resolved": "1.0.0", "contentHash": "G5L8iaJusi7h4zkFtiSEhA8VRX7l6SsupdKRZ7QiHIzYR/860i/HS4wZ+/Uxn2fqkU4WRBuj+aGLFyX1x0VGCw==" - }, - "RocksDB": { - "type": "CentralTransitive", - "requested": "[10.10.1.1747, 10.10.1.1747]", - "resolved": "10.10.1.1747", - "contentHash": "uzwXHMAhfGnfa/iTspRRElvjxzfoUsca/fcRtZvYvBnW4k1CdherPeegz8Fym96V6XdJAAVfj9Xw7HkLq3mAqw==", - "dependencies": { - "ZstdSharp.Port": "0.8.7" - } } } } -} +} \ No newline at end of file diff --git a/src/Nethermind/Nethermind.Serialization.Ssz/Merkleization/Merkle.cs b/src/Nethermind/Nethermind.Serialization.Ssz/Merkleization/Merkle.cs index 5d3405ff1840..b58e31ad55f0 100644 --- a/src/Nethermind/Nethermind.Serialization.Ssz/Merkleization/Merkle.cs +++ b/src/Nethermind/Nethermind.Serialization.Ssz/Merkleization/Merkle.cs @@ -82,16 +82,18 @@ public static void MerkleizeProgressive(out UInt256 root, ReadOnlySpan return; } - int rightChunkCount = (int)Math.Min((ulong)chunks.Length, Math.Min(numLeaves, (ulong)int.MaxValue)); - ReadOnlySpan leftChunks = chunks[rightChunkCount..]; - UInt256 left = UInt256.Zero; - if (!leftChunks.IsEmpty) + // The `numLeaves`-sized subtree is the left child; the remaining chunks recurse into the right. + int subtreeChunkCount = (int)Math.Min((ulong)chunks.Length, Math.Min(numLeaves, (ulong)int.MaxValue)); + Merkleize(out UInt256 subtree, chunks[..subtreeChunkCount], numLeaves); + + ReadOnlySpan remainingChunks = chunks[subtreeChunkCount..]; + UInt256 continuation = UInt256.Zero; + if (!remainingChunks.IsEmpty) { - MerkleizeProgressive(out left, leftChunks, checked(numLeaves * 4)); + MerkleizeProgressive(out continuation, remainingChunks, checked(numLeaves * 4)); } - Merkleize(out UInt256 right, chunks[..rightChunkCount], numLeaves); - root = HashConcatenation(left, right, 0); + root = HashConcatenation(subtree, continuation, 0); } public static void Merkleize(out UInt256 root, ReadOnlySpan value) diff --git a/src/Nethermind/Nethermind.Serialization.SszGenerator.Test/EncodingTest.cs b/src/Nethermind/Nethermind.Serialization.SszGenerator.Test/EncodingTest.cs index 610729d0fac2..2a82a519eef1 100644 --- a/src/Nethermind/Nethermind.Serialization.SszGenerator.Test/EncodingTest.cs +++ b/src/Nethermind/Nethermind.Serialization.SszGenerator.Test/EncodingTest.cs @@ -1000,16 +1000,17 @@ private static void MerkleizeProgressiveSpec(ReadOnlySpan chunks, out U return; } - int rightCount = (int)Math.Min((ulong)chunks.Length, Math.Min(numLeaves, (ulong)int.MaxValue)); - ReadOnlySpan leftChunks = chunks[rightCount..]; - UInt256 left = UInt256.Zero; - if (!leftChunks.IsEmpty) + int subtreeCount = (int)Math.Min((ulong)chunks.Length, Math.Min(numLeaves, (ulong)int.MaxValue)); + Merkle.Merkleize(out UInt256 subtree, chunks[..subtreeCount], numLeaves); + + ReadOnlySpan remainingChunks = chunks[subtreeCount..]; + UInt256 continuation = UInt256.Zero; + if (!remainingChunks.IsEmpty) { - MerkleizeProgressiveSpec(leftChunks, out left, checked(numLeaves * 4)); + MerkleizeProgressiveSpec(remainingChunks, out continuation, checked(numLeaves * 4)); } - Merkle.Merkleize(out UInt256 right, chunks[..rightCount], numLeaves); - root = HashConcat(left, right); + root = HashConcat(subtree, continuation); } private static UInt256 HashConcat(UInt256 left, UInt256 right) diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/ChainConfig.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/ChainConfig.cs deleted file mode 100644 index 7ad80e256887..000000000000 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/ChainConfig.cs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Serialization.Ssz; - -namespace Nethermind.Stateless.Execution.IO; - -[SszContainer] -public partial struct ChainConfig -{ - public ulong ChainId { get; set; } - - public ForkConfig ActiveFork { get; set; } -} diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/ExecutionWitness.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/ExecutionWitness.cs index 9da534b5beba..13e66e3aff2b 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/ExecutionWitness.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/ExecutionWitness.cs @@ -10,10 +10,10 @@ namespace Nethermind.Stateless.Execution.IO; [SszContainer] public partial struct ExecutionWitness { - [SszList(0x40_0000)] + [SszProgressiveList] public SszWitnessState[] State { get; set; } - [SszList(0x4_0000)] + [SszProgressiveList] public SszWitnessCodes[] Codes { get; set; } [SszList(0x100)] diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/ForkConfig.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/ForkConfig.cs deleted file mode 100644 index 1b577ce024b4..000000000000 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/ForkConfig.cs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Core.Specs; -using Nethermind.Serialization.Ssz; - -namespace Nethermind.Stateless.Execution.IO; - -[SszContainer] -public partial struct ForkConfig -{ - public SszForkActivation Activation { get; set; } - - public static ForkConfig From(BlockHeader header, ISpecProvider specProvider) - { - ForkActivation forkActivation = new(header.Number, header.Timestamp); - - for (int i = specProvider.TransitionActivations.Length - 1; i >= 0; i--) - { - ForkActivation activation = specProvider.TransitionActivations[i]; - - if (activation <= forkActivation) - { - forkActivation = activation; - break; - } - } - - return new() - { - Activation = SszForkActivation.From(forkActivation) - }; - } -} diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/InputDecoder.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/InputDecoder.cs index 4a9bf3851603..475ad3c6b341 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/InputDecoder.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/InputDecoder.cs @@ -2,9 +2,10 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Buffers.Binary; +using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Int256; -using Nethermind.Merge.Plugin.SszRest; using Nethermind.Serialization.Ssz; namespace Nethermind.Stateless.Execution.IO; @@ -14,34 +15,47 @@ internal static class InputDecoder /// The schema revision selecting the SSZ StatelessInput payload encoding. internal const byte Revision1 = 0x01; + /// The schema id of a block on the chain's currently deployed fork. + internal const ushort CurrentForkSchemaId = ((ushort)ProtocolFork.Current << 8) | Revision1; + + /// The schema id of an Amsterdam block. + internal const ushort AmsterdamSchemaId = ((ushort)ProtocolFork.Amsterdam << 8) | Revision1; + internal static StatelessPayload Decode(ReadOnlySpan data) { ushort schemaId = BinaryPrimitives.ReadUInt16BigEndian(data); - ProtocolFork fork = (ProtocolFork)(schemaId >> 8); - byte revision = (byte)schemaId; ReadOnlySpan payload = data[sizeof(ushort)..]; - return (fork, revision) switch + return schemaId switch { - (ProtocolFork.Amsterdam, Revision1) => DecodeRevision1(payload, fork), - ( >= ProtocolFork.Cancun and < ProtocolFork.Amsterdam, Revision1) => DecodeRevision1(payload, fork), + AmsterdamSchemaId => DecodeRevision1(payload, schemaId, ProtocolFork.Amsterdam), + CurrentForkSchemaId => DecodeRevision1(payload, schemaId, ProtocolFork.Current), _ => throw new ArgumentException($"Unsupported schema id: 0x{schemaId:x4}", nameof(data)) }; } - private static StatelessPayload DecodeRevision1(ReadOnlySpan data, ProtocolFork protocolFork) - where TExecutionPayload : SszExecutionPayloadV1, ISszExecutionPayloadFactory, ISszCodec, new() + private static StatelessPayload DecodeRevision1( + ReadOnlySpan data, ushort schemaId, ProtocolFork protocolFork) + where TExecutionPayload : SszExecutionPayload, ISszCodec, new() { StatelessInput.Decode(data, out StatelessInput input); NewPayloadRequest.Merkleize(input.NewPayloadRequest, out UInt256 root); + TExecutionPayload executionPayload = input.NewPayloadRequest.ExecutionPayload; + ForkActivation activation = new(executionPayload.BlockNumber, executionPayload.Timestamp); + ISpecProvider specProvider = StatelessSpecProvider.Create(input.ChainId, protocolFork, activation); + NewPayloadRequest newPayloadRequest = input.NewPayloadRequest; + bool requestsEnabled = specProvider.GetSpec(activation).RequestsEnabled; + return new( - Block: input.NewPayloadRequest.ToBlock(requestsEnabled: protocolFork >= ProtocolFork.Prague)!, + GetBlock: () => newPayloadRequest.ToBlock(requestsEnabled)!, Witness: input.Witness, - ChainConfig: input.ChainConfig, + ChainId: input.ChainId, + SchemaId: schemaId, PublicKeys: input.PublicKeys, + VersionedHashes: input.NewPayloadRequest.VersionedHashes, NewPayloadRequestRoot: new Hash256(root.ToLittleEndian()), - ProtocolFork: protocolFork + SpecProvider: specProvider ); } } diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/NewPayloadRequest.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/NewPayloadRequest.cs index 4f6bca4b1381..8ffb375a176e 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/NewPayloadRequest.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/NewPayloadRequest.cs @@ -6,27 +6,25 @@ using Nethermind.Core.Crypto; using Nethermind.Core.ExecutionRequest; using Nethermind.Merge.Plugin.Data; -using Nethermind.Merge.Plugin.SszRest; using Nethermind.Serialization.Ssz; namespace Nethermind.Stateless.Execution.IO; [SszContainer] public partial class NewPayloadRequest - where TExecutionPayload : SszExecutionPayloadV1, ISszExecutionPayloadFactory, ISszCodec, new() + where TExecutionPayload : SszExecutionPayload, ISszCodec, new() { public TExecutionPayload ExecutionPayload { get; set; } = default!; - [SszList(0x1000)] + [SszProgressiveList] public Hash256[] VersionedHashes { get; set; } = []; public Hash256 ParentBeaconBlockRoot { get; set; } = null!; public SszExecutionRequests ExecutionRequests { get; set; } - public static NewPayloadRequest From(Block block) + public static NewPayloadRequest From(Block block, TExecutionPayload payload) { - TExecutionPayload payload = TExecutionPayload.From(block); Hash256 parentBeaconBlockRoot = block.ParentBeaconBlockRoot ?? throw new ArgumentException("Parent beacon block root is missing.", nameof(block)); diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/ProtocolFork.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/ProtocolFork.cs index a884ce5ff95c..5da81032a11e 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/ProtocolFork.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/ProtocolFork.cs @@ -11,10 +11,15 @@ namespace Nethermind.Stateless.Execution.IO; /// /// /// Only forks whose payloads are representable by the stateless input schemas are listed: -/// Cancun through BPO2 use the pre-BAL payload, Amsterdam adds the EIP-7928/EIP-7843 fields. +/// Cancun through BPO2 share the schema's pre-BAL payload, +/// and Amsterdam adds the EIP-7928/EIP-7843 fields. /// public enum ProtocolFork : byte { + /// No pinned fork: the payload's rules come from the chain's own transition schedule. + /// Reserved by the execution specs, whose fork identifiers start at Frontier (0x01). + Current = 0x00, + Cancun = 0x10, Prague = 0x11, Osaka = 0x12, diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionPayload.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionPayload.cs new file mode 100644 index 000000000000..61adcaaccf82 --- /dev/null +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionPayload.cs @@ -0,0 +1,275 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Int256; +using Nethermind.Merge.Plugin.Data; +using Nethermind.Serialization.Ssz; + +namespace Nethermind.Stateless.Execution.IO; + +/// SSZ execution payload of a block on the chain's currently deployed fork. +/// +/// An EIP-7688 progressive container +/// holding the stable field indices 0-16; activates +/// the two fields Amsterdam appends. Every field property below is a pass-through to the +/// like-named member, so the wire layout is defined by the +/// SszField indices rather than by declaration order. +/// +[SszContainer] +public partial class SszExecutionPayload(ExecutionPayloadV3 payload) +{ + /// Creates an empty payload for the SSZ decoder to populate. + public SszExecutionPayload() : this(new ExecutionPayloadV3()) { } + + /// Creates the SSZ execution payload of a block. + /// The block to wrap. + public static SszExecutionPayload From(Block block) => new(ExecutionPayloadV3.Create(block)); + + /// Gets the domain payload the field properties read from and write to. + /// Derived containers narrow this to their own payload type to reach the fields their fork adds. + protected virtual ExecutionPayloadV3 Inner { get; } = payload; + + /// Gets the wrapped domain execution payload. + public virtual ExecutionPayloadV3 AsExecutionPayload() => Inner; + + [SszField(0)] + public Hash256 ParentHash + { + get => Inner.ParentHash; + set => Inner.ParentHash = value; + } + + [SszField(1)] + public Address FeeRecipient + { + get => Inner.FeeRecipient; + set => Inner.FeeRecipient = value; + } + + [SszField(2)] + public Hash256 StateRoot + { + get => Inner.StateRoot; + set => Inner.StateRoot = value; + } + + [SszField(3)] + public Hash256 ReceiptsRoot + { + get => Inner.ReceiptsRoot; + set => Inner.ReceiptsRoot = value; + } + + [SszField(4)] + public Bloom LogsBloom + { + get => Inner.LogsBloom; + set => Inner.LogsBloom = value; + } + + [SszField(5)] + public Hash256 PrevRandao + { + get => Inner.PrevRandao; + set => Inner.PrevRandao = value; + } + + [SszField(6)] + public ulong BlockNumber + { + get => Inner.BlockNumber; + set => Inner.BlockNumber = value; + } + + [SszField(7)] + public ulong GasLimit + { + get => Inner.GasLimit; + set => Inner.GasLimit = value; + } + + [SszField(8)] + public ulong GasUsed + { + get => Inner.GasUsed; + set => Inner.GasUsed = value; + } + + [SszField(9)] + public ulong Timestamp + { + get => Inner.Timestamp; + set => Inner.Timestamp = value; + } + + [SszField(10)] + [SszList(32)] + public byte[] ExtraData + { + get => Inner.ExtraData; + set => Inner.ExtraData = value; + } + + [SszField(11)] + public UInt256 BaseFeePerGas + { + get => Inner.BaseFeePerGas; + set => Inner.BaseFeePerGas = value; + } + + [SszField(12)] + public Hash256 BlockHash + { + get => Inner.BlockHash; + set => Inner.BlockHash = value; + } + + [SszField(13)] + [SszProgressiveList] + public SszProgressiveBytes[] Transactions + { + get + { + if (field is not null) return field; + byte[][] txs = Inner.Transactions; + if (txs.Length == 0) return []; + field = new SszProgressiveBytes[txs.Length]; + for (int i = 0; i < txs.Length; i++) + field[i] = new SszProgressiveBytes { Bytes = txs[i] }; + return field; + } + set + { + field = value; + if (value is null || value.Length == 0) + { + Inner.Transactions = []; + return; + } + byte[][] raw = new byte[value.Length][]; + for (int i = 0; i < value.Length; i++) + raw[i] = value[i].Bytes ?? []; + Inner.Transactions = raw; + } + } + + [SszField(14)] + [SszProgressiveList] + public SszWithdrawal[] Withdrawals + { + get + { + if (field is not null) return field; + Withdrawal[]? ws = Inner.Withdrawals; + if (ws is null || ws.Length == 0) return []; + field = new SszWithdrawal[ws.Length]; + for (int i = 0; i < ws.Length; i++) + field[i] = new SszWithdrawal + { + Index = ws[i].Index, + ValidatorIndex = ws[i].ValidatorIndex, + Address = ws[i].Address, + Amount = ws[i].AmountInGwei + }; + return field; + } + set + { + field = value; + + if (value is null) + { + Inner.Withdrawals = null; + return; + } + + Withdrawal[] result = new Withdrawal[value.Length]; + + for (int i = 0; i < value.Length; i++) + { + result[i] = new Withdrawal + { + Index = value[i].Index, + ValidatorIndex = value[i].ValidatorIndex, + Address = value[i].Address, + AmountInGwei = value[i].Amount + }; + } + + Inner.Withdrawals = result; + } + } + + [SszField(15)] + public ulong BlobGasUsed + { + get => Inner.BlobGasUsed ?? 0; + set => Inner.BlobGasUsed = value; + } + + [SszField(16)] + public ulong ExcessBlobGas + { + get => Inner.ExcessBlobGas ?? 0; + set => Inner.ExcessBlobGas = value; + } +} + +/// SSZ execution payload of an Amsterdam block, adding the EIP-7928 and EIP-7843 fields. +[SszContainer] +public partial class SszExecutionPayloadAmsterdam(ExecutionPayloadV4 payload) : SszExecutionPayload(payload) +{ + /// + public SszExecutionPayloadAmsterdam() : this(new ExecutionPayloadV4()) { } + + /// + public new static SszExecutionPayloadAmsterdam From(Block block) => new(ExecutionPayloadV4.Create(block)); + + protected override ExecutionPayloadV4 Inner => (ExecutionPayloadV4)base.Inner; + + /// + public override ExecutionPayloadV4 AsExecutionPayload() => Inner; + + [SszField(17)] + [SszProgressiveList] + public byte[] BlockAccessList + { + get => Inner.BlockAccessList ?? []; + set => Inner.BlockAccessList = value.Length > 0 ? value : null; + } + + [SszField(18)] + public ulong SlotNumber + { + get => Inner.SlotNumber ?? 0; + set => Inner.SlotNumber = value; + } +} + +/// SSZ ProgressiveByteList as defined by EIP-7688. +[SszContainer(isCollectionItself: true)] +public partial struct SszProgressiveBytes +{ + /// Gets or sets the encoded bytes of one list element, or null for an empty one. + [SszProgressiveList] + public byte[]? Bytes { get; set; } +} + +/// SSZ mirror of a consensus-layer withdrawal, carried by . +[SszContainer] +public partial struct SszWithdrawal +{ + /// + public ulong Index { get; set; } + + /// + public ulong ValidatorIndex { get; set; } + + /// + public Address Address { get; set; } + + /// + public ulong Amount { get; set; } +} diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionRequests.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionRequests.cs index 11daa9b24086..95b2046db34a 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionRequests.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/SszExecutionRequests.cs @@ -8,18 +8,23 @@ namespace Nethermind.Stateless.Execution.IO; [SszContainer] public partial struct SszExecutionRequests { - [SszList(0x2000)] + [SszField(0)] + [SszProgressiveList] public DepositRequest[] Deposits { get; set; } - [SszList(0x10)] + [SszField(1)] + [SszProgressiveList] public WithdrawalRequest[] Withdrawals { get; set; } - [SszList(0x2)] + [SszField(2)] + [SszProgressiveList] public ConsolidationRequest[] Consolidations { get; set; } - [SszList(0x40)] + [SszField(3)] + [SszProgressiveList] public BuilderDepositRequest[] BuilderDeposits { get; set; } - [SszList(0x10)] + [SszField(4)] + [SszProgressiveList] public BuilderExitRequest[] BuilderExits { get; set; } } diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/SszForkActivation.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/SszForkActivation.cs deleted file mode 100644 index 637c50f3cb4c..000000000000 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/SszForkActivation.cs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Core.Specs; -using Nethermind.Serialization.Ssz; - -namespace Nethermind.Stateless.Execution.IO; - -[SszContainer] -public partial struct SszForkActivation -{ - [SszList(1)] - public ulong[] BlockNumber { get; set; } - - [SszList(1)] - public ulong[] Timestamp { get; set; } - - public static SszForkActivation From(ForkActivation forkActivation) - { - if (forkActivation.Timestamp is { } timestamp) - { - return new() - { - BlockNumber = [], - Timestamp = [timestamp] - }; - } - - return new() - { - BlockNumber = [forkActivation.BlockNumber], - Timestamp = [] - }; - } - - public readonly ForkActivation ToForkActivation() - { - Validate(); - - return (BlockNumber.Length, Timestamp.Length) switch - { - (0, 1) => ForkActivation.TimestampOnly(Timestamp[0]), - (1, 0) => new(BlockNumber[0]), - _ => new(BlockNumber[0], Timestamp[0]) - }; - } - - /// - /// Returns whether every configured activation bound is active for the supplied block. - /// - internal readonly bool IsActive(BlockHeader header) - { - Validate(); - - return (BlockNumber.Length == 0 || header.Number >= BlockNumber[0]) - && (Timestamp.Length == 0 || header.Timestamp >= Timestamp[0]); - } - - private readonly void Validate() - { - if (BlockNumber is not { Length: <= 1 }) - throw new InvalidDataException($"{nameof(BlockNumber)} must have at most one element."); - - if (Timestamp is not { Length: <= 1 }) - throw new InvalidDataException($"{nameof(Timestamp)} must have at most one element."); - - if (BlockNumber.Length == 0 && Timestamp.Length == 0) - throw new InvalidDataException($"{nameof(BlockNumber)} or {nameof(Timestamp)} must have one element."); - } -} diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessInput.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessInput.cs index ba909f79f122..ab5cf941ea4e 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessInput.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessInput.cs @@ -1,22 +1,21 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Merge.Plugin.SszRest; using Nethermind.Serialization.Ssz; namespace Nethermind.Stateless.Execution.IO; [SszContainer] -public partial class StatelessInput where TExecutionPayload - : SszExecutionPayloadV1, ISszExecutionPayloadFactory, ISszCodec, new() +public partial class StatelessInput + where TExecutionPayload : SszExecutionPayload, ISszCodec, new() { public NewPayloadRequest NewPayloadRequest { get; set; } = null!; public ExecutionWitness Witness { get; set; } - public ChainConfig ChainConfig { get; set; } + public ulong ChainId { get; set; } - [SszList(0x8000)] + [SszProgressiveList] public SszPublicKeys[] PublicKeys { get; set; } = []; } diff --git a/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessValidationResult.cs b/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessValidationResult.cs index 4924ffc4f4a5..0bd7f6206d5c 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessValidationResult.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/IO/StatelessValidationResult.cs @@ -13,5 +13,9 @@ public partial struct StatelessValidationResult public bool IsSuccess { get; set; } - public ChainConfig ChainConfig { get; set; } + public ulong ChainId { get; set; } + + /// Gets or sets the schema of the input that was decoded and executed. + /// Zero is the sentinel reported when the input bytes cannot be decoded. + public ushort SchemaId { get; set; } } diff --git a/src/Nethermind/Nethermind.Stateless.Executor/Nethermind.Stateless.Executor.csproj b/src/Nethermind/Nethermind.Stateless.Executor/Nethermind.Stateless.Executor.csproj index dafba1f19d87..00b04d95572c 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/Nethermind.Stateless.Executor.csproj +++ b/src/Nethermind/Nethermind.Stateless.Executor/Nethermind.Stateless.Executor.csproj @@ -19,12 +19,6 @@ - - - - - - diff --git a/src/Nethermind/Nethermind.Stateless.Executor/StatelessExecutor.cs b/src/Nethermind/Nethermind.Stateless.Executor/StatelessExecutor.cs index 21404290b14a..3f9d837ac13a 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/StatelessExecutor.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/StatelessExecutor.cs @@ -12,8 +12,6 @@ using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Logging; -using Nethermind.Specs; -using Nethermind.Specs.ChainSpecStyle; using Nethermind.Stateless.Execution.IO; namespace Nethermind.Stateless.Execution; @@ -36,25 +34,31 @@ public static byte[] Execute(ReadOnlySpan data) return output; } - ReadOnlySpan publicKeys = payload.PublicKeys.Span; - Transaction[] transactions = payload.Block.Transactions; StatelessValidationResult result = new() { NewPayloadRequestRoot = payload.NewPayloadRequestRoot, IsSuccess = false, - ChainConfig = payload.ChainConfig + ChainId = payload.ChainId, + SchemaId = payload.SchemaId }; output = StatelessValidationResult.Encode(result); bool success = false; + // Published before block reconstruction, the first step that can throw, so a failure there + // still reports the decoded metadata rather than the zero sentinel. FailureOutput = output; - if (transactions.Length == publicKeys.Length) + try { - try + Block block = payload.GetBlock(); + ReadOnlySpan publicKeys = payload.PublicKeys.Span; + Transaction[] transactions = block.Transactions; + + if (transactions.Length == publicKeys.Length && + BlobVersionedHashesMatch(transactions, payload.VersionedHashes.Span)) { - ISpecProvider specProvider = GetSpecProvider(payload.ChainConfig, payload.ProtocolFork, payload.Block.Header); - IReleaseSpec spec = specProvider.GetSpec(payload.Block.Header); + ISpecProvider specProvider = payload.SpecProvider; + IReleaseSpec spec = specProvider.GetSpec(block.Header); #if !ZK_EVM if (spec.IsEip4844Enabled && !KzgPolynomialCommitments.IsInitialized) KzgPolynomialCommitments.InitializeAsync().GetAwaiter().GetResult(); @@ -64,13 +68,13 @@ public static byte[] Execute(ReadOnlySpan data) using Witness witness = payload.Witness.ToWitness(); - success = Execute(payload.Block, witness, specProvider); - } - catch (Exception ex) - { - Debug.Fail(ex.Message); + success = Execute(block, witness, specProvider); } } + catch (Exception ex) + { + Debug.Fail(ex.Message); + } if (success) { @@ -157,26 +161,31 @@ public static bool Execute(Block suggestedBlock, Witness witness, ISpecProvider { NewPayloadRequestRoot = Hash256.Zero, IsSuccess = false, - ChainConfig = new ChainConfig - { - ChainId = 0, - ActiveFork = new ForkConfig - { - Activation = new() { BlockNumber = [], Timestamp = [] } - } - } + ChainId = 0, + SchemaId = 0 }; - private static ISpecProvider GetSpecProvider(ChainConfig chainConfig, ProtocolFork protocolFork, BlockHeader header) + /// Returns whether commit to exactly , in order. + internal static bool BlobVersionedHashesMatch(Transaction[] transactions, ReadOnlySpan expected) { - if (!chainConfig.ActiveFork.Activation.IsActive(header)) - throw new ArgumentException("ChainConfig active fork is not active for the payload.", nameof(chainConfig)); + int index = 0; + + foreach (Transaction transaction in transactions) + { + byte[]?[]? hashes = transaction.BlobVersionedHashes; - ChainSpecBasedSpecProvider.KnownProvidersByChainId.TryGetValue(chainConfig.ChainId, out IForkAwareSpecProvider? baseProvider); + if (hashes is null) + continue; - // ActiveFork pins the spec by name on any compatible schedule; unknown chains (e.g. devnets) use Mainnet rules. - baseProvider ??= MainnetSpecProvider.Instance; + foreach (byte[]? hash in hashes) + { + if (index == expected.Length || !expected[index].Bytes.SequenceEqual(hash)) + return false; + + index++; + } + } - return StatelessSpecProvider.Create(baseProvider, chainConfig.ChainId, chainConfig.ActiveFork, protocolFork); + return index == expected.Length; } } diff --git a/src/Nethermind/Nethermind.Stateless.Executor/StatelessPayload.cs b/src/Nethermind/Nethermind.Stateless.Executor/StatelessPayload.cs index ded82e013cae..31c48fa4f84e 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/StatelessPayload.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/StatelessPayload.cs @@ -3,16 +3,23 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; using Nethermind.Stateless.Execution.IO; namespace Nethermind.Stateless.Execution; +/// +/// Reconstructs the block. Deferred because it parses attacker-controlled transaction RLP and can +/// throw, so callers must publish before invoking it. +/// internal readonly record struct StatelessPayload ( - Block Block, + Func GetBlock, ExecutionWitness Witness, - ChainConfig ChainConfig, + ulong ChainId, + ushort SchemaId, ReadOnlyMemory PublicKeys, + ReadOnlyMemory VersionedHashes, Hash256 NewPayloadRequestRoot, - ProtocolFork ProtocolFork + ISpecProvider SpecProvider ); diff --git a/src/Nethermind/Nethermind.Stateless.Executor/StatelessSpecProvider.cs b/src/Nethermind/Nethermind.Stateless.Executor/StatelessSpecProvider.cs index ed2838ee4049..a91b3a227c84 100644 --- a/src/Nethermind/Nethermind.Stateless.Executor/StatelessSpecProvider.cs +++ b/src/Nethermind/Nethermind.Stateless.Executor/StatelessSpecProvider.cs @@ -4,6 +4,8 @@ using Nethermind.Core; using Nethermind.Core.Specs; using Nethermind.Int256; +using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; using Nethermind.Specs.GnosisForks; using Nethermind.Stateless.Execution.IO; @@ -11,9 +13,9 @@ namespace Nethermind.Stateless.Execution; /// -/// Stateless fixtures can pin a named fork independently of the base chain's transition schedule. -/// For activations at or after the supplied active fork, returns -/// the pinned release spec; earlier activations continue to use the base provider. +/// Stateless inputs can pin a named fork independently of the base chain's transition schedule. +/// For activations at or after the payload's own activation, returns +/// the payload's release spec; earlier activations continue to use the base provider. /// Chain id is supplied externally, so any compatible base schedule (e.g. Mainnet rules) can serve /// as a devnet's fork catalog without misreporting the chain id to EIP-155 validation. /// Merge transition metadata (, ) @@ -22,8 +24,8 @@ namespace Nethermind.Stateless.Execution; internal sealed class StatelessSpecProvider( ISpecProvider baseProvider, ulong chainId, - ForkActivation activeForkActivation, - IReleaseSpec activeForkSpec) + ForkActivation payloadActivation, + IReleaseSpec payloadSpec) : ISpecProvider { public ForkActivation? MergeBlockNumber => baseProvider.MergeBlockNumber; @@ -47,34 +49,41 @@ internal sealed class StatelessSpecProvider( public ForkActivation[] TransitionActivations => baseProvider.TransitionActivations; public IReleaseSpec GetSpec(ForkActivation activation) => - activation >= activeForkActivation ? activeForkSpec : baseProvider.GetSpec(activation); + activation >= payloadActivation ? payloadSpec : baseProvider.GetSpec(activation); public void UpdateMergeTransitionInfo(ulong? blockNumber, UInt256? terminalTotalDifficulty = null) => baseProvider.UpdateMergeTransitionInfo(blockNumber, terminalTotalDifficulty); - public static StatelessSpecProvider Create( - IForkAwareSpecProvider baseProvider, - ulong chainId, - ForkConfig forkConfig, - ProtocolFork protocolFork) + /// Creates the spec provider governing the rules of a decoded stateless payload. + /// The chain id the payload was produced on. + /// + /// The fork pinned by the input schema, or to follow the chain's schedule. + /// + /// The activation of the payload's own block. + public static StatelessSpecProvider Create(ulong chainId, ProtocolFork protocolFork, ForkActivation payloadActivation) { - string forkName = protocolFork.GetName(); + ChainSpecBasedSpecProvider.KnownProvidersByChainId.TryGetValue(chainId, out IForkAwareSpecProvider? baseProvider); - IReleaseSpec spec; - if (!baseProvider.TryGetForkSpec(forkName, out IReleaseSpec? configuredSpec) || configuredSpec is null) - { - spec = (chainId, protocolFork) switch - { - (BlockchainIds.Gnosis or BlockchainIds.Chiado, ProtocolFork.Amsterdam) => AmsterdamGnosis.Instance, - (_, ProtocolFork.Amsterdam) => Amsterdam.Instance, - _ => throw new ArgumentException($"Unknown fork: {protocolFork}", nameof(protocolFork)) - }; - } - else - { - spec = configuredSpec; - } + // Unknown chains (e.g. devnets) fall back to Mainnet — for ProtocolFork.Current, to its schedule too. + baseProvider ??= MainnetSpecProvider.Instance; + + return new(baseProvider, chainId, payloadActivation, GetPayloadSpec(baseProvider, chainId, protocolFork, payloadActivation)); + } - return new(baseProvider, chainId, forkConfig.Activation.ToForkActivation(), spec); + private static IReleaseSpec GetPayloadSpec( + IForkAwareSpecProvider baseProvider, ulong chainId, ProtocolFork protocolFork, ForkActivation payloadActivation) + { + if (protocolFork == ProtocolFork.Current) + return baseProvider.GetSpec(payloadActivation); + + if (baseProvider.TryGetForkSpec(protocolFork.GetName(), out IReleaseSpec? configuredSpec) && configuredSpec is not null) + return configuredSpec; + + return (chainId, protocolFork) switch + { + (BlockchainIds.Gnosis or BlockchainIds.Chiado, ProtocolFork.Amsterdam) => AmsterdamGnosis.Instance, + (_, ProtocolFork.Amsterdam) => Amsterdam.Instance, + _ => throw new ArgumentException($"Unknown fork: {protocolFork}", nameof(protocolFork)) + }; } } diff --git a/src/Nethermind/Nethermind.Stateless.ZiskGuest/README.md b/src/Nethermind/Nethermind.Stateless.ZiskGuest/README.md index 6d29c6455689..dfe8f777569d 100644 --- a/src/Nethermind/Nethermind.Stateless.ZiskGuest/README.md +++ b/src/Nethermind/Nethermind.Stateless.ZiskGuest/README.md @@ -33,7 +33,7 @@ The `INPUT` variable must point to a file in the `Nethermind.Stateless.ZiskGuest ### Input serialization -The input data is a version-prefixed SSZ as specified [here](https://github.com/ethereum/execution-specs/blob/projects/zkevm/src/ethereum/forks/amsterdam/stateless_ssz.py): `schema: u16be | ssz_bytes`. For the pre-Amsterdam forks, `schema` is 0. +The input data is a version-prefixed SSZ as specified [here](https://github.com/ethereum/execution-specs/blob/projects/zkevm/src/ethereum/forks/amsterdam/stateless_ssz.py): `schema: u16be | ssz_bytes`, where `schema` is a fork index followed by a schema revision. Revision 1 defines two schemas: `0x0001` for a block of the chain's currently deployed fork, and `0x1501` for an Amsterdam block. Starting from Zisk v0.16.0, the input data (`input.bin`) must be framed as follows when specified with the `--inputs` option: diff --git a/src/Nethermind/Stateless.slnx b/src/Nethermind/Stateless.slnx index 184aa3012981..3775de6b5635 100644 --- a/src/Nethermind/Stateless.slnx +++ b/src/Nethermind/Stateless.slnx @@ -13,6 +13,7 @@ + @@ -24,7 +25,6 @@ - diff --git a/tools/StatelessInputGen/InputGenerator.cs b/tools/StatelessInputGen/InputGenerator.cs index c0708c2ab6c3..3d2dc48f2472 100644 --- a/tools/StatelessInputGen/InputGenerator.cs +++ b/tools/StatelessInputGen/InputGenerator.cs @@ -10,7 +10,6 @@ using Nethermind.Crypto; using Nethermind.JsonRpc.Client; using Nethermind.Logging; -using Nethermind.Merge.Plugin.SszRest; using Nethermind.Serialization.Json; using Nethermind.Serialization.Rlp; using Nethermind.Serialization.Ssz; @@ -46,13 +45,16 @@ internal static async Task Generate(string blockParam, Uri host, string out return 1; } - byte[] encoded = fork == ProtocolFork.Amsterdam - ? EncodeInput(block, witness, chainId.Value, specProvider) - : EncodeInput(block, witness, chainId.Value, specProvider); + // Only Amsterdam has a schema of its own; earlier forks share the current-fork schema. + bool isAmsterdam = fork == ProtocolFork.Amsterdam; + byte[] encoded = isAmsterdam + ? EncodeInput(SszExecutionPayloadAmsterdam.From(block), block, witness, chainId.Value) + : EncodeInput(SszExecutionPayload.From(block), block, witness, chainId.Value); data = new byte[encoded.Length + sizeof(ushort)]; - BinaryPrimitives.WriteUInt16BigEndian(data, fork.ToRevision1SchemaId()); + BinaryPrimitives.WriteUInt16BigEndian( + data, (isAmsterdam ? ProtocolFork.Amsterdam : ProtocolFork.Current).ToRevision1SchemaId()); Buffer.BlockCopy(encoded, 0, data, sizeof(ushort), encoded.Length); } @@ -72,18 +74,15 @@ internal static async Task Generate(string blockParam, Uri host, string out return 0; } - private static byte[] EncodeInput(Block block, Witness witness, ulong chainId, ISpecProvider specProvider) - where TExecutionPayload : SszExecutionPayloadV1, ISszExecutionPayloadFactory, ISszCodec, new() + private static byte[] EncodeInput( + TExecutionPayload payload, Block block, Witness witness, ulong chainId) + where TExecutionPayload : SszExecutionPayload, ISszCodec, new() { StatelessInput input = new() { - NewPayloadRequest = NewPayloadRequest.From(block), + NewPayloadRequest = NewPayloadRequest.From(block, payload), Witness = ExecutionWitness.From(witness), - ChainConfig = new() - { - ChainId = chainId, - ActiveFork = ForkConfig.From(block.Header, specProvider) - }, + ChainId = chainId, PublicKeys = RecoverPublicKeys(block.Transactions, chainId) };