diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/AssociativeCacheTestsBase.cs b/src/Nethermind/Nethermind.Core.Test/Caching/AssociativeCacheTestsBase.cs index 7ac334a96cf1..07857ffb917a 100644 --- a/src/Nethermind/Nethermind.Core.Test/Caching/AssociativeCacheTestsBase.cs +++ b/src/Nethermind/Nethermind.Core.Test/Caching/AssociativeCacheTestsBase.cs @@ -347,6 +347,45 @@ public void All_inserted_keys_retrievable_at_various_capacities(int capacity) } } + [Test] + public void Refreshed_entry_survives_sustained_churn_in_its_set() + { + const int capacity = 64; + int setCount = (int)BitOperations.RoundUpToPowerOf2((uint)((capacity + Ways - 1) / Ways)); + int hashShift = BitOperations.Log2((uint)setCount); + // Every key collides into set 0, so each insert past the way count must evict from it. + static DeterministicHashKey MakeKey(int i, int shift) => new(i, (long)(i + 1) << shift); + + AssociativeCache cache = new(capacity); + AssociativeKeyCache keyCache = new(capacity); + DeterministicHashKey hot = MakeKey(0, hashShift); + TestValue hotValue = new(); + cache.Set(in hot, hotValue); + keyCache.Set(in hot); + + for (int i = 1; i <= 200; i++) + { + // The refreshing lookup gives the hot entry the newest eviction age, so 3-random + // eviction (which removes the oldest of its sample) must never select it. This pins + // the ticker semantics: a recency clock coarse enough to tie a refresh with the + // surrounding churn would make this probabilistic. + using (Assert.EnterMultipleScope()) + { + Assert.That(cache.Get(in hot), Is.SameAs(hotValue), $"hot entry evicted after {i - 1} churn inserts"); + Assert.That(keyCache.Get(in hot), Is.True, $"hot key evicted after {i - 1} churn inserts"); + } + DeterministicHashKey filler = MakeKey(i, hashShift); + cache.Set(in filler, new TestValue()); + keyCache.Set(in filler); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(cache.Get(in hot), Is.SameAs(hotValue)); + Assert.That(keyCache.Get(in hot), Is.True); + } + } + private static DeterministicHashKey[] BuildKeys(int capacity, int count) { int setCount = (int)BitOperations.RoundUpToPowerOf2((uint)((capacity + Ways - 1) / Ways)); diff --git a/src/Nethermind/Nethermind.Core.Test/Threading/StripedLongTests.cs b/src/Nethermind/Nethermind.Core.Test/Threading/StripedLongTests.cs new file mode 100644 index 000000000000..89a5412ba39a --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Threading/StripedLongTests.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading.Tasks; +using Nethermind.Core.Threading; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Threading; + +[Parallelizable(ParallelScope.All)] +public class StripedLongTests +{ + [Test] + public void Sum_is_exact_under_concurrent_mixed_adds() + { + StripedLong counter = new(); + const int threads = 8; + const int iterations = 100_000; + + Parallel.For(0, threads, _ => + { + for (int i = 0; i < iterations; i++) + { + counter.Increment(); + counter.Add(3); + counter.Add(-2); + } + }); + + Assert.That(counter.Sum, Is.EqualTo((long)threads * iterations * 2)); + } + + [Test] + public void Negative_totals_are_representable() + { + StripedLong counter = new(); + counter.Add(-5); + counter.Increment(); + Assert.That(counter.Sum, Is.EqualTo(-4)); + } + + [Test] + public void New_counter_sums_to_zero() => Assert.That(new StripedLong().Sum, Is.Zero); +} diff --git a/src/Nethermind/Nethermind.Core/Caching/AssociativeCache.cs b/src/Nethermind/Nethermind.Core/Caching/AssociativeCache.cs index f8cea79b933c..7aacbe9d142d 100644 --- a/src/Nethermind/Nethermind.Core/Caching/AssociativeCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/AssociativeCache.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -79,13 +80,6 @@ public sealed class AssociativeCache /// private long _epochAndCount; - /// - /// Monotonic counter for eviction-age tracking. Interlocked.Increment is faster - /// than Stopwatch.GetTimestamp() (RDTSC) — ~7.6ns vs ~19ns single-threaded, - /// and scales better under contention (20% faster at 8 threads). - /// - private long _ticker; - public int Count => ReadCount(ref _epochAndCount); public AssociativeCache(int maxCapacity) @@ -166,11 +160,18 @@ private bool TryGetCore(in TKey key, out TValue? value) if (h1 == h2 && storedKey.Equals(in key)) { // JIT eliminates this branch entirely per TRefreshTicker instantiation. + // Eviction age uses the high-resolution clock rather than a shared counter: a + // per-hit Interlocked on a cache-wide field is a serialized cross-core RMW under + // concurrent readers (and it dirtied the line _epochAndCount lives on, which + // every TryGet reads first). Single-threaded the clock read loses a few ns to the + // old Interlocked (and more on hosts whose clocksource is not TSC), but it writes + // only this entry's own line, so hits scale with reader count — the regime that + // motivated the change. Do not flip back to a shared counter for the ns. // Ticker store without the set gate is safe: 8-byte aligned long is atomic on // x64/ARM64 hardware. A race with a concurrent Set only affects eviction ranking, // not key/value correctness — the "losing" ticker value is simply slightly stale. if (TRefreshTicker.IsActive) - e.Ticker = Interlocked.Increment(ref _ticker); + e.Ticker = Stopwatch.GetTimestamp(); value = storedValue; return true; } @@ -232,8 +233,7 @@ private bool SetCore(in TKey key, TValue val, int baseIdx, long hashPart) { if ((h & HashMask) == hashPart && e.Key.Equals(in key)) { - long now = Interlocked.Increment(ref _ticker); - WriteEntry(ref e, h, in key, val, tagToStore, now); + WriteEntry(ref e, h, in key, val, tagToStore, Stopwatch.GetTimestamp()); return false; } } @@ -249,7 +249,7 @@ private bool SetCore(in TKey key, TValue val, int baseIdx, long hashPart) if (ReadEpoch(ref _epochAndCount) != epochTag) continue; - long timestamp = Interlocked.Increment(ref _ticker); + long timestamp = Stopwatch.GetTimestamp(); int target = bestEmpty >= 0 ? bestEmpty : bestStale >= 0 diff --git a/src/Nethermind/Nethermind.Core/Caching/AssociativeKeyCache.cs b/src/Nethermind/Nethermind.Core/Caching/AssociativeKeyCache.cs index f5b2735842cf..123df13d8a90 100644 --- a/src/Nethermind/Nethermind.Core/Caching/AssociativeKeyCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/AssociativeKeyCache.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -29,7 +30,6 @@ public sealed class AssociativeKeyCache private readonly int _hashShift; private readonly int[] _setGates; private long _epochAndCount; - private long _ticker; public int Count => ReadCount(ref _epochAndCount); @@ -95,11 +95,18 @@ private bool GetCore(in TKey key) if (h1 == h2 && storedKey.Equals(in key)) { // JIT eliminates this branch entirely per TRefreshTicker instantiation. + // Eviction age uses the high-resolution clock rather than a shared counter: a + // per-hit Interlocked on a cache-wide field is a serialized cross-core RMW under + // concurrent readers (and it dirtied the line _epochAndCount lives on, which + // every Get reads first). Single-threaded the clock read loses a few ns to the + // old Interlocked (and more on hosts whose clocksource is not TSC), but it writes + // only this entry's own line, so hits scale with reader count — the regime that + // motivated the change. Do not flip back to a shared counter for the ns. // Ticker store without the set gate is safe: 8-byte aligned long is atomic on // x64/ARM64 hardware. A race with a concurrent Set only affects eviction ranking, // not key/value correctness — the "losing" ticker value is simply slightly stale. if (TRefreshTicker.IsActive) - e.Ticker = Interlocked.Increment(ref _ticker); + e.Ticker = Stopwatch.GetTimestamp(); return true; } } @@ -158,7 +165,7 @@ private bool SetCore(in TKey key, int baseIdx, long hashPart) // Unlike AssociativeCache.SetCore (which calls WriteEntry to update the value), // the key-only variant has nothing to write, so a bare ticker store suffices. // The seqlock header is unchanged, which is correct: readers see a stable entry. - e.Ticker = Interlocked.Increment(ref _ticker); + e.Ticker = Stopwatch.GetTimestamp(); return false; } } @@ -174,7 +181,7 @@ private bool SetCore(in TKey key, int baseIdx, long hashPart) if (ReadEpoch(ref _epochAndCount) != epochTag) continue; - long timestamp = Interlocked.Increment(ref _ticker); + long timestamp = Stopwatch.GetTimestamp(); int target = bestEmpty >= 0 ? bestEmpty : bestStale >= 0 diff --git a/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs b/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs index 93d5a0fa4a09..6f3e6f901b7b 100644 --- a/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs +++ b/src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs @@ -87,7 +87,14 @@ protected void MarkAccessed(int position) ref long accessedBitmapWord = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(HasBeenAccessedBitmap), offset); - Interlocked.Or(ref accessedBitmapWord, flags); + // Test first: hot entries are re-marked far more often than the clock hand clears them, + // and an unconditional lock-prefixed Or on a word shared by 64 entries serializes + // concurrent readers. A racing clear between the read and the Or loses at most one + // access mark — the same tolerance the eviction algorithm already has. + if ((Volatile.Read(ref accessedBitmapWord) & flags) != flags) + { + Interlocked.Or(ref accessedBitmapWord, flags); + } } protected void MarkAccessedNonConcurrent(int position) diff --git a/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs b/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs index 9a569ae9f87d..e2066c00bc0b 100644 --- a/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs +++ b/src/Nethermind/Nethermind.Core/Caching/StaticPool.cs @@ -59,8 +59,9 @@ namespace Nethermind.Core.Caching; /// public static T Rent() { - // Try to pop from the global pool — this is only hit when a thread - // has exhausted its own fast slot or is cross-thread renting. + // Every rent reaches the shared queue: there is no per-thread tier here, unlike + // Nethermind.Evm's EvmObjectPool, so each rent and return costs a contended atomic. Hosting + // this on that pool would extend the same win to StackList and the block-access-list pool. if (Volatile.Read(ref _poolCount) > 0 && _pool.TryDequeue(out T? item)) { // We track count manually with Interlocked ops instead of using queue.Count. diff --git a/src/Nethermind/Nethermind.Core/Threading/StripedLong.cs b/src/Nethermind/Nethermind.Core/Threading/StripedLong.cs new file mode 100644 index 000000000000..eb2b87ca351f --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Threading/StripedLong.cs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Nethermind.Core.Threading; + +/// +/// Additive counter safe for hot concurrent paths: increments land on a per-core slot, reads sum +/// the slots. +/// +/// +/// A shared counter word turns every increment into a serialized cross-core cache-line transfer +/// once several threads hit it (RPC workers, prewarm workers). Striping by +/// keeps the RMW local to the core in the common case; +/// the atomic add only guards against threads that share or migrate between cores. Slots are +/// 128-byte spaced — same isolation as (adjacent-line prefetch +/// pairs lines) — with a leading pad stride so the first slot is also isolated from whatever +/// precedes the array in memory. Each instance allocates (stripes + 1) * 128 bytes, where stripes +/// is rounded up to a power of two, captured once at +/// type initialization (later CPU hot-add is not tracked; the mask just folds new ids onto +/// existing slots). Reads are O(stripes) and torn only across slots: fine for metrics, not for +/// invariants. +/// +public sealed class StripedLong +{ + // 16 longs = 128 bytes between live slots. + private const int SlotStride = 16; + private static readonly int s_stripeMask = (int)BitOperations.RoundUpToPowerOf2((uint)Environment.ProcessorCount) - 1; + + private readonly long[] _slots = new long[(s_stripeMask + 2) * SlotStride]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Add(long value) + => Interlocked.Add(ref _slots[((Thread.GetCurrentProcessorId() & s_stripeMask) + 1) * SlotStride], value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Increment() => Add(1); + + public long Sum + { + get + { + long[] slots = _slots; + long sum = 0; + for (int stripe = 0; stripe <= s_stripeMask; stripe++) + { + sum += Volatile.Read(ref slots[(stripe + 1) * SlotStride]); + } + return sum; + } + } +} diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index adb528bfc1a7..84e0e4268db2 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -93,8 +93,10 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor private readonly List _metricsUpdaters = []; - internal CacheLinePaddedLong _allocatedSpan; - private CacheLinePaddedLong _totalReads; + // Striped: every concurrent reader (RPC workers, prewarm workers) updates these per Get, and + // a single shared word per DB serializes them under load. + internal readonly StripedLong _allocatedSpan = new(); + private readonly StripedLong _totalReads = new(); private CacheLinePaddedLong _totalWrites; private readonly DisposableLazy? _iteratorManager; @@ -361,7 +363,7 @@ private void RepairIfCorrupted(DbOptions dbOptions) _fileSystem.File.Delete(corruptMarker); } - protected internal void UpdateReadMetrics() => Interlocked.Increment(ref _totalReads.Value); + protected internal void UpdateReadMetrics() => _totalReads.Increment(); protected internal void UpdateWriteMetrics() => Interlocked.Increment(ref _totalWrites.Value); @@ -380,7 +382,7 @@ public IDbMeta.DbMetric GatherMetric() CacheSize = 0, IndexSize = 0, MemtableSize = 0, - TotalReads = _totalReads.Value, + TotalReads = _totalReads.Sum, TotalWrites = _totalWrites.Value, }; } @@ -390,7 +392,7 @@ public IDbMeta.DbMetric GatherMetric() CacheSize = GetCacheSize(), IndexSize = GetIndexSize(), MemtableSize = GetMemtableSize(), - TotalReads = _totalReads.Value, + TotalReads = _totalReads.Sum, TotalWrites = _totalWrites.Value, }; } @@ -991,8 +993,12 @@ internal Span GetSpanWithColumnFamily(scoped ReadOnlySpan key, Colum if (!span.IsNullOrEmpty()) { - Interlocked.Increment(ref _allocatedSpan.Value); - GC.AddMemoryPressure(span.Length); + _allocatedSpan.Increment(); + // Pressure hints exist so the GC accounts for sizeable native memory held alive by + // managed wrappers. Sub-threshold spans are transient (released within the request) + // and each Add/Remove pair mutates GC-global accounting — a contended cost per DB + // read under concurrent load. The threshold must match DangerousReleaseMemory. + if (span.Length >= GcPressureSpanThreshold) GC.AddMemoryPressure(span.Length); } return span; } @@ -1087,12 +1093,14 @@ internal void MergeWithColumnFamily(ReadOnlySpan key, ColumnFamilyHandle? } } + private const int GcPressureSpanThreshold = 16 * 1024; + public void DangerousReleaseMemory(in ReadOnlySpan span) { if (!span.IsNullOrEmpty()) { - Interlocked.Decrement(ref _allocatedSpan.Value); - GC.RemoveMemoryPressure(span.Length); + _allocatedSpan.Add(-1); + if (span.Length >= GcPressureSpanThreshold) GC.RemoveMemoryPressure(span.Length); } _db.DangerousReleaseMemory(span); } diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 0d8c12e4e652..b1161841b8f5 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -429,10 +429,10 @@ private long AllocatedSpan { if (_db is ColumnDb columnDb) { - return columnDb._mainDb._allocatedSpan.Value; + return columnDb._mainDb._allocatedSpan.Sum; } - return (_db as DbOnTheRocks)._allocatedSpan.Value; + return (_db as DbOnTheRocks)._allocatedSpan.Sum; } } diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs index cd5809dd4ea9..1b7e5b594324 100644 --- a/src/Nethermind/Nethermind.Db/Metrics.cs +++ b/src/Nethermind/Nethermind.Db/Metrics.cs @@ -28,30 +28,45 @@ public static class Metrics private static bool IsBlockProcessingThread => ProcessingThread.IsBlockProcessingThread; + // The block-processing thread keeps its dedicated padded word; every other thread (RPC + // workers, prewarm workers) previously shared ONE "other" word per counter, making each + // per-read update a contended cross-core RMW under concurrent load — striped instead. [CounterMetric] [Description("Number of State Trie cache hits.")] - public static long StateTreeCache => _mainStateTreeCacheHits.Value + _otherStateTreeCacheHits.Value; + public static long StateTreeCache => _mainStateTreeCacheHits.Value + _otherStateTreeCacheHits.Sum; private static CacheLinePaddedLong _mainStateTreeCacheHits; - private static CacheLinePaddedLong _otherStateTreeCacheHits; + private static readonly StripedLong _otherStateTreeCacheHits = new(); // Exposed so consumers (e.g. ProcessingStats) can compute block-level deltas that exclude // background prewarmer activity, which runs with IsBlockProcessingThread = false. internal static long MainThreadStateTreeCache => _mainStateTreeCacheHits.Value; - internal static void AddStateTreeCacheHits(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainStateTreeCacheHits.Value : ref _otherStateTreeCacheHits.Value, count); + internal static void AddStateTreeCacheHits(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainStateTreeCacheHits.Value, count); + else _otherStateTreeCacheHits.Add(count); + } [CounterMetric] [Description("Number of State Trie reads.")] - public static long StateTreeReads => _mainStateTreeReads.Value + _otherStateTreeReads.Value; + public static long StateTreeReads => _mainStateTreeReads.Value + _otherStateTreeReads.Sum; private static CacheLinePaddedLong _mainStateTreeReads; - private static CacheLinePaddedLong _otherStateTreeReads; + private static readonly StripedLong _otherStateTreeReads = new(); internal static long MainThreadStateTreeReads => _mainStateTreeReads.Value; - internal static void AddStateTreeReads(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainStateTreeReads.Value : ref _otherStateTreeReads.Value, count); + internal static void AddStateTreeReads(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainStateTreeReads.Value, count); + else _otherStateTreeReads.Add(count); + } [CounterMetric] [Description("Number of State Reader reads.")] - public static long StateReaderReads => _mainStateReaderReads.Value + _otherStateReaderReads.Value; + public static long StateReaderReads => _mainStateReaderReads.Value + _otherStateReaderReads.Sum; private static CacheLinePaddedLong _mainStateReaderReads; - private static CacheLinePaddedLong _otherStateReaderReads; - internal static void IncrementStateReaderReads() => Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainStateReaderReads.Value : ref _otherStateReaderReads.Value); + private static readonly StripedLong _otherStateReaderReads = new(); + internal static void IncrementStateReaderReads() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainStateReaderReads.Value); + else _otherStateReaderReads.Increment(); + } [CounterMetric] [Description("Number of state trie writes.")] @@ -71,57 +86,81 @@ public static class Metrics [CounterMetric] [Description("Number of storage trie cache hits.")] - public static long StorageTreeCache => _mainStorageTreeCache.Value + _otherStorageTreeCache.Value; + public static long StorageTreeCache => _mainStorageTreeCache.Value + _otherStorageTreeCache.Sum; private static CacheLinePaddedLong _mainStorageTreeCache; - private static CacheLinePaddedLong _otherStorageTreeCache; + private static readonly StripedLong _otherStorageTreeCache = new(); internal static long MainThreadStorageTreeCache => _mainStorageTreeCache.Value; - internal static void AddStorageTreeCache(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainStorageTreeCache.Value : ref _otherStorageTreeCache.Value, count); + internal static void AddStorageTreeCache(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainStorageTreeCache.Value, count); + else _otherStorageTreeCache.Add(count); + } [CounterMetric] [Description("Number of storage trie reads.")] - public static long StorageTreeReads => _mainStorageTreeReads.Value + _otherStorageTreeReads.Value; + public static long StorageTreeReads => _mainStorageTreeReads.Value + _otherStorageTreeReads.Sum; private static CacheLinePaddedLong _mainStorageTreeReads; - private static CacheLinePaddedLong _otherStorageTreeReads; + private static readonly StripedLong _otherStorageTreeReads = new(); internal static long MainThreadStorageTreeReads => _mainStorageTreeReads.Value; - internal static void AddStorageTreeReads(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainStorageTreeReads.Value : ref _otherStorageTreeReads.Value, count); + internal static void AddStorageTreeReads(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainStorageTreeReads.Value, count); + else _otherStorageTreeReads.Add(count); + } [CounterMetric] [Description("Number of pre-block (prewarmer-shared) cache hits for accounts, counted on the consumer scope only (populator probes excluded); first-in-block touches, so hits/(hits+misses) = prewarm coverage.")] - public static long PreBlockCacheAccountHits => _mainPreBlockAccountHits.Value + _otherPreBlockAccountHits.Value; + public static long PreBlockCacheAccountHits => _mainPreBlockAccountHits.Value + _otherPreBlockAccountHits.Sum; private static CacheLinePaddedLong _mainPreBlockAccountHits; - private static CacheLinePaddedLong _otherPreBlockAccountHits; + private static readonly StripedLong _otherPreBlockAccountHits = new(); internal static long MainThreadPreBlockAccountHits => _mainPreBlockAccountHits.Value; - internal static void AddPreBlockAccountHits(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainPreBlockAccountHits.Value : ref _otherPreBlockAccountHits.Value, count); + internal static void AddPreBlockAccountHits(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainPreBlockAccountHits.Value, count); + else _otherPreBlockAccountHits.Add(count); + } [CounterMetric] [Description("Number of pre-block (prewarmer-shared) cache misses for accounts, counted on the consumer scope only (populator probes excluded).")] - public static long PreBlockCacheAccountMisses => _mainPreBlockAccountMisses.Value + _otherPreBlockAccountMisses.Value; + public static long PreBlockCacheAccountMisses => _mainPreBlockAccountMisses.Value + _otherPreBlockAccountMisses.Sum; private static CacheLinePaddedLong _mainPreBlockAccountMisses; - private static CacheLinePaddedLong _otherPreBlockAccountMisses; + private static readonly StripedLong _otherPreBlockAccountMisses = new(); internal static long MainThreadPreBlockAccountMisses => _mainPreBlockAccountMisses.Value; - internal static void AddPreBlockAccountMisses(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainPreBlockAccountMisses.Value : ref _otherPreBlockAccountMisses.Value, count); + internal static void AddPreBlockAccountMisses(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainPreBlockAccountMisses.Value, count); + else _otherPreBlockAccountMisses.Add(count); + } [CounterMetric] [Description("Number of pre-block (prewarmer-shared) cache hits for storage slots, counted on the consumer scope only (populator probes excluded); first-in-block touches, so hits/(hits+misses) = prewarm coverage.")] - public static long PreBlockCacheStorageHits => _mainPreBlockStorageHits.Value + _otherPreBlockStorageHits.Value; + public static long PreBlockCacheStorageHits => _mainPreBlockStorageHits.Value + _otherPreBlockStorageHits.Sum; private static CacheLinePaddedLong _mainPreBlockStorageHits; - private static CacheLinePaddedLong _otherPreBlockStorageHits; + private static readonly StripedLong _otherPreBlockStorageHits = new(); internal static long MainThreadPreBlockStorageHits => _mainPreBlockStorageHits.Value; - internal static void AddPreBlockStorageHits(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainPreBlockStorageHits.Value : ref _otherPreBlockStorageHits.Value, count); + internal static void AddPreBlockStorageHits(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainPreBlockStorageHits.Value, count); + else _otherPreBlockStorageHits.Add(count); + } [CounterMetric] [Description("Number of pre-block (prewarmer-shared) cache misses for storage slots, counted on the consumer scope only (populator probes excluded).")] - public static long PreBlockCacheStorageMisses => _mainPreBlockStorageMisses.Value + _otherPreBlockStorageMisses.Value; + public static long PreBlockCacheStorageMisses => _mainPreBlockStorageMisses.Value + _otherPreBlockStorageMisses.Sum; private static CacheLinePaddedLong _mainPreBlockStorageMisses; - private static CacheLinePaddedLong _otherPreBlockStorageMisses; + private static readonly StripedLong _otherPreBlockStorageMisses = new(); internal static long MainThreadPreBlockStorageMisses => _mainPreBlockStorageMisses.Value; - internal static void AddPreBlockStorageMisses(long count) => Interlocked.Add(ref IsBlockProcessingThread ? ref _mainPreBlockStorageMisses.Value : ref _otherPreBlockStorageMisses.Value, count); + internal static void AddPreBlockStorageMisses(long count) + { + if (IsBlockProcessingThread) Interlocked.Add(ref _mainPreBlockStorageMisses.Value, count); + else _otherPreBlockStorageMisses.Add(count); + } [CounterMetric] [Description("Number of storage reader reads.")] - public static long StorageReaderReads => _storageReaderReads.Value; - private static CacheLinePaddedLong _storageReaderReads; - internal static void IncrementStorageReaderReads() => Interlocked.Increment(ref _storageReaderReads.Value); + public static long StorageReaderReads => _storageReaderReads.Sum; + private static readonly StripedLong _storageReaderReads = new(); + internal static void IncrementStorageReaderReads() => _storageReaderReads.Increment(); [CounterMetric] [Description("Number of storage trie writes.")] diff --git a/src/Nethermind/Nethermind.Evm.Test/EvmObjectPoolTests.cs b/src/Nethermind/Nethermind.Evm.Test/EvmObjectPoolTests.cs index e573d6d37d09..3559ae2ed516 100644 --- a/src/Nethermind/Nethermind.Evm.Test/EvmObjectPoolTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/EvmObjectPoolTests.cs @@ -33,6 +33,7 @@ private sealed class BoundItem(int id) : Item(id); private sealed class CrossThreadItem(int id) : Item(id); private sealed class ChurnItem(int id) : Item(id); private sealed class GuardItem(int id) : Item(id); + private sealed class OrderItem(int id) : Item(id); [Test] public void Empty_pool_reports_no_item() @@ -131,6 +132,34 @@ public void Items_overflowed_on_one_thread_are_rentable_on_another() Assert.That(seen, Is.SameAs(overflowed)); } + [Test] + public void Local_tier_is_preferred_over_a_non_empty_shared_tier() + { + const int localCapacity = 2; + EvmObjectPool pool = new(localCapacity); + + // The local tier fills first, so ids 0..1 stay on this thread and 2..5 overflow to the shared + // queue. The local pair must be handed back first even though the shared queue holds newer + // items - that ordering is what keeps the hot path free of atomics. + for (int id = 0; id < localCapacity * 3; id++) + { + pool.Enqueue(new OrderItem(id)); + } + + Assert.That(pool.TryDequeue(out OrderItem? first), Is.True); + Assert.That(pool.TryDequeue(out OrderItem? second), Is.True); + using (Assert.EnterMultipleScope()) + { + // LIFO within the local tier. + Assert.That(first!.Id, Is.EqualTo(1), "local tier must answer before the shared queue"); + Assert.That(second!.Id, Is.EqualTo(0), "local tier must answer before the shared queue"); + } + + // Only now does the shared tier answer, oldest first. + Assert.That(pool.TryDequeue(out OrderItem? fromShared), Is.True); + Assert.That(fromShared!.Id, Is.EqualTo(2), "shared tier answers once the local tier is dry"); + } + [Test] public void Second_pool_over_the_same_item_type_is_rejected() { diff --git a/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs b/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs index 366244b9666c..741b904f27bc 100644 --- a/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs @@ -408,6 +408,35 @@ public void IncrementalGrowth_preserves_written_data_and_zeroes_new_regions(int Assert.That(first.ToArray(), Is.EqualTo(word), "originally written word must survive re-rent"); } + // Sizes bracket the pooling boundaries: 64 KiB (largest fresh allocation), 256 KiB (largest + // thread-cached buffer) and above (shared pools) — every rent source must hand out zeroed- + // reading memory even after a deliberately dirtied buffer was recycled through it. + [TestCase(64 * 1024 - 32)] + [TestCase(64 * 1024)] + [TestCase(64 * 1024 + 32)] + [TestCase(256 * 1024)] + [TestCase(256 * 1024 + 32)] + [TestCase(1024 * 1024)] + public void Growth_across_pooling_boundaries_reads_zero_and_recycles_clean(int size) + { + byte[] word = TestItem.KeccakA.BytesToArray(); + + EvmPooledMemory original = new(); + Assert.That(original.TrySaveWord(0, word), Is.True); + Assert.That(original.TryLoadSpan((UInt256)(size - EvmPooledMemory.WordSize), (UInt256)EvmPooledMemory.WordSize, out Span tail), Is.True); + Assert.That(tail.ToArray(), Is.EqualTo(new byte[EvmPooledMemory.WordSize]), "grown tail must read as zero"); + Assert.That(original.TryLoadSpan(0, (UInt256)EvmPooledMemory.WordSize, out Span head), Is.True); + Assert.That(head.ToArray(), Is.EqualTo(word), "written word must survive growth"); + Assert.That(original.TryLoadSpan(0, (UInt256)size, out Span whole), Is.True); + whole.Fill(0xff); + original.Dispose(); + + EvmPooledMemory recycled = new(); + Assert.That(recycled.TryLoadSpan(0, (UInt256)size, out Span reused), Is.True); + Assert.That(reused.IndexOfAnyExcept((byte)0), Is.EqualTo(-1), "recycled buffer must read as zero"); + recycled.Dispose(); + } + [TestCaseSource(nameof(ZeroExtendedCopyCases))] public void CopyFromZeroExtendedAfterGas_copies_and_zeroes_only_the_destination( byte[] source, diff --git a/src/Nethermind/Nethermind.Evm/EvmObjectPool.std.cs b/src/Nethermind/Nethermind.Evm/EvmObjectPool.std.cs index e9a9705a63a0..5afdf2990b0e 100644 --- a/src/Nethermind/Nethermind.Evm/EvmObjectPool.std.cs +++ b/src/Nethermind/Nethermind.Evm/EvmObjectPool.std.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; namespace Nethermind.Evm; @@ -16,17 +17,35 @@ namespace Nethermind.Evm; /// Frames rent and return in LIFO order on one thread, so the local tier serves almost every request /// with no atomics; the shared queue only absorbs deep-frame overflow and cross-thread imbalance. /// A single instead funnelled every thread through one segment head, -/// which showed up as in TryDequeue 4.4x worse on arm64 than x64 — a -/// contended CAS costs far more under LL/SC than under x86's TSO. +/// where the contended CAS dominates - markedly worse on arm64 under LL/SC than on x64 under TSO. /// /// The local tier is static per closed generic type, so a second pool over the same -/// would hand out this one's items; a debug assertion guards that. +/// would hand out this one's items; the constructor rejects that. /// /// internal sealed class EvmObjectPool { private const int DefaultLocalCapacity = 16; + /// Items and count in one object, so a pool operation costs one thread-static lookup. + /// + /// Two fields would be a GC and a non-GC thread-static, which + /// live in separate per-thread blocks: the JIT cannot share a base between them, so each of + /// and would pay two out-of-line base lookups on the + /// shared-generic instantiations - four per call frame. + /// + private sealed class LocalTier + { + public T[] Items = null!; + public int Count; + } + + [ThreadStatic] private static LocalTier? _local; + + // Never decremented: the pools are singletons built in static field initialisers, so this is a + // construct-once count, not a live one. A test needing a second pool needs a distinct T. + private static int _instanceCount; + private readonly ConcurrentQueue _shared = new(); private readonly int _localCapacity; private readonly int _maxShared; @@ -38,13 +57,6 @@ internal sealed class EvmObjectPool /// private int _sharedCount; - [ThreadStatic] private static T[]? _local; - [ThreadStatic] private static int _localCount; - - // Never decremented: the pools are singletons built in static field initialisers, so this is a - // construct-once count, not a live one. A test needing a second pool needs a distinct T. - private static int _instanceCount; - /// Items each thread may retain. Overflow goes to the shared queue. /// Items the shared queue may retain; further returns are dropped. public EvmObjectPool(int localCapacity = DefaultLocalCapacity, int maxShared = int.MaxValue) @@ -67,15 +79,19 @@ public EvmObjectPool(int localCapacity = DefaultLocalCapacity, int maxShared = i public bool TryDequeue([MaybeNullWhen(false)] out T item) { - int count = _localCount - 1; - if (count >= 0) + LocalTier? local = _local; + if (local is not null) { - T[] local = _local!; - item = local[count]; - // Don't keep the item reachable while it is rented out. - local[count] = default!; - _localCount = count; - return true; + int count = local.Count - 1; + if (count >= 0) + { + T[] items = local.Items; + item = items[count]; + // Don't keep the item reachable while it is rented out. + items[count] = default!; + local.Count = count; + return true; + } } return TryDequeueShared(out item); @@ -83,18 +99,29 @@ public bool TryDequeue([MaybeNullWhen(false)] out T item) public void Enqueue(T item) { - T[] local = _local ??= new T[_localCapacity]; - int count = _localCount; - if (count < local.Length) + LocalTier local = _local ?? CreateLocalTier(); + T[] items = local.Items; + int count = local.Count; + if ((uint)count < (uint)items.Length) { - local[count] = item; - _localCount = count + 1; + // The array is created in CreateLocalTier as exactly T[] and never escapes this type, so + // the covariance check a plain store would emit is dead weight: on the three reference-type + // pools the generic code is shared, the JIT cannot prove the store type-exact, and it + // lowers to an out-of-line CORINFO_HELP_ARRADDR_ST on the per-frame return path. The + // (uint) comparison above replaces the bounds check that Unsafe.Add skips. + Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(items), (uint)count) = item; + local.Count = count + 1; return; } EnqueueShared(item); } + // Out of line so the array allocation and its generic-dictionary lookup stay off the per-frame + // return path, as TryDequeueShared and EnqueueShared already are. + [MethodImpl(MethodImplOptions.NoInlining)] + private LocalTier CreateLocalTier() => _local = new LocalTier { Items = new T[_localCapacity] }; + [MethodImpl(MethodImplOptions.NoInlining)] private bool TryDequeueShared([MaybeNullWhen(false)] out T item) { diff --git a/src/Nethermind/Nethermind.Evm/EvmObjectPool.zkevm.cs b/src/Nethermind/Nethermind.Evm/EvmObjectPool.zkevm.cs new file mode 100644 index 000000000000..4b3067ea4d20 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/EvmObjectPool.zkevm.cs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; + +namespace Nethermind.Evm; + +/// +/// Object pool for the EVM call machinery, single-threaded guest variant. Same name and shape as the +/// mainline pool, so every call site is unconditional. +/// +/// +/// The guest runs one thread, so the mainline pool's two tiers collapse into one stack: interlocked +/// bookkeeping and segment walking would be pure overhead. The capacity arguments are accepted and +/// ignored - a single thread cannot contend with itself, so there is nothing for the split to buy. +/// +internal sealed class EvmObjectPool +{ + private readonly Stack _items = new(); + + public EvmObjectPool(int localCapacity = 0, int maxShared = 0) + { + // Validated as mainline does, so a bad argument fails the same way in both builds. + ArgumentOutOfRangeException.ThrowIfNegative(localCapacity); + ArgumentOutOfRangeException.ThrowIfNegative(maxShared); + } + + public bool TryDequeue(out T item) => _items.TryPop(out item); + + public void Enqueue(T item) => _items.Push(item); + + public int Count => _items.Count; +} diff --git a/src/Nethermind/Nethermind.Evm/EvmPooledMemory.cs b/src/Nethermind/Nethermind.Evm/EvmPooledMemory.cs index 43fc2a8621d3..f6cb7be1ed6a 100644 --- a/src/Nethermind/Nethermind.Evm/EvmPooledMemory.cs +++ b/src/Nethermind/Nethermind.Evm/EvmPooledMemory.cs @@ -462,11 +462,21 @@ private void EnsureRented() } private const int MinRentSize = 1_024; - private const int MaxCachedArrayLength = 1 << 16; + // Above this, a cache miss rents from the shared pool instead of allocating (pow2 sizes from + // here up are LOH-sized). + private const int MaxNewAllocLength = 1 << 16; + // Buffers up to this stay in the per-thread cache. Frames zero-extend their buffer on growth + // (RentSlow), and a buffer that round-tripped through the shared pool between frames comes + // back cold and coherence-invalidated under concurrent load — so those zeroing stores stall. + // Keeping mid-size buffers on the renting thread keeps the lines warm; the byte budget bounds + // per-thread retention. + private const int MaxThreadCachedArrayLength = 1 << 18; + private const int MaxThreadCachedBytes = 1 << 21; private const int CacheSlots = 16; [ThreadStatic] private static byte[]?[]? _cachedArrays; [ThreadStatic] private static int _cachedArrayCount; + [ThreadStatic] private static int _cachedArrayBytes; // Cached dirty; RentSlow zero-extends past Size in chunks on growth. private static byte[] Rent(int minLength) @@ -479,13 +489,14 @@ private static byte[] Rent(int minLength) if (candidate.Length >= minLength) { _cachedArrayCount = cachedArrayCount; + _cachedArrayBytes -= candidate.Length; cache[i] = cache[cachedArrayCount]; cache[cachedArrayCount] = null; return candidate; } } - if (minLength > MaxCachedArrayLength) + if (minLength > MaxNewAllocLength) { return RentLarge(minLength); } @@ -495,16 +506,22 @@ private static byte[] Rent(int minLength) private static void Return(byte[] array) { - if (array.Length > MaxCachedArrayLength) + if (array.Length <= MaxThreadCachedArrayLength + && _cachedArrayCount < CacheSlots + && _cachedArrayBytes + array.Length <= MaxThreadCachedBytes) { - ReturnLarge(array); + byte[]?[] cache = _cachedArrays ??= new byte[CacheSlots][]; + cache[_cachedArrayCount++] = array; + _cachedArrayBytes += array.Length; return; } - byte[]?[] cache = _cachedArrays ??= new byte[CacheSlots][]; - if (_cachedArrayCount < CacheSlots) + // Provenance: arrays <= MaxNewAllocLength are plain allocations, larger ones came from + // RentLarge — an array must never reach a pool it was not rented from, so sub-threshold + // arrays that miss the thread cache are dropped for the GC rather than pooled. + if (array.Length > MaxNewAllocLength) { - cache[_cachedArrayCount++] = array; + ReturnLarge(array); } } diff --git a/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs b/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs index aab8f6df247d..3313575679a7 100644 --- a/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs +++ b/src/Nethermind/Nethermind.Evm/ExecutionEnvironment.cs @@ -5,12 +5,7 @@ using Nethermind.Core; using Nethermind.Evm.CodeAnalysis; using Nethermind.Int256; -using Queue = -#if ZK_EVM - Nethermind.Evm.ZkEvmQueue; -#else - Nethermind.Evm.EvmObjectPool; -#endif +using Queue = Nethermind.Evm.EvmObjectPool; namespace Nethermind.Evm { diff --git a/src/Nethermind/Nethermind.Evm/Metrics.cs b/src/Nethermind/Nethermind.Evm/Metrics.cs index b6c59bd44bdf..c462b765962d 100644 --- a/src/Nethermind/Nethermind.Evm/Metrics.cs +++ b/src/Nethermind/Nethermind.Evm/Metrics.cs @@ -47,18 +47,21 @@ public partial class Metrics { private static bool IsBlockProcessingThread => ProcessingThread.IsBlockProcessingThread; + // Fires per code lookup, i.e. per call frame: the single shared "other" word made this a + // contended cross-core RMW for every concurrent RPC/prewarm thread — striped instead. [CounterMetric] [Description("Number of Code DB cache reads.")] - public static long CodeDbCache => _mainCodeDbCache.Value + _otherCodeDbCache.Value; + public static long CodeDbCache => _mainCodeDbCache.Value + _otherCodeDbCache.Sum; private static CacheLinePaddedLong _mainCodeDbCache; - private static CacheLinePaddedLong _otherCodeDbCache; + private static readonly StripedLong _otherCodeDbCache = new(); [Description("Number of Code DB cache reads on main processing thread.")] public static long MainThreadCodeDbCache => _mainCodeDbCache.Value; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void IncrementCodeDbCache() { if (!ExecutionMetricsFlag.IsActive) return; - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainCodeDbCache.Value : ref _otherCodeDbCache.Value); + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainCodeDbCache.Value); + else _otherCodeDbCache.Increment(); } [CounterMetric] [Description("Number of EVM exceptions thrown by contracts.")] diff --git a/src/Nethermind/Nethermind.Evm/StackAccessTracker.cs b/src/Nethermind/Nethermind.Evm/StackAccessTracker.cs index 37dadd63e95f..a7c2ff8bb650 100644 --- a/src/Nethermind/Nethermind.Evm/StackAccessTracker.cs +++ b/src/Nethermind/Nethermind.Evm/StackAccessTracker.cs @@ -87,13 +87,9 @@ public void Dispose() private sealed class TrackingState { -#if ZK_EVM - private static readonly ZkEvmQueue _trackerPool = new(); -#else // Rented once per top-level execution, not per frame, so one slot per thread is the whole win; // the collections a returned state keeps sized would otherwise be retained a slot at a time. private static readonly EvmObjectPool _trackerPool = new(localCapacity: 1); -#endif public static TrackingState RentState() { diff --git a/src/Nethermind/Nethermind.Evm/StackPool.cs b/src/Nethermind/Nethermind.Evm/StackPool.cs index 9d64bd45a68d..aebd2218cbb9 100644 --- a/src/Nethermind/Nethermind.Evm/StackPool.cs +++ b/src/Nethermind/Nethermind.Evm/StackPool.cs @@ -3,16 +3,12 @@ namespace Nethermind.Evm; -internal sealed partial class StackPool +// Stacks carry no gas-policy state, so one pool serves every VmState{TGasPolicy} instantiation rather +// than one per closed type. Static rather than a singleton instance: EvmObjectPool's local tier is +// static per pooled type, so a second StackPool would hand out this one's stacks - `static` makes that +// inexpressible instead of merely discouraged. +internal static partial class StackPool { - /// - /// The process-wide pool, and the only instance. Stacks carry no gas-policy state, so one serves - /// every instantiation rather than one per closed type. - /// - public static readonly StackPool Shared = new(); - - private StackPool() { } - // Also have parallel prewarming and Rpc calls private const int MaxStacksPooled = VirtualMachineStatics.MaxCallDepth * 2; public const int StackLength = (EvmStack.MaxStackSize + EvmStack.RegisterLength) * 32; @@ -22,7 +18,7 @@ private readonly struct StackItem(byte[] dataStack) public readonly byte[] DataStack = dataStack; } - public partial void ReturnStacks(byte[] dataStack); + public static partial void ReturnStacks(byte[] dataStack); - public partial byte[] RentStacks(); + public static partial byte[] RentStacks(); } diff --git a/src/Nethermind/Nethermind.Evm/StackPool.std.cs b/src/Nethermind/Nethermind.Evm/StackPool.std.cs index acd945d4de53..28ac3f4232bf 100644 --- a/src/Nethermind/Nethermind.Evm/StackPool.std.cs +++ b/src/Nethermind/Nethermind.Evm/StackPool.std.cs @@ -6,18 +6,21 @@ namespace Nethermind.Evm; -internal sealed partial class StackPool +internal static partial class StackPool { // Stacks are ~32KB and pinned, and MaxStacksPooled bounds only the shared tier, so the pinned // ceiling is MaxStacksPooled + LocalStacksPooled per thread that has run an EVM frame - held until // the thread dies, and RegisterRpcModules raises the thread-pool minimum by ProcessorCount. + // A retiring thread abandons its slots rather than returning them, so thread churn also costs fresh + // pinned allocations, and a parked slot is unreachable to a busy thread with a dry shared tier. Every + // abandoned array is the size of its replacement, so both stay a Gen2 and footprint cost. private const int LocalStacksPooled = 8; - private readonly EvmObjectPool _stackPool = new(LocalStacksPooled, MaxStacksPooled); + private static readonly EvmObjectPool _stackPool = new(LocalStacksPooled, MaxStacksPooled); - public partial void ReturnStacks(byte[] dataStack) => _stackPool.Enqueue(new(dataStack)); + public static partial void ReturnStacks(byte[] dataStack) => _stackPool.Enqueue(new(dataStack)); - public partial byte[] RentStacks() + public static partial byte[] RentStacks() { if (_stackPool.TryDequeue(out StackItem result)) { diff --git a/src/Nethermind/Nethermind.Evm/StackPool.zkevm.cs b/src/Nethermind/Nethermind.Evm/StackPool.zkevm.cs index 67835b475e51..f792bb6fad25 100644 --- a/src/Nethermind/Nethermind.Evm/StackPool.zkevm.cs +++ b/src/Nethermind/Nethermind.Evm/StackPool.zkevm.cs @@ -5,11 +5,11 @@ namespace Nethermind.Evm; -internal sealed partial class StackPool +internal static partial class StackPool { - private readonly ZkEvmQueue _stackPool = new(); + private static readonly EvmObjectPool _stackPool = new(); - public partial void ReturnStacks(byte[] dataStack) + public static partial void ReturnStacks(byte[] dataStack) { // Single-threaded guest: bound directly off the queue's O(1) count, no atomics needed. if (_stackPool.Count >= MaxStacksPooled) @@ -18,7 +18,7 @@ public partial void ReturnStacks(byte[] dataStack) _stackPool.Enqueue(new(dataStack)); } - public partial byte[] RentStacks() + public static partial byte[] RentStacks() { if (_stackPool.TryDequeue(out StackItem result)) return result.DataStack; diff --git a/src/Nethermind/Nethermind.Evm/VmState.cs b/src/Nethermind/Nethermind.Evm/VmState.cs index a0d6bb21a141..96231c6ebf57 100644 --- a/src/Nethermind/Nethermind.Evm/VmState.cs +++ b/src/Nethermind/Nethermind.Evm/VmState.cs @@ -20,13 +20,7 @@ namespace Nethermind.Evm; public class VmState : IDisposable where TGasPolicy : struct, IGasPolicy { - private static readonly -#if ZK_EVM - ZkEvmQueue> -#else - EvmObjectPool> -#endif - _statePool = new(); + private static readonly EvmObjectPool> _statePool = new(); public byte[]? DataStack; public TGasPolicy Gas; @@ -214,7 +208,7 @@ public void Dispose() if (DataStack is not null) { // Only return if initialized - StackPool.Shared.ReturnStacks(DataStack); + StackPool.ReturnStacks(DataStack); DataStack = null; } @@ -276,7 +270,7 @@ public void InitializeStacks(ITxTracer txTracer, ReadOnlySpan codeSpan, ou } [MethodImpl(MethodImplOptions.NoInlining)] - private static byte[] AllocateStacks() => StackPool.Shared.RentStacks(); + private static byte[] AllocateStacks() => StackPool.RentStacks(); private static ref byte As32AlignedRef(byte[] array) { diff --git a/src/Nethermind/Nethermind.Evm/ZkEvmQueue.zkevm.cs b/src/Nethermind/Nethermind.Evm/ZkEvmQueue.zkevm.cs deleted file mode 100644 index 0f04b870e236..000000000000 --- a/src/Nethermind/Nethermind.Evm/ZkEvmQueue.zkevm.cs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Collections.Generic; - -namespace Nethermind.Evm; - -/// -/// Single-threaded object pool for the ZisK guest. Drop-in replacement for the -/// pools used by -/// the EVM call machinery, whose interlocked / segment bookkeeping is pure -/// overhead with a single thread. Exposes the same TryDequeue / Enqueue shape -/// so call sites are unchanged. -/// -internal sealed class ZkEvmQueue -{ - private readonly Stack _items = new(); - - public bool TryDequeue(out T item) => _items.TryPop(out item); - - public void Enqueue(T item) => _items.Push(item); - - public int Count => _items.Count; -} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs index 0bfc398b4a7b..de64f99076e4 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs @@ -10,6 +10,7 @@ using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Exceptions; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; @@ -636,6 +637,36 @@ public void Invocation_limit_exceeded_suppresses_warning() Assert.That(response.Error!.SuppressWarning, Is.True); } + [Test] + public void Overload_rejections_are_counted_from_both_shedding_paths() + { + // Per-path deltas so a double-count on one path cannot masquerade as both paths counted. + // >= rather than == on each: the counter is a global metric other parallel tests may bump. + long beforeInvocation = Metrics.JsonRpcOverloadRejections; + + // During-invocation path: the override-environment cap throws from inside the handler. + IEthRpcModule ethRpcModule = Substitute.For(); + ethRpcModule.eth_getLogs(Arg.Any()).Throws(new ConcurrencyLimitReachedException("cap")); + using JsonRpcErrorResponse invocationRejection = AssertJsonRpcError( + TestRequest(ethRpcModule, "eth_getLogs", "{}"), + ErrorCodes.LimitExceeded, + "Too many requests"); + Assert.That(Metrics.JsonRpcOverloadRejections, Is.GreaterThanOrEqualTo(beforeInvocation + 1), + "invocation-path rejection was not counted"); + + long beforeRental = Metrics.JsonRpcOverloadRejections; + + // Before-invocation path: module rental times out. + IRpcModulePool pool = Substitute.For>(); + pool.GetModule(Arg.Any()).Returns(Task.FromException(new ModuleRentalTimeoutException("timeout"))); + using JsonRpcErrorResponse rentalRejection = AssertJsonRpcError( + TestRequestWithPool(pool, "eth_getLogs", "{}"), + ErrorCodes.ModuleTimeout, + "Timeout"); + Assert.That(Metrics.JsonRpcOverloadRejections, Is.GreaterThanOrEqualTo(beforeRental + 1), + "rental-path rejection was not counted"); + } + [TestCaseSource(nameof(ModuleRentalOverloadExceptions))] public void Module_rental_overload_does_not_log_or_return_exception_data( Exception exception, diff --git a/src/Nethermind/Nethermind.JsonRpc/JsonRpcService.cs b/src/Nethermind/Nethermind.JsonRpc/JsonRpcService.cs index 4a0c0f659a37..853c8d19c18b 100644 --- a/src/Nethermind/Nethermind.JsonRpc/JsonRpcService.cs +++ b/src/Nethermind/Nethermind.JsonRpc/JsonRpcService.cs @@ -80,6 +80,8 @@ private JsonRpcErrorResponse ReturnErrorResponse(JsonRpcRequest rpcRequest, Exce (int errorCode, string errorText, bool suppressWarning) = ex switch { + // suppressWarning doubles as the overload-shedding marker: GetErrorResponse counts + // suppressed LimitExceeded/ModuleTimeout responses in Metrics.JsonRpcOverloadRejections. LimitExceededException or ConcurrencyLimitReachedException => (ErrorCodes.LimitExceeded, "Too many requests", true), ModuleRentalTimeoutException => (ErrorCodes.ModuleTimeout, "Timeout", true), _ => (ErrorCodes.InternalError, "Internal error", false), @@ -529,6 +531,8 @@ private JsonRpcErrorResponse HandleInvocationException(Exception ex, string meth GetErrorResponse(methodName, ErrorCodes.Timeout, $"{methodName} request was canceled due to enabled timeout.", null, in request.IdRef, returnAction), + // suppressWarning doubles as the overload-shedding marker: GetErrorResponse counts + // suppressed LimitExceeded/ModuleTimeout responses in Metrics.JsonRpcOverloadRejections. LimitExceededException or ConcurrencyLimitReachedException or { InnerException: LimitExceededException } or { InnerException: ConcurrencyLimitReachedException } => @@ -915,6 +919,16 @@ private JsonRpcErrorResponse GetErrorResponse( bool suppressWarning = false) { if (_logger.IsDebug) _logger.Debug($"Sending error response, method: {(string.IsNullOrEmpty(methodName) ? "none" : methodName)}, id: {id}, errorType: {errorCode}, message: {errorMessage}, errorData: {errorData}"); + // Counted here, at the funnel every error response passes through: concurrency-cap + // rejections reach this point along two distinct paths (module rental before invocation, + // and the override-environment cap during invocation), and their warnings are suppressed + // by design — without a counter operators cannot see that callers are being shed. + // suppressWarning scopes the count to exactly those shedding sites: batch-size and + // response-body caps also produce LimitExceeded but keep their warnings. + if (suppressWarning && errorCode is ErrorCodes.LimitExceeded or ErrorCodes.ModuleTimeout) + { + Metrics.IncrementJsonRpcOverloadRejections(); + } JsonRpcErrorResponse response = new(in id, disposableAction) { Error = new Error diff --git a/src/Nethermind/Nethermind.JsonRpc/Metrics.cs b/src/Nethermind/Nethermind.JsonRpc/Metrics.cs index a864824d7e20..956a0f4caad9 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Metrics.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Metrics.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.ComponentModel; +using System.Threading; using Nethermind.Core.Attributes; using Nethermind.Core.Metric; @@ -21,6 +22,12 @@ public static class Metrics [Description("Number of JSON RPC requests that were invalid.")] public static long JsonRpcInvalidRequests { get; set; } + [CounterMetric] + [Description("Number of JSON RPC requests rejected or timed out at a concurrency cap (module pool or override-environment limit). A nonzero rate means callers receive 'Too many requests' — consider raising JsonRpc.EthModuleConcurrentInstances.")] + public static long JsonRpcOverloadRejections => _jsonRpcOverloadRejections; + private static long _jsonRpcOverloadRejections; + internal static void IncrementJsonRpcOverloadRejections() => Interlocked.Increment(ref _jsonRpcOverloadRejections); + [CounterMetric] [Description("Number of JSON RPC requests processed with errors.")] public static long JsonRpcErrors { get; set; } diff --git a/src/Nethermind/Nethermind.Trie/Metrics.cs b/src/Nethermind/Nethermind.Trie/Metrics.cs index 8e0b85e40e55..13bddf8bf8bc 100644 --- a/src/Nethermind/Nethermind.Trie/Metrics.cs +++ b/src/Nethermind/Nethermind.Trie/Metrics.cs @@ -13,31 +13,43 @@ public static class Metrics { private static bool IsBlockProcessingThread => ProcessingThread.IsBlockProcessingThread; + // The block-processing thread keeps its dedicated padded word; every other thread (RPC + // workers, prewarm workers) previously shared ONE "other" word per counter, making each + // increment a contended cross-core RMW under concurrent load — striped instead. [CounterMetric] [Description("Number of trie node hash calculations.")] - public static long TreeNodeHashCalculations => _mainTreeNodeHashCalculations.Value + _otherTreeNodeHashCalculations.Value; + public static long TreeNodeHashCalculations => _mainTreeNodeHashCalculations.Value + _otherTreeNodeHashCalculations.Sum; private static CacheLinePaddedLong _mainTreeNodeHashCalculations; - private static CacheLinePaddedLong _otherTreeNodeHashCalculations; + private static readonly StripedLong _otherTreeNodeHashCalculations = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void IncrementTreeNodeHashCalculations() => - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainTreeNodeHashCalculations.Value : ref _otherTreeNodeHashCalculations.Value); + internal static void IncrementTreeNodeHashCalculations() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainTreeNodeHashCalculations.Value); + else _otherTreeNodeHashCalculations.Increment(); + } [CounterMetric] [Description("Number of trie node RLP encodings.")] - public static long TreeNodeRlpEncodings => _mainTreeNodeRlpEncodings.Value + _otherTreeNodeRlpEncodings.Value; + public static long TreeNodeRlpEncodings => _mainTreeNodeRlpEncodings.Value + _otherTreeNodeRlpEncodings.Sum; private static CacheLinePaddedLong _mainTreeNodeRlpEncodings; - private static CacheLinePaddedLong _otherTreeNodeRlpEncodings; + private static readonly StripedLong _otherTreeNodeRlpEncodings = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void IncrementTreeNodeRlpEncodings() => - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainTreeNodeRlpEncodings.Value : ref _otherTreeNodeRlpEncodings.Value); + internal static void IncrementTreeNodeRlpEncodings() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainTreeNodeRlpEncodings.Value); + else _otherTreeNodeRlpEncodings.Increment(); + } [CounterMetric] [Description("Number of trie node RLP decodings.")] - public static long TreeNodeRlpDecodings => _mainTreeNodeRlpDecodings.Value + _otherTreeNodeRlpDecodings.Value; + public static long TreeNodeRlpDecodings => _mainTreeNodeRlpDecodings.Value + _otherTreeNodeRlpDecodings.Sum; private static CacheLinePaddedLong _mainTreeNodeRlpDecodings; - private static CacheLinePaddedLong _otherTreeNodeRlpDecodings; + private static readonly StripedLong _otherTreeNodeRlpDecodings = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void IncrementTreeNodeRlpDecodings() => - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainTreeNodeRlpDecodings.Value : ref _otherTreeNodeRlpDecodings.Value); + internal static void IncrementTreeNodeRlpDecodings() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainTreeNodeRlpDecodings.Value); + else _otherTreeNodeRlpDecodings.Increment(); + } } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/Metrics.cs b/src/Nethermind/Nethermind.Trie/Pruning/Metrics.cs index 75dbd1525a1d..9215c486cc29 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/Metrics.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/Metrics.cs @@ -45,23 +45,32 @@ public static class Metrics [Description("Nodes that have been removed from the cache during pruning because they were no longer needed.")] public static long PrunedTransientNodesCount { get; set; } + // The block-processing thread keeps its dedicated padded word; every other thread (RPC + // workers, prewarm workers) previously shared ONE "other" word per counter, making each + // per-node increment a contended cross-core RMW under concurrent load — striped instead. [CounterMetric] [Description("Number of DB reads.")] - public static long LoadedFromDbNodesCount => _mainLoadedFromDbNodesCount.Value + _otherLoadedFromDbNodesCount.Value; + public static long LoadedFromDbNodesCount => _mainLoadedFromDbNodesCount.Value + _otherLoadedFromDbNodesCount.Sum; private static CacheLinePaddedLong _mainLoadedFromDbNodesCount; - private static CacheLinePaddedLong _otherLoadedFromDbNodesCount; + private static readonly StripedLong _otherLoadedFromDbNodesCount = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void IncrementLoadedFromDbNodesCount() => - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainLoadedFromDbNodesCount.Value : ref _otherLoadedFromDbNodesCount.Value); + public static void IncrementLoadedFromDbNodesCount() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainLoadedFromDbNodesCount.Value); + else _otherLoadedFromDbNodesCount.Increment(); + } [CounterMetric] [Description("Number of reads from the node cache.")] - public static long LoadedFromCacheNodesCount => _mainLoadedFromCacheNodesCount.Value + _otherLoadedFromCacheNodesCount.Value; + public static long LoadedFromCacheNodesCount => _mainLoadedFromCacheNodesCount.Value + _otherLoadedFromCacheNodesCount.Sum; private static CacheLinePaddedLong _mainLoadedFromCacheNodesCount; - private static CacheLinePaddedLong _otherLoadedFromCacheNodesCount; + private static readonly StripedLong _otherLoadedFromCacheNodesCount = new(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void IncrementLoadedFromCacheNodesCount() => - Interlocked.Increment(ref IsBlockProcessingThread ? ref _mainLoadedFromCacheNodesCount.Value : ref _otherLoadedFromCacheNodesCount.Value); + public static void IncrementLoadedFromCacheNodesCount() + { + if (IsBlockProcessingThread) Interlocked.Increment(ref _mainLoadedFromCacheNodesCount.Value); + else _otherLoadedFromCacheNodesCount.Increment(); + } [CounterMetric] [Description("Number of reads from the RLP cache.")]