perf(rpc): fix eth_call degradation under concurrent load (50→300 rps)

src/Nethermind/Nethermind.Api/IInitConfig.cs

@@ -107,6 +107,11 @@ public interface IInitConfig : IConfig
         DefaultValue = "8192",
         HiddenFromDocs = true)]
     long HealCanonicalChainDepth { get; set; }
+
+    [ConfigItem(
+        Description = "The EVM instruction-stream cache size, in code entries, accepted range 1 to 1048576. Each entry retains up to 512 KiB (typical entries are far smaller), bounding worst-case retention at size x 512 KiB. The simultaneously-hot code set grows with concurrent eth_call-style request parallelism, and once the cache conflict-evicts, execution falls back to a slower interpreter path — lower this only on memory-constrained setups.",
+        DefaultValue = "4096")]
+    int InstructionStreamCacheSize { get; set; }
 }
 
 public enum DiagnosticMode

src/Nethermind/Nethermind.Api/InitConfig.cs

@@ -43,6 +43,7 @@ public class InitConfig : IInitConfig
         public string? DataDir { get; set; }
         public bool HealCanonicalChain { get; set; } = false;
         public long HealCanonicalChainDepth { get; set; } = 8192;
+        public int InstructionStreamCacheSize { get; set; } = 4096;
 
         [Obsolete("Use DiagnosticMode with MemDb instead")]
         public bool UseMemDb

src/Nethermind/Nethermind.Core.Test/Caching/AssociativeCacheTestsBase.cs

@@ -347,6 +347,45 @@ public void All_inserted_keys_retrievable_at_various_capacities(int capacity)
         }
     }
 
+    [Test]
+    public void Refreshed_entry_survives_sustained_churn_in_its_set()
+    {
+        const int capacity = 64;
+        int setCount = (int)BitOperations.RoundUpToPowerOf2((uint)((capacity + Ways - 1) / Ways));
+        int hashShift = BitOperations.Log2((uint)setCount);
+        // Every key collides into set 0, so each insert past the way count must evict from it.
+        static DeterministicHashKey MakeKey(int i, int shift) => new(i, (long)(i + 1) << shift);
+
+        AssociativeCache<DeterministicHashKey, TestValue> cache = new(capacity);
+        AssociativeKeyCache<DeterministicHashKey> keyCache = new(capacity);
+        DeterministicHashKey hot = MakeKey(0, hashShift);
+        TestValue hotValue = new();
+        cache.Set(in hot, hotValue);
+        keyCache.Set(in hot);
+
+        for (int i = 1; i <= 200; i++)
+        {
+            // The refreshing lookup gives the hot entry the newest eviction age, so 3-random
+            // eviction (which removes the oldest of its sample) must never select it. This pins
+            // the ticker semantics: a recency clock coarse enough to tie a refresh with the
+            // surrounding churn would make this probabilistic.
+            using (Assert.EnterMultipleScope())
+            {
+                Assert.That(cache.Get(in hot), Is.SameAs(hotValue), $"hot entry evicted after {i - 1} churn inserts");
+                Assert.That(keyCache.Get(in hot), Is.True, $"hot key evicted after {i - 1} churn inserts");
+            }
+            DeterministicHashKey filler = MakeKey(i, hashShift);
+            cache.Set(in filler, new TestValue());
+            keyCache.Set(in filler);
+        }
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(cache.Get(in hot), Is.SameAs(hotValue));
+            Assert.That(keyCache.Get(in hot), Is.True);
+        }
+    }
+
     private static DeterministicHashKey[] BuildKeys(int capacity, int count)
     {
         int setCount = (int)BitOperations.RoundUpToPowerOf2((uint)((capacity + Ways - 1) / Ways));

src/Nethermind/Nethermind.Core.Test/Threading/StripedLongTests.cs

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Threading.Tasks;
+using Nethermind.Core.Threading;
+using NUnit.Framework;
+
+namespace Nethermind.Core.Test.Threading;
+
+[Parallelizable(ParallelScope.All)]
+public class StripedLongTests
+{
+    [Test]
+    public void Sum_is_exact_under_concurrent_mixed_adds()
+    {
+        StripedLong counter = new();
+        const int threads = 8;
+        const int iterations = 100_000;
+
+        Parallel.For(0, threads, _ =>
+        {
+            for (int i = 0; i < iterations; i++)
+            {
+                counter.Increment();
+                counter.Add(3);
+                counter.Add(-2);
+            }
+        });
+
+        Assert.That(counter.Sum, Is.EqualTo((long)threads * iterations * 2));
+    }
+
+    [Test]
+    public void Negative_totals_are_representable()
+    {
+        StripedLong counter = new();
+        counter.Add(-5);
+        counter.Increment();
+        Assert.That(counter.Sum, Is.EqualTo(-4));
+    }
+
+    [Test]
+    public void New_counter_sums_to_zero() => Assert.That(new StripedLong().Sum, Is.Zero);
+}

src/Nethermind/Nethermind.Core/Caching/AssociativeCache.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Diagnostics;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -79,13 +80,6 @@ public sealed class AssociativeCache<TKey, TValue>
     /// </summary>
     private long _epochAndCount;
 
-    /// <summary>
-    /// Monotonic counter for eviction-age tracking. Interlocked.Increment is faster
-    /// than Stopwatch.GetTimestamp() (RDTSC) — ~7.6ns vs ~19ns single-threaded,
-    /// and scales better under contention (20% faster at 8 threads).
-    /// </summary>
-    private long _ticker;
-
     public int Count => ReadCount(ref _epochAndCount);
 
     public AssociativeCache(int maxCapacity)
@@ -166,11 +160,18 @@ private bool TryGetCore<TRefreshTicker>(in TKey key, out TValue? value)
             if (h1 == h2 && storedKey.Equals(in key))
             {
                 // JIT eliminates this branch entirely per TRefreshTicker instantiation.
+                // Eviction age uses the high-resolution clock rather than a shared counter: a
+                // per-hit Interlocked on a cache-wide field is a serialized cross-core RMW under
+                // concurrent readers (and it dirtied the line _epochAndCount lives on, which
+                // every TryGet reads first). Single-threaded the clock read loses a few ns to the
+                // old Interlocked (and more on hosts whose clocksource is not TSC), but it writes
+                // only this entry's own line, so hits scale with reader count — the regime that
+                // motivated the change. Do not flip back to a shared counter for the ns.
                 // Ticker store without the set gate is safe: 8-byte aligned long is atomic on
                 // x64/ARM64 hardware. A race with a concurrent Set only affects eviction ranking,
                 // not key/value correctness — the "losing" ticker value is simply slightly stale.
                 if (TRefreshTicker.IsActive)
-                    e.Ticker = Interlocked.Increment(ref _ticker);
+                    e.Ticker = Stopwatch.GetTimestamp();
                 value = storedValue;
                 return true;
             }
@@ -232,8 +233,7 @@ private bool SetCore(in TKey key, TValue val, int baseIdx, long hashPart)
                 {
                     if ((h & HashMask) == hashPart && e.Key.Equals(in key))
                     {
-                        long now = Interlocked.Increment(ref _ticker);
-                        WriteEntry(ref e, h, in key, val, tagToStore, now);
+                        WriteEntry(ref e, h, in key, val, tagToStore, Stopwatch.GetTimestamp());
                         return false;
                     }
                 }
@@ -249,7 +249,7 @@ private bool SetCore(in TKey key, TValue val, int baseIdx, long hashPart)
 
             if (ReadEpoch(ref _epochAndCount) != epochTag) continue;
 
-            long timestamp = Interlocked.Increment(ref _ticker);
+            long timestamp = Stopwatch.GetTimestamp();
             int target = bestEmpty >= 0
                 ? bestEmpty
                 : bestStale >= 0

src/Nethermind/Nethermind.Core/Caching/AssociativeKeyCache.cs

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
+using System.Diagnostics;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
@@ -29,7 +30,6 @@ public sealed class AssociativeKeyCache<TKey>
     private readonly int _hashShift;
     private readonly int[] _setGates;
     private long _epochAndCount;
-    private long _ticker;
 
     public int Count => ReadCount(ref _epochAndCount);
 
@@ -95,11 +95,18 @@ private bool GetCore<TRefreshTicker>(in TKey key)
             if (h1 == h2 && storedKey.Equals(in key))
             {
                 // JIT eliminates this branch entirely per TRefreshTicker instantiation.
+                // Eviction age uses the high-resolution clock rather than a shared counter: a
+                // per-hit Interlocked on a cache-wide field is a serialized cross-core RMW under
+                // concurrent readers (and it dirtied the line _epochAndCount lives on, which
+                // every Get reads first). Single-threaded the clock read loses a few ns to the
+                // old Interlocked (and more on hosts whose clocksource is not TSC), but it writes
+                // only this entry's own line, so hits scale with reader count — the regime that
+                // motivated the change. Do not flip back to a shared counter for the ns.
                 // Ticker store without the set gate is safe: 8-byte aligned long is atomic on
                 // x64/ARM64 hardware. A race with a concurrent Set only affects eviction ranking,
                 // not key/value correctness — the "losing" ticker value is simply slightly stale.
                 if (TRefreshTicker.IsActive)
-                    e.Ticker = Interlocked.Increment(ref _ticker);
+                    e.Ticker = Stopwatch.GetTimestamp();
                 return true;
             }
         }
@@ -158,7 +165,7 @@ private bool SetCore(in TKey key, int baseIdx, long hashPart)
                         // Unlike AssociativeCache.SetCore (which calls WriteEntry to update the value),
                         // the key-only variant has nothing to write, so a bare ticker store suffices.
                         // The seqlock header is unchanged, which is correct: readers see a stable entry.
-                        e.Ticker = Interlocked.Increment(ref _ticker);
+                        e.Ticker = Stopwatch.GetTimestamp();
                         return false;
                     }
                 }
@@ -174,7 +181,7 @@ private bool SetCore(in TKey key, int baseIdx, long hashPart)
 
             if (ReadEpoch(ref _epochAndCount) != epochTag) continue;
 
-            long timestamp = Interlocked.Increment(ref _ticker);
+            long timestamp = Stopwatch.GetTimestamp();
             int target = bestEmpty >= 0
                 ? bestEmpty
                 : bestStale >= 0

src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs

@@ -87,7 +87,14 @@ protected void MarkAccessed(int position)
 
         ref long accessedBitmapWord = ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(HasBeenAccessedBitmap), offset);
 
-        Interlocked.Or(ref accessedBitmapWord, flags);
+        // Test first: hot entries are re-marked far more often than the clock hand clears them,
+        // and an unconditional lock-prefixed Or on a word shared by 64 entries serializes
+        // concurrent readers. A racing clear between the read and the Or loses at most one
+        // access mark — the same tolerance the eviction algorithm already has.
+        if ((Volatile.Read(ref accessedBitmapWord) & flags) != flags)
+        {
+            Interlocked.Or(ref accessedBitmapWord, flags);
+        }
     }
 
     protected void MarkAccessedNonConcurrent(int position)

src/Nethermind/Nethermind.Core/Threading/StripedLong.cs

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+namespace Nethermind.Core.Threading;
+
+/// <summary>
+/// Additive counter safe for hot concurrent paths: increments land on a per-core slot, reads sum
+/// the slots.
+/// </summary>
+/// <remarks>
+/// A shared counter word turns every increment into a serialized cross-core cache-line transfer
+/// once several threads hit it (RPC workers, prewarm workers). Striping by
+/// <see cref="Thread.GetCurrentProcessorId"/> keeps the RMW local to the core in the common case;
+/// the atomic add only guards against threads that share or migrate between cores. Slots are
+/// 128-byte spaced — same isolation as <see cref="CacheLinePaddedLong"/> (adjacent-line prefetch
+/// pairs lines) — with a leading pad stride so the first slot is also isolated from whatever
+/// precedes the array in memory. Each instance allocates (stripes + 1) * 128 bytes, where stripes
+/// is <see cref="Environment.ProcessorCount"/> rounded up to a power of two, captured once at
+/// type initialization (later CPU hot-add is not tracked; the mask just folds new ids onto
+/// existing slots). Reads are O(stripes) and torn only across slots: fine for metrics, not for
+/// invariants.
+/// </remarks>
+public sealed class StripedLong
+{
+    // 16 longs = 128 bytes between live slots.
+    private const int SlotStride = 16;
+    private static readonly int s_stripeMask = (int)BitOperations.RoundUpToPowerOf2((uint)Environment.ProcessorCount) - 1;
+
+    private readonly long[] _slots = new long[(s_stripeMask + 2) * SlotStride];
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public void Add(long value)
+        => Interlocked.Add(ref _slots[((Thread.GetCurrentProcessorId() & s_stripeMask) + 1) * SlotStride], value);
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public void Increment() => Add(1);
+
+    public long Sum
+    {
+        get
+        {
+            long[] slots = _slots;
+            long sum = 0;
+            for (int stripe = 0; stripe <= s_stripeMask; stripe++)
+            {
+                sum += Volatile.Read(ref slots[(stripe + 1) * SlotStride]);
+            }
+            return sum;
+        }
+    }
+}

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -92,8 +92,10 @@ public partial class DbOnTheRocks : IDb, ITunableDb, IReadOnlyNativeKeyValueStor
 
     private readonly List<IDisposable> _metricsUpdaters = [];
 
-    internal CacheLinePaddedLong _allocatedSpan;
-    private CacheLinePaddedLong _totalReads;
+    // Striped: every concurrent reader (RPC workers, prewarm workers) updates these per Get, and
+    // a single shared word per DB serializes them under load.
+    internal readonly StripedLong _allocatedSpan = new();
+    private readonly StripedLong _totalReads = new();
     private CacheLinePaddedLong _totalWrites;
 
     private readonly DisposableLazy<IteratorManager>? _iteratorManager;
@@ -358,7 +360,7 @@ private void RepairIfCorrupted(DbOptions dbOptions)
         _fileSystem.File.Delete(corruptMarker);
     }
 
-    protected internal void UpdateReadMetrics() => Interlocked.Increment(ref _totalReads.Value);
+    protected internal void UpdateReadMetrics() => _totalReads.Increment();
 
     protected internal void UpdateWriteMetrics() => Interlocked.Increment(ref _totalWrites.Value);
 
@@ -377,7 +379,7 @@ public IDbMeta.DbMetric GatherMetric()
                 CacheSize = 0,
                 IndexSize = 0,
                 MemtableSize = 0,
-                TotalReads = _totalReads.Value,
+                TotalReads = _totalReads.Sum,
                 TotalWrites = _totalWrites.Value,
             };
         }
@@ -387,7 +389,7 @@ public IDbMeta.DbMetric GatherMetric()
             CacheSize = GetCacheSize(),
             IndexSize = GetIndexSize(),
             MemtableSize = GetMemtableSize(),
-            TotalReads = _totalReads.Value,
+            TotalReads = _totalReads.Sum,
             TotalWrites = _totalWrites.Value,
         };
     }
@@ -951,8 +953,12 @@ internal Span<byte> GetSpanWithColumnFamily(scoped ReadOnlySpan<byte> key, Colum
 
             if (!span.IsNullOrEmpty())
             {
-                Interlocked.Increment(ref _allocatedSpan.Value);
-                GC.AddMemoryPressure(span.Length);
+                _allocatedSpan.Increment();
+                // Pressure hints exist so the GC accounts for sizeable native memory held alive by
+                // managed wrappers. Sub-threshold spans are transient (released within the request)
+                // and each Add/Remove pair mutates GC-global accounting — a contended cost per DB
+                // read under concurrent load. The threshold must match DangerousReleaseMemory.
+                if (span.Length >= GcPressureSpanThreshold) GC.AddMemoryPressure(span.Length);
             }
             return span;
         }
@@ -1047,12 +1053,14 @@ internal void MergeWithColumnFamily(ReadOnlySpan<byte> key, ColumnFamilyHandle?
         }
     }
 
+    private const int GcPressureSpanThreshold = 16 * 1024;
+
     public void DangerousReleaseMemory(in ReadOnlySpan<byte> span)
     {
         if (!span.IsNullOrEmpty())
         {
-            Interlocked.Decrement(ref _allocatedSpan.Value);
-            GC.RemoveMemoryPressure(span.Length);
+            _allocatedSpan.Add(-1);
+            if (span.Length >= GcPressureSpanThreshold) GC.RemoveMemoryPressure(span.Length);
         }
         _db.DangerousReleaseMemory(span);
     }

src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs

@@ -429,10 +429,10 @@ private long AllocatedSpan
             {
                 if (_db is ColumnDb columnDb)
                 {
-                    return columnDb._mainDb._allocatedSpan.Value;
+                    return columnDb._mainDb._allocatedSpan.Sum;
                 }
 
-                return (_db as DbOnTheRocks)._allocatedSpan.Value;
+                return (_db as DbOnTheRocks)._allocatedSpan.Sum;
             }
         }