From bc68a29dfdb4d3edf181f071f9e30b2d693d34e6 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Aug 2026 12:59:05 +0100 Subject: [PATCH 01/12] perf(core): tune the AVX-512 Keccak-f[1600] permutation - Restructure into a shared inlined Round helper with shared lane-index vectors for Theta/Chi and pre-broadcast round constants (removes the per-round scalar broadcast) - Start Pi permute chains from their own column, saving a register copy per row - Guard the state length with Debug.Assert, matching the scalar path, and access all lanes unchecked - Rename the portable path to KeccakF1600Scalar (internal) and add an AVX-512-vs-scalar equivalence test --- .../Nethermind.Core.Test/KeccakTests.cs | 34 +++ .../Nethermind.Core/Crypto/KeccakHash.std.cs | 275 ++++++------------ 2 files changed, 123 insertions(+), 186 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index ca07d65c5a6f..0ed68caa1e20 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -5,6 +5,7 @@ using System.Buffers; using System.Collections.Generic; using System.Reflection; +using System.Runtime.Intrinsics.X86; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Serialization.Rlp; @@ -168,6 +169,39 @@ public void Computes_known_hash_for_span_and_array() } } + [Test] + public void Avx512_permutation_matches_scalar() + { + if (!Avx512F.IsSupported) + { + Assert.Ignore("AVX-512F intrinsics are not supported on this machine."); + } + + const int stateLength = 25; + ulong[] expected = new ulong[stateLength]; + ulong[] actual = new ulong[stateLength]; + + for (int testCase = 0; testCase < 64; testCase++) + { + for (int lane = 0; lane < stateLength; lane++) + { + actual[lane] = testCase switch + { + 0 => 0, + 1 => ulong.MaxValue, + _ => unchecked((ulong)(testCase * stateLength + lane + 1) * 0x9e3779b97f4a7c15UL) + }; + } + + actual.CopyTo(expected, 0); + + KeccakHash.KeccakF1600Scalar(expected); + KeccakHash.KeccakF1600Avx512F(actual); + + Assert.That(actual, Is.EqualTo(expected), $"Permutation mismatch for test case {testCase}."); + } + } + [TestCase("0x", "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")] public void Sanity_check(string hexString, string expected) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 3ccc9070bcf9..6d6e0c582925 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -29,16 +29,25 @@ public sealed partial class KeccakHash 0x8000000000008080UL, 0x0000000080000001UL, 0x8000000080008008UL ]; + // Shared lane-index vectors (Theta's [1,2,3,4,0] == Chi's permute1) + private static readonly Vector512 LaneShift1 = Vector512.Create(1UL, 2UL, 3UL, 4UL, 0UL, 5UL, 6UL, 7UL); + private static readonly Vector512 LaneShift2 = Vector512.Create(2UL, 3UL, 4UL, 0UL, 1UL, 5UL, 6UL, 7UL); + private static readonly Vector512 ThetaRot4 = Vector512.Create(4UL, 0UL, 1UL, 2UL, 3UL, 5UL, 6UL, 7UL); + + // Pre-broadcast round constants (kills the per-round vmovq) + private static readonly Vector512[] RoundConstantVec = + Array.ConvertAll(RoundConstants, rc => Vector512.CreateScalar(rc)); + // update the state with given number of rounds private static partial void KeccakF(Span st) { if (Avx512F.IsSupported) KeccakF1600Avx512F(st); else - KeccakF1600(st); + KeccakF1600Scalar(st); } - private static void KeccakF1600(Span st) + internal static void KeccakF1600Scalar(Span st) { Debug.Assert(st.Length == 25); @@ -253,195 +262,89 @@ private static void KeccakF1600(Span st) [SkipLocalsInit] public static void KeccakF1600Avx512F(Span state) { - { - // Redundant statement that removes all the in loop bounds checks - _ = state[24]; - } + Debug.Assert(state.Length == 25); - // Can straight load and over-read for start elements + ref ulong s = ref MemoryMarshal.GetReference(state); Vector512 mask = Vector512.Create(ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, 0UL, 0UL, 0UL); - Vector512 c0 = Unsafe.As>(ref MemoryMarshal.GetReference(state)); - // Clear the over-read values from first vectors - c0 = Vector512.BitwiseAnd(mask, c0); - Vector512 c1 = Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 5)); - c1 = Vector512.BitwiseAnd(mask, c1); - Vector512 c2 = Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 10)); - c2 = Vector512.BitwiseAnd(mask, c2); - Vector512 c3 = Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 15)); - c3 = Vector512.BitwiseAnd(mask, c3); - - // Can't over-read for the last elements (8 items in vector 5 to be remaining) - // so read a Vector256 and ulong then combine - Vector256 c4a = Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 20)); - Vector256 c4b = Vector256.Create(state[24], 0UL, 0UL, 0UL); - Vector512 c4 = Vector512.Create(c4a, c4b); - - Vector512 permute1 = Vector512.Create(1UL, 2UL, 3UL, 4UL, 0UL, 5UL, 6UL, 7UL); - Vector512 permute2 = Vector512.Create(2UL, 3UL, 4UL, 0UL, 1UL, 5UL, 6UL, 7UL); - ulong[] roundConstants = RoundConstants; - - // Use constant for loop so Jit expects to loop; unroll once + + Vector512 c0 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref s)); + Vector512 c1 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 5))); + Vector512 c2 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 10))); + Vector512 c3 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 15))); + Vector512 c4 = Vector512.Create( + Unsafe.As>(ref Unsafe.Add(ref s, 20)), + Vector256.CreateScalar(Unsafe.Add(ref s, 24))); + for (int round = 0; round < ROUNDS; round += 2) { - // Iteration 1 - { - ulong roundConstant = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(roundConstants), round); - // Theta step - Vector512 parity = Avx512F.TernaryLogic(Avx512F.TernaryLogic(c0, c1, c2, 0x96), c3, c4, 0x96); - - // Compute Theta - Vector512 bVecRot1Rotated = Avx512F.RotateLeft(Avx512F.PermuteVar8x64(parity, Vector512.Create(1UL, 2UL, 3UL, 4UL, 0UL, 5UL, 6UL, 7UL)), 1); - Vector512 bVecRot4 = Avx512F.PermuteVar8x64(parity, Vector512.Create(4UL, 0UL, 1UL, 2UL, 3UL, 5UL, 6UL, 7UL)); - Vector512 theta = Avx512F.Xor(bVecRot4, bVecRot1Rotated); - - c0 = Avx512F.Xor(c0, theta); - c1 = Avx512F.Xor(c1, theta); - c2 = Avx512F.Xor(c2, theta); - c3 = Avx512F.Xor(c3, theta); - c4 = Avx512F.Xor(c4, theta); - - // Rho step - Vector512 rhoVec0 = Vector512.Create(0UL, 1UL, 62UL, 28UL, 27UL, 0UL, 0UL, 0UL); - c0 = Avx512F.RotateLeftVariable(c0, rhoVec0); - - Vector512 rhoVec1 = Vector512.Create(36UL, 44UL, 6UL, 55UL, 20UL, 0UL, 0UL, 0UL); - c1 = Avx512F.RotateLeftVariable(c1, rhoVec1); - - Vector512 rhoVec2 = Vector512.Create(3UL, 10UL, 43UL, 25UL, 39UL, 0UL, 0UL, 0UL); - c2 = Avx512F.RotateLeftVariable(c2, rhoVec2); - - Vector512 rhoVec3 = Vector512.Create(41UL, 45UL, 15UL, 21UL, 8UL, 0UL, 0UL, 0UL); - c3 = Avx512F.RotateLeftVariable(c3, rhoVec3); - - Vector512 rhoVec4 = Vector512.Create(18UL, 2UL, 61UL, 56UL, 14UL, 0UL, 0UL, 0UL); - c4 = Avx512F.RotateLeftVariable(c4, rhoVec4); - - // Pi step - Vector512 c0Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(0UL, 8 + 1, 2, 3, 4, 5, 6, 7), c1); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 3, 4, 5, 6, 7), c2); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 4, 5, 6, 7), c4); - - Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(3UL, 8 + 4, 2, 3, 4, 5, 6, 7), c1); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 8 + 0, 3, 4, 5, 6, 7), c2); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 8 + 1, 4, 5, 6, 7), c3); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 2, 5, 6, 7), c4); - - Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(1UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 8 + 3, 3, 4, 5, 6, 7), c2); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 8 + 4, 4, 5, 6, 7), c3); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 0, 5, 6, 7), c4); - - Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(4UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 1, 3, 4, 5, 6, 7), c2); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 8 + 2, 4, 5, 6, 7), c3); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 3, 5, 6, 7), c4); - - Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(2UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); - c0 = c0Pi; - c1 = c1Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 4, 3, 4, 5, 6, 7), c2); - c2 = c2Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 8 + 0, 4, 5, 6, 7), c3); - c3 = c3Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 1, 5, 6, 7), c4); - c4 = c4Pi; - - // Chi step - - c0 = Avx512F.TernaryLogic(c0, Avx512F.PermuteVar8x64(c0, permute1), Avx512F.PermuteVar8x64(c0, permute2), 0xD2); - c1 = Avx512F.TernaryLogic(c1, Avx512F.PermuteVar8x64(c1, permute1), Avx512F.PermuteVar8x64(c1, permute2), 0xD2); - c2 = Avx512F.TernaryLogic(c2, Avx512F.PermuteVar8x64(c2, permute1), Avx512F.PermuteVar8x64(c2, permute2), 0xD2); - c3 = Avx512F.TernaryLogic(c3, Avx512F.PermuteVar8x64(c3, permute1), Avx512F.PermuteVar8x64(c3, permute2), 0xD2); - c4 = Avx512F.TernaryLogic(c4, Avx512F.PermuteVar8x64(c4, permute1), Avx512F.PermuteVar8x64(c4, permute2), 0xD2); - - // Iota step - c0 = Vector512.Xor(c0, Vector512.Create(roundConstant, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL)); - } - // Iteration 2 - { - ulong roundConstant = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(roundConstants), round + 1); - // Theta step - Vector512 parity = Avx512F.TernaryLogic(Avx512F.TernaryLogic(c0, c1, c2, 0x96), c3, c4, 0x96); - - // Compute Theta - Vector512 bVecRot1Rotated = Avx512F.RotateLeft(Avx512F.PermuteVar8x64(parity, Vector512.Create(1UL, 2UL, 3UL, 4UL, 0UL, 5UL, 6UL, 7UL)), 1); - Vector512 bVecRot4 = Avx512F.PermuteVar8x64(parity, Vector512.Create(4UL, 0UL, 1UL, 2UL, 3UL, 5UL, 6UL, 7UL)); - Vector512 theta = Avx512F.Xor(bVecRot4, bVecRot1Rotated); - - c0 = Avx512F.Xor(c0, theta); - c1 = Avx512F.Xor(c1, theta); - c2 = Avx512F.Xor(c2, theta); - c3 = Avx512F.Xor(c3, theta); - c4 = Avx512F.Xor(c4, theta); - - // Rho step - Vector512 rhoVec0 = Vector512.Create(0UL, 1UL, 62UL, 28UL, 27UL, 0UL, 0UL, 0UL); - c0 = Avx512F.RotateLeftVariable(c0, rhoVec0); - - Vector512 rhoVec1 = Vector512.Create(36UL, 44UL, 6UL, 55UL, 20UL, 0UL, 0UL, 0UL); - c1 = Avx512F.RotateLeftVariable(c1, rhoVec1); - - Vector512 rhoVec2 = Vector512.Create(3UL, 10UL, 43UL, 25UL, 39UL, 0UL, 0UL, 0UL); - c2 = Avx512F.RotateLeftVariable(c2, rhoVec2); - - Vector512 rhoVec3 = Vector512.Create(41UL, 45UL, 15UL, 21UL, 8UL, 0UL, 0UL, 0UL); - c3 = Avx512F.RotateLeftVariable(c3, rhoVec3); - - Vector512 rhoVec4 = Vector512.Create(18UL, 2UL, 61UL, 56UL, 14UL, 0UL, 0UL, 0UL); - c4 = Avx512F.RotateLeftVariable(c4, rhoVec4); - - // Pi step - Vector512 c0Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(0UL, 8 + 1, 2, 3, 4, 5, 6, 7), c1); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 3, 4, 5, 6, 7), c2); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 4, 5, 6, 7), c4); - - Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(3UL, 8 + 4, 2, 3, 4, 5, 6, 7), c1); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 8 + 0, 3, 4, 5, 6, 7), c2); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 8 + 1, 4, 5, 6, 7), c3); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 2, 5, 6, 7), c4); - - Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(1UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 8 + 3, 3, 4, 5, 6, 7), c2); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 8 + 4, 4, 5, 6, 7), c3); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 0, 5, 6, 7), c4); - - Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(4UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 1, 3, 4, 5, 6, 7), c2); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 8 + 2, 4, 5, 6, 7), c3); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 3, 5, 6, 7), c4); - - Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(2UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); - c0 = c0Pi; - c1 = c1Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 4, 3, 4, 5, 6, 7), c2); - c2 = c2Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 8 + 0, 4, 5, 6, 7), c3); - c3 = c3Pi; - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 1, 5, 6, 7), c4); - c4 = c4Pi; - - // Chi step - - c0 = Avx512F.TernaryLogic(c0, Avx512F.PermuteVar8x64(c0, permute1), Avx512F.PermuteVar8x64(c0, permute2), 0xD2); - c1 = Avx512F.TernaryLogic(c1, Avx512F.PermuteVar8x64(c1, permute1), Avx512F.PermuteVar8x64(c1, permute2), 0xD2); - c2 = Avx512F.TernaryLogic(c2, Avx512F.PermuteVar8x64(c2, permute1), Avx512F.PermuteVar8x64(c2, permute2), 0xD2); - c3 = Avx512F.TernaryLogic(c3, Avx512F.PermuteVar8x64(c3, permute1), Avx512F.PermuteVar8x64(c3, permute2), 0xD2); - c4 = Avx512F.TernaryLogic(c4, Avx512F.PermuteVar8x64(c4, permute1), Avx512F.PermuteVar8x64(c4, permute2), 0xD2); - - // Iota step - c0 = Vector512.Xor(c0, Vector512.Create(roundConstant, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL)); - } + Round(ref c0, ref c1, ref c2, ref c3, ref c4, round); + Round(ref c0, ref c1, ref c2, ref c3, ref c4, round + 1); } - // Can over-write for first elements - Unsafe.As>(ref MemoryMarshal.GetReference(state)) = c0; - Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 5)) = c1; - Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 10)) = c2; - Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 15)) = c3; - // Can't over-write for last elements so write the upper Vector256 and then ulong - Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetReference(state), 20)) = c4.GetLower(); - state[24] = c4.GetElement(4); + Unsafe.As>(ref s) = c0; + Unsafe.As>(ref Unsafe.Add(ref s, 5)) = c1; + Unsafe.As>(ref Unsafe.Add(ref s, 10)) = c2; + Unsafe.As>(ref Unsafe.Add(ref s, 15)) = c3; + Unsafe.As>(ref Unsafe.Add(ref s, 20)) = c4.GetLower(); + Unsafe.Add(ref s, 24) = c4.GetElement(4); + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Round(ref Vector512 c0, ref Vector512 c1, + ref Vector512 c2, ref Vector512 c3, ref Vector512 c4, int round) + { + // Theta + Vector512 parity = Avx512F.TernaryLogic(Avx512F.TernaryLogic(c0, c1, c2, 0x96), c3, c4, 0x96); + Vector512 theta = Avx512F.Xor( + Avx512F.PermuteVar8x64(parity, ThetaRot4), + Avx512F.RotateLeft(Avx512F.PermuteVar8x64(parity, LaneShift1), 1)); + c0 = Avx512F.Xor(c0, theta); c1 = Avx512F.Xor(c1, theta); + c2 = Avx512F.Xor(c2, theta); c3 = Avx512F.Xor(c3, theta); c4 = Avx512F.Xor(c4, theta); + + // Rho + c0 = Avx512F.RotateLeftVariable(c0, Vector512.Create(0UL, 1UL, 62UL, 28UL, 27UL, 0UL, 0UL, 0UL)); + c1 = Avx512F.RotateLeftVariable(c1, Vector512.Create(36UL, 44UL, 6UL, 55UL, 20UL, 0UL, 0UL, 0UL)); + c2 = Avx512F.RotateLeftVariable(c2, Vector512.Create(3UL, 10UL, 43UL, 25UL, 39UL, 0UL, 0UL, 0UL)); + c3 = Avx512F.RotateLeftVariable(c3, Vector512.Create(41UL, 45UL, 15UL, 21UL, 8UL, 0UL, 0UL, 0UL)); + c4 = Avx512F.RotateLeftVariable(c4, Vector512.Create(18UL, 2UL, 61UL, 56UL, 14UL, 0UL, 0UL, 0UL)); + + // Pi + Vector512 c0Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(0UL, 8 + 1, 2, 3, 4, 5, 6, 7), c1); + c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 3, 4, 5, 6, 7), c2); + c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); + c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 4, 5, 6, 7), c4); + + Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c1, Vector512.Create(0UL, 4UL, 8 + 0, 3, 4, 5, 6, 7), c2); + c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 8 + 1, 4, 5, 6, 7), c3); + c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 2, 5, 6, 7), c4); + c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(8UL + 3, 1, 2, 3, 4, 5, 6, 7), c0); + + Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 3UL, 8 + 4, 4, 5, 6, 7), c3); + c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 0, 5, 6, 7), c4); + c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(8UL + 1, 1, 2, 3, 4, 5, 6, 7), c0); + c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); + + Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c3, Vector512.Create(0UL, 1, 2, 2UL, 8 + 3, 5, 6, 7), c4); + c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(8UL + 4, 1, 2, 3, 4, 5, 6, 7), c0); + c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); + c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 1, 3, 4, 5, 6, 7), c2); + + Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c4, Vector512.Create(8 + 2, 1, 2, 3, 1UL, 5, 6, 7), c0); + c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); + c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 4, 3, 4, 5, 6, 7), c2); + c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 8 + 0, 4, 5, 6, 7), c3); + + c0 = c0Pi; c1 = c1Pi; c2 = c2Pi; c3 = c3Pi; c4 = c4Pi; + + // Chi + c0 = Avx512F.TernaryLogic(c0, Avx512F.PermuteVar8x64(c0, LaneShift1), Avx512F.PermuteVar8x64(c0, LaneShift2), 0xD2); + c1 = Avx512F.TernaryLogic(c1, Avx512F.PermuteVar8x64(c1, LaneShift1), Avx512F.PermuteVar8x64(c1, LaneShift2), 0xD2); + c2 = Avx512F.TernaryLogic(c2, Avx512F.PermuteVar8x64(c2, LaneShift1), Avx512F.PermuteVar8x64(c2, LaneShift2), 0xD2); + c3 = Avx512F.TernaryLogic(c3, Avx512F.PermuteVar8x64(c3, LaneShift1), Avx512F.PermuteVar8x64(c3, LaneShift2), 0xD2); + c4 = Avx512F.TernaryLogic(c4, Avx512F.PermuteVar8x64(c4, LaneShift1), Avx512F.PermuteVar8x64(c4, LaneShift2), 0xD2); + + // Iota - Pre-broadcast constant, no vmovq + c0 = Vector512.Xor(c0, RoundConstantVec[round]); } } From d1e14040ecb6471ef7f9e24d65975efbbd1bc1d9 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Aug 2026 13:03:52 +0100 Subject: [PATCH 02/12] Make KeccakF1600Avx512F internal The method skips bounds checks under a Debug.Assert length contract, so restrict callers to the assembly and friend test assemblies, matching KeccakF1600Scalar. --- src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 6d6e0c582925..400d1bcbda43 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -260,7 +260,7 @@ internal static void KeccakF1600Scalar(Span st) } [SkipLocalsInit] - public static void KeccakF1600Avx512F(Span state) + internal static void KeccakF1600Avx512F(Span state) { Debug.Assert(state.Length == 25); From 04c2649f4a256cb0f73fc2cb16404dc321cd5180 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Aug 2026 13:19:38 +0100 Subject: [PATCH 03/12] Drop dead lane masks and hoist the round-constant ref - Lanes 5-7 of the over-read row loads never reach result lanes 0-4 (Theta/Rho/Pi/Chi map lanes 0-4 only from lanes 0-4) and the stores overwrite them, so the vpandq masking was dead - Pass the pre-broadcast round constant into Round via GetArrayDataReference, eliminating the round + 1 bounds check Tier-1 body shrinks 1562 -> 1528 bytes with no range checks left. --- .../Nethermind.Core/Crypto/KeccakHash.std.cs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 400d1bcbda43..755f29517208 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -265,20 +265,22 @@ internal static void KeccakF1600Avx512F(Span state) Debug.Assert(state.Length == 25); ref ulong s = ref MemoryMarshal.GetReference(state); - Vector512 mask = Vector512.Create(ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, ulong.MaxValue, 0UL, 0UL, 0UL); - Vector512 c0 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref s)); - Vector512 c1 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 5))); - Vector512 c2 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 10))); - Vector512 c3 = Vector512.BitwiseAnd(mask, Unsafe.As>(ref Unsafe.Add(ref s, 15))); + // Lanes 5-7 hold over-read neighbor lanes and need no masking: Theta/Rho/Pi/Chi + // map result lanes 0-4 only from lanes 0-4, and the stores below overwrite 5-7. + Vector512 c0 = Unsafe.As>(ref s); + Vector512 c1 = Unsafe.As>(ref Unsafe.Add(ref s, 5)); + Vector512 c2 = Unsafe.As>(ref Unsafe.Add(ref s, 10)); + Vector512 c3 = Unsafe.As>(ref Unsafe.Add(ref s, 15)); Vector512 c4 = Vector512.Create( Unsafe.As>(ref Unsafe.Add(ref s, 20)), Vector256.CreateScalar(Unsafe.Add(ref s, 24))); + ref Vector512 roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstantVec); for (int round = 0; round < ROUNDS; round += 2) { - Round(ref c0, ref c1, ref c2, ref c3, ref c4, round); - Round(ref c0, ref c1, ref c2, ref c3, ref c4, round + 1); + Round(ref c0, ref c1, ref c2, ref c3, ref c4, Unsafe.Add(ref roundConstants, round)); + Round(ref c0, ref c1, ref c2, ref c3, ref c4, Unsafe.Add(ref roundConstants, round + 1)); } Unsafe.As>(ref s) = c0; @@ -292,7 +294,7 @@ internal static void KeccakF1600Avx512F(Span state) [SkipLocalsInit] [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void Round(ref Vector512 c0, ref Vector512 c1, - ref Vector512 c2, ref Vector512 c3, ref Vector512 c4, int round) + ref Vector512 c2, ref Vector512 c3, ref Vector512 c4, Vector512 roundConstant) { // Theta Vector512 parity = Avx512F.TernaryLogic(Avx512F.TernaryLogic(c0, c1, c2, 0x96), c3, c4, 0x96); @@ -344,7 +346,7 @@ private static void Round(ref Vector512 c0, ref Vector512 c1, c3 = Avx512F.TernaryLogic(c3, Avx512F.PermuteVar8x64(c3, LaneShift1), Avx512F.PermuteVar8x64(c3, LaneShift2), 0xD2); c4 = Avx512F.TernaryLogic(c4, Avx512F.PermuteVar8x64(c4, LaneShift1), Avx512F.PermuteVar8x64(c4, LaneShift2), 0xD2); - // Iota - Pre-broadcast constant, no vmovq - c0 = Vector512.Xor(c0, RoundConstantVec[round]); + // Iota + c0 = Vector512.Xor(c0, roundConstant); } } From 6e309b0e0e2a6b0177b147e334644022a6ce7fc2 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Sun, 16 Aug 2026 13:58:20 +0100 Subject: [PATCH 04/12] Assert the round-constant table matches ROUNDS Unsafe.Add dropped the latent bounds check tying RoundConstantVec to the loop, so assert the length and evenness next to the state assert. Bounding the loop by RoundConstantVec.Length instead was measured with JitAsm and rejected: the non-const bound stops the JIT hoisting the 28 vector constants, re-loading all of them from rodata every iteration (loop grows 143 -> 165 instructions per 2 rounds). --- src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 755f29517208..08461c0d24ca 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -263,6 +263,7 @@ internal static void KeccakF1600Scalar(Span st) internal static void KeccakF1600Avx512F(Span state) { Debug.Assert(state.Length == 25); + Debug.Assert(RoundConstantVec.Length == ROUNDS && ROUNDS % 2 == 0); ref ulong s = ref MemoryMarshal.GetReference(state); @@ -276,6 +277,8 @@ internal static void KeccakF1600Avx512F(Span state) Unsafe.As>(ref Unsafe.Add(ref s, 20)), Vector256.CreateScalar(Unsafe.Add(ref s, 24))); + // The const bound lets the JIT hoist the vector constants out of the loop; + // bounding by RoundConstantVec.Length makes it re-load all of them every iteration. ref Vector512 roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstantVec); for (int round = 0; round < ROUNDS; round += 2) { From ea274c9d1b39ce38f3da5720e602bfa4e3471dc2 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 18 Aug 2026 14:05:48 +0100 Subject: [PATCH 05/12] Clean up the outer Keccak methods - Take the Keccak-f[1600] state as `ref ulong` in the internal AVX-512 and scalar permutations; the KeccakF dispatch extracts the ref once, so call sites stop materializing a Span for every permutation call - Replace the stackalloc state in ComputeHash with an [InlineArray] struct local: localloc pinned the method at Tier0-FullOpts (no tiering, dynamic PGO, or inlining) and added GS-cookie and stack-probe overhead per call; as a struct local it tiers to Tier1 and inlines into ValueKeccak.Compute, folding the round size, padding index, and output copy for 32-byte outputs - Add [SkipLocalsInit] to the hot wrappers (ValueKeccak.Compute, InternalCompute, ComputeHash, Update, UpdateFinalTo, GenerateValueHash), removing frame zero-init that Unsafe.SkipInit alone does not skip - Drop `checked` from GetRoundSize: the entry guard bounds the output length to [1, 200], so the arithmetic cannot overflow Verified with DOTNET_JitDisasm at Tier-1: ComputeHash now reaches Tier1 (was permanently Tier0-FullOpts) with an rsp frame, no GS cookie check, no stack probe, and no overflow branches; permutation call sites pass the state pointer directly with no span stores. Wall-clock is unchanged within noise for 20-532 byte inputs (the permutation dominates at ~240 ns per block). Tests: KeccakTests 1051/1051 on x64 with AVX-512, including the AVX-512-vs-scalar equivalence test; Nethermind.Core also builds with -p:EnableZkEvm=true against the unchanged zkevm KeccakF partial. --- .../Nethermind.Core.Test/KeccakTests.cs | 4 ++-- .../Nethermind.Core/Crypto/Keccak.cs | 2 ++ .../Nethermind.Core/Crypto/KeccakHash.cs | 19 ++++++++++++++++-- .../Nethermind.Core/Crypto/KeccakHash.std.cs | 20 ++++++++++++------- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index 0ed68caa1e20..745146b895e5 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -195,8 +195,8 @@ public void Avx512_permutation_matches_scalar() actual.CopyTo(expected, 0); - KeccakHash.KeccakF1600Scalar(expected); - KeccakHash.KeccakF1600Avx512F(actual); + KeccakHash.KeccakF1600Scalar(ref expected[0]); + KeccakHash.KeccakF1600Avx512F(ref actual[0]); Assert.That(actual, Is.EqualTo(expected), $"Permutation mismatch for test case {testCase}."); } diff --git a/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs b/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs index 00eb902c2d70..1232a9f11730 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Keccak.cs @@ -49,6 +49,7 @@ public static ValueHash256 Compute(string input) } [DebuggerStepThrough] + [SkipLocalsInit] public static ValueHash256 Compute(ReadOnlySpan input) { if (input.Length == 0) @@ -61,6 +62,7 @@ public static ValueHash256 Compute(ReadOnlySpan input) return keccak; } + [SkipLocalsInit] internal static ValueHash256 InternalCompute(byte[] input) { Unsafe.SkipInit(out ValueHash256 keccak); diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs index 2341a5f492f0..71b321419ef8 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs @@ -15,6 +15,7 @@ public sealed partial class KeccakHash { private const int HASH_SIZE = 32; private const int STATE_SIZE = 200; + private const int STATE_LANES = STATE_SIZE / sizeof(ulong); private const int HASH_DATA_AREA = 136; private byte[] _remainderBuffer = []; @@ -69,6 +70,7 @@ private KeccakHash(int size) public KeccakHash Copy() => new(this); + [SkipLocalsInit] public static void ComputeHash(ReadOnlySpan input, Span output) { if ((uint)(output.Length - 1) >= STATE_SIZE) @@ -83,7 +85,10 @@ public static void ComputeHash(ReadOnlySpan input, Span output) #endif int roundSize = GetRoundSize(output.Length); - Span state = stackalloc ulong[STATE_SIZE / sizeof(ulong)]; + // A struct local rather than stackalloc: localloc would pin this method at Tier0-FullOpts + // (no tiering or dynamic PGO) and add GS-cookie and stack-probe overhead per call. + KeccakState stateBuffer = default; // the sponge state must start all-zero + Span state = stateBuffer; Span stateBytes = MemoryMarshal.AsBytes(state); if (input.Length == Address.Size) @@ -173,6 +178,7 @@ public static uint[] ComputeBytesToUint(ReadOnlySpan input, int size) return output; } + [SkipLocalsInit] public ValueHash256 GenerateValueHash() { Unsafe.SkipInit(out ValueHash256 output); @@ -180,6 +186,7 @@ public ValueHash256 GenerateValueHash() return output; } + [SkipLocalsInit] public void Update(ReadOnlySpan input) { if (_hash is not null) @@ -259,6 +266,7 @@ public void Update(ReadOnlySpan input) } } + [SkipLocalsInit] public void UpdateFinalTo(Span output) { if (_hash is not null) @@ -364,7 +372,8 @@ public void ResetTo(KeccakHash original) private static partial void KeccakF(Span st); - private static int GetRoundSize(int hashSize) => checked(STATE_SIZE - 2 * hashSize); + // Callers bound hashSize to [1, STATE_SIZE], so the arithmetic cannot overflow. + private static int GetRoundSize(int hashSize) => STATE_SIZE - 2 * hashSize; private byte[] GenerateHash() { @@ -469,6 +478,12 @@ private static unsafe void XorVectors(Span state, ReadOnlySpan input private static void ThrowInvalidOutputSize(int length) => throw new ArgumentOutOfRangeException( nameof(length), length, $"Must be between 1 and {STATE_SIZE}."); + [InlineArray(STATE_LANES)] + private struct KeccakState + { + private ulong _lane0; + } + private static class Pool { private const int MaxPooledPerThread = 4; diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 08461c0d24ca..ca34791c15c6 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -41,15 +41,20 @@ public sealed partial class KeccakHash // update the state with given number of rounds private static partial void KeccakF(Span st) { + Debug.Assert(st.Length == STATE_LANES); + + ref ulong state = ref MemoryMarshal.GetReference(st); if (Avx512F.IsSupported) - KeccakF1600Avx512F(st); + KeccakF1600Avx512F(ref state); else - KeccakF1600Scalar(st); + KeccakF1600Scalar(ref state); } - internal static void KeccakF1600Scalar(Span st) + /// Portable Keccak-f[1600] permutation. + /// Lane 0 of a 25-lane state; all 25 lanes are read and written. + internal static void KeccakF1600Scalar(ref ulong state) { - Debug.Assert(st.Length == 25); + Span st = MemoryMarshal.CreateSpan(ref state, STATE_LANES); ulong aba, abe, abi, abo, abu; ulong aga, age, agi, ago, agu; @@ -259,13 +264,14 @@ internal static void KeccakF1600Scalar(Span st) st[0] = aba; } + /// AVX-512 Keccak-f[1600] permutation. + /// Lane 0 of a 25-lane state; all 25 lanes are read and written. [SkipLocalsInit] - internal static void KeccakF1600Avx512F(Span state) + internal static void KeccakF1600Avx512F(ref ulong state) { - Debug.Assert(state.Length == 25); Debug.Assert(RoundConstantVec.Length == ROUNDS && ROUNDS % 2 == 0); - ref ulong s = ref MemoryMarshal.GetReference(state); + ref ulong s = ref state; // Lanes 5-7 hold over-read neighbor lanes and need no masking: Theta/Rho/Pi/Chi // map result lanes 0-4 only from lanes 0-4, and the stores below overwrite 5-7. From 786ce944f80eb1aafb3cc01d1cad01794c6c664d Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 18 Aug 2026 16:49:47 +0100 Subject: [PATCH 06/12] Balance AVX-512 Keccak Pi merges Arrange each Pi gather as two independent source-pair merges followed by a pair merge and the final c4 insertion. This keeps the instruction count unchanged while shortening the serial vpermt2q dependency chain from four levels to three. Restrict the existing benchmark inputs to the production-relevant 20, 32, and 64 byte sizes. BenchmarkDotNet --quick, ValueKeccak, AMD Ryzen 9 9950X, Windows 11, .NET 10.0.11: - 20 B: 249.1 ns -> 234.1 ns (-6.0%) - 32 B: 248.8 ns -> 233.3 ns (-6.2%) - 64 B: 262.5 ns -> 238.5 ns (-9.1%) A separate scalar run with COMPlus_EnableAVX512=0 measured 176.4, 175.2, and 174.1 ns respectively. FullOpts JitAsm keeps 40 two-source permutes per two rounds and reduces generated code from 1,535 to 1,487 bytes. Validated against the scalar permutation oracle and with the Nethermind.Core.Test suite (1,051 passed). --- .../Core/Keccak256Benchmarks.cs | 15 ++------ .../Nethermind.Core/Crypto/KeccakHash.std.cs | 37 ++++++++++--------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/Nethermind/Nethermind.Benchmark/Core/Keccak256Benchmarks.cs b/src/Nethermind/Nethermind.Benchmark/Core/Keccak256Benchmarks.cs index 9d0c5d7f6907..23941519bd5b 100644 --- a/src/Nethermind/Nethermind.Benchmark/Core/Keccak256Benchmarks.cs +++ b/src/Nethermind/Nethermind.Benchmark/Core/Keccak256Benchmarks.cs @@ -4,7 +4,6 @@ using System; using BenchmarkDotNet.Attributes; using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; //using Nethermind.HashLib; namespace Nethermind.Benchmarks.Core @@ -15,19 +14,11 @@ public class Keccak256Benchmarks private byte[] _a; - private byte[][] _scenarios = - { - Array.Empty(), - new byte[]{1}, - new byte[100000], - TestItem.AddressA.Bytes.ToArray() - }; - - [Params(1)] - public int ScenarioIndex { get; set; } + [Params(20, 32, 64)] + public int Length { get; set; } [GlobalSetup] - public void Setup() => _a = _scenarios[ScenarioIndex]; + public void Setup() => _a = new byte[Length]; [Benchmark] public void MeadowHashSpan() => MeadowHashBenchmarks.ComputeHash(_a); diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index ca34791c15c6..88e30fb58c5d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -321,30 +321,31 @@ private static void Round(ref Vector512 c0, ref Vector512 c1, c4 = Avx512F.RotateLeftVariable(c4, Vector512.Create(18UL, 2UL, 61UL, 56UL, 14UL, 0UL, 0UL, 0UL)); // Pi + // Merge source pairs in parallel to shorten the cross-lane dependency chain. Vector512 c0Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(0UL, 8 + 1, 2, 3, 4, 5, 6, 7), c1); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 3, 4, 5, 6, 7), c2); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); + Vector512 upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); + c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 4, 5, 6, 7), c4); - Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c1, Vector512.Create(0UL, 4UL, 8 + 0, 3, 4, 5, 6, 7), c2); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 8 + 1, 4, 5, 6, 7), c3); + Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(3UL, 8 + 4, 2, 3, 4, 5, 6, 7), c1); + upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 0, 8 + 1, 4, 5, 6, 7), c3); + c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 2, 5, 6, 7), c4); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(8UL + 3, 1, 2, 3, 4, 5, 6, 7), c0); - Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 3UL, 8 + 4, 4, 5, 6, 7), c3); + Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(1UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); + upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 3, 8 + 4, 4, 5, 6, 7), c3); + c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 0, 5, 6, 7), c4); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(8UL + 1, 1, 2, 3, 4, 5, 6, 7), c0); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); - - Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c3, Vector512.Create(0UL, 1, 2, 2UL, 8 + 3, 5, 6, 7), c4); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(8UL + 4, 1, 2, 3, 4, 5, 6, 7), c0); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 1, 3, 4, 5, 6, 7), c2); - - Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c4, Vector512.Create(8 + 2, 1, 2, 3, 1UL, 5, 6, 7), c0); - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 4, 3, 4, 5, 6, 7), c2); - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 8 + 0, 4, 5, 6, 7), c3); + + Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(4UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); + upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 1, 8 + 2, 4, 5, 6, 7), c3); + c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); + c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 3, 5, 6, 7), c4); + + Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(2UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); + upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 4, 8 + 0, 4, 5, 6, 7), c3); + c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); + c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 1, 5, 6, 7), c4); c0 = c0Pi; c1 = c1Pi; c2 = c2Pi; c3 = c3Pi; c4 = c4Pi; From 37b8b81f57c722d16d35282c29fc21637787ce5d Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Tue, 18 Aug 2026 18:16:52 +0100 Subject: [PATCH 07/12] Batch keyed-nonce slot hashes with AVX-512 Add an internal eight-way Keccak-256 kernel for consecutive 64-byte inputs. It transposes independent states across ZMM lanes so Rho/Pi becomes register renaming and all eight permutations share each vector instruction. Use it for EIP-8250 nonce sets of 8-16 keys; smaller sets, oversized primitive calls, and machines without AVX-512 retain the existing per-key path. The kernel uses one outer input pin, scalar-memory round-constant broadcasts, immediate rotates, and no allocations. FullOpts JitAsm emits no calls or variable rotates, keeps the round state in registers, uses a 696-byte frame, and totals 2,105 bytes. BenchmarkDotNet --quick, AMD Ryzen 9 9950X, Windows 11, .NET 10.0.11: - Eight raw 64-byte hashes: 1,916.8 ns -> 183.3 ns (10.45x) - 8 keyed-nonce slot indices: 2,012.1 ns -> 241.9 ns (8.32x) - 16 keyed-nonce slot indices: 3,986.2 ns -> 422.9 ns (9.43x) - All paths allocate 0 B The caller benchmarks include padded sender/key preimage construction and hash-to-UInt256 conversion. Correctness is checked against independent scalar hashes and individual StorageSlot calculations. Nethermind.Core.Test passed 1,052/1,052; KeyedNonceManagerTests passed 33/33 both with AVX-512 enabled and with COMPlus_EnableAVX512=0. --- .../Evm/KeyedNonceStorageSlotBenchmarks.cs | 46 ++++ .../Nethermind.Core.Test/KeccakTests.cs | 29 +++ .../Crypto/KeccakHash.avx512x8.cs | 198 ++++++++++++++++++ .../Nethermind.Core/InternalsVisibility.cs | 1 + .../KeyedNonceManagerTests.cs | 29 +++ .../KeyedNonceManager.cs | 78 ++++++- 6 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 src/Nethermind/Nethermind.Benchmark/Evm/KeyedNonceStorageSlotBenchmarks.cs create mode 100644 src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs diff --git a/src/Nethermind/Nethermind.Benchmark/Evm/KeyedNonceStorageSlotBenchmarks.cs b/src/Nethermind/Nethermind.Benchmark/Evm/KeyedNonceStorageSlotBenchmarks.cs new file mode 100644 index 000000000000..f673fdac4251 --- /dev/null +++ b/src/Nethermind/Nethermind.Benchmark/Evm/KeyedNonceStorageSlotBenchmarks.cs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using BenchmarkDotNet.Attributes; +using Nethermind.Core; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Int256; + +namespace Nethermind.Benchmarks.Evm; + +public class KeyedNonceStorageSlotBenchmarks +{ + private readonly UInt256[] _keys = new UInt256[Eip8250Constants.MaxNonceKeys]; + private readonly UInt256[] _indices = new UInt256[Eip8250Constants.MaxNonceKeys]; + + [Params(8, Eip8250Constants.MaxNonceKeys)] + public int Count { get; set; } + + [GlobalSetup] + public void Setup() + { + for (int i = 0; i < _keys.Length; i++) + { + _keys[i] = (UInt256)(i + 1); + } + } + + [Benchmark(Baseline = true)] + public UInt256 Individual() + { + for (int i = 0; i < Count; i++) + { + _indices[i] = KeyedNonceManager.StorageSlot(Address.SystemUser, _keys[i]).Index; + } + + return _indices[Count - 1]; + } + + [Benchmark] + public UInt256 Batched() + { + KeyedNonceManager.StorageIndices(Address.SystemUser, _keys.AsSpan(0, Count), _indices); + return _indices[Count - 1]; + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index 745146b895e5..a8147f72f83e 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -202,6 +202,35 @@ public void Avx512_permutation_matches_scalar() } } + [Test] + public void Avx512_eight_way_64_byte_hash_matches_individual_hashes() + { + if (!Avx512F.IsSupported) + { + Assert.Ignore("AVX-512F intrinsics are not supported on this machine."); + } + + const int inputLength = 64; + const int hashLength = 32; + const int batchSize = 8; + byte[] input = new byte[inputLength * batchSize]; + byte[] output = new byte[hashLength * batchSize]; + + Random random = new(42); + for (int iteration = 0; iteration < 16; iteration++) + { + random.NextBytes(input); + KeccakHash.ComputeHash64Bytes8Avx512(ref input[0], ref output[0]); + + for (int i = 0; i < batchSize; i++) + { + ValueHash256 expected = ValueKeccak.Compute(input.AsSpan(i * inputLength, inputLength)); + Assert.That(output.AsSpan(i * hashLength, hashLength).SequenceEqual(expected.Bytes), Is.True, + $"Hash mismatch at iteration {iteration}, batch index {i}."); + } + } + } + [TestCase("0x", "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")] public void Sanity_check(string hexString, string expected) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs new file mode 100644 index 000000000000..2a7cd62a599b --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using System.Runtime.InteropServices; + +namespace Nethermind.Core.Crypto; + +public sealed partial class KeccakHash +{ + /// Hashes eight consecutive 64-byte inputs into eight consecutive 32-byte outputs. + /// The caller must ensure that AVX-512F is supported and that both buffers have the required fixed size. + [SkipLocalsInit] + internal static unsafe void ComputeHash64Bytes8Avx512(ref byte input, ref byte output) + { + Debug.Assert(Avx512F.IsSupported); + + Vector512 a0; + Vector512 a1; + Vector512 a2; + Vector512 a3; + Vector512 a4; + Vector512 a5; + Vector512 a6; + Vector512 a7; + fixed (byte* inputPtr = &input) + { + a0 = Gather64(inputPtr, 0); + a1 = Gather64(inputPtr, 1); + a2 = Gather64(inputPtr, 2); + a3 = Gather64(inputPtr, 3); + a4 = Gather64(inputPtr, 4); + a5 = Gather64(inputPtr, 5); + a6 = Gather64(inputPtr, 6); + a7 = Gather64(inputPtr, 7); + } + Vector512 a8 = Vector512.Create(1UL); + Vector512 a9 = Vector512.Zero; + Vector512 a10 = Vector512.Zero; + Vector512 a11 = Vector512.Zero; + Vector512 a12 = Vector512.Zero; + Vector512 a13 = Vector512.Zero; + Vector512 a14 = Vector512.Zero; + Vector512 a15 = Vector512.Zero; + Vector512 a16 = Vector512.Create(0x8000000000000000UL); + Vector512 a17 = Vector512.Zero; + Vector512 a18 = Vector512.Zero; + Vector512 a19 = Vector512.Zero; + Vector512 a20 = Vector512.Zero; + Vector512 a21 = Vector512.Zero; + Vector512 a22 = Vector512.Zero; + Vector512 a23 = Vector512.Zero; + Vector512 a24 = Vector512.Zero; + + ref ulong roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstants); + for (int round = 0; round < ROUNDS; round++) + { + RoundX8( + ref a0, ref a1, ref a2, ref a3, ref a4, + ref a5, ref a6, ref a7, ref a8, ref a9, + ref a10, ref a11, ref a12, ref a13, ref a14, + ref a15, ref a16, ref a17, ref a18, ref a19, + ref a20, ref a21, ref a22, ref a23, ref a24, + Vector512.Create(Unsafe.Add(ref roundConstants, round))); + } + + StoreHash(ref output, 0, a0, a1, a2, a3); + StoreHash(ref output, 1, a0, a1, a2, a3); + StoreHash(ref output, 2, a0, a1, a2, a3); + StoreHash(ref output, 3, a0, a1, a2, a3); + StoreHash(ref output, 4, a0, a1, a2, a3); + StoreHash(ref output, 5, a0, a1, a2, a3); + StoreHash(ref output, 6, a0, a1, a2, a3); + StoreHash(ref output, 7, a0, a1, a2, a3); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe Vector512 Gather64(byte* input, int lane) + { + ulong* lanePtr = (ulong*)(input + lane * sizeof(ulong)); + Vector256 lower = Avx2.GatherVector256(lanePtr, Vector256.Create(0L, 8L, 16L, 24L), 8); + Vector256 upper = Avx2.GatherVector256(lanePtr, Vector256.Create(32L, 40L, 48L, 56L), 8); + return Vector512.Create(lower, upper); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreHash(ref byte output, int hashIndex, + Vector512 a0, Vector512 a1, Vector512 a2, Vector512 a3) + { + ref byte destination = ref Unsafe.Add(ref output, hashIndex * 32); + Unsafe.WriteUnaligned(ref destination, a0.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), a1.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 16), a2.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 24), a3.GetElement(hashIndex)); + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void RoundX8( + ref Vector512 a0, ref Vector512 a1, ref Vector512 a2, ref Vector512 a3, ref Vector512 a4, + ref Vector512 a5, ref Vector512 a6, ref Vector512 a7, ref Vector512 a8, ref Vector512 a9, + ref Vector512 a10, ref Vector512 a11, ref Vector512 a12, ref Vector512 a13, ref Vector512 a14, + ref Vector512 a15, ref Vector512 a16, ref Vector512 a17, ref Vector512 a18, ref Vector512 a19, + ref Vector512 a20, ref Vector512 a21, ref Vector512 a22, ref Vector512 a23, ref Vector512 a24, + Vector512 roundConstant) + { + Vector512 c0 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a0, a5, a10, 0x96), a15, a20, 0x96); + Vector512 c1 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a1, a6, a11, 0x96), a16, a21, 0x96); + Vector512 c2 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a2, a7, a12, 0x96), a17, a22, 0x96); + Vector512 c3 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a3, a8, a13, 0x96), a18, a23, 0x96); + Vector512 c4 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a4, a9, a14, 0x96), a19, a24, 0x96); + + Vector512 next = Avx512F.RotateLeft(c1, 1); + a0 = Avx512F.TernaryLogic(a0, c4, next, 0x96); + a5 = Avx512F.TernaryLogic(a5, c4, next, 0x96); + a10 = Avx512F.TernaryLogic(a10, c4, next, 0x96); + a15 = Avx512F.TernaryLogic(a15, c4, next, 0x96); + a20 = Avx512F.TernaryLogic(a20, c4, next, 0x96); + + next = Avx512F.RotateLeft(c2, 1); + a1 = Avx512F.TernaryLogic(a1, c0, next, 0x96); + a6 = Avx512F.TernaryLogic(a6, c0, next, 0x96); + a11 = Avx512F.TernaryLogic(a11, c0, next, 0x96); + a16 = Avx512F.TernaryLogic(a16, c0, next, 0x96); + a21 = Avx512F.TernaryLogic(a21, c0, next, 0x96); + + next = Avx512F.RotateLeft(c3, 1); + a2 = Avx512F.TernaryLogic(a2, c1, next, 0x96); + a7 = Avx512F.TernaryLogic(a7, c1, next, 0x96); + a12 = Avx512F.TernaryLogic(a12, c1, next, 0x96); + a17 = Avx512F.TernaryLogic(a17, c1, next, 0x96); + a22 = Avx512F.TernaryLogic(a22, c1, next, 0x96); + + next = Avx512F.RotateLeft(c4, 1); + a3 = Avx512F.TernaryLogic(a3, c2, next, 0x96); + a8 = Avx512F.TernaryLogic(a8, c2, next, 0x96); + a13 = Avx512F.TernaryLogic(a13, c2, next, 0x96); + a18 = Avx512F.TernaryLogic(a18, c2, next, 0x96); + a23 = Avx512F.TernaryLogic(a23, c2, next, 0x96); + + next = Avx512F.RotateLeft(c0, 1); + a4 = Avx512F.TernaryLogic(a4, c3, next, 0x96); + a9 = Avx512F.TernaryLogic(a9, c3, next, 0x96); + a14 = Avx512F.TernaryLogic(a14, c3, next, 0x96); + a19 = Avx512F.TernaryLogic(a19, c3, next, 0x96); + a24 = Avx512F.TernaryLogic(a24, c3, next, 0x96); + + Vector512 current = a1; + Vector512 temp = a10; a10 = Avx512F.RotateLeft(current, 1); current = temp; + temp = a7; a7 = Avx512F.RotateLeft(current, 3); current = temp; + temp = a11; a11 = Avx512F.RotateLeft(current, 6); current = temp; + temp = a17; a17 = Avx512F.RotateLeft(current, 10); current = temp; + temp = a18; a18 = Avx512F.RotateLeft(current, 15); current = temp; + temp = a3; a3 = Avx512F.RotateLeft(current, 21); current = temp; + temp = a5; a5 = Avx512F.RotateLeft(current, 28); current = temp; + temp = a16; a16 = Avx512F.RotateLeft(current, 36); current = temp; + temp = a8; a8 = Avx512F.RotateLeft(current, 45); current = temp; + temp = a21; a21 = Avx512F.RotateLeft(current, 55); current = temp; + temp = a24; a24 = Avx512F.RotateLeft(current, 2); current = temp; + temp = a4; a4 = Avx512F.RotateLeft(current, 14); current = temp; + temp = a15; a15 = Avx512F.RotateLeft(current, 27); current = temp; + temp = a23; a23 = Avx512F.RotateLeft(current, 41); current = temp; + temp = a19; a19 = Avx512F.RotateLeft(current, 56); current = temp; + temp = a13; a13 = Avx512F.RotateLeft(current, 8); current = temp; + temp = a12; a12 = Avx512F.RotateLeft(current, 25); current = temp; + temp = a2; a2 = Avx512F.RotateLeft(current, 43); current = temp; + temp = a20; a20 = Avx512F.RotateLeft(current, 62); current = temp; + temp = a14; a14 = Avx512F.RotateLeft(current, 18); current = temp; + temp = a22; a22 = Avx512F.RotateLeft(current, 39); current = temp; + temp = a9; a9 = Avx512F.RotateLeft(current, 61); current = temp; + temp = a6; a6 = Avx512F.RotateLeft(current, 20); current = temp; + a1 = Avx512F.RotateLeft(current, 44); + + ChiRow(ref a0, ref a1, ref a2, ref a3, ref a4); + ChiRow(ref a5, ref a6, ref a7, ref a8, ref a9); + ChiRow(ref a10, ref a11, ref a12, ref a13, ref a14); + ChiRow(ref a15, ref a16, ref a17, ref a18, ref a19); + ChiRow(ref a20, ref a21, ref a22, ref a23, ref a24); + a0 = Avx512F.Xor(a0, roundConstant); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ChiRow(ref Vector512 a0, ref Vector512 a1, ref Vector512 a2, + ref Vector512 a3, ref Vector512 a4) + { + Vector512 b0 = a0; + Vector512 b1 = a1; + a0 = Avx512F.TernaryLogic(a0, a1, a2, 0xD2); + a1 = Avx512F.TernaryLogic(a1, a2, a3, 0xD2); + a2 = Avx512F.TernaryLogic(a2, a3, a4, 0xD2); + a3 = Avx512F.TernaryLogic(a3, a4, b0, 0xD2); + a4 = Avx512F.TernaryLogic(a4, b0, b1, 0xD2); + } +} diff --git a/src/Nethermind/Nethermind.Core/InternalsVisibility.cs b/src/Nethermind/Nethermind.Core/InternalsVisibility.cs index 2b44258ba1ec..1d8017e94c55 100644 --- a/src/Nethermind/Nethermind.Core/InternalsVisibility.cs +++ b/src/Nethermind/Nethermind.Core/InternalsVisibility.cs @@ -6,3 +6,4 @@ [assembly: InternalsVisibleTo("Nethermind.Core.Test")] [assembly: InternalsVisibleTo("Nethermind.Blockchain.Test")] [assembly: InternalsVisibleTo("Nethermind.Clique.Test")] +[assembly: InternalsVisibleTo("Nethermind.Evm")] diff --git a/src/Nethermind/Nethermind.Evm.Test/KeyedNonceManagerTests.cs b/src/Nethermind/Nethermind.Evm.Test/KeyedNonceManagerTests.cs index 248b3d3d5353..ec0c0e3321b6 100644 --- a/src/Nethermind/Nethermind.Evm.Test/KeyedNonceManagerTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/KeyedNonceManagerTests.cs @@ -49,6 +49,35 @@ public void StorageSlot_is_deterministic_and_distinct_per_sender_and_key() Assert.That(slotB1.Index, Is.Not.EqualTo(slotA1.Index), "distinct senders must yield distinct slots"); } + [TestCase(8)] + [TestCase(Eip8250Constants.MaxNonceKeys)] + public void Batched_storage_indices_match_individual_slots(int count) + { + UInt256[] keys = StrictlyIncreasing(count); + UInt256[] indices = new UInt256[count]; + + KeyedNonceManager.StorageIndices(TestItem.AddressA, keys, indices); + + for (int i = 0; i < count; i++) + { + Assert.That(indices[i], Is.EqualTo(KeyedNonceManager.StorageSlot(TestItem.AddressA, keys[i]).Index)); + } + } + + [Test] + public void Batched_nonce_set_is_consumed_and_validated() + { + UInt256[] keys = StrictlyIncreasing(Eip8250Constants.MaxNonceKeys); + + KeyedNonceManager.ConsumeNonceSet(_state, TestItem.AddressA, keys, nonceSeq: 41); + + foreach (UInt256 key in keys) + { + Assert.That(KeyedNonceManager.CurrentNonceSeq(_state, TestItem.AddressA, key), Is.EqualTo(42UL)); + } + Assert.That(KeyedNonceManager.IsNonceSetValid(_state, TestItem.AddressA, keys, nonceSeq: 42), Is.True); + } + [Test] public void CurrentNonceSeq_for_key_zero_returns_account_nonce() { diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs index 425ec1497392..06d5ea57a309 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs @@ -3,6 +3,8 @@ using System; using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -15,6 +17,7 @@ namespace Nethermind.Evm.TransactionProcessing; public static class KeyedNonceManager { private const int SlotPreimageLength = 2 * 32; + private const int HashBatchSize = 8; public static StorageCell StorageSlot(Address sender, in UInt256 nonceKey) { @@ -33,7 +36,12 @@ public static ulong CurrentNonceSeq(IWorldState state, Address sender, in UInt25 return state.GetNonce(sender); } - UInt256 stored = new(state.Get(StorageSlot(sender, nonceKey)), isBigEndian: true); + return CurrentNonceSeq(state, StorageSlot(sender, nonceKey)); + } + + private static ulong CurrentNonceSeq(IWorldState state, in StorageCell slot) + { + UInt256 stored = new(state.Get(slot), isBigEndian: true); // Clamp so a crafted high-bit slot cannot false-match a valid nonce_seq < MAX_NONCE_SEQ. return stored > Eip8250Constants.MaxNonceSeq ? ulong.MaxValue : (ulong)stored; } @@ -52,6 +60,18 @@ public static void ConsumeNonceSet(IWorldState state, Address sender, ReadOnlySp Span buffer = stackalloc byte[32]; ((UInt256)nonceSeq + UInt256.One).ToBigEndian(buffer); byte[] nextSeq = buffer.WithoutLeadingZeros().ToArray(); + + if (Avx512F.IsSupported && nonceKeys.Length is >= HashBatchSize and <= Eip8250Constants.MaxNonceKeys) + { + Span indices = stackalloc UInt256[Eip8250Constants.MaxNonceKeys]; + StorageIndices(sender, nonceKeys, indices); + for (int i = 0; i < nonceKeys.Length; i++) + { + state.Set(new StorageCell(Eip8250Constants.NonceManagerAddress, indices[i]), nextSeq); + } + return; + } + foreach (UInt256 nonceKey in nonceKeys) { // EIP-8250 rejects key 0 in a non-[0] set; the decode-time validity check owns that, this guards the primitive. @@ -108,6 +128,21 @@ public static bool IsNonceSetValid(IWorldState state, Address sender, ReadOnlySp return false; } + if (Avx512F.IsSupported && nonceKeys.Length >= HashBatchSize) + { + Span indices = stackalloc UInt256[Eip8250Constants.MaxNonceKeys]; + StorageIndices(sender, nonceKeys, indices); + for (int i = 0; i < nonceKeys.Length; i++) + { + StorageCell slot = new(Eip8250Constants.NonceManagerAddress, indices[i]); + if (CurrentNonceSeq(state, slot) != nonceSeq) + { + return false; + } + } + return true; + } + foreach (ref readonly UInt256 nonceKey in nonceKeys) { if (CurrentNonceSeq(state, sender, nonceKey) != nonceSeq) @@ -118,4 +153,45 @@ public static bool IsNonceSetValid(IWorldState state, Address sender, ReadOnlySp return true; } + + [SkipLocalsInit] + internal static void StorageIndices(Address sender, ReadOnlySpan nonceKeys, Span indices) + { + Debug.Assert(indices.Length >= nonceKeys.Length); + + int keyIndex = 0; + if (Avx512F.IsSupported && nonceKeys.Length >= HashBatchSize) + { + Span preimages = stackalloc byte[SlotPreimageLength * HashBatchSize]; + Span hashes = stackalloc byte[Keccak.Size * HashBatchSize]; + + for (int i = 0; i < HashBatchSize; i++) + { + Span senderBlock = preimages.Slice(i * SlotPreimageLength, 32); + senderBlock[..(32 - Address.Size)].Clear(); + sender.Bytes.CopyTo(senderBlock[(32 - Address.Size)..]); + } + + do + { + for (int i = 0; i < HashBatchSize; i++) + { + nonceKeys[keyIndex + i].ToBigEndian(preimages.Slice(i * SlotPreimageLength + 32, 32)); + } + + KeccakHash.ComputeHash64Bytes8Avx512(ref preimages[0], ref hashes[0]); + for (int i = 0; i < HashBatchSize; i++) + { + indices[keyIndex + i] = new UInt256(hashes.Slice(i * Keccak.Size, Keccak.Size), isBigEndian: true); + } + + keyIndex += HashBatchSize; + } while (keyIndex <= nonceKeys.Length - HashBatchSize); + } + + for (; keyIndex < nonceKeys.Length; keyIndex++) + { + indices[keyIndex] = StorageSlot(sender, nonceKeys[keyIndex]).Index; + } + } } From 50526940df0f134a8d32aa52fa43de6aa4e20a06 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 19 Aug 2026 06:14:10 +0100 Subject: [PATCH 08/12] Use lane-per-register AVX-512VL for Keccak Replace the row-oriented AVX-512F permutation with a lane-per-register AVX-512VL kernel, retain a fused path for 20, 32, and 64-byte Keccak-256 inputs, and fall back to scalar when VL is unavailable. BenchmarkDotNet on Ryzen 9 9950X, .NET 10.0.11: Direct Keccak-f[1600]: AVX-512F 516.9 ns; scalar 194.0 ns; AVX-512VL 164.3 ns. VL is 68.2% faster than the old AVX-512F implementation and 15.3% faster than scalar. ValueKeccak generic VL vs fused VL: 20 bytes 171.1 vs 161.6 ns (-5.5%); 32 bytes 169.1 vs 162.2 ns (-4.1%); 64 bytes 171.9 vs 162.3 ns (-5.6%). JIT disassembly shows all 25 state lanes remain in registers through the VL round loop with no algorithmic spills. Verified by all 1,055 Keccak tests, including 64 full-state comparisons against the scalar permutation and independent known vectors for the specialized input sizes. --- .../Nethermind.Core.Test/KeccakTests.cs | 31 ++- .../Crypto/KeccakHash.avx512vl.cs | 260 ++++++++++++++++++ .../Nethermind.Core/Crypto/KeccakHash.cs | 10 + .../Nethermind.Core/Crypto/KeccakHash.std.cs | 111 +------- 4 files changed, 299 insertions(+), 113 deletions(-) create mode 100644 src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index a8147f72f83e..3b3002da9b76 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -170,11 +170,11 @@ public void Computes_known_hash_for_span_and_array() } [Test] - public void Avx512_permutation_matches_scalar() + public void Avx512VL_permutation_matches_scalar() { - if (!Avx512F.IsSupported) + if (!Avx512F.VL.IsSupported) { - Assert.Ignore("AVX-512F intrinsics are not supported on this machine."); + Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); } const int stateLength = 25; @@ -196,7 +196,7 @@ public void Avx512_permutation_matches_scalar() actual.CopyTo(expected, 0); KeccakHash.KeccakF1600Scalar(ref expected[0]); - KeccakHash.KeccakF1600Avx512F(ref actual[0]); + KeccakHash.KeccakF1600Avx512VL(ref actual[0]); Assert.That(actual, Is.EqualTo(expected), $"Permutation mismatch for test case {testCase}."); } @@ -231,6 +231,29 @@ public void Avx512_eight_way_64_byte_hash_matches_individual_hashes() } } + // Expected hashes were generated with PyCryptodome's independent Keccak-256 implementation. + [TestCase(20, "50c02dbeee2be79b9595060fe30efbd78f06acedf7a1fe8cb05df7ddd76f2b1b")] + [TestCase(32, "d064c972ea7cbd9f1237bbd922fd5f08ca57895c13bc9ea2b91913f7099809a1")] + [TestCase(64, "52c1f4616862f9d5011ed6a2a77d89a2102e51ee7db2db045bb5fb267fba98d1")] + public void Avx512VL_common_input_lengths_match_known_hash(int inputLength, string expected) + { + if (!Avx512F.VL.IsSupported) + { + Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); + } + + byte[] input = new byte[inputLength]; + byte[] output = new byte[32]; + for (int i = 0; i < input.Length; i++) + { + input[i] = (byte)(i * 37 + 11); + } + + KeccakHash.ComputeHash(input, output); + + Assert.That(output.ToHexString(), Is.EqualTo(expected)); + } + [TestCase("0x", "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")] public void Sanity_check(string hexString, string expected) { diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs new file mode 100644 index 000000000000..c1acfc05b1ca --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace Nethermind.Core.Crypto; + +public sealed partial class KeccakHash +{ + /// AVX-512VL Keccak-f[1600] permutation. + /// Lane 0 of a 25-lane state; all 25 lanes are read and written. + [SkipLocalsInit] + internal static void KeccakF1600Avx512VL(ref ulong state) + { + Debug.Assert(Avx512F.VL.IsSupported); + + Vector128 a0 = Vector128.CreateScalarUnsafe(state); + Vector128 a1 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 1)); + Vector128 a2 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 2)); + Vector128 a3 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 3)); + Vector128 a4 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 4)); + Vector128 a5 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 5)); + Vector128 a6 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 6)); + Vector128 a7 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 7)); + Vector128 a8 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 8)); + Vector128 a9 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 9)); + Vector128 a10 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 10)); + Vector128 a11 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 11)); + Vector128 a12 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 12)); + Vector128 a13 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 13)); + Vector128 a14 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 14)); + Vector128 a15 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 15)); + Vector128 a16 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 16)); + Vector128 a17 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 17)); + Vector128 a18 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 18)); + Vector128 a19 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 19)); + Vector128 a20 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 20)); + Vector128 a21 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 21)); + Vector128 a22 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 22)); + Vector128 a23 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 23)); + Vector128 a24 = Vector128.CreateScalarUnsafe(Unsafe.Add(ref state, 24)); + + ref ulong roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstants); + for (int round = 0; round < ROUNDS; round++) + { + RoundX1( + ref a0, ref a1, ref a2, ref a3, ref a4, + ref a5, ref a6, ref a7, ref a8, ref a9, + ref a10, ref a11, ref a12, ref a13, ref a14, + ref a15, ref a16, ref a17, ref a18, ref a19, + ref a20, ref a21, ref a22, ref a23, ref a24, + Vector128.CreateScalarUnsafe(Unsafe.Add(ref roundConstants, round))); + } + + state = a0.GetElement(0); + Unsafe.Add(ref state, 1) = a1.GetElement(0); + Unsafe.Add(ref state, 2) = a2.GetElement(0); + Unsafe.Add(ref state, 3) = a3.GetElement(0); + Unsafe.Add(ref state, 4) = a4.GetElement(0); + Unsafe.Add(ref state, 5) = a5.GetElement(0); + Unsafe.Add(ref state, 6) = a6.GetElement(0); + Unsafe.Add(ref state, 7) = a7.GetElement(0); + Unsafe.Add(ref state, 8) = a8.GetElement(0); + Unsafe.Add(ref state, 9) = a9.GetElement(0); + Unsafe.Add(ref state, 10) = a10.GetElement(0); + Unsafe.Add(ref state, 11) = a11.GetElement(0); + Unsafe.Add(ref state, 12) = a12.GetElement(0); + Unsafe.Add(ref state, 13) = a13.GetElement(0); + Unsafe.Add(ref state, 14) = a14.GetElement(0); + Unsafe.Add(ref state, 15) = a15.GetElement(0); + Unsafe.Add(ref state, 16) = a16.GetElement(0); + Unsafe.Add(ref state, 17) = a17.GetElement(0); + Unsafe.Add(ref state, 18) = a18.GetElement(0); + Unsafe.Add(ref state, 19) = a19.GetElement(0); + Unsafe.Add(ref state, 20) = a20.GetElement(0); + Unsafe.Add(ref state, 21) = a21.GetElement(0); + Unsafe.Add(ref state, 22) = a22.GetElement(0); + Unsafe.Add(ref state, 23) = a23.GetElement(0); + Unsafe.Add(ref state, 24) = a24.GetElement(0); + } + + /// Computes Keccak-256 for a supported one-block input with one Keccak lane per AVX-512VL vector. + [SkipLocalsInit] + private static void ComputeHash256Avx512VL(ref byte input, int inputLength, ref byte output) + { + Debug.Assert(Avx512F.VL.IsSupported); + Debug.Assert(inputLength is Address.Size or 32 or 64); + + Vector128 a0 = LoadScalar128(ref input, 0); + Vector128 a1 = LoadScalar128(ref input, 8); + Vector128 a2 = Vector128.Zero; + Vector128 a3 = Vector128.Zero; + Vector128 a4 = Vector128.Zero; + Vector128 a5 = Vector128.Zero; + Vector128 a6 = Vector128.Zero; + Vector128 a7 = Vector128.Zero; + Vector128 a8 = Vector128.Zero; + Vector128 a9 = Vector128.Zero; + Vector128 a10 = Vector128.Zero; + Vector128 a11 = Vector128.Zero; + Vector128 a12 = Vector128.Zero; + Vector128 a13 = Vector128.Zero; + Vector128 a14 = Vector128.Zero; + Vector128 a15 = Vector128.Zero; + Vector128 a16 = Vector128.CreateScalarUnsafe(0x8000000000000000UL); + Vector128 a17 = Vector128.Zero; + Vector128 a18 = Vector128.Zero; + Vector128 a19 = Vector128.Zero; + Vector128 a20 = Vector128.Zero; + Vector128 a21 = Vector128.Zero; + Vector128 a22 = Vector128.Zero; + Vector128 a23 = Vector128.Zero; + Vector128 a24 = Vector128.Zero; + + switch (inputLength) + { + case Address.Size: + a2 = Vector128.CreateScalarUnsafe( + Unsafe.ReadUnaligned(ref Unsafe.Add(ref input, 16)) | (1UL << 32)); + break; + case 32: + a2 = LoadScalar128(ref input, 16); + a3 = LoadScalar128(ref input, 24); + a4 = Vector128.CreateScalarUnsafe(1UL); + break; + case 64: + a2 = LoadScalar128(ref input, 16); + a3 = LoadScalar128(ref input, 24); + a4 = LoadScalar128(ref input, 32); + a5 = LoadScalar128(ref input, 40); + a6 = LoadScalar128(ref input, 48); + a7 = LoadScalar128(ref input, 56); + a8 = Vector128.CreateScalarUnsafe(1UL); + break; + } + + ref ulong roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstants); + for (int round = 0; round < ROUNDS; round++) + { + RoundX1( + ref a0, ref a1, ref a2, ref a3, ref a4, + ref a5, ref a6, ref a7, ref a8, ref a9, + ref a10, ref a11, ref a12, ref a13, ref a14, + ref a15, ref a16, ref a17, ref a18, ref a19, + ref a20, ref a21, ref a22, ref a23, ref a24, + Vector128.CreateScalarUnsafe(Unsafe.Add(ref roundConstants, round))); + } + + Unsafe.WriteUnaligned(ref output, a0.GetElement(0)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, 8), a1.GetElement(0)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, 16), a2.GetElement(0)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, 24), a3.GetElement(0)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadScalar128(ref byte input, int offset) => + Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned(ref Unsafe.Add(ref input, offset))); + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void RoundX1( + ref Vector128 a0, ref Vector128 a1, ref Vector128 a2, ref Vector128 a3, ref Vector128 a4, + ref Vector128 a5, ref Vector128 a6, ref Vector128 a7, ref Vector128 a8, ref Vector128 a9, + ref Vector128 a10, ref Vector128 a11, ref Vector128 a12, ref Vector128 a13, ref Vector128 a14, + ref Vector128 a15, ref Vector128 a16, ref Vector128 a17, ref Vector128 a18, ref Vector128 a19, + ref Vector128 a20, ref Vector128 a21, ref Vector128 a22, ref Vector128 a23, ref Vector128 a24, + Vector128 roundConstant) + { + Vector128 c0 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a0, a5, a10, 0x96), a15, a20, 0x96); + Vector128 c1 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a1, a6, a11, 0x96), a16, a21, 0x96); + Vector128 c2 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a2, a7, a12, 0x96), a17, a22, 0x96); + Vector128 c3 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a3, a8, a13, 0x96), a18, a23, 0x96); + Vector128 c4 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a4, a9, a14, 0x96), a19, a24, 0x96); + + Vector128 next = Avx512F.VL.RotateLeft(c1, 1); + a0 = Avx512F.VL.TernaryLogic(a0, c4, next, 0x96); + a5 = Avx512F.VL.TernaryLogic(a5, c4, next, 0x96); + a10 = Avx512F.VL.TernaryLogic(a10, c4, next, 0x96); + a15 = Avx512F.VL.TernaryLogic(a15, c4, next, 0x96); + a20 = Avx512F.VL.TernaryLogic(a20, c4, next, 0x96); + + next = Avx512F.VL.RotateLeft(c2, 1); + a1 = Avx512F.VL.TernaryLogic(a1, c0, next, 0x96); + a6 = Avx512F.VL.TernaryLogic(a6, c0, next, 0x96); + a11 = Avx512F.VL.TernaryLogic(a11, c0, next, 0x96); + a16 = Avx512F.VL.TernaryLogic(a16, c0, next, 0x96); + a21 = Avx512F.VL.TernaryLogic(a21, c0, next, 0x96); + + next = Avx512F.VL.RotateLeft(c3, 1); + a2 = Avx512F.VL.TernaryLogic(a2, c1, next, 0x96); + a7 = Avx512F.VL.TernaryLogic(a7, c1, next, 0x96); + a12 = Avx512F.VL.TernaryLogic(a12, c1, next, 0x96); + a17 = Avx512F.VL.TernaryLogic(a17, c1, next, 0x96); + a22 = Avx512F.VL.TernaryLogic(a22, c1, next, 0x96); + + next = Avx512F.VL.RotateLeft(c4, 1); + a3 = Avx512F.VL.TernaryLogic(a3, c2, next, 0x96); + a8 = Avx512F.VL.TernaryLogic(a8, c2, next, 0x96); + a13 = Avx512F.VL.TernaryLogic(a13, c2, next, 0x96); + a18 = Avx512F.VL.TernaryLogic(a18, c2, next, 0x96); + a23 = Avx512F.VL.TernaryLogic(a23, c2, next, 0x96); + + next = Avx512F.VL.RotateLeft(c0, 1); + a4 = Avx512F.VL.TernaryLogic(a4, c3, next, 0x96); + a9 = Avx512F.VL.TernaryLogic(a9, c3, next, 0x96); + a14 = Avx512F.VL.TernaryLogic(a14, c3, next, 0x96); + a19 = Avx512F.VL.TernaryLogic(a19, c3, next, 0x96); + a24 = Avx512F.VL.TernaryLogic(a24, c3, next, 0x96); + + Vector128 current = a1; + Vector128 temp = a10; a10 = Avx512F.VL.RotateLeft(current, 1); current = temp; + temp = a7; a7 = Avx512F.VL.RotateLeft(current, 3); current = temp; + temp = a11; a11 = Avx512F.VL.RotateLeft(current, 6); current = temp; + temp = a17; a17 = Avx512F.VL.RotateLeft(current, 10); current = temp; + temp = a18; a18 = Avx512F.VL.RotateLeft(current, 15); current = temp; + temp = a3; a3 = Avx512F.VL.RotateLeft(current, 21); current = temp; + temp = a5; a5 = Avx512F.VL.RotateLeft(current, 28); current = temp; + temp = a16; a16 = Avx512F.VL.RotateLeft(current, 36); current = temp; + temp = a8; a8 = Avx512F.VL.RotateLeft(current, 45); current = temp; + temp = a21; a21 = Avx512F.VL.RotateLeft(current, 55); current = temp; + temp = a24; a24 = Avx512F.VL.RotateLeft(current, 2); current = temp; + temp = a4; a4 = Avx512F.VL.RotateLeft(current, 14); current = temp; + temp = a15; a15 = Avx512F.VL.RotateLeft(current, 27); current = temp; + temp = a23; a23 = Avx512F.VL.RotateLeft(current, 41); current = temp; + temp = a19; a19 = Avx512F.VL.RotateLeft(current, 56); current = temp; + temp = a13; a13 = Avx512F.VL.RotateLeft(current, 8); current = temp; + temp = a12; a12 = Avx512F.VL.RotateLeft(current, 25); current = temp; + temp = a2; a2 = Avx512F.VL.RotateLeft(current, 43); current = temp; + temp = a20; a20 = Avx512F.VL.RotateLeft(current, 62); current = temp; + temp = a14; a14 = Avx512F.VL.RotateLeft(current, 18); current = temp; + temp = a22; a22 = Avx512F.VL.RotateLeft(current, 39); current = temp; + temp = a9; a9 = Avx512F.VL.RotateLeft(current, 61); current = temp; + temp = a6; a6 = Avx512F.VL.RotateLeft(current, 20); current = temp; + a1 = Avx512F.VL.RotateLeft(current, 44); + + ChiRowX1(ref a0, ref a1, ref a2, ref a3, ref a4); + ChiRowX1(ref a5, ref a6, ref a7, ref a8, ref a9); + ChiRowX1(ref a10, ref a11, ref a12, ref a13, ref a14); + ChiRowX1(ref a15, ref a16, ref a17, ref a18, ref a19); + ChiRowX1(ref a20, ref a21, ref a22, ref a23, ref a24); + a0 = Vector128.Xor(a0, roundConstant); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ChiRowX1(ref Vector128 a0, ref Vector128 a1, ref Vector128 a2, + ref Vector128 a3, ref Vector128 a4) + { + Vector128 b0 = a0; + Vector128 b1 = a1; + a0 = Avx512F.VL.TernaryLogic(a0, a1, a2, 0xD2); + a1 = Avx512F.VL.TernaryLogic(a1, a2, a3, 0xD2); + a2 = Avx512F.VL.TernaryLogic(a2, a3, a4, 0xD2); + a3 = Avx512F.VL.TernaryLogic(a3, a4, b0, 0xD2); + a4 = Avx512F.VL.TernaryLogic(a4, b0, b1, 0xD2); + } +} diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs index 71b321419ef8..a6ec2a69cd2e 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs @@ -7,6 +7,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using System.Security.Cryptography; namespace Nethermind.Core.Crypto; @@ -83,6 +84,15 @@ public static void ComputeHash(ReadOnlySpan input, Span output) return; } #endif + int inputLength = input.Length; + if (Avx512F.VL.IsSupported && output.Length == HASH_SIZE && + (inputLength == Address.Size || inputLength == Vector256.Count || inputLength == Vector512.Count)) + { + ComputeHash256Avx512VL( + ref MemoryMarshal.GetReference(input), inputLength, ref MemoryMarshal.GetReference(output)); + return; + } + int roundSize = GetRoundSize(output.Length); // A struct local rather than stackalloc: localloc would pin this method at Tier0-FullOpts diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index 88e30fb58c5d..a33b1a286e1a 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -3,9 +3,7 @@ using System; using System.Diagnostics; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; using static System.Numerics.BitOperations; @@ -29,23 +27,14 @@ public sealed partial class KeccakHash 0x8000000000008080UL, 0x0000000080000001UL, 0x8000000080008008UL ]; - // Shared lane-index vectors (Theta's [1,2,3,4,0] == Chi's permute1) - private static readonly Vector512 LaneShift1 = Vector512.Create(1UL, 2UL, 3UL, 4UL, 0UL, 5UL, 6UL, 7UL); - private static readonly Vector512 LaneShift2 = Vector512.Create(2UL, 3UL, 4UL, 0UL, 1UL, 5UL, 6UL, 7UL); - private static readonly Vector512 ThetaRot4 = Vector512.Create(4UL, 0UL, 1UL, 2UL, 3UL, 5UL, 6UL, 7UL); - - // Pre-broadcast round constants (kills the per-round vmovq) - private static readonly Vector512[] RoundConstantVec = - Array.ConvertAll(RoundConstants, rc => Vector512.CreateScalar(rc)); - // update the state with given number of rounds private static partial void KeccakF(Span st) { Debug.Assert(st.Length == STATE_LANES); ref ulong state = ref MemoryMarshal.GetReference(st); - if (Avx512F.IsSupported) - KeccakF1600Avx512F(ref state); + if (Avx512F.VL.IsSupported) + KeccakF1600Avx512VL(ref state); else KeccakF1600Scalar(ref state); } @@ -263,100 +252,4 @@ internal static void KeccakF1600Scalar(ref ulong state) st[1] = abe; st[0] = aba; } - - /// AVX-512 Keccak-f[1600] permutation. - /// Lane 0 of a 25-lane state; all 25 lanes are read and written. - [SkipLocalsInit] - internal static void KeccakF1600Avx512F(ref ulong state) - { - Debug.Assert(RoundConstantVec.Length == ROUNDS && ROUNDS % 2 == 0); - - ref ulong s = ref state; - - // Lanes 5-7 hold over-read neighbor lanes and need no masking: Theta/Rho/Pi/Chi - // map result lanes 0-4 only from lanes 0-4, and the stores below overwrite 5-7. - Vector512 c0 = Unsafe.As>(ref s); - Vector512 c1 = Unsafe.As>(ref Unsafe.Add(ref s, 5)); - Vector512 c2 = Unsafe.As>(ref Unsafe.Add(ref s, 10)); - Vector512 c3 = Unsafe.As>(ref Unsafe.Add(ref s, 15)); - Vector512 c4 = Vector512.Create( - Unsafe.As>(ref Unsafe.Add(ref s, 20)), - Vector256.CreateScalar(Unsafe.Add(ref s, 24))); - - // The const bound lets the JIT hoist the vector constants out of the loop; - // bounding by RoundConstantVec.Length makes it re-load all of them every iteration. - ref Vector512 roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstantVec); - for (int round = 0; round < ROUNDS; round += 2) - { - Round(ref c0, ref c1, ref c2, ref c3, ref c4, Unsafe.Add(ref roundConstants, round)); - Round(ref c0, ref c1, ref c2, ref c3, ref c4, Unsafe.Add(ref roundConstants, round + 1)); - } - - Unsafe.As>(ref s) = c0; - Unsafe.As>(ref Unsafe.Add(ref s, 5)) = c1; - Unsafe.As>(ref Unsafe.Add(ref s, 10)) = c2; - Unsafe.As>(ref Unsafe.Add(ref s, 15)) = c3; - Unsafe.As>(ref Unsafe.Add(ref s, 20)) = c4.GetLower(); - Unsafe.Add(ref s, 24) = c4.GetElement(4); - } - - [SkipLocalsInit] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void Round(ref Vector512 c0, ref Vector512 c1, - ref Vector512 c2, ref Vector512 c3, ref Vector512 c4, Vector512 roundConstant) - { - // Theta - Vector512 parity = Avx512F.TernaryLogic(Avx512F.TernaryLogic(c0, c1, c2, 0x96), c3, c4, 0x96); - Vector512 theta = Avx512F.Xor( - Avx512F.PermuteVar8x64(parity, ThetaRot4), - Avx512F.RotateLeft(Avx512F.PermuteVar8x64(parity, LaneShift1), 1)); - c0 = Avx512F.Xor(c0, theta); c1 = Avx512F.Xor(c1, theta); - c2 = Avx512F.Xor(c2, theta); c3 = Avx512F.Xor(c3, theta); c4 = Avx512F.Xor(c4, theta); - - // Rho - c0 = Avx512F.RotateLeftVariable(c0, Vector512.Create(0UL, 1UL, 62UL, 28UL, 27UL, 0UL, 0UL, 0UL)); - c1 = Avx512F.RotateLeftVariable(c1, Vector512.Create(36UL, 44UL, 6UL, 55UL, 20UL, 0UL, 0UL, 0UL)); - c2 = Avx512F.RotateLeftVariable(c2, Vector512.Create(3UL, 10UL, 43UL, 25UL, 39UL, 0UL, 0UL, 0UL)); - c3 = Avx512F.RotateLeftVariable(c3, Vector512.Create(41UL, 45UL, 15UL, 21UL, 8UL, 0UL, 0UL, 0UL)); - c4 = Avx512F.RotateLeftVariable(c4, Vector512.Create(18UL, 2UL, 61UL, 56UL, 14UL, 0UL, 0UL, 0UL)); - - // Pi - // Merge source pairs in parallel to shorten the cross-lane dependency chain. - Vector512 c0Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(0UL, 8 + 1, 2, 3, 4, 5, 6, 7), c1); - Vector512 upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 2, 8 + 3, 4, 5, 6, 7), c3); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); - c0Pi = Avx512F.PermuteVar8x64x2(c0Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 4, 5, 6, 7), c4); - - Vector512 c1Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(3UL, 8 + 4, 2, 3, 4, 5, 6, 7), c1); - upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 0, 8 + 1, 4, 5, 6, 7), c3); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); - c1Pi = Avx512F.PermuteVar8x64x2(c1Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 2, 5, 6, 7), c4); - - Vector512 c2Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(1UL, 8 + 2, 2, 3, 4, 5, 6, 7), c1); - upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 3, 8 + 4, 4, 5, 6, 7), c3); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); - c2Pi = Avx512F.PermuteVar8x64x2(c2Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 0, 5, 6, 7), c4); - - Vector512 c3Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(4UL, 8 + 0, 2, 3, 4, 5, 6, 7), c1); - upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 1, 8 + 2, 4, 5, 6, 7), c3); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); - c3Pi = Avx512F.PermuteVar8x64x2(c3Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 3, 5, 6, 7), c4); - - Vector512 c4Pi = Avx512F.PermuteVar8x64x2(c0, Vector512.Create(2UL, 8 + 3, 2, 3, 4, 5, 6, 7), c1); - upper = Avx512F.PermuteVar8x64x2(c2, Vector512.Create(0UL, 1, 4, 8 + 0, 4, 5, 6, 7), c3); - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 8 + 2, 8 + 3, 4, 5, 6, 7), upper); - c4Pi = Avx512F.PermuteVar8x64x2(c4Pi, Vector512.Create(0UL, 1, 2, 3, 8 + 1, 5, 6, 7), c4); - - c0 = c0Pi; c1 = c1Pi; c2 = c2Pi; c3 = c3Pi; c4 = c4Pi; - - // Chi - c0 = Avx512F.TernaryLogic(c0, Avx512F.PermuteVar8x64(c0, LaneShift1), Avx512F.PermuteVar8x64(c0, LaneShift2), 0xD2); - c1 = Avx512F.TernaryLogic(c1, Avx512F.PermuteVar8x64(c1, LaneShift1), Avx512F.PermuteVar8x64(c1, LaneShift2), 0xD2); - c2 = Avx512F.TernaryLogic(c2, Avx512F.PermuteVar8x64(c2, LaneShift1), Avx512F.PermuteVar8x64(c2, LaneShift2), 0xD2); - c3 = Avx512F.TernaryLogic(c3, Avx512F.PermuteVar8x64(c3, LaneShift1), Avx512F.PermuteVar8x64(c3, LaneShift2), 0xD2); - c4 = Avx512F.TernaryLogic(c4, Avx512F.PermuteVar8x64(c4, LaneShift1), Avx512F.PermuteVar8x64(c4, LaneShift2), 0xD2); - - // Iota - c0 = Vector512.Xor(c0, roundConstant); - } } From 26ef37ed38e16075f687c40649f5fed6969d1d23 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 19 Aug 2026 13:01:08 +0100 Subject: [PATCH 09/12] Tidy the AVX-512 Keccak kernels for review One statement per line in the rho-pi cycle, named vpternlog immediates (Xor3, Chi), spec-step comments (theta, rho+pi, chi, iota), and FIPS 202 padding notes. Rename Gather64 to GatherLane and document the batch helpers. Use explicit input sizes in the ComputeHash fast-path check. Add the zero-key debug assert to the batched consume loop. The in-place rho-pi swap chain and ChiRow helpers are kept as the data flow: rewriting them with a fresh local per lane made the JIT spill and measured ~2.6x slower, so only the formatting and names changed. --- .../Crypto/KeccakHash.avx512vl.cs | 197 ++++++++++------ .../Crypto/KeccakHash.avx512x8.cs | 222 +++++++++++------- .../Nethermind.Core/Crypto/KeccakHash.cs | 3 +- .../KeyedNonceManager.cs | 4 + 4 files changed, 277 insertions(+), 149 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs index c1acfc05b1ca..c36c55c07780 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs @@ -11,6 +11,10 @@ namespace Nethermind.Core.Crypto; public sealed partial class KeccakHash { + // vpternlog immediates: bit n of the immediate is the output for input bits (a, b, c) = binary n. + private const byte Xor3 = 0x96; // a ^ b ^ c + private const byte Chi = 0xD2; // a ^ (~b & c) + /// AVX-512VL Keccak-f[1600] permutation. /// Lane 0 of a 25-lane state; all 25 lanes are read and written. [SkipLocalsInit] @@ -106,6 +110,8 @@ private static void ComputeHash256Avx512VL(ref byte input, int inputLength, ref Vector128 a13 = Vector128.Zero; Vector128 a14 = Vector128.Zero; Vector128 a15 = Vector128.Zero; + // Multi-rate padding (FIPS 202 sec. 5.1): 0x80 at byte 135, the last byte of the + // 136-byte rate, is the top bit of lane 16; 0x01 goes right after the message below. Vector128 a16 = Vector128.CreateScalarUnsafe(0x8000000000000000UL); Vector128 a17 = Vector128.Zero; Vector128 a18 = Vector128.Zero; @@ -119,13 +125,14 @@ private static void ComputeHash256Avx512VL(ref byte input, int inputLength, ref switch (inputLength) { case Address.Size: + // Lane 2 holds the final 4 address bytes with the 0x01 pad in the byte above them (input offset 20). a2 = Vector128.CreateScalarUnsafe( Unsafe.ReadUnaligned(ref Unsafe.Add(ref input, 16)) | (1UL << 32)); break; case 32: a2 = LoadScalar128(ref input, 16); a3 = LoadScalar128(ref input, 24); - a4 = Vector128.CreateScalarUnsafe(1UL); + a4 = Vector128.CreateScalarUnsafe(1UL); // 0x01 pad at input offset 32 break; case 64: a2 = LoadScalar128(ref input, 16); @@ -134,7 +141,7 @@ private static void ComputeHash256Avx512VL(ref byte input, int inputLength, ref a5 = LoadScalar128(ref input, 40); a6 = LoadScalar128(ref input, 48); a7 = LoadScalar128(ref input, 56); - a8 = Vector128.CreateScalarUnsafe(1UL); + a8 = Vector128.CreateScalarUnsafe(1UL); // 0x01 pad at input offset 64 break; } @@ -170,91 +177,147 @@ private static void RoundX1( ref Vector128 a20, ref Vector128 a21, ref Vector128 a22, ref Vector128 a23, ref Vector128 a24, Vector128 roundConstant) { - Vector128 c0 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a0, a5, a10, 0x96), a15, a20, 0x96); - Vector128 c1 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a1, a6, a11, 0x96), a16, a21, 0x96); - Vector128 c2 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a2, a7, a12, 0x96), a17, a22, 0x96); - Vector128 c3 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a3, a8, a13, 0x96), a18, a23, 0x96); - Vector128 c4 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a4, a9, a14, 0x96), a19, a24, 0x96); + // Theta: column parities C[x] = A[x,0] ^ A[x,1] ^ A[x,2] ^ A[x,3] ^ A[x,4]. + Vector128 c0 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a0, a5, a10, Xor3), a15, a20, Xor3); + Vector128 c1 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a1, a6, a11, Xor3), a16, a21, Xor3); + Vector128 c2 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a2, a7, a12, Xor3), a17, a22, Xor3); + Vector128 c3 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a3, a8, a13, Xor3), a18, a23, Xor3); + Vector128 c4 = Avx512F.VL.TernaryLogic(Avx512F.VL.TernaryLogic(a4, a9, a14, Xor3), a19, a24, Xor3); - Vector128 next = Avx512F.VL.RotateLeft(c1, 1); - a0 = Avx512F.VL.TernaryLogic(a0, c4, next, 0x96); - a5 = Avx512F.VL.TernaryLogic(a5, c4, next, 0x96); - a10 = Avx512F.VL.TernaryLogic(a10, c4, next, 0x96); - a15 = Avx512F.VL.TernaryLogic(a15, c4, next, 0x96); - a20 = Avx512F.VL.TernaryLogic(a20, c4, next, 0x96); + // Theta: A[x,y] ^= C[x-1] ^ ROL(C[x+1], 1); both XORs fuse into one ternary op per lane. + Vector128 rolC1 = Avx512F.VL.RotateLeft(c1, 1); + a0 = Avx512F.VL.TernaryLogic(a0, c4, rolC1, Xor3); + a5 = Avx512F.VL.TernaryLogic(a5, c4, rolC1, Xor3); + a10 = Avx512F.VL.TernaryLogic(a10, c4, rolC1, Xor3); + a15 = Avx512F.VL.TernaryLogic(a15, c4, rolC1, Xor3); + a20 = Avx512F.VL.TernaryLogic(a20, c4, rolC1, Xor3); - next = Avx512F.VL.RotateLeft(c2, 1); - a1 = Avx512F.VL.TernaryLogic(a1, c0, next, 0x96); - a6 = Avx512F.VL.TernaryLogic(a6, c0, next, 0x96); - a11 = Avx512F.VL.TernaryLogic(a11, c0, next, 0x96); - a16 = Avx512F.VL.TernaryLogic(a16, c0, next, 0x96); - a21 = Avx512F.VL.TernaryLogic(a21, c0, next, 0x96); + Vector128 rolC2 = Avx512F.VL.RotateLeft(c2, 1); + a1 = Avx512F.VL.TernaryLogic(a1, c0, rolC2, Xor3); + a6 = Avx512F.VL.TernaryLogic(a6, c0, rolC2, Xor3); + a11 = Avx512F.VL.TernaryLogic(a11, c0, rolC2, Xor3); + a16 = Avx512F.VL.TernaryLogic(a16, c0, rolC2, Xor3); + a21 = Avx512F.VL.TernaryLogic(a21, c0, rolC2, Xor3); - next = Avx512F.VL.RotateLeft(c3, 1); - a2 = Avx512F.VL.TernaryLogic(a2, c1, next, 0x96); - a7 = Avx512F.VL.TernaryLogic(a7, c1, next, 0x96); - a12 = Avx512F.VL.TernaryLogic(a12, c1, next, 0x96); - a17 = Avx512F.VL.TernaryLogic(a17, c1, next, 0x96); - a22 = Avx512F.VL.TernaryLogic(a22, c1, next, 0x96); + Vector128 rolC3 = Avx512F.VL.RotateLeft(c3, 1); + a2 = Avx512F.VL.TernaryLogic(a2, c1, rolC3, Xor3); + a7 = Avx512F.VL.TernaryLogic(a7, c1, rolC3, Xor3); + a12 = Avx512F.VL.TernaryLogic(a12, c1, rolC3, Xor3); + a17 = Avx512F.VL.TernaryLogic(a17, c1, rolC3, Xor3); + a22 = Avx512F.VL.TernaryLogic(a22, c1, rolC3, Xor3); - next = Avx512F.VL.RotateLeft(c4, 1); - a3 = Avx512F.VL.TernaryLogic(a3, c2, next, 0x96); - a8 = Avx512F.VL.TernaryLogic(a8, c2, next, 0x96); - a13 = Avx512F.VL.TernaryLogic(a13, c2, next, 0x96); - a18 = Avx512F.VL.TernaryLogic(a18, c2, next, 0x96); - a23 = Avx512F.VL.TernaryLogic(a23, c2, next, 0x96); + Vector128 rolC4 = Avx512F.VL.RotateLeft(c4, 1); + a3 = Avx512F.VL.TernaryLogic(a3, c2, rolC4, Xor3); + a8 = Avx512F.VL.TernaryLogic(a8, c2, rolC4, Xor3); + a13 = Avx512F.VL.TernaryLogic(a13, c2, rolC4, Xor3); + a18 = Avx512F.VL.TernaryLogic(a18, c2, rolC4, Xor3); + a23 = Avx512F.VL.TernaryLogic(a23, c2, rolC4, Xor3); - next = Avx512F.VL.RotateLeft(c0, 1); - a4 = Avx512F.VL.TernaryLogic(a4, c3, next, 0x96); - a9 = Avx512F.VL.TernaryLogic(a9, c3, next, 0x96); - a14 = Avx512F.VL.TernaryLogic(a14, c3, next, 0x96); - a19 = Avx512F.VL.TernaryLogic(a19, c3, next, 0x96); - a24 = Avx512F.VL.TernaryLogic(a24, c3, next, 0x96); + Vector128 rolC0 = Avx512F.VL.RotateLeft(c0, 1); + a4 = Avx512F.VL.TernaryLogic(a4, c3, rolC0, Xor3); + a9 = Avx512F.VL.TernaryLogic(a9, c3, rolC0, Xor3); + a14 = Avx512F.VL.TernaryLogic(a14, c3, rolC0, Xor3); + a19 = Avx512F.VL.TernaryLogic(a19, c3, rolC0, Xor3); + a24 = Avx512F.VL.TernaryLogic(a24, c3, rolC0, Xor3); - Vector128 current = a1; - Vector128 temp = a10; a10 = Avx512F.VL.RotateLeft(current, 1); current = temp; - temp = a7; a7 = Avx512F.VL.RotateLeft(current, 3); current = temp; - temp = a11; a11 = Avx512F.VL.RotateLeft(current, 6); current = temp; - temp = a17; a17 = Avx512F.VL.RotateLeft(current, 10); current = temp; - temp = a18; a18 = Avx512F.VL.RotateLeft(current, 15); current = temp; - temp = a3; a3 = Avx512F.VL.RotateLeft(current, 21); current = temp; - temp = a5; a5 = Avx512F.VL.RotateLeft(current, 28); current = temp; - temp = a16; a16 = Avx512F.VL.RotateLeft(current, 36); current = temp; - temp = a8; a8 = Avx512F.VL.RotateLeft(current, 45); current = temp; - temp = a21; a21 = Avx512F.VL.RotateLeft(current, 55); current = temp; - temp = a24; a24 = Avx512F.VL.RotateLeft(current, 2); current = temp; - temp = a4; a4 = Avx512F.VL.RotateLeft(current, 14); current = temp; - temp = a15; a15 = Avx512F.VL.RotateLeft(current, 27); current = temp; - temp = a23; a23 = Avx512F.VL.RotateLeft(current, 41); current = temp; - temp = a19; a19 = Avx512F.VL.RotateLeft(current, 56); current = temp; - temp = a13; a13 = Avx512F.VL.RotateLeft(current, 8); current = temp; - temp = a12; a12 = Avx512F.VL.RotateLeft(current, 25); current = temp; - temp = a2; a2 = Avx512F.VL.RotateLeft(current, 43); current = temp; - temp = a20; a20 = Avx512F.VL.RotateLeft(current, 62); current = temp; - temp = a14; a14 = Avx512F.VL.RotateLeft(current, 18); current = temp; - temp = a22; a22 = Avx512F.VL.RotateLeft(current, 39); current = temp; - temp = a9; a9 = Avx512F.VL.RotateLeft(current, 61); current = temp; - temp = a6; a6 = Avx512F.VL.RotateLeft(current, 20); current = temp; - a1 = Avx512F.VL.RotateLeft(current, 44); + // Rho + Pi: walk the single 24-lane Pi cycle, rotating each lane into its permuted + // position; lane 0 is the cycle's fixed point. The two temporaries update the lanes + // in place, which keeps all 25 lanes enregistered; a fresh local per lane instead + // makes the JIT spill (measured ~2.6x slower). + Vector128 source = a1; + Vector128 displaced; + displaced = a10; + a10 = Avx512F.VL.RotateLeft(source, 1); + source = displaced; + displaced = a7; + a7 = Avx512F.VL.RotateLeft(source, 3); + source = displaced; + displaced = a11; + a11 = Avx512F.VL.RotateLeft(source, 6); + source = displaced; + displaced = a17; + a17 = Avx512F.VL.RotateLeft(source, 10); + source = displaced; + displaced = a18; + a18 = Avx512F.VL.RotateLeft(source, 15); + source = displaced; + displaced = a3; + a3 = Avx512F.VL.RotateLeft(source, 21); + source = displaced; + displaced = a5; + a5 = Avx512F.VL.RotateLeft(source, 28); + source = displaced; + displaced = a16; + a16 = Avx512F.VL.RotateLeft(source, 36); + source = displaced; + displaced = a8; + a8 = Avx512F.VL.RotateLeft(source, 45); + source = displaced; + displaced = a21; + a21 = Avx512F.VL.RotateLeft(source, 55); + source = displaced; + displaced = a24; + a24 = Avx512F.VL.RotateLeft(source, 2); + source = displaced; + displaced = a4; + a4 = Avx512F.VL.RotateLeft(source, 14); + source = displaced; + displaced = a15; + a15 = Avx512F.VL.RotateLeft(source, 27); + source = displaced; + displaced = a23; + a23 = Avx512F.VL.RotateLeft(source, 41); + source = displaced; + displaced = a19; + a19 = Avx512F.VL.RotateLeft(source, 56); + source = displaced; + displaced = a13; + a13 = Avx512F.VL.RotateLeft(source, 8); + source = displaced; + displaced = a12; + a12 = Avx512F.VL.RotateLeft(source, 25); + source = displaced; + displaced = a2; + a2 = Avx512F.VL.RotateLeft(source, 43); + source = displaced; + displaced = a20; + a20 = Avx512F.VL.RotateLeft(source, 62); + source = displaced; + displaced = a14; + a14 = Avx512F.VL.RotateLeft(source, 18); + source = displaced; + displaced = a22; + a22 = Avx512F.VL.RotateLeft(source, 39); + source = displaced; + displaced = a9; + a9 = Avx512F.VL.RotateLeft(source, 61); + source = displaced; + displaced = a6; + a6 = Avx512F.VL.RotateLeft(source, 20); + source = displaced; + a1 = Avx512F.VL.RotateLeft(source, 44); + // Chi: A[x,y] = B[x,y] ^ (~B[x+1,y] & B[x+2,y]), applied in place one row at a time. ChiRowX1(ref a0, ref a1, ref a2, ref a3, ref a4); ChiRowX1(ref a5, ref a6, ref a7, ref a8, ref a9); ChiRowX1(ref a10, ref a11, ref a12, ref a13, ref a14); ChiRowX1(ref a15, ref a16, ref a17, ref a18, ref a19); ChiRowX1(ref a20, ref a21, ref a22, ref a23, ref a24); + // Iota: fold the round constant into lane 0. a0 = Vector128.Xor(a0, roundConstant); } + /// Applies the Keccak chi mapping to one row of five lanes in place. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ChiRowX1(ref Vector128 a0, ref Vector128 a1, ref Vector128 a2, ref Vector128 a3, ref Vector128 a4) { Vector128 b0 = a0; Vector128 b1 = a1; - a0 = Avx512F.VL.TernaryLogic(a0, a1, a2, 0xD2); - a1 = Avx512F.VL.TernaryLogic(a1, a2, a3, 0xD2); - a2 = Avx512F.VL.TernaryLogic(a2, a3, a4, 0xD2); - a3 = Avx512F.VL.TernaryLogic(a3, a4, b0, 0xD2); - a4 = Avx512F.VL.TernaryLogic(a4, b0, b1, 0xD2); + a0 = Avx512F.VL.TernaryLogic(a0, a1, a2, Chi); + a1 = Avx512F.VL.TernaryLogic(a1, a2, a3, Chi); + a2 = Avx512F.VL.TernaryLogic(a2, a3, a4, Chi); + a3 = Avx512F.VL.TernaryLogic(a3, a4, b0, Chi); + a4 = Avx512F.VL.TernaryLogic(a4, b0, b1, Chi); } } diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs index 2a7cd62a599b..93fafbb5501d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs @@ -4,9 +4,9 @@ using System; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; -using System.Runtime.InteropServices; namespace Nethermind.Core.Crypto; @@ -29,15 +29,17 @@ internal static unsafe void ComputeHash64Bytes8Avx512(ref byte input, ref byte o Vector512 a7; fixed (byte* inputPtr = &input) { - a0 = Gather64(inputPtr, 0); - a1 = Gather64(inputPtr, 1); - a2 = Gather64(inputPtr, 2); - a3 = Gather64(inputPtr, 3); - a4 = Gather64(inputPtr, 4); - a5 = Gather64(inputPtr, 5); - a6 = Gather64(inputPtr, 6); - a7 = Gather64(inputPtr, 7); + a0 = GatherLane(inputPtr, 0); + a1 = GatherLane(inputPtr, 1); + a2 = GatherLane(inputPtr, 2); + a3 = GatherLane(inputPtr, 3); + a4 = GatherLane(inputPtr, 4); + a5 = GatherLane(inputPtr, 5); + a6 = GatherLane(inputPtr, 6); + a7 = GatherLane(inputPtr, 7); } + // Multi-rate padding (FIPS 202 sec. 5.1): 0x01 right after each 64-byte input, and 0x80 + // at byte 135, the last byte of the 136-byte rate, which is the top bit of lane 16. Vector512 a8 = Vector512.Create(1UL); Vector512 a9 = Vector512.Zero; Vector512 a10 = Vector512.Zero; @@ -78,8 +80,9 @@ internal static unsafe void ComputeHash64Bytes8Avx512(ref byte input, ref byte o StoreHash(ref output, 7, a0, a1, a2, a3); } + /// Gathers Keccak lane from each of the eight consecutive 64-byte inputs. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector512 Gather64(byte* input, int lane) + private static unsafe Vector512 GatherLane(byte* input, int lane) { ulong* lanePtr = (ulong*)(input + lane * sizeof(ulong)); Vector256 lower = Avx2.GatherVector256(lanePtr, Vector256.Create(0L, 8L, 16L, 24L), 8); @@ -87,6 +90,7 @@ private static unsafe Vector512 Gather64(byte* input, int lane) return Vector512.Create(lower, upper); } + /// Writes the 32-byte hash of batch element from the first four state lanes. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void StoreHash(ref byte output, int hashIndex, Vector512 a0, Vector512 a1, Vector512 a2, Vector512 a3) @@ -108,91 +112,147 @@ private static void RoundX8( ref Vector512 a20, ref Vector512 a21, ref Vector512 a22, ref Vector512 a23, ref Vector512 a24, Vector512 roundConstant) { - Vector512 c0 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a0, a5, a10, 0x96), a15, a20, 0x96); - Vector512 c1 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a1, a6, a11, 0x96), a16, a21, 0x96); - Vector512 c2 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a2, a7, a12, 0x96), a17, a22, 0x96); - Vector512 c3 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a3, a8, a13, 0x96), a18, a23, 0x96); - Vector512 c4 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a4, a9, a14, 0x96), a19, a24, 0x96); - - Vector512 next = Avx512F.RotateLeft(c1, 1); - a0 = Avx512F.TernaryLogic(a0, c4, next, 0x96); - a5 = Avx512F.TernaryLogic(a5, c4, next, 0x96); - a10 = Avx512F.TernaryLogic(a10, c4, next, 0x96); - a15 = Avx512F.TernaryLogic(a15, c4, next, 0x96); - a20 = Avx512F.TernaryLogic(a20, c4, next, 0x96); - - next = Avx512F.RotateLeft(c2, 1); - a1 = Avx512F.TernaryLogic(a1, c0, next, 0x96); - a6 = Avx512F.TernaryLogic(a6, c0, next, 0x96); - a11 = Avx512F.TernaryLogic(a11, c0, next, 0x96); - a16 = Avx512F.TernaryLogic(a16, c0, next, 0x96); - a21 = Avx512F.TernaryLogic(a21, c0, next, 0x96); - - next = Avx512F.RotateLeft(c3, 1); - a2 = Avx512F.TernaryLogic(a2, c1, next, 0x96); - a7 = Avx512F.TernaryLogic(a7, c1, next, 0x96); - a12 = Avx512F.TernaryLogic(a12, c1, next, 0x96); - a17 = Avx512F.TernaryLogic(a17, c1, next, 0x96); - a22 = Avx512F.TernaryLogic(a22, c1, next, 0x96); - - next = Avx512F.RotateLeft(c4, 1); - a3 = Avx512F.TernaryLogic(a3, c2, next, 0x96); - a8 = Avx512F.TernaryLogic(a8, c2, next, 0x96); - a13 = Avx512F.TernaryLogic(a13, c2, next, 0x96); - a18 = Avx512F.TernaryLogic(a18, c2, next, 0x96); - a23 = Avx512F.TernaryLogic(a23, c2, next, 0x96); - - next = Avx512F.RotateLeft(c0, 1); - a4 = Avx512F.TernaryLogic(a4, c3, next, 0x96); - a9 = Avx512F.TernaryLogic(a9, c3, next, 0x96); - a14 = Avx512F.TernaryLogic(a14, c3, next, 0x96); - a19 = Avx512F.TernaryLogic(a19, c3, next, 0x96); - a24 = Avx512F.TernaryLogic(a24, c3, next, 0x96); - - Vector512 current = a1; - Vector512 temp = a10; a10 = Avx512F.RotateLeft(current, 1); current = temp; - temp = a7; a7 = Avx512F.RotateLeft(current, 3); current = temp; - temp = a11; a11 = Avx512F.RotateLeft(current, 6); current = temp; - temp = a17; a17 = Avx512F.RotateLeft(current, 10); current = temp; - temp = a18; a18 = Avx512F.RotateLeft(current, 15); current = temp; - temp = a3; a3 = Avx512F.RotateLeft(current, 21); current = temp; - temp = a5; a5 = Avx512F.RotateLeft(current, 28); current = temp; - temp = a16; a16 = Avx512F.RotateLeft(current, 36); current = temp; - temp = a8; a8 = Avx512F.RotateLeft(current, 45); current = temp; - temp = a21; a21 = Avx512F.RotateLeft(current, 55); current = temp; - temp = a24; a24 = Avx512F.RotateLeft(current, 2); current = temp; - temp = a4; a4 = Avx512F.RotateLeft(current, 14); current = temp; - temp = a15; a15 = Avx512F.RotateLeft(current, 27); current = temp; - temp = a23; a23 = Avx512F.RotateLeft(current, 41); current = temp; - temp = a19; a19 = Avx512F.RotateLeft(current, 56); current = temp; - temp = a13; a13 = Avx512F.RotateLeft(current, 8); current = temp; - temp = a12; a12 = Avx512F.RotateLeft(current, 25); current = temp; - temp = a2; a2 = Avx512F.RotateLeft(current, 43); current = temp; - temp = a20; a20 = Avx512F.RotateLeft(current, 62); current = temp; - temp = a14; a14 = Avx512F.RotateLeft(current, 18); current = temp; - temp = a22; a22 = Avx512F.RotateLeft(current, 39); current = temp; - temp = a9; a9 = Avx512F.RotateLeft(current, 61); current = temp; - temp = a6; a6 = Avx512F.RotateLeft(current, 20); current = temp; - a1 = Avx512F.RotateLeft(current, 44); + // Theta: column parities C[x] = A[x,0] ^ A[x,1] ^ A[x,2] ^ A[x,3] ^ A[x,4]. + Vector512 c0 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a0, a5, a10, Xor3), a15, a20, Xor3); + Vector512 c1 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a1, a6, a11, Xor3), a16, a21, Xor3); + Vector512 c2 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a2, a7, a12, Xor3), a17, a22, Xor3); + Vector512 c3 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a3, a8, a13, Xor3), a18, a23, Xor3); + Vector512 c4 = Avx512F.TernaryLogic(Avx512F.TernaryLogic(a4, a9, a14, Xor3), a19, a24, Xor3); + + // Theta: A[x,y] ^= C[x-1] ^ ROL(C[x+1], 1); both XORs fuse into one ternary op per lane. + Vector512 rolC1 = Avx512F.RotateLeft(c1, 1); + a0 = Avx512F.TernaryLogic(a0, c4, rolC1, Xor3); + a5 = Avx512F.TernaryLogic(a5, c4, rolC1, Xor3); + a10 = Avx512F.TernaryLogic(a10, c4, rolC1, Xor3); + a15 = Avx512F.TernaryLogic(a15, c4, rolC1, Xor3); + a20 = Avx512F.TernaryLogic(a20, c4, rolC1, Xor3); + + Vector512 rolC2 = Avx512F.RotateLeft(c2, 1); + a1 = Avx512F.TernaryLogic(a1, c0, rolC2, Xor3); + a6 = Avx512F.TernaryLogic(a6, c0, rolC2, Xor3); + a11 = Avx512F.TernaryLogic(a11, c0, rolC2, Xor3); + a16 = Avx512F.TernaryLogic(a16, c0, rolC2, Xor3); + a21 = Avx512F.TernaryLogic(a21, c0, rolC2, Xor3); + + Vector512 rolC3 = Avx512F.RotateLeft(c3, 1); + a2 = Avx512F.TernaryLogic(a2, c1, rolC3, Xor3); + a7 = Avx512F.TernaryLogic(a7, c1, rolC3, Xor3); + a12 = Avx512F.TernaryLogic(a12, c1, rolC3, Xor3); + a17 = Avx512F.TernaryLogic(a17, c1, rolC3, Xor3); + a22 = Avx512F.TernaryLogic(a22, c1, rolC3, Xor3); + + Vector512 rolC4 = Avx512F.RotateLeft(c4, 1); + a3 = Avx512F.TernaryLogic(a3, c2, rolC4, Xor3); + a8 = Avx512F.TernaryLogic(a8, c2, rolC4, Xor3); + a13 = Avx512F.TernaryLogic(a13, c2, rolC4, Xor3); + a18 = Avx512F.TernaryLogic(a18, c2, rolC4, Xor3); + a23 = Avx512F.TernaryLogic(a23, c2, rolC4, Xor3); + + Vector512 rolC0 = Avx512F.RotateLeft(c0, 1); + a4 = Avx512F.TernaryLogic(a4, c3, rolC0, Xor3); + a9 = Avx512F.TernaryLogic(a9, c3, rolC0, Xor3); + a14 = Avx512F.TernaryLogic(a14, c3, rolC0, Xor3); + a19 = Avx512F.TernaryLogic(a19, c3, rolC0, Xor3); + a24 = Avx512F.TernaryLogic(a24, c3, rolC0, Xor3); + + // Rho + Pi: walk the single 24-lane Pi cycle, rotating each lane into its permuted + // position; lane 0 is the cycle's fixed point. The two temporaries update the lanes + // in place, which keeps all 25 lanes enregistered; a fresh local per lane instead + // makes the JIT spill (measured ~2.6x slower). + Vector512 source = a1; + Vector512 displaced; + displaced = a10; + a10 = Avx512F.RotateLeft(source, 1); + source = displaced; + displaced = a7; + a7 = Avx512F.RotateLeft(source, 3); + source = displaced; + displaced = a11; + a11 = Avx512F.RotateLeft(source, 6); + source = displaced; + displaced = a17; + a17 = Avx512F.RotateLeft(source, 10); + source = displaced; + displaced = a18; + a18 = Avx512F.RotateLeft(source, 15); + source = displaced; + displaced = a3; + a3 = Avx512F.RotateLeft(source, 21); + source = displaced; + displaced = a5; + a5 = Avx512F.RotateLeft(source, 28); + source = displaced; + displaced = a16; + a16 = Avx512F.RotateLeft(source, 36); + source = displaced; + displaced = a8; + a8 = Avx512F.RotateLeft(source, 45); + source = displaced; + displaced = a21; + a21 = Avx512F.RotateLeft(source, 55); + source = displaced; + displaced = a24; + a24 = Avx512F.RotateLeft(source, 2); + source = displaced; + displaced = a4; + a4 = Avx512F.RotateLeft(source, 14); + source = displaced; + displaced = a15; + a15 = Avx512F.RotateLeft(source, 27); + source = displaced; + displaced = a23; + a23 = Avx512F.RotateLeft(source, 41); + source = displaced; + displaced = a19; + a19 = Avx512F.RotateLeft(source, 56); + source = displaced; + displaced = a13; + a13 = Avx512F.RotateLeft(source, 8); + source = displaced; + displaced = a12; + a12 = Avx512F.RotateLeft(source, 25); + source = displaced; + displaced = a2; + a2 = Avx512F.RotateLeft(source, 43); + source = displaced; + displaced = a20; + a20 = Avx512F.RotateLeft(source, 62); + source = displaced; + displaced = a14; + a14 = Avx512F.RotateLeft(source, 18); + source = displaced; + displaced = a22; + a22 = Avx512F.RotateLeft(source, 39); + source = displaced; + displaced = a9; + a9 = Avx512F.RotateLeft(source, 61); + source = displaced; + displaced = a6; + a6 = Avx512F.RotateLeft(source, 20); + source = displaced; + a1 = Avx512F.RotateLeft(source, 44); + // Chi: A[x,y] = B[x,y] ^ (~B[x+1,y] & B[x+2,y]), applied in place one row at a time. ChiRow(ref a0, ref a1, ref a2, ref a3, ref a4); ChiRow(ref a5, ref a6, ref a7, ref a8, ref a9); ChiRow(ref a10, ref a11, ref a12, ref a13, ref a14); ChiRow(ref a15, ref a16, ref a17, ref a18, ref a19); ChiRow(ref a20, ref a21, ref a22, ref a23, ref a24); + // Iota: fold the round constant into lane 0. a0 = Avx512F.Xor(a0, roundConstant); } + /// Applies the Keccak chi mapping to one row of five lanes in place. [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ChiRow(ref Vector512 a0, ref Vector512 a1, ref Vector512 a2, ref Vector512 a3, ref Vector512 a4) { Vector512 b0 = a0; Vector512 b1 = a1; - a0 = Avx512F.TernaryLogic(a0, a1, a2, 0xD2); - a1 = Avx512F.TernaryLogic(a1, a2, a3, 0xD2); - a2 = Avx512F.TernaryLogic(a2, a3, a4, 0xD2); - a3 = Avx512F.TernaryLogic(a3, a4, b0, 0xD2); - a4 = Avx512F.TernaryLogic(a4, b0, b1, 0xD2); + a0 = Avx512F.TernaryLogic(a0, a1, a2, Chi); + a1 = Avx512F.TernaryLogic(a1, a2, a3, Chi); + a2 = Avx512F.TernaryLogic(a2, a3, a4, Chi); + a3 = Avx512F.TernaryLogic(a3, a4, b0, Chi); + a4 = Avx512F.TernaryLogic(a4, b0, b1, Chi); } } diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs index a6ec2a69cd2e..2759802594d2 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs @@ -85,8 +85,9 @@ public static void ComputeHash(ReadOnlySpan input, Span output) } #endif int inputLength = input.Length; + // One-block fast path for the dominant EVM input sizes: address (20), word or hash (32), two words (64). if (Avx512F.VL.IsSupported && output.Length == HASH_SIZE && - (inputLength == Address.Size || inputLength == Vector256.Count || inputLength == Vector512.Count)) + inputLength is Address.Size or 32 or 64) { ComputeHash256Avx512VL( ref MemoryMarshal.GetReference(input), inputLength, ref MemoryMarshal.GetReference(output)); diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs index 06d5ea57a309..59ac4d91d90d 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs @@ -17,6 +17,7 @@ namespace Nethermind.Evm.TransactionProcessing; public static class KeyedNonceManager { private const int SlotPreimageLength = 2 * 32; + /// Batch width of . private const int HashBatchSize = 8; public static StorageCell StorageSlot(Address sender, in UInt256 nonceKey) @@ -67,6 +68,7 @@ public static void ConsumeNonceSet(IWorldState state, Address sender, ReadOnlySp StorageIndices(sender, nonceKeys, indices); for (int i = 0; i < nonceKeys.Length; i++) { + Debug.Assert(!nonceKeys[i].IsZero, "key 0 must not appear in a non-[0] nonce_keys set"); state.Set(new StorageCell(Eip8250Constants.NonceManagerAddress, indices[i]), nextSeq); } return; @@ -128,6 +130,7 @@ public static bool IsNonceSetValid(IWorldState state, Address sender, ReadOnlySp return false; } + // Well-formedness above bounds Length to MaxNonceKeys, so the stackalloc below always fits. if (Avx512F.IsSupported && nonceKeys.Length >= HashBatchSize) { Span indices = stackalloc UInt256[Eip8250Constants.MaxNonceKeys]; @@ -154,6 +157,7 @@ public static bool IsNonceSetValid(IWorldState state, Address sender, ReadOnlySp return true; } + /// Computes the NONCE_MANAGER storage index for each key in , batching the Keccak hashes with AVX-512 where possible. [SkipLocalsInit] internal static void StorageIndices(Address sender, ReadOnlySpan nonceKeys, Span indices) { From 11f618bc890592f7aa4e6e1486f88154fdbbfb96 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 19 Aug 2026 14:25:31 +0100 Subject: [PATCH 10/12] Batch full trie branch hashes with AVX-512VL Full branch nodes whose 16 children are hashes have a fixed 532-byte RLP. Defer those sibling hashes during serial branch encoding and process them in pairs with a lane-per-message AVX-512VL Keccak kernel. A 16-bit child mask avoids carrying a reference buffer through the recursive traversal; an odd final candidate keeps the scalar path. Benchmarked on an AMD Ryzen 9 9950X. The focused 532-byte benchmark improved two hashes from 1485.6 ns scalar to 724.5 ns batched (-51.2%), and an eight-full-child synthetic trie improved from 8517.4 ns to 6430.6 ns (-24.5%). A 65,536-entry trie workload used the median of three independent process runs; each run took the median of nine batches of 16 pre-built updates. Default-parallel account hashing for 300 updates improved from 184.3 us to 175.6 us (-4.7%). Storage hashing improved from 7.8 to 7.6 us for 2 updates (-2.6%), 13.6 to 12.7 us for 4 (-6.6%), 23.4 to 20.4 us for 8 (-12.8%), 43.6 to 39.2 us for 16 (-10.1%), 148.9 to 134.1 us for 64 (-9.9%), and 170.4 to 160.1 us for 300 parallel updates (-6.0%). JitAsm confirms the x2 permutation reaches Tier 1 normally and keeps all 25 state lanes in registers, with only the required callee-saved XMM save/restore traffic. Tests: 1056 Keccak tests passed; the full trie suite passed 471 with 7 existing skips; TrieNodeTests passed 75 with AVX-512 disabled and the intrinsic-specific test skipped. --- .../Nethermind.Core.Test/KeccakTests.cs | 29 ++++ .../Crypto/KeccakHash.avx512vl.cs | 130 ++++++++++++++++++ .../Nethermind.Core/InternalsVisibility.cs | 1 + .../Nethermind.Trie.Test/TrieNodeTests.cs | 38 +++++ .../Nethermind.Trie/TrieNode.Decoder.cs | 108 ++++++++++++++- src/Nethermind/Nethermind.Trie/TrieNode.cs | 35 ++++- 6 files changed, 331 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index 3b3002da9b76..24d8b307b7d6 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -231,6 +231,35 @@ public void Avx512_eight_way_64_byte_hash_matches_individual_hashes() } } + [Test] + public void Avx512vl_two_way_532_byte_hash_matches_individual_hashes() + { + if (!Avx512F.VL.IsSupported) + { + Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); + } + + byte[] input0 = new byte[532]; + byte[] input1 = new byte[532]; + byte[][] inputs = [input0, input1]; + byte[] output = new byte[2 * Hash256.Size]; + Random random = new(42); + + for (int iteration = 0; iteration < 16; iteration++) + { + random.NextBytes(input0); + random.NextBytes(input1); + KeccakHash.ComputeHash532Bytes2Avx512VL(ref input0[0], ref input1[0], ref output[0]); + + for (int i = 0; i < inputs.Length; i++) + { + ValueHash256 expected = ValueKeccak.Compute(inputs[i]); + Assert.That(output.AsSpan(i * Hash256.Size, Hash256.Size).SequenceEqual(expected.Bytes), Is.True, + $"Hash mismatch at iteration {iteration}, batch index {i}."); + } + } + } + // Expected hashes were generated with PyCryptodome's independent Keccak-256 implementation. [TestCase(20, "50c02dbeee2be79b9595060fe30efbd78f06acedf7a1fe8cb05df7ddd76f2b1b")] [TestCase(32, "d064c972ea7cbd9f1237bbd922fd5f08ca57895c13bc9ea2b91913f7099809a1")] diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs index c36c55c07780..83c28329ee37 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs @@ -11,6 +11,14 @@ namespace Nethermind.Core.Crypto; public sealed partial class KeccakHash { + // A struct local preserves tiering and dynamic PGO. Stackalloc would pin the hashing method at + // Tier0 FullOpts and add per-call GS-cookie and stack-probe checks. + [InlineArray(STATE_LANES)] + private struct KeccakStateX2 + { + private Vector128 _lane0; + } + // vpternlog immediates: bit n of the immediate is the output for input bits (a, b, c) = binary n. private const byte Xor3 = 0x96; // a ^ b ^ c private const byte Chi = 0xD2; // a ^ (~b & c) @@ -163,6 +171,128 @@ private static void ComputeHash256Avx512VL(ref byte input, int inputLength, ref Unsafe.WriteUnaligned(ref Unsafe.Add(ref output, 24), a3.GetElement(0)); } + /// Hashes two 532-byte inputs into two consecutive 32-byte outputs. + /// The caller must ensure that AVX-512VL is supported and both inputs have the required fixed size. + [SkipLocalsInit] + internal static void ComputeHash532Bytes2Avx512VL(ref byte input0, ref byte input1, ref byte output) + { + Debug.Assert(Avx512F.VL.IsSupported); + + KeccakStateX2 stateBuffer = default; + ref Vector128 state = ref stateBuffer[0]; + for (int blockOffset = 0; blockOffset < 3 * HASH_DATA_AREA; blockOffset += HASH_DATA_AREA) + { + for (int lane = 0; lane < HASH_DATA_AREA / sizeof(ulong); lane++) + { + int offset = blockOffset + lane * sizeof(ulong); + Unsafe.Add(ref state, lane) ^= LoadPair(ref input0, ref input1, offset); + } + + KeccakF1600x2Avx512VL(ref state); + } + + const int finalBlockOffset = 3 * HASH_DATA_AREA; + for (int lane = 0; lane < 15; lane++) + { + int offset = finalBlockOffset + lane * sizeof(ulong); + Unsafe.Add(ref state, lane) ^= LoadPair(ref input0, ref input1, offset); + } + + Unsafe.Add(ref state, 15) ^= Vector128.Create( + Unsafe.ReadUnaligned(ref Unsafe.Add(ref input0, finalBlockOffset + 120)) | (1UL << 32), + Unsafe.ReadUnaligned(ref Unsafe.Add(ref input1, finalBlockOffset + 120)) | (1UL << 32)); + Unsafe.Add(ref state, 16) ^= Vector128.Create(0x8000000000000000UL); + KeccakF1600x2Avx512VL(ref state); + + StoreHashPair(ref output, 0, state, Unsafe.Add(ref state, 1), Unsafe.Add(ref state, 2), Unsafe.Add(ref state, 3)); + StoreHashPair(ref output, 1, state, Unsafe.Add(ref state, 1), Unsafe.Add(ref state, 2), Unsafe.Add(ref state, 3)); + } + + [SkipLocalsInit] + private static void KeccakF1600x2Avx512VL(ref Vector128 state) + { + Vector128 a0 = state; + Vector128 a1 = Unsafe.Add(ref state, 1); + Vector128 a2 = Unsafe.Add(ref state, 2); + Vector128 a3 = Unsafe.Add(ref state, 3); + Vector128 a4 = Unsafe.Add(ref state, 4); + Vector128 a5 = Unsafe.Add(ref state, 5); + Vector128 a6 = Unsafe.Add(ref state, 6); + Vector128 a7 = Unsafe.Add(ref state, 7); + Vector128 a8 = Unsafe.Add(ref state, 8); + Vector128 a9 = Unsafe.Add(ref state, 9); + Vector128 a10 = Unsafe.Add(ref state, 10); + Vector128 a11 = Unsafe.Add(ref state, 11); + Vector128 a12 = Unsafe.Add(ref state, 12); + Vector128 a13 = Unsafe.Add(ref state, 13); + Vector128 a14 = Unsafe.Add(ref state, 14); + Vector128 a15 = Unsafe.Add(ref state, 15); + Vector128 a16 = Unsafe.Add(ref state, 16); + Vector128 a17 = Unsafe.Add(ref state, 17); + Vector128 a18 = Unsafe.Add(ref state, 18); + Vector128 a19 = Unsafe.Add(ref state, 19); + Vector128 a20 = Unsafe.Add(ref state, 20); + Vector128 a21 = Unsafe.Add(ref state, 21); + Vector128 a22 = Unsafe.Add(ref state, 22); + Vector128 a23 = Unsafe.Add(ref state, 23); + Vector128 a24 = Unsafe.Add(ref state, 24); + + ref ulong roundConstants = ref MemoryMarshal.GetArrayDataReference(RoundConstants); + for (int round = 0; round < ROUNDS; round++) + { + RoundX1( + ref a0, ref a1, ref a2, ref a3, ref a4, + ref a5, ref a6, ref a7, ref a8, ref a9, + ref a10, ref a11, ref a12, ref a13, ref a14, + ref a15, ref a16, ref a17, ref a18, ref a19, + ref a20, ref a21, ref a22, ref a23, ref a24, + Vector128.Create(Unsafe.Add(ref roundConstants, round))); + } + + state = a0; + Unsafe.Add(ref state, 1) = a1; + Unsafe.Add(ref state, 2) = a2; + Unsafe.Add(ref state, 3) = a3; + Unsafe.Add(ref state, 4) = a4; + Unsafe.Add(ref state, 5) = a5; + Unsafe.Add(ref state, 6) = a6; + Unsafe.Add(ref state, 7) = a7; + Unsafe.Add(ref state, 8) = a8; + Unsafe.Add(ref state, 9) = a9; + Unsafe.Add(ref state, 10) = a10; + Unsafe.Add(ref state, 11) = a11; + Unsafe.Add(ref state, 12) = a12; + Unsafe.Add(ref state, 13) = a13; + Unsafe.Add(ref state, 14) = a14; + Unsafe.Add(ref state, 15) = a15; + Unsafe.Add(ref state, 16) = a16; + Unsafe.Add(ref state, 17) = a17; + Unsafe.Add(ref state, 18) = a18; + Unsafe.Add(ref state, 19) = a19; + Unsafe.Add(ref state, 20) = a20; + Unsafe.Add(ref state, 21) = a21; + Unsafe.Add(ref state, 22) = a22; + Unsafe.Add(ref state, 23) = a23; + Unsafe.Add(ref state, 24) = a24; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadPair(ref byte input0, ref byte input1, int offset) => + Vector128.Create( + Unsafe.ReadUnaligned(ref Unsafe.Add(ref input0, offset)), + Unsafe.ReadUnaligned(ref Unsafe.Add(ref input1, offset))); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreHashPair(ref byte output, int hashIndex, + Vector128 a0, Vector128 a1, Vector128 a2, Vector128 a3) + { + ref byte destination = ref Unsafe.Add(ref output, hashIndex * Hash256.Size); + Unsafe.WriteUnaligned(ref destination, a0.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), a1.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 16), a2.GetElement(hashIndex)); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 24), a3.GetElement(hashIndex)); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 LoadScalar128(ref byte input, int offset) => Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned(ref Unsafe.Add(ref input, offset))); diff --git a/src/Nethermind/Nethermind.Core/InternalsVisibility.cs b/src/Nethermind/Nethermind.Core/InternalsVisibility.cs index 1d8017e94c55..1facbf163195 100644 --- a/src/Nethermind/Nethermind.Core/InternalsVisibility.cs +++ b/src/Nethermind/Nethermind.Core/InternalsVisibility.cs @@ -7,3 +7,4 @@ [assembly: InternalsVisibleTo("Nethermind.Blockchain.Test")] [assembly: InternalsVisibleTo("Nethermind.Clique.Test")] [assembly: InternalsVisibleTo("Nethermind.Evm")] +[assembly: InternalsVisibleTo("Nethermind.Trie")] diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index b740c84c8b22..799bfdb0f24a 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -214,6 +214,44 @@ public void Can_encode_decode_heavy_branch() Assert.That(decodedTiniest.Keccak, Is.EqualTo(decoded.GetChildHash(11)), "value"); } + [Test] + public void Resolves_full_branch_children_to_their_individual_hashes() + { + if (!System.Runtime.Intrinsics.X86.Avx512F.VL.IsSupported) + { + Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); + } + + const int branchCount = 8; + TrieNode root = new(NodeType.Branch); + TrieNode[] branches = new TrieNode[branchCount]; + + for (int i = 0; i < branchCount; i++) + { + TrieNode branch = branches[i] = new TrieNode(NodeType.Branch); + root.SetChild(i, branch); + for (int childIndex = 0; childIndex < TrieNode.BranchesCount; childIndex++) + { + Hash256 childHash = Keccak.Compute([(byte)i, (byte)childIndex]); + branch.SetChild(childIndex, new TrieNode(NodeType.Unknown, childHash)); + } + } + + TreePath path = TreePath.Empty; + root.ResolveKey(NullTrieNodeResolver.Instance, ref path, canBeParallel: false); + + using (Assert.EnterMultipleScope()) + { + for (int i = 0; i < branchCount; i++) + { + Assert.That(branches[i].FullRlp.Length, Is.EqualTo(532), $"RLP length at branch {i}"); + Assert.That(branches[i].Keccak, Is.EqualTo(Keccak.Compute(branches[i].FullRlp.AsSpan())), $"hash at branch {i}"); + } + + Assert.That(root.Keccak, Is.EqualTo(Keccak.Compute(root.FullRlp.AsSpan())), "root hash"); + } + } + [Test] public void Can_encode_decode_tiny_extension() { diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs index 7ee1327caefc..724162cad2e3 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs @@ -5,7 +5,9 @@ using System.Buffers; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics.X86; using System.Threading; using Nethermind.Core.Buffers; using Nethermind.Core.Cpu; @@ -27,6 +29,9 @@ public partial class TrieNode private class TrieNodeDecoder { + private const int FullBranchRlpLength = 532; + private const int MinHashBatchSize = 2; + [SkipLocalsInit] public static CappedArray EncodeExtension(TrieNode item, ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? bufferPool, bool canBeParallel) { @@ -175,6 +180,54 @@ static bool UseParallel(bool canBeParallel, TrieNode item) } } + private static void HashPreparedBranches(TrieNode item, ushort candidateMask) + { + int firstIndex = BitOperations.TrailingZeroCount(candidateMask); + candidateMask ^= (ushort)(1 << firstIndex); + if (candidateMask == 0) + { + Unsafe.As(item._nodeData[firstIndex]).ResolvePreparedKey(); + return; + } + + HashPreparedBranchPairs(item, firstIndex, candidateMask); + } + + [SkipLocalsInit] + [MethodImpl(MethodImplOptions.NoInlining)] + private static void HashPreparedBranchPairs(TrieNode item, int firstIndex, ushort candidateMask) + { + Span hashes = stackalloc byte[MinHashBatchSize * Hash256.Size]; + + while (true) + { + int secondIndex = BitOperations.TrailingZeroCount(candidateMask); + candidateMask ^= (ushort)(1 << secondIndex); + TrieNode first = Unsafe.As(item._nodeData[firstIndex]); + TrieNode second = Unsafe.As(item._nodeData[secondIndex]); + Span rlp0 = first.FullRlp.AsSpan(); + Span rlp1 = second.FullRlp.AsSpan(); + KeccakHash.ComputeHash532Bytes2Avx512VL(ref rlp0[0], ref rlp1[0], ref hashes[0]); + ValueHash256 firstHash = new(hashes[..Hash256.Size]); + ValueHash256 secondHash = new(hashes[Hash256.Size..]); + first.SetPreparedKey(in firstHash); + second.SetPreparedKey(in secondHash); + + if (candidateMask == 0) + { + return; + } + + firstIndex = BitOperations.TrailingZeroCount(candidateMask); + candidateMask ^= (ushort)(1 << firstIndex); + if (candidateMask == 0) + { + Unsafe.As(item._nodeData[firstIndex]).ResolvePreparedKey(); + return; + } + } + } + private static int GetChildrenRlpLengthForBranch(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool, bool canBeParallel) => // Tail call optimized. item.HasRlp @@ -225,6 +278,7 @@ private static int GetChildrenRlpLengthForBranchNonRlpParallel(ITrieNodeResolver private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool bufferPool, bool canBeParallel) { int totalLength = 0; + ushort candidateMask = 0; for (int i = 0; i < BranchesCount; i++) { object? data = item._nodeData[i]; @@ -240,11 +294,34 @@ private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, r { path.AppendMut(i); TrieNode childNode = Unsafe.As(data); - childNode.ResolveKey(tree, ref path, bufferPool: bufferPool, canBeParallel: canBeParallel); + if (Avx512F.VL.IsSupported && childNode is { IsBranch: true, Keccak: null }) + { + CappedArray rlp = childNode.PrepareRlp(tree, ref path, bufferPool, canBeParallel); + if (rlp.Length == FullBranchRlpLength) + { + candidateMask |= (ushort)(1 << i); + totalLength += Rlp.LengthOfKeccakRlp; + } + else + { + childNode.ResolvePreparedKey(); + totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } + } + else + { + childNode.ResolveKey(tree, ref path, bufferPool: bufferPool, canBeParallel: canBeParallel); + totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } path.TruncateOne(); - totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; } } + + if (candidateMask != 0) + { + HashPreparedBranches(item, candidateMask); + } + return totalLength; } @@ -293,6 +370,7 @@ private static int GetChildrenRlpLengthForBranchRlpParallel(ITrieNodeResolver tr private static int GetChildrenRlpLengthForBranchRlp(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool, bool canBeParallel) { int totalLength = 0; + ushort candidateMask = 0; RlpReader rlpReader = item.RlpReader; item.SeekChild(ref rlpReader, 0); for (int i = 0; i < BranchesCount; i++) @@ -319,15 +397,37 @@ private static int GetChildrenRlpLengthForBranchRlp(ITrieNodeResolver tree, ref path.AppendMut(i); Debug.Assert(data is TrieNode, "Data is not TrieNode"); TrieNode childNode = Unsafe.As(data); - childNode.ResolveKey(tree, ref path, bufferPool: bufferPool, canBeParallel: canBeParallel); + if (Avx512F.VL.IsSupported && childNode is { IsBranch: true, Keccak: null }) + { + CappedArray rlp = childNode.PrepareRlp(tree, ref path, bufferPool, canBeParallel); + if (rlp.Length == FullBranchRlpLength) + { + candidateMask |= (ushort)(1 << i); + totalLength += Rlp.LengthOfKeccakRlp; + } + else + { + childNode.ResolvePreparedKey(); + totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } + } + else + { + childNode.ResolveKey(tree, ref path, bufferPool: bufferPool, canBeParallel: canBeParallel); + totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } path.TruncateOne(); - totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; } rlpReader.SkipItem(); } } + if (candidateMask != 0) + { + HashPreparedBranches(item, candidateMask); + } + return totalLength; } diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index 76cbc758103d..23e8b4689dcf 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -534,6 +534,22 @@ public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, public Hash256? GenerateKey(ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? bufferPool = null, bool canBeParallel = true) + { + bool isRoot = path.Length == 0; + CappedArray rlp = PrepareRlp(tree, ref path, bufferPool, canBeParallel); + + // Descendant nodes with RLP shorter than a hash are embedded in their parent. + if (rlp.Length >= 32 || isRoot) + { + Metrics.IncrementTreeNodeHashCalculations(); + return Nethermind.Core.Crypto.Keccak.Compute(rlp.AsSpan()); + } + + return null; + } + + internal CappedArray PrepareRlp(ITrieNodeResolver tree, ref TreePath path, + ICappedArrayPool? bufferPool, bool canBeParallel) { bool isRoot = path.Length == 0; CappedArray rlp = ReadRlp(); @@ -553,16 +569,23 @@ public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, WriteRlp(rlp = fullRlp); } - /* nodes that are descendants of other nodes are stored inline - * if their serialized length is less than Keccak length - * */ - if (rlp.Length >= 32 || isRoot) + return rlp; + } + + internal void ResolvePreparedKey() + { + CappedArray rlp = ReadRlp(); + if (rlp.Length >= 32) { Metrics.IncrementTreeNodeHashCalculations(); - return Nethermind.Core.Crypto.Keccak.Compute(rlp.AsSpan()); + Keccak = Nethermind.Core.Crypto.Keccak.Compute(rlp.AsSpan()); } + } - return null; + internal void SetPreparedKey(in ValueHash256 hash) + { + Metrics.IncrementTreeNodeHashCalculations(); + Keccak = new Hash256(in hash); } internal CappedArray RlpEncode(ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? bufferPool = null, bool canBeParallel = false) From 86252d28eb2c0d0844d2d3f6a715bc2d61cce79a Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 19 Aug 2026 15:32:41 +0100 Subject: [PATCH 11/12] Fix AVX-512 ZK EVM build and lint --- .../Nethermind.Core/Crypto/KeccakHash.avx512x8.cs | 1 - src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs | 13 +++++++++++++ .../Nethermind.Core/Crypto/KeccakHash.std.cs | 12 ------------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs index 93fafbb5501d..5d9d6d787266 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs index 2759802594d2..d971381f6a5f 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs @@ -18,6 +18,19 @@ public sealed partial class KeccakHash private const int STATE_SIZE = 200; private const int STATE_LANES = STATE_SIZE / sizeof(ulong); private const int HASH_DATA_AREA = 136; + private const int ROUNDS = 24; + + private static readonly ulong[] RoundConstants = + [ + 0x0000000000000001UL, 0x0000000000008082UL, 0x800000000000808aUL, + 0x8000000080008000UL, 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, 0x000000000000008aUL, + 0x0000000000000088UL, 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, 0x8000000000008089UL, + 0x8000000000008003UL, 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, 0x8000000080008081UL, + 0x8000000000008080UL, 0x0000000080000001UL, 0x8000000080008008UL + ]; private byte[] _remainderBuffer = []; private ulong[] _state = []; diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs index a33b1a286e1a..15968b201ad8 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.std.cs @@ -12,20 +12,8 @@ namespace Nethermind.Core.Crypto; public sealed partial class KeccakHash { - private const int ROUNDS = 24; private const int LANE_BITS = 8 * 8; private const int TEMP_BUFF_SIZE = 144; - private static readonly ulong[] RoundConstants = - [ - 0x0000000000000001UL, 0x0000000000008082UL, 0x800000000000808aUL, - 0x8000000080008000UL, 0x000000000000808bUL, 0x0000000080000001UL, - 0x8000000080008081UL, 0x8000000000008009UL, 0x000000000000008aUL, - 0x0000000000000088UL, 0x0000000080008009UL, 0x000000008000000aUL, - 0x000000008000808bUL, 0x800000000000008bUL, 0x8000000000008089UL, - 0x8000000000008003UL, 0x8000000000008002UL, 0x8000000000000080UL, - 0x000000000000800aUL, 0x800000008000000aUL, 0x8000000080008081UL, - 0x8000000000008080UL, 0x0000000080000001UL, 0x8000000080008008UL - ]; // update the state with given number of rounds private static partial void KeccakF(Span st) From 1aa609389d0faf768ae954e59872cdeb3f757cd3 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 19 Aug 2026 16:36:17 +0100 Subject: [PATCH 12/12] Address AVX-512 review hardening Reuse prepared branch RLP values, fail loudly before a fixed-size pair read if their length changes, document the keyed-nonce zero-key invariant, and broaden the dispatch and trie batch coverage. Benchmark sanity check on Ryzen 9 9950X / .NET 10.0.11: the dense eight-eligible-child trie measured 6,489.7 ns batched versus 7,181.7 ns scalar. Three-process medians for the 65,536-entry workload were 150.4 us for 300 parallel account updates and 6.7/11.4/18.2/35.2/116.8/131.0 us for 2/4/8/16/64/300 storage updates, with no observed regression against the PR measurements. --- .../Nethermind.Core.Test/KeccakTests.cs | 7 +-- .../Crypto/KeccakHash.avx512vl.cs | 2 +- .../Crypto/KeccakHash.avx512x8.cs | 6 ++- .../KeyedNonceManager.cs | 3 +- .../Nethermind.Trie.Test/TrieNodeTests.cs | 43 +++++++++++++++---- .../Nethermind.Trie/TrieNode.Decoder.cs | 27 ++++++++---- src/Nethermind/Nethermind.Trie/TrieNode.cs | 5 +++ 7 files changed, 67 insertions(+), 26 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index 24d8b307b7d6..6a8eaa757d8d 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -264,13 +264,8 @@ public void Avx512vl_two_way_532_byte_hash_matches_individual_hashes() [TestCase(20, "50c02dbeee2be79b9595060fe30efbd78f06acedf7a1fe8cb05df7ddd76f2b1b")] [TestCase(32, "d064c972ea7cbd9f1237bbd922fd5f08ca57895c13bc9ea2b91913f7099809a1")] [TestCase(64, "52c1f4616862f9d5011ed6a2a77d89a2102e51ee7db2db045bb5fb267fba98d1")] - public void Avx512VL_common_input_lengths_match_known_hash(int inputLength, string expected) + public void Common_input_lengths_match_known_hash(int inputLength, string expected) { - if (!Avx512F.VL.IsSupported) - { - Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); - } - byte[] input = new byte[inputLength]; byte[] output = new byte[32]; for (int i = 0; i < input.Length; i++) diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs index 83c28329ee37..293c752909cc 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512vl.cs @@ -286,7 +286,7 @@ private static Vector128 LoadPair(ref byte input0, ref byte input1, int o private static void StoreHashPair(ref byte output, int hashIndex, Vector128 a0, Vector128 a1, Vector128 a2, Vector128 a3) { - ref byte destination = ref Unsafe.Add(ref output, hashIndex * Hash256.Size); + ref byte destination = ref Unsafe.Add(ref output, hashIndex * HASH_SIZE); Unsafe.WriteUnaligned(ref destination, a0.GetElement(hashIndex)); Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), a1.GetElement(hashIndex)); Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 16), a2.GetElement(hashIndex)); diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs index 5d9d6d787266..0c310341300d 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.avx512x8.cs @@ -12,11 +12,12 @@ namespace Nethermind.Core.Crypto; public sealed partial class KeccakHash { /// Hashes eight consecutive 64-byte inputs into eight consecutive 32-byte outputs. - /// The caller must ensure that AVX-512F is supported and that both buffers have the required fixed size. + /// The caller must ensure that AVX-512F is supported and provide 512 input bytes and 256 output bytes. [SkipLocalsInit] internal static unsafe void ComputeHash64Bytes8Avx512(ref byte input, ref byte output) { Debug.Assert(Avx512F.IsSupported); + Debug.Assert(Avx2.IsSupported); Vector512 a0; Vector512 a1; @@ -26,6 +27,7 @@ internal static unsafe void ComputeHash64Bytes8Avx512(ref byte input, ref byte o Vector512 a5; Vector512 a6; Vector512 a7; + // Each gather reads one lane from all eight 64-byte inputs; lane 7 reaches the final input byte. fixed (byte* inputPtr = &input) { a0 = GatherLane(inputPtr, 0); @@ -94,7 +96,7 @@ private static unsafe Vector512 GatherLane(byte* input, int lane) private static void StoreHash(ref byte output, int hashIndex, Vector512 a0, Vector512 a1, Vector512 a2, Vector512 a3) { - ref byte destination = ref Unsafe.Add(ref output, hashIndex * 32); + ref byte destination = ref Unsafe.Add(ref output, hashIndex * HASH_SIZE); Unsafe.WriteUnaligned(ref destination, a0.GetElement(hashIndex)); Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 8), a1.GetElement(hashIndex)); Unsafe.WriteUnaligned(ref Unsafe.Add(ref destination, 16), a2.GetElement(hashIndex)); diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs index 59ac4d91d90d..d762e30c3ffc 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/KeyedNonceManager.cs @@ -130,9 +130,10 @@ public static bool IsNonceSetValid(IWorldState state, Address sender, ReadOnlySp return false; } - // Well-formedness above bounds Length to MaxNonceKeys, so the stackalloc below always fits. + // A well-formed multi-key set is bounded and cannot contain key 0, so every key uses a storage slot. if (Avx512F.IsSupported && nonceKeys.Length >= HashBatchSize) { + Debug.Assert(!nonceKeys[0].IsZero, "key 0 cannot appear in a well-formed multi-key set"); Span indices = stackalloc UInt256[Eip8250Constants.MaxNonceKeys]; StorageIndices(sender, nonceKeys, indices); for (int i = 0; i < nonceKeys.Length; i++) diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs index 799bfdb0f24a..6476244f1dc2 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieNodeTests.cs @@ -214,20 +214,28 @@ public void Can_encode_decode_heavy_branch() Assert.That(decodedTiniest.Keccak, Is.EqualTo(decoded.GetChildHash(11)), "value"); } - [Test] - public void Resolves_full_branch_children_to_their_individual_hashes() + [TestCase(0x0001)] + [TestCase(0x0003)] + [TestCase(0x0007)] + [TestCase(0x5555)] + [TestCase(0xffff)] + public void Resolves_full_branch_children_to_their_individual_hashes(int branchMask) { if (!System.Runtime.Intrinsics.X86.Avx512F.VL.IsSupported) { Assert.Ignore("AVX-512VL intrinsics are not supported on this machine."); } - const int branchCount = 8; TrieNode root = new(NodeType.Branch); - TrieNode[] branches = new TrieNode[branchCount]; + TrieNode?[] branches = new TrieNode?[TrieNode.BranchesCount]; - for (int i = 0; i < branchCount; i++) + for (int i = 0; i < TrieNode.BranchesCount; i++) { + if ((branchMask & (1 << i)) == 0) + { + continue; + } + TrieNode branch = branches[i] = new TrieNode(NodeType.Branch); root.SetChild(i, branch); for (int childIndex = 0; childIndex < TrieNode.BranchesCount; childIndex++) @@ -237,17 +245,36 @@ public void Resolves_full_branch_children_to_their_individual_hashes() } } + TrieNode? nonCandidate = null; + if (branchMask != 0xffff) + { + const int nonCandidateIndex = TrieNode.BranchesCount - 1; + nonCandidate = new TrieNode(NodeType.Branch); + nonCandidate.SetChild(0, new TrieNode(NodeType.Unknown, Keccak.Compute([0xff]))); + root.SetChild(nonCandidateIndex, nonCandidate); + } + TreePath path = TreePath.Empty; root.ResolveKey(NullTrieNodeResolver.Instance, ref path, canBeParallel: false); using (Assert.EnterMultipleScope()) { - for (int i = 0; i < branchCount; i++) + for (int i = 0; i < branches.Length; i++) { - Assert.That(branches[i].FullRlp.Length, Is.EqualTo(532), $"RLP length at branch {i}"); - Assert.That(branches[i].Keccak, Is.EqualTo(Keccak.Compute(branches[i].FullRlp.AsSpan())), $"hash at branch {i}"); + TrieNode? branch = branches[i]; + if (branch is not null) + { + Assert.That(branch.FullRlp.Length, Is.EqualTo(532), $"RLP length at branch {i}"); + Assert.That(branch.Keccak, Is.EqualTo(Keccak.Compute(branch.FullRlp.AsSpan())), $"hash at branch {i}"); + } } + if (nonCandidate is not null) + { + Assert.That(nonCandidate.FullRlp.Length, Is.Not.EqualTo(532), "non-candidate RLP length"); + Assert.That(nonCandidate.Keccak, Is.EqualTo(Keccak.Compute(nonCandidate.FullRlp.AsSpan())), + "non-candidate hash"); + } Assert.That(root.Keccak, Is.EqualTo(Keccak.Compute(root.FullRlp.AsSpan())), "root hash"); } } diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs index 724162cad2e3..781013956ad4 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs @@ -30,7 +30,7 @@ public partial class TrieNode private class TrieNodeDecoder { private const int FullBranchRlpLength = 532; - private const int MinHashBatchSize = 2; + private const int HashPairSize = 2; [SkipLocalsInit] public static CappedArray EncodeExtension(TrieNode item, ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? bufferPool, bool canBeParallel) @@ -197,7 +197,7 @@ private static void HashPreparedBranches(TrieNode item, ushort candidateMask) [MethodImpl(MethodImplOptions.NoInlining)] private static void HashPreparedBranchPairs(TrieNode item, int firstIndex, ushort candidateMask) { - Span hashes = stackalloc byte[MinHashBatchSize * Hash256.Size]; + Span hashes = stackalloc byte[HashPairSize * Hash256.Size]; while (true) { @@ -205,8 +205,15 @@ private static void HashPreparedBranchPairs(TrieNode item, int firstIndex, ushor candidateMask ^= (ushort)(1 << secondIndex); TrieNode first = Unsafe.As(item._nodeData[firstIndex]); TrieNode second = Unsafe.As(item._nodeData[secondIndex]); - Span rlp0 = first.FullRlp.AsSpan(); - Span rlp1 = second.FullRlp.AsSpan(); + CappedArray firstRlp = first.FullRlp; + CappedArray secondRlp = second.FullRlp; + if (firstRlp.Length != FullBranchRlpLength || secondRlp.Length != FullBranchRlpLength) + { + ThrowUnexpectedPreparedBranchLength(); + } + + Span rlp0 = firstRlp.AsSpan(); + Span rlp1 = secondRlp.AsSpan(); KeccakHash.ComputeHash532Bytes2Avx512VL(ref rlp0[0], ref rlp1[0], ref hashes[0]); ValueHash256 firstHash = new(hashes[..Hash256.Size]); ValueHash256 secondHash = new(hashes[Hash256.Size..]); @@ -226,6 +233,10 @@ private static void HashPreparedBranchPairs(TrieNode item, int firstIndex, ushor return; } } + + [DoesNotReturn, StackTraceHidden] + static void ThrowUnexpectedPreparedBranchLength() => + throw new TrieException("A prepared full branch changed before batched hashing."); } private static int GetChildrenRlpLengthForBranch(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool, bool canBeParallel) => @@ -304,8 +315,8 @@ private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, r } else { - childNode.ResolvePreparedKey(); - totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + childNode.ResolvePreparedKey(in rlp); + totalLength += childNode.Keccak is null ? rlp.Length : Rlp.LengthOfKeccakRlp; } } else @@ -407,8 +418,8 @@ private static int GetChildrenRlpLengthForBranchRlp(ITrieNodeResolver tree, ref } else { - childNode.ResolvePreparedKey(); - totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + childNode.ResolvePreparedKey(in rlp); + totalLength += childNode.Keccak is null ? rlp.Length : Rlp.LengthOfKeccakRlp; } } else diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index 23e8b4689dcf..12db71186570 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -575,6 +575,11 @@ internal CappedArray PrepareRlp(ITrieNodeResolver tree, ref TreePath path, internal void ResolvePreparedKey() { CappedArray rlp = ReadRlp(); + ResolvePreparedKey(in rlp); + } + + private void ResolvePreparedKey(in CappedArray rlp) + { if (rlp.Length >= 32) { Metrics.IncrementTreeNodeHashCalculations();