diff --git a/.github/workflows/nethermind-tests.yml b/.github/workflows/nethermind-tests.yml index 3648164d0326..872cd7701754 100644 --- a/.github/workflows/nethermind-tests.yml +++ b/.github/workflows/nethermind-tests.yml @@ -74,6 +74,7 @@ jobs: - Nethermind.HealthChecks.Test - Nethermind.History.Test - Nethermind.Hive.Test + - Nethermind.Init.Snapshot.Test - Nethermind.JsonRpc.Test - Nethermind.JsonRpc.TraceStore.Test - Nethermind.KeyStore.Test diff --git a/src/Nethermind/Nethermind.Init.Snapshot.Test/InitDatabaseSnapshotTests.cs b/src/Nethermind/Nethermind.Init.Snapshot.Test/InitDatabaseSnapshotTests.cs new file mode 100644 index 000000000000..ed01534eece1 --- /dev/null +++ b/src/Nethermind/Nethermind.Init.Snapshot.Test/InitDatabaseSnapshotTests.cs @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Formats.Tar; +using System.IO; +using System.IO.Abstractions; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Api; +using Nethermind.Core.Test.IO; +using Nethermind.Logging; +using NSubstitute; +using NUnit.Framework; +using Testably.Abstractions; + +namespace Nethermind.Init.Snapshot.Test; + +public class InitDatabaseSnapshotTests +{ + private const int SnapshotPayloadSize = 100_000; + + private TempPath _tempDir = null!; + private string _dbPath = null!; + private string _snapshotPath = null!; + private SnapshotConfig _snapshotConfig = null!; + private INethermindApi _api = null!; + + [SetUp] + public void SetUp() + { + _tempDir = TempPath.GetTempDirectory(); + string snapshotDirectory = Path.Combine(_tempDir.Path, "snapshot"); + Directory.CreateDirectory(snapshotDirectory); + _dbPath = Path.Combine(_tempDir.Path, "db"); + _snapshotConfig = new SnapshotConfig + { + Enabled = true, + DownloadUrl = "http://127.0.0.1:1/snapshot.tar", + SnapshotDirectory = snapshotDirectory, + SnapshotFileName = "snapshot.tar", + StripComponents = 1 + }; + _snapshotPath = Path.Combine(snapshotDirectory, _snapshotConfig.SnapshotFileName); + + _api = Substitute.For(); + _api.Config().Returns(_snapshotConfig); + _api.Config().Returns(new InitConfig { BaseDbPath = _dbPath }); + _api.LogManager.Returns(LimboLogs.Instance); + _api.FileSystem.Returns(new RealFileSystem()); + } + + [TearDown] + public void TearDown() => _tempDir.Dispose(); + + [Test] + public async Task Execute_FreeSpaceCoversExtractionButNotLegacyMultiplier_ExtractsSnapshot() + { + long snapshotSize = WriteSnapshotTar(); + AdvanceCheckpoint(SnapshotStage.Verified); + long freeSpace = snapshotSize * 2; + Assert.That(freeSpace, Is.GreaterThanOrEqualTo(InitDatabaseSnapshot.GetRequiredSpaceForExtraction(snapshotSize)), + "precondition: free space must cover the extraction estimate"); + Assert.That(freeSpace, Is.LessThan((long)(snapshotSize * 2.5)), + "precondition: free space must be below the legacy 2.5x requirement to prove the regression is fixed"); + InitDatabaseSnapshot step = new(_api, DrivesWithFreeSpace(freeSpace)); + + await step.Execute(CancellationToken.None); + + Assert.That(File.Exists(Path.Combine(_dbPath, "state.bin")), Is.True, + "the snapshot content should be extracted into the database directory"); + Assert.That(File.Exists(_snapshotPath), Is.False, + "the snapshot archive should be deleted after a successful extraction"); + } + + [Test] + public void Execute_FreeSpaceBelowExtractionEstimate_ThrowsIOException() + { + long snapshotSize = WriteSnapshotTar(); + AdvanceCheckpoint(SnapshotStage.Verified); + InitDatabaseSnapshot step = new(_api, DrivesWithFreeSpace(snapshotSize)); + + IOException exception = Assert.ThrowsAsync(() => step.Execute(CancellationToken.None))!; + + Assert.That(exception.Message, Does.Contain("Insufficient disk space"), + "the extraction should be rejected when free space is below the extraction estimate"); + Assert.That(Directory.Exists(_dbPath), Is.False, + "nothing should be extracted when free space is insufficient"); + } + + [Test] + public void Execute_FreeSpaceBelowDownloadRequirement_ThrowsIOExceptionBeforeDownloading() + { + using SnapshotServer server = SnapshotServer.Start(contentLength: 1_000_000); + _snapshotConfig.DownloadUrl = server.Url; + InitDatabaseSnapshot step = new(_api, DrivesWithFreeSpace(1_000_000)); + + IOException exception = Assert.ThrowsAsync(() => step.Execute(CancellationToken.None))!; + + Assert.That(exception.Message, Does.Contain("Insufficient disk space"), + "the download should be rejected when free space cannot fit the snapshot and its extraction"); + Assert.That(File.Exists(_snapshotPath), Is.False, + "no bytes should be downloaded when free space is insufficient"); + } + + [Test] + public async Task Execute_ServerDoesNotReportSize_SkipsPreDownloadCheckAndCompletes() + { + byte[] tarBytes = BuildSnapshotTar(); + using SnapshotServer server = SnapshotServer.Start(payload: tarBytes); + _snapshotConfig.DownloadUrl = server.Url; + InitDatabaseSnapshot step = new(_api, DrivesWithFreeSpace(tarBytes.Length * 2L)); + + await step.Execute(CancellationToken.None); + + Assert.That(File.Exists(Path.Combine(_dbPath, "state.bin")), Is.True, + "the download and extraction should proceed when the server reports no snapshot size"); + } + + [Test] + public async Task Execute_SizeProbeFailsTransiently_SkipsPreDownloadCheckAndCompletes() + { + byte[] tarBytes = BuildSnapshotTar(); + using SnapshotServer server = SnapshotServer.Start(contentLength: tarBytes.Length, payload: tarBytes, failFirstRequests: 1); + _snapshotConfig.DownloadUrl = server.Url; + InitDatabaseSnapshot step = new(_api, DrivesWithFreeSpace(tarBytes.Length * 2L)); + + await step.Execute(CancellationToken.None); + + Assert.That(File.Exists(Path.Combine(_dbPath, "state.bin")), Is.True, + "the download and extraction should proceed when the size probe fails transiently"); + } + + [TestCase(1_000, 0, 2_500, TestName = "FreshDownload")] + [TestCase(1_000, 400, 2_100, TestName = "ResumedDownload")] + public void GetRequiredSpaceForDownload_ForGivenSizes_AddsRemainingBytesToExtractionEstimate( + long totalSize, long existingSize, long expected) => + Assert.That(InitDatabaseSnapshot.GetRequiredSpaceForDownload(totalSize, existingSize), Is.EqualTo(expected), + "the pre-download requirement should be the remaining bytes plus the extraction estimate of the full snapshot"); + + private long WriteSnapshotTar() + { + byte[] tarBytes = BuildSnapshotTar(); + File.WriteAllBytes(_snapshotPath, tarBytes); + return tarBytes.Length; + } + + private static byte[] BuildSnapshotTar() + { + using MemoryStream tarStream = new(); + using (TarWriter tarWriter = new(tarStream, leaveOpen: true)) + { + tarWriter.WriteEntry(new PaxTarEntry(TarEntryType.Directory, "data")); + PaxTarEntry fileEntry = new(TarEntryType.RegularFile, "data/state.bin") + { + DataStream = new MemoryStream(new byte[SnapshotPayloadSize]) + }; + tarWriter.WriteEntry(fileEntry); + } + + return tarStream.ToArray(); + } + + private void AdvanceCheckpoint(SnapshotStage stage) => + new SnapshotCheckpoint(_snapshotConfig, LimboLogs.Instance).Advance(stage); + + private static IDriveInfo[] DrivesWithFreeSpace(long freeSpace) + { + IDriveInfo drive = Substitute.For(); + drive.AvailableFreeSpace.Returns(freeSpace); + drive.RootDirectory.FullName.Returns("/"); + return [drive]; + } + + private sealed class SnapshotServer : IDisposable + { + private readonly HttpListener _listener; + + public string Url { get; } + + private SnapshotServer(HttpListener listener, string url) + { + _listener = listener; + Url = url; + } + + public static SnapshotServer Start(long? contentLength = null, byte[] payload = null, int failFirstRequests = 0) + { + (HttpListener listener, int port) = StartListener(); + int requestIndex = 0; + + _ = Task.Run(async () => + { + while (listener.IsListening) + { + HttpListenerContext context; + try + { + context = await listener.GetContextAsync(); + } + catch (Exception) + { + return; + } + + try + { + if (requestIndex++ < failFirstRequests) + { + context.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable; + context.Response.Close(); + continue; + } + + byte[] body = payload ?? new byte[1024]; + if (contentLength is not null) + context.Response.ContentLength64 = contentLength.Value; + else + context.Response.SendChunked = true; + await context.Response.OutputStream.WriteAsync(body); + context.Response.OutputStream.Close(); + } + catch (Exception) + { + } + } + }); + + return new SnapshotServer(listener, $"http://127.0.0.1:{port}/snapshot.tar"); + } + + private static (HttpListener Listener, int Port) StartListener() + { + for (int attempt = 0; ; attempt++) + { + TcpListener portProbe = new(IPAddress.Loopback, 0); + portProbe.Start(); + int port = ((IPEndPoint)portProbe.LocalEndpoint).Port; + portProbe.Stop(); + + HttpListener listener = new(); + listener.Prefixes.Add($"http://127.0.0.1:{port}/"); + try + { + listener.Start(); + return (listener, port); + } + catch (HttpListenerException) when (attempt < 5) + { + listener.Close(); + } + } + } + + public void Dispose() + { + _listener.Stop(); + _listener.Close(); + } + } +} diff --git a/src/Nethermind/Nethermind.Init.Snapshot.Test/Nethermind.Init.Snapshot.Test.csproj b/src/Nethermind/Nethermind.Init.Snapshot.Test/Nethermind.Init.Snapshot.Test.csproj new file mode 100644 index 000000000000..41845560d254 --- /dev/null +++ b/src/Nethermind/Nethermind.Init.Snapshot.Test/Nethermind.Init.Snapshot.Test.csproj @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs index 316862eb0699..7a30584b1cba 100644 --- a/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs +++ b/src/Nethermind/Nethermind.Init.Snapshot/InitDatabaseSnapshot.cs @@ -26,6 +26,7 @@ public class InitDatabaseSnapshot( INethermindApi api, [KeyFilter(nameof(IInitConfig.BaseDbPath))] IDriveInfo[] drives) : IStep { + private const double ExtractionSpaceMultiplier = 1.5; private const int ExtractionRestartDelaySeconds = 5; private const int InitialRetryDelaySeconds = 5; private const int MaxRetryDelaySeconds = 300; @@ -109,6 +110,8 @@ private async Task DownloadWithRetryAsync( if (checkpoint.Read() >= SnapshotStage.Downloaded) return; + await CheckDiskSpaceBeforeDownloadAsync(downloader, url, destinationPath, cancellationToken).ConfigureAwait(false); + TimeSpan retryDelay = TimeSpan.FromSeconds(InitialRetryDelaySeconds); long lastSize = GetFileSize(destinationPath); @@ -191,30 +194,88 @@ private async Task ExtractAsync( if (checkpoint.Read() >= SnapshotStage.Extracted) return; - CheckDiskSpace(snapshotPath); + CheckDiskSpace(GetRequiredSpaceForExtraction(GetFileSize(snapshotPath)), "extract"); SnapshotExtractor extractor = new(api.LogManager); await extractor.ExtractAsync(snapshotPath, dbPath, stripComponents, cancellationToken).ConfigureAwait(false); checkpoint.Advance(SnapshotStage.Extracted); } - private void CheckDiskSpace(string snapshotPath) + private async Task CheckDiskSpaceBeforeDownloadAsync( + SnapshotDownloader downloader, string url, string destinationPath, CancellationToken cancellationToken) + { + long existingSize = GetFileSize(destinationPath); + long? totalSize; + try + { + totalSize = await downloader.GetTotalSizeAsync(url, existingSize, cancellationToken).ConfigureAwait(false); + } + catch (Exception e) when (e is IOException or HttpRequestException + || (e is OperationCanceledException && !cancellationToken.IsCancellationRequested)) + { + if (_logger.IsWarn) + _logger.Warn($"Could not determine the snapshot size upfront. Skipping the pre-download disk space check. Error: {e.Message}"); + return; + } + + if (totalSize is null) + { + if (_logger.IsWarn) + _logger.Warn("The server did not report the snapshot size. Skipping the pre-download disk space check."); + return; + } + + CheckDiskSpaceBeforeDownload(totalSize.Value, existingSize, Path.GetDirectoryName(destinationPath)!); + } + + private void CheckDiskSpaceBeforeDownload(long totalSize, long existingSize, string snapshotDirectory) { - if (drives.Length == 0) + IDriveInfo[] snapshotDrives = api.FileSystem.GetDriveInfos(snapshotDirectory); + if (snapshotDrives.Length == 0) + { + CheckDiskSpace(GetRequiredSpaceForDownload(totalSize, existingSize), "download and extract"); return; + } - long snapshotSize = api.FileSystem.FileInfo.New(snapshotPath).Length; - long required = (long)(snapshotSize * 2.5); + long remainingDownload = totalSize - existingSize; + long extraction = GetRequiredSpaceForExtraction(totalSize); foreach (IDriveInfo drive in drives) { - if (drive.AvailableFreeSpace < required) - throw new IOException( - $"Insufficient disk space on '{drive.RootDirectory.FullName}' to extract snapshot: " + - $"need at least {required} bytes, {drive.AvailableFreeSpace} available."); + bool holdsArchive = snapshotDrives.Any(snapshotDrive => IsSameDrive(snapshotDrive, drive)); + CheckDiskSpace(drive, holdsArchive ? remainingDownload + extraction : extraction, "download and extract"); + } + + foreach (IDriveInfo snapshotDrive in snapshotDrives) + { + if (!drives.Any(drive => IsSameDrive(drive, snapshotDrive))) + CheckDiskSpace(snapshotDrive, remainingDownload, "download"); } } + private static bool IsSameDrive(IDriveInfo first, IDriveInfo second) => + string.Equals(first.RootDirectory.FullName, second.RootDirectory.FullName, StringComparison.Ordinal); + + internal static long GetRequiredSpaceForDownload(long totalSize, long existingSize) => + totalSize - existingSize + GetRequiredSpaceForExtraction(totalSize); + + internal static long GetRequiredSpaceForExtraction(long snapshotSize) => + (long)(snapshotSize * ExtractionSpaceMultiplier); + + private void CheckDiskSpace(long required, string operation) + { + foreach (IDriveInfo drive in drives) + CheckDiskSpace(drive, required, operation); + } + + private static void CheckDiskSpace(IDriveInfo drive, long required, string operation) + { + if (drive.AvailableFreeSpace < required) + throw new IOException( + $"Insufficient disk space on '{drive.RootDirectory.FullName}' to {operation} the snapshot: " + + $"need at least {required} bytes, {drive.AvailableFreeSpace} available."); + } + private async Task ComputeChecksumAsync(string filePath, CancellationToken cancellationToken) { long fileSize = new FileInfo(filePath).Length; diff --git a/src/Nethermind/Nethermind.Init.Snapshot/Nethermind.Init.Snapshot.csproj b/src/Nethermind/Nethermind.Init.Snapshot/Nethermind.Init.Snapshot.csproj index e9f2b426edf8..7e06814b5747 100644 --- a/src/Nethermind/Nethermind.Init.Snapshot/Nethermind.Init.Snapshot.csproj +++ b/src/Nethermind/Nethermind.Init.Snapshot/Nethermind.Init.Snapshot.csproj @@ -12,6 +12,10 @@ + + + <_Parameter1>Nethermind.Init.Snapshot.Test + diff --git a/src/Nethermind/Nethermind.Init.Snapshot/SnapshotDownloader.cs b/src/Nethermind/Nethermind.Init.Snapshot/SnapshotDownloader.cs index 535c4f728b6d..36b100e9801b 100644 --- a/src/Nethermind/Nethermind.Init.Snapshot/SnapshotDownloader.cs +++ b/src/Nethermind/Nethermind.Init.Snapshot/SnapshotDownloader.cs @@ -83,6 +83,16 @@ public async Task DownloadAsync(string url, string destinationPath, Cancellation _logger.Info($"Snapshot downloaded to {destinationPath}."); } + public async Task GetTotalSizeAsync(string url, long existingSize, CancellationToken cancellationToken) + { + using HttpResponseMessage response = await SendWithRangeAsync(_httpClient, url, existingSize, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.RequestedRangeNotSatisfiable) + return existingSize; + + return ResolveCopyStrategy(response.StatusCode, existingSize, response.Content.Headers.ContentLength).totalSize; + } + public void Dispose() => _httpClient.Dispose(); private static (FileMode fileMode, long bytesToSkip, long? totalSize) ResolveCopyStrategy( diff --git a/src/Nethermind/Nethermind.slnx b/src/Nethermind/Nethermind.slnx index c7884b9e584d..fe8f6044f5cb 100644 --- a/src/Nethermind/Nethermind.slnx +++ b/src/Nethermind/Nethermind.slnx @@ -76,6 +76,7 @@ +