Add exponential backoff to downloads

DepotDownloader/Backoff.cs

@@ -0,0 +1,44 @@
+// This file is subject to the terms and conditions defined
+// in file 'LICENSE', which is part of this source code package.
+
+using System;
+using System.Threading.Tasks;
+
+namespace DepotDownloader
+{
+    class Backoff
+    {
+        public readonly double ScalingFactor;
+        public readonly double MinDelayS;
+        public readonly double MaxDelayS;
+        public int Attempts = 0;
+
+        public Backoff(double MinDelayS = 1.0, double MaxDelayS = 120.0, double ScalingFactor = 1.3)
+        {
+            this.MinDelayS = MinDelayS;
+            this.MaxDelayS = MaxDelayS;
+            this.ScalingFactor = ScalingFactor;
+        }
+
+        public void RecordAttempt()
+        {
+            this.Attempts += 1;
+        }
+
+        public Task Sleep()
+        {
+            if (Attempts == 0)
+            {
+                return Task.Delay(0);
+            }
+            var sleepTimeS = MinDelayS * Math.Pow(ScalingFactor, Attempts - 1);
+            if (sleepTimeS > MaxDelayS)
+            {
+                sleepTimeS = MaxDelayS;
+            }
+            var sleepTimeMS = (int)Math.Round(sleepTimeS * 1000.0);
+            Console.WriteLine($"Waiting {sleepTimeS:F2} seconds");
+            return Task.Delay(sleepTimeMS);
+        }
+    }
+}

DepotDownloader/ContentDownloader.cs

@@ -794,10 +794,14 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
                     ulong manifestRequestCode = 0;
                     var manifestRequestCodeExpiration = DateTime.MinValue;
 
+                    var backoff = new Backoff();
+
                     do
                     {
                         cts.Token.ThrowIfCancellationRequested();
 
+                        backoff.RecordAttempt();
+
                         Server connection = null;
 
                         try
@@ -850,7 +854,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
                         }
                         catch (TaskCanceledException)
                         {
-                            Console.WriteLine("Connection timeout downloading depot manifest {0} {1}. Retrying.", depot.DepotId, depot.ManifestId);
+                            Console.WriteLine("Connection timeout downloading depot manifest {0} {1}", depot.DepotId, depot.ManifestId);
                         }
                         catch (SteamKitWebRequestException e)
                         {
@@ -889,6 +893,9 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
                             cdnPool.ReturnBrokenConnection(connection);
                             Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
                         }
+
+                        Console.WriteLine("Retrying.");
+                        await backoff.Sleep();
                     } while (newManifest == null);
 
                     if (newManifest == null)
@@ -1244,10 +1251,13 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
 
             try
             {
+                var backoff = new Backoff();
                 do
                 {
                     cts.Token.ThrowIfCancellationRequested();
 
+                    backoff.RecordAttempt();
+
                     Server connection = null;
 
                     try
@@ -1313,6 +1323,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
                         cdnPool.ReturnBrokenConnection(connection);
                         Console.WriteLine("Encountered unexpected error downloading chunk {0}: {1}", chunkID, e.Message);
                     }
+
+                    Console.WriteLine("Retrying.");
+                    await backoff.Sleep();
                 } while (written == 0);
 
                 if (written == 0)