Improvements and Contributions to DepotDownloader

.gitattributes

@@ -1,4 +1,4 @@
 *.cs text eol=lf
 *.csproj text eol=lf
 *.config eol=lf
-*.json eol=lf
+*.json eol=lf

.gitignore

@@ -4,7 +4,7 @@
 
 # mstest test results
 TestResults
-
+.history
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
 
@@ -118,4 +118,6 @@ cryptopp/
 
 # misc
 Thumbs.db
+.vscode
+DepotDownloader/Properties
 launchSettings.json

DepotDownloader/ContentDownloader.cs

@@ -68,9 +68,8 @@ static bool CreateDirectories(uint depotId, uint depotVersion, out string instal
                 else
                 {
                     Directory.CreateDirectory(Config.InstallDirectory);
-
-                    installDir = Config.InstallDirectory;
-
+                    installDir = Config.DepotLayout ? Path.Combine(Config.InstallDirectory, depotId.ToString()) : Config.InstallDirectory;
+                    Directory.CreateDirectory(installDir);
                     Directory.CreateDirectory(Path.Combine(installDir, CONFIG_DIR));
                     Directory.CreateDirectory(Path.Combine(installDir, STAGING_DIR));
                 }
@@ -435,7 +434,9 @@ public static async Task DownloadAppAsync(uint appId, List<(uint depotId, ulong
                 else
                 {
                     var contentName = GetAppName(appId);
-                    throw new ContentDownloaderException(string.Format("App {0} ({1}) is not available from this account.", appId, contentName));
+                    Console.WriteLine(string.Format("Error: App {0} ({1}) is not available from this account.", appId, contentName));
+                    // Skip this app and continue processing other apps in the list since we don't have access to download it
+                    return;
                 }
             }
 
@@ -600,8 +601,7 @@ static async Task<DepotDownloadInfo> GetDepotInfo(uint depotId, uint appId, ulon
                 Console.WriteLine("Error: Unable to create install directories!");
                 return null;
             }
-
-            // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload
+            // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id
             var containingAppId = appId;
             var proxyAppId = GetSteam3DepotProxyAppId(depotId, appId);
             if (proxyAppId != INVALID_APP_ID)
@@ -655,57 +655,151 @@ private class DepotDownloadCounter
             public ulong depotBytesUncompressed;
         }
 
+        /// <summary>
+        /// Monitors and reports download progress at regular intervals
+        /// </summary>
+        private sealed class DownloadMonitor : IDisposable
+        {
+            private readonly GlobalDownloadCounter globalDownloadCounter;
+            private readonly int reportIntervalMs;
+            private Thread monitorThread;
+            private volatile bool running = false;
+            private long lastDownloadSize = 0;
+            private readonly AutoResetEvent threadWait = new(false);
+            private readonly object lockObject = new();
+
+            public DownloadMonitor(
+                GlobalDownloadCounter globalDownloadCounter,
+                int reportIntervalMs = 60000)
+            {
+                this.globalDownloadCounter = globalDownloadCounter;
+                this.reportIntervalMs = reportIntervalMs;
+            }
+
+            public void Start()
+            {
+                if (monitorThread != null)
+                    return;
+
+                running = true;
+                monitorThread = new Thread(ReportAlive);
+                monitorThread.Start();
+            }
+
+            public void Stop()
+            {
+                if (!running)
+                    return;
+
+                running = false;
+                threadWait.Set();
+                monitorThread?.Join();
+                monitorThread = null;
+            }
+
+            private void ReportAlive()
+            {
+                try
+                {
+                    while (running)
+                    {
+                        ulong currentDownloadedBytes;
+                        lock (lockObject)
+                        {
+                            currentDownloadedBytes = globalDownloadCounter.totalBytesUncompressed;
+                        }
+
+                        if (currentDownloadedBytes > (ulong)Interlocked.Read(ref lastDownloadSize))
+                        {
+                            Interlocked.Exchange(ref lastDownloadSize, (long)currentDownloadedBytes);
+                            Console.WriteLine("Downloaded : {0}", Util.FormatSize((long)currentDownloadedBytes));
+                        }
+                        threadWait.WaitOne(reportIntervalMs);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine($"Error in download monitor: {ex.Message}");
+                }
+            }
+
+            public void Dispose()
+            {
+                Stop();
+                threadWait.Dispose();
+            }
+        }
         private static async Task DownloadSteam3Async(List<DepotDownloadInfo> depots)
         {
             Ansi.Progress(Ansi.ProgressState.Indeterminate);
 
             await cdnPool.UpdateServerList();
 
             var cts = new CancellationTokenSource();
+
             var downloadCounter = new GlobalDownloadCounter();
+            var downloadMonitor = new DownloadMonitor(downloadCounter);
             var depotsToDownload = new List<DepotFilesData>(depots.Count);
             var allFileNamesAllDepots = new HashSet<string>();
 
-            // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
-            foreach (var depot in depots)
+            try
             {
-                var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter);
-
-                if (depotFileData != null)
+                downloadMonitor.Start();
+                // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup
+                foreach (var depot in depots)
                 {
-                    depotsToDownload.Add(depotFileData);
-                    allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
+                    var depotFileData = await ProcessDepotManifestAndFiles(cts, depot, downloadCounter);
+
+                    if (depotFileData != null)
+                    {
+                        depotsToDownload.Add(depotFileData);
+                        allFileNamesAllDepots.UnionWith(depotFileData.allFileNames);
+                    }
+                    else
+                    {
+                        Console.WriteLine("Error: skipping depot {0} from app {1} with manifest {2}. could not get depot data", depot.DepotId, depot.AppId, depot.ManifestId);
+                        continue;
+                    }
+                    cts.Token.ThrowIfCancellationRequested();
                 }
 
-                cts.Token.ThrowIfCancellationRequested();
-            }
+                if (!Config.DepotLayout)
+                {
 
-            // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path
-            // This is in last-depot-wins order, from Steam or the list of depots supplied by the user
-            if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0)
-            {
-                var claimedFileNames = new HashSet<string>();
+                    if (!string.IsNullOrWhiteSpace(Config.InstallDirectory) && depotsToDownload.Count > 0)
+                    {
+                        var claimedFileNames = new HashSet<String>();
 
-                for (var i = depotsToDownload.Count - 1; i >= 0; i--)
-                {
-                    // For each depot, remove all files from the list that have been claimed by a later depot
-                    depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName));
+                        for (var i = depotsToDownload.Count - 1; i >= 0; i--)
+                        {
+                            // For each depot, remove all files from the list that have been claimed by a later depot
+                            depotsToDownload[i].filteredFiles.RemoveAll(file => claimedFileNames.Contains(file.FileName));
 
-                    claimedFileNames.UnionWith(depotsToDownload[i].allFileNames);
+                            claimedFileNames.UnionWith(depotsToDownload[i].allFileNames);
+                        }
+                    }
                 }
-            }
 
-            foreach (var depotFileData in depotsToDownload)
-            {
-                await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
-            }
 
-            Ansi.Progress(Ansi.ProgressState.Hidden);
+                foreach (var depotFileData in depotsToDownload)
+                {
+                    await DownloadSteam3AsyncDepotFiles(cts, downloadCounter, depotFileData, allFileNamesAllDepots);
+                }
 
-            Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
-                downloadCounter.totalBytesCompressed, downloadCounter.totalBytesUncompressed, depots.Count);
-        }
+                Ansi.Progress(Ansi.ProgressState.Hidden);
 
+                Console.WriteLine("Total downloaded: {0} bytes ({1} bytes uncompressed) from {2} depots",
+                    Util.FormatSize((long)downloadCounter.totalBytesCompressed), Util.FormatSize((long)downloadCounter.totalBytesUncompressed), depots.Count);
+            }
+            catch (IOException ex)
+            {
+                Console.WriteLine("Failed to download steam app: {0}", ex.Message);
+            }
+            finally
+            {
+                downloadMonitor.Stop();
+            }
+        }
         private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(CancellationTokenSource cts, DepotDownloadInfo depot, GlobalDownloadCounter downloadCounter)
         {
             var depotCounter = new DepotDownloadCounter();
@@ -785,6 +879,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
                                 if (manifestRequestCode == 0)
                                 {
                                     cts.Cancel();
+                                    return null;
                                 }
                             }
 
@@ -972,7 +1067,7 @@ await DownloadSteam3AsyncDepotFileChunk(
             DepotConfigStore.Instance.InstalledManifestIDs[depot.DepotId] = depot.ManifestId;
             DepotConfigStore.Save();
 
-            Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, depotCounter.depotBytesCompressed, depotCounter.depotBytesUncompressed);
+            Console.WriteLine("Depot {0} - Downloaded {1} bytes ({2} bytes uncompressed)", depot.DepotId, Util.FormatSize((long)depotCounter.depotBytesCompressed), Util.FormatSize((long)depotCounter.depotBytesUncompressed));
         }
 
         private static void DownloadSteam3AsyncDepotFile(
@@ -1004,6 +1099,20 @@ private static void DownloadSteam3AsyncDepotFile(
             }
 
             List<DepotManifest.ChunkData> neededChunks;
+            // Check if existing file needs updating when leveraging bandwidth 
+            // If outdated , delete to re-download            
+            // Staging and updating blocks can be slower than fresh download on fast connections
+            if (Config.LeverageBandwidth && File.Exists(fileFinalPath))
+            {
+                if ((oldManifestFile == null) || (!oldManifestFile.FileHash.SequenceEqual(file.FileHash)))
+                {
+                    File.Delete(fileFinalPath);
+                    oldProtoManifest = null;
+                    Console.WriteLine("Re-downloading outdated file {0}", file.FileName);
+                }
+
+            }
+
             var fi = new FileInfo(fileFinalPath);
             var fileDidExist = fi.Exists;
             if (!fileDidExist)

DepotDownloader/DepotDownloader.csproj

@@ -11,7 +11,7 @@
     <ApplicationIcon>..\Icon\DepotDownloader.ico</ApplicationIcon>
     <Deterministic>true</Deterministic>
     <TreatWarningsAsErrors Condition="'$(ContinuousIntegrationBuild)' == 'true'">true</TreatWarningsAsErrors>
-    <InvariantGlobalization>true</InvariantGlobalization>
+	<InvariantGlobalization>true</InvariantGlobalization>
   </PropertyGroup>
 
   <ItemGroup>

DepotDownloader/DownloadConfig.cs

@@ -32,5 +32,7 @@ class DownloadConfig
 
         public bool UseQrCode { get; set; }
         public bool SkipAppConfirmation { get; set; }
+        public bool DepotLayout { get; set; }
+        public bool LeverageBandwidth { get; set; }
     }
 }

DepotDownloader/Program.cs

@@ -135,6 +135,8 @@ static async Task<int> Main(string[] args)
             }
 
             ContentDownloader.Config.InstallDirectory = GetParameter<string>(args, "-dir");
+            ContentDownloader.Config.DepotLayout = HasParameter(args, "-depot-layout");
+            ContentDownloader.Config.LeverageBandwidth = HasParameter(args, "-leverage-bandwidth");
 
             ContentDownloader.Config.VerifyAll = HasParameter(args, "-verify-all") || HasParameter(args, "-verify_all") || HasParameter(args, "-validate");
 
@@ -520,16 +522,19 @@ static void PrintUsage()
             Console.WriteLine("  -no-mobile               - prefer entering a 2FA code instead of prompting to accept in the Steam mobile app");
             Console.WriteLine();
             Console.WriteLine("  -dir <installdir>        - the directory in which to place downloaded files.");
-            Console.WriteLine("  -filelist <file.txt>     - the name of a local file that contains a list of files to download (from the manifest).");
-            Console.WriteLine("                             prefix file path with `regex:` if you want to match with regex. each file path should be on their own line.");
+            Console.WriteLine("  -filelist <file.txt>     - a list of files to download (from the manifest). Prefix file path with 'regex:' if you want to match with regex.");
+            Console.WriteLine("  -validate                - Include checksum verification of files already downloaded");
             Console.WriteLine();
-            Console.WriteLine("  -validate                - include checksum verification of files already downloaded");
             Console.WriteLine("  -manifest-only           - downloads a human readable manifest for any depots that would be downloaded.");
             Console.WriteLine("  -cellid <#>              - the overridden CellID of the content server to download from.");
+            Console.WriteLine("  -max-servers <#>         - maximum number of content servers to use. (default: 20).");
             Console.WriteLine("  -max-downloads <#>       - maximum number of chunks to download concurrently. (default: 8).");
             Console.WriteLine("  -loginid <#>             - a unique 32-bit integer Steam LogonID in decimal, required if running multiple instances of DepotDownloader concurrently.");
             Console.WriteLine("  -use-lancache            - forces downloads over the local network via a Lancache instance.");
             Console.WriteLine();
+            Console.WriteLine("  -depot-layout            - store each depot's files in separate folder named by depot ID.This can be useful for game development.");
+            Console.WriteLine("  -leverage-bandwidth      - prefer network bandwidth over CPU usage by re-downloading changed files instead of patching them");
+            Console.WriteLine();
             Console.WriteLine("  -debug                   - enable verbose debug logging.");
             Console.WriteLine("  -V or --version          - print version and runtime.");
         }

DepotDownloader/Steam3Session.cs

@@ -242,19 +242,20 @@ public async Task RequestDepotKey(uint depotId, uint appid = 0)
             if (DepotKeys.ContainsKey(depotId) || bAborted)
                 return;
 
-            var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid);
-
-            Console.WriteLine("Got depot key for {0} result: {1}", depotKey.DepotID, depotKey.Result);
-
-            if (depotKey.Result != EResult.OK)
+            await Util.ExecuteWithRetry(async () =>
             {
-                return;
-            }
+                var depotKey = await steamApps.GetDepotDecryptionKey(depotId, appid);
+                Console.WriteLine($"Got depot key for {depotKey.DepotID} result: {depotKey.Result}");
 
-            DepotKeys[depotKey.DepotID] = depotKey.DepotKey;
-        }
+                if (depotKey.Result != EResult.OK)
+                {
+                    return;
+                }
 
+                DepotKeys[depotKey.DepotID] = depotKey.DepotKey;
+            }, 5);
 
+        }
         public async Task<ulong> GetDepotManifestRequestCodeAsync(uint depotId, uint appId, ulong manifestId, string branch)
         {
             if (bAborted)