From 069ba2b6b3bd503bcedf1318b677fd14d89089c7 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 16:43:09 -0500 Subject: [PATCH 01/26] Replace usage of ResumableDownloader with NonReusableDownloader for files under an arbitrary size NonReusableDownloader leverages the Downloader library for executing downloads. This presents some performance overhead when downloading large number files, as is often done when downloading larger modlists. Moved these Downloader classes to the Downloader folder in solution to better align with solution structure. Added service extension for DI Cleaned up impacted files Removed unusued method from ResumableDownloader --- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.CLI/Program.cs | 7 +- .../PerformanceSettings.cs | 2 + Wabbajack.DTOs/Archive.cs | 13 +- .../DownloadClientFactory.cs | 34 ++++ .../DownloaderService.cs | 36 ++++ .../NonResumableDownloadClient.cs | 35 ++++ .../ResumableDownloadClient.cs | 45 ++--- .../ServiceExtensions.cs | 16 ++ .../Wabbajack.Downloader.Services.csproj | 25 +++ .../IDownloadClient.cs | 10 ++ Wabbajack.Installer/AInstaller.cs | 40 +++-- Wabbajack.Launcher/Program.cs | 8 +- .../ServiceExtensions.cs | 12 -- .../SingleThreadedDownloader.cs | 162 ------------------ .../Wabbajack.Networking.Http.csproj | 1 + .../ServiceExtensions.cs | 6 +- .../Wabbajack.Services.OSIntegrated.csproj | 2 + Wabbajack.sln | 9 +- 19 files changed, 228 insertions(+), 237 deletions(-) create mode 100644 Wabbajack.Downloader.Clients/DownloadClientFactory.cs create mode 100644 Wabbajack.Downloader.Clients/DownloaderService.cs create mode 100644 Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs rename Wabbajack.Networking.Http/ResumableDownloader.cs => Wabbajack.Downloader.Clients/ResumableDownloadClient.cs (80%) create mode 100644 Wabbajack.Downloader.Clients/ServiceExtensions.cs create mode 100644 Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj create mode 100644 Wabbajack.Downloaders.Interfaces/IDownloadClient.cs delete mode 100644 Wabbajack.Networking.Http/ServiceExtensions.cs delete mode 100644 Wabbajack.Networking.Http/SingleThreadedDownloader.cs diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 5c0aa72be..6043cb1de 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - WinExe + Exe net9.0-windows true x64 diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 94208892b..370037726 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -1,7 +1,6 @@ using System; using System.CommandLine; using System.CommandLine.IO; -using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -11,14 +10,13 @@ using NLog.Targets; using Octokit; using Wabbajack.DTOs.Interventions; -using Wabbajack.Networking.Http; -using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths.IO; using Wabbajack.Server.Lib; using Wabbajack.Services.OSIntegrated; using Wabbajack.VFS; using Client = Wabbajack.Networking.GitHub.Client; using Wabbajack.CLI.Builder; +using Wabbajack.Downloader.Clients; namespace Wabbajack.CLI; @@ -31,8 +29,7 @@ private static async Task Main(string[] args) .ConfigureServices((host, services) => { services.AddSingleton(new JsonSerializerOptions()); - services.AddSingleton(); - services.AddSingleton(); + services.AddDownloaderService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs index 93dff2406..64dd6d8aa 100644 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ b/Wabbajack.Configuration/PerformanceSettings.cs @@ -3,4 +3,6 @@ public class PerformanceSettings { public int MaximumMemoryPerDownloadThreadMb { get; set; } + + public long MinimumFileSizeForResumableDownload { get; set; } = (long)1024 * 1024 * 500; // 500MB } \ No newline at end of file diff --git a/Wabbajack.DTOs/Archive.cs b/Wabbajack.DTOs/Archive.cs index 4cf9906d7..b6fe20ec9 100644 --- a/Wabbajack.DTOs/Archive.cs +++ b/Wabbajack.DTOs/Archive.cs @@ -1,13 +1,24 @@ +using System; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; namespace Wabbajack.DTOs; -public class Archive +public class Archive : IComparable { public Hash Hash { get; set; } public string Meta { get; set; } = ""; public string Name { get; set; } public long Size { get; set; } public IDownloadState State { get; set; } + + public int CompareTo(object obj) + { + if (obj == null) return 1; + Archive otherArchive = obj as Archive; + if (otherArchive != null) + return this.Size.CompareTo(otherArchive.Size); + else + throw new ArgumentException("Object is not an Archive"); + } } \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs new file mode 100644 index 000000000..c85c12333 --- /dev/null +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Configuration; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.Paths; +using Wabbajack.RateLimiter; + +namespace Wabbajack.Downloader.Clients; + +public interface IDownloadClientFactory +{ + public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job); +} + +public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory +{ + private readonly ILogger _nonResuableDownloaderLogger = _loggerFactory.CreateLogger(); + private readonly ILogger _resumableDownloaderLogger = _loggerFactory.CreateLogger(); + + private NonResumableDownloadClient? _nonReusableDownloader = default; + + public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) + { + if (job.Size >= _performanceSettings.MinimumFileSizeForResumableDownload) + { + return new ResumableDownloadClient(msg, outputPath, job, _performanceSettings, _resumableDownloaderLogger); + } + else + { + _nonReusableDownloader ??= new NonResumableDownloadClient(msg, outputPath, _nonResuableDownloaderLogger, _httpClientFactory); + + return new NonResumableDownloadClient(msg, outputPath, _nonResuableDownloaderLogger, _httpClientFactory); + } + } +} diff --git a/Wabbajack.Downloader.Clients/DownloaderService.cs b/Wabbajack.Downloader.Clients/DownloaderService.cs new file mode 100644 index 000000000..a4ce17368 --- /dev/null +++ b/Wabbajack.Downloader.Clients/DownloaderService.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Paths; +using Wabbajack.RateLimiter; + +namespace Wabbajack.Downloader.Clients; + +public class DownloaderService(ILogger _logger, IDownloadClientFactory _httpDownloaderFactory) : IHttpDownloader +{ + public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, + CancellationToken token) + { + Exception downloadError = null!; + + var downloader = _httpDownloaderFactory.GetDownloader(message, outputPath, job); + + for (var i = 0; i < 3; i++) + { + try + { + return await downloader.Download(token, 3); + } + catch (Exception ex) + { + downloadError = ex; + _logger.LogDebug("Download for '{name}' failed. Retrying...", outputPath.FileName.ToString()); + } + } + + _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); + return new Hash(); + + + } +} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs new file mode 100644 index 000000000..00d7ea74e --- /dev/null +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Downloader.Clients; + +internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, ILogger _logger, IHttpClientFactory _httpClientFactory) : IDownloadClient +{ + public async Task Download(CancellationToken token, int retry = 3) + { + try + { + var httpClient = _httpClientFactory.CreateClient("SmallFilesClient"); + var response = await httpClient.GetStreamAsync(_msg.RequestUri!.ToString()); + await using var fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await response.CopyToAsync(fileStream, token); + fileStream.Close(); + await using var file = _outputPath.Open(FileMode.Open); + return await file.Hash(token); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download '{name}' after 3 tries.", _outputPath.FileName.ToString()); + + if (retry <= 3) + { + return await Download(token, retry--); + } + + return new Hash(); + } + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/ResumableDownloader.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs similarity index 80% rename from Wabbajack.Networking.Http/ResumableDownloader.cs rename to Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index 078de77d4..30e1c1da7 100644 --- a/Wabbajack.Networking.Http/ResumableDownloader.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -1,10 +1,5 @@ -using System; -using System.ComponentModel; -using System.IO; -using System.Net.Http; +using System.ComponentModel; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Configuration; @@ -12,32 +7,18 @@ using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; +using Wabbajack.Networking.Http; +using Wabbajack.Downloaders.Interfaces; -namespace Wabbajack.Networking.Http; +namespace Wabbajack.Downloader.Clients; -internal class ResumableDownloader +internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient { - private readonly IJob _job; - private readonly HttpRequestMessage _msg; - private readonly AbsolutePath _outputPath; - private readonly AbsolutePath _packagePath; - private readonly PerformanceSettings _performanceSettings; - private readonly ILogger _logger; private CancellationToken _token; private Exception? _error; + private AbsolutePath _packagePath = _outputPath.WithExtension(Extension.FromPath(".download_package")); - - public ResumableDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job, PerformanceSettings performanceSettings, ILogger logger) - { - _job = job; - _msg = msg; - _outputPath = outputPath; - _packagePath = outputPath.WithExtension(Extension.FromPath(".download_package")); - _performanceSettings = performanceSettings; - _logger = logger; - } - - public async Task Download(CancellationToken token) + public async Task Download(CancellationToken token, int retry = 0) { _token = token; @@ -80,17 +61,17 @@ public async Task Download(CancellationToken token) } else { - _logger.LogError(_error,"Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); + _logger.LogError(_error, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); } throw _error; - } - if (downloader.Status == DownloadStatus.Completed) - { - DeletePackage(); + if (downloader.Status == DownloadStatus.Completed) + { + DeletePackage(); + } } - + if (!_outputPath.FileExists()) { return new Hash(); diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs new file mode 100644 index 000000000..5ee53cfad --- /dev/null +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.DependencyInjection; +using Wabbajack.Configuration; +using Wabbajack.Networking.Http.Interfaces; + +namespace Wabbajack.Downloader.Clients; + +public static class ServiceExtensions +{ + public static void AddDownloaderService(this IServiceCollection services) + { + services.AddHttpClient("SmallFilesClient").ConfigureHttpClient(c => c.Timeout = TimeSpan.FromMinutes(5)); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } +} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj b/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj new file mode 100644 index 000000000..56b94732b --- /dev/null +++ b/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj @@ -0,0 +1,25 @@ + + + + net9.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs new file mode 100644 index 000000000..445fe05fa --- /dev/null +++ b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs @@ -0,0 +1,10 @@ +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.Hashing.xxHash64; + +namespace Wabbajack.Downloaders.Interfaces; + +public interface IDownloadClient +{ + public Task Download(CancellationToken token, int retry = 0); +} diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index c15fb76f9..e6cfaeb8d 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Drawing.Text; using System.IO; using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders; @@ -25,6 +27,7 @@ using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.VFS; +using YamlDotNet.Core.Tokens; namespace Wabbajack.Installer; @@ -370,31 +373,34 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - + await missing - .Shuffle() .Where(a => a.State is not Manual) + .Shuffle() .PDoAll(async archive => { - _logger.LogInformation("Downloading {Archive}", archive.Name); - var outputPath = _configuration.Downloads.Combine(archive.Name); - var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); - - if (download) - if (outputPath.FileExists() && !downloadPackagePath.FileExists()) - { - var origName = Path.GetFileNameWithoutExtension(archive.Name); - var ext = Path.GetExtension(archive.Name); - var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); - outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); - outputPath.Delete(); - } - - var hash = await DownloadArchive(archive, download, token, outputPath); + await DownloadArchiveAsync(archive, token, download); UpdateProgress(1); }); } + private async Task DownloadArchiveAsync(Archive archive, CancellationToken token, bool download) + { + _logger.LogInformation("Downloading {Archive}", archive.Name); + var outputPath = _configuration.Downloads.Combine(archive.Name); + var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + //if (download) + //if (outputPath.FileExists() && !downloadPackagePath.FileExists()) + //{ + // var origName = Path.GetFileNameWithoutExtension(archive.Name); + // var ext = Path.GetExtension(archive.Name); + // var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); + // outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); + // outputPath.Delete(); + //} + var hash = await DownloadArchive(archive, download, token, outputPath); + } + private async Task SendDownloadMetrics(List missing) { var grouped = missing.GroupBy(m => m.State.GetType()); diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index 1f5e898b2..536cf0451 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -52,15 +52,15 @@ public static void Main(string[] args) services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); - services.AddSingleton(); + services.AddSingleton(); services.AddAllSingleton>(s => new Resource("Web Requests", 4)); - services.AddAllSingleton(); - + services.AddHttpDownloader(); + var version = $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; services.AddSingleton(s => new ApplicationInfo diff --git a/Wabbajack.Networking.Http/ServiceExtensions.cs b/Wabbajack.Networking.Http/ServiceExtensions.cs deleted file mode 100644 index 42a9796e7..000000000 --- a/Wabbajack.Networking.Http/ServiceExtensions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Wabbajack.Networking.Http.Interfaces; - -namespace Wabbajack.Networking.Http; - -public static class ServiceExtensions -{ - public static void AddHttpDownloader(this IServiceCollection services) - { - services.AddSingleton(); - } -} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs b/Wabbajack.Networking.Http/SingleThreadedDownloader.cs deleted file mode 100644 index ed88e3bba..000000000 --- a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs +++ /dev/null @@ -1,162 +0,0 @@ -using System; -using System.Buffers; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.Configuration; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Networking.Http.Interfaces; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.RateLimiter; - -namespace Wabbajack.Networking.Http; - -public class SingleThreadedDownloader : IHttpDownloader -{ - private readonly HttpClient _client; - private readonly ILogger _logger; - private readonly PerformanceSettings _settings; - - public SingleThreadedDownloader(ILogger logger, HttpClient client, MainSettings settings) - { - _logger = logger; - _client = client; - _settings = settings.PerformanceSettings; - } - - public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, - CancellationToken token) - { - Exception downloadError = null!; - var downloader = new ResumableDownloader(message, outputPath, job, _settings, _logger); - for (var i = 0; i < 3; i++) - { - try - { - return await downloader.Download(token); - } - catch (Exception ex) - { - downloadError = ex; - _logger.LogDebug("Download for '{name}' failed. Retrying...", outputPath.FileName.ToString()); - } - } - - _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); - return new Hash(); - - // using var response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); - // if (!response.IsSuccessStatusCode) - // throw new HttpException(response); - // - // if (job.Size == 0) - // job.Size = response.Content.Headers.ContentLength ?? 0; - // - // /* Need to make this mulitthreaded to be much use - // if ((response.Content.Headers.ContentLength ?? 0) != 0 && - // response.Headers.AcceptRanges.FirstOrDefault() == "bytes") - // { - // return await ResettingDownloader(response, message, outputPath, job, token); - // } - // */ - // - // await using var stream = await response.Content.ReadAsStreamAsync(token); - // await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write); - // return await stream.HashingCopy(outputStream, token, job); - } - - private const int CHUNK_SIZE = 1024 * 1024 * 8; - - private async Task ResettingDownloader(HttpResponseMessage response, HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) - { - - using var rented = MemoryPool.Shared.Rent(CHUNK_SIZE); - var buffer = rented.Memory; - - var hasher = new xxHashAlgorithm(0); - - var running = true; - ulong finalHash = 0; - - var inputStream = await response.Content.ReadAsStreamAsync(token); - await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); - long writePosition = 0; - - while (running && !token.IsCancellationRequested) - { - var totalRead = 0; - - while (totalRead != buffer.Length) - { - var read = await inputStream.ReadAsync(buffer.Slice(totalRead, buffer.Length - totalRead), - token); - - - if (read == 0) - { - running = false; - break; - } - - if (job != null) - await job.Report(read, token); - - totalRead += read; - } - - var pendingWrite = outputStream.WriteAsync(buffer[..totalRead], token); - if (running) - { - hasher.TransformByteGroupsInternal(buffer.Span); - await pendingWrite; - } - else - { - var preSize = (totalRead >> 5) << 5; - if (preSize > 0) - { - hasher.TransformByteGroupsInternal(buffer[..preSize].Span); - finalHash = hasher.FinalizeHashValueInternal(buffer[preSize..totalRead].Span); - await pendingWrite; - break; - } - - finalHash = hasher.FinalizeHashValueInternal(buffer[..totalRead].Span); - await pendingWrite; - break; - } - - { - writePosition += totalRead; - if (job != null) - await job.Report(totalRead, token); - message = CloneMessage(message); - message.Headers.Range = new RangeHeaderValue(writePosition, writePosition + CHUNK_SIZE); - await inputStream.DisposeAsync(); - response.Dispose(); - response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); - HttpException.ThrowOnFailure(response); - inputStream = await response.Content.ReadAsStreamAsync(token); - } - } - - await outputStream.FlushAsync(token); - - return new Hash(finalHash); - } - - private HttpRequestMessage CloneMessage(HttpRequestMessage message) - { - var newMsg = new HttpRequestMessage(message.Method, message.RequestUri); - foreach (var header in message.Headers) - { - newMsg.Headers.Add(header.Key, header.Value); - } - return newMsg; - } -} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj index 595ab3d1b..4f158d080 100644 --- a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj +++ b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj @@ -9,6 +9,7 @@ + diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index a2c3c5181..bc89e9785 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -11,6 +11,7 @@ using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; +using Wabbajack.Downloader.Clients; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.Downloaders.ModDB; @@ -23,7 +24,6 @@ using Wabbajack.Installer; using Wabbajack.Networking.BethesdaNet; using Wabbajack.Networking.Discord; -using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; using Wabbajack.Networking.Steam; @@ -157,7 +157,9 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service // Networking service.AddSingleton(); - service.AddAllSingleton(); + + // Downloader + service.AddDownloaderService(); service.AddSteam(); diff --git a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj index 55e7bfb54..39b248fc8 100644 --- a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj +++ b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj @@ -18,10 +18,12 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/Wabbajack.sln b/Wabbajack.sln index bedde649f..642f5ab20 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -108,8 +108,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".solutionItems", ".solutionItems", "{109037C8-CF2F-4179-B064-A66147BC18C5}" ProjectSection(SolutionItems) = preProject .gitignore = .gitignore - nuget.config = nuget.config CHANGELOG.md = CHANGELOG.md + nuget.config = nuget.config README.md = README.md EndProjectSection EndProject @@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Verif EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Configuration", "Wabbajack.Configuration\Wabbajack.Configuration.csproj", "{E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloader.Services", "Wabbajack.Downloader.Clients\Wabbajack.Downloader.Services.csproj", "{258D44F2-956F-43A3-BD29-11A28D03F406}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -401,6 +403,10 @@ Global {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Debug|Any CPU.Build.0 = Debug|Any CPU {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Release|Any CPU.ActiveCfg = Release|Any CPU {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Release|Any CPU.Build.0 = Release|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Debug|Any CPU.Build.0 = Debug|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Release|Any CPU.ActiveCfg = Release|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -451,6 +457,7 @@ Global {7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871} {E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C} {D9560C73-4E58-4463-9DB9-D06491E0E1C8} = {98B731EE-4FC0-4482-A069-BCBA25497871} + {258D44F2-956F-43A3-BD29-11A28D03F406} = {98B731EE-4FC0-4482-A069-BCBA25497871} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8} From dc6847e81a7f4087ee09e26506272e345b7c3454 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 16:44:19 -0500 Subject: [PATCH 02/26] Reverted unneeded change to AInstaller --- Wabbajack.Installer/AInstaller.cs | 32 +++++++++++++------------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index e6cfaeb8d..3f42903ba 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -379,28 +379,22 @@ await missing .Shuffle() .PDoAll(async archive => { - await DownloadArchiveAsync(archive, token, download); - UpdateProgress(1); + _logger.LogInformation("Downloading {Archive}", archive.Name); + var outputPath = _configuration.Downloads.Combine(archive.Name); + var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + if (download) + if (outputPath.FileExists() && !downloadPackagePath.FileExists()) + { + var origName = Path.GetFileNameWithoutExtension(archive.Name); + var ext = Path.GetExtension(archive.Name); + var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); + outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); + outputPath.Delete(); + } + var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); }); } - private async Task DownloadArchiveAsync(Archive archive, CancellationToken token, bool download) - { - _logger.LogInformation("Downloading {Archive}", archive.Name); - var outputPath = _configuration.Downloads.Combine(archive.Name); - var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); - //if (download) - //if (outputPath.FileExists() && !downloadPackagePath.FileExists()) - //{ - // var origName = Path.GetFileNameWithoutExtension(archive.Name); - // var ext = Path.GetExtension(archive.Name); - // var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); - // outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); - // outputPath.Delete(); - //} - var hash = await DownloadArchive(archive, download, token, outputPath); - } - private async Task SendDownloadMetrics(List missing) { var grouped = missing.GroupBy(m => m.State.GetType()); From 28a7f4eab4e5430e952c5f6463238c8a90cf18e8 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 17:06:14 -0500 Subject: [PATCH 03/26] Added more explicit error logging --- .../NonResumableDownloadClient.cs | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs index 00d7ea74e..db4fd5817 100644 --- a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -10,26 +10,53 @@ internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath { public async Task Download(CancellationToken token, int retry = 3) { + Stream? fileStream; + try { + fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not open file path '{filePath}'. Throwing...", _outputPath.FileName.ToString()); + + throw; + } + + try + { + _logger.LogDebug("Download for '{name}' is starting from scratch...", _outputPath.FileName.ToString()); + var httpClient = _httpClientFactory.CreateClient("SmallFilesClient"); var response = await httpClient.GetStreamAsync(_msg.RequestUri!.ToString()); - await using var fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); await response.CopyToAsync(fileStream, token); fileStream.Close(); - await using var file = _outputPath.Open(FileMode.Open); - return await file.Hash(token); + } catch (Exception ex) { - _logger.LogError(ex, "Failed to download '{name}' after 3 tries.", _outputPath.FileName.ToString()); - if (retry <= 3) { + _logger.LogError(ex, "Download for '{name}' encountered error. Retrying...", _outputPath.FileName.ToString()); + return await Download(token, retry--); } - return new Hash(); + _logger.LogError(ex, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); + + throw; + } + + try + { + await using var file = _outputPath.Open(FileMode.Open); + return await file.Hash(token); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not hash file '{filePath}'. Throwing...", _outputPath.FileName.ToString()); + + throw; } } } \ No newline at end of file From 861e7df30d76f777760488001d00ab7a742b95d4 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 17:17:02 -0500 Subject: [PATCH 04/26] Synced Namespaces Cleaned up code Removed retry login in NonResumableDownload as DownloadService already retries --- Wabbajack.CLI/Program.cs | 2 +- Wabbajack.Downloader.Clients/DownloadClientFactory.cs | 2 +- Wabbajack.Downloader.Clients/DownloaderService.cs | 10 ++++------ .../NonResumableDownloadClient.cs | 11 ++--------- .../ResumableDownloadClient.cs | 8 ++++---- Wabbajack.Downloader.Clients/ServiceExtensions.cs | 2 +- Wabbajack.Downloaders.Interfaces/IDownloadClient.cs | 2 +- Wabbajack.Services.OSIntegrated/ServiceExtensions.cs | 2 +- 8 files changed, 15 insertions(+), 24 deletions(-) diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 370037726..bb13ed675 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -16,7 +16,7 @@ using Wabbajack.VFS; using Client = Wabbajack.Networking.GitHub.Client; using Wabbajack.CLI.Builder; -using Wabbajack.Downloader.Clients; +using Wabbajack.Downloader.Services; namespace Wabbajack.CLI; diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index c85c12333..ce3467a78 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -4,7 +4,7 @@ using Wabbajack.Paths; using Wabbajack.RateLimiter; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public interface IDownloadClientFactory { diff --git a/Wabbajack.Downloader.Clients/DownloaderService.cs b/Wabbajack.Downloader.Clients/DownloaderService.cs index a4ce17368..15de71491 100644 --- a/Wabbajack.Downloader.Clients/DownloaderService.cs +++ b/Wabbajack.Downloader.Clients/DownloaderService.cs @@ -4,12 +4,11 @@ using Wabbajack.Paths; using Wabbajack.RateLimiter; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public class DownloaderService(ILogger _logger, IDownloadClientFactory _httpDownloaderFactory) : IHttpDownloader { - public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, - CancellationToken token) + public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) { Exception downloadError = null!; @@ -19,7 +18,7 @@ public async Task Download(HttpRequestMessage message, AbsolutePath output { try { - return await downloader.Download(token, 3); + return await downloader.Download(token); } catch (Exception ex) { @@ -29,8 +28,7 @@ public async Task Download(HttpRequestMessage message, AbsolutePath output } _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); - return new Hash(); - + return new Hash(); } } \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs index db4fd5817..3b0203d66 100644 --- a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -4,11 +4,11 @@ using Wabbajack.Paths; using Wabbajack.Paths.IO; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, ILogger _logger, IHttpClientFactory _httpClientFactory) : IDownloadClient { - public async Task Download(CancellationToken token, int retry = 3) + public async Task Download(CancellationToken token) { Stream? fileStream; @@ -35,13 +35,6 @@ public async Task Download(CancellationToken token, int retry = 3) } catch (Exception ex) { - if (retry <= 3) - { - _logger.LogError(ex, "Download for '{name}' encountered error. Retrying...", _outputPath.FileName.ToString()); - - return await Download(token, retry--); - } - _logger.LogError(ex, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); throw; diff --git a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index 30e1c1da7..ecdd5ac9d 100644 --- a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -10,7 +10,7 @@ using Wabbajack.Networking.Http; using Wabbajack.Downloaders.Interfaces; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient { @@ -18,7 +18,7 @@ internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _ou private Exception? _error; private AbsolutePath _packagePath = _outputPath.WithExtension(Extension.FromPath(".download_package")); - public async Task Download(CancellationToken token, int retry = 0) + public async Task Download(CancellationToken token) { _token = token; @@ -71,7 +71,7 @@ public async Task Download(CancellationToken token, int retry = 0) DeletePackage(); } } - + if (!_outputPath.FileExists()) { return new Hash(); @@ -93,7 +93,7 @@ private DownloadConfiguration CreateConfiguration(HttpRequestMessage message) { Headers = message.Headers.ToWebHeaderCollection(), ProtocolVersion = message.Version, - UserAgent = message.Headers.UserAgent.ToString() + UserAgent = message.Headers.UserAgent.ToString() } }; diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs index 5ee53cfad..d361b4513 100644 --- a/Wabbajack.Downloader.Clients/ServiceExtensions.cs +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -2,7 +2,7 @@ using Wabbajack.Configuration; using Wabbajack.Networking.Http.Interfaces; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public static class ServiceExtensions { diff --git a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs index 445fe05fa..759105bb8 100644 --- a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs +++ b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs @@ -6,5 +6,5 @@ namespace Wabbajack.Downloaders.Interfaces; public interface IDownloadClient { - public Task Download(CancellationToken token, int retry = 0); + public Task Download(CancellationToken token); } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index bc89e9785..d60d39c04 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -11,7 +11,7 @@ using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; -using Wabbajack.Downloader.Clients; +using Wabbajack.Downloader.Services; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.Downloaders.ModDB; From 44fe6950a8452fbafabd9e5157f8e1a2f955cf39 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:27:01 -0500 Subject: [PATCH 05/26] Adding UI for setting minimum file size at which to use resumable downloader --- Wabbajack.App.Wpf/Settings.cs | 27 ++++++++++++++++++- .../Settings/PerformanceSettingsView.xaml | 24 +++++++++++++++++ .../Settings/PerformanceSettingsView.xaml.cs | 13 +++++++++ .../PerformanceSettings.cs | 2 +- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index 4ad1517c3..887b57169 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -1,4 +1,5 @@ -using Wabbajack.Downloaders; +using SteamKit2.GC.Dota.Internal; +using Wabbajack.Downloaders; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Paths; using Wabbajack.RateLimiter; @@ -18,6 +19,7 @@ public class PerformanceSettings : ViewModel { private readonly Configuration.MainSettings _settings; private readonly int _defaultMaximumMemoryPerDownloadThreadMb; + private readonly long _defaultMinimumFileSizeForResumableDownload; public PerformanceSettings(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { @@ -26,15 +28,23 @@ public PerformanceSettings(Configuration.MainSettings settings, IResource _minimumFileSizeForResumableDownload; + set + { + RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownload, value); + _settings.PerformanceSettings.MinimumFileSizeForResumableDownload = value; + } + } + public void ResetMaximumMemoryPerDownloadThreadMb() { MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMb; } + + public void ResetMinimumFileSizeForResumableDownload() + { + MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownload; + } } } diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml index d976f2b94..f2a6ebc2e 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml @@ -71,6 +71,30 @@ HorizontalAlignment="Left"> Reset + + + diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs index d2a0ee5c4..142f0d911 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs @@ -21,16 +21,29 @@ public PerformanceSettingsView() x => x.MaximumMemoryPerDownloadThreadMb, x => x.MaximumMemoryPerDownloadThreadIntegerUpDown.Value) .DisposeWith(disposable); + + this.BindStrict( + ViewModel, + x => x.MinimumFileSizeForResumableDownload, + x => x.MinimumFileSizeForResumableDownloadIntegerUpDown.Value) + .DisposeWith(disposable); + this.EditResourceSettings.Command = ReactiveCommand.Create(() => { UIUtils.OpenFile( KnownFolders.WabbajackAppLocal.Combine("saved_settings", "resource_settings.json")); Environment.Exit(0); }); + ResetMaximumMemoryPerDownloadThread.Command = ReactiveCommand.Create(() => { ViewModel.ResetMaximumMemoryPerDownloadThreadMb(); }); + + ResetMinimumFileSizeForResumableDownload.Command = ReactiveCommand.Create(() => + { + ViewModel.ResetMinimumFileSizeForResumableDownload(); + }); }); } } diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs index 64dd6d8aa..adefc1e27 100644 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ b/Wabbajack.Configuration/PerformanceSettings.cs @@ -4,5 +4,5 @@ public class PerformanceSettings { public int MaximumMemoryPerDownloadThreadMb { get; set; } - public long MinimumFileSizeForResumableDownload { get; set; } = (long)1024 * 1024 * 500; // 500MB + public long MinimumFileSizeForResumableDownload { get; set; } = 0; } \ No newline at end of file From fa7b2a08ad46fc5aeba79f213216a2dacc3a0cfe Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:34:26 -0500 Subject: [PATCH 06/26] Reverting unintended changes --- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Installer/AInstaller.cs | 82 +++++++++++----------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 6043cb1de..5c0aa72be 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - Exe + WinExe net9.0-windows true x64 diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 3f42903ba..efb4bdd00 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Drawing.Text; using System.IO; using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders; @@ -27,7 +25,6 @@ using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.VFS; -using YamlDotNet.Core.Tokens; namespace Wabbajack.Installer; @@ -226,7 +223,7 @@ public async Task InstallArchives(CancellationToken token) NextStep(Consts.StepInstalling, "Installing files", ModList.Directives.Sum(d => d.Size), x => x.ToFileSizeString()); var grouped = ModList.Directives .OfType() - .Select(a => new {VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) + .Select(a => new { VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a }) .GroupBy(a => a.VF) .ToDictionary(a => a.Key); @@ -247,27 +244,27 @@ await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) => switch (file) { case PatchedFromArchive pfa: - { - await using var s = await sf.GetStream(); - s.Position = 0; - await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { - await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); - ThrowOnNonMatchingHash(file, hash); + await using var s = await sf.GetStream(); + s.Position = 0; + await using var patchDataStream = await InlinedFileStream(pfa.PatchID); + { + await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); + } } - } break; case TransformedTexture tt: - { - await using var s = await sf.GetStream(); - await using var of = destPath.Open(FileMode.Create, FileAccess.Write); - _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); - await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, - of, token); - } + { + await using var s = await sf.GetStream(); + await using var of = destPath.Open(FileMode.Create, FileAccess.Write); + _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); + await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, + of, token); + } break; @@ -290,7 +287,7 @@ await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.Im } await FileHashCache.FileHashWriteCache(destPath, file.Hash); - await job.Report((int) directive.VF.Size, token); + await job.Report((int)directive.VF.Size, token); } }, token); } @@ -305,8 +302,8 @@ private void ThrowNonMatchingError(Directive file, Hash gotHash) _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); } - - + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) { if (hash == directive.Hash) return; @@ -335,14 +332,14 @@ public async Task DownloadArchives(CancellationToken token) { var matches = mirrors[archive.Hash].ToArray(); if (!matches.Any()) continue; - + archive.State = matches.First().State; _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString()); _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name, matches.First().State.PrimaryKeyString); } - - + + foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { @@ -375,13 +372,14 @@ public async Task DownloadMissingArchives(List missing, CancellationTok } await missing - .Where(a => a.State is not Manual) .Shuffle() + .Where(a => a.State is not Manual) .PDoAll(async archive => { _logger.LogInformation("Downloading {Archive}", archive.Name); var outputPath = _configuration.Downloads.Combine(archive.Name); var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + if (download) if (outputPath.FileExists() && !downloadPackagePath.FileExists()) { @@ -391,7 +389,9 @@ await missing outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); outputPath.Delete(); } - var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); + + var hash = await DownloadArchive(archive, download, token, outputPath); + UpdateProgress(1); }); } @@ -502,7 +502,7 @@ protected async Task OptimizeModlist(CancellationToken token) _logger.LogInformation("Optimizing ModList directives"); UnoptimizedArchives = ModList.Archives; UnoptimizedDirectives = ModList.Directives; - + var indexed = ModList.Directives.ToDictionary(d => d.To); var bsasToBuild = await ModList.Directives @@ -537,11 +537,11 @@ FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuil var profileFolder = _configuration.Install.Combine("profiles"); - var savePath = (RelativePath) "saves"; + var savePath = (RelativePath)"saves"; NextStep(Consts.StepPreparing, "Looking for files to delete", 0); await _configuration.Install.EnumerateFiles() - .PMapAllBatched(_limiter, f => + .PMapAllBatched(_limiter, f => { var relativeTo = f.RelativeTo(_configuration.Install); if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) @@ -570,7 +570,7 @@ await _configuration.Install.EnumerateFiles() { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] - var split = ((string) path.RelativeTo(_configuration.Install)).Split('\\'); + var split = ((string)path.RelativeTo(_configuration.Install)).Split('\\'); return Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))); }) .Distinct() @@ -598,15 +598,15 @@ await _configuration.Install.EnumerateFiles() NextStep(Consts.StepPreparing, "Looking for unmodified files", 0); await indexed.Values.PMapAllBatchedAsync(_limiter, async d => - { - // Bit backwards, but we want to return null for - // all files we *want* installed. We return the files - // to remove from the install list. - var path = _configuration.Install.Combine(d.To); - if (!existingfiles.Contains(path)) return null; - - return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; - }) + { + // Bit backwards, but we want to return null for + // all files we *want* installed. We return the files + // to remove from the install list. + var path = _configuration.Install.Combine(d.To); + if (!existingfiles.Contains(path)) return null; + + return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; + }) .Do(d => { if (d != null) @@ -621,7 +621,7 @@ await indexed.Values.PMapAllBatchedAsync(_limiter, async d => .GroupBy(d => d.ArchiveHashPath.Hash) .Select(d => d.Key) .ToHashSet(); - + ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); } From 5ef9812fb5cffc1fb173cb8a9f2877e82ba23b68 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:38:05 -0500 Subject: [PATCH 07/26] Reverting unneeded changes --- Wabbajack.DTOs/Archive.cs | 13 +----- Wabbajack.Installer/AInstaller.cs | 74 +++++++++++++++---------------- 2 files changed, 38 insertions(+), 49 deletions(-) diff --git a/Wabbajack.DTOs/Archive.cs b/Wabbajack.DTOs/Archive.cs index b6fe20ec9..4cf9906d7 100644 --- a/Wabbajack.DTOs/Archive.cs +++ b/Wabbajack.DTOs/Archive.cs @@ -1,24 +1,13 @@ -using System; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; namespace Wabbajack.DTOs; -public class Archive : IComparable +public class Archive { public Hash Hash { get; set; } public string Meta { get; set; } = ""; public string Name { get; set; } public long Size { get; set; } public IDownloadState State { get; set; } - - public int CompareTo(object obj) - { - if (obj == null) return 1; - Archive otherArchive = obj as Archive; - if (otherArchive != null) - return this.Size.CompareTo(otherArchive.Size); - else - throw new ArgumentException("Object is not an Archive"); - } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index efb4bdd00..c15fb76f9 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -223,7 +223,7 @@ public async Task InstallArchives(CancellationToken token) NextStep(Consts.StepInstalling, "Installing files", ModList.Directives.Sum(d => d.Size), x => x.ToFileSizeString()); var grouped = ModList.Directives .OfType() - .Select(a => new { VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a }) + .Select(a => new {VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) .GroupBy(a => a.VF) .ToDictionary(a => a.Key); @@ -244,27 +244,27 @@ await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) => switch (file) { case PatchedFromArchive pfa: + { + await using var s = await sf.GetStream(); + s.Position = 0; + await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { - await using var s = await sf.GetStream(); - s.Position = 0; - await using var patchDataStream = await InlinedFileStream(pfa.PatchID); - { - await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); - ThrowOnNonMatchingHash(file, hash); - } + await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); } + } break; case TransformedTexture tt: - { - await using var s = await sf.GetStream(); - await using var of = destPath.Open(FileMode.Create, FileAccess.Write); - _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); - await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, - of, token); - } + { + await using var s = await sf.GetStream(); + await using var of = destPath.Open(FileMode.Create, FileAccess.Write); + _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); + await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, + of, token); + } break; @@ -287,7 +287,7 @@ await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.Im } await FileHashCache.FileHashWriteCache(destPath, file.Hash); - await job.Report((int)directive.VF.Size, token); + await job.Report((int) directive.VF.Size, token); } }, token); } @@ -302,8 +302,8 @@ private void ThrowNonMatchingError(Directive file, Hash gotHash) _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); } - - + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) { if (hash == directive.Hash) return; @@ -332,14 +332,14 @@ public async Task DownloadArchives(CancellationToken token) { var matches = mirrors[archive.Hash].ToArray(); if (!matches.Any()) continue; - + archive.State = matches.First().State; _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString()); _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name, matches.First().State.PrimaryKeyString); } - - + + foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { @@ -370,7 +370,7 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - + await missing .Shuffle() .Where(a => a.State is not Manual) @@ -502,7 +502,7 @@ protected async Task OptimizeModlist(CancellationToken token) _logger.LogInformation("Optimizing ModList directives"); UnoptimizedArchives = ModList.Archives; UnoptimizedDirectives = ModList.Directives; - + var indexed = ModList.Directives.ToDictionary(d => d.To); var bsasToBuild = await ModList.Directives @@ -537,11 +537,11 @@ FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuil var profileFolder = _configuration.Install.Combine("profiles"); - var savePath = (RelativePath)"saves"; + var savePath = (RelativePath) "saves"; NextStep(Consts.StepPreparing, "Looking for files to delete", 0); await _configuration.Install.EnumerateFiles() - .PMapAllBatched(_limiter, f => + .PMapAllBatched(_limiter, f => { var relativeTo = f.RelativeTo(_configuration.Install); if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) @@ -570,7 +570,7 @@ await _configuration.Install.EnumerateFiles() { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] - var split = ((string)path.RelativeTo(_configuration.Install)).Split('\\'); + var split = ((string) path.RelativeTo(_configuration.Install)).Split('\\'); return Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))); }) .Distinct() @@ -598,15 +598,15 @@ await _configuration.Install.EnumerateFiles() NextStep(Consts.StepPreparing, "Looking for unmodified files", 0); await indexed.Values.PMapAllBatchedAsync(_limiter, async d => - { - // Bit backwards, but we want to return null for - // all files we *want* installed. We return the files - // to remove from the install list. - var path = _configuration.Install.Combine(d.To); - if (!existingfiles.Contains(path)) return null; - - return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; - }) + { + // Bit backwards, but we want to return null for + // all files we *want* installed. We return the files + // to remove from the install list. + var path = _configuration.Install.Combine(d.To); + if (!existingfiles.Contains(path)) return null; + + return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; + }) .Do(d => { if (d != null) @@ -621,7 +621,7 @@ await indexed.Values.PMapAllBatchedAsync(_limiter, async d => .GroupBy(d => d.ArchiveHashPath.Hash) .Select(d => d.Key) .ToHashSet(); - + ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); } From c25f43922b6e2866c698d550639ca77f45e376c3 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 01:35:54 -0500 Subject: [PATCH 08/26] Remove deprecated nexus auth method --- Wabbajack.CLI/VerbRegistration.cs | 2 -- Wabbajack.CLI/Verbs/SetNexusApiKey.cs | 40 --------------------------- 2 files changed, 42 deletions(-) delete mode 100644 Wabbajack.CLI/Verbs/SetNexusApiKey.cs diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index 69634dde2..93ed9e134 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -45,8 +45,6 @@ public static void AddCLIVerbs(this IServiceCollection services) { services.AddSingleton(); CommandLineBuilder.RegisterCommand(ModlistReport.Definition, c => ((ModlistReport)c).Run); services.AddSingleton(); -CommandLineBuilder.RegisterCommand(SetNexusApiKey.Definition, c => ((SetNexusApiKey)c).Run); -services.AddSingleton(); CommandLineBuilder.RegisterCommand(SteamDownloadFile.Definition, c => ((SteamDownloadFile)c).Run); services.AddSingleton(); CommandLineBuilder.RegisterCommand(SteamDumpAppInfo.Definition, c => ((SteamDumpAppInfo)c).Run); diff --git a/Wabbajack.CLI/Verbs/SetNexusApiKey.cs b/Wabbajack.CLI/Verbs/SetNexusApiKey.cs deleted file mode 100644 index a66576441..000000000 --- a/Wabbajack.CLI/Verbs/SetNexusApiKey.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.CLI.Builder; -using Wabbajack.DTOs.Logins; -using Wabbajack.Services.OSIntegrated; - -namespace Wabbajack.CLI.Verbs; - -public class SetNexusApiKey -{ - private readonly EncryptedJsonTokenProvider _tokenProvider; - private readonly ILogger _logger; - - public SetNexusApiKey(EncryptedJsonTokenProvider tokenProvider, ILogger logger) - { - _tokenProvider = tokenProvider; - _logger = logger; - } - - public static VerbDefinition Definition = new("set-nexus-api-key", - "Sets the Nexus API key to the specified value", - [ - new OptionDefinition(typeof(string), "k", "key", "The Nexus API key") - ]); - - public async Task Run(string key) - { - if (string.IsNullOrEmpty(key)) - { - _logger.LogInformation("Not setting Nexus API key, that looks like an empty string to me."); - return -1; - } - else - { - await _tokenProvider.SetToken(new() { ApiKey = key }); - _logger.LogInformation("Set Nexus API Key to {key}", key); - return 0; - } - } -} \ No newline at end of file From 54863c65d07979bf53c0252eaaf20a9fefe59c3c Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 13:26:45 -0500 Subject: [PATCH 09/26] Refactored performance settings and its dependency injection --- Wabbajack.App.Wpf/Settings.cs | 14 ++++---- .../View Models/Settings/SettingsVM.cs | 4 +-- .../Settings/PerformanceSettingsView.xaml | 2 +- .../Settings/PerformanceSettingsView.xaml.cs | 2 +- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Configuration/MainSettings.cs | 32 ++++++++++++++++--- .../PerformanceSettings.cs | 8 ----- .../DownloadClientFactory.cs | 6 ++-- .../ResumableDownloadClient.cs | 6 ++-- .../ServiceExtensions.cs | 2 -- Wabbajack.Installer/AInstaller.cs | 20 +++++++++--- Wabbajack.Launcher/Program.cs | 24 ++------------ .../ServiceExtensions.cs | 17 +++------- .../SettingsManager.cs | 13 ++++++++ 14 files changed, 82 insertions(+), 70 deletions(-) delete mode 100644 Wabbajack.Configuration/PerformanceSettings.cs diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index 887b57169..ac7c207e1 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -15,13 +15,13 @@ public class Mo2ModlistInstallationSettings public bool AutomaticallyOverrideExistingInstall { get; set; } } - public class PerformanceSettings : ViewModel + public class PerformanceSettingsViewModel : ViewModel { private readonly Configuration.MainSettings _settings; private readonly int _defaultMaximumMemoryPerDownloadThreadMb; private readonly long _defaultMinimumFileSizeForResumableDownload; - public PerformanceSettings(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) + public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { var p = systemParams.Create(); @@ -29,15 +29,15 @@ public PerformanceSettings(Configuration.MainSettings settings, IResource logger, IServiceProvider provider) AuthorFile = new AuthorFilesVM(provider.GetRequiredService>()!, provider.GetRequiredService()!, provider.GetRequiredService()!, this); OpenTerminalCommand = ReactiveCommand.CreateFromTask(OpenTerminal); - Performance = new PerformanceSettings( + Performance = new PerformanceSettingsViewModel( _settings, provider.GetRequiredService>(), provider.GetRequiredService()); diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml index f2a6ebc2e..ddc9b950b 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml @@ -9,7 +9,7 @@ xmlns:xwpf="http://schemas.xceed.com/wpf/xaml/toolkit" d:DesignHeight="450" d:DesignWidth="800" - x:TypeArguments="local:PerformanceSettings" + x:TypeArguments="local:PerformanceSettingsViewModel" mc:Ignorable="d"> /// Interaction logic for PerformanceSettingsView.xaml /// - public partial class PerformanceSettingsView : ReactiveUserControl + public partial class PerformanceSettingsView : ReactiveUserControl { public PerformanceSettingsView() { diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 5c0aa72be..6043cb1de 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - WinExe + Exe net9.0-windows true x64 diff --git a/Wabbajack.Configuration/MainSettings.cs b/Wabbajack.Configuration/MainSettings.cs index 5b8bf4eb8..0006f61a0 100644 --- a/Wabbajack.Configuration/MainSettings.cs +++ b/Wabbajack.Configuration/MainSettings.cs @@ -1,13 +1,31 @@ -namespace Wabbajack.Configuration; +using System.Text.Json.Serialization; + +namespace Wabbajack.Configuration; public class MainSettings { public const string SettingsFileName = "app_settings"; - private const int SettingsVersion = 1; + [JsonPropertyName("CurrentSettingsVersion")] public int CurrentSettingsVersion { get; set; } - public PerformanceSettings PerformanceSettings { get; set; } = new(); + public int MaximumMemoryPerDownloadThreadInMB + { + get => Performance.MaximumMemoryPerDownloadThreadMb; + set => Performance.MaximumMemoryPerDownloadThreadMb = value; + } + + public long MinimumFileSizeForResumableDownloadMB { + get => Performance.MinimumFileSizeForResumableDownload; + set => Performance.MinimumFileSizeForResumableDownload = value; + } + + private const int SettingsVersion = 1; + + [JsonInclude] + [JsonPropertyName("PerformanceSettings")] + private PerformanceSettings Performance { get; set; } = new(); + public bool Upgrade() { @@ -18,10 +36,16 @@ public bool Upgrade() if (CurrentSettingsVersion < 1) { - PerformanceSettings.MaximumMemoryPerDownloadThreadMb = -1; + Performance.MaximumMemoryPerDownloadThreadMb = -1; } CurrentSettingsVersion = SettingsVersion; return true; } + + internal class PerformanceSettings + { + public int MaximumMemoryPerDownloadThreadMb { get; set; } = -1; + public long MinimumFileSizeForResumableDownload { get; set; } = -1; + } } \ No newline at end of file diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs deleted file mode 100644 index adefc1e27..000000000 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Wabbajack.Configuration; - -public class PerformanceSettings -{ - public int MaximumMemoryPerDownloadThreadMb { get; set; } - - public long MinimumFileSizeForResumableDownload { get; set; } = 0; -} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index ce3467a78..4cc228683 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -11,7 +11,7 @@ public interface IDownloadClientFactory public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job); } -public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory +public class DownloadClientFactory(MainSettings _settings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory { private readonly ILogger _nonResuableDownloaderLogger = _loggerFactory.CreateLogger(); private readonly ILogger _resumableDownloaderLogger = _loggerFactory.CreateLogger(); @@ -20,9 +20,9 @@ public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILo public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) { - if (job.Size >= _performanceSettings.MinimumFileSizeForResumableDownload) + if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB) { - return new ResumableDownloadClient(msg, outputPath, job, _performanceSettings, _resumableDownloaderLogger); + return new ResumableDownloadClient(msg, outputPath, job, _settings.MaximumMemoryPerDownloadThreadInMB, _resumableDownloaderLogger); } else { diff --git a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index ecdd5ac9d..dc5a75b46 100644 --- a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -12,7 +12,7 @@ namespace Wabbajack.Downloader.Services; -internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient +internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, int _maxMemoryPerDownloadThread, ILogger _logger) : IDownloadClient { private CancellationToken _token; private Exception? _error; @@ -83,10 +83,10 @@ public async Task Download(CancellationToken token) private DownloadConfiguration CreateConfiguration(HttpRequestMessage message) { - var maximumMemoryPerDownloadThreadMb = Math.Max(0, _performanceSettings.MaximumMemoryPerDownloadThreadMb); + var maximumMemoryPerDownloadThreadMb = Math.Max(0, _maxMemoryPerDownloadThread); var configuration = new DownloadConfiguration { - MaximumMemoryBufferBytes = maximumMemoryPerDownloadThreadMb * 1024 * 1024, + MaximumMemoryBufferBytes = maximumMemoryPerDownloadThreadMb, Timeout = (int)TimeSpan.FromSeconds(120).TotalMilliseconds, ReserveStorageSpaceBeforeStartingDownload = true, RequestConfiguration = new RequestConfiguration diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs index d361b4513..d00e1fef6 100644 --- a/Wabbajack.Downloader.Clients/ServiceExtensions.cs +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Wabbajack.Configuration; using Wabbajack.Networking.Http.Interfaces; namespace Wabbajack.Downloader.Services; @@ -9,7 +8,6 @@ public static class ServiceExtensions public static void AddDownloaderService(this IServiceCollection services) { services.AddHttpClient("SmallFilesClient").ConfigureHttpClient(c => c.Timeout = TimeSpan.FromMinutes(5)); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); } diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index c15fb76f9..bad41e058 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -370,11 +370,16 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - - await missing - .Shuffle() + + var missingBatches = missing .Where(a => a.State is not Manual) - .PDoAll(async archive => + .Batch(100) + .ToList(); + + List batchTasks = []; + foreach (var batch in missingBatches) + { + batchTasks.Add(batch.PDoAll(async archive => { _logger.LogInformation("Downloading {Archive}", archive.Name); var outputPath = _configuration.Downloads.Combine(archive.Name); @@ -392,7 +397,12 @@ await missing var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); - }); + })); + + await Task.Delay(TimeSpan.FromSeconds(10)); // Hitting a Nexus API limit when spinning these downloads up too fast. Need to slow this down. + } + + await Task.WhenAll(batchTasks); } private async Task SendDownloadMetrics(List missing) diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index 536cf0451..a577ffda9 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -1,20 +1,17 @@ using System; using System.Net.Http; using System.Runtime.InteropServices; -using System.Threading.Tasks; using Avalonia; using Avalonia.ReactiveUI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Wabbajack.Common; using Wabbajack.Configuration; using Wabbajack.Downloaders.Http; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Logins; using Wabbajack.Launcher.ViewModels; -using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; using Wabbajack.Paths; @@ -40,7 +37,8 @@ public static void Main(string[] args) services.AddNexusApi(); services.AddDTOConverters(); services.AddDTOSerializer(); - + services.AddSettings(); + services.AddSingleton(s => new Services.OSIntegrated.Configuration { EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), @@ -49,11 +47,7 @@ public static void Main(string[] args) LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") }); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); - + services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); @@ -81,18 +75,6 @@ public static void Main(string[] args) .StartWithClassicDesktopLifetime(args); } - private static MainSettings GetAppSettings(IServiceProvider provider, string name) - { - var settingsManager = provider.GetRequiredService(); - var settings = Task.Run(() => settingsManager.Load(name)).Result; - if (settings.Upgrade()) - { - settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); - } - - return settings; - } - public static IServiceProvider Services { get; set; } // Avalonia configuration, don't remove; also used by visual designer. diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index d60d39c04..89c67d8ea 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -121,9 +121,7 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") }); - service.AddSingleton(); - service.AddSingleton(); - service.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); + service.AddSettings(); // Resources @@ -230,16 +228,11 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service return service; } - public static MainSettings GetAppSettings(IServiceProvider provider, string name) + public static IServiceCollection AddSettings(this IServiceCollection services) { - var settingsManager = provider.GetRequiredService(); - var settings = Task.Run(() => settingsManager.Load(name)).Result; - if (settings.Upgrade()) - { - settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); - } - - return settings; + services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(); + return services; } private static void CleanAllTempData(AbsolutePath path) diff --git a/Wabbajack.Services.OSIntegrated/SettingsManager.cs b/Wabbajack.Services.OSIntegrated/SettingsManager.cs index ca04ff765..81d6f9728 100644 --- a/Wabbajack.Services.OSIntegrated/SettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/SettingsManager.cs @@ -3,8 +3,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.Common; +using Wabbajack.Configuration; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Paths; using Wabbajack.Paths.IO; @@ -30,6 +32,17 @@ private AbsolutePath GetPath(string key) return _configuration.SavedSettingsLocation.Combine(key).WithExtension(Ext.Json); } + public MainSettings GetAppSettings(IServiceProvider provider, string name) + { + var settings = Task.Run(() => Load(name)).Result; + if (settings.Upgrade()) + { + Save(MainSettings.SettingsFileName, settings).FireAndForget(); + } + + return settings; + } + public async Task Save(string key, T value) { var tmp = GetPath(key).WithExtension(Ext.Temp); From e8fa2be6b7adc7970d89f9869e184d4fa52a49b7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 14:03:18 -0500 Subject: [PATCH 10/26] Replaced references to concrete SettingsManager with interface Deduplicated two more DI method across OS Integrated and Launcher --- .../View Models/Compilers/CompilerVM.cs | 4 +- .../View Models/Gallery/ModListGalleryVM.cs | 8 ++-- .../View Models/Installers/InstallerVM.cs | 6 +-- .../View Models/Settings/SettingsVM.cs | 4 +- Wabbajack.Launcher/Program.cs | 31 ++----------- .../Configuration.cs | 2 +- .../ResourceSettingsManager.cs | 4 +- .../ServiceExtensions.cs | 44 +++++++++++++------ .../SettingsManager.cs | 20 +++------ 9 files changed, 53 insertions(+), 70 deletions(-) diff --git a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs index f514aea9f..4d586e0b7 100644 --- a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs @@ -46,7 +46,7 @@ public class CompilerVM : BackNavigatingVM, ICpuStatusVM { private const string LastSavedCompilerSettings = "last-saved-compiler-settings"; private readonly DTOSerializer _dtos; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly ResourceMonitor _resourceMonitor; @@ -106,7 +106,7 @@ public class CompilerVM : BackNavigatingVM, ICpuStatusVM [Reactive] public ErrorResponse ErrorState { get; private set; } - public CompilerVM(ILogger logger, DTOSerializer dtos, SettingsManager settingsManager, + public CompilerVM(ILogger logger, DTOSerializer dtos, ISettingsManager settingsManager, IServiceProvider serviceProvider, LogStream loggerProvider, ResourceMonitor resourceMonitor, CompilerSettingsInferencer inferencer, Client wjClient, IEnumerable logins, DownloadDispatcher downloadDispatcher) : base(logger) { diff --git a/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs b/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs index 48045dcf9..202278f6d 100644 --- a/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs +++ b/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs @@ -1,6 +1,4 @@ - - -using System; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -77,13 +75,13 @@ public GameTypeEntry SelectedGameTypeEntry private readonly ILogger _logger; private readonly GameLocator _locator; private readonly ModListDownloadMaintainer _maintainer; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly CancellationToken _cancellationToken; public ICommand ClearFiltersCommand { get; set; } public ModListGalleryVM(ILogger logger, Client wjClient, GameLocator locator, - SettingsManager settingsManager, ModListDownloadMaintainer maintainer, CancellationToken cancellationToken) + ISettingsManager settingsManager, ModListDownloadMaintainer maintainer, CancellationToken cancellationToken) : base(logger) { _wjClient = wjClient; diff --git a/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs b/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs index 99918e1bc..41ff8b755 100644 --- a/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs @@ -15,7 +15,6 @@ using System.Threading; using System.Threading.Tasks; using System.Windows.Shell; -using System.Windows.Threading; using Microsoft.Extensions.Logging; using Microsoft.WindowsAPICodePack.Dialogs; using Wabbajack.Common; @@ -34,7 +33,6 @@ using Wabbajack.Paths.IO; using Wabbajack.Services.OSIntegrated; using Wabbajack.Util; -using System.Windows.Forms; using Microsoft.Extensions.DependencyInjection; using Wabbajack.CLI.Verbs; using Wabbajack.VFS; @@ -114,7 +112,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM private readonly DTOSerializer _dtos; private readonly ILogger _logger; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly IServiceProvider _serviceProvider; private readonly SystemParametersConstructor _parametersConstructor; private readonly IGameLocator _gameLocator; @@ -156,7 +154,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM public ReactiveCommand VerifyCommand { get; } - public InstallerVM(ILogger logger, DTOSerializer dtos, SettingsManager settingsManager, IServiceProvider serviceProvider, + public InstallerVM(ILogger logger, DTOSerializer dtos, ISettingsManager settingsManager, IServiceProvider serviceProvider, SystemParametersConstructor parametersConstructor, IGameLocator gameLocator, LogStream loggerProvider, ResourceMonitor resourceMonitor, Wabbajack.Services.OSIntegrated.Configuration configuration, HttpClient client, DownloadDispatcher dispatcher, IEnumerable logins, CancellationToken cancellationToken) : base(logger) diff --git a/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs b/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs index c203212a5..06f5641ab 100644 --- a/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs +++ b/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs @@ -24,7 +24,7 @@ namespace Wabbajack public class SettingsVM : BackNavigatingVM { private readonly Configuration.MainSettings _settings; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; public LoginManagerVM Login { get; } public PerformanceSettingsViewModel Performance { get; } @@ -36,7 +36,7 @@ public SettingsVM(ILogger logger, IServiceProvider provider) : base(logger) { _settings = provider.GetRequiredService(); - _settingsManager = provider.GetRequiredService(); + _settingsManager = provider.GetRequiredService(); Login = new LoginManagerVM(provider.GetRequiredService>(), this, provider.GetRequiredService>()); diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index a577ffda9..8fe905a96 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -1,12 +1,10 @@ using System; using System.Net.Http; -using System.Runtime.InteropServices; using Avalonia; using Avalonia.ReactiveUI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Wabbajack.Configuration; using Wabbajack.Downloaders.Http; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; @@ -14,8 +12,6 @@ using Wabbajack.Launcher.ViewModels; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.Services.OSIntegrated; using Wabbajack.Services.OSIntegrated.TokenProviders; @@ -38,36 +34,17 @@ public static void Main(string[] args) services.AddDTOConverters(); services.AddDTOSerializer(); services.AddSettings(); - - services.AddSingleton(s => new Services.OSIntegrated.Configuration - { - EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), - ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), - SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), - LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), - ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") - }); + services.AddKnownFoldersConfiguration(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); - services.AddSingleton(); + services.AddSingleton(); services.AddAllSingleton>(s => new Resource("Web Requests", 4)); services.AddHttpDownloader(); - var version = - $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; - services.AddSingleton(s => new ApplicationInfo - { - ApplicationSlug = "Wabbajack", - ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack", - ApplicationSha = ThisAssembly.Git.Sha, - Platform = RuntimeInformation.ProcessArchitecture.ToString(), - OperatingSystemDescription = RuntimeInformation.OSDescription, - RuntimeIdentifier = RuntimeInformation.RuntimeIdentifier, - OSVersion = Environment.OSVersion.VersionString, - Version = version - }); + var version = services.AddApplicationInfo(); + }).Build(); Services = host.Services; diff --git a/Wabbajack.Services.OSIntegrated/Configuration.cs b/Wabbajack.Services.OSIntegrated/Configuration.cs index 7adf650bf..9aa8dd595 100644 --- a/Wabbajack.Services.OSIntegrated/Configuration.cs +++ b/Wabbajack.Services.OSIntegrated/Configuration.cs @@ -1,4 +1,5 @@ using Wabbajack.Paths; +using Wabbajack.Paths.IO; namespace Wabbajack.Services.OSIntegrated; @@ -8,6 +9,5 @@ public class Configuration public AbsolutePath SavedSettingsLocation { get; set; } public AbsolutePath EncryptedDataLocation { get; set; } public AbsolutePath LogLocation { get; set; } - public AbsolutePath ImageCacheLocation { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs b/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs index 55c669f13..80165f21e 100644 --- a/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs @@ -7,10 +7,10 @@ namespace Wabbajack.Services.OSIntegrated; public class ResourceSettingsManager { - private readonly SettingsManager _manager; + private readonly ISettingsManager _manager; private Dictionary? _settings; - public ResourceSettingsManager(SettingsManager manager) + public ResourceSettingsManager(ISettingsManager manager) { _manager = manager; } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 89c67d8ea..fd15fd712 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.Linq; using System.Net.Http; @@ -8,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using ProtoBuf.Meta; using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; @@ -112,15 +114,7 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service // Settings - service.AddSingleton(s => new Configuration - { - EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), - ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), - SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), - LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), - ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") - }); - + service.AddKnownFoldersConfiguration(); service.AddSettings(); // Resources @@ -208,10 +202,16 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service service.AddScoped(); service.AddSingleton(); - // Application Info + var version = service.AddApplicationInfo(); + + return service; + } + + public static string AddApplicationInfo(this IServiceCollection services) + { var version = $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; - service.AddSingleton(s => new ApplicationInfo + services.AddSingleton(s => new ApplicationInfo { ApplicationSlug = "Wabbajack", ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack", @@ -223,14 +223,30 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service Version = version }); + return version; + } + public static IServiceCollection AddKnownFoldersConfiguration(this IServiceCollection services) + { + var savedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"); + savedSettingsLocation.CreateDirectory(); - return service; + services.AddSingleton(s => new Configuration + { + EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), + ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), + SavedSettingsLocation = savedSettingsLocation, + LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), + ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") + }); + + return services; } - + public static IServiceCollection AddSettings(this IServiceCollection services) { - services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(); + services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); services.AddSingleton(); return services; } diff --git a/Wabbajack.Services.OSIntegrated/SettingsManager.cs b/Wabbajack.Services.OSIntegrated/SettingsManager.cs index 81d6f9728..dc52dd4af 100644 --- a/Wabbajack.Services.OSIntegrated/SettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/SettingsManager.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Configuration; @@ -13,20 +12,15 @@ namespace Wabbajack.Services.OSIntegrated; -public class SettingsManager +public interface ISettingsManager { - private readonly Configuration _configuration; - private readonly DTOSerializer _dtos; - private readonly ILogger _logger; - - public SettingsManager(ILogger logger, Configuration configuration, DTOSerializer dtos) - { - _logger = logger; - _dtos = dtos; - _configuration = configuration; - _configuration.SavedSettingsLocation.CreateDirectory(); - } + Task Save(string key, T value); + Task Load(string key) where T : new(); + MainSettings GetAppSettings(IServiceProvider provider, string name); +} +internal class SettingsManager(ILogger _logger, Configuration _configuration, DTOSerializer _dtos) : ISettingsManager +{ private AbsolutePath GetPath(string key) { return _configuration.SavedSettingsLocation.Combine(key).WithExtension(Ext.Json); From 76cd4dc8c03a3998ea06441619e432a64fb97e24 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 14:03:41 -0500 Subject: [PATCH 11/26] Removed two unnecessary usings --- Wabbajack.Services.OSIntegrated/ServiceExtensions.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index fd15fd712..dd599af60 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.ComponentModel; using System.Diagnostics; using System.Linq; using System.Net.Http; @@ -9,7 +8,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using ProtoBuf.Meta; using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; From f0d8b64364a0e6ea174b7d002d11bd8fe1d226a7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 16:46:24 -0500 Subject: [PATCH 12/26] Fixed DownloadClientFactory to compare file size correctly --- Wabbajack.Downloader.Clients/DownloadClientFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index 4cc228683..f58180ecc 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -20,7 +20,7 @@ public class DownloadClientFactory(MainSettings _settings, ILoggerFactory _logge public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) { - if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB) + if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB * 1024 * 1024) { return new ResumableDownloadClient(msg, outputPath, job, _settings.MaximumMemoryPerDownloadThreadInMB, _resumableDownloaderLogger); } From 66545cdbfa12d552e9b716aad1bc820348cd50c2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 17:05:38 -0500 Subject: [PATCH 13/26] One more cleanup pass --- Wabbajack.App.Wpf/Settings.cs | 28 +++++++++++----------- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Configuration/MainSettings.cs | 14 +++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index ac7c207e1..c380bfae5 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -18,8 +18,8 @@ public class Mo2ModlistInstallationSettings public class PerformanceSettingsViewModel : ViewModel { private readonly Configuration.MainSettings _settings; - private readonly int _defaultMaximumMemoryPerDownloadThreadMb; - private readonly long _defaultMinimumFileSizeForResumableDownload; + private readonly int _defaultMaximumMemoryPerDownloadThreadMB; + private readonly long _defaultMinimumFileSizeForResumableDownloadMB; public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { @@ -27,10 +27,10 @@ public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResour _settings = settings; // Split half of available memory among download threads - _defaultMaximumMemoryPerDownloadThreadMb = (int)(p.SystemMemorySize / downloadResources.MaxTasks / 1024 / 1024) / 2; - _defaultMinimumFileSizeForResumableDownload = long.MaxValue; - _maximumMemoryPerDownloadThreadMb = settings.MaximumMemoryPerDownloadThreadInMB; - _minimumFileSizeForResumableDownload = settings.MinimumFileSizeForResumableDownloadMB; + _defaultMaximumMemoryPerDownloadThreadMB = (int)(p.SystemMemorySize / downloadResources.MaxTasks / 1024 / 1024) / 2; + _defaultMinimumFileSizeForResumableDownloadMB = long.MaxValue; + _maximumMemoryPerDownloadThreadMB = settings.MaximumMemoryPerDownloadThreadInMB; + _minimumFileSizeForResumableDownloadMB = settings.MinimumFileSizeForResumableDownloadMB; if (MaximumMemoryPerDownloadThreadMb < 0) { @@ -43,37 +43,37 @@ public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResour } } - private int _maximumMemoryPerDownloadThreadMb; - private long _minimumFileSizeForResumableDownload; + private int _maximumMemoryPerDownloadThreadMB; + private long _minimumFileSizeForResumableDownloadMB; public int MaximumMemoryPerDownloadThreadMb { - get => _maximumMemoryPerDownloadThreadMb; + get => _maximumMemoryPerDownloadThreadMB; set { - RaiseAndSetIfChanged(ref _maximumMemoryPerDownloadThreadMb, value); + RaiseAndSetIfChanged(ref _maximumMemoryPerDownloadThreadMB, value); _settings.MaximumMemoryPerDownloadThreadInMB = value; } } public long MinimumFileSizeForResumableDownload { - get => _minimumFileSizeForResumableDownload; + get => _minimumFileSizeForResumableDownloadMB; set { - RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownload, value); + RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownloadMB, value); _settings.MinimumFileSizeForResumableDownloadMB = value; } } public void ResetMaximumMemoryPerDownloadThreadMb() { - MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMb; + MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMB; } public void ResetMinimumFileSizeForResumableDownload() { - MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownload; + MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownloadMB; } } } diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 6043cb1de..5c0aa72be 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - Exe + WinExe net9.0-windows true x64 diff --git a/Wabbajack.Configuration/MainSettings.cs b/Wabbajack.Configuration/MainSettings.cs index 0006f61a0..c9b866183 100644 --- a/Wabbajack.Configuration/MainSettings.cs +++ b/Wabbajack.Configuration/MainSettings.cs @@ -11,13 +11,13 @@ public class MainSettings public int MaximumMemoryPerDownloadThreadInMB { - get => Performance.MaximumMemoryPerDownloadThreadMb; - set => Performance.MaximumMemoryPerDownloadThreadMb = value; + get => Performance.MaximumMemoryPerDownloadThreadMB; + set => Performance.MaximumMemoryPerDownloadThreadMB = value; } public long MinimumFileSizeForResumableDownloadMB { - get => Performance.MinimumFileSizeForResumableDownload; - set => Performance.MinimumFileSizeForResumableDownload = value; + get => Performance.MinimumFileSizeForResumableDownloadMB; + set => Performance.MinimumFileSizeForResumableDownloadMB = value; } private const int SettingsVersion = 1; @@ -36,7 +36,7 @@ public bool Upgrade() if (CurrentSettingsVersion < 1) { - Performance.MaximumMemoryPerDownloadThreadMb = -1; + Performance.MaximumMemoryPerDownloadThreadMB = -1; } CurrentSettingsVersion = SettingsVersion; @@ -45,7 +45,7 @@ public bool Upgrade() internal class PerformanceSettings { - public int MaximumMemoryPerDownloadThreadMb { get; set; } = -1; - public long MinimumFileSizeForResumableDownload { get; set; } = -1; + public int MaximumMemoryPerDownloadThreadMB { get; set; } = -1; + public long MinimumFileSizeForResumableDownloadMB { get; set; } = -1; } } \ No newline at end of file From 5ff915ca8eb4db20d465c82d458524b9b9c6c471 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 16:43:09 -0500 Subject: [PATCH 14/26] Replace usage of ResumableDownloader with NonReusableDownloader for files under an arbitrary size NonReusableDownloader leverages the Downloader library for executing downloads. This presents some performance overhead when downloading large number files, as is often done when downloading larger modlists. Moved these Downloader classes to the Downloader folder in solution to better align with solution structure. Added service extension for DI Cleaned up impacted files Removed unusued method from ResumableDownloader --- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.CLI/Program.cs | 7 +- .../PerformanceSettings.cs | 2 + Wabbajack.DTOs/Archive.cs | 13 +- .../DownloadClientFactory.cs | 34 ++++ .../DownloaderService.cs | 36 ++++ .../NonResumableDownloadClient.cs | 35 ++++ .../ResumableDownloadClient.cs | 45 ++--- .../ServiceExtensions.cs | 16 ++ .../Wabbajack.Downloader.Services.csproj | 25 +++ .../IDownloadClient.cs | 10 ++ Wabbajack.Installer/AInstaller.cs | 40 +++-- Wabbajack.Launcher/Program.cs | 8 +- .../ServiceExtensions.cs | 12 -- .../SingleThreadedDownloader.cs | 162 ------------------ .../Wabbajack.Networking.Http.csproj | 1 + .../ServiceExtensions.cs | 6 +- .../Wabbajack.Services.OSIntegrated.csproj | 2 + Wabbajack.sln | 9 +- 19 files changed, 228 insertions(+), 237 deletions(-) create mode 100644 Wabbajack.Downloader.Clients/DownloadClientFactory.cs create mode 100644 Wabbajack.Downloader.Clients/DownloaderService.cs create mode 100644 Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs rename Wabbajack.Networking.Http/ResumableDownloader.cs => Wabbajack.Downloader.Clients/ResumableDownloadClient.cs (80%) create mode 100644 Wabbajack.Downloader.Clients/ServiceExtensions.cs create mode 100644 Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj create mode 100644 Wabbajack.Downloaders.Interfaces/IDownloadClient.cs delete mode 100644 Wabbajack.Networking.Http/ServiceExtensions.cs delete mode 100644 Wabbajack.Networking.Http/SingleThreadedDownloader.cs diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 5c0aa72be..6043cb1de 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - WinExe + Exe net9.0-windows true x64 diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 94208892b..370037726 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -1,7 +1,6 @@ using System; using System.CommandLine; using System.CommandLine.IO; -using System.Net.Http; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -11,14 +10,13 @@ using NLog.Targets; using Octokit; using Wabbajack.DTOs.Interventions; -using Wabbajack.Networking.Http; -using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Paths.IO; using Wabbajack.Server.Lib; using Wabbajack.Services.OSIntegrated; using Wabbajack.VFS; using Client = Wabbajack.Networking.GitHub.Client; using Wabbajack.CLI.Builder; +using Wabbajack.Downloader.Clients; namespace Wabbajack.CLI; @@ -31,8 +29,7 @@ private static async Task Main(string[] args) .ConfigureServices((host, services) => { services.AddSingleton(new JsonSerializerOptions()); - services.AddSingleton(); - services.AddSingleton(); + services.AddDownloaderService(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs index 93dff2406..64dd6d8aa 100644 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ b/Wabbajack.Configuration/PerformanceSettings.cs @@ -3,4 +3,6 @@ public class PerformanceSettings { public int MaximumMemoryPerDownloadThreadMb { get; set; } + + public long MinimumFileSizeForResumableDownload { get; set; } = (long)1024 * 1024 * 500; // 500MB } \ No newline at end of file diff --git a/Wabbajack.DTOs/Archive.cs b/Wabbajack.DTOs/Archive.cs index 4cf9906d7..b6fe20ec9 100644 --- a/Wabbajack.DTOs/Archive.cs +++ b/Wabbajack.DTOs/Archive.cs @@ -1,13 +1,24 @@ +using System; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; namespace Wabbajack.DTOs; -public class Archive +public class Archive : IComparable { public Hash Hash { get; set; } public string Meta { get; set; } = ""; public string Name { get; set; } public long Size { get; set; } public IDownloadState State { get; set; } + + public int CompareTo(object obj) + { + if (obj == null) return 1; + Archive otherArchive = obj as Archive; + if (otherArchive != null) + return this.Size.CompareTo(otherArchive.Size); + else + throw new ArgumentException("Object is not an Archive"); + } } \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs new file mode 100644 index 000000000..c85c12333 --- /dev/null +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Configuration; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.Paths; +using Wabbajack.RateLimiter; + +namespace Wabbajack.Downloader.Clients; + +public interface IDownloadClientFactory +{ + public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job); +} + +public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory +{ + private readonly ILogger _nonResuableDownloaderLogger = _loggerFactory.CreateLogger(); + private readonly ILogger _resumableDownloaderLogger = _loggerFactory.CreateLogger(); + + private NonResumableDownloadClient? _nonReusableDownloader = default; + + public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) + { + if (job.Size >= _performanceSettings.MinimumFileSizeForResumableDownload) + { + return new ResumableDownloadClient(msg, outputPath, job, _performanceSettings, _resumableDownloaderLogger); + } + else + { + _nonReusableDownloader ??= new NonResumableDownloadClient(msg, outputPath, _nonResuableDownloaderLogger, _httpClientFactory); + + return new NonResumableDownloadClient(msg, outputPath, _nonResuableDownloaderLogger, _httpClientFactory); + } + } +} diff --git a/Wabbajack.Downloader.Clients/DownloaderService.cs b/Wabbajack.Downloader.Clients/DownloaderService.cs new file mode 100644 index 000000000..a4ce17368 --- /dev/null +++ b/Wabbajack.Downloader.Clients/DownloaderService.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.Http.Interfaces; +using Wabbajack.Paths; +using Wabbajack.RateLimiter; + +namespace Wabbajack.Downloader.Clients; + +public class DownloaderService(ILogger _logger, IDownloadClientFactory _httpDownloaderFactory) : IHttpDownloader +{ + public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, + CancellationToken token) + { + Exception downloadError = null!; + + var downloader = _httpDownloaderFactory.GetDownloader(message, outputPath, job); + + for (var i = 0; i < 3; i++) + { + try + { + return await downloader.Download(token, 3); + } + catch (Exception ex) + { + downloadError = ex; + _logger.LogDebug("Download for '{name}' failed. Retrying...", outputPath.FileName.ToString()); + } + } + + _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); + return new Hash(); + + + } +} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs new file mode 100644 index 000000000..00d7ea74e --- /dev/null +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Logging; +using Wabbajack.Downloaders.Interfaces; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; + +namespace Wabbajack.Downloader.Clients; + +internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, ILogger _logger, IHttpClientFactory _httpClientFactory) : IDownloadClient +{ + public async Task Download(CancellationToken token, int retry = 3) + { + try + { + var httpClient = _httpClientFactory.CreateClient("SmallFilesClient"); + var response = await httpClient.GetStreamAsync(_msg.RequestUri!.ToString()); + await using var fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await response.CopyToAsync(fileStream, token); + fileStream.Close(); + await using var file = _outputPath.Open(FileMode.Open); + return await file.Hash(token); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download '{name}' after 3 tries.", _outputPath.FileName.ToString()); + + if (retry <= 3) + { + return await Download(token, retry--); + } + + return new Hash(); + } + } +} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/ResumableDownloader.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs similarity index 80% rename from Wabbajack.Networking.Http/ResumableDownloader.cs rename to Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index 078de77d4..30e1c1da7 100644 --- a/Wabbajack.Networking.Http/ResumableDownloader.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -1,10 +1,5 @@ -using System; -using System.ComponentModel; -using System.IO; -using System.Net.Http; +using System.ComponentModel; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Configuration; @@ -12,32 +7,18 @@ using Wabbajack.Paths; using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; +using Wabbajack.Networking.Http; +using Wabbajack.Downloaders.Interfaces; -namespace Wabbajack.Networking.Http; +namespace Wabbajack.Downloader.Clients; -internal class ResumableDownloader +internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient { - private readonly IJob _job; - private readonly HttpRequestMessage _msg; - private readonly AbsolutePath _outputPath; - private readonly AbsolutePath _packagePath; - private readonly PerformanceSettings _performanceSettings; - private readonly ILogger _logger; private CancellationToken _token; private Exception? _error; + private AbsolutePath _packagePath = _outputPath.WithExtension(Extension.FromPath(".download_package")); - - public ResumableDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job, PerformanceSettings performanceSettings, ILogger logger) - { - _job = job; - _msg = msg; - _outputPath = outputPath; - _packagePath = outputPath.WithExtension(Extension.FromPath(".download_package")); - _performanceSettings = performanceSettings; - _logger = logger; - } - - public async Task Download(CancellationToken token) + public async Task Download(CancellationToken token, int retry = 0) { _token = token; @@ -80,17 +61,17 @@ public async Task Download(CancellationToken token) } else { - _logger.LogError(_error,"Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); + _logger.LogError(_error, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); } throw _error; - } - if (downloader.Status == DownloadStatus.Completed) - { - DeletePackage(); + if (downloader.Status == DownloadStatus.Completed) + { + DeletePackage(); + } } - + if (!_outputPath.FileExists()) { return new Hash(); diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs new file mode 100644 index 000000000..5ee53cfad --- /dev/null +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.DependencyInjection; +using Wabbajack.Configuration; +using Wabbajack.Networking.Http.Interfaces; + +namespace Wabbajack.Downloader.Clients; + +public static class ServiceExtensions +{ + public static void AddDownloaderService(this IServiceCollection services) + { + services.AddHttpClient("SmallFilesClient").ConfigureHttpClient(c => c.Timeout = TimeSpan.FromMinutes(5)); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } +} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj b/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj new file mode 100644 index 000000000..56b94732b --- /dev/null +++ b/Wabbajack.Downloader.Clients/Wabbajack.Downloader.Services.csproj @@ -0,0 +1,25 @@ + + + + net9.0 + enable + enable + + + + + + + + + + + + + + + + + + + diff --git a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs new file mode 100644 index 000000000..445fe05fa --- /dev/null +++ b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs @@ -0,0 +1,10 @@ +using System.Threading; +using System.Threading.Tasks; +using Wabbajack.Hashing.xxHash64; + +namespace Wabbajack.Downloaders.Interfaces; + +public interface IDownloadClient +{ + public Task Download(CancellationToken token, int retry = 0); +} diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index c15fb76f9..e6cfaeb8d 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Drawing.Text; using System.IO; using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders; @@ -25,6 +27,7 @@ using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.VFS; +using YamlDotNet.Core.Tokens; namespace Wabbajack.Installer; @@ -370,31 +373,34 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - + await missing - .Shuffle() .Where(a => a.State is not Manual) + .Shuffle() .PDoAll(async archive => { - _logger.LogInformation("Downloading {Archive}", archive.Name); - var outputPath = _configuration.Downloads.Combine(archive.Name); - var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); - - if (download) - if (outputPath.FileExists() && !downloadPackagePath.FileExists()) - { - var origName = Path.GetFileNameWithoutExtension(archive.Name); - var ext = Path.GetExtension(archive.Name); - var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); - outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); - outputPath.Delete(); - } - - var hash = await DownloadArchive(archive, download, token, outputPath); + await DownloadArchiveAsync(archive, token, download); UpdateProgress(1); }); } + private async Task DownloadArchiveAsync(Archive archive, CancellationToken token, bool download) + { + _logger.LogInformation("Downloading {Archive}", archive.Name); + var outputPath = _configuration.Downloads.Combine(archive.Name); + var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + //if (download) + //if (outputPath.FileExists() && !downloadPackagePath.FileExists()) + //{ + // var origName = Path.GetFileNameWithoutExtension(archive.Name); + // var ext = Path.GetExtension(archive.Name); + // var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); + // outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); + // outputPath.Delete(); + //} + var hash = await DownloadArchive(archive, download, token, outputPath); + } + private async Task SendDownloadMetrics(List missing) { var grouped = missing.GroupBy(m => m.State.GetType()); diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index 1f5e898b2..536cf0451 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -52,15 +52,15 @@ public static void Main(string[] args) services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); - services.AddSingleton(); + services.AddSingleton(); services.AddAllSingleton>(s => new Resource("Web Requests", 4)); - services.AddAllSingleton(); - + services.AddHttpDownloader(); + var version = $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; services.AddSingleton(s => new ApplicationInfo diff --git a/Wabbajack.Networking.Http/ServiceExtensions.cs b/Wabbajack.Networking.Http/ServiceExtensions.cs deleted file mode 100644 index 42a9796e7..000000000 --- a/Wabbajack.Networking.Http/ServiceExtensions.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Wabbajack.Networking.Http.Interfaces; - -namespace Wabbajack.Networking.Http; - -public static class ServiceExtensions -{ - public static void AddHttpDownloader(this IServiceCollection services) - { - services.AddSingleton(); - } -} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs b/Wabbajack.Networking.Http/SingleThreadedDownloader.cs deleted file mode 100644 index ed88e3bba..000000000 --- a/Wabbajack.Networking.Http/SingleThreadedDownloader.cs +++ /dev/null @@ -1,162 +0,0 @@ -using System; -using System.Buffers; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Wabbajack.Configuration; -using Wabbajack.Hashing.xxHash64; -using Wabbajack.Networking.Http.Interfaces; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; -using Wabbajack.RateLimiter; - -namespace Wabbajack.Networking.Http; - -public class SingleThreadedDownloader : IHttpDownloader -{ - private readonly HttpClient _client; - private readonly ILogger _logger; - private readonly PerformanceSettings _settings; - - public SingleThreadedDownloader(ILogger logger, HttpClient client, MainSettings settings) - { - _logger = logger; - _client = client; - _settings = settings.PerformanceSettings; - } - - public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, - CancellationToken token) - { - Exception downloadError = null!; - var downloader = new ResumableDownloader(message, outputPath, job, _settings, _logger); - for (var i = 0; i < 3; i++) - { - try - { - return await downloader.Download(token); - } - catch (Exception ex) - { - downloadError = ex; - _logger.LogDebug("Download for '{name}' failed. Retrying...", outputPath.FileName.ToString()); - } - } - - _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); - return new Hash(); - - // using var response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); - // if (!response.IsSuccessStatusCode) - // throw new HttpException(response); - // - // if (job.Size == 0) - // job.Size = response.Content.Headers.ContentLength ?? 0; - // - // /* Need to make this mulitthreaded to be much use - // if ((response.Content.Headers.ContentLength ?? 0) != 0 && - // response.Headers.AcceptRanges.FirstOrDefault() == "bytes") - // { - // return await ResettingDownloader(response, message, outputPath, job, token); - // } - // */ - // - // await using var stream = await response.Content.ReadAsStreamAsync(token); - // await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write); - // return await stream.HashingCopy(outputStream, token, job); - } - - private const int CHUNK_SIZE = 1024 * 1024 * 8; - - private async Task ResettingDownloader(HttpResponseMessage response, HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) - { - - using var rented = MemoryPool.Shared.Rent(CHUNK_SIZE); - var buffer = rented.Memory; - - var hasher = new xxHashAlgorithm(0); - - var running = true; - ulong finalHash = 0; - - var inputStream = await response.Content.ReadAsStreamAsync(token); - await using var outputStream = outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); - long writePosition = 0; - - while (running && !token.IsCancellationRequested) - { - var totalRead = 0; - - while (totalRead != buffer.Length) - { - var read = await inputStream.ReadAsync(buffer.Slice(totalRead, buffer.Length - totalRead), - token); - - - if (read == 0) - { - running = false; - break; - } - - if (job != null) - await job.Report(read, token); - - totalRead += read; - } - - var pendingWrite = outputStream.WriteAsync(buffer[..totalRead], token); - if (running) - { - hasher.TransformByteGroupsInternal(buffer.Span); - await pendingWrite; - } - else - { - var preSize = (totalRead >> 5) << 5; - if (preSize > 0) - { - hasher.TransformByteGroupsInternal(buffer[..preSize].Span); - finalHash = hasher.FinalizeHashValueInternal(buffer[preSize..totalRead].Span); - await pendingWrite; - break; - } - - finalHash = hasher.FinalizeHashValueInternal(buffer[..totalRead].Span); - await pendingWrite; - break; - } - - { - writePosition += totalRead; - if (job != null) - await job.Report(totalRead, token); - message = CloneMessage(message); - message.Headers.Range = new RangeHeaderValue(writePosition, writePosition + CHUNK_SIZE); - await inputStream.DisposeAsync(); - response.Dispose(); - response = await _client.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, token); - HttpException.ThrowOnFailure(response); - inputStream = await response.Content.ReadAsStreamAsync(token); - } - } - - await outputStream.FlushAsync(token); - - return new Hash(finalHash); - } - - private HttpRequestMessage CloneMessage(HttpRequestMessage message) - { - var newMsg = new HttpRequestMessage(message.Method, message.RequestUri); - foreach (var header in message.Headers) - { - newMsg.Headers.Add(header.Key, header.Value); - } - return newMsg; - } -} \ No newline at end of file diff --git a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj index 595ab3d1b..4f158d080 100644 --- a/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj +++ b/Wabbajack.Networking.Http/Wabbajack.Networking.Http.csproj @@ -9,6 +9,7 @@ + diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index a2c3c5181..bc89e9785 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -11,6 +11,7 @@ using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; +using Wabbajack.Downloader.Clients; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.Downloaders.ModDB; @@ -23,7 +24,6 @@ using Wabbajack.Installer; using Wabbajack.Networking.BethesdaNet; using Wabbajack.Networking.Discord; -using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; using Wabbajack.Networking.Steam; @@ -157,7 +157,9 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service // Networking service.AddSingleton(); - service.AddAllSingleton(); + + // Downloader + service.AddDownloaderService(); service.AddSteam(); diff --git a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj index 55e7bfb54..39b248fc8 100644 --- a/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj +++ b/Wabbajack.Services.OSIntegrated/Wabbajack.Services.OSIntegrated.csproj @@ -18,10 +18,12 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/Wabbajack.sln b/Wabbajack.sln index bedde649f..642f5ab20 100644 --- a/Wabbajack.sln +++ b/Wabbajack.sln @@ -108,8 +108,8 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".solutionItems", ".solutionItems", "{109037C8-CF2F-4179-B064-A66147BC18C5}" ProjectSection(SolutionItems) = preProject .gitignore = .gitignore - nuget.config = nuget.config CHANGELOG.md = CHANGELOG.md + nuget.config = nuget.config README.md = README.md EndProjectSection EndProject @@ -147,6 +147,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloaders.Verif EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Configuration", "Wabbajack.Configuration\Wabbajack.Configuration.csproj", "{E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wabbajack.Downloader.Services", "Wabbajack.Downloader.Clients\Wabbajack.Downloader.Services.csproj", "{258D44F2-956F-43A3-BD29-11A28D03F406}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -401,6 +403,10 @@ Global {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Debug|Any CPU.Build.0 = Debug|Any CPU {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Release|Any CPU.ActiveCfg = Release|Any CPU {E7CDACA6-D3FF-4CF6-8EF8-05FCD27F6FBE}.Release|Any CPU.Build.0 = Release|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Debug|Any CPU.Build.0 = Debug|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Release|Any CPU.ActiveCfg = Release|Any CPU + {258D44F2-956F-43A3-BD29-11A28D03F406}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -451,6 +457,7 @@ Global {7FC4F129-F0FA-46B7-B7C4-532E371A6326} = {98B731EE-4FC0-4482-A069-BCBA25497871} {E4BDB22D-11A4-452F-8D10-D9CA9777EA22} = {F677890D-5109-43BC-97C7-C4CD47C8EE0C} {D9560C73-4E58-4463-9DB9-D06491E0E1C8} = {98B731EE-4FC0-4482-A069-BCBA25497871} + {258D44F2-956F-43A3-BD29-11A28D03F406} = {98B731EE-4FC0-4482-A069-BCBA25497871} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AA30275-0F38-4A7D-B645-F5505178DDE8} From 4120d41920c8cd2a762e75089286ca4a64a9614c Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 16:44:19 -0500 Subject: [PATCH 15/26] Reverted unneeded change to AInstaller --- Wabbajack.Installer/AInstaller.cs | 32 +++++++++++++------------------ 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index e6cfaeb8d..3f42903ba 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -379,28 +379,22 @@ await missing .Shuffle() .PDoAll(async archive => { - await DownloadArchiveAsync(archive, token, download); - UpdateProgress(1); + _logger.LogInformation("Downloading {Archive}", archive.Name); + var outputPath = _configuration.Downloads.Combine(archive.Name); + var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + if (download) + if (outputPath.FileExists() && !downloadPackagePath.FileExists()) + { + var origName = Path.GetFileNameWithoutExtension(archive.Name); + var ext = Path.GetExtension(archive.Name); + var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); + outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); + outputPath.Delete(); + } + var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); }); } - private async Task DownloadArchiveAsync(Archive archive, CancellationToken token, bool download) - { - _logger.LogInformation("Downloading {Archive}", archive.Name); - var outputPath = _configuration.Downloads.Combine(archive.Name); - var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); - //if (download) - //if (outputPath.FileExists() && !downloadPackagePath.FileExists()) - //{ - // var origName = Path.GetFileNameWithoutExtension(archive.Name); - // var ext = Path.GetExtension(archive.Name); - // var uniqueKey = archive.State.PrimaryKeyString.StringSha256Hex(); - // outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); - // outputPath.Delete(); - //} - var hash = await DownloadArchive(archive, download, token, outputPath); - } - private async Task SendDownloadMetrics(List missing) { var grouped = missing.GroupBy(m => m.State.GetType()); From d6ec4b25279f91d5641682e565d0d21ef17068e2 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 17:06:14 -0500 Subject: [PATCH 16/26] Added more explicit error logging --- .../NonResumableDownloadClient.cs | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs index 00d7ea74e..db4fd5817 100644 --- a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -10,26 +10,53 @@ internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath { public async Task Download(CancellationToken token, int retry = 3) { + Stream? fileStream; + try { + fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not open file path '{filePath}'. Throwing...", _outputPath.FileName.ToString()); + + throw; + } + + try + { + _logger.LogDebug("Download for '{name}' is starting from scratch...", _outputPath.FileName.ToString()); + var httpClient = _httpClientFactory.CreateClient("SmallFilesClient"); var response = await httpClient.GetStreamAsync(_msg.RequestUri!.ToString()); - await using var fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); await response.CopyToAsync(fileStream, token); fileStream.Close(); - await using var file = _outputPath.Open(FileMode.Open); - return await file.Hash(token); + } catch (Exception ex) { - _logger.LogError(ex, "Failed to download '{name}' after 3 tries.", _outputPath.FileName.ToString()); - if (retry <= 3) { + _logger.LogError(ex, "Download for '{name}' encountered error. Retrying...", _outputPath.FileName.ToString()); + return await Download(token, retry--); } - return new Hash(); + _logger.LogError(ex, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); + + throw; + } + + try + { + await using var file = _outputPath.Open(FileMode.Open); + return await file.Hash(token); + } + catch (Exception ex) + { + _logger.LogError(ex, "Could not hash file '{filePath}'. Throwing...", _outputPath.FileName.ToString()); + + throw; } } } \ No newline at end of file From 6fc715a670fa1258c979b0748a1e8d33aadf602f Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 17:17:02 -0500 Subject: [PATCH 17/26] Synced Namespaces Cleaned up code Removed retry login in NonResumableDownload as DownloadService already retries --- Wabbajack.CLI/Program.cs | 2 +- Wabbajack.Downloader.Clients/DownloadClientFactory.cs | 2 +- Wabbajack.Downloader.Clients/DownloaderService.cs | 10 ++++------ .../NonResumableDownloadClient.cs | 11 ++--------- .../ResumableDownloadClient.cs | 8 ++++---- Wabbajack.Downloader.Clients/ServiceExtensions.cs | 2 +- Wabbajack.Downloaders.Interfaces/IDownloadClient.cs | 2 +- Wabbajack.Services.OSIntegrated/ServiceExtensions.cs | 2 +- 8 files changed, 15 insertions(+), 24 deletions(-) diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 370037726..bb13ed675 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -16,7 +16,7 @@ using Wabbajack.VFS; using Client = Wabbajack.Networking.GitHub.Client; using Wabbajack.CLI.Builder; -using Wabbajack.Downloader.Clients; +using Wabbajack.Downloader.Services; namespace Wabbajack.CLI; diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index c85c12333..ce3467a78 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -4,7 +4,7 @@ using Wabbajack.Paths; using Wabbajack.RateLimiter; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public interface IDownloadClientFactory { diff --git a/Wabbajack.Downloader.Clients/DownloaderService.cs b/Wabbajack.Downloader.Clients/DownloaderService.cs index a4ce17368..15de71491 100644 --- a/Wabbajack.Downloader.Clients/DownloaderService.cs +++ b/Wabbajack.Downloader.Clients/DownloaderService.cs @@ -4,12 +4,11 @@ using Wabbajack.Paths; using Wabbajack.RateLimiter; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public class DownloaderService(ILogger _logger, IDownloadClientFactory _httpDownloaderFactory) : IHttpDownloader { - public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, - CancellationToken token) + public async Task Download(HttpRequestMessage message, AbsolutePath outputPath, IJob job, CancellationToken token) { Exception downloadError = null!; @@ -19,7 +18,7 @@ public async Task Download(HttpRequestMessage message, AbsolutePath output { try { - return await downloader.Download(token, 3); + return await downloader.Download(token); } catch (Exception ex) { @@ -29,8 +28,7 @@ public async Task Download(HttpRequestMessage message, AbsolutePath output } _logger.LogError(downloadError, "Failed to download '{name}' after 3 tries.", outputPath.FileName.ToString()); - return new Hash(); - + return new Hash(); } } \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs index db4fd5817..3b0203d66 100644 --- a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -4,11 +4,11 @@ using Wabbajack.Paths; using Wabbajack.Paths.IO; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, ILogger _logger, IHttpClientFactory _httpClientFactory) : IDownloadClient { - public async Task Download(CancellationToken token, int retry = 3) + public async Task Download(CancellationToken token) { Stream? fileStream; @@ -35,13 +35,6 @@ public async Task Download(CancellationToken token, int retry = 3) } catch (Exception ex) { - if (retry <= 3) - { - _logger.LogError(ex, "Download for '{name}' encountered error. Retrying...", _outputPath.FileName.ToString()); - - return await Download(token, retry--); - } - _logger.LogError(ex, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); throw; diff --git a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index 30e1c1da7..ecdd5ac9d 100644 --- a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -10,7 +10,7 @@ using Wabbajack.Networking.Http; using Wabbajack.Downloaders.Interfaces; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient { @@ -18,7 +18,7 @@ internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _ou private Exception? _error; private AbsolutePath _packagePath = _outputPath.WithExtension(Extension.FromPath(".download_package")); - public async Task Download(CancellationToken token, int retry = 0) + public async Task Download(CancellationToken token) { _token = token; @@ -71,7 +71,7 @@ public async Task Download(CancellationToken token, int retry = 0) DeletePackage(); } } - + if (!_outputPath.FileExists()) { return new Hash(); @@ -93,7 +93,7 @@ private DownloadConfiguration CreateConfiguration(HttpRequestMessage message) { Headers = message.Headers.ToWebHeaderCollection(), ProtocolVersion = message.Version, - UserAgent = message.Headers.UserAgent.ToString() + UserAgent = message.Headers.UserAgent.ToString() } }; diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs index 5ee53cfad..d361b4513 100644 --- a/Wabbajack.Downloader.Clients/ServiceExtensions.cs +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -2,7 +2,7 @@ using Wabbajack.Configuration; using Wabbajack.Networking.Http.Interfaces; -namespace Wabbajack.Downloader.Clients; +namespace Wabbajack.Downloader.Services; public static class ServiceExtensions { diff --git a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs index 445fe05fa..759105bb8 100644 --- a/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs +++ b/Wabbajack.Downloaders.Interfaces/IDownloadClient.cs @@ -6,5 +6,5 @@ namespace Wabbajack.Downloaders.Interfaces; public interface IDownloadClient { - public Task Download(CancellationToken token, int retry = 0); + public Task Download(CancellationToken token); } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index bc89e9785..d60d39c04 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -11,7 +11,7 @@ using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; -using Wabbajack.Downloader.Clients; +using Wabbajack.Downloader.Services; using Wabbajack.Downloaders; using Wabbajack.Downloaders.GameFile; using Wabbajack.Downloaders.ModDB; From c87b3029bcc6d2b721fac5e1db6a92da32bdf9ee Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:27:01 -0500 Subject: [PATCH 18/26] Adding UI for setting minimum file size at which to use resumable downloader --- Wabbajack.App.Wpf/Settings.cs | 27 ++++++++++++++++++- .../Settings/PerformanceSettingsView.xaml | 24 +++++++++++++++++ .../Settings/PerformanceSettingsView.xaml.cs | 13 +++++++++ .../PerformanceSettings.cs | 2 +- 4 files changed, 64 insertions(+), 2 deletions(-) diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index 4ad1517c3..887b57169 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -1,4 +1,5 @@ -using Wabbajack.Downloaders; +using SteamKit2.GC.Dota.Internal; +using Wabbajack.Downloaders; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Paths; using Wabbajack.RateLimiter; @@ -18,6 +19,7 @@ public class PerformanceSettings : ViewModel { private readonly Configuration.MainSettings _settings; private readonly int _defaultMaximumMemoryPerDownloadThreadMb; + private readonly long _defaultMinimumFileSizeForResumableDownload; public PerformanceSettings(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { @@ -26,15 +28,23 @@ public PerformanceSettings(Configuration.MainSettings settings, IResource _minimumFileSizeForResumableDownload; + set + { + RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownload, value); + _settings.PerformanceSettings.MinimumFileSizeForResumableDownload = value; + } + } + public void ResetMaximumMemoryPerDownloadThreadMb() { MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMb; } + + public void ResetMinimumFileSizeForResumableDownload() + { + MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownload; + } } } diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml index d976f2b94..f2a6ebc2e 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml @@ -71,6 +71,30 @@ HorizontalAlignment="Left"> Reset + + + diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs index d2a0ee5c4..142f0d911 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml.cs @@ -21,16 +21,29 @@ public PerformanceSettingsView() x => x.MaximumMemoryPerDownloadThreadMb, x => x.MaximumMemoryPerDownloadThreadIntegerUpDown.Value) .DisposeWith(disposable); + + this.BindStrict( + ViewModel, + x => x.MinimumFileSizeForResumableDownload, + x => x.MinimumFileSizeForResumableDownloadIntegerUpDown.Value) + .DisposeWith(disposable); + this.EditResourceSettings.Command = ReactiveCommand.Create(() => { UIUtils.OpenFile( KnownFolders.WabbajackAppLocal.Combine("saved_settings", "resource_settings.json")); Environment.Exit(0); }); + ResetMaximumMemoryPerDownloadThread.Command = ReactiveCommand.Create(() => { ViewModel.ResetMaximumMemoryPerDownloadThreadMb(); }); + + ResetMinimumFileSizeForResumableDownload.Command = ReactiveCommand.Create(() => + { + ViewModel.ResetMinimumFileSizeForResumableDownload(); + }); }); } } diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs index 64dd6d8aa..adefc1e27 100644 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ b/Wabbajack.Configuration/PerformanceSettings.cs @@ -4,5 +4,5 @@ public class PerformanceSettings { public int MaximumMemoryPerDownloadThreadMb { get; set; } - public long MinimumFileSizeForResumableDownload { get; set; } = (long)1024 * 1024 * 500; // 500MB + public long MinimumFileSizeForResumableDownload { get; set; } = 0; } \ No newline at end of file From 19c75837e30567b27f8b581052fa98c9eeffde6d Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:34:26 -0500 Subject: [PATCH 19/26] Reverting unintended changes --- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Installer/AInstaller.cs | 82 +++++++++++----------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 6043cb1de..5c0aa72be 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - Exe + WinExe net9.0-windows true x64 diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 3f42903ba..efb4bdd00 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Drawing.Text; using System.IO; using System.IO.Compression; using System.Linq; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Downloader; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Downloaders; @@ -27,7 +25,6 @@ using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.VFS; -using YamlDotNet.Core.Tokens; namespace Wabbajack.Installer; @@ -226,7 +223,7 @@ public async Task InstallArchives(CancellationToken token) NextStep(Consts.StepInstalling, "Installing files", ModList.Directives.Sum(d => d.Size), x => x.ToFileSizeString()); var grouped = ModList.Directives .OfType() - .Select(a => new {VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) + .Select(a => new { VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a }) .GroupBy(a => a.VF) .ToDictionary(a => a.Key); @@ -247,27 +244,27 @@ await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) => switch (file) { case PatchedFromArchive pfa: - { - await using var s = await sf.GetStream(); - s.Position = 0; - await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { - await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); - ThrowOnNonMatchingHash(file, hash); + await using var s = await sf.GetStream(); + s.Position = 0; + await using var patchDataStream = await InlinedFileStream(pfa.PatchID); + { + await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); + } } - } break; case TransformedTexture tt: - { - await using var s = await sf.GetStream(); - await using var of = destPath.Open(FileMode.Create, FileAccess.Write); - _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); - await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, - of, token); - } + { + await using var s = await sf.GetStream(); + await using var of = destPath.Open(FileMode.Create, FileAccess.Write); + _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); + await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, + of, token); + } break; @@ -290,7 +287,7 @@ await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.Im } await FileHashCache.FileHashWriteCache(destPath, file.Hash); - await job.Report((int) directive.VF.Size, token); + await job.Report((int)directive.VF.Size, token); } }, token); } @@ -305,8 +302,8 @@ private void ThrowNonMatchingError(Directive file, Hash gotHash) _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); } - - + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) { if (hash == directive.Hash) return; @@ -335,14 +332,14 @@ public async Task DownloadArchives(CancellationToken token) { var matches = mirrors[archive.Hash].ToArray(); if (!matches.Any()) continue; - + archive.State = matches.First().State; _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString()); _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name, matches.First().State.PrimaryKeyString); } - - + + foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { @@ -375,13 +372,14 @@ public async Task DownloadMissingArchives(List missing, CancellationTok } await missing - .Where(a => a.State is not Manual) .Shuffle() + .Where(a => a.State is not Manual) .PDoAll(async archive => { _logger.LogInformation("Downloading {Archive}", archive.Name); var outputPath = _configuration.Downloads.Combine(archive.Name); var downloadPackagePath = outputPath.WithExtension(Ext.DownloadPackage); + if (download) if (outputPath.FileExists() && !downloadPackagePath.FileExists()) { @@ -391,7 +389,9 @@ await missing outputPath = _configuration.Downloads.Combine(origName + "_" + uniqueKey + "_" + ext); outputPath.Delete(); } - var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); + + var hash = await DownloadArchive(archive, download, token, outputPath); + UpdateProgress(1); }); } @@ -502,7 +502,7 @@ protected async Task OptimizeModlist(CancellationToken token) _logger.LogInformation("Optimizing ModList directives"); UnoptimizedArchives = ModList.Archives; UnoptimizedDirectives = ModList.Directives; - + var indexed = ModList.Directives.ToDictionary(d => d.To); var bsasToBuild = await ModList.Directives @@ -537,11 +537,11 @@ FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuil var profileFolder = _configuration.Install.Combine("profiles"); - var savePath = (RelativePath) "saves"; + var savePath = (RelativePath)"saves"; NextStep(Consts.StepPreparing, "Looking for files to delete", 0); await _configuration.Install.EnumerateFiles() - .PMapAllBatched(_limiter, f => + .PMapAllBatched(_limiter, f => { var relativeTo = f.RelativeTo(_configuration.Install); if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) @@ -570,7 +570,7 @@ await _configuration.Install.EnumerateFiles() { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] - var split = ((string) path.RelativeTo(_configuration.Install)).Split('\\'); + var split = ((string)path.RelativeTo(_configuration.Install)).Split('\\'); return Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))); }) .Distinct() @@ -598,15 +598,15 @@ await _configuration.Install.EnumerateFiles() NextStep(Consts.StepPreparing, "Looking for unmodified files", 0); await indexed.Values.PMapAllBatchedAsync(_limiter, async d => - { - // Bit backwards, but we want to return null for - // all files we *want* installed. We return the files - // to remove from the install list. - var path = _configuration.Install.Combine(d.To); - if (!existingfiles.Contains(path)) return null; - - return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; - }) + { + // Bit backwards, but we want to return null for + // all files we *want* installed. We return the files + // to remove from the install list. + var path = _configuration.Install.Combine(d.To); + if (!existingfiles.Contains(path)) return null; + + return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; + }) .Do(d => { if (d != null) @@ -621,7 +621,7 @@ await indexed.Values.PMapAllBatchedAsync(_limiter, async d => .GroupBy(d => d.ArchiveHashPath.Hash) .Select(d => d.Key) .ToHashSet(); - + ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); } From 2fcad645521d00bdf7d732a9532e4e9a7f5790dc Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 27 Dec 2024 22:38:05 -0500 Subject: [PATCH 20/26] Reverting unneeded changes --- Wabbajack.DTOs/Archive.cs | 13 +----- Wabbajack.Installer/AInstaller.cs | 74 +++++++++++++++---------------- 2 files changed, 38 insertions(+), 49 deletions(-) diff --git a/Wabbajack.DTOs/Archive.cs b/Wabbajack.DTOs/Archive.cs index b6fe20ec9..4cf9906d7 100644 --- a/Wabbajack.DTOs/Archive.cs +++ b/Wabbajack.DTOs/Archive.cs @@ -1,24 +1,13 @@ -using System; using Wabbajack.DTOs.DownloadStates; using Wabbajack.Hashing.xxHash64; namespace Wabbajack.DTOs; -public class Archive : IComparable +public class Archive { public Hash Hash { get; set; } public string Meta { get; set; } = ""; public string Name { get; set; } public long Size { get; set; } public IDownloadState State { get; set; } - - public int CompareTo(object obj) - { - if (obj == null) return 1; - Archive otherArchive = obj as Archive; - if (otherArchive != null) - return this.Size.CompareTo(otherArchive.Size); - else - throw new ArgumentException("Object is not an Archive"); - } } \ No newline at end of file diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index efb4bdd00..c15fb76f9 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -223,7 +223,7 @@ public async Task InstallArchives(CancellationToken token) NextStep(Consts.StepInstalling, "Installing files", ModList.Directives.Sum(d => d.Size), x => x.ToFileSizeString()); var grouped = ModList.Directives .OfType() - .Select(a => new { VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a }) + .Select(a => new {VF = _vfs.Index.FileForArchiveHashPath(a.ArchiveHashPath), Directive = a}) .GroupBy(a => a.VF) .ToDictionary(a => a.Key); @@ -244,27 +244,27 @@ await _vfs.Extract(grouped.Keys.ToHashSet(), async (vf, sf) => switch (file) { case PatchedFromArchive pfa: + { + await using var s = await sf.GetStream(); + s.Position = 0; + await using var patchDataStream = await InlinedFileStream(pfa.PatchID); { - await using var s = await sf.GetStream(); - s.Position = 0; - await using var patchDataStream = await InlinedFileStream(pfa.PatchID); - { - await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); - var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); - ThrowOnNonMatchingHash(file, hash); - } + await using var os = destPath.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None); + var hash = await BinaryPatching.ApplyPatch(s, patchDataStream, os); + ThrowOnNonMatchingHash(file, hash); } + } break; case TransformedTexture tt: - { - await using var s = await sf.GetStream(); - await using var of = destPath.Open(FileMode.Create, FileAccess.Write); - _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); - await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, - of, token); - } + { + await using var s = await sf.GetStream(); + await using var of = destPath.Open(FileMode.Create, FileAccess.Write); + _logger.LogInformation("Recompressing {Filename}", tt.To.FileName); + await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.ImageState.MipLevels, tt.ImageState.Format, + of, token); + } break; @@ -287,7 +287,7 @@ await ImageLoader.Recompress(s, tt.ImageState.Width, tt.ImageState.Height, tt.Im } await FileHashCache.FileHashWriteCache(destPath, file.Hash); - await job.Report((int)directive.VF.Size, token); + await job.Report((int) directive.VF.Size, token); } }, token); } @@ -302,8 +302,8 @@ private void ThrowNonMatchingError(Directive file, Hash gotHash) _logger.LogError("Hashes for {Path} did not match, expected {Expected} got {Got}", file.To, file.Hash, gotHash); throw new Exception($"Hashes for {file.To} did not match, expected {file.Hash} got {gotHash}"); } - - + + protected void ThrowOnNonMatchingHash(CreateBSA bsa, Directive directive, AFile state, Hash hash) { if (hash == directive.Hash) return; @@ -332,14 +332,14 @@ public async Task DownloadArchives(CancellationToken token) { var matches = mirrors[archive.Hash].ToArray(); if (!matches.Any()) continue; - + archive.State = matches.First().State; _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString()); _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name, matches.First().State.PrimaryKeyString); } - - + + foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { @@ -370,7 +370,7 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - + await missing .Shuffle() .Where(a => a.State is not Manual) @@ -502,7 +502,7 @@ protected async Task OptimizeModlist(CancellationToken token) _logger.LogInformation("Optimizing ModList directives"); UnoptimizedArchives = ModList.Archives; UnoptimizedDirectives = ModList.Directives; - + var indexed = ModList.Directives.ToDictionary(d => d.To); var bsasToBuild = await ModList.Directives @@ -537,11 +537,11 @@ FromArchive a when a.To.StartsWith($"{Consts.BSACreationDir}") => !bsasToNotBuil var profileFolder = _configuration.Install.Combine("profiles"); - var savePath = (RelativePath)"saves"; + var savePath = (RelativePath) "saves"; NextStep(Consts.StepPreparing, "Looking for files to delete", 0); await _configuration.Install.EnumerateFiles() - .PMapAllBatched(_limiter, f => + .PMapAllBatched(_limiter, f => { var relativeTo = f.RelativeTo(_configuration.Install); if (indexed.ContainsKey(relativeTo) || f.InFolder(_configuration.Downloads)) @@ -570,7 +570,7 @@ await _configuration.Install.EnumerateFiles() { // Get all the folders and all the folder parents // so for foo\bar\baz\qux.txt this emits ["foo", "foo\\bar", "foo\\bar\\baz"] - var split = ((string)path.RelativeTo(_configuration.Install)).Split('\\'); + var split = ((string) path.RelativeTo(_configuration.Install)).Split('\\'); return Enumerable.Range(1, split.Length - 1).Select(t => string.Join("\\", split.Take(t))); }) .Distinct() @@ -598,15 +598,15 @@ await _configuration.Install.EnumerateFiles() NextStep(Consts.StepPreparing, "Looking for unmodified files", 0); await indexed.Values.PMapAllBatchedAsync(_limiter, async d => - { - // Bit backwards, but we want to return null for - // all files we *want* installed. We return the files - // to remove from the install list. - var path = _configuration.Install.Combine(d.To); - if (!existingfiles.Contains(path)) return null; - - return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; - }) + { + // Bit backwards, but we want to return null for + // all files we *want* installed. We return the files + // to remove from the install list. + var path = _configuration.Install.Combine(d.To); + if (!existingfiles.Contains(path)) return null; + + return await FileHashCache.FileHashCachedAsync(path, token) == d.Hash ? d : null; + }) .Do(d => { if (d != null) @@ -621,7 +621,7 @@ await indexed.Values.PMapAllBatchedAsync(_limiter, async d => .GroupBy(d => d.ArchiveHashPath.Hash) .Select(d => d.Key) .ToHashSet(); - + ModList.Archives = ModList.Archives.Where(a => requiredArchives.Contains(a.Hash)).ToArray(); ModList.Directives = indexed.Values.ToArray(); } From 2d77b125b776c1955ddb58955bab0d3c47f29d22 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 13:26:45 -0500 Subject: [PATCH 21/26] Refactored performance settings and its dependency injection --- Wabbajack.App.Wpf/Settings.cs | 14 ++++---- .../View Models/Settings/SettingsVM.cs | 4 +-- .../Settings/PerformanceSettingsView.xaml | 2 +- .../Settings/PerformanceSettingsView.xaml.cs | 2 +- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Configuration/MainSettings.cs | 32 ++++++++++++++++--- .../PerformanceSettings.cs | 8 ----- .../DownloadClientFactory.cs | 6 ++-- .../ResumableDownloadClient.cs | 6 ++-- .../ServiceExtensions.cs | 2 -- Wabbajack.Installer/AInstaller.cs | 20 +++++++++--- Wabbajack.Launcher/Program.cs | 24 ++------------ .../ServiceExtensions.cs | 17 +++------- .../SettingsManager.cs | 13 ++++++++ 14 files changed, 82 insertions(+), 70 deletions(-) delete mode 100644 Wabbajack.Configuration/PerformanceSettings.cs diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index 887b57169..ac7c207e1 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -15,13 +15,13 @@ public class Mo2ModlistInstallationSettings public bool AutomaticallyOverrideExistingInstall { get; set; } } - public class PerformanceSettings : ViewModel + public class PerformanceSettingsViewModel : ViewModel { private readonly Configuration.MainSettings _settings; private readonly int _defaultMaximumMemoryPerDownloadThreadMb; private readonly long _defaultMinimumFileSizeForResumableDownload; - public PerformanceSettings(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) + public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { var p = systemParams.Create(); @@ -29,15 +29,15 @@ public PerformanceSettings(Configuration.MainSettings settings, IResource logger, IServiceProvider provider) AuthorFile = new AuthorFilesVM(provider.GetRequiredService>()!, provider.GetRequiredService()!, provider.GetRequiredService()!, this); OpenTerminalCommand = ReactiveCommand.CreateFromTask(OpenTerminal); - Performance = new PerformanceSettings( + Performance = new PerformanceSettingsViewModel( _settings, provider.GetRequiredService>(), provider.GetRequiredService()); diff --git a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml index f2a6ebc2e..ddc9b950b 100644 --- a/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml +++ b/Wabbajack.App.Wpf/Views/Settings/PerformanceSettingsView.xaml @@ -9,7 +9,7 @@ xmlns:xwpf="http://schemas.xceed.com/wpf/xaml/toolkit" d:DesignHeight="450" d:DesignWidth="800" - x:TypeArguments="local:PerformanceSettings" + x:TypeArguments="local:PerformanceSettingsViewModel" mc:Ignorable="d"> /// Interaction logic for PerformanceSettingsView.xaml /// - public partial class PerformanceSettingsView : ReactiveUserControl + public partial class PerformanceSettingsView : ReactiveUserControl { public PerformanceSettingsView() { diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 5c0aa72be..6043cb1de 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - WinExe + Exe net9.0-windows true x64 diff --git a/Wabbajack.Configuration/MainSettings.cs b/Wabbajack.Configuration/MainSettings.cs index 5b8bf4eb8..0006f61a0 100644 --- a/Wabbajack.Configuration/MainSettings.cs +++ b/Wabbajack.Configuration/MainSettings.cs @@ -1,13 +1,31 @@ -namespace Wabbajack.Configuration; +using System.Text.Json.Serialization; + +namespace Wabbajack.Configuration; public class MainSettings { public const string SettingsFileName = "app_settings"; - private const int SettingsVersion = 1; + [JsonPropertyName("CurrentSettingsVersion")] public int CurrentSettingsVersion { get; set; } - public PerformanceSettings PerformanceSettings { get; set; } = new(); + public int MaximumMemoryPerDownloadThreadInMB + { + get => Performance.MaximumMemoryPerDownloadThreadMb; + set => Performance.MaximumMemoryPerDownloadThreadMb = value; + } + + public long MinimumFileSizeForResumableDownloadMB { + get => Performance.MinimumFileSizeForResumableDownload; + set => Performance.MinimumFileSizeForResumableDownload = value; + } + + private const int SettingsVersion = 1; + + [JsonInclude] + [JsonPropertyName("PerformanceSettings")] + private PerformanceSettings Performance { get; set; } = new(); + public bool Upgrade() { @@ -18,10 +36,16 @@ public bool Upgrade() if (CurrentSettingsVersion < 1) { - PerformanceSettings.MaximumMemoryPerDownloadThreadMb = -1; + Performance.MaximumMemoryPerDownloadThreadMb = -1; } CurrentSettingsVersion = SettingsVersion; return true; } + + internal class PerformanceSettings + { + public int MaximumMemoryPerDownloadThreadMb { get; set; } = -1; + public long MinimumFileSizeForResumableDownload { get; set; } = -1; + } } \ No newline at end of file diff --git a/Wabbajack.Configuration/PerformanceSettings.cs b/Wabbajack.Configuration/PerformanceSettings.cs deleted file mode 100644 index adefc1e27..000000000 --- a/Wabbajack.Configuration/PerformanceSettings.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Wabbajack.Configuration; - -public class PerformanceSettings -{ - public int MaximumMemoryPerDownloadThreadMb { get; set; } - - public long MinimumFileSizeForResumableDownload { get; set; } = 0; -} \ No newline at end of file diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index ce3467a78..4cc228683 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -11,7 +11,7 @@ public interface IDownloadClientFactory public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job); } -public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory +public class DownloadClientFactory(MainSettings _settings, ILoggerFactory _loggerFactory, IHttpClientFactory _httpClientFactory) : IDownloadClientFactory { private readonly ILogger _nonResuableDownloaderLogger = _loggerFactory.CreateLogger(); private readonly ILogger _resumableDownloaderLogger = _loggerFactory.CreateLogger(); @@ -20,9 +20,9 @@ public class DownloadClientFactory(PerformanceSettings _performanceSettings, ILo public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) { - if (job.Size >= _performanceSettings.MinimumFileSizeForResumableDownload) + if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB) { - return new ResumableDownloadClient(msg, outputPath, job, _performanceSettings, _resumableDownloaderLogger); + return new ResumableDownloadClient(msg, outputPath, job, _settings.MaximumMemoryPerDownloadThreadInMB, _resumableDownloaderLogger); } else { diff --git a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs index ecdd5ac9d..dc5a75b46 100644 --- a/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/ResumableDownloadClient.cs @@ -12,7 +12,7 @@ namespace Wabbajack.Downloader.Services; -internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, PerformanceSettings _performanceSettings, ILogger _logger) : IDownloadClient +internal class ResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath _outputPath, IJob _job, int _maxMemoryPerDownloadThread, ILogger _logger) : IDownloadClient { private CancellationToken _token; private Exception? _error; @@ -83,10 +83,10 @@ public async Task Download(CancellationToken token) private DownloadConfiguration CreateConfiguration(HttpRequestMessage message) { - var maximumMemoryPerDownloadThreadMb = Math.Max(0, _performanceSettings.MaximumMemoryPerDownloadThreadMb); + var maximumMemoryPerDownloadThreadMb = Math.Max(0, _maxMemoryPerDownloadThread); var configuration = new DownloadConfiguration { - MaximumMemoryBufferBytes = maximumMemoryPerDownloadThreadMb * 1024 * 1024, + MaximumMemoryBufferBytes = maximumMemoryPerDownloadThreadMb, Timeout = (int)TimeSpan.FromSeconds(120).TotalMilliseconds, ReserveStorageSpaceBeforeStartingDownload = true, RequestConfiguration = new RequestConfiguration diff --git a/Wabbajack.Downloader.Clients/ServiceExtensions.cs b/Wabbajack.Downloader.Clients/ServiceExtensions.cs index d361b4513..d00e1fef6 100644 --- a/Wabbajack.Downloader.Clients/ServiceExtensions.cs +++ b/Wabbajack.Downloader.Clients/ServiceExtensions.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Wabbajack.Configuration; using Wabbajack.Networking.Http.Interfaces; namespace Wabbajack.Downloader.Services; @@ -9,7 +8,6 @@ public static class ServiceExtensions public static void AddDownloaderService(this IServiceCollection services) { services.AddHttpClient("SmallFilesClient").ConfigureHttpClient(c => c.Timeout = TimeSpan.FromMinutes(5)); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); } diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index c15fb76f9..bad41e058 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -370,11 +370,16 @@ public async Task DownloadMissingArchives(List missing, CancellationTok UpdateProgress(1); } } - - await missing - .Shuffle() + + var missingBatches = missing .Where(a => a.State is not Manual) - .PDoAll(async archive => + .Batch(100) + .ToList(); + + List batchTasks = []; + foreach (var batch in missingBatches) + { + batchTasks.Add(batch.PDoAll(async archive => { _logger.LogInformation("Downloading {Archive}", archive.Name); var outputPath = _configuration.Downloads.Combine(archive.Name); @@ -392,7 +397,12 @@ await missing var hash = await DownloadArchive(archive, download, token, outputPath); UpdateProgress(1); - }); + })); + + await Task.Delay(TimeSpan.FromSeconds(10)); // Hitting a Nexus API limit when spinning these downloads up too fast. Need to slow this down. + } + + await Task.WhenAll(batchTasks); } private async Task SendDownloadMetrics(List missing) diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index 536cf0451..a577ffda9 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -1,20 +1,17 @@ using System; using System.Net.Http; using System.Runtime.InteropServices; -using System.Threading.Tasks; using Avalonia; using Avalonia.ReactiveUI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Wabbajack.Common; using Wabbajack.Configuration; using Wabbajack.Downloaders.Http; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; using Wabbajack.DTOs.Logins; using Wabbajack.Launcher.ViewModels; -using Wabbajack.Networking.Http; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; using Wabbajack.Paths; @@ -40,7 +37,8 @@ public static void Main(string[] args) services.AddNexusApi(); services.AddDTOConverters(); services.AddDTOSerializer(); - + services.AddSettings(); + services.AddSingleton(s => new Services.OSIntegrated.Configuration { EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), @@ -49,11 +47,7 @@ public static void Main(string[] args) LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") }); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); - + services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); @@ -81,18 +75,6 @@ public static void Main(string[] args) .StartWithClassicDesktopLifetime(args); } - private static MainSettings GetAppSettings(IServiceProvider provider, string name) - { - var settingsManager = provider.GetRequiredService(); - var settings = Task.Run(() => settingsManager.Load(name)).Result; - if (settings.Upgrade()) - { - settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); - } - - return settings; - } - public static IServiceProvider Services { get; set; } // Avalonia configuration, don't remove; also used by visual designer. diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index d60d39c04..89c67d8ea 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -121,9 +121,7 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") }); - service.AddSingleton(); - service.AddSingleton(); - service.AddSingleton(s => GetAppSettings(s, MainSettings.SettingsFileName)); + service.AddSettings(); // Resources @@ -230,16 +228,11 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service return service; } - public static MainSettings GetAppSettings(IServiceProvider provider, string name) + public static IServiceCollection AddSettings(this IServiceCollection services) { - var settingsManager = provider.GetRequiredService(); - var settings = Task.Run(() => settingsManager.Load(name)).Result; - if (settings.Upgrade()) - { - settingsManager.Save(MainSettings.SettingsFileName, settings).FireAndForget(); - } - - return settings; + services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(); + return services; } private static void CleanAllTempData(AbsolutePath path) diff --git a/Wabbajack.Services.OSIntegrated/SettingsManager.cs b/Wabbajack.Services.OSIntegrated/SettingsManager.cs index ca04ff765..81d6f9728 100644 --- a/Wabbajack.Services.OSIntegrated/SettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/SettingsManager.cs @@ -3,8 +3,10 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.Common; +using Wabbajack.Configuration; using Wabbajack.DTOs.JsonConverters; using Wabbajack.Paths; using Wabbajack.Paths.IO; @@ -30,6 +32,17 @@ private AbsolutePath GetPath(string key) return _configuration.SavedSettingsLocation.Combine(key).WithExtension(Ext.Json); } + public MainSettings GetAppSettings(IServiceProvider provider, string name) + { + var settings = Task.Run(() => Load(name)).Result; + if (settings.Upgrade()) + { + Save(MainSettings.SettingsFileName, settings).FireAndForget(); + } + + return settings; + } + public async Task Save(string key, T value) { var tmp = GetPath(key).WithExtension(Ext.Temp); From 7c73b2428cbc12ac0ed248f9a86b8be06cc7822b Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 14:03:18 -0500 Subject: [PATCH 22/26] Replaced references to concrete SettingsManager with interface Deduplicated two more DI method across OS Integrated and Launcher --- .../View Models/Compilers/CompilerVM.cs | 4 +- .../View Models/Gallery/ModListGalleryVM.cs | 8 ++-- .../View Models/Installers/InstallerVM.cs | 6 +-- .../View Models/Settings/SettingsVM.cs | 4 +- Wabbajack.Launcher/Program.cs | 31 ++----------- .../Configuration.cs | 2 +- .../ResourceSettingsManager.cs | 4 +- .../ServiceExtensions.cs | 44 +++++++++++++------ .../SettingsManager.cs | 20 +++------ 9 files changed, 53 insertions(+), 70 deletions(-) diff --git a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs index f514aea9f..4d586e0b7 100644 --- a/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Compilers/CompilerVM.cs @@ -46,7 +46,7 @@ public class CompilerVM : BackNavigatingVM, ICpuStatusVM { private const string LastSavedCompilerSettings = "last-saved-compiler-settings"; private readonly DTOSerializer _dtos; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly ResourceMonitor _resourceMonitor; @@ -106,7 +106,7 @@ public class CompilerVM : BackNavigatingVM, ICpuStatusVM [Reactive] public ErrorResponse ErrorState { get; private set; } - public CompilerVM(ILogger logger, DTOSerializer dtos, SettingsManager settingsManager, + public CompilerVM(ILogger logger, DTOSerializer dtos, ISettingsManager settingsManager, IServiceProvider serviceProvider, LogStream loggerProvider, ResourceMonitor resourceMonitor, CompilerSettingsInferencer inferencer, Client wjClient, IEnumerable logins, DownloadDispatcher downloadDispatcher) : base(logger) { diff --git a/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs b/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs index 48045dcf9..202278f6d 100644 --- a/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs +++ b/Wabbajack.App.Wpf/View Models/Gallery/ModListGalleryVM.cs @@ -1,6 +1,4 @@ - - -using System; +using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; @@ -77,13 +75,13 @@ public GameTypeEntry SelectedGameTypeEntry private readonly ILogger _logger; private readonly GameLocator _locator; private readonly ModListDownloadMaintainer _maintainer; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly CancellationToken _cancellationToken; public ICommand ClearFiltersCommand { get; set; } public ModListGalleryVM(ILogger logger, Client wjClient, GameLocator locator, - SettingsManager settingsManager, ModListDownloadMaintainer maintainer, CancellationToken cancellationToken) + ISettingsManager settingsManager, ModListDownloadMaintainer maintainer, CancellationToken cancellationToken) : base(logger) { _wjClient = wjClient; diff --git a/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs b/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs index 99918e1bc..41ff8b755 100644 --- a/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs +++ b/Wabbajack.App.Wpf/View Models/Installers/InstallerVM.cs @@ -15,7 +15,6 @@ using System.Threading; using System.Threading.Tasks; using System.Windows.Shell; -using System.Windows.Threading; using Microsoft.Extensions.Logging; using Microsoft.WindowsAPICodePack.Dialogs; using Wabbajack.Common; @@ -34,7 +33,6 @@ using Wabbajack.Paths.IO; using Wabbajack.Services.OSIntegrated; using Wabbajack.Util; -using System.Windows.Forms; using Microsoft.Extensions.DependencyInjection; using Wabbajack.CLI.Verbs; using Wabbajack.VFS; @@ -114,7 +112,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM private readonly DTOSerializer _dtos; private readonly ILogger _logger; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; private readonly IServiceProvider _serviceProvider; private readonly SystemParametersConstructor _parametersConstructor; private readonly IGameLocator _gameLocator; @@ -156,7 +154,7 @@ public class InstallerVM : BackNavigatingVM, IBackNavigatingVM, ICpuStatusVM public ReactiveCommand VerifyCommand { get; } - public InstallerVM(ILogger logger, DTOSerializer dtos, SettingsManager settingsManager, IServiceProvider serviceProvider, + public InstallerVM(ILogger logger, DTOSerializer dtos, ISettingsManager settingsManager, IServiceProvider serviceProvider, SystemParametersConstructor parametersConstructor, IGameLocator gameLocator, LogStream loggerProvider, ResourceMonitor resourceMonitor, Wabbajack.Services.OSIntegrated.Configuration configuration, HttpClient client, DownloadDispatcher dispatcher, IEnumerable logins, CancellationToken cancellationToken) : base(logger) diff --git a/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs b/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs index c203212a5..06f5641ab 100644 --- a/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs +++ b/Wabbajack.App.Wpf/View Models/Settings/SettingsVM.cs @@ -24,7 +24,7 @@ namespace Wabbajack public class SettingsVM : BackNavigatingVM { private readonly Configuration.MainSettings _settings; - private readonly SettingsManager _settingsManager; + private readonly ISettingsManager _settingsManager; public LoginManagerVM Login { get; } public PerformanceSettingsViewModel Performance { get; } @@ -36,7 +36,7 @@ public SettingsVM(ILogger logger, IServiceProvider provider) : base(logger) { _settings = provider.GetRequiredService(); - _settingsManager = provider.GetRequiredService(); + _settingsManager = provider.GetRequiredService(); Login = new LoginManagerVM(provider.GetRequiredService>(), this, provider.GetRequiredService>()); diff --git a/Wabbajack.Launcher/Program.cs b/Wabbajack.Launcher/Program.cs index a577ffda9..8fe905a96 100644 --- a/Wabbajack.Launcher/Program.cs +++ b/Wabbajack.Launcher/Program.cs @@ -1,12 +1,10 @@ using System; using System.Net.Http; -using System.Runtime.InteropServices; using Avalonia; using Avalonia.ReactiveUI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; -using Wabbajack.Configuration; using Wabbajack.Downloaders.Http; using Wabbajack.DTOs; using Wabbajack.DTOs.JsonConverters; @@ -14,8 +12,6 @@ using Wabbajack.Launcher.ViewModels; using Wabbajack.Networking.Http.Interfaces; using Wabbajack.Networking.NexusApi; -using Wabbajack.Paths; -using Wabbajack.Paths.IO; using Wabbajack.RateLimiter; using Wabbajack.Services.OSIntegrated; using Wabbajack.Services.OSIntegrated.TokenProviders; @@ -38,36 +34,17 @@ public static void Main(string[] args) services.AddDTOConverters(); services.AddDTOSerializer(); services.AddSettings(); - - services.AddSingleton(s => new Services.OSIntegrated.Configuration - { - EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), - ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), - SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), - LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), - ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") - }); + services.AddKnownFoldersConfiguration(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton, NexusApiTokenProvider>(); - services.AddSingleton(); + services.AddSingleton(); services.AddAllSingleton>(s => new Resource("Web Requests", 4)); services.AddHttpDownloader(); - var version = - $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; - services.AddSingleton(s => new ApplicationInfo - { - ApplicationSlug = "Wabbajack", - ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack", - ApplicationSha = ThisAssembly.Git.Sha, - Platform = RuntimeInformation.ProcessArchitecture.ToString(), - OperatingSystemDescription = RuntimeInformation.OSDescription, - RuntimeIdentifier = RuntimeInformation.RuntimeIdentifier, - OSVersion = Environment.OSVersion.VersionString, - Version = version - }); + var version = services.AddApplicationInfo(); + }).Build(); Services = host.Services; diff --git a/Wabbajack.Services.OSIntegrated/Configuration.cs b/Wabbajack.Services.OSIntegrated/Configuration.cs index 7adf650bf..9aa8dd595 100644 --- a/Wabbajack.Services.OSIntegrated/Configuration.cs +++ b/Wabbajack.Services.OSIntegrated/Configuration.cs @@ -1,4 +1,5 @@ using Wabbajack.Paths; +using Wabbajack.Paths.IO; namespace Wabbajack.Services.OSIntegrated; @@ -8,6 +9,5 @@ public class Configuration public AbsolutePath SavedSettingsLocation { get; set; } public AbsolutePath EncryptedDataLocation { get; set; } public AbsolutePath LogLocation { get; set; } - public AbsolutePath ImageCacheLocation { get; set; } } \ No newline at end of file diff --git a/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs b/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs index 55c669f13..80165f21e 100644 --- a/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/ResourceSettingsManager.cs @@ -7,10 +7,10 @@ namespace Wabbajack.Services.OSIntegrated; public class ResourceSettingsManager { - private readonly SettingsManager _manager; + private readonly ISettingsManager _manager; private Dictionary? _settings; - public ResourceSettingsManager(SettingsManager manager) + public ResourceSettingsManager(ISettingsManager manager) { _manager = manager; } diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index 89c67d8ea..fd15fd712 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.Diagnostics; using System.Linq; using System.Net.Http; @@ -8,6 +9,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using ProtoBuf.Meta; using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; @@ -112,15 +114,7 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service // Settings - service.AddSingleton(s => new Configuration - { - EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), - ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), - SavedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"), - LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), - ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") - }); - + service.AddKnownFoldersConfiguration(); service.AddSettings(); // Resources @@ -208,10 +202,16 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service service.AddScoped(); service.AddSingleton(); - // Application Info + var version = service.AddApplicationInfo(); + + return service; + } + + public static string AddApplicationInfo(this IServiceCollection services) + { var version = $"{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Major}.{ThisAssembly.Git.SemVer.Patch}{ThisAssembly.Git.SemVer.DashLabel}"; - service.AddSingleton(s => new ApplicationInfo + services.AddSingleton(s => new ApplicationInfo { ApplicationSlug = "Wabbajack", ApplicationName = Environment.ProcessPath?.ToAbsolutePath().FileName.ToString() ?? "Wabbajack", @@ -223,14 +223,30 @@ public static IServiceCollection AddOSIntegrated(this IServiceCollection service Version = version }); + return version; + } + public static IServiceCollection AddKnownFoldersConfiguration(this IServiceCollection services) + { + var savedSettingsLocation = KnownFolders.WabbajackAppLocal.Combine("saved_settings"); + savedSettingsLocation.CreateDirectory(); - return service; + services.AddSingleton(s => new Configuration + { + EncryptedDataLocation = KnownFolders.WabbajackAppLocal.Combine("encrypted"), + ModListsDownloadLocation = KnownFolders.EntryPoint.Combine("downloaded_mod_lists"), + SavedSettingsLocation = savedSettingsLocation, + LogLocation = KnownFolders.LauncherAwarePath.Combine("logs"), + ImageCacheLocation = KnownFolders.WabbajackAppLocal.Combine("image_cache") + }); + + return services; } - + public static IServiceCollection AddSettings(this IServiceCollection services) { - services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); + services.AddSingleton(); + services.AddSingleton(s => s.GetRequiredService().GetAppSettings(s, MainSettings.SettingsFileName)); services.AddSingleton(); return services; } diff --git a/Wabbajack.Services.OSIntegrated/SettingsManager.cs b/Wabbajack.Services.OSIntegrated/SettingsManager.cs index 81d6f9728..dc52dd4af 100644 --- a/Wabbajack.Services.OSIntegrated/SettingsManager.cs +++ b/Wabbajack.Services.OSIntegrated/SettingsManager.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Wabbajack.Common; using Wabbajack.Configuration; @@ -13,20 +12,15 @@ namespace Wabbajack.Services.OSIntegrated; -public class SettingsManager +public interface ISettingsManager { - private readonly Configuration _configuration; - private readonly DTOSerializer _dtos; - private readonly ILogger _logger; - - public SettingsManager(ILogger logger, Configuration configuration, DTOSerializer dtos) - { - _logger = logger; - _dtos = dtos; - _configuration = configuration; - _configuration.SavedSettingsLocation.CreateDirectory(); - } + Task Save(string key, T value); + Task Load(string key) where T : new(); + MainSettings GetAppSettings(IServiceProvider provider, string name); +} +internal class SettingsManager(ILogger _logger, Configuration _configuration, DTOSerializer _dtos) : ISettingsManager +{ private AbsolutePath GetPath(string key) { return _configuration.SavedSettingsLocation.Combine(key).WithExtension(Ext.Json); From 31626cf17b2a0df2932c213f08adc466a2615469 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 14:03:41 -0500 Subject: [PATCH 23/26] Removed two unnecessary usings --- Wabbajack.Services.OSIntegrated/ServiceExtensions.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs index fd15fd712..dd599af60 100644 --- a/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs +++ b/Wabbajack.Services.OSIntegrated/ServiceExtensions.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.ComponentModel; using System.Diagnostics; using System.Linq; using System.Net.Http; @@ -9,7 +8,6 @@ using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using ProtoBuf.Meta; using Wabbajack.Common; using Wabbajack.Compiler; using Wabbajack.Configuration; From a3bbcbfa78dc499230f2971684a4cc9d5f1cc93a Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 16:46:24 -0500 Subject: [PATCH 24/26] Fixed DownloadClientFactory to compare file size correctly --- Wabbajack.Downloader.Clients/DownloadClientFactory.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs index 4cc228683..f58180ecc 100644 --- a/Wabbajack.Downloader.Clients/DownloadClientFactory.cs +++ b/Wabbajack.Downloader.Clients/DownloadClientFactory.cs @@ -20,7 +20,7 @@ public class DownloadClientFactory(MainSettings _settings, ILoggerFactory _logge public IDownloadClient GetDownloader(HttpRequestMessage msg, AbsolutePath outputPath, IJob job) { - if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB) + if (job.Size >= _settings.MinimumFileSizeForResumableDownloadMB * 1024 * 1024) { return new ResumableDownloadClient(msg, outputPath, job, _settings.MaximumMemoryPerDownloadThreadInMB, _resumableDownloaderLogger); } From 192c34bd6442dac97cac908d028085953f0c93f2 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 17:05:38 -0500 Subject: [PATCH 25/26] One more cleanup pass --- Wabbajack.App.Wpf/Settings.cs | 28 +++++++++++----------- Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj | 2 +- Wabbajack.Configuration/MainSettings.cs | 14 +++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Wabbajack.App.Wpf/Settings.cs b/Wabbajack.App.Wpf/Settings.cs index ac7c207e1..c380bfae5 100644 --- a/Wabbajack.App.Wpf/Settings.cs +++ b/Wabbajack.App.Wpf/Settings.cs @@ -18,8 +18,8 @@ public class Mo2ModlistInstallationSettings public class PerformanceSettingsViewModel : ViewModel { private readonly Configuration.MainSettings _settings; - private readonly int _defaultMaximumMemoryPerDownloadThreadMb; - private readonly long _defaultMinimumFileSizeForResumableDownload; + private readonly int _defaultMaximumMemoryPerDownloadThreadMB; + private readonly long _defaultMinimumFileSizeForResumableDownloadMB; public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResource downloadResources, SystemParametersConstructor systemParams) { @@ -27,10 +27,10 @@ public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResour _settings = settings; // Split half of available memory among download threads - _defaultMaximumMemoryPerDownloadThreadMb = (int)(p.SystemMemorySize / downloadResources.MaxTasks / 1024 / 1024) / 2; - _defaultMinimumFileSizeForResumableDownload = long.MaxValue; - _maximumMemoryPerDownloadThreadMb = settings.MaximumMemoryPerDownloadThreadInMB; - _minimumFileSizeForResumableDownload = settings.MinimumFileSizeForResumableDownloadMB; + _defaultMaximumMemoryPerDownloadThreadMB = (int)(p.SystemMemorySize / downloadResources.MaxTasks / 1024 / 1024) / 2; + _defaultMinimumFileSizeForResumableDownloadMB = long.MaxValue; + _maximumMemoryPerDownloadThreadMB = settings.MaximumMemoryPerDownloadThreadInMB; + _minimumFileSizeForResumableDownloadMB = settings.MinimumFileSizeForResumableDownloadMB; if (MaximumMemoryPerDownloadThreadMb < 0) { @@ -43,37 +43,37 @@ public PerformanceSettingsViewModel(Configuration.MainSettings settings, IResour } } - private int _maximumMemoryPerDownloadThreadMb; - private long _minimumFileSizeForResumableDownload; + private int _maximumMemoryPerDownloadThreadMB; + private long _minimumFileSizeForResumableDownloadMB; public int MaximumMemoryPerDownloadThreadMb { - get => _maximumMemoryPerDownloadThreadMb; + get => _maximumMemoryPerDownloadThreadMB; set { - RaiseAndSetIfChanged(ref _maximumMemoryPerDownloadThreadMb, value); + RaiseAndSetIfChanged(ref _maximumMemoryPerDownloadThreadMB, value); _settings.MaximumMemoryPerDownloadThreadInMB = value; } } public long MinimumFileSizeForResumableDownload { - get => _minimumFileSizeForResumableDownload; + get => _minimumFileSizeForResumableDownloadMB; set { - RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownload, value); + RaiseAndSetIfChanged(ref _minimumFileSizeForResumableDownloadMB, value); _settings.MinimumFileSizeForResumableDownloadMB = value; } } public void ResetMaximumMemoryPerDownloadThreadMb() { - MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMb; + MaximumMemoryPerDownloadThreadMb = _defaultMaximumMemoryPerDownloadThreadMB; } public void ResetMinimumFileSizeForResumableDownload() { - MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownload; + MinimumFileSizeForResumableDownload = _defaultMinimumFileSizeForResumableDownloadMB; } } } diff --git a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj index 6043cb1de..5c0aa72be 100644 --- a/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj +++ b/Wabbajack.App.Wpf/Wabbajack.App.Wpf.csproj @@ -1,7 +1,7 @@ - Exe + WinExe net9.0-windows true x64 diff --git a/Wabbajack.Configuration/MainSettings.cs b/Wabbajack.Configuration/MainSettings.cs index 0006f61a0..c9b866183 100644 --- a/Wabbajack.Configuration/MainSettings.cs +++ b/Wabbajack.Configuration/MainSettings.cs @@ -11,13 +11,13 @@ public class MainSettings public int MaximumMemoryPerDownloadThreadInMB { - get => Performance.MaximumMemoryPerDownloadThreadMb; - set => Performance.MaximumMemoryPerDownloadThreadMb = value; + get => Performance.MaximumMemoryPerDownloadThreadMB; + set => Performance.MaximumMemoryPerDownloadThreadMB = value; } public long MinimumFileSizeForResumableDownloadMB { - get => Performance.MinimumFileSizeForResumableDownload; - set => Performance.MinimumFileSizeForResumableDownload = value; + get => Performance.MinimumFileSizeForResumableDownloadMB; + set => Performance.MinimumFileSizeForResumableDownloadMB = value; } private const int SettingsVersion = 1; @@ -36,7 +36,7 @@ public bool Upgrade() if (CurrentSettingsVersion < 1) { - Performance.MaximumMemoryPerDownloadThreadMb = -1; + Performance.MaximumMemoryPerDownloadThreadMB = -1; } CurrentSettingsVersion = SettingsVersion; @@ -45,7 +45,7 @@ public bool Upgrade() internal class PerformanceSettings { - public int MaximumMemoryPerDownloadThreadMb { get; set; } = -1; - public long MinimumFileSizeForResumableDownload { get; set; } = -1; + public int MaximumMemoryPerDownloadThreadMB { get; set; } = -1; + public long MinimumFileSizeForResumableDownloadMB { get; set; } = -1; } } \ No newline at end of file From a3e901625640f7ef1605ac3431e8c68cd404cae7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 28 Dec 2024 21:58:26 -0500 Subject: [PATCH 26/26] Refactored downloader to resume on network failures --- .../NonResumableDownloadClient.cs | 53 ++++++++++++------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs index 3b0203d66..bf2da0a68 100644 --- a/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs +++ b/Wabbajack.Downloader.Clients/NonResumableDownloadClient.cs @@ -1,4 +1,6 @@ using Microsoft.Extensions.Logging; +using System.Net.Http.Headers; +using System.Net.Sockets; using Wabbajack.Downloaders.Interfaces; using Wabbajack.Hashing.xxHash64; using Wabbajack.Paths; @@ -10,46 +12,61 @@ internal class NonResumableDownloadClient(HttpRequestMessage _msg, AbsolutePath { public async Task Download(CancellationToken token) { - Stream? fileStream; + if (_msg.RequestUri == null) + { + throw new ArgumentException("Request URI is null"); + } try { - fileStream = _outputPath.Open(FileMode.Create, FileAccess.Write, FileShare.None); + return await DownloadStreamDirectlyToFile(_msg.RequestUri, token, _outputPath, 5); } catch (Exception ex) { - _logger.LogError(ex, "Could not open file path '{filePath}'. Throwing...", _outputPath.FileName.ToString()); + _logger.LogError(ex, "Failed to download '{name}'", _outputPath.FileName.ToString()); throw; } + } + private async Task DownloadStreamDirectlyToFile(Uri rquestURI, CancellationToken token, AbsolutePath filePath, int retry = 5) + { try { - _logger.LogDebug("Download for '{name}' is starting from scratch...", _outputPath.FileName.ToString()); - var httpClient = _httpClientFactory.CreateClient("SmallFilesClient"); - var response = await httpClient.GetStreamAsync(_msg.RequestUri!.ToString()); + using Stream fileStream = GetFileStream(filePath); + var startingPosition = fileStream.Length; + + _logger.LogDebug("Download for '{name}' is starting from {position}...", _outputPath.FileName.ToString(), startingPosition); + httpClient.DefaultRequestHeaders.Range = new RangeHeaderValue(startingPosition, null); //GetStreamAsync does not accept a HttpRequestMessage so we have to set headers on the client itself + + var response = await httpClient.GetStreamAsync(rquestURI, token); await response.CopyToAsync(fileStream, token); - fileStream.Close(); + return await fileStream.Hash(token); } - catch (Exception ex) + catch (Exception ex) when (ex is SocketException || ex is IOException) { - _logger.LogError(ex, "Download for '{name}' encountered error. Throwing...", _outputPath.FileName.ToString()); + _logger.LogWarning(ex, "Failed to download '{name}' due to network error. Retrying...", _outputPath.FileName.ToString()); - throw; + if(retry == 0) + { + throw; + } + + return await DownloadStreamDirectlyToFile(rquestURI, token, _outputPath, retry--); } + } - try + private Stream GetFileStream(AbsolutePath filePath) + { + if (filePath.FileExists()) { - await using var file = _outputPath.Open(FileMode.Open); - return await file.Hash(token); + return filePath.Open(FileMode.Append, FileAccess.Write, FileShare.None); } - catch (Exception ex) + else { - _logger.LogError(ex, "Could not hash file '{filePath}'. Throwing...", _outputPath.FileName.ToString()); - - throw; + return filePath.Open(FileMode.Create, FileAccess.Write, FileShare.None); } } -} \ No newline at end of file +}