Merge #3645 Improvements for failed repo updates

CHANGELOG.md

@@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.
 - [Multiple] Many improvements for failed downloads (#3635, #3637, #3642 by: HebaruSan; reviewed: techman83)
 - [GUI] Show reverse relationships (#3638 by: HebaruSan; reviewed: techman83)
 - [GUI] Debounce search events on all platforms (#3641 by: HebaruSan; reviewed: techman83)
+- [Multiple] Improvements for failed repo updates (#3645 by: HebaruSan; reviewed: techman83)
 
 ## Bugfixes
 

Cmdline/Action/Update.cs

@@ -40,22 +40,9 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
                 compatible_prior = registry.CompatibleModules(instance.VersionCriteria()).ToList();
             }
 
-            // If no repository is selected, select all.
-            if (options.repo == null)
-            {
-                options.update_all = true;
-            }
-
             try
             {
-                if (options.update_all)
-                {
-                    UpdateRepository(instance);
-                }
-                else
-                {
-                    UpdateRepository(instance, options.repo);
-                }
+                UpdateRepository(instance);
             }
             catch (ReinstallModuleKraken rmk)
             {
@@ -148,13 +135,13 @@ private void PrintModules(string message, IEnumerable<CkanModule> modules)
         /// </summary>
         /// <param name="instance">The KSP instance to work on.</param>
         /// <param name="repository">Repository to update. If null all repositories are used.</param>
-        private void UpdateRepository(CKAN.GameInstance instance, string repository = null)
+        private void UpdateRepository(CKAN.GameInstance instance)
         {
             RegistryManager registry_manager = RegistryManager.Instance(instance);
 
-            var updated = repository == null
-                ? CKAN.Repo.UpdateAllRepositories(registry_manager, instance, manager.Cache, user) != CKAN.RepoUpdateResult.Failed
-                : CKAN.Repo.Update(registry_manager, instance, user, repository);
+            var downloader = new NetAsyncDownloader(user);
+
+            CKAN.Repo.UpdateAllRepositories(registry_manager, instance, downloader, manager.Cache, user);
 
             user.RaiseMessage(Properties.Resources.UpdateSummary, registry_manager.registry.CompatibleModules(instance.VersionCriteria()).Count());
         }

Cmdline/Options.cs

@@ -494,13 +494,6 @@ internal class ConsoleUIOptions : InstanceSpecificOptions
 
     internal class UpdateOptions : InstanceSpecificOptions
     {
-        // This option is really meant for devs testing their CKAN-meta forks.
-        [Option('r', "repo", HelpText = "CKAN repository to use (experimental!)")]
-        public string repo { get; set; }
-
-        [Option("all", DefaultValue = false, HelpText = "Upgrade all available updated modules")]
-        public bool update_all { get; set; }
-
         [Option("list-changes", DefaultValue = false, HelpText = "List new and removed modules")]
         public bool list_changes { get; set; }
     }

ConsoleUI/ModListScreen.cs

@@ -437,9 +437,12 @@ private bool UpdateRegistry(ConsoleTheme theme, bool showNewModsPrompt = true)
                 );
                 recent.Clear();
                 try {
+                    var downloader = new NetAsyncDownloader(ps);
+
                     Repo.UpdateAllRepositories(
                         RegistryManager.Instance(manager.CurrentInstance),
                         manager.CurrentInstance,
+                        downloader,
                         manager.Cache,
                         ps
                     );

Core/Extensions/EnumerableExtensions.cs

@@ -67,6 +67,18 @@ public static void RemoveWhere<K, V>(
         public static TimeSpan Sum(this IEnumerable<TimeSpan> source)
             => source.Aggregate(TimeSpan.Zero,
                                 (a, b) => a + b);
+
+
+        /// <summary>
+        /// Select : SelectMany :: Zip : ZipMany
+        /// </summary>
+        /// <param name="seq1">Sequence from which to get first values</param>
+        /// <param name="seq2">Sequence from which to get second values</param>
+        /// <param name="func">Function to transform a value from each input sequence into a sequence of multiple outputs</param>
+        /// <returns>Flattened sequence of values from func applies to seq1 and seq2</returns>
+        public static IEnumerable<V> ZipMany<T, U, V>(this IEnumerable<T> seq1, IEnumerable<U> seq2, Func<T, U, IEnumerable<V>> func)
+            => seq1.Zip(seq2, func).SelectMany(seqs => seqs);
+
     }
 
     /// <summary>

Core/Net/Net.cs

@@ -180,20 +180,19 @@ public static string DownloadWithProgress(Uri url, string filename = null, IUser
 
         public static void DownloadWithProgress(ICollection<DownloadTarget> downloadTargets, IUser user = null)
         {
-            new NetAsyncDownloader(user ?? new NullUser())
+            var downloader = new NetAsyncDownloader(user ?? new NullUser());
+            downloader.onOneCompleted += (url, filename, error, etag) =>
             {
-                onOneCompleted = (url, filename, error) =>
+                if (error != null)
                 {
-                    if (error != null)
-                    {
-                        user?.RaiseError(error.ToString());
-                    }
-                    else
-                    {
-                        File.Move(filename, downloadTargets.First(p => p.url == url).filename);
-                    }
+                    user?.RaiseError(error.ToString());
                 }
-            }.DownloadAndWait(downloadTargets);
+                else
+                {
+                    File.Move(filename, downloadTargets.First(p => p.url == url).filename);
+                }
+            };
+            downloader.DownloadAndWait(downloadTargets);
         }
 
         /// <summary>

Core/Net/NetAsyncDownloader.cs

@@ -6,10 +6,12 @@
 using System.Net;
 using System.Text.RegularExpressions;
 using System.Threading;
+
 using Autofac;
-using CKAN.Configuration;
 using log4net;
 
+using CKAN.Configuration;
+
 namespace CKAN
 {
     /// <summary>
@@ -30,18 +32,18 @@ private class NetAsyncDownloaderDownloadPart
             public Exception error;
             public int lastProgressUpdateSize;
 
-            public event DownloadProgressChangedEventHandler Progress;
-            public event AsyncCompletedEventHandler          Done;
+            public event DownloadProgressChangedEventHandler             Progress;
+            public event Action<object, AsyncCompletedEventArgs, string> Done;
 
             private string mimeType;
             private WebClient agent;
 
-            public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target, string path = null)
+            public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target)
             {
                 this.target = target;
                 this.mimeType = target.mimeType;
                 this.triedFallback = false;
-                this.path = path ?? Path.GetTempFileName();
+                this.path = target.filename ?? Path.GetTempFileName();
                 this.size = bytesLeft = target.size;
                 this.lastProgressUpdateTime = DateTime.Now;
             }
@@ -72,7 +74,8 @@ private void ResetAgent()
 
                 // Check whether to use an auth token for this host
                 string token;
-                if (ServiceLocator.Container.Resolve<IConfiguration>().TryGetAuthToken(this.target.url.Host, out token)
+                if (this.target.url.IsAbsoluteUri
+                    && ServiceLocator.Container.Resolve<IConfiguration>().TryGetAuthToken(this.target.url.Host, out token)
                         && !string.IsNullOrEmpty(token))
                 {
                     log.InfoFormat("Using auth token for {0}", this.target.url.Host);
@@ -81,17 +84,16 @@ private void ResetAgent()
                 }
 
                 // Forward progress and completion events to our listeners
-                agent.DownloadProgressChanged += (sender, args) => {
-                    if (Progress != null)
-                    {
-                        Progress(sender, args);
-                    }
+                agent.DownloadProgressChanged += (sender, args) =>
+                {
+                    Progress?.Invoke(sender, args);
                 };
-                agent.DownloadFileCompleted += (sender, args) => {
-                    if (Done != null)
-                    {
-                        Done(sender, args);
-                    }
+                agent.DownloadFileCompleted += (sender, args) =>
+                {
+                    Done?.Invoke(sender, args,
+                                 args.Cancelled || args.Error != null
+                                     ? null
+                                     : agent.ResponseHeaders?.Get("ETag")?.Replace("\"", ""));
                 };
             }
         }
@@ -114,8 +116,7 @@ private void ResetAgent()
         private volatile bool download_canceled;
         private readonly ManualResetEvent complete_or_canceled;
 
-        public delegate void NetAsyncOneCompleted(Uri url, string filename, Exception error);
-        public NetAsyncOneCompleted onOneCompleted;
+        public event Action<Uri, string, Exception, string> onOneCompleted;
 
         /// <summary>
         /// Returns a perfectly boring NetAsyncDownloader.
@@ -169,8 +170,8 @@ private void DownloadModule(Net.DownloadTarget target)
                         args.TotalBytesToReceive);
 
                 // And schedule a notification if we're done (or if something goes wrong)
-                dl.Done += (sender, args) =>
-                    FileDownloadComplete(index, args.Error, args.Cancelled);
+                dl.Done += (sender, args, etag) =>
+                    FileDownloadComplete(index, args.Error, args.Cancelled, etag);
 
                 // Start the download!
                 dl.Download(dl.target.url, dl.path);
@@ -189,7 +190,7 @@ private void DownloadModule(Net.DownloadTarget target)
         private bool shouldQueue(Net.DownloadTarget target)
         {
             return downloads.Any(dl =>
-                    dl.target.url.Host == target.url.Host
+                    (!dl.target.url.IsAbsoluteUri || dl.target.url.Host == target.url.Host)
                     && dl.bytesLeft > 0
                     // Consider done if already tried and failed
                     && dl.error == null);
@@ -267,7 +268,8 @@ public void DownloadAndWait(ICollection<Net.DownloadTarget> urls)
                             // Handle HTTP 403 used for throttling
                             case HttpStatusCode.Forbidden:
                                 Uri infoUrl;
-                                if (Net.ThrottledHosts.TryGetValue(downloads[i].target.url.Host, out infoUrl))
+                                if (downloads[i].target.url.IsAbsoluteUri
+                                    && Net.ThrottledHosts.TryGetValue(downloads[i].target.url.Host, out infoUrl))
                                 {
                                     throw new DownloadThrottledKraken(downloads[i].target.url, infoUrl);
                                 }
@@ -385,7 +387,7 @@ private void FileProgressReport(int index, int percent, long bytesDownloaded, lo
         /// This method gets called back by `WebClient` when a download is completed.
         /// It in turncalls the onCompleted hook when *all* downloads are finished.
         /// </summary>
-        private void FileDownloadComplete(int index, Exception error, bool canceled)
+        private void FileDownloadComplete(int index, Exception error, bool canceled, string etag)
         {
             // Make sure the threads don't trip on one another
             lock (dlMutex)
@@ -420,15 +422,15 @@ private void FileDownloadComplete(int index, Exception error, bool canceled)
                 }
 
                 var next = queuedDownloads.FirstOrDefault(dl =>
-                    dl.url.Host == downloads[index].target.url.Host);
+                    !dl.url.IsAbsoluteUri || dl.url.Host == downloads[index].target.url.Host);
                 if (next != null)
                 {
                     // Start this host's next queued download
                     queuedDownloads.Remove(next);
                     DownloadModule(next);
                 }
 
-                onOneCompleted.Invoke(downloads[index].target.url, downloads[index].path, downloads[index].error);
+                onOneCompleted?.Invoke(downloads[index].target.url, downloads[index].path, downloads[index].error, etag);
 
                 if (++completed_downloads >= downloads.Count + queuedDownloads.Count)
                 {

Core/Net/NetAsyncModulesDownloader.cs

@@ -35,11 +35,9 @@ private IUser User
         public NetAsyncModulesDownloader(IUser user, NetModuleCache cache)
         {
             modules    = new List<CkanModule>();
-            downloader = new NetAsyncDownloader(user)
-            {
-                // Schedule us to process each module on completion.
-                onOneCompleted = ModuleDownloadComplete
-            };
+            downloader = new NetAsyncDownloader(user);
+            // Schedule us to process each module on completion.
+            downloader.onOneCompleted += ModuleDownloadComplete;
             downloader.Progress += (target, remaining, total) =>
             {
                 var mod = modules.FirstOrDefault(m => m.download == target.url);
@@ -104,7 +102,7 @@ public void DownloadModules(IEnumerable<CkanModule> modules)
             }
         }
 
-        private void ModuleDownloadComplete(Uri url, string filename, Exception error)
+        private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag)
         {
             if (error != null)
             {