Merge #2904 Restructure Net.Download and Net.DownloadText

CHANGELOG.md

@@ -27,6 +27,7 @@ All notable changes to this project will be documented in this file.
 - [GUI] Show target version for upgrades in change set (#2888 by: HebaruSan; reviewed: DasSkelett)
 - [Netkan] Merge resources and include metanetkan (#2913 by: HebaruSan; reviewed: DasSkelett)
 - [GUI] Force redraw of recommendation listview headers on Mono (#2920 by: HebaruSan; reviewed: DasSkelett)
+- [Core] Restructure Net.Download and Net.DownloadText (#2904 by: DasSkelett; reviewed: HebaruSan)
 
 ### Internal
 

Core/Net/Curl.cs

@@ -33,7 +33,7 @@ public static void Init()
 
         /// <summary>
         /// Release any resources used by libcurl. NOT THREADSAFE AT ALL.
-        /// Do this after all other threads are done. 
+        /// Do this after all other threads are done.
         /// </summary>
         public static void CleanUp()
         {
@@ -46,11 +46,12 @@ public static void CleanUp()
         /// <summary>
         /// Creates a CurlEasy object that calls the given writeback function
         /// when data is received.
+        /// Can also write back the header.
         /// </summary>
-        /// <returns>The CurlEasy obect</returns>
-        /// 
+        /// <returns>The CurlEasy object</returns>
+        ///
         /// Adapted from MultiDemo.cs in the curlsharp repo
-        public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
+        public static CurlEasy CreateEasy(string url, CurlWriteCallback wf, CurlHeaderCallback hwf = null)
         {
             if (!_initComplete)
             {
@@ -62,21 +63,15 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
             easy.Url = url;
             easy.WriteData = null;
             easy.WriteFunction = wf;
+            if (hwf != null)
+            {
+                easy.HeaderFunction = hwf;
+            }
             easy.Encoding = "deflate, gzip";
             easy.FollowLocation = true; // Follow redirects
             easy.UserAgent = Net.UserAgentString;
             easy.SslVerifyPeer = true;
 
-            // ksp.sarbian.com uses a SSL cert that libcurl can't
-            // verify, so we skip verification. Yeah, that sucks, I know,
-            // but this sucks less than our previous solution that disabled
-            // SSL checking entirely.
-
-            if (url.StartsWith("https://ksp.sarbian.com/"))
-            {
-                easy.SslVerifyPeer = false;
-            }
-
             var caBundle = ResolveCurlCaBundle();
             if (caBundle != null)
             {
@@ -88,15 +83,16 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
 
         /// <summary>
         /// Creates a CurlEasy object that writes to the given stream.
+        /// Can call a writeback function for the header.
         /// </summary>
-        public static CurlEasy CreateEasy(string url, FileStream stream)
+        public static CurlEasy CreateEasy(string url, FileStream stream, CurlHeaderCallback hwf = null)
         {
             // Let's make a happy closure around this stream!
             return CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData)
             {
                 stream.Write(buf, 0, size * nmemb);
                 return size * nmemb;
-            });
+            }, hwf);
         }
 
         public static CurlEasy CreateEasy(Uri url, FileStream stream)

Core/Net/Net.cs

@@ -105,33 +105,54 @@ public static string Download(string url, out string etag, string filename = nul
                 }
 
                 HttpWebResponse response = ex.Response as HttpWebResponse;
-                if (response.StatusCode != HttpStatusCode.Redirect)
+                if (response?.StatusCode != HttpStatusCode.Redirect)
                 {
                     throw;
                 }
 
-                return Net.Download(response.GetResponseHeader("Location"), out etag, filename, user);
+                return Download(response.GetResponseHeader("Location"), out etag, filename, user);
             }
-            catch (Exception ex)
+            catch (Exception e)
             {
-                log.InfoFormat("Download failed, trying with curlsharp...");
+                log.InfoFormat("Native download failed, trying with CurlSharp...");
                 etag = null;
 
                 try
                 {
                     Curl.Init();
 
                     using (FileStream stream = File.OpenWrite(filename))
-                    using (var curl = Curl.CreateEasy(url, stream))
                     {
-                        CurlCode result = curl.Perform();
-                        if (result != CurlCode.Ok)
+                        string header  = string.Empty;
+
+                        var client = Curl.CreateEasy(url, stream, delegate(byte[] buf, int size, int nmemb, object extraData)
                         {
-                            throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
-                        }
-                        else
+                            header += Encoding.UTF8.GetString(buf);
+                            return size * nmemb;
+                        });
+
+                        using (client)
                         {
-                            log.Debug("curlsharp download successful");
+                            var result = client.Perform();
+                            var returnCode = client.ResponseCode;
+
+                            if (result != CurlCode.Ok || returnCode >= 300)
+                            {
+                                // Always log if it's an error.
+                                log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
+
+                                WebException curlException =
+                                    new WebException($"Curl download failed with status {returnCode}.");
+                                throw new NativeAndCurlDownloadFailedKraken(
+                                    new List<Exception> {e, curlException},
+                                    url.ToString(), header, null, returnCode
+                                );
+                            }
+                            else
+                            {
+                                // Only log if debug flag is set.
+                                log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
+                            }
                         }
                     }
 
@@ -157,9 +178,9 @@ public static string Download(string url, out string etag, string filename = nul
                 }
 
                 // Look for an exception regarding the authentication.
-                if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
+                if (Regex.IsMatch(e.ToString(), "The authentication or decryption has failed."))
                 {
-                    throw new MissingCertificateKraken("Failed downloading " + url, ex);
+                    throw new MissingCertificateKraken("Failed downloading " + url, e);
                 }
 
                 // Not the exception we were looking for! Throw it further upwards!
@@ -241,39 +262,55 @@ public static string DownloadText(Uri url, string authToken = "")
                     agent.Headers.Add("Authorization", $"token {authToken}");
                 }
 
-                return agent.DownloadString(url.OriginalString);
+                string content = agent.DownloadString(url.OriginalString);
+                string header  = agent.ResponseHeaders.ToString();
+
+                log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+
+                return content;
             }
             catch (Exception e)
             {
-                log.InfoFormat(e.ToString());
-                log.InfoFormat("Download failed, trying with curlsharp...");
+                log.InfoFormat("Native download failed, trying with CurlSharp...");
 
-                var content = string.Empty;
+                string content = string.Empty;
+                string header  = string.Empty;
 
-                var client = Curl.CreateEasy(url.OriginalString, delegate (byte[] buf, int size, int nmemb, object extraData)
-                {
-                    content += Encoding.UTF8.GetString(buf);
-                    return size * nmemb;
-                });
+                var client = Curl.CreateEasy(url.OriginalString,
+                    delegate (byte[] buf, int size, int nmemb, object extraData)
+                    {
+                        content += Encoding.UTF8.GetString(buf);
+                        return size * nmemb;
+                    },
+                    delegate(byte[] buf, int size, int nmemb, object extraData)
+                    {
+                        header += Encoding.UTF8.GetString(buf);
+                        return size * nmemb;
+                    }
+                );
 
-                client.SetOpt(CurlOption.FailOnError, true);
-
                 using (client)
                 {
                     var result = client.Perform();
                     var returnCode = client.ResponseCode;
 
-                    if (result != CurlCode.Ok)
+                    if (result != CurlCode.Ok || returnCode >= 300 )
                     {
-                        throw new WebException(
-                            String.Format("Curl download failed with error {0} ({1})", result, returnCode), 
-                            e
-                            );
+                        // Always log if it's an error.
+                        log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+
+                        WebException curlException = new WebException($"Curl download failed with status {returnCode}.");
+                        throw new NativeAndCurlDownloadFailedKraken(
+                            new List<Exception> {e, curlException},
+                            url.ToString(), header, content, returnCode
+                        );
+                    }
+                    else
+                    {
+                        // Only log if debug flag is set
+                        log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+                        return content;
                     }
-
-                    log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);
-
-                    return content;
                 }
             }
         }

Core/Net/NetAsyncDownloader.cs

@@ -154,7 +154,7 @@ private void Download(ICollection<Net.DownloadTarget> targets)
         }
 
         /// <summary>
-        /// Download all our files using the native .NET hanlders.
+        /// Download all our files using the native .NET handlers.
         /// </summary>
         /// <returns>The native.</returns>
         private void DownloadNative()

Core/Types/Kraken.cs

@@ -274,7 +274,41 @@ public override string ToString()
         {
             return "Uh oh, the following things went wrong when downloading...\r\n\r\n" + String.Join("\r\n", exceptions);
         }
+    }
+
+    /// <summary>
+    /// We often try downloading using native .NET methods and CurlSharp as a fallback.
+    /// If both downloads fail, use this Kraken to combine the exceptions.
+    /// It assumes that both methods got an equal response.
+    /// Has a nice ToString() method including the response header and content.
+    /// </summary>
+    public class NativeAndCurlDownloadFailedKraken : Kraken
+    {
+        public readonly List<Exception> exceptions;
+        public readonly string URL;
+        public readonly string responseHeader;
+        public readonly string responseContent;
+        public readonly int    responseStatus;
+
+        public NativeAndCurlDownloadFailedKraken(List<Exception> errors, string URL, string responseHeader, string responseContent, int responseStatus)
+            : this($"Native and cURL download failed downloading from {URL}, status {responseStatus}", errors, URL, responseHeader, responseContent, responseStatus)
+        {}
 
+        public NativeAndCurlDownloadFailedKraken(string message, List<Exception> errors, string URL, string responseHeader, string responseContent, int responseStatus)
+            : base(message)
+        {
+            exceptions = errors;
+            this.URL = URL;
+            this.responseHeader = responseHeader;
+            this.responseContent = responseContent;
+            this.responseStatus = responseStatus;
+        }
+
+        public override string ToString()
+        {
+            return $"Native and cURL download failed downloading from {URL}, status {responseStatus}:\r\n" +
+                   $"{String.Join("\r\n\r\n", exceptions)}\r\n{responseHeader}\r\n{responseContent}";
+        }
     }
 
     /// <summary>

Netkan/Program.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Net;
 using System.Diagnostics;
 using System.IO;
@@ -11,9 +10,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Model;
-using CKAN.NetKAN.Services;
 using CKAN.NetKAN.Transformers;
-using CKAN.NetKAN.Validators;
 using CKAN.NetKAN.Processors;
 
 namespace CKAN.NetKAN
@@ -106,7 +103,7 @@ public static int Main(string[] args)
             }
             catch (Exception e)
             {
-                e = e.GetBaseException() ?? e;
+                e = e.GetBaseException();
 
                 Log.Fatal(e.Message);
 

Netkan/Sources/Curse/CurseApi.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Diagnostics.Eventing.Reader;
+using System.IO;
 using System.Net;
 using CKAN.NetKAN.Services;
 using log4net;
@@ -22,15 +24,21 @@ public CurseApi(IHttpService http)
 
         public CurseMod GetMod(string nameOrId)
         {
-            var json = Call(nameOrId);
+            string json;
+            try
+            {
+                json = Call(nameOrId);
+            }
+            catch (NativeAndCurlDownloadFailedKraken e)
+            {
+                // CurseForge returns a valid json with an error message in some cases.
+                json = e.responseContent;
+            }
             // Check if the mod has been removed from Curse and if it corresponds to a KSP mod.
             var error = JsonConvert.DeserializeObject<CurseError>(json);
             if (!string.IsNullOrWhiteSpace(error.error))
             {
-                throw new Kraken(string.Format(
-                    "Could not get the mod from Curse, reason: {0}.",
-                    error.message
-                ));
+                throw new Kraken($"Could not get the mod from Curse, reason: {error.message}.");
             }
             return CurseMod.FromJson(json);
         }
@@ -42,7 +50,24 @@ public static Uri ResolveRedirect(Uri url)
             HttpWebRequest request = (HttpWebRequest) WebRequest.Create(redirUrl);
             request.AllowAutoRedirect = false;
             request.UserAgent = Net.UserAgentString;
-            HttpWebResponse response = (HttpWebResponse) request.GetResponse();
+
+            HttpWebResponse response;
+            try
+            {
+                response = (HttpWebResponse) request.GetResponse();
+            }
+            catch (WebException e)
+            {
+                if (e.Status == WebExceptionStatus.ProtocolError)
+                {
+                    response = e.Response as HttpWebResponse;
+                    if (response?.StatusCode == HttpStatusCode.Forbidden)
+                    {
+                        throw new Kraken("CKAN blocked by CurseForge");
+                    }
+                }
+                throw;
+            }
             response.Close();
             while (response.Headers["Location"] != null)
             {

Netkan/Sources/Github/GithubApi.cs

@@ -102,9 +102,12 @@ private string Call(string path)
             {
                 return _http.DownloadText(url, _oauthToken);
             }
-            catch (WebException webEx)
+            catch (NativeAndCurlDownloadFailedKraken k)
             {
-                Log.ErrorFormat("WebException while accessing {0}: {1}", url, webEx);
+                if (k.responseStatus == 403 && k.responseHeader.Contains("X-RateLimit-Remaining: 0"))
+                {
+                    throw new Kraken("GitHub API rate limit exceeded.");
+                }
                 throw;
             }
         }