Restructure Net.Download and Net.DownloadText

Core/Net/Curl.cs

@@ -33,7 +33,7 @@ public static void Init()
 
  /// <summary>
  /// Release any resources used by libcurl. NOT THREADSAFE AT ALL.
- /// Do this after all other threads are done. 
+ /// Do this after all other threads are done.
  /// </summary>
  public static void CleanUp()
  {
@@ -46,11 +46,12 @@ public static void CleanUp()
  /// <summary>
  /// Creates a CurlEasy object that calls the given writeback function
  /// when data is received.
+ /// Can also write back the header.
  /// </summary>
- /// <returns>The CurlEasy obect</returns>
- /// 
+ /// <returns>The CurlEasy object</returns>
+ ///
  /// Adapted from MultiDemo.cs in the curlsharp repo
- public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
+ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf, CurlHeaderCallback hwf = null)
  {
  if (!_initComplete)
  {
@@ -62,21 +63,15 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
  easy.Url = url;
  easy.WriteData = null;
  easy.WriteFunction = wf;
+ if (hwf != null)
+ {
+ easy.HeaderFunction = hwf;
+ }
  easy.Encoding = "deflate, gzip";
  easy.FollowLocation = true; // Follow redirects
  easy.UserAgent = Net.UserAgentString;
  easy.SslVerifyPeer = true;
 
- // ksp.sarbian.com uses a SSL cert that libcurl can't
- // verify, so we skip verification. Yeah, that sucks, I know,
- // but this sucks less than our previous solution that disabled
- // SSL checking entirely.
-
- if (url.StartsWith("https://ksp.sarbian.com/"))
- {
- easy.SslVerifyPeer = false;
- }
-
  var caBundle = ResolveCurlCaBundle();
  if (caBundle != null)
  {
@@ -88,15 +83,16 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
 
  /// <summary>
  /// Creates a CurlEasy object that writes to the given stream.
+ /// Can call a writeback function for the header.
  /// </summary>
- public static CurlEasy CreateEasy(string url, FileStream stream)
+ public static CurlEasy CreateEasy(string url, FileStream stream, CurlHeaderCallback hwf = null)
  {
  // Let's make a happy closure around this stream!
  return CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData)
  {
  stream.Write(buf, 0, size * nmemb);
  return size * nmemb;
- });
+ }, hwf);
  }
 
  public static CurlEasy CreateEasy(Uri url, FileStream stream)

Core/Net/Net.cs

@@ -105,33 +105,54 @@ public static string Download(string url, out string etag, string filename = nul
  }
 
  HttpWebResponse response = ex.Response as HttpWebResponse;
- if (response.StatusCode != HttpStatusCode.Redirect)
+ if (response?.StatusCode != HttpStatusCode.Redirect)
  {
  throw;
  }
 
- return Net.Download(response.GetResponseHeader("Location"), out etag, filename, user);
+ return Download(response.GetResponseHeader("Location"), out etag, filename, user);
  }
- catch (Exception ex)
+ catch (Exception e)
  {
- log.InfoFormat("Download failed, trying with curlsharp...");
+ log.InfoFormat("Native download failed, trying with CurlSharp...");
  etag = null;
 
  try
  {
  Curl.Init();
 
  using (FileStream stream = File.OpenWrite(filename))
- using (var curl = Curl.CreateEasy(url, stream))
  {
- CurlCode result = curl.Perform();
- if (result != CurlCode.Ok)
+ string header = string.Empty;
+
+ var client = Curl.CreateEasy(url, stream, delegate(byte[] buf, int size, int nmemb, object extraData)
  {
- throw new Kraken("curl download of " + url + " failed with CurlCode " + result);
- }
- else
+ header += Encoding.UTF8.GetString(buf);
+ return size * nmemb;
+ });
+
+ using (client)
  {
- log.Debug("curlsharp download successful");
+ var result = client.Perform();
+ var returnCode = client.ResponseCode;
+
+ if (result != CurlCode.Ok || returnCode >= 300)
+ {
+ // Always log if it's an error.
+ log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
+
+ WebException curlException =
+ new WebException($"Curl download failed with status {returnCode}.");
+ throw new NativeAndCurlDownloadFailedKraken(
+ new List<Exception> {e, curlException},
+ url.ToString(), header, null, returnCode
+ );
+ }
+ else
+ {
+ // Only log if debug flag is set.
+ log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, "Content not logged because it is likely a file.");
+ }
  }
  }
 
@@ -157,9 +178,9 @@ public static string Download(string url, out string etag, string filename = nul
  }
 
  // Look for an exception regarding the authentication.
- if (Regex.IsMatch(ex.ToString(), "The authentication or decryption has failed."))
+ if (Regex.IsMatch(e.ToString(), "The authentication or decryption has failed."))
  {
- throw new MissingCertificateKraken("Failed downloading " + url, ex);
+ throw new MissingCertificateKraken("Failed downloading " + url, e);
  }
 
  // Not the exception we were looking for! Throw it further upwards!
@@ -241,39 +262,55 @@ public static string DownloadText(Uri url, string authToken = "")
  agent.Headers.Add("Authorization", $"token {authToken}");
  }
 
- return agent.DownloadString(url.OriginalString);
+ string content = agent.DownloadString(url.OriginalString);
+ string header = agent.ResponseHeaders.ToString();
+
+ log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+
+ return content;
  }
  catch (Exception e)
  {
- log.InfoFormat(e.ToString());
- log.InfoFormat("Download failed, trying with curlsharp...");
+ log.InfoFormat("Native download failed, trying with CurlSharp...");
 
- var content = string.Empty;
+ string content = string.Empty;
+ string header = string.Empty;
 
- var client = Curl.CreateEasy(url.OriginalString, delegate (byte[] buf, int size, int nmemb, object extraData)
- {
- content += Encoding.UTF8.GetString(buf);
- return size * nmemb;
- });
+ var client = Curl.CreateEasy(url.OriginalString,
+ delegate (byte[] buf, int size, int nmemb, object extraData)
+ {
+ content += Encoding.UTF8.GetString(buf);
+ return size * nmemb;
+ },
+ delegate(byte[] buf, int size, int nmemb, object extraData)
+ {
+ header += Encoding.UTF8.GetString(buf);
+ return size * nmemb;
+ }
+ );
 
- client.SetOpt(CurlOption.FailOnError, true);
-
  using (client)
  {
  var result = client.Perform();
  var returnCode = client.ResponseCode;
 
- if (result != CurlCode.Ok)
+ if (result != CurlCode.Ok || returnCode >= 300 )
  {
- throw new WebException(
- String.Format("Curl download failed with error {0} ({1})", result, returnCode), 
- e
- );
+ // Always log if it's an error.
+ log.ErrorFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+
+ WebException curlException = new WebException($"Curl download failed with status {returnCode}.");
+ throw new NativeAndCurlDownloadFailedKraken(
+ new List<Exception> {e, curlException},
+ url.ToString(), header, content, returnCode
+ );
+ }
+ else
+ {
+ // Only log if debug flag is set
+ log.DebugFormat("Response from {0}:\r\n\r\n{1}\r\n{2}", url, header, content);
+ return content;
  }
-
- log.DebugFormat("Download from {0}:\r\n\r\n{1}", url, content);
-
- return content;
  }
  }
  }

Core/Net/NetAsyncDownloader.cs

@@ -154,7 +154,7 @@ private void Download(ICollection<Net.DownloadTarget> targets)
  }
 
  /// <summary>
- /// Download all our files using the native .NET hanlders.
+ /// Download all our files using the native .NET handlers.
  /// </summary>
  /// <returns>The native.</returns>
  private void DownloadNative()

Core/Types/Kraken.cs

@@ -274,7 +274,41 @@ public override string ToString()
  {
  return "Uh oh, the following things went wrong when downloading...\r\n\r\n" + String.Join("\r\n", exceptions);
  }
+ }
+
+ /// <summary>
+ /// We often try downloading using native .NET methods and CurlSharp as a fallback.
+ /// If both downloads fail, use this Kraken to combine the exceptions.
+ /// It assumes that both methods got an equal response.
+ /// Has a nice ToString() method including the response header and content.
+ /// </summary>
+ public class NativeAndCurlDownloadFailedKraken : Kraken
+ {
+ public readonly List<Exception> exceptions;
+ public readonly string URL;
+ public readonly string responseHeader;
+ public readonly string responseContent;
+ public readonly int responseStatus;
+
+ public NativeAndCurlDownloadFailedKraken(List<Exception> errors, string URL, string responseHeader, string responseContent, int responseStatus)
+ : this($"Native and cURL download failed downloading from {URL}, status {responseStatus}", errors, URL, responseHeader, responseContent, responseStatus)
+ {}
 
+ public NativeAndCurlDownloadFailedKraken(string message, List<Exception> errors, string URL, string responseHeader, string responseContent, int responseStatus)
+ : base(message)
+ {
+ exceptions = errors;
+ this.URL = URL;
+ this.responseHeader = responseHeader;
+ this.responseContent = responseContent;
+ this.responseStatus = responseStatus;
+ }
+
+ public override string ToString()
+ {
+ return $"Native and cURL download failed downloading from {URL}, status {responseStatus}:\r\n" +
+ $"{String.Join("\r\n\r\n", exceptions)}\r\n{responseHeader}\r\n{responseContent}";
+ }
  }
 
  /// <summary>

Netkan/Program.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Net;
 using System.Diagnostics;
 using System.IO;
@@ -11,9 +10,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Model;
-using CKAN.NetKAN.Services;
 using CKAN.NetKAN.Transformers;
-using CKAN.NetKAN.Validators;
 using CKAN.NetKAN.Processors;
 
 namespace CKAN.NetKAN
@@ -106,7 +103,7 @@ public static int Main(string[] args)
  }
  catch (Exception e)
  {
- e = e.GetBaseException() ?? e;
+ e = e.GetBaseException();
 
  Log.Fatal(e.Message);
 

Netkan/Sources/Curse/CurseApi.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Diagnostics.Eventing.Reader;
+using System.IO;
 using System.Net;
 using CKAN.NetKAN.Services;
 using log4net;
@@ -22,15 +24,21 @@ public CurseApi(IHttpService http)
 
  public CurseMod GetMod(string nameOrId)
  {
- var json = Call(nameOrId);
+ string json;
+ try
+ {
+ json = Call(nameOrId);
+ }
+ catch (NativeAndCurlDownloadFailedKraken e)
+ {
+ // CurseForge returns a valid json with an error message in some cases.
+ json = e.responseContent;
+ }
  // Check if the mod has been removed from Curse and if it corresponds to a KSP mod.
  var error = JsonConvert.DeserializeObject<CurseError>(json);
  if (!string.IsNullOrWhiteSpace(error.error))
  {
- throw new Kraken(string.Format(
- "Could not get the mod from Curse, reason: {0}.",
- error.message
- ));
+ throw new Kraken($"Could not get the mod from Curse, reason: {error.message}.");
  }
  return CurseMod.FromJson(json);
  }
@@ -42,7 +50,24 @@ public static Uri ResolveRedirect(Uri url)
  HttpWebRequest request = (HttpWebRequest) WebRequest.Create(redirUrl);
  request.AllowAutoRedirect = false;
  request.UserAgent = Net.UserAgentString;
- HttpWebResponse response = (HttpWebResponse) request.GetResponse();
+
+ HttpWebResponse response;
+ try
+ {
+ response = (HttpWebResponse) request.GetResponse();
+ }
+ catch (WebException e)
+ {
+ if (e.Status == WebExceptionStatus.ProtocolError)
+ {
+ response = e.Response as HttpWebResponse;
+ if (response?.StatusCode == HttpStatusCode.Forbidden)
+ {
+ throw new Kraken("CKAN blocked by CurseForge");
+ }
+ }
+ throw;
+ }
  response.Close();
  while (response.Headers["Location"] != null)
  {

Netkan/Sources/Github/GithubApi.cs

@@ -102,9 +102,12 @@ private string Call(string path)
  {
  return _http.DownloadText(url, _oauthToken);
  }
- catch (WebException webEx)
+ catch (NativeAndCurlDownloadFailedKraken k)
  {
- Log.ErrorFormat("WebException while accessing {0}: {1}", url, webEx);
+ if (k.responseStatus == 403 && k.responseHeader.Contains("X-RateLimit-Remaining: 0"))
+ {
+ throw new Kraken("GitHub API rate limit exceeded.");
+ }
  throw;
  }
  }