Merge #2946 Coerce GitHub URLs into the authenticated API in Netkan

CHANGELOG.md

@@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file.
 - [Netkan] Better parsing errors for version files (#2939 by: HebaruSan; reviewed: DasSkelett)
 - [Netkan] Retain URL hash cache, cache file hashes (#2940 by: HebaruSan; reviewed: DasSkelett)
 - [Netkan] Get org members as authors, time-sort authors, sort tags to middle (#2942 by: HebaruSan; reviewed: DasSkelett)
+- [Netkan] Coerce GitHub URLs into the authenticated API in Netkan (#2946 by: HebaruSan; reviewed: DasSkelett)
 
 ## v1.26.6 (Leonov)
 

Core/Net/Net.cs

@@ -244,7 +244,7 @@ public static void DownloadWithProgress(ICollection<DownloadTarget> downloadTarg
             }.DownloadAndWait(downloadTargets);
         }
 
-        public static string DownloadText(Uri url, string authToken = "")
+        public static string DownloadText(Uri url, string authToken = "", string mimeType = null)
         {
             log.DebugFormat("About to download {0}", url.OriginalString);
 
@@ -261,6 +261,11 @@ public static string DownloadText(Uri url, string authToken = "")
                     // Send our auth token to the GitHub API (or whoever else needs one)
                     agent.Headers.Add("Authorization", $"token {authToken}");
                 }
+                if (!string.IsNullOrEmpty(mimeType))
+                {
+                    log.InfoFormat("Setting MIME type {0}", mimeType);
+                    agent.Headers.Add("Accept", mimeType);
+                }
 
                 string content = agent.DownloadString(url.OriginalString);
                 string header  = agent.ResponseHeaders.ToString();

Netkan/Services/CachingHttpService.cs

@@ -80,9 +80,9 @@ public string DownloadText(Uri url)
         {
             return Net.DownloadText(url);
         }
-        public string DownloadText(Uri url, string authToken)
+        public string DownloadText(Uri url, string authToken, string mimeType = null)
         {
-            return Net.DownloadText(url, authToken);
+            return Net.DownloadText(url, authToken, mimeType);
         }
 
         public IEnumerable<Uri> RequestedURLs { get { return _requestedURLs; } }

Netkan/Services/IHttpService.cs

@@ -7,7 +7,7 @@ internal interface IHttpService
     {
         string DownloadPackage(Uri url, string identifier, DateTime? updated);
         string DownloadText(Uri url);
-        string DownloadText(Uri url, string authToken);
+        string DownloadText(Uri url, string authToken, string mimeType);
 
         IEnumerable<Uri> RequestedURLs { get; }
         void ClearRequestedURLs();

Netkan/Sources/Github/GithubApi.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Text.RegularExpressions;
 using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -21,6 +22,26 @@ internal sealed class GithubApi : IGithubApi
         private readonly IHttpService _http;
         private readonly string       _oauthToken;
 
+        private const string rawMediaType = "application/vnd.github.v3.raw";
+
+        // https://github.com/<OWNER>/<REPO>/blob/<BRANCH>/<PATH>
+        // https://github.com/<OWNER>/<REPO>/tree/<BRANCH>/<PATH>
+        // https://github.com/<OWNER>/<REPO>/raw/<BRANCH>/<PATH>
+        private static readonly Regex githubUrlRegex = new Regex(
+            @"^/(?<owner>[^/]+)/(?<repo>[^/]+)/(?<type>[^/]+)/(?<branch>[^/]+)/(?<path>.+)$",
+            RegexOptions.Compiled);
+
+        private static readonly HashSet<string> urlTypes = new HashSet<string>()
+        {
+            "blob", "tree", "raw"
+        };
+
+        // https://raw.githubusercontent.com/<OWNER>/<REPO>/<BRANCH>/<PATH>
+        private static readonly Regex githubUserContentUrlRegex = new Regex(
+            @"^/(?<owner>[^/]+)/(?<repo>[^/]+)/(?<branch>[^/]+)/(?<path>.+)$",
+            RegexOptions.Compiled
+        );
+
         public GithubApi(IHttpService http, string oauthToken = null)
         {
             _http       = http;
@@ -100,14 +121,75 @@ public List<GithubUser> getOrgMembers(GithubUser organization)
             );
         }        
 
-        private string Call(string path)
+        /// <summary>
+        /// Download a URL via the GitHubAPI.
+        /// Will use a token if we have one.
+        /// </summary>
+        /// <param name="url">The URL to download</param>
+        /// <returns>
+        /// null if the URL isn't on GitHub, otherwise the contents of the download
+        /// </returns>
+        public string DownloadText(Uri url)
+        {
+            if (TryGetGitHubPath(url, out string ghOwner, out string ghRepo, out string ghBranch, out string ghPath))
+            {
+                Log.Info("Found GitHub URL, retrieving with API");
+                return Call(
+                    $"repos/{ghOwner}/{ghRepo}/contents/{ghPath}?ref={ghBranch}",
+                    rawMediaType
+                );
+            }
+            else
+            {
+                Log.DebugFormat("Not a GitHub URL: {0}", url.ToString());
+                return null;
+            }
+        }
+
+        private bool TryGetGitHubPath(Uri url, out string owner, out string repo, out string branch, out string path)
+        {
+            switch (url.Host)
+            {
+                case "github.com":
+                    Log.DebugFormat("Found standard GitHub host, checking format");
+                    Match ghMatch = githubUrlRegex.Match(url.AbsolutePath);
+                    if (ghMatch.Success && urlTypes.Contains(ghMatch.Groups["type"].Value))
+                    {
+                        Log.DebugFormat("Matched standard GitHub format");
+                        owner  = ghMatch.Groups["owner"].Value;
+                        repo   = ghMatch.Groups["repo"].Value;
+                        branch = ghMatch.Groups["branch"].Value;
+                        path   = ghMatch.Groups["path"].Value;
+                        return true;
+                    }
+                    break;
+
+                case "raw.githubusercontent.com":
+                    Log.DebugFormat("Found raw GitHub host, checking format");
+                    Match rawMatch = githubUserContentUrlRegex.Match(url.AbsolutePath);
+                    if (rawMatch.Success)
+                    {
+                        Log.DebugFormat("Matched raw GitHub format");
+                        owner  = rawMatch.Groups["owner"].Value;
+                        repo   = rawMatch.Groups["repo"].Value;
+                        branch = rawMatch.Groups["branch"].Value;
+                        path   = rawMatch.Groups["path"].Value;
+                        return true;
+                    }
+                    break;
+            }
+            owner = repo = branch = path = null;
+            return false;
+        }
+
+        private string Call(string path, string mimeType = null)
         {
             var url = new Uri(ApiBase, path);
             Log.DebugFormat("Calling {0}", url);
 
             try
             {
-                return _http.DownloadText(url, _oauthToken);
+                return _http.DownloadText(url, _oauthToken, mimeType);
             }
             catch (NativeAndCurlDownloadFailedKraken k)
             {

Netkan/Sources/Github/IGithubApi.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 
 namespace CKAN.NetKAN.Sources.Github
@@ -8,5 +9,6 @@ internal interface IGithubApi
         GithubRelease GetLatestRelease(GithubRef reference);
         IEnumerable<GithubRelease> GetAllReleases(GithubRef reference);
         List<GithubUser> getOrgMembers(GithubUser organization);
+        string DownloadText(Uri url);
     }
 }

Netkan/Transformers/AvcKrefTransformer.cs

@@ -1,12 +1,13 @@
 using System;
 using System.Collections.Generic;
+using log4net;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Extensions;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Services;
 using CKAN.NetKAN.Sources.Avc;
-using log4net;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
+using CKAN.NetKAN.Sources.Github;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -18,11 +19,13 @@ internal sealed class AvcKrefTransformer : ITransformer
         private static readonly ILog Log = LogManager.GetLogger(typeof(AvcKrefTransformer));
 
         public string Name { get { return "avc-kref"; } }
-        private IHttpService httpSvc;
+        private readonly IHttpService httpSvc;
+        private readonly IGithubApi   githubSrc;
 
-        public AvcKrefTransformer(IHttpService http)
+        public AvcKrefTransformer(IHttpService http, IGithubApi github)
         {
-            httpSvc = http;
+            httpSvc   = http;
+            githubSrc = github;
         }
 
         public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
@@ -34,8 +37,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 Log.InfoFormat("Executing KSP-AVC $kref transformation with {0}", metadata.Kref);
                 Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
 
+                var url = new Uri(metadata.Kref.Id);
                 AvcVersion remoteAvc = JsonConvert.DeserializeObject<AvcVersion>(
-                    httpSvc.DownloadText(CKAN.Net.GetRawUri(new Uri(metadata.Kref.Id)))
+                    githubSrc?.DownloadText(url)
+                        ?? httpSvc.DownloadText(CKAN.Net.GetRawUri(url))
                 );
 
                 json.SafeAdd("name",     remoteAvc.Name);

Netkan/Transformers/AvcTransformer.cs

@@ -1,14 +1,15 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using log4net;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Extensions;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Services;
 using CKAN.NetKAN.Sources.Avc;
 using CKAN.Versioning;
-using log4net;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
+using CKAN.NetKAN.Sources.Github;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -20,15 +21,17 @@ internal sealed class AvcTransformer : ITransformer
     {
         private static readonly ILog Log = LogManager.GetLogger(typeof(AvcTransformer));
 
-        private readonly IHttpService _http;
+        private readonly IHttpService   _http;
         private readonly IModuleService _moduleService;
+        private readonly IGithubApi     _github;
 
         public string Name { get { return "avc"; } }
 
-        public AvcTransformer(IHttpService http, IModuleService moduleService)
+        public AvcTransformer(IHttpService http, IModuleService moduleService, IGithubApi github)
         {
-            _http = http;
+            _http          = http;
             _moduleService = moduleService;
+            _github        = github;
         }
 
         public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
@@ -67,7 +70,8 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                     {
                         try
                         {
-                            var remoteJson = _http.DownloadText(remoteUri);
+                            var remoteJson = _github?.DownloadText(remoteUri)
+                                ?? _http.DownloadText(remoteUri);
                             var remoteAvc = JsonConvert.DeserializeObject<AvcVersion>(remoteJson);
 
                             if (avc.version.CompareTo(remoteAvc.version) == 0)

Netkan/Transformers/MetaNetkanTransformer.cs

@@ -6,6 +6,7 @@
 using CKAN.NetKAN.Extensions;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Services;
+using CKAN.NetKAN.Sources.Github;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -19,12 +20,14 @@ internal sealed class MetaNetkanTransformer : ITransformer
         private const string KrefSource = "netkan";
 
         private readonly IHttpService _http;
+        private readonly IGithubApi   _github;
 
         public string Name { get { return "metanetkan"; } }
 
-        public MetaNetkanTransformer(IHttpService http)
+        public MetaNetkanTransformer(IHttpService http, IGithubApi github)
         {
-            _http = http;
+            _http   = http;
+            _github = github;
         }
 
         public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
@@ -43,7 +46,8 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 resourcesJson.SafeAdd("metanetkan", metadata.Kref.Id);
 
                 var uri = new Uri(metadata.Kref.Id);
-                var targetFileText = _http.DownloadText(CKAN.Net.GetRawUri(uri));
+                var targetFileText = _github?.DownloadText(uri)
+                    ?? _http.DownloadText(CKAN.Net.GetRawUri(uri));
 
                 Log.DebugFormat("Target netkan:{0}{1}", Environment.NewLine, targetFileText);
 

Netkan/Transformers/NetkanTransformer.cs

@@ -26,17 +26,18 @@ public NetkanTransformer(
             bool prerelease
         )
         {
+            var ghApi = new GithubApi(http, githubToken);
             _transformers = InjectVersionedOverrideTransformers(new List<ITransformer>
             {
-                new MetaNetkanTransformer(http),
+                new MetaNetkanTransformer(http, ghApi),
                 new SpacedockTransformer(new SpacedockApi(http)),
                 new CurseTransformer(new CurseApi(http)),
-                new GithubTransformer(new GithubApi(http, githubToken), prerelease),
+                new GithubTransformer(ghApi, prerelease),
                 new HttpTransformer(),
                 new JenkinsTransformer(new JenkinsApi(http)),
-                new AvcKrefTransformer(http),
+                new AvcKrefTransformer(http, ghApi),
                 new InternalCkanTransformer(http, moduleService),
-                new AvcTransformer(http, moduleService),
+                new AvcTransformer(http, moduleService, ghApi),
                 new LocalizationsTransformer(http, moduleService),
                 new VersionEditTransformer(),
                 new ForcedVTransformer(),

Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs

@@ -67,7 +67,7 @@ private Metadata TryKref(IHttpService http, string kref)
             }
 
             // Act
-            var tran = new AvcKrefTransformer(http);
+            var tran = new AvcKrefTransformer(http, null);
             return tran.Transform(new Metadata(json), opts).First();
         }
 

Tests/NetKAN/Transformers/AvcTransformerTests.cs

@@ -32,7 +32,7 @@ public void AddsMissingVersionInfo()
             mModuleService.Setup(i => i.GetInternalAvc(It.IsAny<CkanModule>(), It.IsAny<string>(), It.IsAny<string>()))
                 .Returns(avcVersion);
 
-            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object);
+            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object, null);
 
             var json = new JObject();
             json["spec_version"] = 1;
@@ -70,7 +70,7 @@ public void PreferentiallyAddsRangedKspVersionInfo()
             mModuleService.Setup(i => i.GetInternalAvc(It.IsAny<CkanModule>(), It.IsAny<string>(), It.IsAny<string>()))
                 .Returns(avcVersion);
 
-            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object);
+            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object, null);
 
             var json = new JObject();
             json["spec_version"] = 1;
@@ -208,7 +208,7 @@ public void CorrectlyCalculatesKspVersionInfo(
             mModuleService.Setup(i => i.GetInternalAvc(It.IsAny<CkanModule>(), It.IsAny<string>(), It.IsAny<string>()))
                 .Returns(avcVersion);
 
-            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object);
+            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object, null);
 
             // Act
             var result = sut.Transform(new Metadata(json), opts).First();
@@ -240,7 +240,7 @@ public void DoesNotOverrideExistingVersionInfo()
             mModuleService.Setup(i => i.GetInternalAvc(It.IsAny<CkanModule>(), It.IsAny<string>(), It.IsAny<string>()))
                 .Returns(avcVersion);
 
-            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object);
+            var sut = new AvcTransformer(mHttp.Object, mModuleService.Object, null);
 
             var json = new JObject();
             json["spec_version"] = 1;
@@ -271,7 +271,7 @@ public void Transform_TrustVersionFileTrue_OverridesExistingInfo()
                     version = new ModuleVersion("1.2.3")
                 });
 
-            ITransformer sut = new AvcTransformer(mHttp.Object, mModuleService.Object);
+            ITransformer sut = new AvcTransformer(mHttp.Object, mModuleService.Object, null);
 
             JObject json = new JObject();
             json["spec_version"]                = 1;