Skip to content

Commit

Permalink
Merge #2950 Request fewer GitHub releases and cache string URLs in Ne…
Browse files Browse the repository at this point in the history 
…tkan
  • Loading branch information
@HebaruSan
HebaruSan committed Dec 24, 2019
2 parents 78438b3 + 5ad7d62 commit 13054b2
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 37 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file.
- [Netkan] Retain URL hash cache, cache file hashes (#2940 by: HebaruSan; reviewed: DasSkelett)
- [Netkan] Get org members as authors, time-sort authors, sort tags to middle (#2942 by: HebaruSan; reviewed: DasSkelett)
- [Netkan] Coerce GitHub URLs into the authenticated API in Netkan (#2946 by: HebaruSan; reviewed: DasSkelett)
- [Netkan] Request fewer GitHub releases and cache string URLs in Netkan (#2950 by: HebaruSan)

## v1.26.6 (Leonov)

Expand Down
39 changes: 37 additions & 2 deletions Netkan/Services/CachingHttpService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ internal sealed class CachingHttpService : IHttpService
private readonly NetFileCache _cache;
private HashSet<Uri> _requestedURLs = new HashSet<Uri>();
private bool _overwriteCache = false;
private Dictionary<Uri, StringCacheEntry> _stringCache = new Dictionary<Uri, StringCacheEntry>();

// Re-use string value URLs within 2 minutes
private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 2, 0);

public CachingHttpService(NetFileCache cache, bool overwrite = false)
{
Expand Down Expand Up @@ -78,11 +82,35 @@ public string DownloadPackage(Uri url, string identifier, DateTime? updated)

public string DownloadText(Uri url)
{
return Net.DownloadText(url);
return TryGetCached(url, () => Net.DownloadText(url));
}
public string DownloadText(Uri url, string authToken, string mimeType = null)
{
return Net.DownloadText(url, authToken, mimeType);
return TryGetCached(url, () => Net.DownloadText(url, authToken, mimeType));
}

private string TryGetCached(Uri url, Func<string> uncached)
{
if (_stringCache.TryGetValue(url, out StringCacheEntry entry))
{
if (DateTime.Now - entry.Timestamp < stringCacheLifetime)
{
// Re-use recent cached request of this URL
return entry.Value;
}
else
{
// Too old, purge it
_stringCache.Remove(url);
}
}
string val = uncached();
_stringCache.Add(url, new StringCacheEntry()
{
Value = val,
Timestamp = DateTime.Now
});
return val;
}

public IEnumerable<Uri> RequestedURLs { get { return _requestedURLs; } }
Expand All @@ -93,4 +121,11 @@ public void ClearRequestedURLs()

private static readonly ILog log = LogManager.GetLogger(typeof(CachingHttpService));
}

public class StringCacheEntry
{
public string Value;
public DateTime Timestamp;
}

}
80 changes: 45 additions & 35 deletions Netkan/Sources/Github/GithubApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,55 +62,65 @@ public GithubRelease GetLatestRelease(GithubRef reference)

public IEnumerable<GithubRelease> GetAllReleases(GithubRef reference)
{
var json = Call($"repos/{reference.Repository}/releases?per_page=100");
Log.Debug("Parsing JSON...");
var releases = JArray.Parse(json);
const int perPage = 10;
for (int page = 1; true; ++page)
{
var json = Call($"repos/{reference.Repository}/releases?per_page={perPage}&page={page}");
Log.Debug("Parsing JSON...");
var releases = JArray.Parse(json);

// Finding the most recent *stable* release means filtering
// out on pre-releases.
// Finding the most recent *stable* release means filtering
// out on pre-releases.

foreach (var release in releases)
{
// First, check for prerelease status...
if (reference.UsePrerelease == (bool)release["prerelease"])
foreach (var release in releases)
{
var version = new ModuleVersion((string)release["tag_name"]);
var author = (string)release["author"]["login"];
// First, check for prerelease status...
if (reference.UsePrerelease == (bool)release["prerelease"])
{
var version = new ModuleVersion((string)release["tag_name"]);
var author = (string)release["author"]["login"];

Uri download = null;
DateTime? updated = null;
DateTime parsed;
Uri download = null;
DateTime? updated = null;
DateTime parsed;

if (reference.UseSourceArchive)
{
Log.Debug("Using GitHub source archive");
download = new Uri((string)release["zipball_url"]);
if (DateTime.TryParse(release["published_at"].ToString(), out parsed))
if (reference.UseSourceArchive)
{
updated = parsed;
Log.Debug("Using GitHub source archive");
download = new Uri((string)release["zipball_url"]);
if (DateTime.TryParse(release["published_at"].ToString(), out parsed))
{
updated = parsed;
}
}
}
else
{
var assets = (JArray)release["assets"];

foreach (var asset in assets.Where(asset => reference.Filter.IsMatch((string)asset["name"])))
else
{
Log.DebugFormat("Using GitHub asset: {0}", asset["name"]);
download = new Uri((string)asset["browser_download_url"]);
if (DateTime.TryParse(asset["updated_at"].ToString(), out parsed))
var assets = (JArray)release["assets"];

foreach (var asset in assets.Where(asset => reference.Filter.IsMatch((string)asset["name"])))
{
updated = parsed;
Log.DebugFormat("Using GitHub asset: {0}", asset["name"]);
download = new Uri((string)asset["browser_download_url"]);
if (DateTime.TryParse(asset["updated_at"].ToString(), out parsed))
{
updated = parsed;
}
break;
}
break;
}
}

if (download != null)
{
yield return new GithubRelease(author, version, download, updated);
if (download != null)
{
yield return new GithubRelease(author, version, download, updated);
}
}
}

if (releases.Count < perPage)
{
// That's all folks!
break;
}
}
}

Expand Down

0 comments on commit 13054b2

Please sign in to comment.