Skip to content

Commit

Permalink
fix/implement caching upstream #93
Browse files Browse the repository at this point in the history
For read-through caching, queries to registration and package service
must be directed to upstream server. Otherwise client would get always
404 because there is nothing in the cache to begin with.
We can add caching queries too, but that requires integrating catalog
reader to refresh responses for already cached packages.
  • Loading branch information
tomzo committed Sep 28, 2018
1 parent a9a3297 commit f76b37f
Show file tree
Hide file tree
Showing 8 changed files with 227 additions and 6 deletions.
4 changes: 2 additions & 2 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/src/BaGet/bin/Debug/netcoreapp2.0/BaGet.dll",
"program": "${workspaceFolder}/src/BaGet/bin/Debug/netcoreapp2.1/BaGet.dll",
"args": [],
"cwd": "${workspaceFolder}",
"cwd": "${workspaceFolder}/src/BaGet",
"stopAtEntry": false,
"internalConsoleOptions": "openOnSessionStart",
"launchBrowser": {
Expand Down
3 changes: 2 additions & 1 deletion src/BaGet.Core/BaGet.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@

<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.Relational" Version="2.1.1" />
<PackageReference Include="NuGet.Packaging" Version="4.7.0" />
<PackageReference Include="NuGet.Packaging" Version="4.8.0" />
<PackageReference Include="NuGet.Protocol" Version="4.8.0" />
</ItemGroup>

</Project>
17 changes: 16 additions & 1 deletion src/BaGet.Core/Mirror/FakeMirrorService.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System.Threading;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NuGet.Protocol.Core.Types;
using NuGet.Versioning;

namespace BaGet.Core.Mirror
Expand All @@ -9,6 +11,19 @@ namespace BaGet.Core.Mirror
/// </summary>
public class FakeMirrorService : IMirrorService
{
Task<IReadOnlyList<string>> emptyVersions = Task.Factory.StartNew(() => new List<string>() as IReadOnlyList<string>);
Task<IEnumerable<IPackageSearchMetadata>> emptyMeta = Task.Factory.StartNew(() => new List<IPackageSearchMetadata>() as IEnumerable<IPackageSearchMetadata>);

public Task<IReadOnlyList<string>> FindUpstreamAsync(string id, CancellationToken ct)
{
return emptyVersions;
}

public Task<IEnumerable<IPackageSearchMetadata>> FindUpstreamMetadataAsync(string id, CancellationToken ct)
{
return emptyMeta;
}

public Task MirrorAsync(
string id,
NuGetVersion version,
Expand Down
8 changes: 7 additions & 1 deletion src/BaGet.Core/Mirror/IMirrorService.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System.Threading;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NuGet.Protocol.Core.Types;
using NuGet.Versioning;

namespace BaGet.Core.Mirror
Expand All @@ -17,5 +19,9 @@ public interface IMirrorService
/// <param name="cancellationToken">The token to cancel the mirroring</param>
/// <returns>A task that completes when the package has been mirrored.</returns>
Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken);

Task<IReadOnlyList<string>> FindUpstreamAsync(string id, CancellationToken ct);

Task<IEnumerable<IPackageSearchMetadata>> FindUpstreamMetadataAsync(string id, CancellationToken ct);
}
}
33 changes: 33 additions & 0 deletions src/BaGet.Core/Mirror/MirrorService.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,15 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using BaGet.Core.Services;
using Microsoft.Extensions.Logging;
using NuGet.Configuration;
using NuGet.Protocol.Core.Types;
using NuGet.Versioning;
using NuGet.Protocol;
using NuGet.Common;
using System.Linq;

namespace BaGet.Core.Mirror
{
Expand All @@ -14,6 +20,11 @@ public class MirrorService : IMirrorService
private readonly IPackageDownloader _downloader;
private readonly IIndexingService _indexer;
private readonly ILogger<MirrorService> _logger;
private readonly SourceRepository _sourceRepository;
private SourceCacheContext _cacheContext;
NuGetLoggerAdapter<MirrorService> _loggerAdapter;
private PackageMetadataResourceV3 _metadataSearch;
private RemoteV3FindPackageByIdResource _versionSearch;

public MirrorService(
Uri packageBaseAddress,
Expand All @@ -27,6 +38,28 @@ public MirrorService(
_downloader = downloader ?? throw new ArgumentNullException(nameof(downloader));
_indexer = indexer ?? throw new ArgumentNullException(nameof(indexer));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
this._loggerAdapter = new NuGetLoggerAdapter<MirrorService>(_logger);
List<Lazy<INuGetResourceProvider>> providers = new List<Lazy<INuGetResourceProvider>>();
providers.AddRange(Repository.Provider.GetCoreV3());
providers.Add(new Lazy<INuGetResourceProvider>(() => new PackageMetadataResourceV3Provider()));
PackageSource packageSource = new PackageSource("https://api.nuget.org/v3/index.json"); //TODO needs options
_sourceRepository = new SourceRepository(packageSource, providers);
_cacheContext = new SourceCacheContext();
var httpSource = _sourceRepository.GetResource<HttpSourceResource>();
RegistrationResourceV3 regResource = _sourceRepository.GetResource<RegistrationResourceV3>();
ReportAbuseResourceV3 reportAbuseResource = _sourceRepository.GetResource<ReportAbuseResourceV3>();
_metadataSearch = new PackageMetadataResourceV3(httpSource.HttpSource, regResource, reportAbuseResource);
_versionSearch = new RemoteV3FindPackageByIdResource(_sourceRepository, httpSource.HttpSource);
}

public async Task<IEnumerable<IPackageSearchMetadata>> FindUpstreamMetadataAsync(string id, CancellationToken ct) {
return await _metadataSearch.GetMetadataAsync(id, true, false, _cacheContext, _loggerAdapter, ct);
}

public async Task<IReadOnlyList<string>> FindUpstreamAsync(string id, CancellationToken ct)
{
var versions = await _versionSearch.GetAllVersionsAsync(id, _cacheContext, _loggerAdapter, ct);
return versions.Select(v => v.ToNormalizedString()).ToList();
}

public async Task MirrorAsync(string id, NuGetVersion version, CancellationToken cancellationToken)
Expand Down
92 changes: 92 additions & 0 deletions src/BaGet.Core/NuGetLoggerAdapter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;

namespace BaGet.Core
{
public class NuGetLoggerAdapter<T> : NuGet.Common.ILogger
{
Microsoft.Extensions.Logging.ILogger<T> logger;

public NuGetLoggerAdapter(Microsoft.Extensions.Logging.ILogger<T> logger) {
this.logger = logger ?? throw new ArgumentNullException("logger");
}


public void Log(NuGet.Common.LogLevel level, string data)
{
switch(level) {
case NuGet.Common.LogLevel.Error:
this.logger.LogError(data);
break;
case NuGet.Common.LogLevel.Debug:
this.logger.LogDebug(data);
break;
case NuGet.Common.LogLevel.Minimal:
this.logger.LogDebug(data);
break;
case NuGet.Common.LogLevel.Information:
this.logger.LogInformation(data);
break;
case NuGet.Common.LogLevel.Verbose:
this.logger.LogTrace(data);
break;
case NuGet.Common.LogLevel.Warning:
this.logger.LogWarning(data);
break;
}
}

public void Log(NuGet.Common.ILogMessage message)
{
this.Log(message.Level, message.Message);
}

public Task LogAsync(NuGet.Common.LogLevel level, string data)
{
this.Log(level, data);
return Task.CompletedTask;
}

public Task LogAsync(NuGet.Common.ILogMessage message)
{
this.Log(message.Level, message.Message);
return Task.CompletedTask;
}

public void LogDebug(string data)
{
this.logger.LogDebug(data);
}

public void LogError(string data)
{
this.logger.LogError(data);
}

public void LogInformation(string data)
{
this.logger.LogInformation(data);
}

public void LogInformationSummary(string data)
{
this.logger.LogInformation(data);
}

public void LogMinimal(string data)
{
this.logger.LogDebug(data);
}

public void LogVerbose(string data)
{
this.logger.LogDebug(data);
}

public void LogWarning(string data)
{
this.logger.LogWarning(data);
}
}
}
8 changes: 8 additions & 0 deletions src/BaGet.Web/Controllers/PackageController.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -29,6 +30,13 @@ public async Task<IActionResult> Versions(string id)

if (!packages.Any())
{
IReadOnlyList<string> upstreamVersions = await _mirror.FindUpstreamAsync(id, CancellationToken.None);
if(upstreamVersions.Any()) {
return Json(new
{
Versions = upstreamVersions.ToList()
});
}
return NotFound();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BaGet.Core.Entities;
using BaGet.Core.Mirror;
using BaGet.Core.Services;
using BaGet.Web.Extensions;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using NuGet.Protocol.Core.Types;

namespace BaGet.Controllers.Web.Registration
{
Expand All @@ -15,10 +18,12 @@ namespace BaGet.Controllers.Web.Registration
/// </summary>
public class RegistrationIndexController : Controller
{
private readonly IMirrorService _mirror;
private readonly IPackageService _packages;

public RegistrationIndexController(IPackageService packages)
public RegistrationIndexController(IMirrorService mirror, IPackageService packages)
{
this._mirror = mirror;
_packages = packages ?? throw new ArgumentNullException(nameof(packages));
}

Expand All @@ -32,6 +37,23 @@ public async Task<IActionResult> Get(string id)

if (!packages.Any())
{
var upstreamPackages = (await _mirror.FindUpstreamMetadataAsync(id, CancellationToken.None)).ToList();
if(upstreamPackages.Any()) {
return Json(new
{
Count = upstreamPackages.Count,
TotalDownloads = upstreamPackages.Sum(s => s.DownloadCount),
Items = new[]
{
new RegistrationIndexItem(
packageId: id,
items: upstreamPackages.Select(ToRegistrationIndexLeaf).ToList(),
lower: upstreamPackages.Select(p => p.Identity.Version).Min().ToNormalizedString(),
upper: upstreamPackages.Select(p => p.Identity.Version).Max().ToNormalizedString()
),
}
});
}
return NotFound();
}

Expand All @@ -54,6 +76,15 @@ public async Task<IActionResult> Get(string id)
});
}

private RegistrationIndexLeaf ToRegistrationIndexLeaf(IPackageSearchMetadata package) =>
new RegistrationIndexLeaf(
packageId: package.Identity.Id,
catalogEntry: new CatalogEntry(
package: package,
catalogUri: $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Identity.Id}.{package.Identity.Version}.json",
packageContent: Url.PackageDownload(package.Identity.Id, package.Identity.Version)),
packageContent: Url.PackageDownload(package.Identity.Id, package.Identity.Version));

private RegistrationIndexLeaf ToRegistrationIndexLeaf(Package package) =>
new RegistrationIndexLeaf(
packageId: package.Id,
Expand Down Expand Up @@ -141,6 +172,41 @@ public CatalogEntry(Package package, string catalogUri, string packageContent)
Title = package.Title;
}

public CatalogEntry(IPackageSearchMetadata package, string catalogUri, string packageContent)
{
if (package == null) throw new ArgumentNullException(nameof(package));

CatalogUri = catalogUri ?? throw new ArgumentNullException(nameof(catalogUri));

PackageId = package.Identity.Id;
Version = package.Identity.Version.ToFullString();
Authors = string.Join(", ", package.Authors);
Description = package.Description;
Downloads = package.DownloadCount.GetValueOrDefault(0);
HasReadme = false; //
IconUrl = NullSafeToString(package.IconUrl);
Language = null; //
LicenseUrl = NullSafeToString(package.LicenseUrl);
Listed = package.IsListed;
//MinClientVersion =
PackageContent = packageContent;
ProjectUrl = NullSafeToString(package.ProjectUrl);
//RepositoryUrl = package.RepositoryUrlString;
//RepositoryType = package.RepositoryType;
//Published = package.Published.GetValueOrDefault(DateTimeOffset.MinValue);
RequireLicenseAcceptance = package.RequireLicenseAcceptance;
Summary = package.Summary;
Tags = package.Tags == null ? null : package.Tags.Split(",");
Title = package.Title;
}

private string NullSafeToString(object prop)
{
if(prop == null)
return null;
return prop.ToString();
}

[JsonProperty(PropertyName = "@id")]
public string CatalogUri { get; }

Expand Down

0 comments on commit f76b37f

Please sign in to comment.