New MaxVersionsPerPackage to clean packages

docs/docs/configuration.md

@@ -154,6 +154,20 @@ downloaded if you know the package's id and version. You can override this behav
 }
 ```
 
+## Enable package auto-deletion
+
+If your build server generates many nuget packages, your BaGet server can quickly run out of space. To avoid this issue, `MaxVersionsPerPackage` can be configured to auto-delete packages older packages when a new one is uploaded. This will use the `HardDelete` option detailed above and will unlist and delete the files for the older packages. By default this value is not configured and no packages will be deleted automatically.
+
+```json
+{
+    ...
+
+    "MaxVersionsPerPackage ": 5,
+
+    ...
+}
+```
+
 ## Enable package overwrites
 
 Normally, BaGetter will reject a package upload if the id and version are already taken. This is to maintain the [immutability of semantically versioned packages](https://learn.microsoft.com/azure/devops/artifacts/artifacts-key-concepts?view=azure-devops#immutability).

global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "8.0.100",
+    "version": "8.0.401",
     "rollForward": "latestFeature",
     "allowPrerelease": false
   }

src/BaGetter.Core/Configuration/BaGetterOptions.cs

@@ -47,6 +47,12 @@ public class BaGetterOptions
     /// </summary>
     public uint MaxPackageSizeGiB { get; set; } = 8;
 
+    /// <summary>
+    /// If this is set to a value, it will limit the number of versions that can be pushed for a package.
+    /// the older versions will be deleted.
+    /// </summary>
+    public uint? MaxVersionsPerPackage { get; set; } = null;
+
     public DatabaseOptions Database { get; set; }
 
     public StorageOptions Storage { get; set; }

src/BaGetter.Core/Indexing/IPackageDeletionService.cs

@@ -6,6 +6,15 @@ namespace BaGetter.Core;
 
 public interface IPackageDeletionService
 {
+    /// <summary>
+    /// Delete old versions of packages
+    /// </summary>
+    /// <param name="package">Current package object to clean</param>
+    /// <param name="maxPackagesToKeep">Maximum number of packages to keep</param>
+    /// <param name="cancellationToken"></param>
+    /// <returns>Number of packages deleted</returns>
+    Task<int> DeleteOldVersionsAsync(Package package, uint maxPackagesToKeep, CancellationToken cancellationToken);
+
     /// <summary>
     /// Attempt to delete a package.
     /// </summary>

src/BaGetter.Core/Indexing/PackageDeletionService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -88,4 +89,22 @@ private async Task<bool> TryHardDeletePackageAsync(string id, NuGetVersion versi
 
         return found;
     }
+
+    public async Task<int> DeleteOldVersionsAsync(Package package, uint maxPackages, CancellationToken cancellationToken)
+    {
+        // list all versions of the package
+        var versions = await _packages.FindAsync(package.Id, includeUnlisted: true, cancellationToken);
+        if (versions is null || versions.Count <= maxPackages) return 0;
+        // sort by version and take everything except the last maxPackages
+        var versionsToDelete = versions
+            .OrderByDescending(p => p.Version)
+            .Skip((int)maxPackages)
+            .ToList();
+        var deleted = 0;
+        foreach (var version in versionsToDelete)
+        {
+            if (await TryHardDeletePackageAsync(package.Id, version.Version, cancellationToken)) deleted++;
+        }
+        return deleted;
+    }
 }

src/BaGetter.Core/Indexing/PackageIndexingService.cs

@@ -16,10 +16,12 @@ public class PackageIndexingService : IPackageIndexingService
     private readonly SystemTime _time;
     private readonly IOptionsSnapshot<BaGetterOptions> _options;
     private readonly ILogger<PackageIndexingService> _logger;
+    private readonly IPackageDeletionService _packageDeletionService;
 
     public PackageIndexingService(
         IPackageDatabase packages,
         IPackageStorageService storage,
+        IPackageDeletionService packageDeletionService,
         ISearchIndexer search,
         SystemTime time,
         IOptionsSnapshot<BaGetterOptions> options,
@@ -31,6 +33,7 @@ public PackageIndexingService(
         _time = time ?? throw new ArgumentNullException(nameof(time));
         _options = options ?? throw new ArgumentNullException(nameof(options));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _packageDeletionService = packageDeletionService ?? throw new ArgumentNullException(nameof(packageDeletionService));
     }
 
     public async Task<PackageIndexingResult> IndexAsync(Stream packageStream, CancellationToken cancellationToken)
@@ -153,6 +156,33 @@ await _storage.SavePackageContentAsync(
 
         await _search.IndexAsync(package, cancellationToken);
 
+        if (_options.Value.MaxVersionsPerPackage.HasValue)
+        {
+            try { 
+                _logger.LogInformation(
+                    "Deleting older packages for package {PackageId} {PackageVersion}",
+                    package.Id,
+                    package.NormalizedVersionString);
+                var deleted = await _packageDeletionService.DeleteOldVersionsAsync(package, _options.Value.MaxVersionsPerPackage.Value, cancellationToken);
+                if (deleted > 0)
+                {
+                    _logger.LogInformation(
+                        "Deleted {packages} older packages for package {PackageId} {PackageVersion}",
+                        deleted,
+                        package.Id,
+                        package.NormalizedVersionString);
+                }
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(
+                    e,
+                    "Failed to cleanup older versions of package {PackageId} {PackageVersion}",
+                    package.Id,
+                    package.NormalizedVersionString);
+            }
+        }
+
         _logger.LogInformation(
             "Successfully indexed package {Id} {Version} in search",
             package.Id,