Configurable cache size limit

Cmdline/Action/Cache.cs

@@ -25,6 +25,12 @@ private class CacheSubOptions : VerbCommandOptions
  [VerbOption("reset", HelpText = "Set the download cache path to the default")]
  public CommonOptions ResetOptions { get; set; }
 
+ [VerbOption("showlimit", HelpText = "Show the cache size limit")]
+ public CommonOptions ShowLimitOptions { get; set; }
+
+ [VerbOption("setlimit", HelpText = "Set the cache size limit")]
+ public SetLimitOptions SetLimitOptions { get; set; }
+
  [HelpVerbOption]
  public string GetUsage(string verb)
  {
@@ -45,11 +51,15 @@ public string GetUsage(string verb)
  case "set":
  ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options] path");
  break;
+ case "setlimit":
+ ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options] megabytes");
+ break;
 
  // Now the commands with only --flag type options
  case "list":
  case "clear":
  case "reset":
+ case "showlimit":
  default:
  ht.AddPreOptionsLine($"Usage: ckan cache {verb} [options]");
  break;
@@ -65,6 +75,12 @@ private class SetOptions : CommonOptions
  public string Path { get; set; }
  }
 
+ private class SetLimitOptions : CommonOptions
+ {
+ [ValueOption(0)]
+ public long Megabytes { get; set; } = -1;
+ }
+
  /// <summary>
  /// Execute a cache subcommand
  /// </summary>
@@ -111,6 +127,14 @@ public int RunSubCommand(KSPManager mgr, CommonOptions opts, SubCommandOptions u
  exitCode = ResetCacheDirectory((CommonOptions)suboptions);
  break;
 
+ case "showlimit":
+ exitCode = ShowCacheSizeLimit((CommonOptions)suboptions);
+ break;
+
+ case "setlimit":
+ exitCode = SetCacheSizeLimit((SetLimitOptions)suboptions);
+ break;
+
  default:
  user.RaiseMessage("Unknown command: cache {0}", option);
  exitCode = Exit.BADOPT;
@@ -176,6 +200,34 @@ private int ResetCacheDirectory(CommonOptions options)
  return Exit.OK;
  }
 
+ private int ShowCacheSizeLimit(CommonOptions options)
+ {
+ IWin32Registry winReg = new Win32Registry();
+ if (winReg.CacheSizeLimit.HasValue)
+ {
+ user.RaiseMessage(CkanModule.FmtSize(winReg.CacheSizeLimit.Value));
+ }
+ else
+ {
+ user.RaiseMessage("Unlimited");
+ }
+ return Exit.OK;
+ }
+
+ private int SetCacheSizeLimit(SetLimitOptions options)
+ {
+ IWin32Registry winReg = new Win32Registry();
+ if (options.Megabytes < 0)
+ {
+ winReg.CacheSizeLimit = null;
+ }
+ else
+ {
+ winReg.CacheSizeLimit = options.Megabytes * (long)1024 * (long)1024;
+ }
+ return ShowCacheSizeLimit(null);
+ }
+
  private void printCacheInfo()
  {
  int fileCount;
@@ -187,7 +239,7 @@ private void printCacheInfo()
  private KSPManager manager;
  private IUser user;
 
- private static readonly ILog  log = LogManager.GetLogger(typeof(Cache));
+ private static readonly ILog log = LogManager.GetLogger(typeof(Cache));
  }
 
 }

Core/ModuleInstaller.cs

@@ -219,6 +219,8 @@ public void InstallList(ICollection<CkanModule> modules, RelationshipResolverOpt
 
  }
 
+ EnforceCacheSizeLimit();
+
  // We can scan GameData as a separate transaction. Installing the mods
  // leaves everything consistent, and this is just gravy. (And ScanGameData
  // acts as a Tx, anyway, so we don't need to provide our own.)
@@ -253,6 +255,8 @@ public void InstallList(ModuleResolution modules, RelationshipResolverOptions op
  User.RaiseProgress("Committing filesystem changes", 80);
 
  transaction.Complete();
+
+ EnforceCacheSizeLimit();
  }
  }
 
@@ -996,6 +1000,8 @@ public void AddRemove(IEnumerable<CkanModule> add = null, IEnumerable<string> re
  registry_manager.Save(enforceConsistency);
 
  tx.Complete();
+
+ EnforceCacheSizeLimit();
  }
  }
 
@@ -1157,6 +1163,18 @@ public void ImportFiles(HashSet<FileInfo> files, IUser user, Action<CkanModule>
  f.Delete();
  }
  }
+
+ EnforceCacheSizeLimit();
+ }
+
+ private void EnforceCacheSizeLimit()
+ {
+ // Purge old downloads if we're over the limit
+ Win32Registry winReg = new Win32Registry();
+ if (winReg.CacheSizeLimit.HasValue)
+ {
+ Cache.EnforceSizeLimit(winReg.CacheSizeLimit.Value, registry_manager.registry);
+ }
  }
 
  /// <summary>

Core/Net/NetFileCache.cs

@@ -11,6 +11,7 @@
 using ICSharpCode.SharpZipLib.Zip;
 using log4net;
 using CKAN.Extensions;
+using CKAN.Versioning;
 
 namespace CKAN
 {
@@ -290,6 +291,103 @@ private HashSet<string> legacyDirs()
  .ToHashSet();
  }
 
+ public void EnforceSizeLimit(long bytes, Registry registry)
+ {
+ int numFiles;
+ long curBytes;
+ GetSizeInfo(out numFiles, out curBytes);
+ if (curBytes > bytes)
+ {
+ // This object will let us determine whether a module is compatible with any of our instances
+ KspVersionCriteria aggregateCriteria = manager?.Instances.Values
+ .Where(ksp => ksp.Valid)
+ .Select(ksp => ksp.VersionCriteria())
+ .Aggregate((a, b) => a.Union(b));
+
+ // This object lets us find the modules associated with a cached file
+ Dictionary<string, List<CkanModule>> hashMap = registry.GetDownloadHashIndex();
+
+ // Prune the module lists to only those that are compatible
+ foreach (var kvp in hashMap)
+ {
+ kvp.Value.RemoveAll(mod => !mod.IsCompatibleKSP(aggregateCriteria));
+ }
+
+ // Now get all the files in all the caches...
+ List<FileInfo> files = allFiles();
+ // ... and sort them by compatibilty and timestamp...
+ files.Sort((a, b) => compareFiles(
+ hashMap, aggregateCriteria, a, b
+ ));
+
+ // ... and delete them till we're under the limit
+ foreach (FileInfo fi in files)
+ {
+ curBytes -= fi.Length;
+ fi.Delete();
+ if (curBytes <= bytes)
+ {
+ // Limit met, all done!
+ break;
+ }
+ }
+ OnCacheChanged();
+ }
+ }
+
+ private int compareFiles(Dictionary<string, List<CkanModule>> hashMap, KspVersionCriteria crit, FileInfo a, FileInfo b)
+ {
+ // Compatible modules for file A
+ List<CkanModule> modulesA;
+ hashMap.TryGetValue(a.Name.Substring(0, 8), out modulesA);
+ bool compatA = modulesA?.Any() ?? false;
+
+ // Compatible modules for file B
+ List<CkanModule> modulesB;
+ hashMap.TryGetValue(b.Name.Substring(0, 8), out modulesB);
+ bool compatB = modulesB?.Any() ?? false;
+
+ if (modulesA == null && modulesB != null)
+ {
+ // A isn't indexed but B is, delete A first
+ return -1;
+ }
+ else if (modulesA != null && modulesB == null)
+ {
+ // A is indexed but B isn't, delete B first
+ return 1;
+ }
+ else if (!compatA && compatB)
+ {
+ // A isn't compatible but B is, delete A first
+ return -1;
+ }
+ else if (compatA && !compatB)
+ {
+ // A is compatible but B isn't, delete B first
+ return 1;
+ }
+ else
+ {
+ // Both are either compatible or incompatible
+ // Go by file age, oldest first
+ return (int)(a.CreationTime - b.CreationTime).TotalSeconds;
+ }
+ return 0;
+ }
+
+ private List<FileInfo> allFiles()
+ {
+ DirectoryInfo mainDir = new DirectoryInfo(cachePath);
+ var files = mainDir.EnumerateFiles();
+ foreach (string legacyDir in legacyDirs())
+ {
+ DirectoryInfo legDir = new DirectoryInfo(legacyDir);
+ files = files.Union(legDir.EnumerateFiles());
+ }
+ return files.ToList();
+ }
+
  /// <summary>
  /// Check whether a ZIP file is valid
  /// </summary>

Core/Net/NetModuleCache.cs

@@ -74,6 +74,10 @@ public void GetSizeInfo(out int numFiles, out long numBytes)
  {
  cache.GetSizeInfo(out numFiles, out numBytes);
  }
+ public void EnforceSizeLimit(long bytes, Registry registry)
+ {
+ cache.EnforceSizeLimit(bytes, registry);
+ }
 
  /// <summary>
  /// Calculate the SHA1 hash of a file

Core/Registry/Registry.cs

@@ -1097,14 +1097,20 @@ public HashSet<string> FindReverseDependencies(IEnumerable<string> modules_to_re
  public Dictionary<string, List<CkanModule>> GetSha1Index()
  {
  var index = new Dictionary<string, List<CkanModule>>();
- foreach (var kvp in available_modules) {
+ foreach (var kvp in available_modules)
+ {
  AvailableModule am = kvp.Value;
- foreach (var kvp2 in am.module_version) {
+ foreach (var kvp2 in am.module_version)
+ {
  CkanModule mod = kvp2.Value;
- if (mod.download_hash != null) {
- if (index.ContainsKey(mod.download_hash.sha1)) {
+ if (mod.download_hash != null)
+ {
+ if (index.ContainsKey(mod.download_hash.sha1))
+ {
  index[mod.download_hash.sha1].Add(mod);
- } else {
+ }
+ else
+ {
  index.Add(mod.download_hash.sha1, new List<CkanModule>() {mod});
  }
  }
@@ -1113,5 +1119,38 @@ public Dictionary<string, List<CkanModule>> GetSha1Index()
  return index;
  }
 
+ /// <summary>
+ /// Get a dictionary of all mod versions indexed by their download URLs' hash.
+ /// Useful for finding the mods for a group of URLs without repeatedly searching the entire registry.
+ /// </summary>
+ /// <returns>
+ /// dictionary[urlHash] = {mod1, mod2, mod3};
+ /// </returns>
+ public Dictionary<string, List<CkanModule>> GetDownloadHashIndex()
+ {
+ var index = new Dictionary<string, List<CkanModule>>();
+ foreach (var kvp in available_modules)
+ {
+ AvailableModule am = kvp.Value;
+ foreach (var kvp2 in am.module_version)
+ {
+ CkanModule mod = kvp2.Value;
+ if (mod.download != null)
+ {
+ string hash = NetFileCache.CreateURLHash(mod.download);
+ if (index.ContainsKey(hash))
+ {
+ index[hash].Add(mod);
+ }
+ else
+ {
+ index.Add(hash, new List<CkanModule>() {mod});
+ }
+ }
+ }
+ }
+ return index;
+ }
+
  }
 }

Core/Versioning/KspVersionCriteria.cs

@@ -6,7 +6,7 @@ namespace CKAN.Versioning
 {
  public class KspVersionCriteria
  {
- private List<KspVersion> _versions = new List<KspVersion> ();
+ private List<KspVersion> _versions = new List<KspVersion>();
 
  public KspVersionCriteria (KspVersion v)
  {
@@ -34,6 +34,14 @@ public IList<KspVersion> Versions
  }
  }
 
+ public KspVersionCriteria Union(KspVersionCriteria other)
+ {
+ return new KspVersionCriteria(
+ null,
+ _versions.Union(other.Versions).ToList()
+ );
+ }
+
  public override String ToString()
  {
  return "[Versions: " + _versions.ToString() + "]";

Core/Win32Registry.cs

@@ -14,6 +14,7 @@ public interface IWin32Registry
  string GetKSPBuilds();
  void SetKSPBuilds(string buildMap);
  string DownloadCacheDir { get; set; }
+ long? CacheSizeLimit { get; set; }
  }
 
  public class Win32Registry : IWin32Registry
@@ -58,6 +59,30 @@ public string DownloadCacheDir
  }
  }
 
+ /// <summary>
+ /// Get and set the maximum number of bytes allowed in the cache.
+ /// Unlimited if null.
+ /// </summary>
+ public long? CacheSizeLimit
+ {
+ get
+ {
+ string val = GetRegistryValue<string>(@"CacheSizeLimit", null);
+ return string.IsNullOrEmpty(val) ? null : (long?)Convert.ToInt64(val);
+ }
+ set
+ {
+ if (!value.HasValue)
+ {
+ DeleteRegistryValue(@"CacheSizeLimit");
+ }
+ else
+ {
+ SetRegistryValue(@"CacheSizeLimit", value.Value);
+ }
+ }
+ }
+
  private int InstanceCount
  {
  get { return GetRegistryValue(@"KSPInstanceCount", 0); }