Autogenerate spec_version and make it optional in netkans

Netkan/Model/Metadata.cs

@@ -15,7 +15,7 @@ internal sealed class Metadata
  {
  private const string KrefPropertyName = "$kref";
  private const string VrefPropertyName = "$vref";
- private const string SpecVersionPropertyName = "spec_version";
+ public  const string SpecVersionPropertyName = "spec_version";
  private const string VersionPropertyName = "version";
  private const string DownloadPropertyName = "download";
  public const string UpdatedPropertyName = "release_date";
@@ -85,10 +85,6 @@ public Metadata(JObject json)
  ));
  }
  }
- else
- {
- throw new Kraken(string.Format("{0} must be specified.", SpecVersionPropertyName));
- }
 
  if (json.TryGetValue(VersionPropertyName, out JToken versionToken))
  {

Netkan/Processors/Inflator.cs

@@ -48,6 +48,7 @@ internal IEnumerable<Metadata> Inflate(string filename, Metadata[] netkans, Tran
  .SelectMany(netkan => transformer.Transform(netkan, opts))
  .GroupBy(module => module.Version)
  .Select(grp => Metadata.Merge(grp.ToArray()))
+ .SelectMany(merged => specVersionTransformer.Transform(merged, opts))
  .ToList();
  log.Debug("Finished transformation");
 
@@ -120,6 +121,7 @@ private static void PurgeDownloads(IHttpService http, NetFileCache cache)
  private readonly IHttpService http;
 
  private readonly NetkanTransformer transformer;
+ private readonly SpecVersionTransformer specVersionTransformer = new SpecVersionTransformer();
 
  private readonly NetkanValidator netkanValidator = new NetkanValidator();
  private readonly CkanValidator ckanValidator;

Netkan/Program.cs

@@ -4,20 +4,20 @@
 using System.Linq;
 using System.Net;
 using System.Text;
-using CommandLine;
 
+using CommandLine;
 using log4net;
 using log4net.Core;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using YamlDotNet.RepresentationModel;
 
 using CKAN.Games;
 using CKAN.Versioning;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Processors;
 using CKAN.NetKAN.Transformers;
 using CKAN.NetKAN.Extensions;
-using YamlDotNet.RepresentationModel;
 
 namespace CKAN.NetKAN
 {

Netkan/Transformers/GitlabTransformer.cs

@@ -12,7 +12,7 @@
 namespace CKAN.NetKAN.Transformers
 {
  /// <summary>
- /// An <see cref="ITransformer"/> that looks up data from GitHub.
+ /// An <see cref="ITransformer"/> that looks up data from GitLab.
  /// </summary>
  internal sealed class GitlabTransformer : ITransformer
  {

Netkan/Transformers/InternalCkanTransformer.cs

@@ -61,9 +61,6 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
  json.SafeAdd(property.Name, property.Value);
  }
 
- json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, new Metadata(internalJson).SpecVersion)
- .ToSpecVersionJson();
-
  json.SafeMerge("resources", internalJson["resources"]);
 
  Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);

Netkan/Transformers/MetaNetkanTransformer.cs

@@ -66,9 +66,6 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
  var targetMetadata = new Metadata(targetJson);
  if (targetMetadata.Kref == null || targetMetadata.Kref.Source != "netkan")
  {
- json["spec_version"] = ModuleVersion.Max(metadata.SpecVersion, targetMetadata.SpecVersion)
- .ToSpecVersionJson();
-
  if (targetJson["$kref"] != null)
  {
  json["$kref"] = targetJson["$kref"];

Netkan/Transformers/SpecVersionTransformer.cs

@@ -0,0 +1,153 @@
+using System.Collections.Generic;
+using System.Linq;
+
+using Newtonsoft.Json.Linq;
+using log4net;
+
+using CKAN.Versioning;
+using CKAN.NetKAN.Model;
+using CKAN.NetKAN.Extensions;
+
+namespace CKAN.NetKAN.Transformers
+{
+ /// <summary>
+ /// An <see cref="ITransformer"/> that sets the spec version to match the metadata.
+ /// </summary>
+ internal sealed class SpecVersionTransformer : ITransformer
+ {
+ /// <summary>
+ /// Defines the name of this transformer
+ /// </summary>
+ public string Name => "spec_version";
+
+ public IEnumerable<Metadata> Transform(Metadata metadata,
+ TransformOptions opts)
+ {
+ var json = metadata.Json();
+ var minVersion = MinimumSpecVersion(json);
+ if (metadata.SpecVersion == null || metadata.SpecVersion != minVersion)
+ {
+ log.InfoFormat("Setting spec version {0}", minVersion);
+ json[Metadata.SpecVersionPropertyName] = minVersion.ToSpecVersionJson();
+ yield return new Metadata(json);
+ }
+ else
+ {
+ yield return metadata;
+ }
+ }
+
+ private static ModuleVersion MinimumSpecVersion(JObject json)
+ // Add new stuff at the top, versions in this function should be in descending order
+ => json["download_hash"] is JObject hashes
+ && (!hashes.ContainsKey("sha256") || !hashes.ContainsKey("sha1")) ? v1p35
+
+ : json["download"] is JArray ? v1p34
+
+ : AllRelationships(json).Any(rel => rel.ContainsKey("any_of")
+ && rel.ContainsKey("choice_help_text")) ? v1p31
+
+ : HasLicense(json, "MPL-2.0") ? v1p30
+
+ : (json["install"] as JArray)?.OfType<JObject>().Any(stanza =>
+ (string)stanza["install_to"] == "Ships" && (
+ // find: .../Script, install_to: Ships
+ ((string)stanza["find"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script"
+ // file: .../Script, install_to: Ships
+ || ((string)stanza["file"])?.Split(new char[] {'/'})?.LastOrDefault() == "Script"
+ // install_to: Ships, as: Script
+ || (((string)stanza["as"])?.EndsWith("Script") ?? false))) ?? false ? v1p29
+
+ : (string)json["kind"] == "dlc" ? v1p28
+
+ : json.ContainsKey("replaced_by") ? v1p26
+
+ : AllRelationships(json).Any(rel => rel.ContainsKey("any_of")) ? v1p26
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => (string)stanza["install_to"] == "Missions") ?? false ? v1p25
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => stanza.ContainsKey("include_only")
+ || stanza.ContainsKey("include_only_regexp")) ?? false ? v1p24
+
+ : HasLicense(json, "Unlicense") ? v1p18
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => stanza.ContainsKey("as")) ?? false ? v1p18
+
+ : json.ContainsKey("ksp_version_strict") ? v1p16
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/@thumbs")) ?? false ? v1p16
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => stanza.ContainsKey("find_matches_files")) ?? false ? v1p16
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => (string)stanza["install_to"] == "Scenarios") ?? false ? v1p14
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => ((string)stanza["install_to"]).StartsWith("Ships/")) ?? false ? v1p12
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => stanza.ContainsKey("find_regexp")
+ || stanza.ContainsKey("filter_regexp")) ?? false ? v1p10
+
+ : json["license"] is JArray ? v1p8
+
+ : (string)json["kind"] == "metapackage" ? v1p6
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => stanza.ContainsKey("find")) ?? false ? v1p4
+
+ : HasLicense(json, "WTFPL") ? v1p2
+
+ : json.ContainsKey("supports") ? v1p2
+
+ : (json["install"] as JArray)?.OfType<JObject>()
+ .Any(stanza => ((string)stanza["install_to"]).StartsWith("GameData/")) ?? false ? v1p2
+
+ : v1p0;
+
+ private static bool HasLicense(JObject json,
+ string name)
+ => json["license"] is JArray array ? array.Contains(name)
+ : json["license"] is JToken token && ((string)token) == name;
+
+ private static IEnumerable<JObject> AllRelationships(JObject json)
+ => relProps.SelectMany(p => json[p] is JArray array ? array.OfType<JObject>()
+ : Enumerable.Empty<JObject>());
+
+ private static readonly string[] relProps = new string[]
+ {
+ "depends",
+ "recommends",
+ "suggests",
+ "conflicts",
+ "supports"
+ };
+
+ private static readonly ModuleVersion v1p0 = new ModuleVersion("v1.0");
+ private static readonly ModuleVersion v1p2 = new ModuleVersion("v1.2");
+ private static readonly ModuleVersion v1p4 = new ModuleVersion("v1.4");
+ private static readonly ModuleVersion v1p6 = new ModuleVersion("v1.6");
+ private static readonly ModuleVersion v1p8 = new ModuleVersion("v1.8");
+ private static readonly ModuleVersion v1p10 = new ModuleVersion("v1.10");
+ private static readonly ModuleVersion v1p12 = new ModuleVersion("v1.12");
+ private static readonly ModuleVersion v1p14 = new ModuleVersion("v1.14");
+ private static readonly ModuleVersion v1p16 = new ModuleVersion("v1.16");
+ private static readonly ModuleVersion v1p18 = new ModuleVersion("v1.18");
+ private static readonly ModuleVersion v1p24 = new ModuleVersion("v1.24");
+ private static readonly ModuleVersion v1p25 = new ModuleVersion("v1.25");
+ private static readonly ModuleVersion v1p26 = new ModuleVersion("v1.26");
+ private static readonly ModuleVersion v1p28 = new ModuleVersion("v1.28");
+ private static readonly ModuleVersion v1p29 = new ModuleVersion("v1.29");
+ private static readonly ModuleVersion v1p30 = new ModuleVersion("v1.30");
+ private static readonly ModuleVersion v1p31 = new ModuleVersion("v1.31");
+ private static readonly ModuleVersion v1p34 = new ModuleVersion("v1.34");
+ private static readonly ModuleVersion v1p35 = new ModuleVersion("v1.35");
+
+ private static readonly ILog log = LogManager.GetLogger(typeof(SpecVersionTransformer));
+ }
+}

Netkan/Validators/CkanValidator.cs

@@ -18,6 +18,10 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService, IGam
  new DownloadArrayValidator(),
  new TagsValidator(),
  new LicensesValidator(),
+ new RelationshipsValidator(),
+ new VersionStrictValidator(),
+ new ReplacedByValidator(),
+ new InstallValidator(),
  new InstallsFilesValidator(downloader, moduleService, game),
  new MatchesKnownGameVersionsValidator(game),
  new ObeysCKANSchemaValidator(),

Netkan/Validators/NetkanValidator.cs

@@ -12,17 +12,12 @@ public NetkanValidator()
  {
  _validators = new List<IValidator>()
  {
- new SpecVersionFormatValidator(),
  new HasIdentifierValidator(),
  new KrefValidator(),
  new AlphaNumericIdentifierValidator(),
- new RelationshipsValidator(),
  new KrefDownloadMutexValidator(),
  new DownloadVersionValidator(),
  new OverrideValidator(),
- new VersionStrictValidator(),
- new ReplacedByValidator(),
- new InstallValidator(),
  };
  }
 

Spec.md

@@ -127,8 +127,8 @@ reference CKAN client that will read this file.
 For compatibility with pre-release clients, and the v1.0 client, the special
 *integer* `1` should be used.
 
-This document describes the CKAN specification 'v1.26'. Changes since spec `1`
-are marked with **v1.2** through to **v1.26** respectively. For maximum
+This document describes the CKAN specification 'v1.34'. Changes since spec `1`
+are marked with **v1.2** through to **v1.34** respectively. For maximum
 compatibility, using older spec versions is preferred when newer features are
 not required.
 

Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs

@@ -95,46 +95,5 @@ public void DoesNotOverrideExistingProperties()
  "InternalCkanTransformer should not override existing properties."
  );
  }
-
- [TestCase("v1.2", "v1.4", "v1.4")]
- [TestCase("v1.4", "v1.2", "v1.4")]
- public void HigherOfTwoSpecVersionsIsChosen(
- string specVersion, string internalSpecVersion, string expectedSpecVersion
- )
- {
- // Arrange
- const string filePath = "/DoesNotExist.zip";
-
- var internalCkan = new JObject();
- internalCkan["spec_version"] = internalSpecVersion;
-
- var mHttp = new Mock<IHttpService>();
- var mModuleService = new Mock<IModuleService>();
-
- mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
- .Returns(filePath);
-
- mModuleService.Setup(i => i.GetInternalCkan(
- It.IsAny<CkanModule>(), It.IsAny<string>(),
- It.IsAny<GameInstance>()))
- .Returns(internalCkan);
-
- var sut = new InternalCkanTransformer(mHttp.Object, mModuleService.Object, new KerbalSpaceProgram());
-
- var json = new JObject();
- json["spec_version"] = specVersion;
- json["identifier"] = "DoesNotExist";
- json["version"] = "1.0";
- json["download"] = "https://awesomemod.example/AwesomeMod.zip";
-
- // Act
- var result = sut.Transform(new Metadata(json), opts).First();
- var transformedJson = result.Json();
-
- // Assert
- Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expectedSpecVersion),
- "InternalCkanTransformer should use the higher of the two spec_versions."
- );
- }
  }
 }

Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs

@@ -123,34 +123,5 @@ public void TargetMetadataDoesNotOverrideExistingProperty()
  "MetaNetkanTransformer should not override existing properties."
  );
  }
-
- [TestCase("v1.2", "v1.4", "v1.4")]
- [TestCase("v1.4", "v1.2", "v1.4")]
- public void SelectsTheHigherSpecVresion(string specVersion, string targetSpecVersion, string expected)
- {
- // Arrange
- var targetJson = new JObject();
- targetJson["spec_version"] = targetSpecVersion;
-
- var mHttp = new Mock<IHttpService>();
-
- mHttp.Setup(i => i.DownloadText(It.IsAny<Uri>()))
- .Returns(targetJson.ToString());
-
- var sut = new MetaNetkanTransformer(mHttp.Object, null);
-
- var json = new JObject();
- json["spec_version"] = specVersion;
- json["$kref"] = "#/ckan/netkan/http://awesomemod.example/AwesomeMod.netkan";
-
- // Act
- var result = sut.Transform(new Metadata(json), opts).First();
- var transformedJson = result.Json();
-
- // Assert
- Assert.That((string)transformedJson["spec_version"], Is.EqualTo(expected),
- "MetaNetkanTransformer should select the higher of the two spec_versions."
- );
- }
  }
 }

Tests/NetKAN/Validators/IsCkanModuleValidatorTests.cs

@@ -35,7 +35,6 @@ public void DoesNotThrowOnValidCkan()
  );
  }
 
- [TestCase("spec_version")]
  [TestCase("identifier")]
  [TestCase("version")]
  [TestCase("download")]