Update feed2catalog and catalog2registration to properly handle SemVe…

…r 2.0.0 (#145)

* Update feed2catalog and catalog2registration to properly handle SemVer 2.0.0
* Add testing for NuGetVersionUtility
* Add integration tests for creation of catalog items (feed2catalog)
* Add tests for PackageCatalogItem
* Add unit tests for RegistrationCatalogEntry and PackagesFolderPackagePathProvider
* Only call the SemVer 2.0.0 storage factory if it was provided
* Add tests for the SemVer 2.0.0 storage factory
* Update Feed2Catalog tests to add a SemVer2 package
* Add functional tests for registration collector
* Addresses NuGet/NuGetGallery#3560
* Addresses NuGet/NuGetGallery#3680

src/Catalog/CollectorBase.cs

@@ -1,5 +1,6 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
 using System;
 using System.Diagnostics;
 using System.Net;
@@ -15,6 +16,11 @@ public abstract class CollectorBase
 
         public CollectorBase(Uri index, Func<HttpMessageHandler> handlerFunc = null)
         {
+            if (index == null)
+            {
+                throw new ArgumentNullException(nameof(index));
+            }
+
             _handlerFunc = handlerFunc;
             Index = index;
             ServicePointManager.DefaultConnectionLimit = 4;

src/Catalog/Helpers/NuGetVersionUtility.cs

@@ -0,0 +1,118 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using NuGet.Versioning;
+using VDS.RDF;
+using VDS.RDF.Query;
+
+namespace NuGet.Services.Metadata.Catalog.Helpers
+{
+    public static class NuGetVersionUtility
+    {
+        public static string NormalizeVersion(string version)
+        {
+            NuGetVersion parsedVersion;
+            if (!NuGetVersion.TryParse(version, out parsedVersion))
+            {
+                return version;
+            }
+
+            return parsedVersion.ToNormalizedString();
+        }
+
+        public static string NormalizeVersionRange(string versionRange)
+        {
+            VersionRange parsedVersionRange;
+            if (!VersionRange.TryParse(versionRange, out parsedVersionRange))
+            {
+                return versionRange;
+            }
+
+            return parsedVersionRange.ToNormalizedString();
+        }
+
+        public static string GetFullVersionString(string version)
+        {
+            NuGetVersion parsedVersion;
+            if (!NuGetVersion.TryParse(version, out parsedVersion))
+            {
+                return version;
+            }
+
+            return parsedVersion.ToFullString();
+        }
+
+        public static bool IsVersionSemVer2(string version)
+        {
+            NuGetVersion parsedVersion;
+            if (!NuGetVersion.TryParse(version, out parsedVersion))
+            {
+                return false;
+            }
+
+            return parsedVersion.IsSemVer2;
+        }
+
+        public static bool IsVersionRangeSemVer2(string versionRange)
+        {
+            VersionRange parsedVersionRange;
+            if (!VersionRange.TryParse(versionRange, out parsedVersionRange))
+            {
+                return false;
+            }
+
+            if (parsedVersionRange.HasLowerBound && parsedVersionRange.MinVersion.IsSemVer2)
+            {
+                return true;
+            }
+
+            if (parsedVersionRange.HasUpperBound && parsedVersionRange.MaxVersion.IsSemVer2)
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        public static bool IsGraphSemVer2(string version, string resourceUri, IGraph graph)
+        {
+            // Is the package version itself SemVer 2.0.0?
+            if (IsVersionSemVer2(version))
+            {
+                return true;
+            }
+
+            if (resourceUri == null)
+            {
+                throw new ArgumentNullException(nameof(resourceUri));
+            }
+
+            if (graph == null)
+            {
+                throw new ArgumentNullException(nameof(graph));
+            }
+
+            // Are any of the dependency version ranges SemVer 2.0.0?
+            var sparql = new SparqlParameterizedString
+            {
+                CommandText = Utils.GetResource("sparql.SelectDistinctDependencyVersionRanges.rq")
+            };
+            sparql.SetUri("resourceUri", new Uri(resourceUri));
+            var query = sparql.ToString();
+
+            TripleStore store = new TripleStore();
+            store.Add(graph, true);
+            foreach (SparqlResult row in SparqlHelpers.Select(store, query))
+            {
+                var unparsedVersionRange = row["versionRange"].ToString();
+                if (IsVersionRangeSemVer2(unparsedVersionRange))
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+    }
+}

src/Catalog/Helpers/XsltHelper.cs

@@ -1,10 +1,12 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-using NuGet.Versioning;
+
 using System.Collections.Generic;
 using System.Linq;
 using System.Xml;
 using System.Xml.XPath;
+using NuGet.Services.Metadata.Catalog.Helpers;
+using NuGet.Versioning;
 
 namespace NuGet.Services.Metadata.Catalog
 {
@@ -40,22 +42,17 @@ public string LowerCase(string original)
 
         public string NormalizeVersion(string original)
         {
-            NuGetVersion nugetVersion;
-            if (NuGetVersion.TryParse(original, out nugetVersion))
-            {
-                return nugetVersion.ToNormalizedString();
-            }
-            return original;
+            return NuGetVersionUtility.NormalizeVersion(original);
+        }
+
+        public string GetFullVersionString(string original)
+        {
+            return NuGetVersionUtility.GetFullVersionString(original);
         }
 
         public string NormalizeVersionRange(string original)
         {
-            VersionRange versionRange;
-            if (VersionRange.TryParse(original, out versionRange))
-            {
-                return versionRange.ToString();
-            }
-            return original;
+            return NuGetVersionUtility.NormalizeVersionRange(original);
         }
 
         public string IsPrerelease(string original)

src/Catalog/NuGet.Services.Metadata.Catalog.csproj

@@ -114,6 +114,7 @@
     <Compile Include="CloudBlobStorageExtensions.cs" />
     <Compile Include="CommitCollector.cs" />
     <Compile Include="CommitMetadata.cs" />
+    <Compile Include="Helpers\NuGetVersionUtility.cs" />
     <Compile Include="Persistence\StorageListItem.cs" />
     <Compile Include="Utilities.cs" />
     <Compile Include="DeleteCatalogItem.cs" />
@@ -232,6 +233,7 @@
     <EmbeddedResource Include="sparql\ConstructRegistrationPageContentGraph.rq" />
     <None Include="NuGet.Services.Metadata.Catalog.nuspec" />
     <None Include="packages.config" />
+    <EmbeddedResource Include="sparql\SelectDistinctDependencyVersionRanges.rq" />
   </ItemGroup>
   <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

src/Catalog/PackageCatalogItem.cs

@@ -1,22 +1,24 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
 using System;
 using System.Linq;
-using System.Xml.Linq;
 using Newtonsoft.Json.Linq;
+using NuGet.Services.Metadata.Catalog.Helpers;
 using NuGet.Services.Metadata.Catalog.Persistence;
 using VDS.RDF;
 
 namespace NuGet.Services.Metadata.Catalog
 {
     public class PackageCatalogItem : AppendOnlyCatalogItem
     {
-        NupkgMetadata _nupkgMetadata;
-        DateTime? _createdDate;
-        DateTime? _lastEditedDate;
-        DateTime? _publishedDate;
-        string _id;
-        string _version;
+        private readonly NupkgMetadata _nupkgMetadata;
+        private readonly DateTime? _createdDate;
+        private readonly DateTime? _lastEditedDate;
+        private readonly DateTime? _publishedDate;
+        private string _id;
+        private string _fullVersion;
+        private string _normalizedVersion;
 
         public PackageCatalogItem(NupkgMetadata nupkgMetadata, DateTime? createdDate = null, DateTime? lastEditedDate = null, DateTime? publishedDate = null, string licenseNames = null, string licenseReportUrl = null)
         {
@@ -119,7 +121,8 @@ protected void SetIdVersionFromGraph(IGraph graph)
             Triple version = graph.GetTriplesWithSubjectPredicate(resource.Subject, versionPredicate).FirstOrDefault();
             if (version != null)
             {
-                _version = ((ILiteralNode)version.Object).Value;
+                _fullVersion = ((ILiteralNode)version.Object).Value;
+                _normalizedVersion = NuGetVersionUtility.NormalizeVersion(_fullVersion);
             }
         }
 
@@ -169,17 +172,17 @@ public override IGraph CreatePageContent(CatalogContext context)
                 graph.Assert(subject, idPredicate, graph.CreateLiteralNode(_id));
             }
 
-            if (_version != null)
+            if (_fullVersion != null)
             {
-                graph.Assert(subject, versionPredicate, graph.CreateLiteralNode(_version));
+                graph.Assert(subject, versionPredicate, graph.CreateLiteralNode(_fullVersion));
             }
 
             return graph;
         }
 
         protected override string GetItemIdentity()
         {
-            return (_id + "." + _version).ToLowerInvariant();
+            return (_id + "." + _normalizedVersion).ToLowerInvariant();
         }
     }
 }

src/Catalog/Registration/PackagesFolderPackagePathProvider.cs

@@ -1,12 +1,16 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
+using NuGet.Services.Metadata.Catalog.Helpers;
+
 namespace NuGet.Services.Metadata.Catalog.Registration
 {
     public class PackagesFolderPackagePathProvider : IPackagePathProvider
     {
         public string GetPackagePath(string id, string version)
         {
+            version = NuGetVersionUtility.NormalizeVersion(version);
+
             return $"packages/{id.ToLowerInvariant()}.{version.ToLowerInvariant()}.nupkg";
         }
     }

src/Catalog/Registration/RegistrationCatalogEntry.cs

@@ -1,12 +1,25 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using VDS.RDF;
 
 namespace NuGet.Services.Metadata.Catalog.Registration
 {
+    /// <summary>
+    /// A delegate used to determine whether a package should be included in a registration hive. The delegate is
+    /// important because some registration hives exclude specific packages (typically packages that break older
+    /// clients). The first example of this difference is SemVer 2.0.0 packages, which should be excluded from the legacy
+    /// registration hives.
+    /// </summary>
+    /// <param name="key">The package key. This contains the ID and version of the package.</param>
+    /// <param name="resourceUri">The URI (identifier) of the package in the RDF graph.</param>
+    /// <param name="graph">The RDF graph containing metadata about the package.</param>
+    /// <returns>True if the package should be included in the registration hive. False otherwise.</returns>
+    public delegate bool ShouldIncludeRegistrationPackage(RegistrationEntryKey key, string resourceUri, IGraph graph);
+
     public class RegistrationCatalogEntry
     {
         public RegistrationCatalogEntry(string resourceUri, IGraph graph, bool isExistingItem)
@@ -20,16 +33,22 @@ public RegistrationCatalogEntry(string resourceUri, IGraph graph, bool isExistin
         public IGraph Graph { get; set; }
         public bool IsExistingItem { get; set; }
 
-        public static KeyValuePair<RegistrationEntryKey, RegistrationCatalogEntry> Promote(string resourceUri, IGraph graph, bool isExistingItem)
+        public static KeyValuePair<RegistrationEntryKey, RegistrationCatalogEntry> Promote(
+            string resourceUri,
+            IGraph graph,
+            ShouldIncludeRegistrationPackage shouldInclude,
+            bool isExistingItem)
         {
             INode subject = graph.CreateUriNode(new Uri(resourceUri));
             string version = graph.GetTriplesWithSubjectPredicate(subject, graph.CreateUriNode(Schema.Predicates.Version)).First().Object.ToString();
 
             RegistrationEntryKey registrationEntryKey = new RegistrationEntryKey(RegistrationKey.Promote(resourceUri, graph), version);
 
-            RegistrationCatalogEntry registrationCatalogEntry = IsDelete(subject, graph) 
-                ? null 
-                : new RegistrationCatalogEntry(resourceUri, graph, isExistingItem);
+            RegistrationCatalogEntry registrationCatalogEntry = null;
+            if (!IsDelete(subject, graph) && shouldInclude(registrationEntryKey, resourceUri, graph))
+            {
+                registrationCatalogEntry = new RegistrationCatalogEntry(resourceUri, graph, isExistingItem);
+            }
 
             return new KeyValuePair<RegistrationEntryKey, RegistrationCatalogEntry>(registrationEntryKey, registrationCatalogEntry);
         }