Initial Component catalog refactor

Microsoft.ML.sln

@@ -115,6 +115,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Analyzer", "sr
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.StaticPipelineTesting", "test\Microsoft.ML.StaticPipelineTesting\Microsoft.ML.StaticPipelineTesting.csproj", "{8B38BF24-35F4-4787-A9C5-22D35987106E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.DnnAnalyzer", "src\Microsoft.ML.DnnAnalyzer\Microsoft.ML.DnnAnalyzer\Microsoft.ML.DnnAnalyzer.csproj", "{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -419,6 +421,14 @@ Global
 		{8B38BF24-35F4-4787-A9C5-22D35987106E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{8B38BF24-35F4-4787-A9C5-22D35987106E}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
 		{8B38BF24-35F4-4787-A9C5-22D35987106E}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Debug-Intrinsics|Any CPU.Build.0 = Debug|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Release-Intrinsics|Any CPU.ActiveCfg = Release|Any CPU
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA}.Release-Intrinsics|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -466,6 +476,7 @@ Global
 		{570A0B8A-5463-44D2-8521-54C0CA4CACA9} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{6DEF0F40-3853-47B3-8165-5F24BA5E14DF} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{8B38BF24-35F4-4787-A9C5-22D35987106E} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{73DAAC82-D308-48CC-8FFE-3B037F8BBCCA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

build.proj

@@ -7,6 +7,7 @@
 
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), Directory.Build.props))\Directory.Build.props" />
 
+  <Import Project="build/ExternalBenchmarkDataFiles.props" />
   <Import Project="$(ToolsDir)VersionTools.targets" Condition="Exists('$(ToolsDir)VersionTools.targets')" />
   <UsingTask TaskName="DownloadFilesFromUrl" AssemblyFile="$(ToolsDir)Microsoft.DotNet.Build.Tasks.dll"/>
   <PropertyGroup>
@@ -78,11 +79,16 @@
   <ItemGroup>
     <TestFile Include="$(MSBuildThisFileDirectory)/test/data/external/winequality-white.csv"
       Url="https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv"
-      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/winequality-white.csv" />
-
-    <TestFile Condition="'$(IncludeBenchmarkData)' == 'true'" Include="$(MSBuildThisFileDirectory)/test/data/external/WikiDetoxAnnotated160kRows.tsv"
-      Url="https://aka.ms/tlc-resources/benchmarks/WikiDetoxAnnotated160kRows.tsv"
-      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/WikiDetoxAnnotated160kRows.tsv" />
+      DestinationFile="$(MSBuildThisFileDirectory)test/data/external/winequality-white.csv" />    
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IncludeBenchmarkData)' == 'true'">
+    <BenchmarkFile Update="@(BenchmarkFile)">
+      <Url>https://aka.ms/tlc-resources/benchmarks/%(Identity)</Url>
+      <DestinationFile>$(MSBuildThisFileDirectory)/test/data/external/%(Identity)</DestinationFile>
+    </BenchmarkFile>
+
+    <TestFile Include="@(BenchmarkFile->'$(MSBuildThisFileDirectory)/test/data/external/%(Identity)')" />
   </ItemGroup>
 
   <Target Name="DownloadExternalTestFiles" Inputs="@(TestFile)" Outputs="%(TestFile.DestinationFile)">

build/Dependencies.props

@@ -13,7 +13,7 @@
     <BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
     <TensorFlowVersion>1.10.0</TensorFlowVersion>
     <SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
-
+    <SystemMemoryVersion>4.5.1</SystemMemoryVersion>
     <MicrosoftCodeAnalysisCSharpVersion>2.9.0</MicrosoftCodeAnalysisCSharpVersion>
     <MicrosoftCSharpVersion>4.5.0</MicrosoftCSharpVersion>
     <SystemCompositionVersion>1.2.0</SystemCompositionVersion>

build/ExternalBenchmarkDataFiles.props

@@ -0,0 +1,8 @@
+<Project>
+  <ItemGroup>
+    <BenchmarkFile Include="MSLRWeb10KTest240kRows.tsv" />
+    <BenchmarkFile Include="MSLRWeb10KTrain720kRows.tsv" />
+    <BenchmarkFile Include="MSLRWeb10KValidate240kRows.tsv" />
+    <BenchmarkFile Include="WikiDetoxAnnotated160kRows.tsv" />
+  </ItemGroup>
+</Project>

pkg/Microsoft.ML/Microsoft.ML.nupkgproj

@@ -12,6 +12,7 @@
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPackageVersion)" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="$(SystemThreadingTasksDataflowPackageVersion)" />
     <PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/Common/AssemblyLoadingUtils.cs

@@ -0,0 +1,178 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.ML.Runtime.Internal.Utilities;
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Reflection;
+
+namespace Microsoft.ML.Runtime
+{
+    internal static class AssemblyLoadingUtils
+    {
+        /// <summary>
+        /// Make sure the given assemblies are loaded and that their loadable classes have been catalogued.
+        /// </summary>
+        public static void LoadAndRegister(IHostEnvironment env, string[] assemblies)
+        {
+            Contracts.AssertValue(env);
+
+            if (Utils.Size(assemblies) > 0)
+            {
+                foreach (string path in assemblies)
+                {
+                    Exception ex = null;
+                    try
+                    {
+                        // REVIEW: Will LoadFrom ever return null?
+                        Contracts.CheckNonEmpty(path, nameof(path));
+                        var assem = LoadAssembly(env, path);
+                        if (assem != null)
+                            continue;
+                    }
+                    catch (Exception e)
+                    {
+                        ex = e;
+                    }
+
+                    // If it is a zip file, load it that way.
+                    ZipArchive zip;
+                    try
+                    {
+                        zip = ZipFile.OpenRead(path);
+                    }
+                    catch (Exception e)
+                    {
+                        // Couldn't load as an assembly and not a zip, so warn the user.
+                        ex = ex ?? e;
+                        Console.Error.WriteLine("Warning: Could not load '{0}': {1}", path, ex.Message);
+                        continue;
+                    }
+
+                    string dir;
+                    try
+                    {
+                        dir = CreateTempDirectory();
+                    }
+                    catch (Exception e)
+                    {
+                        throw Contracts.ExceptIO(e, "Creating temp directory for extra assembly zip extraction failed: '{0}'", path);
+                    }
+
+                    try
+                    {
+                        zip.ExtractToDirectory(dir);
+                    }
+                    catch (Exception e)
+                    {
+                        throw Contracts.ExceptIO(e, "Extracting extra assembly zip failed: '{0}'", path);
+                    }
+
+                    LoadAssembliesInDir(env, dir);
+                }
+            }
+        }
+
+        public static IDisposable CreateAssemblyRegistrar(IHostEnvironment env, string loadAssembliesPath = null)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValueOrNull(loadAssembliesPath);
+
+            return new AssemblyRegistrar(env, loadAssembliesPath);
+        }
+
+        public static void RegisterCurrentLoadedAssemblies(IHostEnvironment env)
+        {
+            Contracts.CheckValue(env, nameof(env));
+
+            foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
+            {
+                // Ignore dynamic assemblies.
+                if (a.IsDynamic)
+                    continue;
+
+                env.ComponentCatalog.RegisterAssembly(a);
+            }
+        }
+
+        private static string CreateTempDirectory()
+        {
+            string dir = GetTempPath();
+            Directory.CreateDirectory(dir);
+            return dir;
+        }
+
+        private static string GetTempPath()
+        {
+            Guid guid = Guid.NewGuid();
+            return Path.GetFullPath(Path.Combine(Path.GetTempPath(), "MLNET_" + guid.ToString()));
+        }
+
+        private static void LoadAssembliesInDir(IHostEnvironment env, string dir)
+        {
+            if (!Directory.Exists(dir))
+                return;
+
+            // Load all dlls in the given directory.
+            var paths = Directory.EnumerateFiles(dir, "*.dll");
+            foreach (string path in paths)
+            {
+                LoadAssembly(env, path);
+            }
+        }
+
+        /// <summary>
+        /// Given an assembly path, load the assembly and register it with the ComponentCatalog.
+        /// </summary>
+        private static Assembly LoadAssembly(IHostEnvironment env, string path)
+        {
+            try
+            {
+                var assembly = Assembly.LoadFrom(path);
+                env.ComponentCatalog.RegisterAssembly(assembly);
+                return assembly;
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+
+        private sealed class AssemblyRegistrar : IDisposable
+        {
+            private readonly IHostEnvironment _env;
+
+            public AssemblyRegistrar(IHostEnvironment env, string path)
+            {
+                _env = env;
+
+                RegisterCurrentLoadedAssemblies(_env);
+
+                if (!string.IsNullOrEmpty(path))
+                {
+                    LoadAssembliesInDir(_env, path);
+                    path = Path.Combine(path, "AutoLoad");
+                    LoadAssembliesInDir(_env, path);
+                }
+
+                AppDomain.CurrentDomain.AssemblyLoad += CurrentDomainAssemblyLoad;
+            }
+
+            public void Dispose()
+            {
+                AppDomain.CurrentDomain.AssemblyLoad -= CurrentDomainAssemblyLoad;
+            }
+
+            private void CurrentDomainAssemblyLoad(object sender, AssemblyLoadEventArgs args)
+            {
+                // Don't try to index dynamic generated assembly
+                if (args.LoadedAssembly.IsDynamic)
+                    return;
+
+                _env.ComponentCatalog.RegisterAssembly(args.LoadedAssembly);
+            }
+        }
+    }
+}

src/Microsoft.ML.Api/ApiUtils.cs

@@ -19,11 +19,10 @@ private static OpCode GetAssignmentOpCode(Type t)
         {
             // REVIEW: This should be a Dictionary<Type, OpCode> based solution.
             // DvTypes, strings, arrays, all nullable types, VBuffers and UInt128.
-            if (t == typeof(DvInt8) || t == typeof(DvInt4) || t == typeof(DvInt2) || t == typeof(DvInt1) ||
-                t == typeof(DvBool) || t == typeof(DvText) || t == typeof(string) || t.IsArray ||
+            if (t == typeof(ReadOnlyMemory<char>) || t == typeof(string) || t.IsArray ||
                 (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(VBuffer<>)) ||
                 (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Nullable<>)) ||
-                t == typeof(DvDateTime) || t == typeof(DvDateTimeZone) || t == typeof(DvTimeSpan) || t == typeof(UInt128))
+                t == typeof(DateTime) || t == typeof(DateTimeOffset) || t == typeof(TimeSpan) || t == typeof(UInt128))
             {
                 return OpCodes.Stobj;
             }

src/Microsoft.ML.Api/ComponentCreation.cs

@@ -439,7 +439,7 @@ private static TRes CreateCore<TRes, TArgs, TSig>(IHostEnvironment env, TArgs ar
         {
             env.CheckValue(args, nameof(args));
 
-            var classes = ComponentCatalog.FindLoadableClasses<TArgs, TSig>();
+            var classes = env.ComponentCatalog.FindLoadableClasses<TArgs, TSig>();
             if (classes.Length == 0)
                 throw env.Except("Couldn't find a {0} class that accepts {1} as arguments.", typeof(TRes).Name, typeof(TArgs).FullName);
             if (classes.Length > 1)