[build] Remove Mono submodule (#3371)

Context: https://github.com/xamarin/xamarin-android/projects/10

One of the goals behind the Mono SDK archive was to enable product
builds without having the entire Mono source tree present, which
means it should be possible to remove the Mono git submodule and
build the entire xamarin-android repo without any issues.

At the same time, we want to keep the ability to build Mono from
source, if necessary, for situations when a developer works on the
BCL or the runtime itself and there's no appropriate binary archive
present.

To support these two requirements:

  * Remove the Mono submodule
  * Put Mono repo commit reference into the `.external` file
  * Make distinction between "commercial" and "regular" bits in the
    `.external` file so that we don't attempt to check out everything
    mentioned in there in situations when a developer doesn't have
    access to the commercial bits.
  * Use Mono linker sources from the archive
  * Remove `zlib-helper.c` from the `libmonodroid` build - the API
    implemented in there is no longer used by Mono.
  * `Xamarin.Android.Cecil.dll` and `Xamarin.Android.Cecil.Mdb.dll`
    are no longer built from source.  Instead we use Cecil to reshape
    `Mono.Cecil` and conjure the two renamed and re-signed assemblies.
  * All the code which used `Xamarin.Android.Cecil` project reference
    now simply references the conjured assemblies (which are also
    added to the bundle)
  * Third Party Notices are generated from licenses found in the
    Mono archive.
  * Windows build downloads also the Darwin archive - in order to be
    able to use Mono Linker sources as well as licenses contained in
    this archive.

.external

@@ -1 +1,2 @@
 xamarin/monodroid:master@237e0bd9f105b9842778ef82161a20c6d4497a40
+mono/mono:2019-06@afcf28a3660b069721e5c441dfe5bb7548e3204f

.gitignore

@@ -3,7 +3,6 @@
 bin
 Configuration.Override.props
 Configuration.OperatingSystem.props
-msfinal.pub
 obj
 packages
 .DS_Store

.gitmodules

@@ -22,10 +22,6 @@
     path = external/mman-win32
     url = https://github.com/witwall/mman-win32.git
     branch = master
-[submodule "external/mono"]
-    path = external/mono
-    url = https://github.com/mono/mono.git
-    branch = 2019-06
 [submodule "external/mxe"]
     path = external/mxe
     url = https://github.com/xamarin/mxe.git

Configuration.props

@@ -88,7 +88,7 @@
     <MonoRequiredMaximumVersion Condition=" '$(MonoRequiredMaximumVersion)' == '' ">6.5.0</MonoRequiredMaximumVersion>
     <IgnoreMaxMonoVersion Condition=" '$(IgnoreMaxMonoVersion)' == '' ">True</IgnoreMaxMonoVersion>
     <MonoRequiredDarwinMinimumVersion>$(MonoRequiredMinimumVersion).98</MonoRequiredDarwinMinimumVersion>
-    <LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\external\linker\src</LinkerSourceDirectory>
+    <LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\sdks\out\android-sources\external\linker\src</LinkerSourceDirectory>
     <OpenTKSourceDirectory>$(MSBuildThisFileDirectory)external\opentk</OpenTKSourceDirectory>
     <MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Linux' ">\usr</MingwZlibRootDirectory>
     <MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Darwin' ">$(HostHomebrewPrefix)\opt\mingw-zlib\usr</MingwZlibRootDirectory>

build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+using Mono.Cecil;
+
+public class ConjureXamarinAndroidCecil
+{
+	const string BaseNameReplacement = "Xamarin.Android.Cecil";
+	const string CecilAssemblyName = BaseNameReplacement;
+	const string CecilMdbAssemblyName = BaseNameReplacement + ".Mdb";
+
+	static readonly List<string> internalsVisibleTo = new List<string> {
+		"Xamarin.Android.Cecil.Pdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db",
+		"Xamarin.Android.Cecil.Mdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db"
+	};
+
+	public static int Main (string[] args)
+	{
+		if (args.Length < 2) {
+			Console.WriteLine ("Usage: <input directory> <output directory>");
+			Console.WriteLine ("  <input directory> must have Mono.Cecil.dll and Mono.Cecil.Mdb.dll assemblies");
+			return 1;
+		}
+
+		string inputDir = args [0];
+		string inputFilePath = Path.Combine (inputDir, "Mono.Cecil.dll");
+		string outputDirPath = args [1];
+
+		var resolver = new DefaultAssemblyResolver ();
+		resolver.AddSearchDirectory (Path.GetDirectoryName (inputFilePath));
+		var rp = new ReaderParameters () {
+			AssemblyResolver = resolver,
+			ReadSymbols = true
+		};
+		var monoCecil = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
+		monoCecil.Name.Name = CecilAssemblyName;
+
+		var ivtCtor = monoCecil.MainModule.ImportReference (typeof (System.Runtime.CompilerServices.InternalsVisibleToAttribute).GetConstructor (new []{typeof(string)}));
+		foreach (string ivtParam in internalsVisibleTo) {
+			var ca = new CustomAttribute (ivtCtor);
+			ca.ConstructorArguments.Add (new CustomAttributeArgument (monoCecil.MainModule.TypeSystem.String, ivtParam));
+			monoCecil.CustomAttributes.Add (ca);
+		}
+
+		var wp = new WriterParameters {
+			WriteSymbols = true
+		};
+
+		monoCecil.Write (Path.Combine (outputDirPath, $"{CecilAssemblyName}.dll"), wp);
+
+		inputFilePath = Path.Combine (inputDir, "Mono.Cecil.Mdb.dll");
+		var monoCecilMdb = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
+		monoCecilMdb.Name.Name = CecilMdbAssemblyName;
+
+		AssemblyNameReference monoCecilRef = monoCecilMdb.MainModule.AssemblyReferences.Single (r => String.Compare ("Mono.Cecil", r.Name, StringComparison.Ordinal) == 0);
+		monoCecilRef.Name = CecilAssemblyName;
+		monoCecilMdb.Write (Path.Combine (outputDirPath, $"{CecilMdbAssemblyName}.dll"), wp);
+
+		return 0;
+	}
+}

build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.csproj

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{C876DA71-8573-4CEF-9149-716D72455ED4}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>Xamarin.Android.Prepare</RootNamespace>
+    <AssemblyName>conjure-xamarin-android-cecil</AssemblyName>
+  </PropertyGroup>
+  <Import Project="..\..\Configuration.props" />
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <Optimize>false</Optimize>
+    <OutputPath>..\..\bin\BuildDebug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <Optimize>true</Optimize>
+    <OutputPath>..\..\bin\BuildRelease</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ExternalConsole>true</ExternalConsole>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="Mono.Cecil">
+      <HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="conjure-xamarin-android-cecil.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+</Project>

build-tools/remap-assembly-ref/remap-assembly-ref.csproj

@@ -27,10 +27,10 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />
-    <ProjectReference Include="..\..\external\Java.Interop\src\Xamarin.Android.Cecil\Xamarin.Android.Cecil.csproj">
-      <Project>{15945D4B-FF56-4BCC-B598-2718D199DD08}</Project>
-      <Name>Xamarin.Android.Cecil</Name>
-    </ProjectReference>  </ItemGroup>
+    <Reference Include="Mono.Cecil">
+      <HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
+    </Reference>
+  </ItemGroup>
   <ItemGroup>
     <Compile Include="remap-assembly-ref.cs" />
   </ItemGroup>

build-tools/scripts/XAVersionInfo.targets

@@ -6,15 +6,13 @@
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitsInRange" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitTime" />
   <UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll"  TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitInfo" />
+  <Import Project="..\..\bin\Build$(Configuration)\MonoGitHash.props" />
   <Target Name="_GetSubmodulesVersionInfo">
     <ItemGroup>
       <!-- If anything changes in this list, make sure to update:
            src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets (the _GenerateXACommonProps target)
            build-tools/create-vsix/create-vsix.targets (the _CreateDependencies target)
       -->
-      <_SubmoduleBranchInfo Include="external/mono">
-        <OutputPropertyName>_BuildInfo_MonoCommit</OutputPropertyName>
-      </_SubmoduleBranchInfo>
       <_SubmoduleBranchInfo Include="external/Java.Interop">
         <OutputPropertyName>_BuildInfo_JavaInteropCommit</OutputPropertyName>
       </_SubmoduleBranchInfo>

build-tools/xaprepare/xaprepare/Application/BuildInfo.cs

@@ -105,7 +105,13 @@ void DetermineBundleHashes (Context context)
 			LibZipHash = EnsureHash ("LibZip", Utilities.ShortenGitHash (FullLibZipHash));
 
 			Log.StatusLine ($"  {context.Characters.Bullet} Mono commit hash", ConsoleColor.Gray);
-			FullMonoHash = git.GetTopCommitHash (context.Properties.GetRequiredValue (KnownProperties.MonoSourceFullPath), shortHash: false);
+			List<ExternalGitDependency> externalDependencies = ExternalGitDependency.GetDependencies (context, Configurables.Paths.ExternalGitDepsFilePath, quiet: true);
+			ExternalGitDependency mono = externalDependencies?.Where (
+				eg => eg != null &&
+				      String.Compare ("mono", eg.Owner, StringComparison.Ordinal) == 0 &&
+				      String.Compare ("mono", eg.Name, StringComparison.Ordinal) == 0).FirstOrDefault ();
+
+			FullMonoHash = mono?.Commit?.Trim ();
 			MonoHash = EnsureHash ("Mono", Utilities.ShortenGitHash (FullMonoHash));
 
 			if (Configurables.Paths.BundleVersionHashFiles == null || Configurables.Paths.BundleVersionHashFiles.Count == 0) {

build-tools/xaprepare/xaprepare/Application/Context.cs

@@ -329,6 +329,13 @@ public string DebugFileExtension {
 		/// </summary>
 		public RefreshableComponent ComponentsToRefresh { get; set; }
 
+		/// <summary>
+		///   Set by <see cref="Step_DownloadMonoArchive"/> if the archive has been downloaded and validated, so
+		///   that the <see cref="Step_BuildMonoRuntimes"/> step doesn't have to re-download the archive, should
+		///   bundle be absent and <see cref="Step_PrepareBundle"/> fails.
+		/// </summary>
+		public bool MonoAlreadyBuilt { get; set; }
+
 		static Context ()
 		{
 			Instance = new Context ();
@@ -742,8 +749,10 @@ public async Task<bool> Init (string scenarioName = null)
 			Banner ("Updating Git submodules");
 
 			var git = new GitRunner (this);
-			if (!await git.SubmoduleUpdate ())
-				Log.WarningLine ("Failed to update Git submodules");
+			if (!await git.SubmoduleUpdate ()) {
+				Log.ErrorLine ("Failed to update Git submodules");
+				return false;
+			}
 
 			BuildInfo = new BuildInfo ();
 			await BuildInfo.GatherGitInfo (this);

build-tools/xaprepare/xaprepare/Application/ExternalGitDependency.cs

@@ -33,11 +33,13 @@ sealed class ExternalGitDependency : AppObject
 		public string Name   { get; private set; }
 		public string Owner  { get; private set; }
 
-		public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath)
+		public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath, bool quiet = false)
 		{
-			Log.Instance.StatusLine ($"  {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
+			if (!quiet)
+				Log.Instance.StatusLine ($"  {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
 			string[] unparsedExternals = File.ReadAllLines (externalFilePath);
 			var externals = new List<ExternalGitDependency> (unparsedExternals.Length);
+			bool includeCommercial = context.CheckCondition (KnownConditions.IncludeCommercial);
 
 			foreach (string external in unparsedExternals) {
 				Match match = externalRegex.Match (external);
@@ -47,14 +49,22 @@ public static List<ExternalGitDependency> GetDependencies (Context context, stri
 						continue;
 					}
 
+					string owner = match.Groups["owner"].Value;
+					string repo = match.Groups["repo"].Value;
+					if (!includeCommercial && Configurables.Defaults.CommercialExternalDependencies.Contains ($"{owner}/{repo}")) {
+						Log.Instance.DebugLine ($"Ignoring external commercial dependency '{owner}/{repo}'");
+						continue;
+					}
+
 					var e = new ExternalGitDependency {
 						Branch = match.Groups["branch"].Value,
 						Commit = match.Groups["commit"].Value,
-						Name   = match.Groups["repo"].Value,
-						Owner  = match.Groups["owner"].Value,
+						Name   = repo,
+						Owner  = owner,
 					};
 					externals.Add (e);
-					Log.Instance.StatusLine ($"    {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
+					if (!quiet)
+						Log.Instance.StatusLine ($"    {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
 				}
 			}
 

build-tools/xaprepare/xaprepare/Application/KnownConditions.cs

@@ -18,5 +18,11 @@ public enum KnownConditions
 		///   only a single program (e.g. the UpdateMono scenario) but not the rest. Default: unset. <see cref="Scenario_UpdateMono" />
 		/// </summary>
 		IgnoreMissingPrograms,
+
+		/// <summary>
+		///   If set, will cause checkout of the commercial dependencies mentioned in the `.external` file at
+		///   the top of the repository. Default: unset
+		/// </summary>
+		IncludeCommercial,
 	}
 }

build-tools/xaprepare/xaprepare/Application/Utilities.cs

@@ -28,6 +28,84 @@ static partial class Utilities
 
 		public static readonly Encoding UTF8NoBOM = new UTF8Encoding (false);
 
+		public static bool AbiChoiceChanged (Context context)
+		{
+			string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
+			if (!File.Exists (cacheFile)) {
+				Log.DebugLine ($"Enabled ABI cache file not found at {cacheFile}");
+				return true;
+			}
+
+			var oldAbis = new HashSet<string> (StringComparer.Ordinal);
+			foreach (string l in File.ReadAllLines (cacheFile)) {
+				string line = l?.Trim ();
+				if (String.IsNullOrEmpty (line) || oldAbis.Contains (line))
+					continue;
+				oldAbis.Add (line);
+			}
+
+			HashSet<string> currentAbis = null;
+			FillCurrentAbis (context, ref currentAbis);
+
+			if (oldAbis.Count != currentAbis.Count)
+				return true;
+
+			foreach (string abi in oldAbis) {
+				if (!currentAbis.Contains (abi))
+					return true;
+			}
+
+			return false;
+		}
+
+		public static void SaveAbiChoice (Context context)
+		{
+			HashSet<string> currentAbis = null;
+			FillCurrentAbis (context, ref currentAbis);
+
+			string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
+			Log.DebugLine ($"Writing ABI cache file {cacheFile}");
+			File.WriteAllLines (cacheFile, currentAbis);
+		}
+
+		static void FillCurrentAbis (Context context, ref HashSet<string> currentAbis)
+		{
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis).Trim (), ref currentAbis);
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetAotAbis).Trim (), ref currentAbis);
+			Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedHostJitAbis).Trim (), ref currentAbis);
+		}
+
+		public static void PropagateXamarinAndroidCecil (Context context)
+		{
+			const string CecilAssembly = "Xamarin.Android.Cecil.dll";
+
+			CopyFile (
+				Path.Combine (Configurables.Paths.InstallMSBuildDir, CecilAssembly),
+				Path.Combine (Configurables.Paths.ExternalJavaInteropDir, "bin", context.Configuration, CecilAssembly)
+			);
+		}
+
+		public static async Task<bool> BuildRemapRef (Context context, bool haveManagedRuntime, string managedRuntime, bool quiet = false)
+		{
+			if (!quiet)
+				Log.StatusLine ("Building remap-assembly-ref");
+
+			var msbuild = new MSBuildRunner (context);
+			string projectPath = Path.Combine (Configurables.Paths.BuildToolsDir, "remap-assembly-ref", "remap-assembly-ref.csproj");
+			bool result = await msbuild.Run (
+				projectPath: projectPath,
+				logTag: "remap-assembly-ref",
+				binlogName: "build-remap-assembly-ref"
+			);
+
+			if (!result) {
+				Log.ErrorLine ("Failed to build remap-assembly-ref");
+				return false;
+			}
+
+			return true;
+		}
+
 		public static string ToXamarinAndroidPropertyValue (ICollection<string> coll)
 		{
 			if (coll == null)

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Unix.cs

@@ -12,6 +12,7 @@ partial class Defaults
 		partial class Paths
 		{
 			static string BundleOSType                  => Context.Instance.OS.Type;
+			static string ArchiveOSType                 => Context.Instance.OS.Type;
 
 			public static string BCLTestsSourceDir      => GetCachedPath (ref bclTestsSourceDir, () => Path.Combine (MonoProfileDir, "tests"));
 			public static string BCLAssembliesSourceDir => MonoProfileDir;

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs

@@ -34,6 +34,7 @@ partial class Defaults
 		partial class Paths
 		{
 			static string BundleOSType                                            => "Darwin"; // Windows doesn't build the bundle
+			static string ArchiveOSType                                           => "Darwin"; // Windows need sources from there
 
 			// Windows doesn't build the bundle so we need to look in the XA framework dir as installed
 			public static string BCLAssembliesSourceDir                           => InstallBCLFrameworkDir;