[native] Real shared libraries in APK /lib directories

Documentation/docs-mobile/messages/index.md

@@ -1,4 +1,4 @@
----
+--
 title: .NET for Android errors and warnings reference
 description: Build and deployment error and warning codes in .NET for Android, their meanings, and guidance on how to address them.
 ms.date: 04/11/2024
@@ -104,6 +104,7 @@ or 'Help->Report a Problem' in Visual Studio for Mac.
 + [XA0139](xa0139.md): `@(AndroidAsset)` `{0}` has invalid `DeliveryType` metadata of `{1}`. Supported values are `installtime`, `ondemand` or `fastfollow`
 + [XA0140](xa0140.md):
 + [XA0141](xa0141.md): NuGet package '{0}' version '{1}' contains a shared library '{2}' which is not correctly aligned. See https://developer.android.com/guide/practices/page-sizes for more details
++ [XA0142](xa0142.md): Command '{0}' failed.\n{1}
 
 ## XA1xxx: Project related
 

Documentation/docs-mobile/messages/xa0142.md

@@ -0,0 +1,17 @@
+---
+title: .NET for Android error XA0142
+description: XA0141 error code
+ms.date: 11/09/2024
+---
+# .NET for Android warning XA0142
+
+## Issue
+
+Command '{0}' failed.\n{1}
+
+## Solution
+
+Examine logged output of the failed command for indications of what caused the issue.  If no immediate
+solution is suggested by the logged messages, please file an issue at https://github.com/dotnet/android
+including the full error message and steps that led to the command failing (possibly including a small
+repro application).

build-tools/create-packs/Microsoft.Android.Runtime.proj

@@ -42,6 +42,7 @@ projects that use the Microsoft.Android framework in .NET 6+.
       <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libmono-android.release.so" />
       <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-debug-app-helper.so" />
       <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libxamarin-native-tracing.so" />
+      <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libarchive-dso-stub.so" />
       <_AndroidRuntimePackAssets Include="$(MicrosoftAndroidSdkOutDir)lib\$(AndroidRID)\libunwind_xamarin.a" />
       <FrameworkListFileClass Include="@(_AndroidRuntimePackAssemblies->'%(Filename)%(Extension)')" Profile="Android" />
       <FrameworkListFileClass Include="@(_AndroidRuntimePackAssets->'%(Filename)%(Extension)')" Profile="Android" />

build-tools/create-packs/SignList.xml

@@ -16,6 +16,7 @@
     <ThirdParty Include="SgmlReaderDll.dll" />
     <ThirdParty Include="aapt2.exe" />
     <ThirdParty Include="llvm-mc.exe" />
+    <ThirdParty Include="llvm-objcopy.exe" />
     <ThirdParty Include="llvm-strip.exe" />
     <ThirdParty Include="aarch64-linux-android-ld.exe" />
     <ThirdParty Include="arm-linux-androideabi-ld.exe" />

build-tools/installers/create-installers.targets

@@ -207,6 +207,7 @@
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\ld.exe" />
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llc.exe" />
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-mc.exe" />
+    <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-objcopy.exe" />
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\llvm-strip.exe" />
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\aarch64-linux-android-as.cmd" />
     <_MSBuildFilesWin Include="$(MicrosoftAndroidSdkOutDir)binutils\bin\aarch64-linux-android-ld.cmd" />

build-tools/installers/unix-binutils.projitems

@@ -19,6 +19,7 @@
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\ld" />
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llc" />
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-mc" />
+    <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-objcopy" />
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\llvm-strip" />
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\x86_64-linux-android-as" />
     <_BinUtilsFilesUnixSignAndHarden Include="$(MicrosoftAndroidSdkOutDir)$(HostOS)\binutils\bin\x86_64-linux-android-ld" />

build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs

@@ -15,7 +15,7 @@ namespace Xamarin.Android.Prepare
 	//
 	partial class Configurables
 	{
-		const string BinutilsVersion                = "L_18.1.6-8.0.0";
+		const string BinutilsVersion                = "L_18.1.6-8.0.0-1";
 
 		const string MicrosoftOpenJDK17Version      = "17.0.12";
 		const string MicrosoftOpenJDK17Release      = "17.0.12";
@@ -157,6 +157,7 @@ public static partial class Defaults
 				new NDKTool (name: "ld"),
 				new NDKTool (name: "llc"),
 				new NDKTool (name: "llvm-mc"),
+				new NDKTool (name: "llvm-objcopy"),
 				new NDKTool (name: "llvm-strip"),
 			};
 		}

src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets

@@ -13,6 +13,7 @@ _ResolveAssemblies MSBuild target.
   <UsingTask TaskName="Xamarin.Android.Tasks.ProcessAssemblies"      AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
   <UsingTask TaskName="Xamarin.Android.Tasks.ProcessNativeLibraries" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
   <UsingTask TaskName="Xamarin.Android.Tasks.StripNativeLibraries" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
+  <UsingTask TaskName="Xamarin.Android.Tasks.PrepareDSOWrapperState" AssemblyFile="$(_XamarinAndroidBuildTasksAssembly)" />
 
   <!-- HACK: workaround for: https://github.com/dotnet/sdk/issues/25679 -->
   <Target Name="_RemoveLinuxFrameworkReferences"
@@ -122,6 +123,17 @@ _ResolveAssemblies MSBuild target.
       <_ResolvedJavaLibraries Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.jar' " />
     </ItemGroup>
 
+    <ItemGroup>
+      <_ResolvedArchiveDSOStub Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Filename)%(ResolvedFileToPublish.Extension)' == 'libarchive-dso-stub.so' " />
+      <ResolvedFileToPublish Remove="@(_ResolvedArchiveDSOStub)" />
+    </ItemGroup>
+
+    <!-- This must run as early as possible, as soon as we have locations of the .so files and before any wrapping/packaging is done -->
+    <PrepareDSOWrapperState
+        ArchiveDSOStubs="@(_ResolvedArchiveDSOStub)"
+        AndroidBinUtilsDirectory="$(AndroidBinUtilsDirectory)"
+        BaseOutputDirectory="$(IntermediateOutputPath)" />
+
     <!-- All assemblies must be per-RID, thus no `->Distinct()` on `InputAssemblies` or `ResolvedSymbols` items -->
     <ProcessAssemblies
         RuntimeIdentifiers="@(_RIDs)"

src/Xamarin.Android.Build.Tasks/Properties/Resources.resx

@@ -1072,4 +1072,8 @@ To use a custom JDK path for a command line build, set the 'JavaSdkDirectory' MS
     <comment>The following are literal names and should not be translated: Maven, group_id, artifact_id
 {0} - A Maven artifact specification</comment>
   </data>
+  <data name="XA0142" xml:space="preserve">
+    <value>Command '{0}' failed.\n{1}</value>
+    <comment>'{0}' is a failed command name (potentially with path) followed by all the arguments passed to it. {1} is the combined output on the standard error and standard output streams.</comment>
+  </data>
 </root>

src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs

@@ -371,6 +371,7 @@ public override bool RunTask ()
 			OutputFiles = outputFiles.Select (a => new TaskItem (a)).ToArray ();
 
 			Log.LogDebugTaskItems ("  [Output] OutputFiles :", OutputFiles);
+			DSOWrapperGenerator.CleanUp (BuildEngine4, Log);
 
 			return !Log.HasLoggedErrors;
 		}
@@ -404,7 +405,8 @@ void AddRuntimeConfigBlob (ZipArchiveEx apk)
 					// Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we
 					// like since we can finish scanning the zip central directory earlier at startup.
 					string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so");
-					AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath));
+					string wrappedSourcePath = DSOWrapperGenerator.WrapIt (MonoAndroidHelper.AbiToTargetArch (abi), RuntimeConfigBinFilePath, Path.GetFileName (inArchivePath), BuildEngine4, Log);
+					AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath));
 				}
 			}
 		}
@@ -448,7 +450,8 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<And
 			foreach (var kvp in assemblyStorePaths) {
 				string abi = MonoAndroidHelper.ArchToAbi (kvp.Key);
 				inArchivePath = MakeArchiveLibPath (abi, "lib" + Path.GetFileName (kvp.Value));
-				AddFileToArchiveIfNewer (apk, kvp.Value, inArchivePath, GetCompressionMethod (inArchivePath));
+				string wrappedSourcePath = DSOWrapperGenerator.WrapIt (kvp.Key, kvp.Value, Path.GetFileName (inArchivePath), BuildEngine4, Log);
+				AddFileToArchiveIfNewer (apk, wrappedSourcePath, inArchivePath, GetCompressionMethod (inArchivePath));
 			}
 
 			void AddAssembliesFromCollection (ITaskItem[] assemblies)
@@ -469,11 +472,11 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies)
 
 				foreach (var kvp in perArchAssemblies) {
 					Log.LogDebugMessage ($"Adding assemblies for architecture '{kvp.Key}'");
-					DoAddAssembliesFromArchCollection (kvp.Value);
+					DoAddAssembliesFromArchCollection (kvp.Key, kvp.Value);
 				}
 			}
 
-			void DoAddAssembliesFromArchCollection (Dictionary<string, ITaskItem> assemblies)
+			void DoAddAssembliesFromArchCollection (AndroidTargetArch arch, Dictionary<string, ITaskItem> assemblies)
 			{
 				// In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is,
 				// their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory.
@@ -492,7 +495,8 @@ void DoAddAssembliesFromArchCollection (Dictionary<string, ITaskItem> assemblies
 					if (UseAssemblyStore) {
 						storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assembly);
 					} else {
-						AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath));
+						string wrappedSourcePath = DSOWrapperGenerator.WrapIt (arch, sourcePath, Path.GetFileName (assemblyPath), BuildEngine4, Log);
+						AddFileToArchiveIfNewer (apk, wrappedSourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath));
 					}
 
 					// Try to add config if exists
@@ -502,7 +506,7 @@ void DoAddAssembliesFromArchCollection (Dictionary<string, ITaskItem> assemblies
 							storeAssemblyInfo.ConfigFile = new FileInfo (config);
 						}
 					} else {
-						AddAssemblyConfigEntry (apk, assemblyDirectory, config);
+						AddAssemblyConfigEntry (apk, arch, assemblyDirectory, config);
 					}
 
 					// Try to add symbols if Debug
@@ -519,9 +523,10 @@ void DoAddAssembliesFromArchCollection (Dictionary<string, ITaskItem> assemblies
 								storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath);
 							} else {
 								string archiveSymbolsPath = assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols));
+								string wrappedSymbolsPath = DSOWrapperGenerator.WrapIt (arch, symbolsPath, Path.GetFileName (archiveSymbolsPath), BuildEngine4, Log);
 								AddFileToArchiveIfNewer (
 									apk,
-									symbolsPath,
+									wrappedSymbolsPath,
 									archiveSymbolsPath,
 									compressionMethod: GetCompressionMethod (archiveSymbolsPath)
 								);
@@ -607,7 +612,7 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat
 			return true;
 		}
 
-		void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string configFile)
+		void AddAssemblyConfigEntry (ZipArchiveEx apk, AndroidTargetArch arch, string assemblyPath, string configFile)
 		{
 			string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile));
 			existingEntries.Remove (inArchivePath);
@@ -623,13 +628,8 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi
 			}
 
 			Log.LogDebugMessage ($"Adding {configFile} as the archive file is out of date.");
-			using (var source = File.OpenRead (configFile)) {
-				var dest = new MemoryStream ();
-				source.CopyTo (dest);
-				dest.WriteByte (0);
-				dest.Position = 0;
-				apk.AddEntryAndFlush (inArchivePath, dest, compressionMethod);
-			}
+			string wrappedConfigFile = DSOWrapperGenerator.WrapIt (arch, configFile, Path.GetFileName (inArchivePath), BuildEngine4, Log);
+			apk.AddFileAndFlush (wrappedConfigFile, inArchivePath, compressionMethod);
 		}
 
 		/// <summary>

src/Xamarin.Android.Build.Tasks/Tasks/PrepareDSOWrapperState.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+using Microsoft.Build.Framework;
+using Microsoft.Android.Build.Tasks;
+using Xamarin.Android.Tools;
+
+namespace Xamarin.Android.Tasks;
+
+/// <summary>
+/// Registers a state object used by the DSOWrapperGenerator class later on during
+/// the build.  This is to avoid having to pass parameters to some tasks (esp. BuildApk)
+/// which do not necessarily need those parameters directly.  Registering the state here
+/// also avoids having to update monodroid whenever any required parameter is added to
+/// BuildApk.
+/// </summary>
+public class PrepareDSOWrapperState : AndroidTask
+{
+	public override string TaskPrefix => "PDWS";
+
+	[Required]
+	public ITaskItem[] ArchiveDSOStubs     { get; set; }
+
+	[Required]
+	public string AndroidBinUtilsDirectory { get; set; }
+
+	[Required]
+	public string BaseOutputDirectory      { get; set; }
+
+	public override bool RunTask ()
+	{
+		var stubPaths = new Dictionary<AndroidTargetArch, ITaskItem> ();
+
+		foreach (ITaskItem stubItem in ArchiveDSOStubs) {
+			string rid = stubItem.GetRequiredMetadata ("ArchiveDSOStub", "RuntimeIdentifier", Log);
+			AndroidTargetArch arch = MonoAndroidHelper.RidToArch (rid);
+			if (stubPaths.ContainsKey (arch)) {
+				throw new InvalidOperationException ($"Internal error: duplicate archive DSO stub architecture '{arch}' (RID: '{rid}')");
+			}
+
+			if (!File.Exists (stubItem.ItemSpec)) {
+				throw new InvalidOperationException ($"Internal error: archive DSO stub file '{stubItem.ItemSpec}' does not exist");
+			}
+
+			stubPaths.Add (arch, stubItem);
+		}
+
+		var config = new DSOWrapperGenerator.Config (stubPaths, AndroidBinUtilsDirectory, BaseOutputDirectory);
+		BuildEngine4.RegisterTaskObjectAssemblyLocal (DSOWrapperGenerator.RegisteredConfigKey, config, RegisteredTaskObjectLifetime.Build);
+
+		return !Log.HasLoggedErrors;
+	}
+}

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs

@@ -64,8 +64,57 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true,
 				ret = ReadZipEntry (path, arch, uncompressIfNecessary);
 			}
 
-			ret?.Seek (0, SeekOrigin.Begin);
-			return ret;
+			if (ret == null) {
+				return null;
+			}
+
+			ret.Flush ();
+			ret.Seek (0, SeekOrigin.Begin);
+			(ulong elfPayloadOffset, ulong elfPayloadSize, ELFPayloadError error) = Xamarin.Android.AssemblyStore.Utils.FindELFPayloadSectionOffsetAndSize (ret);
+
+			if (error != ELFPayloadError.None) {
+				string message = error switch {
+					ELFPayloadError.NotELF           => $"Entry '{path}' is not a valid ELF binary",
+					ELFPayloadError.LoadFailed       => $"Entry '{path}' could not be loaded",
+					ELFPayloadError.NotSharedLibrary => $"Entry '{path}' is not a shared ELF library",
+					ELFPayloadError.NotLittleEndian  => $"Entry '{path}' is not a little-endian ELF image",
+					ELFPayloadError.NoPayloadSection => $"Entry '{path}' does not contain the 'payload' section",
+					_                                => $"Unknown ELF payload section error for entry '{path}': {error}"
+				};
+				Console.WriteLine (message);
+			} else {
+				Console.WriteLine ($"Extracted content from ELF image '{path}'");
+			}
+
+			if (elfPayloadOffset == 0) {
+				ret.Seek (0, SeekOrigin.Begin);
+				return ret;
+			}
+
+			// Make a copy of JUST the payload section, so that it contains only the data the tests expect and support
+			var payload = new MemoryStream ();
+			var data = buffers.Rent (16384);
+			int toRead = data.Length;
+			int nRead = 0;
+			ulong remaining = elfPayloadSize;
+
+			ret.Seek ((long)elfPayloadOffset, SeekOrigin.Begin);
+			while (remaining > 0 && (nRead = ret.Read (data, 0, toRead)) > 0) {
+				payload.Write (data, 0, nRead);
+				remaining -= (ulong)nRead;
+
+				if (remaining < (ulong)data.Length) {
+					// Make sure the last chunk doesn't gobble in more than we need
+					toRead = (int)remaining;
+				}
+			}
+			buffers.Return (data);
+
+			payload.Flush ();
+			ret.Dispose ();
+
+			payload.Seek (0, SeekOrigin.Begin);
+			return payload;
 		}
 
 		Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary)