[Xamarin.Android.Build.Tasks] Bump ZipFlushFilesLimit (#7957)

Context: https://dev.azure.com/DevDiv/DevDiv/_workitems/edit/1782014
Context: 9166e03

On Windows customers are seeing the following error:

	error XABLD7000: Xamarin.Tools.Zip.ZipException: Renaming temporary file failed: Permission denied
	 at Xamarin.Tools.Zip.ZipArchive.Close() in /Users/runner/work/1/s/LibZipSharp/Xamarin.Tools.Zip/ZipArchive.cs:line 939

We have traced this to the use of the [`MoveFileExW()`][0] /
`MoveFileExA()` API: when libzip is trying to move its temp file to
the final location, Windows is raising this error:

	Renaming temporary file failed: Permission denied

Turning off Anti Virus seems to help, however adding an exclusion
does not.  This is very confusing, so we are unsure why this error is
being raised.  The process has the correct permissions and the file
being moved is in the same directory, so its not a TEMP folder issue.

Perhaps it's the number of temp files we create?  Part of the
`<BuildApk/>` system is that as we add files we `Flush()` the zip
file to commit those changes to disk.  This is partly to work around
how libzip works: it does not write any data to the main file until
[`zip_close()`][1] is called.  To work around issues around too many
files being open (9166e03), we added this flush.

The limit of 50 files was picked out of a hat.  Try pushing the limit
up a bit to see if that helps.

Additionally, introduce the following two (private!) MSBuild
properties:

  * `$(_ZipFlushFilesLimit)`: Call `Flush()` after
    `$(_ZipFlushFilesLimit)` files have been added to the `.apk`.

  * `$(_ZipFlushSizeLimit)`:  Call `Flush()` after
    `$(_ZipFlushSizeLimit)` bytes of data have been added to the
    `.apk`.

[0]: https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw
[1]: https://libzip.org/documentation/zip_close.html

Author: dellis1972

src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs

@@ -95,6 +95,10 @@ public class BuildApk : AndroidTask
 
 		public bool UseAssemblyStore { get; set; }
 
+		public string ZipFlushFilesLimit { get; set; }
+
+		public string ZipFlushSizeLimit { get; set; }
+
 		[Required]
 		public string ProjectFullPath { get; set; }
 
@@ -136,6 +140,12 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
 				refresh = false;
 			}
 			using (var apk = new ZipArchiveEx (apkOutputPath, File.Exists (apkOutputPath) ? FileMode.Open : FileMode.Create )) {
+				if (int.TryParse (ZipFlushFilesLimit, out int flushFilesLimit)) {
+					apk.ZipFlushFilesLimit = flushFilesLimit;
+				}
+				if (int.TryParse (ZipFlushSizeLimit, out int flushSizeLimit)) {
+					apk.ZipFlushSizeLimit = flushSizeLimit;
+				}
 				if (refresh) {
 					for (long i = 0; i < apk.Archive.EntryCount; i++) {
 						ZipEntry e = apk.Archive.ReadEntry ((ulong) i);
@@ -206,7 +216,6 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
 					AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, $"{AssembliesPath}rc.bin", compressionMethod: UncompressedMethod);
 				}
 
-				int count = 0;
 				foreach (var file in files) {
 					var item = Path.Combine (file.archivePath.Replace (Path.DirectorySeparatorChar, '/'));
 					existingEntries.Remove (item);
@@ -216,12 +225,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
 						continue;
 					}
 					Log.LogDebugMessage ("\tAdding {0}", file.filePath);
-					apk.Archive.AddFile (file.filePath, item, compressionMethod: compressionMethod);
-					count++;
-					if (count >= ZipArchiveEx.ZipFlushFilesLimit) {
-						apk.Flush();
-						count = 0;
-					}
+					apk.AddFileAndFlush (file.filePath, item, compressionMethod: compressionMethod);
 				}
 
 				var jarFiles = (JavaSourceFiles != null) ? JavaSourceFiles.Where (f => f.ItemSpec.EndsWith (".jar", StringComparison.OrdinalIgnoreCase)) : null;
@@ -236,7 +240,6 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
 				var jarFilePaths = libraryProjectJars.Concat (jarFiles != null ? jarFiles.Select (j => j.ItemSpec) : Enumerable.Empty<string> ());
 				jarFilePaths = MonoAndroidHelper.DistinctFilesByContent (jarFilePaths);
 
-				count = 0;
 				foreach (var jarFile in jarFilePaths) {
 					using (var stream = File.OpenRead (jarFile))
 					using (var jar = ZipArchive.Open (stream)) {
@@ -278,14 +281,9 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut
 								data = d.ToArray ();
 							}
 							Log.LogDebugMessage ($"Adding {path} from {jarFile} as the archive file is out of date.");
-							apk.Archive.AddEntry (data, path);
+							apk.AddEntryAndFlush (data, path);
 						}
 					}
-					count++;
-					if (count >= ZipArchiveEx.ZipFlushFilesLimit) {
-						apk.Flush();
-						count = 0;
-					}
 				}
 				// Clean up Removed files.
 				foreach (var entry in existingEntries) {
@@ -377,7 +375,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<str
 				storeGenerator = null;
 			}
 
-			int count = 0;
 			AssemblyStoreAssemblyInfo storeAssembly = null;
 
 			//
@@ -398,7 +395,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary<str
 			AddAssembliesFromCollection (ResolvedUserAssemblies);
 
 			// Add framework assemblies
-			count = 0;
 			AddAssembliesFromCollection (ResolvedFrameworkAssemblies);
 
 			if (!UseAssemblyStore) {
@@ -482,12 +478,6 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies)
 
 					if (UseAssemblyStore) {
 						storeGenerator.Add (assemblyStoreApkName, storeAssembly);
-					} else {
-						count++;
-						if (count >= ZipArchiveEx.ZipFlushFilesLimit) {
-							apk.Flush();
-							count = 0;
-						}
 					}
 				}
 			}
@@ -554,7 +544,7 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat
 				return false;
 			}
 			Log.LogDebugMessage ($"Adding {file} as the archive file is out of date.");
-			apk.Archive.AddFile (file, inArchivePath, compressionMethod: compressionMethod);
+			apk.AddFileAndFlush (file, inArchivePath, compressionMethod: compressionMethod);
 			return true;
 		}
 
@@ -578,7 +568,7 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi
 				source.CopyTo (dest);
 				dest.WriteByte (0);
 				dest.Position = 0;
-				apk.Archive.AddEntry (inArchivePath, dest, compressionMethod);
+				apk.AddEntryAndFlush (inArchivePath, dest, compressionMethod);
 			}
 		}
 
@@ -625,7 +615,7 @@ void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemP
 				return;
 			}
 			Log.LogDebugMessage ($"Adding native library: {filesystemPath} (APK path: {archivePath})");
-			apk.Archive.AddEntry (archivePath, File.OpenRead (filesystemPath), compressionMethod);
+			apk.AddEntryAndFlush (archivePath, File.OpenRead (filesystemPath), compressionMethod);
 		}
 
 		void AddRuntimeLibraries (ZipArchiveEx apk, string [] supportedAbis)

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs

@@ -962,6 +962,46 @@ public void CheckExcludedFilesAreMissing ()
 			}
 		}
 
+		[Test]
+		[TestCase (1, -1)]
+		[TestCase (5, -1)]
+		[TestCase (50, -1)]
+		[TestCase (100, -1)]
+		[TestCase (512, -1)]
+		[TestCase (1024, -1)]
+		[TestCase (-1, 1)]
+		[TestCase (-1, 5)]
+		[TestCase (-1, 10)]
+		[TestCase (-1, 100)]
+		[TestCase (-1, 200)]
+		public void BuildApkWithZipFlushLimits (int filesLimit, int sizeLimit)
+		{
+			var proj = new XamarinAndroidApplicationProject  {
+				IsRelease = false,
+				PackageReferences = {
+					KnownPackages.SupportDesign_27_0_2_1,
+					KnownPackages.SupportV7CardView_27_0_2_1,
+					KnownPackages.AndroidSupportV4_27_0_2_1,
+					KnownPackages.SupportCoreUtils_27_0_2_1,
+					KnownPackages.SupportMediaCompat_27_0_2_1,
+					KnownPackages.SupportFragment_27_0_2_1,
+					KnownPackages.SupportCoreUI_27_0_2_1,
+					KnownPackages.SupportCompat_27_0_2_1,
+					KnownPackages.SupportV7AppCompat_27_0_2_1,
+					KnownPackages.SupportV7MediaRouter_27_0_2_1,
+				},
+			};
+			proj.SetProperty ("EmbedAssembliesIntoApk", "true");
+			if (filesLimit > 0)
+				proj.SetProperty ("_ZipFlushFilesLimit", filesLimit.ToString ());
+			if (sizeLimit > 0)
+				proj.SetProperty ("_ZipFlushSizeLimit", (sizeLimit * 1024 * 1024).ToString ());
+			using (var b = CreateApkBuilder ()) {
+				Assert.IsTrue (b.Build (proj), "Build should have succeeded.");
+
+			}
+		}
+
 		[Test]
 		public void ExtractNativeLibsTrue ()
 		{

src/Xamarin.Android.Build.Tasks/Utilities/ZipArchiveEx.cs

@@ -8,8 +8,8 @@ namespace Xamarin.Android.Tasks
 	public class ZipArchiveEx : IDisposable
 	{
 
-		public static int ZipFlushSizeLimit = 50 * 1024 * 1024;
-		public static int ZipFlushFilesLimit = 50;
+		const int DEFAULT_FLUSH_SIZE_LIMIT = 100 * 1024 * 1024;
+		const int DEFAULT_FLUSH_FILES_LIMIT = 512;
 
 		ZipArchive zip;
 		string archive;
@@ -24,6 +24,9 @@ public ZipArchive Archive {
 
 		public bool CreateDirectoriesInZip { get; set; } = true;
 
+		public int ZipFlushSizeLimit { get; set; } = DEFAULT_FLUSH_SIZE_LIMIT;
+		public int ZipFlushFilesLimit { get; set; } = DEFAULT_FLUSH_FILES_LIMIT;
+
 		public ZipArchiveEx (string archive) : this (archive, FileMode.CreateNew)
 		{
 		}
@@ -65,7 +68,31 @@ void AddFileAndFlush (string filename, long fileLength, string archiveFileName,
 		{
 			filesWrittenTotalSize += fileLength;
 			zip.AddFile (filename, archiveFileName, compressionMethod: compressionMethod);
-			if ((filesWrittenTotalSize >= ZipArchiveEx.ZipFlushSizeLimit || filesWrittenTotalCount >= ZipArchiveEx.ZipFlushFilesLimit) && AutoFlush) {
+			if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) {
+				Flush ();
+			}
+		}
+
+		public void AddFileAndFlush (string filename, string archiveFileName, CompressionMethod compressionMethod)
+		{
+			var fi = new FileInfo (filename);
+			AddFileAndFlush (filename, fi.Length, archiveFileName, compressionMethod);
+		}
+
+		public void AddEntryAndFlush (byte[] data, string archiveFileName)
+		{
+			filesWrittenTotalSize += data.Length;
+			zip.AddEntry (data, archiveFileName);
+			if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) {
+				Flush ();
+			}
+		}
+
+		public void AddEntryAndFlush (string archiveFileName, Stream data, CompressionMethod method)
+		{
+			filesWrittenTotalSize += data.Length;
+			zip.AddEntry (archiveFileName, data, method);
+			if ((filesWrittenTotalSize >= ZipFlushSizeLimit || filesWrittenTotalCount >= ZipFlushFilesLimit) && AutoFlush) {
 				Flush ();
 			}
 		}

src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets

@@ -2152,6 +2152,8 @@ because xbuild doesn't support framework reference assemblies.
     CheckedBuild="$(_AndroidCheckedBuild)"
     RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)"
     ExcludeFiles="@(AndroidPackagingOptionsExclude)"
+    ZipFlushFilesLimit="$(_ZipFlushFilesLimit)"
+    ZipFlushSizeLimit="$(_ZipFlushSizeLimit)"
     UseAssemblyStore="$(AndroidUseAssemblyStore)">
     <Output TaskParameter="OutputFiles" ItemName="ApkFiles" />
   </BuildApk>
@@ -2185,6 +2187,8 @@ because xbuild doesn't support framework reference assemblies.
       CheckedBuild="$(_AndroidCheckedBuild)"
       RuntimeConfigBinFilePath="$(_BinaryRuntimeConfigPath)"
       ExcludeFiles="@(AndroidPackagingOptionsExclude)"
+      ZipFlushFilesLimit="$(_ZipFlushFilesLimit)"
+      ZipFlushSizeLimit="$(_ZipFlushSizeLimit)"
       UseAssemblyStore="$(AndroidUseAssemblyStore)">
     <Output TaskParameter="OutputFiles" ItemName="BaseZipFile" />
   </BuildBaseAppBundle>