Add Lzma library files to the repository

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets

@@ -34,6 +34,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <!-- Use libbrotlicommon.a as the sentinel for the three brotli libs. -->
       <UseSystemBrotli Condition="'$(UseSystemBrotli)' == '' and !Exists('$(IlcFrameworkNativePath)libbrotlicommon.a')">true</UseSystemBrotli>
       <UseSystemZstd Condition="'$(UseSystemZstd)' == '' and !Exists('$(IlcFrameworkNativePath)libzstd.a')">true</UseSystemZstd>
+      <UseSystemLzma Condition="'$(UseSystemLzma)' == '' and !Exists('$(IlcFrameworkNativePath)liblzma.a')">true</UseSystemLzma>
 
       <FullRuntimeName>libRuntime.WorkstationGC</FullRuntimeName>
       <FullRuntimeName Condition="'$(ServerGarbageCollection)' == 'true' or '$(IlcLinkServerGC)' == 'true'">libRuntime.ServerGC</FullRuntimeName>
@@ -188,6 +189,11 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Condition="'$(UseSystemZstd)' != 'true'" Include="$(IlcFrameworkNativePath)libzstd.a" />
     </ItemGroup>
 
+    <ItemGroup>
+      <!-- xz must be added after System.IO.Compression.Native due to dependency. -->
+      <NativeLibrary Condition="'$(UseSystemLzma)' != 'true'" Include="$(IlcFrameworkNativePath)liblzma.a" />
+    </ItemGroup>
+
     <ItemGroup Condition="'$(StaticICULinking)' == 'true' and '$(NativeLib)' != 'Static' and '$(InvariantGlobalization)' != 'true'">
       <NativeLibrary Include="$(IntermediateOutputPath)libs/System.Globalization.Native/build/libSystem.Globalization.Native.a" />
       <DirectPInvoke Include="System.Globalization.Native" />
@@ -228,6 +234,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeSystemLibrary Include="z" Condition="'$(UseSystemZlib)' == 'true'" />
       <NativeSystemLibrary Include="brotlienc;brotlidec;brotlicommon" Condition="'$(UseSystemBrotli)' == 'true'" />
       <NativeSystemLibrary Include="zstd" Condition="'$(UseSystemZstd)' == 'true'" />
+      <NativeSystemLibrary Include="lzma" Condition="'$(UseSystemXz)' == 'true'" />
       <NativeSystemLibrary Include="rt" Condition="'$(_IsApplePlatform)' != 'true' and '$(_linuxLibcFlavor)' != 'bionic'" />
       <NativeSystemLibrary Include="log" Condition="'$(_linuxLibcFlavor)' == 'bionic'" />
       <NativeSystemLibrary Include="icucore" Condition="'$(_IsApplePlatform)' == 'true'" />

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Windows.targets

@@ -51,6 +51,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Include="$(IlcSdkPath)brotlienc$(LibFileExt)" />
       <NativeLibrary Include="$(IlcSdkPath)brotlidec$(LibFileExt)" />
       <NativeLibrary Include="$(IlcSdkPath)zstd_static$(LibFileExt)" />
+      <NativeLibrary Include="$(IlcSdkPath)lzma$(LibFileExt)" />
     </ItemGroup>
 
     <ItemGroup>

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -302,6 +302,9 @@
     <!-- zstd-specific files -->
     <PlatformManifestFileEntry Include="libzstd.a" IsNative="true" />
     <PlatformManifestFileEntry Include="zstd_static.lib" IsNative="true" />
+    <!-- xz/lzma-specific files -->
+    <PlatformManifestFileEntry Include="liblzma.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="lzma.lib" IsNative="true" />
   </ItemGroup>
 
  <ItemGroup>

src/native/external/cgmanifest.json

@@ -2,6 +2,16 @@
   "$schema": "https://json.schemastore.org/component-detection-manifest.json",
   "version": 1,
   "registrations": [
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "repositoryUrl": "https://github.com/tukaani-project/xz",
+          "commitHash": "3d078b52adbff566ccfc51067dfbf742ecf3ef86"
+        }
+      },
+      "developmentDependency": false
+    },
     {
       "component": {
         "type": "git",

src/native/external/xz-patch.diff

@@ -0,0 +1,295 @@
+diff --git a/src/liblzma/check/crc_x86_clmul.h b/src/liblzma/check/crc_x86_clmul.h
+index b302d6cf..35653623 100644
+--- a/src/liblzma/check/crc_x86_clmul.h
++++ b/src/liblzma/check/crc_x86_clmul.h
+@@ -344,7 +344,7 @@ is_arch_extension_supported(void)
+ #if defined(_MSC_VER)
+ 	// This needs <intrin.h> with MSVC. ICC has it as a built-in
+ 	// on all platforms.
+-	__cpuid(r, 1);
++	__cpuid((int *)r, 1);
+ #elif defined(HAVE_CPUID_H)
+ 	// Compared to just using __asm__ to run CPUID, this also checks
+ 	// that CPUID is supported and saves and restores ebx as that is
+diff --git a/src/liblzma/common/block_buffer_encoder.c b/src/liblzma/common/block_buffer_encoder.c
+index df3b90e8..3963c15c 100644
+--- a/src/liblzma/common/block_buffer_encoder.c
++++ b/src/liblzma/common/block_buffer_encoder.c
+@@ -143,7 +143,7 @@ block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size,
+ 		// Size of the uncompressed chunk
+ 		const size_t copy_size
+ 				= my_min(in_size - in_pos, LZMA2_CHUNK_MAX);
+-		out[(*out_pos)++] = (copy_size - 1) >> 8;
++		out[(*out_pos)++] = (uint8_t)((copy_size - 1) >> 8);
+ 		out[(*out_pos)++] = (copy_size - 1) & 0xFF;
+
+ 		// The actual data
+diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c
+index 45e57a26..33840d1b 100644
+--- a/src/liblzma/common/block_header_encoder.c
++++ b/src/liblzma/common/block_header_encoder.c
+@@ -81,7 +81,7 @@ lzma_block_header_encode(const lzma_block *block, uint8_t *out)
+ 	const size_t out_size = block->header_size - 4;
+
+ 	// Store the Block Header Size.
+-	out[0] = out_size / 4;
++	out[0] = (uint8_t)(out_size / 4);
+
+ 	// We write Block Flags in pieces.
+ 	out[1] = 0x00;
+diff --git a/src/liblzma/delta/delta_encoder.c b/src/liblzma/delta/delta_encoder.c
+index ba4a50b1..c3f30273 100644
+--- a/src/liblzma/delta/delta_encoder.c
++++ b/src/liblzma/delta/delta_encoder.c
+@@ -126,7 +126,7 @@ lzma_delta_props_encode(const void *options, uint8_t *out)
+ 		return LZMA_PROG_ERROR;
+
+ 	const lzma_options_delta *opt = options;
+-	out[0] = opt->dist - LZMA_DELTA_DIST_MIN;
++	out[0] = (uint8_t)(opt->dist - LZMA_DELTA_DIST_MIN);
+
+ 	return LZMA_OK;
+ }
+diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h
+index 2698e016..d26963a9 100644
+--- a/src/liblzma/lz/lz_decoder.h
++++ b/src/liblzma/lz/lz_decoder.h
+@@ -205,7 +205,7 @@ dict_repeat(lzma_dict *restrict dict,
+ {
+ 	// Don't write past the end of the dictionary.
+ 	const size_t dict_avail = dict->limit - dict->pos;
+-	uint32_t left = my_min(dict_avail, *len);
++	uint32_t left = (uint32_t)my_min(dict_avail, *len);
+ 	*len -= left;
+
+ 	size_t back = dict->pos - distance - 1;
+diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c
+index e5c4057d..0062882a 100644
+--- a/src/liblzma/lz/lz_encoder.c
++++ b/src/liblzma/lz/lz_encoder.c
+@@ -107,7 +107,7 @@ fill_window(lzma_coder *coder, const lzma_allocator *allocator,
+ 				coder->mf.size, action);
+ 	}
+
+-	coder->mf.write_pos = write_pos;
++	coder->mf.write_pos = (uint32_t)write_pos;
+
+ 	// Silence Valgrind. lzma_memcmplen() can read extra bytes
+ 	// and Valgrind will give warnings if those bytes are uninitialized
+@@ -199,10 +199,10 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
+ 			|| lz_options->nice_len > lz_options->match_len_max)
+ 		return true;
+
+-	mf->keep_size_before = lz_options->before_size + lz_options->dict_size;
++	mf->keep_size_before = (uint32_t)(lz_options->before_size + lz_options->dict_size);
+
+-	mf->keep_size_after = lz_options->after_size
+-			+ lz_options->match_len_max;
++	mf->keep_size_after = (uint32_t)(lz_options->after_size
++			+ lz_options->match_len_max);
+
+ 	// To avoid constant memmove()s, allocate some extra space. Since
+ 	// memmove()s become more expensive when the size of the buffer
+@@ -215,12 +215,12 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
+ 	//     to size_t.
+ 	//   - Memory usage calculation needs something too, e.g. use uint64_t
+ 	//     for mf->size.
+-	uint32_t reserve = lz_options->dict_size / 2;
++	uint32_t reserve = (uint32_t)(lz_options->dict_size / 2);
+ 	if (reserve > (UINT32_C(1) << 30))
+ 		reserve /= 2;
+
+-	reserve += (lz_options->before_size + lz_options->match_len_max
+-			+ lz_options->after_size) / 2 + (UINT32_C(1) << 19);
++	reserve += (uint32_t)((lz_options->before_size + lz_options->match_len_max
++			+ lz_options->after_size) / 2 + (UINT32_C(1) << 19));
+
+ 	const uint32_t old_size = mf->size;
+ 	mf->size = mf->keep_size_before + reserve + mf->keep_size_after;
+@@ -233,8 +233,8 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
+ 	}
+
+ 	// Match finder options
+-	mf->match_len_max = lz_options->match_len_max;
+-	mf->nice_len = lz_options->nice_len;
++	mf->match_len_max = (uint32_t)lz_options->match_len_max;
++	mf->nice_len = (uint32_t)lz_options->nice_len;
+
+ 	// cyclic_size has to stay smaller than 2 Gi. Note that this doesn't
+ 	// mean limiting dictionary size to less than 2 GiB. With a match
+@@ -251,7 +251,7 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
+ 	// memory to keep the code simpler. The current way is simple and
+ 	// still allows pretty big dictionaries, so I don't expect these
+ 	// limits to change.
+-	mf->cyclic_size = lz_options->dict_size + 1;
++	mf->cyclic_size = (uint32_t)(lz_options->dict_size + 1);
+
+ 	// Validate the match finder ID and setup the function pointers.
+ 	switch (lz_options->match_finder) {
+@@ -308,7 +308,7 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator,
+ 	} else {
+ 		// Round dictionary size up to the next 2^n - 1 so it can
+ 		// be used as a hash mask.
+-		hs = lz_options->dict_size - 1;
++		hs = (uint32_t)(lz_options->dict_size - 1);
+ 		hs |= hs >> 1;
+ 		hs |= hs >> 2;
+ 		hs |= hs >> 4;
+diff --git a/src/liblzma/lzma/lzma2_encoder.c b/src/liblzma/lzma/lzma2_encoder.c
+index 71cfd9b4..957fa31e 100644
+--- a/src/liblzma/lzma/lzma2_encoder.c
++++ b/src/liblzma/lzma/lzma2_encoder.c
+@@ -81,14 +81,14 @@ lzma2_header_lzma(lzma_lzma2_coder *coder)
+
+ 	// Uncompressed size
+ 	size_t size = coder->uncompressed_size - 1;
+-	coder->buf[pos++] += size >> 16;
+-	coder->buf[pos++] = (size >> 8) & 0xFF;
+-	coder->buf[pos++] = size & 0xFF;
++	coder->buf[pos++] += (uint8_t)(size >> 16);
++	coder->buf[pos++] = (uint8_t)((size >> 8) & 0xFF);
++	coder->buf[pos++] = (uint8_t)(size & 0xFF);
+
+ 	// Compressed size
+ 	size = coder->compressed_size - 1;
+-	coder->buf[pos++] = size >> 8;
+-	coder->buf[pos++] = size & 0xFF;
++	coder->buf[pos++] = (uint8_t)(size >> 8);
++	coder->buf[pos++] = (uint8_t)(size & 0xFF);
+
+ 	// Properties, if needed
+ 	if (coder->need_properties)
+@@ -122,7 +122,7 @@ lzma2_header_uncompressed(lzma_lzma2_coder *coder)
+ 	coder->need_dictionary_reset = false;
+
+ 	// "Compressed" size
+-	coder->buf[1] = (coder->uncompressed_size - 1) >> 8;
++	coder->buf[1] = (uint8_t)((coder->uncompressed_size - 1) >> 8);
+ 	coder->buf[2] = (coder->uncompressed_size - 1) & 0xFF;
+
+ 	// Set the start position for copying.
+@@ -164,8 +164,8 @@ lzma2_encode(void *coder_ptr, lzma_mf *restrict mf,
+ 	case SEQ_LZMA_ENCODE: {
+ 		// Calculate how much more uncompressed data this chunk
+ 		// could accept.
+-		const uint32_t left = LZMA2_UNCOMPRESSED_MAX
+-				- coder->uncompressed_size;
++		const uint32_t left = (uint32_t)(LZMA2_UNCOMPRESSED_MAX
++				- coder->uncompressed_size);
+ 		uint32_t limit;
+
+ 		if (left < mf->match_len_max) {
+@@ -394,7 +394,7 @@ lzma_lzma2_props_encode(const void *options, uint8_t *out)
+ 	if (d == UINT32_MAX)
+ 		out[0] = 40;
+ 	else
+-		out[0] = get_dist_slot(d + 1) - 24;
++		out[0] = (uint8_t)(get_dist_slot(d + 1) - 24);
+
+ 	return LZMA_OK;
+ }
+diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c
+index 2088a2fa..627d2ca5 100644
+--- a/src/liblzma/lzma/lzma_decoder.c
++++ b/src/liblzma/lzma/lzma_decoder.c
+@@ -378,7 +378,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr,
+ 			}
+
+ 			// Write decoded literal to dictionary
+-			dict_put(&dict, symbol);
++			dict_put(&dict, (uint8_t)symbol);
+ 			continue;
+ 		}
+
+@@ -742,7 +742,7 @@ slow:
+ 			}
+
+ 	case SEQ_LITERAL_WRITE:
+-			if (dict_put_safe(&dict, symbol)) {
++			if (dict_put_safe(&dict, (uint8_t)symbol)) {
+ 				coder->sequence = SEQ_LITERAL_WRITE;
+ 				goto out;
+ 			}
+@@ -1203,7 +1203,7 @@ lzma_lzma_lclppb_decode(lzma_options_lzma *options, uint8_t byte)
+
+ 	// See the file format specification to understand this.
+ 	options->pb = byte / (9 * 5);
+-	byte -= options->pb * 9 * 5;
++	byte -= (uint8_t)(options->pb * 9 * 5);
+ 	options->lp = byte / 9;
+ 	options->lc = byte - options->lp * 9;
+
+diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c
+index 543ca321..20dc62d3 100644
+--- a/src/liblzma/lzma/lzma_encoder.c
++++ b/src/liblzma/lzma/lzma_encoder.c
+@@ -753,7 +753,7 @@ lzma_lzma_lclppb_encode(const lzma_options_lzma *options, uint8_t *byte)
+ 	if (!is_lclppb_valid(options))
+ 		return true;
+
+-	*byte = (options->pb * 5 + options->lp) * 9 + options->lc;
++	*byte = (uint8_t)((options->pb * 5 + options->lp) * 9 + options->lc);
+ 	assert(*byte <= (4 * 5 + 4) * 9 + 8);
+
+ 	return false;
+diff --git a/src/liblzma/simple/arm.c b/src/liblzma/simple/arm.c
+index f9d9c08b..159671a7 100644
+--- a/src/liblzma/simple/arm.c
++++ b/src/liblzma/simple/arm.c
+@@ -35,9 +35,9 @@ arm_code(void *simple lzma_attribute((__unused__)),
+ 				dest = src - (now_pos + (uint32_t)(i) + 8);
+
+ 			dest >>= 2;
+-			buffer[i + 2] = (dest >> 16);
+-			buffer[i + 1] = (dest >> 8);
+-			buffer[i + 0] = dest;
++			buffer[i + 2] = (uint8_t)(dest >> 16);
++			buffer[i + 1] = (uint8_t)(dest >> 8);
++			buffer[i + 0] = (uint8_t)dest;
+ 		}
+ 	}
+
+diff --git a/src/liblzma/simple/armthumb.c b/src/liblzma/simple/armthumb.c
+index 368b51c7..36233550 100644
+--- a/src/liblzma/simple/armthumb.c
++++ b/src/liblzma/simple/armthumb.c
+@@ -42,9 +42,9 @@ armthumb_code(void *simple lzma_attribute((__unused__)),
+
+ 			dest >>= 1;
+ 			buffer[i + 1] = 0xF0 | ((dest >> 19) & 0x7);
+-			buffer[i + 0] = (dest >> 11);
++			buffer[i + 0] = (uint8_t)(dest >> 11);
+ 			buffer[i + 3] = 0xF8 | ((dest >> 8) & 0x7);
+-			buffer[i + 2] = (dest);
++			buffer[i + 2] = (uint8_t)(dest);
+ 			i += 2;
+ 		}
+ 	}
+diff --git a/src/liblzma/simple/powerpc.c b/src/liblzma/simple/powerpc.c
+index ea47d14d..2bcc638b 100644
+--- a/src/liblzma/simple/powerpc.c
++++ b/src/liblzma/simple/powerpc.c
+@@ -39,8 +39,8 @@ powerpc_code(void *simple lzma_attribute((__unused__)),
+ 				dest = src - (now_pos + (uint32_t)(i));
+
+ 			buffer[i + 0] = 0x48 | ((dest >> 24) &  0x03);
+-			buffer[i + 1] = (dest >> 16);
+-			buffer[i + 2] = (dest >> 8);
++			buffer[i + 1] = (uint8_t)(dest >> 16);
++			buffer[i + 2] = (uint8_t)(dest >> 8);
+ 			buffer[i + 3] &= 0x03;
+ 			buffer[i + 3] |= dest;
+ 		}
+diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c
+index 5cbfa822..930a532d 100644
+--- a/src/liblzma/simple/simple_coder.c
++++ b/src/liblzma/simple/simple_coder.c
+@@ -59,7 +59,7 @@ call_filter(lzma_simple_coder *coder, uint8_t *buffer, size_t size)
+ 	const size_t filtered = coder->filter(coder->simple,
+ 			coder->now_pos, coder->is_encoder,
+ 			buffer, size);
+-	coder->now_pos += filtered;
++	coder->now_pos += (uint32_t)filtered;
+ 	return filtered;
+ }
+

src/native/external/xz-version.txt

@@ -0,0 +1,12 @@
+v5.8.2
+https://github.com/tukaani-project/xz/releases/tag/v5.8.2
+
+To update the code in the future, follow these steps:
+1. Download the source code tarball for the matching version from https://tukaani.org/xz/ or https://github.com/tukaani-project/xz/releases/
+2. Delete existing files in 'src/native/external/xz/'
+3. From the extracted tarball, copy the 'cmake', 'lib' and 'src' directories into 'src/native/external/xz/'
+4. Rebuild the native components and run the relevant tests
+
+Only the 'cmake', 'lib' and 'src' directories from the original source were included to minimize unnecessary code.
+
+To fix build-time warnings in the currently released version, the xz-patch.diff file was applied. These changes should be upstreamed to the xz project and should not be necessary when upgrading to a future version.