Update commons-compress to 1.26.1

Fixes bazelbuild#20269. Update commons-compress to 1.26.1 and swap use of GZIPInputStream to commons-compress' GzipCompressorInputStream, which [deals correctly with concatenated gz files](https://github.com/apache/commons-compress/blob/53c5e19208caaf63946a41d2763cda1f1b7eadc8/src/main/java/org/apache/commons/compress/compressors/gzip/GzipCompressorInputStream.java#L38-L70). Add a test to demonstrate this fixes the ruff extraction (thanks, fmeum) and update all related lockfiles. Closes bazelbuild#22213. PiperOrigin-RevId: 631509796 Change-Id: I4038244bfbdfbace747554e988587663ca580c16
Kila2 · May 13, 2024 · fabc6b0 · fabc6b0
1 parent aa16af6
commit fabc6b0
Show file tree

Hide file tree

Showing 9 changed files with 174 additions and 21 deletions.
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -156,7 +156,7 @@ maven.install(
         "javax.inject:javax.inject:1",
         "net.bytebuddy:byte-buddy:1.14.5",
         "net.bytebuddy:byte-buddy-agent:1.14.5",
-        "org.apache.commons:commons-compress:1.20",
+        "org.apache.commons:commons-compress:1.26.1",
         "org.apache.commons:commons-pool2:2.8.0",
         "org.apache.tomcat:tomcat-annotations-api:8.0.5",
         "org.apache.velocity:velocity:1.7",

diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
diff --git a/maven_install.json b/maven_install.json
@@ -1,7 +1,7 @@
 {
   "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL",
-  "__INPUT_ARTIFACTS_HASH": -1562612404,
-  "__RESOLVED_ARTIFACTS_HASH": 1921205679,
+  "__INPUT_ARTIFACTS_HASH": -983466987,
+  "__RESOLVED_ARTIFACTS_HASH": 1834716018,
   "conflict_resolution": {
     "com.google.auto.value:auto-value-annotations:1.9": "com.google.auto.value:auto-value-annotations:1.10.4",
     "com.google.code.gson:gson:2.8.9": "com.google.code.gson:gson:2.9.0",
@@ -312,12 +312,24 @@
       },
       "version": "1.12.0"
     },
+    "commons-codec:commons-codec": {
+      "shasums": {
+        "jar": "ec87bfb55f22cbd1b21e2190eeda28b2b312ed2a431ee49fbdcc01812d04a5e4"
+      },
+      "version": "1.16.1"
+    },
     "commons-collections:commons-collections": {
       "shasums": {
         "jar": "eeeae917917144a68a741d4c0dff66aa5c5c5fd85593ff217bced3fc8ca783b8"
       },
       "version": "3.2.2"
     },
+    "commons-io:commons-io": {
+      "shasums": {
+        "jar": "a58af12ee1b68cfd2ebb0c27caef164f084381a00ec81a48cc275fd7ea54e154"
+      },
+      "version": "2.15.1"
+    },
     "commons-lang:commons-lang": {
       "shasums": {
         "jar": "50f11b09f877c294d56f24463f47d28f929cf5044f648661c0f0cfbae9a2f49c"
@@ -588,9 +600,15 @@
     },
     "org.apache.commons:commons-compress": {
       "shasums": {
-        "jar": "0aeb625c948c697ea7b205156e112363b59ed5e2551212cd4e460bdb72c7c06e"
+        "jar": "27bb5d40f37c3bb7205b4a0540247df057715e9f6cbbd97d626ab8b50318bb04"
+      },
+      "version": "1.26.1"
+    },
+    "org.apache.commons:commons-lang3": {
+      "shasums": {
+        "jar": "7b96bf3ee68949abb5bc465559ac270e0551596fa34523fddf890ec418dde13c"
       },
-      "version": "1.20"
+      "version": "3.14.0"
     },
     "org.apache.commons:commons-math3": {
       "shasums": {
@@ -1143,6 +1161,11 @@
     "junit:junit": [
       "org.hamcrest:hamcrest-core"
     ],
+    "org.apache.commons:commons-compress": [
+      "commons-codec:commons-codec",
+      "commons-io:commons-io",
+      "org.apache.commons:commons-lang3"
+    ],
     "org.apache.velocity:velocity": [
       "commons-collections:commons-collections",
       "commons-lang:commons-lang"
@@ -1574,6 +1597,15 @@
     "com.squareup:javapoet": [
       "com.squareup.javapoet"
     ],
+    "commons-codec:commons-codec": [
+      "org.apache.commons.codec",
+      "org.apache.commons.codec.binary",
+      "org.apache.commons.codec.cli",
+      "org.apache.commons.codec.digest",
+      "org.apache.commons.codec.language",
+      "org.apache.commons.codec.language.bm",
+      "org.apache.commons.codec.net"
+    ],
     "commons-collections:commons-collections": [
       "org.apache.commons.collections",
       "org.apache.commons.collections.bag",
@@ -1588,6 +1620,23 @@
       "org.apache.commons.collections.map",
       "org.apache.commons.collections.set"
     ],
+    "commons-io:commons-io": [
+      "org.apache.commons.io",
+      "org.apache.commons.io.build",
+      "org.apache.commons.io.channels",
+      "org.apache.commons.io.charset",
+      "org.apache.commons.io.comparator",
+      "org.apache.commons.io.file",
+      "org.apache.commons.io.file.attribute",
+      "org.apache.commons.io.file.spi",
+      "org.apache.commons.io.filefilter",
+      "org.apache.commons.io.function",
+      "org.apache.commons.io.input",
+      "org.apache.commons.io.input.buffer",
+      "org.apache.commons.io.monitor",
+      "org.apache.commons.io.output",
+      "org.apache.commons.io.serialization"
+    ],
     "commons-lang:commons-lang": [
       "org.apache.commons.lang",
       "org.apache.commons.lang.builder",
@@ -1993,9 +2042,36 @@
       "org.apache.commons.compress.compressors.xz",
       "org.apache.commons.compress.compressors.z",
       "org.apache.commons.compress.compressors.zstandard",
+      "org.apache.commons.compress.harmony",
+      "org.apache.commons.compress.harmony.archive.internal.nls",
+      "org.apache.commons.compress.harmony.pack200",
+      "org.apache.commons.compress.harmony.unpack200",
+      "org.apache.commons.compress.harmony.unpack200.bytecode",
+      "org.apache.commons.compress.harmony.unpack200.bytecode.forms",
+      "org.apache.commons.compress.java.util.jar",
       "org.apache.commons.compress.parallel",
       "org.apache.commons.compress.utils"
     ],
+    "org.apache.commons:commons-lang3": [
+      "org.apache.commons.lang3",
+      "org.apache.commons.lang3.arch",
+      "org.apache.commons.lang3.builder",
+      "org.apache.commons.lang3.compare",
+      "org.apache.commons.lang3.concurrent",
+      "org.apache.commons.lang3.concurrent.locks",
+      "org.apache.commons.lang3.event",
+      "org.apache.commons.lang3.exception",
+      "org.apache.commons.lang3.function",
+      "org.apache.commons.lang3.math",
+      "org.apache.commons.lang3.mutable",
+      "org.apache.commons.lang3.reflect",
+      "org.apache.commons.lang3.stream",
+      "org.apache.commons.lang3.text",
+      "org.apache.commons.lang3.text.translate",
+      "org.apache.commons.lang3.time",
+      "org.apache.commons.lang3.tuple",
+      "org.apache.commons.lang3.util"
+    ],
     "org.apache.commons:commons-math3": [
       "org.apache.commons.math3",
       "org.apache.commons.math3.analysis",
@@ -2371,7 +2447,9 @@
       "com.ryanharter.auto.value:auto-value-gson-factory",
       "com.ryanharter.auto.value:auto-value-gson-runtime",
       "com.squareup:javapoet",
+      "commons-codec:commons-codec",
       "commons-collections:commons-collections",
+      "commons-io:commons-io",
       "commons-lang:commons-lang",
       "io.github.eisop:dataflow-errorprone",
       "io.github.java-diff-utils:java-diff-utils",
@@ -2426,6 +2504,7 @@
       "net.bytebuddy:byte-buddy-agent",
       "net.sf.jopt-simple:jopt-simple",
       "org.apache.commons:commons-compress",
+      "org.apache.commons:commons-lang3",
       "org.apache.commons:commons-math3",
       "org.apache.commons:commons-pool2",
       "org.apache.tomcat:tomcat-annotations-api",

diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh
@@ -155,12 +155,17 @@ function create_deploy_jar() {
   local output=$3
   shift 3
   local packages=""
-  for i in $output/classes/*; do
+  # Only keep the services subdirectory of META-INF (needed for AutoService).
+  for i in $output/classes/META-INF/*; do
     local package=$(basename $i)
-    if [[ "$package" != "META-INF" ]]; then
-      packages="$packages -C $output/classes $package"
+    if [[ "$package" != "services" ]]; then
+      rm -r "$i"
     fi
   done
+  for i in $output/classes/*; do
+    local package=$(basename $i)
+    packages="$packages -C $output/classes $package"
+  done
 
   log "Creating $name.jar..."
   echo "Main-Class: $mainClass" > $output/MANIFEST.MF

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarBz2Function.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarBz2Function.java
@@ -36,6 +36,7 @@ protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
       throws IOException {
     return new BZip2CompressorInputStream(
         new BufferedInputStream(
-            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE));
+            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE),
+        true);
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/TarGzFunction.java
@@ -19,7 +19,7 @@
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.zip.GZIPInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 
 /**
  * Creates a repository by unarchiving a .tar.gz file.
@@ -34,8 +34,9 @@ private TarGzFunction() {
   @Override
   protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
       throws IOException {
-    return new GZIPInputStream(
+    return new GzipCompressorInputStream(
         new BufferedInputStream(
-            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE));
+            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE),
+        true);
   }
 }
diff --git a/src/test/shell/bazel/bazel_workspaces_test.sh b/src/test/shell/bazel/bazel_workspaces_test.sh
@@ -504,6 +504,22 @@ function test_extract_default_zip_non_ascii_utf8_file_names() {
   ensure_output_contains_exactly_once "external/repo/out_dir/Ä_foo_∅.txt" "bar"
 }
 
+function test_sparse_tar() {
+  set_workspace_command "
+  repository_ctx.download_and_extract(
+      url='https://mirror.bazel.build/github.com/astral-sh/ruff/releases/download/v0.1.6/ruff-aarch64-apple-darwin.tar.gz',
+      sha256='0b626e88762b16908b3dbba8327341ddc13b37ebe6ec1a0db3f033ce5a44162d',
+  )"
+
+  build_and_process_log --exclude_rule "repository @@local_config_cc"
+
+  ensure_contains_exactly 'location: .*repos.bzl:3:38' 1
+  ensure_contains_atleast 'context: "repository @@repo"' 2
+  ensure_contains_exactly 'download_and_extract_event' 1
+
+  [[ -f "$(bazel info output_base)/external/repo/ruff" ]] || fail "Expected ruff binary to be extracted"
+}
+
 function test_file() {
   set_workspace_command 'repository_ctx.file("filefile.sh", "echo filefile", True)'
 

diff --git a/src/test/shell/bazel/jdeps_class_denylist.txt b/src/test/shell/bazel/jdeps_class_denylist.txt
@@ -9,3 +9,9 @@
 ./lombok/javac/java6/CommentCollectingScannerFactory.class
 ./lombok/javac/java7/CommentCollectingParser.class
 ./lombok/javac/java7/CommentCollectingScanner.class
+./org/apache/commons/compress/harmony/pack200/Pack200Adapter.class
+./org/apache/commons/compress/java/util/jar/Pack200.class
+./org/apache/commons/compress/java/util/jar/Pack200$Packer.class
+./org/apache/commons/compress/java/util/jar/Pack200$Unpacker.class
+./org/apache/commons/lang3/concurrent/AbstractCircuitBreaker.class
+./org/apache/commons/lang3/concurrent/EventCountCircuitBreaker.class