Skip to content

Commit

Permalink
Support decompressing zstd tar archives for repository rules.
Browse files Browse the repository at this point in the history
This is mostly a rebase of #11968 with a few tweaks of my own.

Fixes #10342.

Co-authored-by: Grzegorz Lukasik <glukasik@nuro.ai>

RELNOTES[new]: `repository_ctx.extract`, and thus `http_archive`, can now decompress zstandard-compressed archives.

Closes #15026.

PiperOrigin-RevId: 436146779
  • Loading branch information
benjaminp authored and copybara-github committed Mar 21, 2022
1 parent a421288 commit 00d74ff
Show file tree
Hide file tree
Showing 8 changed files with 72 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ java_library(
"//third_party:java-diff-utils",
"//third_party:jsr305",
"//third_party:xz",
"@zstd-jni",
],
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,15 @@ static Decompressor getDecompressor(Path archivePath)
return TarGzFunction.INSTANCE;
} else if (baseName.endsWith(".tar.xz") || baseName.endsWith(".txz")) {
return TarXzFunction.INSTANCE;
} else if (baseName.endsWith(".tar.zst") || baseName.endsWith(".tzst")) {
return TarZstFunction.INSTANCE;
} else if (baseName.endsWith(".tar.bz2")) {
return TarBz2Function.INSTANCE;
} else {
throw new RepositoryFunctionException(
Starlark.errorf(
"Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz, "
+ "or .tar.bz2 suffix (got %s)",
"Expected a file with a .zip, .jar, .war, .aar, .tar, .tar.gz, .tgz, .tar.xz, .txz,"
+ " .tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
archivePath),
Transience.PERSISTENT);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.build.lib.bazel.repository;

import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

/** Creates a repository by unarchiving a zstandard-compressed .tar file. */
final class TarZstFunction extends CompressedTarFunction {
static final Decompressor INSTANCE = new TarZstFunction();
private static final int BUFFER_SIZE = 32 * 1024;

private TarZstFunction() {}

@Override
protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
throws IOException {
return new ZstdInputStreamNoFinalizer(
new BufferedInputStream(
new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.txz");
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.zst");
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tzst");
DecompressorDescriptor.builder().setArchivePath(path).build();
path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
DecompressorDescriptor.builder().setArchivePath(path).build();
}
Expand Down
5 changes: 4 additions & 1 deletion src/test/shell/bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,10 @@ sh_test(
name = "external_integration_test",
size = "large",
srcs = ["external_integration_test.sh"],
data = [":test-deps"],
data = [
":test-deps",
"//src/test/shell/bazel/testdata:zstd_test_archive.tar.zst",
],
shard_count = 15,
tags = [
"no_windows",
Expand Down
16 changes: 16 additions & 0 deletions src/test/shell/bazel/external_integration_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,22 @@ function test_http_archive_tar_xz() {
http_archive_helper tar_xz_up
}

function test_http_archive_tar_zstd() {
cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
http_archive(
name = 'test_zstd_repo',
url = 'file://$(rlocation io_bazel/src/test/shell/bazel/testdata/zstd_test_archive.tar.zst)',
sha256 = '12b0116f2a3c804859438e102a8a1d5f494c108d1b026da9f6ca55fb5107c7e9',
build_file_content = 'filegroup(name="x", srcs=glob(["*"]))',
)
EOF
bazel build @test_zstd_repo//...

base_external_path=bazel-out/../external/test_zstd_repo
assert_contains "test content" "${base_external_path}/test_dir/test_file"
}

function test_http_archive_no_server() {
cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
Expand Down
8 changes: 5 additions & 3 deletions src/test/shell/bazel/testdata/BUILD
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
load("//:distdir_deps.bzl", "gen_workspace_stanza")

package(default_testonly = True)

filegroup(
name = "srcs",
testonly = False,
srcs = glob(["**"]),
visibility = ["//src/test/shell/bazel:__pkg__"],
)

filegroup(
name = "git-repos",
testonly = 1,
srcs = [
"outer-planets-repo.tar.gz",
"pluto-repo.tar.gz",
Expand All @@ -20,7 +22,6 @@ filegroup(

filegroup(
name = "embedded_tools_deps_test_data",
testonly = 1,
srcs = [
"//src/test/shell/bazel/testdata:embedded_tools_srcs_deps",
],
Expand All @@ -29,7 +30,6 @@ filegroup(

filegroup(
name = "jdk_http_archives_filegroup",
testonly = 1,
srcs = [
"//src/test/shell/bazel/testdata:jdk_http_archives",
],
Expand Down Expand Up @@ -64,3 +64,5 @@ gen_workspace_stanza(
template = "jdk_http_archives.tmpl",
visibility = ["//:__pkg__"],
)

exports_files(["zstd_test_archive.tar.zst"])
Binary file not shown.

0 comments on commit 00d74ff

Please sign in to comment.