diff --git a/BUILD b/BUILD index 84c15b67735e6e..0540bb8c7b0ed4 100644 --- a/BUILD +++ b/BUILD @@ -96,6 +96,7 @@ genrule( pkg_tar( name = "bootstrap-jars", srcs = [ + "@blake3", "@com_google_protobuf//:protobuf_java", "@com_google_protobuf//:protobuf_java_util", "@com_google_protobuf//:protobuf_javalite", diff --git a/MODULE.bazel b/MODULE.bazel index 4c1409804fd85b..4c4d4fef236810 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,6 +14,7 @@ bazel_dep(name = "platforms", version = "0.0.6") bazel_dep(name = "rules_pkg", version = "0.7.0") bazel_dep(name = "stardoc", version = "0.5.3", repo_name = "io_bazel_skydoc") bazel_dep(name = "zstd-jni", version = "1.5.2-3") +bazel_dep(name = "blake3", version = "1.3.3") bazel_dep(name = "zlib", version = "1.2.13") bazel_dep(name = "rules_cc", version = "0.0.8") bazel_dep(name = "rules_java", version = "6.2.2") diff --git a/distdir_deps.bzl b/distdir_deps.bzl index cf9bd193f81b5d..ddfa94b1d0db23 100644 --- a/distdir_deps.bzl +++ b/distdir_deps.bzl @@ -272,7 +272,7 @@ DIST_DEPS = { "package_version": "1.5.2-3", }, "blake3": { - "archive": "v1.3.3.zip", + "archive": "1.3.3.zip", "sha256": "bb529ba133c0256df49139bd403c17835edbf60d2ecd6463549c6a5fe279364d", "urls": [ "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.3.3.zip", diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index 27ac03a050c926..ac88542a61a969 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -96,6 +96,7 @@ filegroup( "//src/main/java/com/google/devtools/build/lib/util:srcs", "//src/main/java/com/google/devtools/build/lib/versioning:srcs", "//src/main/java/com/google/devtools/build/lib/vfs:srcs", + "//src/main/java/com/google/devtools/build/lib/vfs/bazel:srcs", "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs:srcs", "//src/main/java/com/google/devtools/build/lib/windows:srcs", "//src/main/java/com/google/devtools/build/lib/worker:srcs", @@ -443,6 +444,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:output_service", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/lib/vfs/bazel", "//src/main/java/com/google/devtools/build/lib/windows", "//src/main/java/com/google/devtools/build/lib/worker:worker_metric", "//src/main/java/com/google/devtools/build/skyframe", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index 087438d0f8b0ff..409723946c7301 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -157,6 +157,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/lib/vfs/bazel", "//src/main/java/com/google/devtools/build/lib/windows", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java index 1027f3a6de3e4c..f5af64e7ab9f66 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.JavaIoFileSystem; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.bazel.BazelHashFunctions; import com.google.devtools.build.lib.windows.WindowsFileSystem; import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsParsingResult; @@ -44,6 +45,10 @@ * com.google.devtools.build.lib.vfs.FileSystem} class use {@code SHA256} by default. */ public class BazelFileSystemModule extends BlazeModule { + static { + BazelHashFunctions.ensureRegistered(); + } + @Override public ModuleFileSystem getFileSystem( OptionsParsingResult startupOptions, PathFragment realExecRootBase) diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD new file mode 100644 index 00000000000000..fde90bc571f660 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BUILD @@ -0,0 +1,28 @@ +load("@rules_java//java:defs.bzl", "java_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "bazel", + srcs = glob( + [ + "*.java", + ], + ), + deps = [ + "//src/main/java/com/google/devtools/build/lib/jni", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//third_party:error_prone_annotations", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java new file mode 100644 index 00000000000000..c917441a5be460 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctions.java @@ -0,0 +1,45 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.bazel; + +import com.google.devtools.build.lib.jni.JniLoader; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import java.security.Security; +import javax.annotation.Nullable; + +/** Bazel specific {@link DigestHashFunction}s. */ +public final class BazelHashFunctions { + @Nullable public static final DigestHashFunction BLAKE3; + + static { + DigestHashFunction hashFunction = null; + + if (JniLoader.isJniAvailable()) { + try { + Security.addProvider(new Blake3Provider()); + hashFunction = DigestHashFunction.register(new Blake3HashFunction(), "BLAKE3"); + } catch (UnsatisfiedLinkError ignored) { + // This can happen if bazel was compiled manually (with compile.sh), + // on windows. In that case jni is available, but missing the blake3 + // symbols necessary to register the hasher. + } + } + + BLAKE3 = hashFunction; + } + + public static void ensureRegistered() {} + + private BazelHashFunctions() {} +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java new file mode 100644 index 00000000000000..0b203ba62eea5b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3HashFunction.java @@ -0,0 +1,88 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.bazel; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkPositionIndexes; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** A {@link HashFunction} for BLAKE3. */ +public final class Blake3HashFunction implements HashFunction { + @Override + public int bits() { + return 256; + } + + @Override + public Hasher newHasher() { + return new Blake3Hasher(new Blake3MessageDigest()); + } + + @Override + public Hasher newHasher(int expectedInputSize) { + checkArgument( + expectedInputSize >= 0, "expectedInputSize must be >= 0 but was %s", expectedInputSize); + return newHasher(); + } + + /* The following methods implement the {HashFunction} interface. */ + + @Override + public HashCode hashObject(T instance, Funnel funnel) { + return newHasher().putObject(instance, funnel).hash(); + } + + @Override + public HashCode hashUnencodedChars(CharSequence input) { + int len = input.length(); + return newHasher(len * 2).putUnencodedChars(input).hash(); + } + + @Override + public HashCode hashString(CharSequence input, Charset charset) { + return newHasher().putString(input, charset).hash(); + } + + @Override + public HashCode hashInt(int input) { + return newHasher(4).putInt(input).hash(); + } + + @Override + public HashCode hashLong(long input) { + return newHasher(8).putLong(input).hash(); + } + + @Override + public HashCode hashBytes(byte[] input) { + return hashBytes(input, 0, input.length); + } + + @Override + public HashCode hashBytes(byte[] input, int off, int len) { + checkPositionIndexes(off, off + len, input.length); + return newHasher(len).putBytes(input, off, len).hash(); + } + + @Override + public HashCode hashBytes(ByteBuffer input) { + return newHasher(input.remaining()).putBytes(input).hash(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java new file mode 100644 index 00000000000000..180d62bdd6fc40 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Hasher.java @@ -0,0 +1,146 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.bazel; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +/** A {@link Hasher} for BLAKE3. */ +public final class Blake3Hasher implements Hasher { + private final Blake3MessageDigest messageDigest; + private boolean isDone = false; + + public Blake3Hasher(Blake3MessageDigest blake3MessageDigest) { + messageDigest = blake3MessageDigest; + } + + /* The following methods implement the {Hasher} interface. */ + + @Override + @CanIgnoreReturnValue + public Hasher putBytes(ByteBuffer b) { + messageDigest.engineUpdate(b); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes, int off, int len) { + messageDigest.engineUpdate(bytes, off, len); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes) { + messageDigest.engineUpdate(bytes, 0, bytes.length); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putByte(byte b) { + messageDigest.engineUpdate(b); + return this; + } + + @Override + public HashCode hash() { + checkState(!isDone); + isDone = true; + + return HashCode.fromBytes(messageDigest.engineDigest()); + } + + @Override + @CanIgnoreReturnValue + public final Hasher putBoolean(boolean b) { + return putByte(b ? (byte) 1 : (byte) 0); + } + + @Override + @CanIgnoreReturnValue + public final Hasher putDouble(double d) { + return putLong(Double.doubleToRawLongBits(d)); + } + + @Override + @CanIgnoreReturnValue + public final Hasher putFloat(float f) { + return putInt(Float.floatToRawIntBits(f)); + } + + @Override + @CanIgnoreReturnValue + public Hasher putUnencodedChars(CharSequence charSequence) { + for (int i = 0, len = charSequence.length(); i < len; i++) { + putChar(charSequence.charAt(i)); + } + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putString(CharSequence charSequence, Charset charset) { + return putBytes(charSequence.toString().getBytes(charset)); + } + + @Override + @CanIgnoreReturnValue + public Hasher putShort(short s) { + putByte((byte) s); + putByte((byte) (s >>> 8)); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putInt(int i) { + putByte((byte) i); + putByte((byte) (i >>> 8)); + putByte((byte) (i >>> 16)); + putByte((byte) (i >>> 24)); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putLong(long l) { + for (int i = 0; i < 64; i += 8) { + putByte((byte) (l >>> i)); + } + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putChar(char c) { + putByte((byte) c); + putByte((byte) (c >>> 8)); + return this; + } + + @Override + @CanIgnoreReturnValue + public Hasher putObject(T instance, Funnel funnel) { + funnel.funnel(instance, this); + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java new file mode 100644 index 00000000000000..50d2ace80f51a8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3MessageDigest.java @@ -0,0 +1,139 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.bazel; + +import static java.lang.Math.min; + +import com.google.devtools.build.lib.jni.JniLoader; +import java.nio.ByteBuffer; +import java.security.DigestException; +import java.security.MessageDigest; + +/** A {@link MessageDigest} for BLAKE3. */ +public final class Blake3MessageDigest extends MessageDigest { + // These constants match the native definitions in: + // https://github.com/BLAKE3-team/BLAKE3/blob/master/c/blake3.h + public static final int KEY_LEN = 32; + public static final int OUT_LEN = 32; + + static { + JniLoader.loadJni(); + } + + private static final int STATE_SIZE = hasher_size(); + private static final byte[] INITIAL_STATE = new byte[STATE_SIZE]; + + static { + initialize_hasher(INITIAL_STATE); + } + + // To reduce the number of calls made via JNI, buffer up to this many bytes + // before updating the hasher. + public static final int ONESHOT_THRESHOLD = 8 * 1024; + + private final ByteBuffer buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD); + private final byte[] hasher = new byte[STATE_SIZE]; + + public Blake3MessageDigest() { + super("BLAKE3"); + System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE); + } + + private void flush() { + if (buffer.position() > 0) { + blake3_hasher_update(hasher, buffer.array(), buffer.position()); + buffer.clear(); + } + } + + @Override + public void engineUpdate(byte[] data, int offset, int length) { + while (length > 0) { + int numToCopy = min(length, buffer.remaining()); + buffer.put(data, offset, numToCopy); + length -= numToCopy; + offset += numToCopy; + + if (buffer.remaining() == 0) { + flush(); + } + } + } + + @Override + public void engineUpdate(byte b) { + if (buffer.remaining() == 0) { + flush(); + } + buffer.put(b); + } + + @Override + public void engineUpdate(ByteBuffer input) { + super.engineUpdate(input); + } + + private byte[] getOutput(int outputLength) { + flush(); + + byte[] retByteArray = new byte[outputLength]; + blake3_hasher_finalize(hasher, retByteArray, outputLength); + + engineReset(); + return retByteArray; + } + + @Override + public Object clone() throws CloneNotSupportedException { + throw new CloneNotSupportedException(); + } + + @Override + public void engineReset() { + buffer.clear(); + System.arraycopy(INITIAL_STATE, 0, hasher, 0, STATE_SIZE); + } + + @Override + public int engineGetDigestLength() { + return OUT_LEN; + } + + @Override + public byte[] engineDigest() { + return getOutput(OUT_LEN); + } + + @Override + public int engineDigest(byte[] buf, int off, int len) throws DigestException { + if (len < OUT_LEN) { + throw new DigestException("partial digests not returned"); + } + if (buf.length - off < OUT_LEN) { + throw new DigestException("insufficient space in the output buffer to store the digest"); + } + + byte[] digestBytes = getOutput(OUT_LEN); + System.arraycopy(digestBytes, 0, buf, off, digestBytes.length); + return digestBytes.length; + } + + public static final native int hasher_size(); + + public static final native void initialize_hasher(byte[] hasher); + + public static final native void blake3_hasher_update(byte[] hasher, byte[] input, int inputLen); + + public static final native void blake3_hasher_finalize(byte[] hasher, byte[] out, int outLen); +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java new file mode 100644 index 00000000000000..0c868147dc4572 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/bazel/Blake3Provider.java @@ -0,0 +1,24 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.vfs.bazel; + +import java.security.Provider; + +/** A {@link Provider} for BLAKE3. */ +public final class Blake3Provider extends Provider { + public Blake3Provider() { + super("BLAKE3Provider", "1.0", "A BLAKE3 digest provider"); + put("MessageDigest.BLAKE3", Blake3MessageDigest.class.getName()); + } +} diff --git a/src/main/native/BUILD b/src/main/native/BUILD index 2601d3298cfb30..c011aef6d20c2b 100644 --- a/src/main/native/BUILD +++ b/src/main/native/BUILD @@ -54,6 +54,21 @@ cc_library( includes = ["."], # For jni headers. ) +cc_library( + name = "blake3_jni", + srcs = [ + "blake3_jni.cc", + ":jni.h", + ":jni_md.h", + ], + includes = ["."], # For jni headers. + visibility = ["//src/main/native:__subpackages__"], + deps = [ + "@blake3", + ], + alwayslink = 1, +) + cc_binary( name = "libunix_jni.so", srcs = [ @@ -80,6 +95,7 @@ cc_binary( linkshared = 1, visibility = ["//src/main/java/com/google/devtools/build/lib/jni:__pkg__"], deps = [ + ":blake3_jni", ":latin1_jni_path", "//src/main/cpp/util:logging", "//src/main/cpp/util:md5", diff --git a/src/main/native/blake3_jni.cc b/src/main/native/blake3_jni.cc new file mode 100644 index 00000000000000..56201888e91a56 --- /dev/null +++ b/src/main/native/blake3_jni.cc @@ -0,0 +1,73 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "c/blake3.h" + +namespace blaze_jni { + +jbyte *get_byte_array(JNIEnv *env, jbyteArray java_array) { + return (jbyte *)env->GetPrimitiveArrayCritical(java_array, nullptr); +} + +void release_byte_array(JNIEnv *env, jbyteArray array, jbyte *addr) { + env->ReleasePrimitiveArrayCritical(array, addr, 0); +} + +extern "C" JNIEXPORT int JNICALL +Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_hasher_1size( + JNIEnv *env, jobject obj) { + return (int)sizeof(blake3_hasher); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_initialize_1hasher( + JNIEnv *env, jobject obj, jbyteArray jhasher) { + blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher); + if (hasher) { + blake3_hasher_init(hasher); + release_byte_array(env, jhasher, (jbyte *)hasher); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_blake3_1hasher_1update( + JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray input, + jint input_len) { + blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher); + if (hasher) { + jbyte *input_addr = get_byte_array(env, input); + blake3_hasher_update(hasher, input_addr, input_len); + release_byte_array(env, input, input_addr); + release_byte_array(env, jhasher, (jbyte *)hasher); + } +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_bazel_Blake3MessageDigest_blake3_1hasher_1finalize( + JNIEnv *env, jobject obj, jbyteArray jhasher, jbyteArray out, + jint out_len) { + blake3_hasher *hasher = (blake3_hasher *)get_byte_array(env, jhasher); + if (hasher) { + jbyte *out_addr = get_byte_array(env, out); + blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len); + release_byte_array(env, out, out_addr); + release_byte_array(env, jhasher, (jbyte *)hasher); + } +} + +} // namespace blaze_jni diff --git a/src/main/native/windows/BUILD b/src/main/native/windows/BUILD index 73eaeed150544b..b595e7553353b2 100644 --- a/src/main/native/windows/BUILD +++ b/src/main/native/windows/BUILD @@ -76,6 +76,7 @@ cc_binary( deps = [ ":lib-file", ":lib-process", + "//src/main/native:blake3_jni", ], ) diff --git a/src/test/java/com/google/devtools/build/lib/vfs/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/BUILD index 8b9e00889aaf59..d4856a8ea12407 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/BUILD +++ b/src/test/java/com/google/devtools/build/lib/vfs/BUILD @@ -10,6 +10,7 @@ filegroup( name = "srcs", testonly = 0, srcs = glob(["**"]) + [ + "//src/test/java/com/google/devtools/build/lib/vfs/bazel:srcs", "//src/test/java/com/google/devtools/build/lib/vfs/util:srcs", ], visibility = ["//src:__subpackages__"], diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD new file mode 100644 index 00000000000000..f1d34cef6afb5f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD @@ -0,0 +1,42 @@ +load("@rules_java//java:defs.bzl", "java_library", "java_test") + +package( + default_applicable_licenses = ["//:license"], + default_testonly = 1, + default_visibility = ["//src:__subpackages__"], +) + +filegroup( + name = "srcs", + testonly = 0, + srcs = glob(["**"]), + visibility = ["//src:__subpackages__"], +) + +java_library( + name = "BazelTests_lib", + srcs = glob( + [ + "*.java", + ], + ), + deps = [ + "//src/main/java/com/google/devtools/build/lib/vfs/bazel", + "//src/test/java/com/google/devtools/build/lib/testutil", + "//third_party:guava", + "//third_party:guava-testlib", + "//third_party:junit4", + "//third_party:truth", + "//third_party/protobuf:protobuf_java", + ], +) + +java_test( + name = "BazelTests", + size = "small", + test_class = "com.google.devtools.build.lib.AllTests", + runtime_deps = [ + ":BazelTests_lib", + "//src/test/java/com/google/devtools/build/lib:test_runner", + ], +) diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java b/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java new file mode 100644 index 00000000000000..9fa0cb3929d5c0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/Blake3HasherTest.java @@ -0,0 +1,48 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs.bazel; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link Blake3MessageDigest}. */ +@RunWith(JUnit4.class) +public class Blake3HasherTest { + @Test + public void emptyHash() { + Blake3Hasher h = new Blake3Hasher(new Blake3MessageDigest()); + + byte[] data = new byte[0]; + h.putBytes(data); + + assertEquals( + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", h.hash().toString()); + } + + @Test + public void helloWorld() { + Blake3Hasher h = new Blake3Hasher(new Blake3MessageDigest()); + + byte[] data = "hello world".getBytes(StandardCharsets.US_ASCII); + h.putBytes(data); + + assertEquals( + "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24", h.hash().toString()); + } +} diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD index 6e96f051f1412e..867e33f67c6cec 100644 --- a/third_party/blake3/blake3.BUILD +++ b/third_party/blake3/blake3.BUILD @@ -30,7 +30,9 @@ cc_library( ] + select({ "@bazel_tools//src/conditions:linux_x86_64": [ "c/blake3_avx2_x86-64_unix.S", - "c/blake3_avx512_x86-64_unix.S", + # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 + # lacking the headers to compile AVX512. + # "c/blake3_avx512_x86-64_unix.S", "c/blake3_sse2_x86-64_unix.S", "c/blake3_sse41_x86-64_unix.S", ], @@ -50,16 +52,21 @@ cc_library( "c/blake3_impl.h", ], copts = select({ - "@bazel_tools//src/conditions:linux_x86_64": [], + "@bazel_tools//src/conditions:linux_x86_64": [ + # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 + # lacking the headers to compile AVX512. + "-DBLAKE3_NO_AVX512", + ], "@bazel_tools//src/conditions:windows_x64": [], "@bazel_tools//src/conditions:darwin_arm64": [ "-DBLAKE3_USE_NEON=1", ], "//conditions:default": [ - "-DBLAKE3_NO_SSE2", - "-DBLAKE3_NO_SSE41", "-DBLAKE3_NO_AVX2", "-DBLAKE3_NO_AVX512", + "-DBLAKE3_NO_NEON", + "-DBLAKE3_NO_SSE2", + "-DBLAKE3_NO_SSE41", ], }), includes = ["."],