From b28cb22196b6ecf062abaf9b16265e55a6b7d009 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Tue, 16 Apr 2024 14:01:41 +0200 Subject: [PATCH 1/5] Update blake3 to v1.5.1.bcr.1 Also adds a JMH benchmark pitting BLAKE3 against SHA2-256. --- MODULE.bazel | 4 +- MODULE.bazel.lock | 29 ++++---- maven_install.json | 8 +-- .../google/devtools/build/lib/vfs/bazel/BUILD | 26 +++++++ .../bazel/BazelHashFunctionsBenchmark.java | 68 +++++++++++++++++++ 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java diff --git a/MODULE.bazel b/MODULE.bazel index 766e48483bdb52..e2240d10eb6029 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,7 +20,7 @@ bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_pkg", version = "0.9.1") bazel_dep(name = "stardoc", version = "0.5.6", repo_name = "io_bazel_skydoc") bazel_dep(name = "zstd-jni", version = "1.5.2-3.bcr.1") -bazel_dep(name = "blake3", version = "1.3.3.bcr.1") +bazel_dep(name = "blake3", version = "1.5.1.bcr.1") bazel_dep(name = "sqlite3", version = "3.42.0.bcr.1") bazel_dep(name = "zlib", version = "1.3") bazel_dep(name = "rules_cc", version = "0.0.9") @@ -104,7 +104,7 @@ maven.install( "com.google.http-client:google-http-client-gson:1.42.0", "com.google.http-client:google-http-client:1.42.0", "com.google.j2objc:j2objc-annotations:1.3", - "com.google.turbine:turbine:0.6.0", + "com.google.turbine:turbine:0.5.0", "com.ryanharter.auto.value:auto-value-gson-extension:1.3.1", "com.ryanharter.auto.value:auto-value-gson-runtime:1.3.1", "com.ryanharter.auto.value:auto-value-gson-factory:1.3.1", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 83889a8bbd80e5..3f96a5a13ea9d7 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "41e97ab73eab5fafef6cd8f9b4696823e31384219a6356a052cab52245e1193c", + "moduleFileHash": "080c971252e98bbf09df6fd2ca65329e56fb5dfd43b5816f85444b43bb141b68", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -83,7 +83,7 @@ "com.google.http-client:google-http-client-gson:1.42.0", "com.google.http-client:google-http-client:1.42.0", "com.google.j2objc:j2objc-annotations:1.3", - "com.google.turbine:turbine:0.6.0", + "com.google.turbine:turbine:0.5.0", "com.ryanharter.auto.value:auto-value-gson-extension:1.3.1", "com.ryanharter.auto.value:auto-value-gson-runtime:1.3.1", "com.ryanharter.auto.value:auto-value-gson-factory:1.3.1", @@ -635,7 +635,7 @@ "rules_pkg": "rules_pkg@0.9.1", "io_bazel_skydoc": "stardoc@0.5.6", "zstd-jni": "zstd-jni@1.5.2-3.bcr.1", - "blake3": "blake3@1.3.3.bcr.1", + "blake3": "blake3@1.5.1.bcr.1", "sqlite3": "sqlite3@3.42.0.bcr.1", "zlib": "zlib@1.3", "rules_cc": "rules_cc@0.0.9", @@ -1008,10 +1008,10 @@ } } }, - "blake3@1.3.3.bcr.1": { + "blake3@1.5.1.bcr.1": { "name": "blake3", - "version": "1.3.3.bcr.1", - "key": "blake3@1.3.3.bcr.1", + "version": "1.5.1.bcr.1", + "key": "blake3@1.5.1.bcr.1", "repoName": "blake3", "executionPlatformsToRegister": [], "toolchainsToRegister": [], @@ -1026,13 +1026,14 @@ "ruleClassName": "http_archive", "attributes": { "urls": [ - "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.3.3.tar.gz" + "https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.5.1.tar.gz" ], - "integrity": "sha256-J9K8TuWUW6dUNIWVIQQslJRj7nUU/xeq7zKOI++D/sA=", - "strip_prefix": "BLAKE3-1.3.3", + "integrity": "sha256-gizTf3AVLlmFQz0sUMj2suyDqvEaoxvp/nFIapF0Tzc=", + "strip_prefix": "BLAKE3-1.5.1", "remote_patches": { - "https://bcr.bazel.build/modules/blake3/1.3.3.bcr.1/patches/add_build_file.patch": "sha256-lKVoznUHSqWywOo27+g4J0csjL8lH3FEXjAFRJN5+Kw=", - "https://bcr.bazel.build/modules/blake3/1.3.3.bcr.1/patches/module_dot_bazel.patch": "sha256-4M/MRHdDFjS8iyVaKqy6QIc5Qea9pblUz7oj6I5aHfg=" + "https://bcr.bazel.build/modules/blake3/1.5.1.bcr.1/patches/add_build_file.patch": "sha256-BmZOqWOTfHup68uNZXOh2mP+b971CWY3QufbkIe6eEM=", + "https://bcr.bazel.build/modules/blake3/1.5.1.bcr.1/patches/module_dot_bazel.patch": "sha256-dKnHpXqvNwW2m7vYxnfWEoBhphBysqbeBxAMVX3b5a0=", + "https://bcr.bazel.build/modules/blake3/1.5.1.bcr.1/patches/fix_windows_arm_build_pr_389.patch": "sha256-9G3QDBp5OuyYP7vwPKjqK+uTUZmjhLpKSE7nshz8guc=" }, "remote_patch_strip": 0 } @@ -2883,7 +2884,7 @@ "general": { "bzlTransitiveDigest": "tunTSmgwd2uvTzkCLtdbuCp0AI+WR+ftiPNqZ0rmcZk=", "recordedFileInputs": { - "@@//MODULE.bazel": "41e97ab73eab5fafef6cd8f9b4696823e31384219a6356a052cab52245e1193c", + "@@//MODULE.bazel": "080c971252e98bbf09df6fd2ca65329e56fb5dfd43b5816f85444b43bb141b68", "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "e6b22f35c7bce99c0677f24a19c7aee829f97e7b1b5cc31e600c7cc9b408a292" }, "recordedDirentsInputs": {}, @@ -6003,7 +6004,7 @@ "{ \"group\": \"com.google.http-client\", \"artifact\": \"google-http-client-gson\", \"version\": \"1.42.0\" }", "{ \"group\": \"com.google.http-client\", \"artifact\": \"google-http-client\", \"version\": \"1.42.0\" }", "{ \"group\": \"com.google.j2objc\", \"artifact\": \"j2objc-annotations\", \"version\": \"1.3\" }", - "{ \"group\": \"com.google.turbine\", \"artifact\": \"turbine\", \"version\": \"0.6.0\" }", + "{ \"group\": \"com.google.turbine\", \"artifact\": \"turbine\", \"version\": \"0.5.0\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-extension\", \"version\": \"1.3.1\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-runtime\", \"version\": \"1.3.1\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-factory\", \"version\": \"1.3.1\" }", @@ -8583,7 +8584,7 @@ "{ \"group\": \"com.google.http-client\", \"artifact\": \"google-http-client-gson\", \"version\": \"1.42.0\" }", "{ \"group\": \"com.google.http-client\", \"artifact\": \"google-http-client\", \"version\": \"1.42.0\" }", "{ \"group\": \"com.google.j2objc\", \"artifact\": \"j2objc-annotations\", \"version\": \"1.3\" }", - "{ \"group\": \"com.google.turbine\", \"artifact\": \"turbine\", \"version\": \"0.6.0\" }", + "{ \"group\": \"com.google.turbine\", \"artifact\": \"turbine\", \"version\": \"0.5.0\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-extension\", \"version\": \"1.3.1\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-runtime\", \"version\": \"1.3.1\" }", "{ \"group\": \"com.ryanharter.auto.value\", \"artifact\": \"auto-value-gson-factory\", \"version\": \"1.3.1\" }", diff --git a/maven_install.json b/maven_install.json index ecdce76fdc23ef..1ced30e058406a 100644 --- a/maven_install.json +++ b/maven_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": 1164989950, - "__RESOLVED_ARTIFACTS_HASH": -1287912278, + "__INPUT_ARTIFACTS_HASH": 1516511089, + "__RESOLVED_ARTIFACTS_HASH": -1733639136, "conflict_resolution": { "com.google.auto.value:auto-value-annotations:1.9": "com.google.auto.value:auto-value-annotations:1.10.4", "com.google.code.gson:gson:2.8.9": "com.google.code.gson:gson:2.9.0", @@ -284,9 +284,9 @@ }, "com.google.turbine:turbine": { "shasums": { - "jar": "277b6d59934dc8af95675a6cf6f054e9f43e0708344b08ba2aa927a578174950" + "jar": "3bb883977330ec99dd11d8bba0c9c9084b356189807920903984215a466c81ac" }, - "version": "0.6.0" + "version": "0.5.0" }, "com.ryanharter.auto.value:auto-value-gson-extension": { "shasums": { diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD index f1d34cef6afb5f..54a712588bf61a 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD @@ -19,6 +19,9 @@ java_library( [ "*.java", ], + exclude = [ + "BazelHashFunctionsBenchmark.java", + ], ), deps = [ "//src/main/java/com/google/devtools/build/lib/vfs/bazel", @@ -40,3 +43,26 @@ java_test( "//src/test/java/com/google/devtools/build/lib:test_runner", ], ) + +java_binary( + name = "BazelHashFunctionsBenchmark", + srcs = ["BazelHashFunctionsBenchmark.java"], + main_class = "org.openjdk.jmh.Main", + deps = [ + ":jmh", + "//src/main/java/com/google/devtools/build/lib/vfs/bazel", + "//third_party:guava", + ], +) + +java_library( + name = "jmh", + exported_plugins = [":jmh_benchmark_processor"], + exports = ["@maven//:org_openjdk_jmh_jmh_core"], +) + +java_plugin( + name = "jmh_benchmark_processor", + processor_class = "org.openjdk.jmh.generators.BenchmarkProcessor", + deps = ["@maven//:org_openjdk_jmh_jmh_generator_annprocess"], +) diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java new file mode 100644 index 00000000000000..e4992e33aa9ec4 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java @@ -0,0 +1,68 @@ +package com.google.devtools.build.lib.vfs.bazel; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import java.security.SecureRandom; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; + +@BenchmarkMode(Mode.Throughput) +@State(Scope.Benchmark) +// Realistic usage within Bazel will hash files on multiple threads. +@Threads(4) +public class BazelHashFunctionsBenchmark { + + static { + BazelHashFunctions.ensureRegistered(); + } + + public enum HashFunctionType { + BLAKE3(new Blake3HashFunction()), + SHA2_256(Hashing.sha256()); + + final HashFunction hashFunction; + + HashFunctionType(HashFunction hashFunction) { + this.hashFunction = hashFunction; + } + } + + @Param({"BLAKE3", "SHA2_256"}) + public HashFunctionType type; + + @Param({"1", "16", "128", "512", "1024", "4096", "16384", "1048576"}) + public int size; + + private byte[] data; + + @Setup(Level.Iteration) + public void setup() { + data = new byte[size]; + new SecureRandom().nextBytes(data); + } + + @Benchmark + public HashCode hashBytesOneShot() { + return type.hashFunction.hashBytes(data); + } + + private static final int CHUNK_SIZE = 4096; + + @Benchmark + public HashCode hashBytesChunks() { + Hasher hasher = type.hashFunction.newHasher(); + for (int pos = 0; pos < data.length; pos += CHUNK_SIZE) { + hasher.putBytes(data, pos, Math.min(CHUNK_SIZE, data.length - pos)); + } + return hasher.hash(); + } +} From ae4b660a5d74a9004474725d4144059379bc8b15 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Tue, 16 Apr 2024 21:52:41 +0200 Subject: [PATCH 2/5] Always build the benchmark with `-c opt` --- MODULE.bazel | 1 + MODULE.bazel.lock | 112 +++++++++++------- .../google/devtools/build/lib/vfs/bazel/BUILD | 3 +- .../build/lib/vfs/bazel/java_opt_binary.bzl | 3 + third_party/blake3/blake3.BUILD | 102 ++++++++++++++++ 5 files changed, 179 insertions(+), 42 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl create mode 100644 third_party/blake3/blake3.BUILD diff --git a/MODULE.bazel b/MODULE.bazel index e2240d10eb6029..215e7f22b29e5e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -31,6 +31,7 @@ bazel_dep(name = "rules_jvm_external", version = "6.0") bazel_dep(name = "rules_python", version = "0.28.0") bazel_dep(name = "rules_testing", version = "0.6.0") bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest") +bazel_dep(name = "with_cfg.bzl", version = "0.2.4") # TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR. bazel_dep(name = "remoteapis", version = "") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 3f96a5a13ea9d7..fb051f567f91c6 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "080c971252e98bbf09df6fd2ca65329e56fb5dfd43b5816f85444b43bb141b68", + "moduleFileHash": "a46ff64f51b56ec85fed41a5464644ab6c23f3696a14f9cd171b38495a4cbe2f", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -39,7 +39,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 75, + "line": 76, "column": 22 }, "imports": { @@ -168,7 +168,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 76, + "line": 77, "column": 14 } }, @@ -183,7 +183,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -198,7 +198,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -213,7 +213,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -228,7 +228,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -243,7 +243,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -258,7 +258,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -273,7 +273,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -288,7 +288,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -303,7 +303,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 199, + "line": 200, "column": 19 } }, @@ -331,7 +331,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 339, + "line": 340, "column": 22 } } @@ -345,7 +345,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 220, + "line": 221, "column": 32 }, "imports": { @@ -385,7 +385,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 254, + "line": 255, "column": 23 }, "imports": {}, @@ -399,7 +399,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 255, + "line": 256, "column": 17 } } @@ -413,7 +413,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 257, + "line": 258, "column": 20 }, "imports": { @@ -431,7 +431,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 258, + "line": 259, "column": 10 } } @@ -445,7 +445,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 269, + "line": 270, "column": 33 }, "imports": { @@ -476,7 +476,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 290, + "line": 291, "column": 29 }, "imports": { @@ -493,7 +493,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 293, + "line": 294, "column": 20 }, "imports": { @@ -512,7 +512,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 294, + "line": 295, "column": 12 } } @@ -526,7 +526,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 306, + "line": 307, "column": 32 }, "imports": { @@ -545,7 +545,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 314, + "line": 315, "column": 31 }, "imports": { @@ -562,7 +562,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 317, + "line": 318, "column": 48 }, "imports": { @@ -579,7 +579,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 361, + "line": 362, "column": 35 }, "imports": { @@ -596,7 +596,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 364, + "line": 365, "column": 42 }, "imports": { @@ -614,7 +614,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 367, + "line": 368, "column": 45 }, "imports": { @@ -646,6 +646,7 @@ "rules_python": "rules_python@0.28.0", "rules_testing": "rules_testing@0.6.0", "com_google_googletest": "googletest@1.14.0", + "with_cfg.bzl": "with_cfg.bzl@0.2.4", "remoteapis": "remoteapis@_", "googleapis": "googleapis@_", "apple_support": "apple_support@1.8.1", @@ -1815,6 +1816,35 @@ } } }, + "with_cfg.bzl@0.2.4": { + "name": "with_cfg.bzl", + "version": "0.2.4", + "key": "with_cfg.bzl@0.2.4", + "repoName": "with_cfg.bzl", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [], + "deps": { + "bazel_skylib": "bazel_skylib@1.5.0", + "bazel_tools": "bazel_tools@_", + "local_config_platform": "local_config_platform@_" + }, + "repoSpec": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/fmeum/with_cfg.bzl/releases/download/v0.2.4/with_cfg.bzl-v0.2.4.tar.gz" + ], + "integrity": "sha256-BqKxtWpYxHGrQNivFmxNUfCYLhxrxGN1uAWRWz/AZY4=", + "strip_prefix": "with_cfg.bzl-0.2.4", + "remote_patches": { + "https://bcr.bazel.build/modules/with_cfg.bzl/0.2.4/patches/module_dot_bazel_version.patch": "sha256-nMJ63F1h826i8AWSmXwz9pZYDwKfOnM5QdrZqyCLYMo=" + }, + "remote_patch_strip": 1 + } + } + }, "remoteapis@_": { "name": "remoteapis", "version": "", @@ -2884,7 +2914,7 @@ "general": { "bzlTransitiveDigest": "tunTSmgwd2uvTzkCLtdbuCp0AI+WR+ftiPNqZ0rmcZk=", "recordedFileInputs": { - "@@//MODULE.bazel": "080c971252e98bbf09df6fd2ca65329e56fb5dfd43b5816f85444b43bb141b68", + "@@//MODULE.bazel": "a46ff64f51b56ec85fed41a5464644ab6c23f3696a14f9cd171b38495a4cbe2f", "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "e6b22f35c7bce99c0677f24a19c7aee829f97e7b1b5cc31e600c7cc9b408a292" }, "recordedDirentsInputs": {}, @@ -5389,7 +5419,7 @@ "recordedFileInputs": { "@@//src/tools/android/maven_android_install.json": "09bff3e33d291336046f7c9201630fb5e014f0e60b78b6f09b84e4f5f73ed04f", "@@rules_jvm_external~//rules_jvm_external_deps_install.json": "cafb5d2d8119391eb2b322ce3840d3352ea82d496bdb8cbd4b6779ec4d044dda", - "@@//maven_install.json": "112e83df70ed9983987adeadea45a3ac179484234285c832ed87514ca7499583" + "@@//maven_install.json": "0333a28500b745188bd05d1a64e2facce6883d600ab41b58a0d93c0fc402c893" }, "recordedDirentsInputs": {}, "envVariables": {}, @@ -6468,6 +6498,17 @@ "downloaded_file_path": "v1/com/android/tools/common/30.1.3/common-30.1.3.jar" } }, + "com_google_turbine_turbine_0_5_0": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_file", + "attributes": { + "sha256": "3bb883977330ec99dd11d8bba0c9c9084b356189807920903984215a466c81ac", + "urls": [ + "https://repo1.maven.org/maven2/com/google/turbine/turbine/0.5.0/turbine-0.5.0.jar" + ], + "downloaded_file_path": "v1/com/google/turbine/turbine/0.5.0/turbine-0.5.0.jar" + } + }, "io_grpc_grpc_auth_1_48_1": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_file", @@ -6908,17 +6949,6 @@ "downloaded_file_path": "v1/com/google/code/java-allocation-instrumenter/java-allocation-instrumenter/3.3.0/java-allocation-instrumenter-3.3.0.jar" } }, - "com_google_turbine_turbine_0_6_0": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_file", - "attributes": { - "sha256": "277b6d59934dc8af95675a6cf6f054e9f43e0708344b08ba2aa927a578174950", - "urls": [ - "https://repo1.maven.org/maven2/com/google/turbine/turbine/0.6.0/turbine-0.6.0.jar" - ], - "downloaded_file_path": "v1/com/google/turbine/turbine/0.6.0/turbine-0.6.0.jar" - } - }, "org_jetbrains_kotlin_kotlin_stdlib_jdk7_1_4_32": { "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", "ruleClassName": "http_file", diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD index 54a712588bf61a..4307662b71318c 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BUILD @@ -1,4 +1,5 @@ load("@rules_java//java:defs.bzl", "java_library", "java_test") +load(":java_opt_binary.bzl", "java_opt_binary") package( default_applicable_licenses = ["//:license"], @@ -44,7 +45,7 @@ java_test( ], ) -java_binary( +java_opt_binary( name = "BazelHashFunctionsBenchmark", srcs = ["BazelHashFunctionsBenchmark.java"], main_class = "org.openjdk.jmh.Main", diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl b/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl new file mode 100644 index 00000000000000..22981ee40d3573 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl @@ -0,0 +1,3 @@ +load("@with_cfg.bzl", "with_cfg") + +java_opt_binary, _java_opt_binary = with_cfg(native.java_binary).set("compilation_mode", "opt").build() diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD new file mode 100644 index 00000000000000..d8e76ece8660aa --- /dev/null +++ b/third_party/blake3/blake3.BUILD @@ -0,0 +1,102 @@ +load("@rules_license//rules:license.bzl", "license") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") + +licenses(["notice"]) # BSD/MIT-like license + +exports_files(["LICENSE"]) + +license( + name = "license", + package_name = "blake3", + license_kinds = [ + "@rules_license//licenses/spdx:Apache-2.0", + ], + license_text = "LICENSE", + package_version = "1.3.3", +) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//third_party:__pkg__"], +) + +cc_library( + name = "blake3", + srcs = [ + "c/blake3.c", + "c/blake3_dispatch.c", + "c/blake3_portable.c", + ] + select({ + "@bazel_tools//src/conditions:linux_x86_64": [ + "c/blake3_avx2_x86-64_unix.S", + # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 + # lacking the headers to compile AVX512. + # "c/blake3_avx512_x86-64_unix.S", + "c/blake3_sse2_x86-64_unix.S", + "c/blake3_sse41_x86-64_unix.S", + ], + "@bazel_tools//src/conditions:linux_aarch64": [ + "c/blake3_neon.c", + ], + "@bazel_tools//src/conditions:windows_x64": [ + "c/blake3_avx2_x86-64_windows_msvc.asm", + "c/blake3_avx512_x86-64_windows_msvc.asm", + "c/blake3_sse2_x86-64_windows_msvc.asm", + "c/blake3_sse41_x86-64_windows_msvc.asm", + ], + "@bazel_tools//src/conditions:windows_arm64": [ + "c/blake3_neon.c", + ], + "@bazel_tools//src/conditions:darwin_arm64": [ + "c/blake3_neon.c", + ], + "//conditions:default": [], + }), + hdrs = [ + "c/blake3.h", + "c/blake3_impl.h", + ], + copts = select({ + "@bazel_tools//src/conditions:linux_x86_64": [ + # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 + # lacking the headers to compile AVX512. + "-DBLAKE3_NO_AVX512", + ], + "@bazel_tools//src/conditions:linux_aarch64": [ + "-DBLAKE3_USE_NEON=1", + ], + "@bazel_tools//src/conditions:windows_x64": [], + "@bazel_tools//src/conditions:windows_arm64": [ + "-DBLAKE3_USE_NEON=1", + ], + "@bazel_tools//src/conditions:darwin_arm64": [ + "-DBLAKE3_USE_NEON=1", + ], + "//conditions:default": [ + "-DBLAKE3_NO_AVX2", + "-DBLAKE3_NO_AVX512", + "-DBLAKE3_NO_NEON", + "-DBLAKE3_NO_SSE2", + "-DBLAKE3_NO_SSE41", + ], + }), + includes = ["."], + visibility = ["//visibility:public"], +) + +cc_binary( + name = "example", + srcs = [ + "c/example.c", + ], + copts = [ + "-w", + "-O3", + ], + includes = ["."], + visibility = ["//visibility:public"], + deps = [ + ":blake3", + ], +) From c4cf9de676c8f7a3d33b9a4dae7be2ded284b47e Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 17 Apr 2024 13:11:25 +0200 Subject: [PATCH 3/5] Simplify benchmark --- .../bazel/BazelHashFunctionsBenchmark.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java index e4992e33aa9ec4..a7b3158e20442c 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/BazelHashFunctionsBenchmark.java @@ -17,8 +17,6 @@ @BenchmarkMode(Mode.Throughput) @State(Scope.Benchmark) -// Realistic usage within Bazel will hash files on multiple threads. -@Threads(4) public class BazelHashFunctionsBenchmark { static { @@ -36,17 +34,30 @@ public enum HashFunctionType { } } + public enum Size { + B, + KB, + MB, + GB; + + final int bytes; + + Size() { + bytes = 1 << (ordinal() * 10); + } + } + @Param({"BLAKE3", "SHA2_256"}) public HashFunctionType type; - @Param({"1", "16", "128", "512", "1024", "4096", "16384", "1048576"}) - public int size; + @Param({"B", "KB", "MB", "GB"}) + public Size size; private byte[] data; @Setup(Level.Iteration) public void setup() { - data = new byte[size]; + data = new byte[size.bytes]; new SecureRandom().nextBytes(data); } @@ -54,15 +65,4 @@ public void setup() { public HashCode hashBytesOneShot() { return type.hashFunction.hashBytes(data); } - - private static final int CHUNK_SIZE = 4096; - - @Benchmark - public HashCode hashBytesChunks() { - Hasher hasher = type.hashFunction.newHasher(); - for (int pos = 0; pos < data.length; pos += CHUNK_SIZE) { - hasher.putBytes(data, pos, Math.min(CHUNK_SIZE, data.length - pos)); - } - return hasher.hash(); - } } From 96aa95524e4e8f8c2f0e50b5071f651a5fc9193b Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Mon, 22 Apr 2024 15:09:03 +0200 Subject: [PATCH 4/5] Add doc string --- .../build/lib/vfs/bazel/java_opt_binary.bzl | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl b/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl index 22981ee40d3573..cac1de2b2bc1a4 100644 --- a/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl +++ b/src/test/java/com/google/devtools/build/lib/vfs/bazel/java_opt_binary.bzl @@ -1,3 +1,23 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A wrapper around java_binary that forces it to be built with `-c opt`. + +This is useful for benchmark targets. +""" + load("@with_cfg.bzl", "with_cfg") java_opt_binary, _java_opt_binary = with_cfg(native.java_binary).set("compilation_mode", "opt").build() From 2a2f1696e1f244ec169c4d1881d68006f82d6c23 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 24 Apr 2024 12:35:23 +0200 Subject: [PATCH 5/5] Remove build file --- third_party/blake3/blake3.BUILD | 102 -------------------------------- 1 file changed, 102 deletions(-) delete mode 100644 third_party/blake3/blake3.BUILD diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD deleted file mode 100644 index d8e76ece8660aa..00000000000000 --- a/third_party/blake3/blake3.BUILD +++ /dev/null @@ -1,102 +0,0 @@ -load("@rules_license//rules:license.bzl", "license") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") - -licenses(["notice"]) # BSD/MIT-like license - -exports_files(["LICENSE"]) - -license( - name = "license", - package_name = "blake3", - license_kinds = [ - "@rules_license//licenses/spdx:Apache-2.0", - ], - license_text = "LICENSE", - package_version = "1.3.3", -) - -filegroup( - name = "srcs", - srcs = glob(["**"]), - visibility = ["//third_party:__pkg__"], -) - -cc_library( - name = "blake3", - srcs = [ - "c/blake3.c", - "c/blake3_dispatch.c", - "c/blake3_portable.c", - ] + select({ - "@bazel_tools//src/conditions:linux_x86_64": [ - "c/blake3_avx2_x86-64_unix.S", - # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 - # lacking the headers to compile AVX512. - # "c/blake3_avx512_x86-64_unix.S", - "c/blake3_sse2_x86-64_unix.S", - "c/blake3_sse41_x86-64_unix.S", - ], - "@bazel_tools//src/conditions:linux_aarch64": [ - "c/blake3_neon.c", - ], - "@bazel_tools//src/conditions:windows_x64": [ - "c/blake3_avx2_x86-64_windows_msvc.asm", - "c/blake3_avx512_x86-64_windows_msvc.asm", - "c/blake3_sse2_x86-64_windows_msvc.asm", - "c/blake3_sse41_x86-64_windows_msvc.asm", - ], - "@bazel_tools//src/conditions:windows_arm64": [ - "c/blake3_neon.c", - ], - "@bazel_tools//src/conditions:darwin_arm64": [ - "c/blake3_neon.c", - ], - "//conditions:default": [], - }), - hdrs = [ - "c/blake3.h", - "c/blake3_impl.h", - ], - copts = select({ - "@bazel_tools//src/conditions:linux_x86_64": [ - # Disable to appease bazel-ci which uses ubuntu-18 (EOL) and GCC 7 - # lacking the headers to compile AVX512. - "-DBLAKE3_NO_AVX512", - ], - "@bazel_tools//src/conditions:linux_aarch64": [ - "-DBLAKE3_USE_NEON=1", - ], - "@bazel_tools//src/conditions:windows_x64": [], - "@bazel_tools//src/conditions:windows_arm64": [ - "-DBLAKE3_USE_NEON=1", - ], - "@bazel_tools//src/conditions:darwin_arm64": [ - "-DBLAKE3_USE_NEON=1", - ], - "//conditions:default": [ - "-DBLAKE3_NO_AVX2", - "-DBLAKE3_NO_AVX512", - "-DBLAKE3_NO_NEON", - "-DBLAKE3_NO_SSE2", - "-DBLAKE3_NO_SSE41", - ], - }), - includes = ["."], - visibility = ["//visibility:public"], -) - -cc_binary( - name = "example", - srcs = [ - "c/example.c", - ], - copts = [ - "-w", - "-O3", - ], - includes = ["."], - visibility = ["//visibility:public"], - deps = [ - ":blake3", - ], -)