From dfd6b9797f646f479c8735a5e36181b2a7f071b1 Mon Sep 17 00:00:00 2001 From: Tony Aiuto Date: Wed, 13 Jan 2021 13:38:47 -0500 Subject: [PATCH] Move rules_proto and protocolbuffers/ubp to distdir_deps.bzl - generate @rules_proto workspace inclusions in tests - src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestEnvironment.java will be done in a future CL. That will be the first case where we need to generate Java code. It requires some design up front. Context: #12081 --- WORKSPACE | 44 +------------------ distdir_deps.bzl | 27 ++++++++++++ .../devtools/build/lib/bazel/rules/java/BUILD | 1 + .../lib/bazel/rules/java/jdk.WORKSPACE.tmpl | 11 +---- src/test/shell/BUILD | 1 + src/test/shell/bazel/BUILD | 15 ++++++- .../shell/bazel/bazel_proto_library_test.sh | 13 ++---- src/test/shell/integration/BUILD | 10 +++++ .../integration/modify_execution_info_test.sh | 13 ++---- src/test/shell/testenv.sh.tmpl | 10 +---- 10 files changed, 62 insertions(+), 83 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 86e4b7e3dc45bf..e2311bfcc590d7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -131,20 +131,12 @@ distdir_tar( archives = [ "coverage_output_generator-v2.5.zip", "android_tools_pkg-0.19.0rc3.tar.gz", - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", ], dirname = "derived/distdir", dist_deps = {dep: attrs for dep, attrs in DIST_DEPS.items() if "additional_distfiles" in attrs["used_in"]}, sha256 = { "coverage_output_generator-v2.5.zip": "cd14f1cb4559e4723e63b7e7b06d09fcc3bd7ba58d03f354cdff1439bd936a7d", "android_tools_pkg-0.19.0rc3.tar.gz": "ea5c0589a01e2a9f43c20e5c145d3530e3b3bdbe7322789bc5da38d0ca49b837", - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz": "7992217989f3156f8109931c1fc6db3434b7414957cb82371552377beaeb9d6c", }, urls = { "coverage_output_generator-v2.5.zip": [ @@ -153,16 +145,6 @@ distdir_tar( "android_tools_pkg-0.19.0rc3.tar.gz": [ "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.19.0rc3.tar.gz", ], - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz": [ - "https://mirror.bazel.build/github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", - "https://github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", - ], }, ) @@ -330,16 +312,10 @@ dist_http_archive( patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_FILE_WIN, ) -http_archive( +dist_http_archive( name = "rules_proto", patch_cmds = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE, patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE_WIN, - sha256 = "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - strip_prefix = "rules_proto-7e4afce6fe62dbff0a4a03450143146f9f2d7488", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], ) # For testing, have an distdir_tar with all the archives implicit in every @@ -354,10 +330,6 @@ distdir_tar( "zulu11.37.17-ca-jdk11.0.6-macosx_x64.tar.gz", "zulu11.37.17-ca-jdk11.0.6-win_x64.zip", "android_tools_pkg-0.19.0rc3.tar.gz", - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", ], dirname = "test_WORKSPACE/distdir", dist_deps = {dep: attrs for dep, attrs in DIST_DEPS.items() if "test_WORKSPACE_files" in attrs["used_in"]}, @@ -368,10 +340,6 @@ distdir_tar( "zulu11.37.17-ca-jdk11.0.6-macosx_x64.tar.gz": "e1fe56769f32e2aaac95e0a8f86b5a323da5af3a3b4bba73f3086391a6cc056f", "zulu11.37.17-ca-jdk11.0.6-win_x64.zip": "a9695617b8374bfa171f166951214965b1d1d08f43218db9a2a780b71c665c18", "android_tools_pkg-0.19.0rc3.tar.gz": "ea5c0589a01e2a9f43c20e5c145d3530e3b3bdbe7322789bc5da38d0ca49b837", - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz": "7992217989f3156f8109931c1fc6db3434b7414957cb82371552377beaeb9d6c", }, urls = { "coverage_output_generator-v2.5.zip": ["https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v2.5.zip"], @@ -382,16 +350,6 @@ distdir_tar( "android_tools_pkg-0.19.0rc3.tar.gz": [ "https://mirror.bazel.build/bazel_android_tools/android_tools_pkg-0.19.0rc3.tar.gz", ], - # bazelbuild/rules_proto - "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz": [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], - # protocolbuffers/upb - "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz": [ - "https://mirror.bazel.build/github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", - "https://github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", - ], }, ) diff --git a/distdir_deps.bzl b/distdir_deps.bzl index e35f956e9d3df8..ef6a8cae9d109b 100644 --- a/distdir_deps.bzl +++ b/distdir_deps.bzl @@ -73,6 +73,21 @@ DIST_DEPS = { "test_WORKSPACE_files", ], }, + # TODO(aiuto): Update src/test/java/com/google/devtools/build/lib/blackbox/framework/BlackBoxTestEnvironment.java to use + # this dynamically. + "rules_proto": { + "archive": "7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", + "sha256": "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", + "strip_prefix": "rules_proto-7e4afce6fe62dbff0a4a03450143146f9f2d7488", + "urls": [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", + ], + "used_in": [ + "additional_distfiles", + "test_WORKSPACE_files", + ], + }, ################################################# # # Dependencies which are part of the Bazel binary @@ -93,6 +108,18 @@ DIST_DEPS = { "test_WORKSPACE_files", ], }, + "protocolbuffers": { + "archive": "382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", + "sha256": "7992217989f3156f8109931c1fc6db3434b7414957cb82371552377beaeb9d6c", + "urls": [ + "https://mirror.bazel.build/github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", + "https://github.com/protocolbuffers/upb/archive/382d5afc60e05470c23e8de19b19fc5ad231e732.tar.gz", + ], + "used_in": [ + "additional_distfiles", + "test_WORKSPACE_files", + ], + }, "com_github_grpc_grpc": { "archive": "v1.32.0.tar.gz", "sha256": "f880ebeb2ccf0e47721526c10dd97469200e40b5f101a0d9774eb69efa0bd07a", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD index d952928e416727..833b96cee026d0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD @@ -84,6 +84,7 @@ gen_workspace_stanza( "remote_java_tools_darwin", "rules_cc", "rules_java", + "rules_proto", ], template = "jdk.WORKSPACE.tmpl", use_maybe = True, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl index 206edb838ad3e7..6a3834a530d952 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE.tmpl @@ -221,15 +221,6 @@ maybe( {rules_cc} # Needed only because of java_tools. -maybe( - http_archive, - "rules_proto", - sha256 = "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - strip_prefix = "rules_proto-7e4afce6fe62dbff0a4a03450143146f9f2d7488", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], -) +{rules_proto} register_toolchains("@bazel_tools//tools/jdk:all") diff --git a/src/test/shell/BUILD b/src/test/shell/BUILD index 49155dfbf1b6a7..784a740d160e0a 100644 --- a/src/test/shell/BUILD +++ b/src/test/shell/BUILD @@ -28,6 +28,7 @@ gen_workspace_stanza( repos = [ "rules_cc", "rules_java", + "rules_proto", ], template = "testenv.sh.tmpl", visibility = ["//visibility:public"], diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 605c4ac1b36ed9..9bb21446c54fa9 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -1,3 +1,5 @@ +load("//:distdir_deps.bzl", "gen_workspace_stanza") + package(default_visibility = ["//visibility:private"]) filegroup( @@ -20,6 +22,14 @@ genrule( cmd = "cp $< $@", ) +gen_workspace_stanza( + name = "rules_proto_stanza", + out = "rules_proto_stanza.txt", + repos = [ + "rules_proto", + ], +) + filegroup( name = "test-deps", testonly = 1, @@ -326,7 +336,10 @@ sh_test( name = "bazel_proto_library_test", size = "large", # Downloads and compiles protobuf for *every* *test* *case* srcs = ["bazel_proto_library_test.sh"], - data = [":test-deps"], + data = [ + ":rules_proto_stanza.txt", + ":test-deps", + ], exec_compatible_with = ["//:highcpu_machine"], tags = ["no_windows"], # Doesn't work on Windows for unknown reason ) diff --git a/src/test/shell/bazel/bazel_proto_library_test.sh b/src/test/shell/bazel/bazel_proto_library_test.sh index 05df5df7389ac4..c41b4e1e678c13 100755 --- a/src/test/shell/bazel/bazel_proto_library_test.sh +++ b/src/test/shell/bazel/bazel_proto_library_test.sh @@ -60,16 +60,9 @@ new_local_repository( ) load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -http_archive( - name = "rules_proto", - strip_prefix = "rules_proto-7e4afce6fe62dbff0a4a03450143146f9f2d7488", - sha256 = "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], -) +EOF + cat $(rlocation io_bazel/src/tests/shell/bazel/rules_proto_stanza.txt) >> "$workspace"WORKSPACE + cat >> "$workspace"WORKSPACE << EOF load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") rules_proto_dependencies() rules_proto_toolchains() diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD index ce8b58991a0ac3..2ef3c3dc2b7672 100644 --- a/src/test/shell/integration/BUILD +++ b/src/test/shell/integration/BUILD @@ -1,3 +1,4 @@ +load("//:distdir_deps.bzl", "gen_workspace_stanza") load("@rules_cc//cc:defs.bzl", "cc_binary") package(default_visibility = ["//visibility:private"]) @@ -36,6 +37,14 @@ filegroup( ], ) +gen_workspace_stanza( + name = "rules_proto_stanza", + out = "rules_proto_stanza.txt", + repos = [ + "rules_proto", + ], +) + sh_test( name = "progress_reporting_test", size = "large", @@ -355,6 +364,7 @@ sh_test( name = "modify_execution_info_test", srcs = ["modify_execution_info_test.sh"], data = [ + ":rules_proto_stanza.txt", ":test-deps", "@bazel_tools//tools/bash/runfiles", ], diff --git a/src/test/shell/integration/modify_execution_info_test.sh b/src/test/shell/integration/modify_execution_info_test.sh index 3b2ef9d05d2194..28e485ef0bb60a 100755 --- a/src/test/shell/integration/modify_execution_info_test.sh +++ b/src/test/shell/integration/modify_execution_info_test.sh @@ -162,16 +162,9 @@ new_local_repository( build_file = "$(rlocation io_bazel/third_party/rules_python/BUILD)", workspace_file = "$(rlocation io_bazel/third_party/rules_python/rules_python.WORKSPACE)", ) - -http_archive( - name = "rules_proto", - strip_prefix = "rules_proto-7e4afce6fe62dbff0a4a03450143146f9f2d7488", - sha256 = "8e7d59a5b12b233be5652e3d29f42fba01c7cbab09f6b3a8d0a57ed6d1e9a0da", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - "https://github.com/bazelbuild/rules_proto/archive/7e4afce6fe62dbff0a4a03450143146f9f2d7488.tar.gz", - ], -) +EOF + cat "$(rlocation "io_bazel/src/test/shell/integration/rules_proto_stanza.txt")" >>WORKSPACE + cat >> WORKSPACE << EOF load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") rules_proto_dependencies() rules_proto_toolchains() diff --git a/src/test/shell/testenv.sh.tmpl b/src/test/shell/testenv.sh.tmpl index 4a546ed9cd5a06..e3f52e45ce3d09 100755 --- a/src/test/shell/testenv.sh.tmpl +++ b/src/test/shell/testenv.sh.tmpl @@ -559,15 +559,7 @@ function add_rules_proto_to_workspace() { cat >> "$1"<