From d10013de94a8d8aa19a85130c8c9ff0285852eda Mon Sep 17 00:00:00 2001 From: Ivo List Date: Wed, 25 Nov 2020 14:41:16 -0800 Subject: [PATCH] Separate java_tools into platform independent and prebuilt part. java_tools is repository package containing tools needed during Java compilation: JavaBuilder, patches for Java compiler, ijar, singlejar, ... Most of the files are jars with Java classes. java_tools are released for three platforms: linux, windows and darwin, however the only difference is in two binaries: ijar and singlejar. This is part one of splitting java_tools and releasing split version (following PR makes use of released split version in Bazel) Java_tools used to be released for multiple Java versions, but all the releases were the same except a some string substitutions in BUILD file. I changed to build only a single version, since it already supports Java from 8 to 14. Changes: - BUILD.java_tools is split into BUILD.java_tools_prebuilt (where the second contains prebuilt binaries) - toolchain definitions are removed from BUILD.java_tools and will be added to tools/jdk/BUILD in the second part - java_toolchain_default.bzl.java_tools is removed (default_java_toolchain.bzl will be updated with its features in the second part). - src/BUILD: JAVA_VERSION is removed, targets used to build java_tools.zip are duplicated to build java_tools_prebuilt.zip (done some cleanup as well) - upload_all_java_tools.sh and upload_java_tools.sh: used by Build kite, I removed java_version over the release, but kept it over tests (for different JDKs) - create_java_tools_release.sh: used by the user in the release process - added platform independent part - tests are updated to use platform independent and platform files, some tests had to be disabled and will be reenabled after the release Closes #12546. PiperOrigin-RevId: 344319092 --- src/BUILD | 316 +++++++++--------- src/create_java_tools_release.sh | 61 ++-- src/test/shell/bazel/BUILD | 88 +++-- .../shell/bazel/bazel_coverage_java_test.sh | 24 +- src/test/shell/bazel/bazel_java14_test.sh | 18 +- src/test/shell/bazel/bazel_java15_test.sh | 18 +- src/test/shell/bazel/bazel_java_test.sh | 26 +- .../shell/bazel/bazel_java_tools_dist_test.sh | 6 +- src/test/shell/bazel/bazel_java_tools_test.sh | 122 ++----- src/tools/singlejar/BUILD | 14 +- src/upload_all_java_tools.sh | 67 ++-- src/upload_java_tools.sh | 11 +- tools/jdk/BUILD | 2 +- tools/jdk/BUILD.java_tools | 176 ---------- tools/jdk/BUILD.java_tools_prebuilt | 24 ++ .../jdk/java_toolchain_default.bzl.java_tools | 178 ---------- 16 files changed, 445 insertions(+), 706 deletions(-) create mode 100644 tools/jdk/BUILD.java_tools_prebuilt delete mode 100644 tools/jdk/java_toolchain_default.bzl.java_tools diff --git a/src/BUILD b/src/BUILD index f00e228276ba58..b39dc0d26c0ee6 100644 --- a/src/BUILD +++ b/src/BUILD @@ -552,6 +552,39 @@ sh_binary( visibility = ["//visibility:public"], ) +# The java_tools release process produces following zip files: +# - java_tools_dist.zip: all sources (Java and C++) +# - java_tools.zip: platform independent part - Java classes (deploy jars) and C++ sources +# - java_tools_prebuilt.zip: C++ binaries (for platforms: Linux, Darwin and Windows) +# +# The files are then used in Bazel as @remote_java_tools and @remote_java_tools_{platform} +# repositories. +# The zip files are not embedded or used in Bazel. + +# Following target build java_tools_dist.zip - the sources +genrule( + name = "java_tools_dist", + srcs = [ + "//src/java_tools/buildjar:srcs", + "//src/java_tools/junitrunner:srcs", + "//src/java_tools/singlejar:srcs", + "//src/tools/singlejar:embedded_java_tools", + "//third_party/checker_framework_dataflow:srcs", + "//third_party/checker_framework_javacutil:srcs", + "//third_party/ijar:transitive_sources", + "//third_party/java/jacoco:transitive_sources", + "//third_party/java/proguard:srcs", + "@java_tools_langtools_javac11//:srcs", + ], + outs = ["java_tools_dist.zip"], + cmd = "zip -qXr $@ $$(echo $(SRCS) | sort)", + output_to_bindir = 1, + visibility = ["//src/test/shell/bazel:__pkg__"], +) + +# Following targets build java_tools.zip - platform independent part of java_tools +# It is painstainkingly built by renaming single files, zipping them and merging zips together +# TODO(bazel-team): Replace this with a single target, for example "release_archive" in rules_kotlin JAVA_TOOLS_DEPLOY_JARS = [ "//src/java_tools/buildjar:JavaBuilder_deploy.jar", "//src/java_tools/buildjar:VanillaJavaBuilder_deploy.jar", @@ -559,174 +592,139 @@ JAVA_TOOLS_DEPLOY_JARS = [ "//src/java_tools/buildjar/java/com/google/devtools/build/java/turbine:turbine_direct_binary_deploy.jar", "//src/java_tools/junitrunner/java/com/google/testing/coverage:JacocoCoverage_jarjar_deploy.jar", "//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar", -] + select({ - "//src/conditions:arm": ["//src/java_tools/singlejar/java/com/google/devtools/build/singlejar:bazel-singlejar_deploy.jar"], - "//conditions:default": [], -}) + "//src/java_tools/singlejar/java/com/google/devtools/build/singlejar:bazel-singlejar_deploy.jar", +] -JAVA_VERSIONS = ("11",) +genrule( + name = "jars_java_tools_zip", + srcs = JAVA_TOOLS_DEPLOY_JARS + [ + "@java_tools_langtools_javac11//:jdk_compiler_jar", + "@java_tools_langtools_javac11//:java_compiler_jar", + "@java_tools_langtools_javac11//:javac_jar", + ], + outs = ["jars_java_tools.zip"], + cmd = "zip -qjX $@ $$(echo $(SRCS) | sort)", + visibility = ["//visibility:private"], +) -[ - genrule( - name = "jars_java_tools_java" + java_version + "_zip", - srcs = JAVA_TOOLS_DEPLOY_JARS + [ - "@java_tools_langtools_javac11//:jdk_compiler_jar", - "@java_tools_langtools_javac11//:java_compiler_jar", - "@java_tools_langtools_javac11//:javac_jar", - ], - outs = ["jars_java_tools_java" + java_version + ".zip"], - cmd = "zip -qjX $@ $$(echo $(SRCS) | sort)", - visibility = ["//visibility:private"], - ) - for java_version in JAVA_VERSIONS -] +genrule( + name = "java_tools_build_zip", + srcs = ["//tools/jdk:BUILD.java_tools"], + outs = ["java_tools_build.zip"], + cmd = "cat $(SRCS) > BUILD; zip -qjX $@ BUILD", +) -[ - genrule( - name = "java_tools_dist_java" + java_version, - srcs = [ - "//src/java_tools/buildjar:srcs", - "//src/java_tools/junitrunner:srcs", - "//src/java_tools/singlejar:srcs", - "//src/tools/singlejar:embedded_java_tools", - "//third_party/checker_framework_dataflow:srcs", - "//third_party/checker_framework_javacutil:srcs", - "//third_party/ijar:transitive_sources", - "//third_party/java/jacoco:transitive_sources", - "//third_party/java/proguard:srcs", - "@java_tools_langtools_javac11//:srcs", - ], - outs = ["java_tools_dist_javac" + java_version + ".zip"], - cmd = "zip -qXr $@ $$(echo $(SRCS) | sort)", - output_to_bindir = 1, - visibility = ["//src/test/shell/bazel:__pkg__"], - ) - for java_version in JAVA_VERSIONS -] +genrule( + name = "java_tools_no_build_zip", + srcs = [ + ":jars_java_tools.zip", + "//src/tools/singlejar:singlejar_transitive_zip", + "//third_party/ijar:ijar_transitive_srcs_zip", + "//third_party/java/jacoco:jacoco_jars_zip", + "//third_party/java/proguard:proguard_zip", + ], + outs = ["java_tools_no_build.zip"], + cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)", + output_to_bindir = 1, + tools = ["//src:merge_zip_files"], + visibility = ["//src/test/shell/bazel:__pkg__"], +) -[ - # Targets used by the java_tools_binaries Buildkite pipeline to build the - # java_tools_dist_java* zips and upload them to a tmp directory in GCS. - sh_binary( - name = "upload_java_tools_dist_java" + java_version, - srcs = ["upload_java_tools.sh"], - args = [ - "--java_tools_zip", - "src/java_tools_dist_javac" + java_version + ".zip", - "--gcs_java_tools_dir", - "tmp/sources", - "--java_version", - java_version, - "--platform", - ] + select({ - "//src/conditions:darwin": ["darwin"], - "//src/conditions:windows": ["windows"], - "//src/conditions:linux": ["linux"], - "//conditions:default": ["unknown"], - }), - data = [":java_tools_dist_java" + java_version], - deps = ["@bazel_tools//tools/bash/runfiles"], - ) - for java_version in JAVA_VERSIONS -] +genrule( + name = "java_tools_zip", + srcs = [ + "java_tools_no_build.zip", + "java_tools_build.zip", + ], + outs = ["java_tools.zip"], + cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)", + output_to_bindir = 1, + tools = ["//src:merge_zip_files"], + visibility = ["//src/test/shell/bazel:__pkg__"], +) -[ - # The java_tools releases can have BUILD files that vary depending on the - # javac version they embed. Currently the only difference is in the - # java_toolchain source version which has to be 14 for javac 14 to be able - # to build new Java 14 features. This is not used atm, as the toolchain for - # javac 14 was duplicated, but it might be used in future Bazel releases to - # support new javac release, so that we preserve this step for now. - [ - genrule( - name = "create_java_tools_build_java" + java_version, - srcs = ["//tools/jdk:BUILD.java_tools"], - outs = ["remote_java_tools_java" + java_version + "/BUILD"], - cmd = "sed 's/JAVA_LANGUAGE_LEVEL/" + java_version + "/g' $< > $@", - ), - genrule( - name = "create_java_tools_default_java" + java_version, - srcs = ["//tools/jdk:java_toolchain_default.bzl.java_tools"], - outs = ["remote_java_tools_java" + java_version + "/java_toolchain_default.bzl"], - cmd = "cp $< $@", - ), - ] - for java_version in JAVA_VERSIONS -] +# Following targets build java_tools_prebuilt.zip part of java_tools +# It is painstainkingly built by renaming single files, zipping them and merging zips together +# TODO(bazel-team): Replace this with a single target, for example "release_archive" in rules_kotlin +genrule( + name = "java_tools_prebuilt_build_zip", + srcs = ["//tools/jdk:BUILD.java_tools_prebuilt"], + outs = ["java_tools_prebuilt_build.zip"], + cmd = "cat $(SRCS) > BUILD; zip -qjX $@ BUILD", +) -[ - genrule( - name = "java_tools_java" + java_version + "_build_zip", - srcs = [ - "remote_java_tools_java" + java_version + "/BUILD", - "remote_java_tools_java" + java_version + "/java_toolchain_default.bzl", - ], - outs = ["java_tools_java_" + java_version + "_build.zip"], - cmd = "zip -qjX $@ $(SRCS)", - ) - for java_version in JAVA_VERSIONS -] +genrule( + name = "java_tools_prebuilt_no_build_zip", + srcs = [ + "//src/tools/singlejar:singlejar_deploy_zip", + "//third_party/ijar:ijar_deploy_zip", + ], + outs = ["java_tools_prebuilt_no_build.zip"], + cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)", + output_to_bindir = True, + tools = ["//src:merge_zip_files"], + visibility = ["//src/test/shell/bazel:__pkg__"], +) -# Builds the remote Java tools archive. Not embedded or used in Bazel, but used -# by the Java tools release process. -[ - genrule( - name = "java_tools_java" + java_version + "_no_build_zip", - srcs = [ - ":jars_java_tools_java" + java_version + ".zip", - "//src/tools/singlejar:singlejar_transitive_zip", - "//third_party/ijar:ijar_transitive_zip", - "//third_party/java/jacoco:jacoco_jars_zip", - "//third_party/java/proguard:proguard_zip", - ], - outs = ["java_tools_java" + java_version + "_no_build.zip"], - cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)", - output_to_bindir = 1, - tools = ["//src:merge_zip_files"], - visibility = ["//src/test/shell/bazel:__pkg__"], - ) - for java_version in JAVA_VERSIONS -] +genrule( + name = "java_tools_prebuilt_zip", + srcs = [ + "java_tools_prebuilt_no_build.zip", + "java_tools_prebuilt_build.zip", + ], + outs = ["java_tools_prebuilt.zip"], + cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)", + output_to_bindir = 1, + tools = ["//src:merge_zip_files"], + visibility = ["//src/test/shell/bazel:__pkg__"], +) -[ - genrule( - name = "java_tools_java" + java_version + "_zip", - srcs = [ - "java_tools_java" + java_version + "_no_build.zip", - "java_tools_java_" + java_version + "_build.zip", - ], - outs = ["java_tools_java" + java_version + ".zip"], - cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)", - output_to_bindir = 1, - tools = ["//src:merge_zip_files"], - visibility = ["//src/test/shell/bazel:__pkg__"], - ) - for java_version in JAVA_VERSIONS -] +# Following targets used by the java_tools_binaries Buildkite pipeline to upload +# the java_tools_*.zip to either tmp/sources or tmp/build directories in GCS. +sh_binary( + name = "upload_java_tools_dist", + srcs = ["upload_java_tools.sh"], + args = [ + "--java_tools_zip", + "src/java_tools_dist.zip", + "--gcs_java_tools_dir", + "tmp/sources", + ], + data = [":java_tools_dist"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) -[ - # Targets used by the java_tools_binaries Buildkite pipeline to build the - # java_tools_java* zips and upload them to a tmp directory in GCS. - sh_binary( - name = "upload_java_tools_java" + java_version, - srcs = ["upload_java_tools.sh"], - args = [ - "--gcs_java_tools_dir", - "tmp/build", - "--java_version", - java_version, - "--platform", - ] + select({ - "//src/conditions:darwin": ["darwin"], - "//src/conditions:windows": ["windows"], - "//src/conditions:linux": ["linux"], - "//conditions:default": ["unknown"], - }), - data = [":java_tools_java" + java_version + "_zip"], - deps = ["@bazel_tools//tools/bash/runfiles"], - ) - for java_version in JAVA_VERSIONS -] +sh_binary( + name = "upload_java_tools", + srcs = ["upload_java_tools.sh"], + args = [ + "--java_tools_zip", + "src/java_tools.zip", + "--gcs_java_tools_dir", + "tmp/build", + ], + data = [":java_tools_zip"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) + +sh_binary( + name = "upload_java_tools_prebuilt", + srcs = ["upload_java_tools.sh"], + args = [ + "--java_tools_zip", + "src/java_tools_prebuilt.zip", + "--gcs_java_tools_dir", + "tmp/build", + "--platform", + ] + select({ + "//src/conditions:darwin": ["darwin"], + "//src/conditions:windows": ["windows"], + "//src/conditions:linux": ["linux"], + "//conditions:default": ["unknown"], + }), + data = [":java_tools_prebuilt_zip"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) # Part of the Java tools remote archive. Not embedded or used in Bazel. genrule( diff --git a/src/create_java_tools_release.sh b/src/create_java_tools_release.sh index eb160866c4d979..4f3bbb69f3629a 100755 --- a/src/create_java_tools_release.sh +++ b/src/create_java_tools_release.sh @@ -22,8 +22,6 @@ # The script is using gsutil to copy artifacts. # # Mandatory flags: -# --java_version The JDK version included in the java_tools to be -# released. # --java_tools_version The version number of the java_tools to be released. # --rc The release candidate number of current release. # If --release true then --rc is the number of the rc to @@ -42,12 +40,12 @@ # JDK11, that was built at commit_hash 123456: # # src/create_java_tools_release.sh --commit_hash 123456 \ -# --java_tools_version 2.1 --java_version 11 --rc 1 --release false +# --java_tools_version 2.1 --rc 1 --release false # # To release the release candidate created above: # # src/create_java_tools_release.sh \ -# --java_tools_version 2.1 --java_version 11 --rc 1 --release true +# --java_tools_version 2.1 --rc 1 --release true set -euo pipefail @@ -56,7 +54,6 @@ while [[ -n "$@" ]]; do arg="$1"; shift val="$1"; shift case "$arg" in - "--java_version") java_version="$val" ;; "--java_tools_version") java_tools_version="$val" ;; "--commit_hash") commit_hash="$val" ;; "--rc") rc="$val" ;; @@ -72,33 +69,21 @@ trap "rm -fr $tmp_dir" EXIT gcs_bucket="gs://bazel-mirror/bazel_java_tools" -for platform in linux windows darwin; do - rc_url="release_candidates/javac${java_version}/v${java_tools_version}/java_tools_javac${java_version}_${platform}-v${java_tools_version}-rc${rc}.zip" - rc_sources_url="release_candidates/javac${java_version}/v${java_tools_version}/sources/java_tools_javac${java_version}_${platform}-v${java_tools_version}-rc${rc}.zip" +for platform in "linux" "windows" "darwin"; do + rc_url="release_candidates/java/v${java_tools_version}/java_tools_${platform}-v${java_tools_version}-rc${rc}.zip" if [[ $release == "true" ]]; then - release_artifact="releases/javac${java_version}/v${java_tools_version}/java_tools_javac${java_version}_${platform}-v${java_tools_version}.zip" - release_sources_artifact="releases/javac${java_version}/v${java_tools_version}/sources/java_tools_javac${java_version}_${platform}-v${java_tools_version}.zip" + release_artifact="releases/java/v${java_tools_version}/java_tools_${platform}-v${java_tools_version}.zip" # Make release candidate the release artifact for the current platform. # Don't overwrite existing file. gsutil -q cp -n "${gcs_bucket}/${rc_url}" "${gcs_bucket}/${release_artifact}" - - # Copy the associated zip file that contains the sources of the release zip. - # Don't overwrite existing file. - gsutil -q cp -n "${gcs_bucket}/${rc_sources_url}" "${gcs_bucket}/${release_sources_artifact}" else - tmp_url=$(gsutil ls -lh ${gcs_bucket}/tmp/build/${commit_hash}/java${java_version}/java_tools_javac${java_version}_${platform}* | sort -k 2 | grep gs -m 1 | awk '{print $4}') - + tmp_url=$(gsutil ls -lh ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools_${platform}* | sort -k 2 | grep gs -m 1 | awk '{print $4}') # Make the generated artifact a release candidate for the current platform. # Don't overwrite existing file. gsutil -q cp -n ${tmp_url} "${gcs_bucket}/${rc_url}" release_artifact="${rc_url}" - - # Copy the associated zip file that contains the sources of the release zip. - # Don't overwrite existing file. - tmp_sources_url=$(gsutil ls -lh ${gcs_bucket}/tmp/sources/${commit_hash}/java${java_version}/java_tools_javac${java_version}_${platform}* | sort -k 2 | grep gs -m 1 | awk '{print $4}') - gsutil -q cp -n ${tmp_sources_url} ${gcs_bucket}/${rc_sources_url} fi # Download the file locally to compute its sha256sum (needed to update the @@ -109,3 +94,37 @@ for platform in linux windows darwin; do file_hash=$(sha256sum ${local_zip} | cut -d' ' -f1) echo "${release_artifact} ${file_hash}" done + +rc_url="release_candidates/java/v${java_tools_version}/java_tools-v${java_tools_version}-rc${rc}.zip" +rc_sources_url="release_candidates/java/v${java_tools_version}/sources/java_tools-v${java_tools_version}-rc${rc}.zip" + +if [[ $release == "true" ]]; then + release_artifact="releases/java/v${java_tools_version}/java_tools-v${java_tools_version}.zip" + release_sources_artifact="releases/java/v${java_tools_version}/sources/java_tools-v${java_tools_version}.zip" + # Make release candidate the release artifact for the current platform. + # Don't overwrite existing file. + gsutil -q cp -n "${gcs_bucket}/${rc_url}" "${gcs_bucket}/${release_artifact}" + + # Copy the associated zip file that contains the sources of the release zip. + # Don't overwrite existing file. + gsutil -q cp -n "${gcs_bucket}/${rc_sources_url}" "${gcs_bucket}/${release_sources_artifact}" +else + tmp_url=$(gsutil ls -lh ${gcs_bucket}/tmp/build/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $4}') + + gsutil -q cp -n ${tmp_url} "${gcs_bucket}/${rc_url}" + release_artifact="${rc_url}" + + # Copy the associated zip file that contains the sources of the release zip. + # Don't overwrite existing file. + tmp_sources_url=$(gsutil ls -lh ${gcs_bucket}/tmp/sources/${commit_hash}/java/java_tools-* | sort -k 2 | grep gs -m 1 | awk '{print $4}') + gsutil -q cp -n ${tmp_sources_url} ${gcs_bucket}/${rc_sources_url} +fi + +# Download the file locally to compute its sha256sum (needed to update the +# java_tools in Bazel). +# Don't overwrite existing file. +local_zip="$tmp_dir/java_tools.zip" +gsutil -q cp -n ${gcs_bucket}/${rc_url} ${local_zip} +file_hash=$(sha256sum ${local_zip} | cut -d' ' -f1) +echo "${release_artifact} ${file_hash}" + diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 27f69e91396fba..4cfd2e729c456b 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -183,9 +183,10 @@ sh_test( srcs = ["bazel_java14_test.sh"], args = [ # --java_toolchain and --host_java_toolchain values - "@local_java_tools//:toolchain_jdk_14", + "@bazel_tools//tools/jdk:toolchain_jdk_14", # java_tools zip to test - "src/java_tools_java11.zip", + "src/java_tools.zip", + "src/java_tools_prebuilt.zip", ] + select({ # --javabase and --host_javabase values "//src/conditions:darwin": ["@openjdk14_darwin_archive//:runtime"], @@ -194,11 +195,16 @@ sh_test( }), data = [ ":test-deps", - "//src:java_tools_java11_zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//src/test/shell/bazel/testdata:jdk_http_archives_filegroup", "@bazel_tools//tools/bash/runfiles", ], - tags = ["local"], + # TODO(ilist): reenable after java_tools release + tags = [ + "local", + "manual", + ], ) sh_test( @@ -206,9 +212,10 @@ sh_test( srcs = ["bazel_java15_test.sh"], args = [ # --java_toolchain and --host_java_toolchain values - "@local_java_tools//:toolchain_jdk_15", + "@bazel_tools//tools/jdk:toolchain_jdk_15", # java_tools zip to test - "src/java_tools_java11.zip", + "src/java_tools.zip", + "src/java_tools_prebuilt.zip", ] + select({ # --javabase and --host_javabase values "//src/conditions:darwin": ["@openjdk15_darwin_archive//:runtime"], @@ -217,11 +224,16 @@ sh_test( }), data = [ ":test-deps", - "//src:java_tools_java11_zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//src/test/shell/bazel/testdata:jdk_http_archives_filegroup", "@bazel_tools//tools/bash/runfiles", ], - tags = ["local"], + # TODO(ilist): reenable after java_tools release + tags = [ + "local", + "manual", + ], ) sh_test( @@ -233,6 +245,7 @@ sh_test( args = [ "@bazel_tools//tools/jdk:toolchain", "released", + "released", ], data = [ ":test-deps", @@ -252,9 +265,10 @@ JAVA_VERSIONS = ("11", "14", "15") srcs = ["bazel_java_test.sh"], args = [ # --java_toolchain - "@local_java_tools//:toolchain", - # java_tools zip to test - "src/java_tools_java11.zip", + "@bazel_tools//tools/jdk:toolchain", + # java_tools zips to test + "src/java_tools.zip", + "src/java_tools_prebuilt.zip", # --javabase value ] + select({ "//src/conditions:darwin": ["@openjdk" + java_version + "_darwin_archive//:runtime"], @@ -263,11 +277,14 @@ JAVA_VERSIONS = ("11", "14", "15") }), data = [ ":test-deps", - "//src:java_tools_java11_zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//src/test/shell/bazel/testdata:jdk_http_archives_filegroup", "@bazel_tools//tools/bash/runfiles", ], exec_compatible_with = ["//:highcpu_machine"], + # TODO(ilist): reenable after java_tools release + tags = ["manual"], ) for java_version in JAVA_VERSIONS ] @@ -280,9 +297,10 @@ JAVA_VERSIONS = ("11", "14", "15") srcs = ["bazel_java_test.sh"], args = [ # --java_toolchain - "@local_java_tools//:prebuilt_toolchain", - # java_tools zip to test - "src/java_tools_java11.zip", + "@bazel_tools//tools/jdk:prebuilt_toolchain", + # java_tools zips to test + "src/java_tools.zip", + "src/java_tools_prebuilt.zip", # --javabase value ] + select({ "//src/conditions:darwin": ["@openjdk" + java_version + "_darwin_archive//:runtime"], @@ -291,11 +309,14 @@ JAVA_VERSIONS = ("11", "14", "15") }), data = [ ":test-deps", - "//src:java_tools_java11_zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//src/test/shell/bazel/testdata:jdk_http_archives_filegroup", "@bazel_tools//tools/bash/runfiles", ], exec_compatible_with = ["//:highcpu_machine"], + # TODO(ilist): reenable after java_tools release + tags = ["manual"], ) for java_version in JAVA_VERSIONS ] @@ -308,9 +329,10 @@ JAVA_VERSIONS = ("11", "14", "15") srcs = ["bazel_java_test.sh"], args = [ # --java_toolchain - "@local_java_tools//:toolchain", - # java_tools zip to test + "@bazel_tools//tools/jdk:toolchain", + # java_tools zips to test "$(LOCAL_JAVA_TOOLS_ZIP_URL)", + "$(LOCAL_JAVA_TOOLS_PREBUILT_ZIP_URL)", # --javabase value ] + select({ "//src/conditions:darwin": ["@openjdk" + java_version + "_darwin_archive//:runtime"], @@ -341,25 +363,25 @@ sh_test( ) sh_test( - name = "bazel_java_tools_javac11_test", + name = "bazel_java_tools_test", size = "medium", srcs = ["bazel_java_tools_test.sh"], - args = ["java11"], data = [ ":test-deps", - "//src:java_tools_java11.zip", + "//src:java_tools.zip", + "//src:java_tools_prebuilt.zip", "@bazel_tools//tools/bash/runfiles", ], ) sh_test( - name = "bazel_java_tools_dist_javac11_test", + name = "bazel_java_tools_dist_test", size = "small", srcs = ["bazel_java_tools_dist_test.sh"], args = ["javac11"], data = [ ":test-deps", - "//src:java_tools_dist_javac11.zip", + "//src:java_tools_dist.zip", "@bazel_tools//tools/bash/runfiles", ], ) @@ -499,6 +521,7 @@ sh_test( "@bazel_tools//tools/jdk:toolchain", "released", "released", + "released", ], data = [ ":test-deps", @@ -514,13 +537,11 @@ sh_test( sh_test( name = "bazel_coverage_java_jdk" + java_version + "_toolchain_released_test", srcs = ["bazel_coverage_java_test.sh"], - args = select({ - "//src/conditions:darwin": ["@remote_java_tools_javac11_test_darwin//:toolchain"], - "//src/conditions:windows": ["@remote_java_tools_javac11_test_windows//:toolchain"], - "//src/conditions:linux": ["@remote_java_tools_javac11_test_linux//:toolchain"], - }) + [ + args = [ + "@bazel_tools//tools/jdk:toolchain", # java_tools zip to test "released", + "released", # coverage_report_generator to test "released", # --javabase value @@ -547,9 +568,10 @@ sh_test( srcs = ["bazel_coverage_java_test.sh"], args = [ # --java_toolchain - "@local_java_tools//:toolchain", - # java_tools zip to test - "src/java_tools_java11.zip", + "@bazel_tools//tools/jdk:toolchain", + # java_tools zips to test + "src/java_tools.zip", + "src/java_tools_prebuilt.zip", # coverage output generator to test "tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/coverage", # --javabase value @@ -560,11 +582,13 @@ sh_test( }), data = [ ":test-deps", - "//src:java_tools_java11_zip", + "//src:java_tools_prebuilt_zip", + "//src:java_tools_zip", "//src/test/shell/bazel/testdata:jdk_http_archives_filegroup", "//tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_repo", ], tags = [ + "manual", # TODO(ilist): reenable after java_tools release "no_windows", ], ) diff --git a/src/test/shell/bazel/bazel_coverage_java_test.sh b/src/test/shell/bazel/bazel_coverage_java_test.sh index deafebec367f18..8d16aa815231be 100755 --- a/src/test/shell/bazel/bazel_coverage_java_test.sh +++ b/src/test/shell/bazel/bazel_coverage_java_test.sh @@ -35,6 +35,16 @@ if [[ "${JAVA_TOOLS_ZIP}" != "released" ]]; then fi JAVA_TOOLS_ZIP_FILE_URL=${JAVA_TOOLS_ZIP_FILE_URL:-} +JAVA_TOOLS_PREBUILT_ZIP="$1"; shift +if [[ "${JAVA_TOOLS_PREBUILT_ZIP}" != "released" ]]; then + if [[ "${JAVA_TOOLS_PREBUILT_ZIP}" == file* ]]; then + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="${JAVA_TOOLS_PREBUILT_ZIP}" + else + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="file://$(rlocation io_bazel/$JAVA_TOOLS_PREBUILT_ZIP)" + fi +fi +JAVA_TOOLS_PREBUILT_ZIP_FILE_URL=${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL:-} + COVERAGE_GENERATOR_DIR="$1"; shift if [[ "${COVERAGE_GENERATOR_DIR}" != "released" ]]; then COVERAGE_GENERATOR_DIR="$(rlocation io_bazel/$COVERAGE_GENERATOR_DIR)" @@ -56,9 +66,21 @@ EOF if [[ ! -z "${JAVA_TOOLS_ZIP_FILE_URL}" ]]; then cat >>WORKSPACE <> WORKSPACE } diff --git a/src/test/shell/bazel/bazel_java15_test.sh b/src/test/shell/bazel/bazel_java15_test.sh index 124e40a2b130b0..fd09a33af105a2 100755 --- a/src/test/shell/bazel/bazel_java15_test.sh +++ b/src/test/shell/bazel/bazel_java15_test.sh @@ -56,6 +56,7 @@ fi JAVA_TOOLCHAIN="$1"; shift JAVA_TOOLS_ZIP="$1"; shift +JAVA_TOOLS_PREBUILT_ZIP="$1"; shift JAVA_RUNTIME="$1"; shift echo "JAVA_TOOLS_ZIP=$JAVA_TOOLS_ZIP" @@ -65,10 +66,13 @@ JAVA_TOOLS_RLOCATION=$(rlocation io_bazel/$JAVA_TOOLS_ZIP) if "$is_windows"; then JAVA_TOOLS_ZIP_FILE_URL="file:///${JAVA_TOOLS_RLOCATION}" + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="file:///$(rlocation io_bazel/$JAVA_TOOLS_PREBUILT_ZIP)" else JAVA_TOOLS_ZIP_FILE_URL="file://${JAVA_TOOLS_RLOCATION}" + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="file://$(rlocation io_bazel/$JAVA_TOOLS_PREBUILT_ZIP)" fi JAVA_TOOLS_ZIP_FILE_URL=${JAVA_TOOLS_ZIP_FILE_URL:-} +JAVA_TOOLS_PREBUILT_ZIP_FILE_URL=${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL:-} add_to_bazelrc "build --java_toolchain=${JAVA_TOOLCHAIN}" add_to_bazelrc "build --host_java_toolchain=${JAVA_TOOLCHAIN}" @@ -81,9 +85,21 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # java_tools versions only used to test Bazel with various JDK toolchains. http_archive( - name = "local_java_tools", + name = "remote_java_tools", urls = ["${JAVA_TOOLS_ZIP_FILE_URL}"] ) +http_archive( + name = "remote_java_tools_linux", + urls = ["${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL}"] +) +http_archive( + name = "remote_java_tools_windows", + urls = ["${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL}"] +) +http_archive( + name = "remote_java_tools_darwin", + urls = ["${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL}"] +) EOF cat $(rlocation io_bazel/src/test/shell/bazel/testdata/jdk_http_archives) >> WORKSPACE } diff --git a/src/test/shell/bazel/bazel_java_test.sh b/src/test/shell/bazel/bazel_java_test.sh index b544b21fd551cd..8eeb55d5c8a389 100755 --- a/src/test/shell/bazel/bazel_java_test.sh +++ b/src/test/shell/bazel/bazel_java_test.sh @@ -71,6 +71,18 @@ if [[ "${JAVA_TOOLS_ZIP}" != "released" ]]; then fi JAVA_TOOLS_ZIP_FILE_URL=${JAVA_TOOLS_ZIP_FILE_URL:-} +JAVA_TOOLS_PREBUILT_ZIP="$1"; shift +if [[ "${JAVA_TOOLS_PREBUILT_ZIP}" != "released" ]]; then + if [[ "${JAVA_TOOLS_PREBUILT_ZIP}" == file* ]]; then + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="${JAVA_TOOLS_PREBUILT_ZIP}" + elif "$is_windows"; then + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="file:///$(rlocation io_bazel/JAVA_TOOLS_PREBUILT_ZIP)" + else + JAVA_TOOLS_PREBUILT_ZIP_FILE_URL="file://$(rlocation io_bazel/JAVA_TOOLS_PREBUILT_ZIP)" + fi +fi +JAVA_TOOLS_PREBUILT_ZIP_FILE_URL=${JAVA_TOOLS_PREBUILT_ZIP_FILE_URL:-} + if [[ $# -gt 0 ]]; then JAVABASE_VALUE="$1"; shift add_to_bazelrc "build --javabase=${JAVABASE_VALUE}" @@ -88,9 +100,21 @@ EOF if [[ ! -z "${JAVA_TOOLS_ZIP_FILE_URL}" ]]; then cat >>WORKSPACE <& ${TEST_log} || fail "Path $path not found in java_tools_${JAVA_TOOLS_JAVA_VERSION}.zip" + (zipinfo -1 $(rlocation io_bazel/src/java_tools_dist.zip) \ + | grep -c "$path") >& ${TEST_log} || fail "Path $path not found in java_tools_dist.zip" } function test_java_tools_has_ijar() { diff --git a/src/test/shell/bazel/bazel_java_tools_test.sh b/src/test/shell/bazel/bazel_java_tools_test.sh index e554a58e42740f..db1c8aa44338c3 100755 --- a/src/test/shell/bazel/bazel_java_tools_test.sh +++ b/src/test/shell/bazel/bazel_java_tools_test.sh @@ -17,8 +17,6 @@ # Load the test setup defined in the parent directory set -euo pipefail -JAVA_TOOLS_JAVA_VERSION="$1"; shift - # --- begin runfiles.bash initialization --- if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then if [[ -f "$0.runfiles_manifest" ]]; then @@ -66,33 +64,51 @@ if "$is_windows"; then fi function set_up() { - local java_tools_rlocation=$(rlocation io_bazel/src/java_tools_${JAVA_TOOLS_JAVA_VERSION}.zip) + local java_tools_rlocation=$(rlocation io_bazel/src/java_tools.zip) local java_tools_zip_file_url="file://${java_tools_rlocation}" if "$is_windows"; then java_tools_zip_file_url="file:///${java_tools_rlocation}" fi + local java_tools_prebuilt_rlocation=$(rlocation io_bazel/src/java_tools_prebuilt.zip) + local java_tools_prebuilt_zip_file_url="file://${java_tools_prebuilt_rlocation}" + if "$is_windows"; then + java_tools_prebuilt_zip_file_url="file:///${java_tools_prebuilt_rlocation}" + fi cat > WORKSPACE < BUILD < BUILD < BUILD < BUILD < BUILD < BUILD < BUILD <= 0] != []: - fail("Relative labels are not supported in jvm_opts parameter.") - jvm_opts = [opt.replace(" //:", " @{repo}//:").format(repo = repo) for opt in jvm_opts] - return dict(toolchain_args, jvm_opts = jvm_opts) - -def java_toolchain_default(name, configuration = dict(), **kwargs): - """Defines a java_toolchain with appropriate defaults for Bazel.""" - - toolchain_args = dict(_BASE_TOOLCHAIN_CONFIGURATION) - toolchain_args.update(configuration) - toolchain_args.update(kwargs) - toolchain_args = {k: _to_label(k, v) for k, v in toolchain_args.items()} - toolchain_args = _format_jvm_opts(toolchain_args, Label("//x").workspace_name) - native.java_toolchain( - name = name, - **toolchain_args - )