Skip to content

Commit

Permalink
Separate java_tools into platform independent and prebuilt part.
Browse files Browse the repository at this point in the history 
java_tools is repository package containing tools needed during Java compilation: JavaBuilder, patches for Java compiler, ijar, singlejar, ...
Most of the files are jars with Java classes. java_tools are released for three platforms: linux, windows and darwin, however the only difference is in two binaries: ijar and singlejar.

This is part one of splitting java_tools and releasing split version (following PR makes use of released split version in Bazel)

Java_tools used to be released for multiple Java versions, but all the releases were the same except a some string substitutions in BUILD file. I changed to build only a single version, since it already supports Java from 8 to 14.

Changes:
- BUILD.java_tools is split into BUILD.java_tools_prebuilt (where the second contains prebuilt binaries)
- toolchain definitions are removed from BUILD.java_tools and will be added to tools/jdk/BUILD in the second part
- java_toolchain_default.bzl.java_tools is removed (default_java_toolchain.bzl will be updated with its features in the second part).
- src/BUILD: JAVA_VERSION is removed, targets used to build java_tools.zip are duplicated to build java_tools_prebuilt.zip (done some cleanup as well)
- upload_all_java_tools.sh and upload_java_tools.sh: used by Build kite, I removed java_version over the release, but kept it over tests (for different JDKs)
- create_java_tools_release.sh: used by the user in the release process - added platform independent part
- tests are updated to use platform independent and platform files, some tests had to be disabled and will be reenabled after the release

Closes #12546.

PiperOrigin-RevId: 344319092
  • Loading branch information
@comius @copybara-github
comius authored and copybara-github committed Nov 25, 2020
1 parent 079bb7d commit d10013d
Show file tree
Hide file tree
Showing 16 changed files with 445 additions and 706 deletions.
316 changes: 157 additions & 159 deletions src/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,181 +552,179 @@ sh_binary(
visibility = ["//visibility:public"],
)

# The java_tools release process produces following zip files:
# - java_tools_dist.zip: all sources (Java and C++)
# - java_tools.zip: platform independent part - Java classes (deploy jars) and C++ sources
# - java_tools_prebuilt.zip: C++ binaries (for platforms: Linux, Darwin and Windows)
#
# The files are then used in Bazel as @remote_java_tools and @remote_java_tools_{platform}
# repositories.
# The zip files are not embedded or used in Bazel.

# Following target build java_tools_dist.zip - the sources
genrule(
name = "java_tools_dist",
srcs = [
"//src/java_tools/buildjar:srcs",
"//src/java_tools/junitrunner:srcs",
"//src/java_tools/singlejar:srcs",
"//src/tools/singlejar:embedded_java_tools",
"//third_party/checker_framework_dataflow:srcs",
"//third_party/checker_framework_javacutil:srcs",
"//third_party/ijar:transitive_sources",
"//third_party/java/jacoco:transitive_sources",
"//third_party/java/proguard:srcs",
"@java_tools_langtools_javac11//:srcs",
],
outs = ["java_tools_dist.zip"],
cmd = "zip -qXr $@ $$(echo $(SRCS) | sort)",
output_to_bindir = 1,
visibility = ["//src/test/shell/bazel:__pkg__"],
)

# Following targets build java_tools.zip - platform independent part of java_tools
# It is painstainkingly built by renaming single files, zipping them and merging zips together
# TODO(bazel-team): Replace this with a single target, for example "release_archive" in rules_kotlin
JAVA_TOOLS_DEPLOY_JARS = [
"//src/java_tools/buildjar:JavaBuilder_deploy.jar",
"//src/java_tools/buildjar:VanillaJavaBuilder_deploy.jar",
"//src/java_tools/buildjar/java/com/google/devtools/build/buildjar/genclass:GenClass_deploy.jar",
"//src/java_tools/buildjar/java/com/google/devtools/build/java/turbine:turbine_direct_binary_deploy.jar",
"//src/java_tools/junitrunner/java/com/google/testing/coverage:JacocoCoverage_jarjar_deploy.jar",
"//src/java_tools/junitrunner/java/com/google/testing/junit/runner:Runner_deploy.jar",
] + select({
"//src/conditions:arm": ["//src/java_tools/singlejar/java/com/google/devtools/build/singlejar:bazel-singlejar_deploy.jar"],
"//conditions:default": [],
})
"//src/java_tools/singlejar/java/com/google/devtools/build/singlejar:bazel-singlejar_deploy.jar",
]

JAVA_VERSIONS = ("11",)
genrule(
name = "jars_java_tools_zip",
srcs = JAVA_TOOLS_DEPLOY_JARS + [
"@java_tools_langtools_javac11//:jdk_compiler_jar",
"@java_tools_langtools_javac11//:java_compiler_jar",
"@java_tools_langtools_javac11//:javac_jar",
],
outs = ["jars_java_tools.zip"],
cmd = "zip -qjX $@ $$(echo $(SRCS) | sort)",
visibility = ["//visibility:private"],
)

[
genrule(
name = "jars_java_tools_java" + java_version + "_zip",
srcs = JAVA_TOOLS_DEPLOY_JARS + [
"@java_tools_langtools_javac11//:jdk_compiler_jar",
"@java_tools_langtools_javac11//:java_compiler_jar",
"@java_tools_langtools_javac11//:javac_jar",
],
outs = ["jars_java_tools_java" + java_version + ".zip"],
cmd = "zip -qjX $@ $$(echo $(SRCS) | sort)",
visibility = ["//visibility:private"],
)
for java_version in JAVA_VERSIONS
]
genrule(
name = "java_tools_build_zip",
srcs = ["//tools/jdk:BUILD.java_tools"],
outs = ["java_tools_build.zip"],
cmd = "cat $(SRCS) > BUILD; zip -qjX $@ BUILD",
)

[
genrule(
name = "java_tools_dist_java" + java_version,
srcs = [
"//src/java_tools/buildjar:srcs",
"//src/java_tools/junitrunner:srcs",
"//src/java_tools/singlejar:srcs",
"//src/tools/singlejar:embedded_java_tools",
"//third_party/checker_framework_dataflow:srcs",
"//third_party/checker_framework_javacutil:srcs",
"//third_party/ijar:transitive_sources",
"//third_party/java/jacoco:transitive_sources",
"//third_party/java/proguard:srcs",
"@java_tools_langtools_javac11//:srcs",
],
outs = ["java_tools_dist_javac" + java_version + ".zip"],
cmd = "zip -qXr $@ $$(echo $(SRCS) | sort)",
output_to_bindir = 1,
visibility = ["//src/test/shell/bazel:__pkg__"],
)
for java_version in JAVA_VERSIONS
]
genrule(
name = "java_tools_no_build_zip",
srcs = [
":jars_java_tools.zip",
"//src/tools/singlejar:singlejar_transitive_zip",
"//third_party/ijar:ijar_transitive_srcs_zip",
"//third_party/java/jacoco:jacoco_jars_zip",
"//third_party/java/proguard:proguard_zip",
],
outs = ["java_tools_no_build.zip"],
cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)",
output_to_bindir = 1,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)

[
# Targets used by the java_tools_binaries Buildkite pipeline to build the
# java_tools_dist_java* zips and upload them to a tmp directory in GCS.
sh_binary(
name = "upload_java_tools_dist_java" + java_version,
srcs = ["upload_java_tools.sh"],
args = [
"--java_tools_zip",
"src/java_tools_dist_javac" + java_version + ".zip",
"--gcs_java_tools_dir",
"tmp/sources",
"--java_version",
java_version,
"--platform",
] + select({
"//src/conditions:darwin": ["darwin"],
"//src/conditions:windows": ["windows"],
"//src/conditions:linux": ["linux"],
"//conditions:default": ["unknown"],
}),
data = [":java_tools_dist_java" + java_version],
deps = ["@bazel_tools//tools/bash/runfiles"],
)
for java_version in JAVA_VERSIONS
]
genrule(
name = "java_tools_zip",
srcs = [
"java_tools_no_build.zip",
"java_tools_build.zip",
],
outs = ["java_tools.zip"],
cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)",
output_to_bindir = 1,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)

[
# The java_tools releases can have BUILD files that vary depending on the
# javac version they embed. Currently the only difference is in the
# java_toolchain source version which has to be 14 for javac 14 to be able
# to build new Java 14 features. This is not used atm, as the toolchain for
# javac 14 was duplicated, but it might be used in future Bazel releases to
# support new javac release, so that we preserve this step for now.
[
genrule(
name = "create_java_tools_build_java" + java_version,
srcs = ["//tools/jdk:BUILD.java_tools"],
outs = ["remote_java_tools_java" + java_version + "/BUILD"],
cmd = "sed 's/JAVA_LANGUAGE_LEVEL/" + java_version + "/g' $< > $@",
),
genrule(
name = "create_java_tools_default_java" + java_version,
srcs = ["//tools/jdk:java_toolchain_default.bzl.java_tools"],
outs = ["remote_java_tools_java" + java_version + "/java_toolchain_default.bzl"],
cmd = "cp $< $@",
),
]
for java_version in JAVA_VERSIONS
]
# Following targets build java_tools_prebuilt.zip part of java_tools
# It is painstainkingly built by renaming single files, zipping them and merging zips together
# TODO(bazel-team): Replace this with a single target, for example "release_archive" in rules_kotlin
genrule(
name = "java_tools_prebuilt_build_zip",
srcs = ["//tools/jdk:BUILD.java_tools_prebuilt"],
outs = ["java_tools_prebuilt_build.zip"],
cmd = "cat $(SRCS) > BUILD; zip -qjX $@ BUILD",
)

[
genrule(
name = "java_tools_java" + java_version + "_build_zip",
srcs = [
"remote_java_tools_java" + java_version + "/BUILD",
"remote_java_tools_java" + java_version + "/java_toolchain_default.bzl",
],
outs = ["java_tools_java_" + java_version + "_build.zip"],
cmd = "zip -qjX $@ $(SRCS)",
)
for java_version in JAVA_VERSIONS
]
genrule(
name = "java_tools_prebuilt_no_build_zip",
srcs = [
"//src/tools/singlejar:singlejar_deploy_zip",
"//third_party/ijar:ijar_deploy_zip",
],
outs = ["java_tools_prebuilt_no_build.zip"],
cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)",
output_to_bindir = True,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)

# Builds the remote Java tools archive. Not embedded or used in Bazel, but used
# by the Java tools release process.
[
genrule(
name = "java_tools_java" + java_version + "_no_build_zip",
srcs = [
":jars_java_tools_java" + java_version + ".zip",
"//src/tools/singlejar:singlejar_transitive_zip",
"//third_party/ijar:ijar_transitive_zip",
"//third_party/java/jacoco:jacoco_jars_zip",
"//third_party/java/proguard:proguard_zip",
],
outs = ["java_tools_java" + java_version + "_no_build.zip"],
cmd = "$(location //src:merge_zip_files) java_tools $@ $(SRCS)",
output_to_bindir = 1,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)
for java_version in JAVA_VERSIONS
]
genrule(
name = "java_tools_prebuilt_zip",
srcs = [
"java_tools_prebuilt_no_build.zip",
"java_tools_prebuilt_build.zip",
],
outs = ["java_tools_prebuilt.zip"],
cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)",
output_to_bindir = 1,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)

[
genrule(
name = "java_tools_java" + java_version + "_zip",
srcs = [
"java_tools_java" + java_version + "_no_build.zip",
"java_tools_java_" + java_version + "_build.zip",
],
outs = ["java_tools_java" + java_version + ".zip"],
cmd = "$(location //src:merge_zip_files) - $@ $(SRCS)",
output_to_bindir = 1,
tools = ["//src:merge_zip_files"],
visibility = ["//src/test/shell/bazel:__pkg__"],
)
for java_version in JAVA_VERSIONS
]
# Following targets used by the java_tools_binaries Buildkite pipeline to upload
# the java_tools_*.zip to either tmp/sources or tmp/build directories in GCS.
sh_binary(
name = "upload_java_tools_dist",
srcs = ["upload_java_tools.sh"],
args = [
"--java_tools_zip",
"src/java_tools_dist.zip",
"--gcs_java_tools_dir",
"tmp/sources",
],
data = [":java_tools_dist"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)

[
# Targets used by the java_tools_binaries Buildkite pipeline to build the
# java_tools_java* zips and upload them to a tmp directory in GCS.
sh_binary(
name = "upload_java_tools_java" + java_version,
srcs = ["upload_java_tools.sh"],
args = [
"--gcs_java_tools_dir",
"tmp/build",
"--java_version",
java_version,
"--platform",
] + select({
"//src/conditions:darwin": ["darwin"],
"//src/conditions:windows": ["windows"],
"//src/conditions:linux": ["linux"],
"//conditions:default": ["unknown"],
}),
data = [":java_tools_java" + java_version + "_zip"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)
for java_version in JAVA_VERSIONS
]
sh_binary(
name = "upload_java_tools",
srcs = ["upload_java_tools.sh"],
args = [
"--java_tools_zip",
"src/java_tools.zip",
"--gcs_java_tools_dir",
"tmp/build",
],
data = [":java_tools_zip"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)

sh_binary(
name = "upload_java_tools_prebuilt",
srcs = ["upload_java_tools.sh"],
args = [
"--java_tools_zip",
"src/java_tools_prebuilt.zip",
"--gcs_java_tools_dir",
"tmp/build",
"--platform",
] + select({
"//src/conditions:darwin": ["darwin"],
"//src/conditions:windows": ["windows"],
"//src/conditions:linux": ["linux"],
"//conditions:default": ["unknown"],
}),
data = [":java_tools_prebuilt_zip"],
deps = ["@bazel_tools//tools/bash/runfiles"],
)

# Part of the Java tools remote archive. Not embedded or used in Bazel.
genrule(
Expand Down
Loading

0 comments on commit d10013d

Please sign in to comment.