From 3f58250088d60a303ba51b3288bf06703f619775 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 3 Aug 2023 09:59:32 +0200 Subject: [PATCH] draft of bazelified protoc build_artifacts --- tools/bazelify_tests/build_defs.bzl | 63 +++++++++++++++++++ .../grpc_build_artifact_task.sh | 38 +++++++++++ tools/bazelify_tests/test/BUILD | 8 ++- .../test/build_artifact_protoc.sh | 23 +++++++ 4 files changed, 131 insertions(+), 1 deletion(-) create mode 100755 tools/bazelify_tests/grpc_build_artifact_task.sh create mode 100755 tools/bazelify_tests/test/build_artifact_protoc.sh diff --git a/tools/bazelify_tests/build_defs.bzl b/tools/bazelify_tests/build_defs.bzl index ec5054e6ddf4b..7b75ae52d9a9a 100644 --- a/tools/bazelify_tests/build_defs.bzl +++ b/tools/bazelify_tests/build_defs.bzl @@ -101,3 +101,66 @@ def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout native.sh_test( **test_args ) + +def grpc_build_artifact_task(name, srcs = [], timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, build_script = None): + """Execute an run_tests.py-harness style test under bazel. + + Args: + name: The name of the test. + args: The args to supply to the test binary. + data: Data dependencies. + size: The size of the test. + timeout: The test timeout. + tags: The tags for the test. + exec_compatible_with: A list of constraint values that must be + satisifed for the platform. + exec_properties: A dictionary of strings that will be added to the + exec_properties of a platform selected for this target. + flaky: Whether this test is flaky. + """ + + out_archive_name = str(name + ".tar.gz") + + outs = [ + out_archive_name, + ] + + srcs = [ + "//tools/bazelify_tests:grpc_build_artifact_task.sh", + "//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz", + build_script, + ] + srcs + + cmd = "$(location //tools/bazelify_tests:grpc_build_artifact_task.sh) $(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz) $(location " + out_archive_name + ") $(location " + build_script + ") 1>&2" + + # TODO: figure out how to use .currentversion files as input for configuring attributes + # TODO: use rbe_exec_properties helpers + + exec_properties = { + "dockerNetwork": "standard", # look into deactivating network for some actions + "label:workload": "misc", # always use a dedicated "misc" pool for running bazelified tests + "label:machine_size": "misc_large", # needed to override the default value of "small". + } + if docker_image_version: + image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None) + if not image_spec: + fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version) + exec_properties["container-image"] = image_spec + + # TODO: check that test can only run under docker sandbox or remotely + + genrule_args = { + "name": name, + "cmd": cmd, + "srcs": srcs, + "tags": tags, + "flaky": flaky, + "timeout": timeout, + "exec_compatible_with": exec_compatible_with, + "exec_properties": exec_properties, + "outs": outs, + } + + native.genrule( + **genrule_args + ) diff --git a/tools/bazelify_tests/grpc_build_artifact_task.sh b/tools/bazelify_tests/grpc_build_artifact_task.sh new file mode 100755 index 0000000000000..0b97978621d4a --- /dev/null +++ b/tools/bazelify_tests/grpc_build_artifact_task.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +ARCHIVE_WITH_SUBMODULES="$1" +OUTFILE="$2" +BUILD_SCRIPT="$3" +shift 3 + +# Extract grpc repo archive +tar -xopf ${ARCHIVE_WITH_SUBMODULES} +cd grpc + +mkdir -p artifacts + +../"${BUILD_SCRIPT}" "$@" || FAILED="true" + +# TODO: deterministic tar +tar -czvf ../"${OUTFILE}" artifacts + +if [ "$FAILED" != "" ] +then + exit 1 +fi + diff --git a/tools/bazelify_tests/test/BUILD b/tools/bazelify_tests/test/BUILD index 5aedb1b20e667..b3269bc7546f3 100644 --- a/tools/bazelify_tests/test/BUILD +++ b/tools/bazelify_tests/test/BUILD @@ -13,7 +13,7 @@ # limitations under the License. load("//bazel:grpc_build_system.bzl", "grpc_package") -load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") +load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test", "grpc_build_artifact_task") load(":portability_tests.bzl", "generate_run_tests_portability_tests") licenses(["notice"]) @@ -22,6 +22,12 @@ grpc_package(name = "tools/bazelify_tests/test") generate_run_tests_portability_tests(name = "portability_tests_linux") +grpc_build_artifact_task( + name = "artifact_protoc_linux_x64", + build_script = "build_artifact_protoc.sh", + docker_image_version = "tools/dockerfile/grpc_artifact_centos6_x64.current_version", +) + # C/C++ grpc_run_tests_py_test( name = "runtests_c_linux_dbg", diff --git a/tools/bazelify_tests/test/build_artifact_protoc.sh b/tools/bazelify_tests/test/build_artifact_protoc.sh new file mode 100755 index 0000000000000..5b9d874a08c7c --- /dev/null +++ b/tools/bazelify_tests/test/build_artifact_protoc.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +export LDFLAGS="${LDFLAGS} -static-libgcc -static-libstdc++ -s" + +mkdir -p artifacts + +ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_protoc.sh +