From 7be81c006cc8dd5833b8772bfb5fc3b3eb845aaf Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Thu, 9 Feb 2023 14:27:08 -0600 Subject: [PATCH] Add caching and log artifacts to GitHub actions Signed-off-by: Michael Carroll --- .github/actions/bazel-ci-jammy/Dockerfile | 10 + .github/actions/bazel-ci-jammy/entrypoint.sh | 2 +- .github/actions/bazel-ci-jammy/run.sh | 71 ++-- .github/workflows/ci.yml | 29 +- workspace/pkg_config.BUILD.tpl | 21 - workspace/pkg_config.bzl | 388 ------------------- 6 files changed, 75 insertions(+), 446 deletions(-) delete mode 100644 workspace/pkg_config.BUILD.tpl delete mode 100644 workspace/pkg_config.bzl diff --git a/.github/actions/bazel-ci-jammy/Dockerfile b/.github/actions/bazel-ci-jammy/Dockerfile index 25ba350..12b5e40 100644 --- a/.github/actions/bazel-ci-jammy/Dockerfile +++ b/.github/actions/bazel-ci-jammy/Dockerfile @@ -1,6 +1,16 @@ FROM ghcr.io/gazebo-tooling/gz-ubuntu:garden-jammy +ARG UID=1000 + +# Add a runner user and group to match github actions +# Additionally, use passwordless sudo +RUN groupadd -g 121 runner \ + && useradd -mr -d /home/runner -u 1001 -g 121 runner \ + && usermod -aG sudo runner \ + && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers COPY ["run.sh", "/run.sh"] COPY ["entrypoint.sh", "/entrypoint.sh"] +USER runner + ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/bazel-ci-jammy/entrypoint.sh b/.github/actions/bazel-ci-jammy/entrypoint.sh index d43cecd..268fc7f 100755 --- a/.github/actions/bazel-ci-jammy/entrypoint.sh +++ b/.github/actions/bazel-ci-jammy/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -l -sudo bash /run.sh $@ +bash /run.sh $@ diff --git a/.github/actions/bazel-ci-jammy/run.sh b/.github/actions/bazel-ci-jammy/run.sh index 57f12ae..a1fcd77 100755 --- a/.github/actions/bazel-ci-jammy/run.sh +++ b/.github/actions/bazel-ci-jammy/run.sh @@ -4,61 +4,66 @@ set -x set -e BAZEL_ARGS=$1 +WORKSPACE=~/gz +BAZEL_CACHE=$GITHUB_WORKSPACE/bazel_cache +BAZEL=$BAZEL_CACHE/bazelisk-linux-amd64 -echo ::group::Install tools: apt -apt update 2>&1 -apt -y install \ - build-essential \ - cppcheck \ - curl \ - git \ - gnupg \ - lsb-release \ - python3-pip \ - wget - -cd "$GITHUB_WORKSPACE" SYSTEM_VERSION=`lsb_release -cs` SOURCE_DEPENDENCIES="`pwd`/.github/ci/dependencies.yaml" SOURCE_DEPENDENCIES_VERSIONED="`pwd`/.github/ci-$SYSTEM_VERSION/dependencies.yaml" -echo ::endgroup:: -echo ::group::Install tools: pip -pip3 install -U pip vcstool colcon-common-extensions -echo ::endgroup:: +cd "$GITHUB_WORKSPACE" -# Install bazelisk -wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 -mv ./bazelisk-linux-amd64 /usr/bin/bazel -chmod +x /usr/bin/bazel +# Restore cache +if [ ! -d "$BAZEL_CACHE" ]; then + mkdir -p ${BAZEL_CACHE} +fi +rm -rf ~/.cache +ln -sf ${BAZEL_CACHE} ~/.cache + +# Install baselisk if it is not already installed +if [ ! -f ${BAZEL} ]; then + wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 -O ${BAZEL} +fi +chmod +x ${BAZEL} # Import repos -mkdir -p /gz -cd /gz -cp -R /github/workspace /gz/bazel -vcs import . < /github/workspace/example/bazel.repos +mkdir -p ${WORKSPACE}/bazel +cd ${WORKSPACE} +shopt -s extglob +# Copy relevant bazel files into the build space +cp -R ${GITHUB_WORKSPACE}/!(gz|.git|.github|bazel_cache) ${WORKSPACE}/bazel +vcs import . < ${WORKSPACE}/bazel/example/bazel.repos echo ::group::Install dependencies from binaries EXCLUDE_APT="libignition|libgz|libsdformat|libogre|dart" UBUNTU_VERSION=`lsb_release -cs` ALL_PACKAGES=$( \ sort -u $(find . -iname 'packages-'$UBUNTU_VERSION'.apt' -o -iname 'packages.apt') | grep -Ev $EXCLUDE_APT | tr '\n' ' ') -apt-get install --no-install-recommends --quiet --yes $ALL_PACKAGES +sudo apt install --no-install-recommends --quiet --yes $ALL_PACKAGES echo ::endgroup:: -ln -sf /gz/bazel/example/WORKSPACE.example /gz/WORKSPACE -ln -sf /gz/bazel/example/BUILD.bazel.example /gz/BUILD.bazel -ln -sf /gz/bazel/example/bazelrc.example /gz/.bazelrc -ln -sf /gz/bazel/example/bazelproject.example /gz/.bazelproject +ln -sf ${WORKSPACE}/bazel/example/WORKSPACE.example ${WORKSPACE}/WORKSPACE +ln -sf ${WORKSPACE}/bazel/example/BUILD.bazel.example ${WORKSPACE}/BUILD.bazel +ln -sf ${WORKSPACE}/bazel/example/bazelrc.example ${WORKSPACE}/.bazelrc +ln -sf ${WORKSPACE}/bazel/example/bazelproject.example ${WORKSPACE}/.bazelproject echo ::group::Bazel query -bazel query $BAZEL_ARGS +${BAZEL} query $BAZEL_ARGS echo ::endgroup:: echo ::group::Bazel build -bazel build $BAZEL_ARGS +${BAZEL} build $BAZEL_ARGS echo ::endgroup:: echo ::group::Bazel test -bazel test $BAZEL_ARGS +${BAZEL} test $BAZEL_ARGS echo ::endgroup:: + +echo ::group::Bazel test +ls -la ${WORKSPACE} +ls -la ${WORKSPACE}/bazel-testlogs +ls -la ${BAZEL_CACHE} +echo ::endgroup:: + +cp -RL ${WORKSPACE}/bazel-testlogs ${GITHUB_WORKSPACE}/bazel-testlogs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 937ee33..210fa6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,13 +7,36 @@ jobs: runs-on: ubuntu-latest name: Ubuntu Jammy CI steps: + - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to GHCR - run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + - name: Cache bazel + uses: actions/cache@v3 + env: + cache-name: bazel-cache-3 + with: + path: | + ${{ github.workspace }}/bazel_cache + key: cache-bazel-${{ hashFiles('example/bazel.repos') }} + restore-keys: | + cache-bazel- - name: Compile and test uses: ./.github/actions/bazel-ci-jammy with: bazel-args: //... + + - name: 'Upload Test Logs' + uses: actions/upload-artifact@v3 + with: + name: bazel-testlogs + path: ${{ github.workspace }}/bazel-testlogs + retention-days: 5 diff --git a/workspace/pkg_config.BUILD.tpl b/workspace/pkg_config.BUILD.tpl deleted file mode 100644 index 03e6140..0000000 --- a/workspace/pkg_config.BUILD.tpl +++ /dev/null @@ -1,21 +0,0 @@ -# -*- python -*- - -# %{topcomment} - -licenses(%{licenses}) - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = %{name}, - srcs = %{srcs}, - hdrs = %{hdrs}, - copts = %{copts}, - defines = %{defines}, - includes = %{includes}, - linkopts = %{linkopts}, - deps = %{deps}, - deprecation = %{extra_deprecation}, -) - -%{build_epilog} diff --git a/workspace/pkg_config.bzl b/workspace/pkg_config.bzl deleted file mode 100644 index 845e3cc..0000000 --- a/workspace/pkg_config.bzl +++ /dev/null @@ -1,388 +0,0 @@ -# -*- python -*- - -load("@gz//bazel/workspace:execute.bzl", "path", "which") -load("@gz//bazel/workspace:os.bzl", "determine_os") - -_DEFAULT_TEMPLATE = Label("@gz//bazel/workspace:pkg_config.BUILD.tpl") - -_DEFAULT_STATIC = False - -def _run_pkg_config(repository_ctx, command_line, pkg_config_paths): - """Run command_line with PKG_CONFIG_PATH = pkg_config_paths and return its - tokenized output.""" - pkg_config_path = ":".join(pkg_config_paths) - result = repository_ctx.execute( - command_line, - environment = { - "PKG_CONFIG_PATH": pkg_config_path, - }, - ) - if result.return_code != 0: - return struct(error = "error {} from {}: {}{}".format( - result.return_code, - command_line, - result.stdout, - result.stderr, - )) - tokens = [x for x in result.stdout.strip().split(" ") if x] - return struct(tokens = tokens, error = None) - -def setup_pkg_config_repository(repository_ctx): - """This is the macro form of the pkg_config_repository() rule below. - Refer to that rule's API documentation for details. - - This flavor of this rule is intended to be called by other repository_rule - implementation functions. The pkg_config_repository flavor of this rule is - intended to be called directly from the WORKSPACE file, or from a macro - that was called by the WORKSPACE file. - """ - - # First locate pkg-config. - tool_path = which(repository_ctx, "pkg-config") - if not tool_path: - return struct(error = "Could not find pkg-config on PATH={}".format( - path(repository_ctx), - )) - args = [tool_path, repository_ctx.attr.modname] - - # Grab any extra paths requested by the user. - pkg_config_paths = list(getattr( - repository_ctx.attr, - "pkg_config_paths", - [], - )) - - os_result = determine_os(repository_ctx) - - if os_result.is_macos or os_result.is_macos_wheel: - # Find the desired homebrew search path, if any. - homebrew_prefix = os_result.homebrew_prefix - homebrew_subdir = getattr( - repository_ctx.attr, - "homebrew_subdir", - "", - ) - if homebrew_prefix and homebrew_subdir: - pkg_config_paths.insert(0, "{}/{}".format( - homebrew_prefix, - homebrew_subdir, - )) - - if os_result.is_manylinux or os_result.is_macos_wheel: - pkg_config_paths.insert(0, "/opt/drake-dependencies/share/pkgconfig") - pkg_config_paths.insert(0, "/opt/drake-dependencies/lib/pkgconfig") - - # Check if we can find the required *.pc file of any version. - result = _run_pkg_config(repository_ctx, args, pkg_config_paths) - if result.error != None: - return result - - # If we have a minimum version, enforce that. - atleast_version = getattr(repository_ctx.attr, "atleast_version", "") - if atleast_version: - result = _run_pkg_config(repository_ctx, args + [ - "--atleast-version", - atleast_version, - ], pkg_config_paths) - if result.error != None: - return struct(error = result.error + "during version check") - - # Determine linkopts. - static = getattr(repository_ctx.attr, "static", _DEFAULT_STATIC) - libs_args = args + ["--libs"] - if static: - libs_args = libs_args + ["--static"] - result = _run_pkg_config(repository_ctx, libs_args, pkg_config_paths) - if result.error != None: - return result - linkopts = result.tokens - - # Bazel "linkopts=" must be either switches ("-foo"), variables ("$(FOO)"), - # or labels ("foo"). We should only get switches from `pkg-config --libs`. - # However, sometimes it produces "-framework CoreFoundation" or similar, - # which is *supposed* to be a single switch, but our split heuristic - # chopped it up. We recombine non-switch args with their preceding arg as - # a repair. We process args in reserve order to keep our loop index - # unchanged by a pop. - for i in reversed(range(len(linkopts))): - linkopt = linkopts[i] - - # Absolute system paths to *.dylib and *.so files get turned into -l - # instead. - if linkopt.endswith(".dylib") or linkopt.endswith(".so"): - if linkopt.endswith(".dylib"): - possible_libdirs = [ - "/usr/lib", - "/usr/local/lib", - ] - suffix = ".dylib" - elif linkopt.endswith(".so"): - possible_libdirs = [ - "/usr/lib", - "/usr/lib/x86_64-linux-gnu", - ] - suffix = ".so" - else: - return struct(error = ("expected linkopt {} to end with " + - ".dylib or .so").format(linkopt)) - for dir in possible_libdirs: - prefix = dir + "/lib" - if linkopt.startswith(prefix): - name = linkopt[len(prefix):-len(suffix)] - if "/" not in name: - linkopt = "-l" + name - linkopts[i] = linkopt - break - - # Add `-Wl,-rpath,` for `-L`. - # See https://github.com/RobotLocomotion/drake/issues/7387#issuecomment-359952616 # noqa - if linkopt.startswith("-L"): - linkopts.insert(i, "-Wl,-rpath," + linkopt[2:]) - continue - - # Switches stay put. - if linkopt.startswith("-"): - continue - - # A non-switch arg should be recombined with the preceding arg. - non_switch_arg = linkopts.pop(i) - if i == 0: - return struct(error = "malformed linkopts: " + repr(linkopts)) - linkopts[i - 1] += " " + non_switch_arg - - # Determine cflags; we'll split into includes and defines in a moment. - result = _run_pkg_config( - repository_ctx, - args + ["--cflags"], - pkg_config_paths, - ) - if result.error != None: - return result - cflags = result.tokens - - # Split cflags into includes and defines. The -I paths from pkg-config - # will be absolute paths; we'll make them relative in a moment. - absolute_includes = [] - defines = [] - unknown_cflags = [] - - # Blacklist various system include paths on macOS. - blacklisted_includes = [ - "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include", # noqa - "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.0.sdk/usr/include", # noqa - "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.0.sdk/usr/include", # noqa - "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include", # noqa - "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include", - "/Library/Developer/CommandLineTools/SDKs/MacOSX11.0.sdk/usr/include", - "/Library/Developer/CommandLineTools/SDKs/MacOSX12.0.sdk/usr/include", - "/Library/Developer/CommandLineTools/usr/include", - ] - - # We process in reserve order to keep our loop index unchanged by a pop. - for cflag in cflags: - if cflag.startswith("-I"): - value = cflag[2:] - if value in blacklisted_includes: - continue - if value not in absolute_includes: - absolute_includes.append(value) - elif cflag.startswith("-D"): - value = cflag[2:] - if value not in defines: - defines.append(value) - elif cflag == "-pthread": - # The pkg-config output has told us to use -pthread when compiling. - # When compiling the typical effect of -pthread is to -D_REENTRANT; - # when linking the typical effect of -pthread is to -lpthread. In - # Bazel, we can't pass -pthread in a cc_library's defines (it's not - # a preprocessor definition), and we shouldn't pass -pthread only - # in a cc_library's copts (i.e., non-transitively), since - # respecting transitivity might be important for some toolchains. - # Instead, when compiling our code that uses this library, we'll - # decide to just ignore pkg-config's advice to use -pthread when - # compiling and instead apply -pthread only when linking. - if "-pthread" not in linkopts: - linkopts.append("-pthread") - elif cflag in [ - "-frounding-math", - "-ffloat-store", - "-msse", - "-msse2", - "-msse3", - "-msse4", - "-msse4.1", - "-msse4.2", - "-mfpmath", - ]: - # We know these are okay to ignore. - pass - else: - unknown_cflags += [cflag] - if unknown_cflags: - print("pkg-config of {} returned flags that we will ignore: {}".format( - repository_ctx.attr.modname, - unknown_cflags, - )) - - # Symlink the absolute include paths into our repository, to obtain - # relative paths for them as required by cc_library's attributes. - includes = [] - hdrs_path = repository_ctx.path("include") - for item in absolute_includes: - if item == "/usr/include" or item == "/usr/local/include": - print(("pkg-config of {} returned an include path that " + - "contains {} that may contain unrelated headers").format( - repository_ctx.attr.modname, - item, - )) - symlink_dest = item.replace("/", "_") - repository_ctx.symlink( - repository_ctx.path(item), - hdrs_path.get_child(symlink_dest), - ) - includes += ["include/" + symlink_dest] - hdrs_prologue = "glob([\"include/**\"]) + " - - extra_deprecation = getattr( - repository_ctx.attr, - "extra_deprecation", - "", - ) - if extra_deprecation == "": - extra_deprecation = None - - # Write out the BUILD.bazel file. - substitutions = { - "%{topcomment}": "DO NOT EDIT: generated by pkg_config_repository()", - "%{licenses}": repr( - getattr(repository_ctx.attr, "licenses", []), - ), - "%{name}": repr( - repository_ctx.name, - ), - "%{srcs}": repr( - getattr(repository_ctx.attr, "extra_srcs", []), - ), - "%{hdrs}": ( - hdrs_prologue + repr( - getattr(repository_ctx.attr, "extra_hdrs", []), - ) - ), - "%{copts}": repr( - getattr(repository_ctx.attr, "extra_copts", []), - ), - "%{defines}": repr( - defines + getattr(repository_ctx.attr, "extra_defines", []), - ), - "%{includes}": repr( - includes + getattr(repository_ctx.attr, "extra_includes", []), - ), - "%{linkopts}": repr( - linkopts + getattr(repository_ctx.attr, "extra_linkopts", []), - ), - "%{deps}": repr( - getattr(repository_ctx.attr, "extra_deps", []), - ), - "%{build_epilog}": getattr(repository_ctx.attr, "build_epilog", ""), - "%{extra_deprecation}": repr(extra_deprecation), - } - template = getattr( - repository_ctx.attr, - "build_file_template", - _DEFAULT_TEMPLATE, - ) - repository_ctx.template("BUILD.bazel", template, substitutions) - - return struct(value = True, error = None) - -def _impl(repository_ctx): - result = setup_pkg_config_repository(repository_ctx) - if result.error != None: - fail("Unable to complete pkg-config setup for " + - "@{} repository: {}".format( - repository_ctx.name, - result.error, - )) - -pkg_config_repository = repository_rule( - # TODO(jamiesnape): Make licenses mandatory. - # TODO(jamiesnape): Use of this rule may cause additional transitive - # dependencies to be linked and their licenses must also be enumerated. - attrs = { - "licenses": attr.string_list(), - "modname": attr.string(mandatory = True), - "atleast_version": attr.string(), - "static": attr.bool(default = _DEFAULT_STATIC), - "build_file_template": attr.label( - default = _DEFAULT_TEMPLATE, - allow_files = True, - ), - "extra_srcs": attr.string_list(), - "extra_hdrs": attr.string_list(), - "extra_copts": attr.string_list(), - "extra_defines": attr.string_list(), - "extra_includes": attr.string_list(), - "extra_linkopts": attr.string_list(), - "extra_deps": attr.string_list(), - "build_epilog": attr.string(), - "pkg_config_paths": attr.string_list(), - "homebrew_subdir": attr.string(), - "extra_deprecation": attr.string(), - }, - local = True, - configure = True, - implementation = _impl, -) - -"""Creates a repository that contains a single library target, based on the -results of invoking pkg-config. - -The pkg_config_repository flavor of this rule is intended to be called directly -from the WORKSPACE file, or from a macro that was called by the WORKSPACE file. -The setup_pkg_config_repository flavor of this rule is intended to be called by -other repository_rule implementation functions. - -Example: - WORKSPACE: - load("@drake//bazel/workspace:pkg_config.bzl", "pkg_config_repository") - pkg_config_repository( - name = "foo", - modname = "foo-2.0", - ) - - BUILD: - cc_library( - name = "foobar", - deps = ["@foo"], - srcs = ["bar.cc"], - ) - -Args: - name: A unique name for this rule. - licenses: Licenses of the library. Valid license types include restricted, - reciprocal, notice, permissive, and unencumbered. See - https://docs.bazel.build/versions/master/be/functions.html#licenses_args - for more information. - modname: The library name as known to pkg-config. - atleast_version: (Optional) The --atleast-version to pkg-config. - static: (Optional) Add linkopts for static linking to the library target. - build_file_template: (Optional) (Advanced) Override the BUILD template. - extra_srcs: (Optional) Extra items to add to the library target. - extra_hdrs: (Optional) Extra items to add to the library target. - extra_copts: (Optional) Extra items to add to the library target. - extra_defines: (Optional) Extra items to add to the library target. - extra_includes: (Optional) Extra items to add to the library target. - extra_linkopts: (Optional) Extra items to add to the library target. - extra_deps: (Optional) Extra items to add to the library target. - build_epilog: (Optional) Extra text to add to the generated BUILD.bazel. - pkg_config_paths: (Optional) Paths to find pkg-config files (.pc). Note - that we ignore the environment variable PKG_CONFIG_PATH - set by the user. - homebrew_subdir: (Optional) If running on macOS, then this path under the - homebrew prefix will also be searched. For example, - homebrew_subdir = "opt/libpng/lib/pkgconfig" would search - "/usr/local/opt/libpng/lib/pkgconfig" or - "/opt/homebrew/opt/libpng/lib/pkgconfig". - extra_deprecation: (Optional) Add a deprecation message to the library - BUILD target. -"""