external_data: Add stub of Bazel structure for isolated and meta-test…

…ing.

WORKSPACE

@@ -8,3 +8,9 @@ workspace(name = "drake")
 load("//tools/workspace:default.bzl", "add_default_repositories")
 
 add_default_repositories()
+
+# These are test repositories only needed for local testing of `external_data`,
+# and should not be needed for downstream projects.
+load("@drake//tools/external_data/test:external_data_workspace_test.bzl", "add_external_data_test_repositories") # noqa
+
+add_external_data_test_repositories(__workspace_dir__)

tools/external_data/BUILD.bazel

@@ -0,0 +1,12 @@
+load(":expose_all_files.bzl", "expose_all_files")
+
+# Bazel does not let us glob into `test/**`, even though the workspaces are not
+# a package of this workspace. The solution is to declare the lint tests
+# externally.
+expose_all_files(
+ sub_packages = ["bazel_external_data"],
+ # We do not care about `test` for direct consumption, so do not expose it.
+)
+
+# Linting is not done here to simplify test dependencies.
+# See :test/BUILD.bazel for how it is done.

tools/external_data/README.md

@@ -0,0 +1,33 @@
+# About
+
+This is a stub implementation of incorporating external data.
+More functionality to follow.
+
+# For Consumers of `external_data`
+
+The main file of interest is:
+
+* `external_data.bzl` - Macros for using external data.
+* `cli` - (To be added) Command-line interface.
+
+# For Developers of `external_data`
+
+This presently performs meta-testing with Bazel to ensure that we achieve the
+desired workflows with Bazel. This is all structured such that `bazel test ...`
+is valid from Drake, and from each test workspace under `test/`.
+
+Main structure:
+
+* `BUILD.bazel` - Effectively a no-op; only for exposing files.
+* `expose_all_files.bzl` - Macros to allow for (recursive) consumption of
+files from separate packages / workspaces.
+* `bazel_external_data/` - Python code for CLI interface (for both end users
+and Bazel `genrule`s).
+* `test/`
+ * `BUILD.bazel` - Declares tests (unlike other Drake packages), declares
+ linting for all of `//tools/external_data/...`.
+ * `external_data_workspace_tests.bzl` - Provides a list of workspaces to
+ be tested, repository declarations, convenience wrappings for linting
+ files, and macro for `external_data_workspace_test.sh`.
+ * `external_data_*_test` - Workspaces for testing downstream behavior.
+ * `external_data_pkg_test` - Stub for now.

tools/external_data/bazel_external_data/BUILD.bazel

@@ -0,0 +1,15 @@
+# -*- python -*-
+
+load("//tools/external_data:expose_all_files.bzl", "expose_all_files")
+
+# For macro testing.
+exports_files(["stub_test.py"])
+
+# Stub file pending Python code.
+py_test(
+ name = "stub_test",
+ srcs = ["stub_test.py"],
+ visibility = ["//visibility:public"],
+)
+
+expose_all_files()

tools/external_data/bazel_external_data/stub_test.py

@@ -0,0 +1,2 @@
+# Simple Python test file, to ensure we get linting.
+print("Stub test ran.")

tools/external_data/expose_all_files.bzl

@@ -0,0 +1,94 @@
+def _recursive_filegroup_impl(ctx):
+ files = depset()
+ for d in ctx.attr.data:
+ files += d.data_runfiles.files
+ return [DefaultInfo(
+ files = files,
+ data_runfiles = ctx.runfiles(
+ files = list(files),
+ ),
+ )]
+
+"""
+Provides all files (including `data` dependencies) at one level such that they
+are expandable via `$(locations ...)`.
+
+@param data
+ Upstream data targets. This will consume both the `srcs` and `data`
+ portions of an existing `filegroup`.
+"""
+
+recursive_filegroup = rule(
+ attrs = {
+ "data": attr.label_list(
+ cfg = "data",
+ allow_files = True,
+ mandatory = True,
+ ),
+ },
+ implementation = _recursive_filegroup_impl,
+)
+
+# Patterns to be exposed.
+_patterns_map = dict(
+ all_files = [
+ "*",
+ ],
+ # To minimize the number of dependencies need to consume `external_data`,
+ # this list is copied (not imported) from `//tools/lint:bazel_lint.bzl`.
+ bazel_lint_files = [
+ "*.bzl",
+ "*.BUILD",
+ "*.BUILD.bazel",
+ "BUILD",
+ "BUILD.bazel",
+ "WORKSPACE",
+ ],
+ python_lint_files = [
+ "*.py",
+ ],
+)
+
+def expose_all_files(
+ sub_packages = [],
+ sub_dirs = [],
+ visibility = ["//visibility:public"]):
+ """
+ Declares files to be consumed externally (for Bazel workspace tests,
+ linting, etc).
+ Creates rules "${type}_files" and "${type}_files_recursive", where `type`
+ will be all of {"all", "bazel_lint", "python_lint"}.
+
+ @param sub_packages
+ Child packages, only the first level.
+ @param sub_dirs
+ Any directories that are not packages.
+ """
+ # @note It'd be nice if this could respect *ignore files, but meh.
+ # Also, it'd be **super** nice if Bazel did not let `**` globs leak into
+ # other packages and then error out.
+ package_name = native.package_name()
+ if package_name:
+ package_prefix = "//" + package_name + "/"
+ else:
+ package_prefix = "//" # Root case.
+ for name, patterns in _patterns_map.items():
+ srcs = native.glob(patterns)
+ for sub_dir in sub_dirs:
+ srcs += native.glob([
+ sub_dir + "/" + pattern for pattern in patterns])
+ native.filegroup(
+ name = name,
+ srcs = srcs,
+ # Trying to use `data = deps` here only exposes the files in
+ # runfiles, but not for expansion via `$(locations...)`.
+ visibility = visibility,
+ )
+ # Expose all files recursively (from one level).
+ deps = [package_prefix + sub_package + ":" + name + "_recursive"
+ for sub_package in sub_packages]
+ recursive_filegroup(
+ name = name + "_recursive",
+ data = [name] + deps,
+ visibility = visibility,
+ )

tools/external_data/external_data.bzl

@@ -0,0 +1,10 @@
+# -*- python -*-
+
+def external_data_stub_test():
+ # Define stub test using upstream package's file.
+ file = "@drake//tools/external_data/bazel_external_data:stub_test.py"
+ native.py_test(
+ name = "stub_test",
+ srcs = [file],
+ main = file,
+ )

tools/external_data/test/BUILD.bazel

@@ -0,0 +1,29 @@
+load("//tools/lint:lint.bzl", "add_lint_tests")
+load(
+ ":external_data_workspace_test.bzl",
+ "collect_external_data_lint_files",
+ "external_data_workspace_test",
+)
+
+external_data_workspace_test(
+ name = "external_data_pkg_test",
+)
+
+exports_files(
+ srcs = ["workspace_test.sh"],
+)
+
+# To simplify testing `load` dependencies, linting is done in this package
+# only. This macro collects all lint files under `//tools/external_data/...`,
+# excluding the files in this package (which are implicitly added by
+# `add_lint_tests`).
+collect_external_data_lint_files()
+
+add_lint_tests(
+ bazel_lint_extra_srcs = [
+ ":all_bazel_lint_files",
+ ],
+ python_lint_extra_srcs = [
+ ":all_python_lint_files",
+ ],
+)

tools/external_data/test/external_data_pkg_test/BUILD.bazel

@@ -0,0 +1,11 @@
+# -*- python -*-
+
+load("@drake//tools/external_data:expose_all_files.bzl", "expose_all_files")
+load(
+ "@drake//tools/external_data:external_data.bzl",
+ "external_data_stub_test",
+)
+
+external_data_stub_test()
+
+expose_all_files()

tools/external_data/test/external_data_pkg_test/WORKSPACE

@@ -0,0 +1,8 @@
+workspace(name = "external_data_pkg_test")
+
+load("//:drake_path.bzl", "get_drake_path")
+
+local_repository(
+ name = "drake",
+ path = get_drake_path(__workspace_dir__),
+)

tools/external_data/test/external_data_pkg_test/drake_path.bzl

@@ -0,0 +1,12 @@
+# -*- python -*-
+
+def _dirname(p, remove = 1):
+ # Returns parent directory name for a path `p`.
+ pieces = p.split('/')
+ return '/'.join(pieces[0:-remove])
+
+def get_drake_path(workspace_dir):
+ """Returns path of Drake. This is replaced under
+ `external_data_workspace_test.sh`.
+ """
+ return _dirname(workspace_dir, 4)

tools/external_data/test/external_data_workspace_test.bzl

@@ -0,0 +1,80 @@
+load(
+ "//tools/external_data:expose_all_files.bzl",
+ "recursive_filegroup",
+)
+load(
+ "//tools/external_data/test:workspace_test.bzl",
+ "workspace_test",
+ "CMD_DEFAULT",
+)
+
+_workspace_list = [
+ "external_data_pkg_test",
+]
+
+_upstream_files = [
+ "//tools/external_data:all_files_recursive",
+]
+
+def external_data_workspace_test(
+ name,
+ args = [CMD_DEFAULT],
+ data = []):
+ package = "@{}//".format(name)
+ package_files = package + ":all_files_recursive"
+ pkg_reldir = "external/" + name
+ script = "external_data_workspace_test.sh"
+ workspace_test(
+ name = name,
+ args = [
+ "$(location {})".format(script),
+ pkg_reldir,
+ ] + args,
+ data = [
+ package_files,
+ script,
+ ] + _upstream_files + data,
+ )
+
+def collect_external_data_lint_files():
+ """
+ Creates a recursive_filegroup "all_${type}_files", where `type` will be
+ all of {"bazel_lint", "python_lint"}, such that they can be consumed by
+ `add_lint_tests` as extra files.
+ """
+ packages = ["//tools/external_data"]
+ for workspace in _workspace_list:
+ package = "@" + workspace + "//"
+ # Prepare to expose all files recursively.
+ packages.append(package)
+ # Join files for linting, to permit $(locations ...) expansion directly
+ # on transitive file dependencies.
+ for name in ["bazel_lint_files", "python_lint_files"]:
+ data = [package + ":" + name + "_recursive" for package in packages]
+ recursive_filegroup(
+ name = "all_" + name,
+ data = data,
+ )
+
+def add_external_data_test_repositories(workspace_dir):
+ """
+ Adds test workspace directories as repositories so that their files can be
+ consumed and their tests can be ignored by `bazel test ...` from Drake.
+ """
+ # WARNING: Bazel also craps out here if `workspace_dir + path` is used
+ # rather than just `path`.
+ # N.B. This error is *stateful*. You will get different behavior depending
+ # on what has been built / run previously in Bazel. In one mode, the error
+ # will be:
+ # Encountered error while [...]
+ # /home/${USER}/.cache/bazel/_bazel_${USER}/${HASH}/external/external_data_pkg_test # noqa
+ # must be an existing directory
+ # In another mode, you will get Java errors:
+ # java.lang.IllegalArgumentException: PathFragment
+ # tools/external_data/workspace is not beneath
+ # /home/${USER}/${WORKSPACE_DIR}/tools/external_data/workspace
+ for workspace in _workspace_list:
+ native.local_repository(
+ name = workspace,
+ path = "tools/external_data/test/" + workspace,
+ )

tools/external_data/test/external_data_workspace_test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# Ensure this is only run with `TEST_TMPDIR` present (called from
+# `workspace_test.sh`).
+[[ -n "${TEST_TMPDIR}" ]]
+
+pkg_reldir="${1}"
+shift
+
+# Create mock drake/WORKSPACE file.
+! test -f ./WORKSPACE
+echo 'workspace(name = "drake")' > ./WORKSPACE
+# Record directory.
+drake_dir="${PWD}"
+
+# Change to the workspace directory.
+cd "${pkg_reldir}"
+# Ensure path to Drake is corrected.
+echo "def get_drake_path(_): return \"${drake_dir}\"" > ./drake_path.bzl
+# Get rid of Bazel symlinks if they already exist.
+rm bazel-* 2> /dev/null || :
+
+# Run command.
+eval "$@"

tools/external_data/test/remove_bazel_symlinks.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eux -o pipefail
+
+# @file
+# Removes `bazel-*` symlinks (which mess up Bazel's own package scanning)
+# underneath `//tools/external_data/test/...`.
+# `bazel test ...` does not like having workspace sym-links, as it tries to
+# analyze them (tripping itself up) rather than ignore them.
+
+cd $(dirname "${0}")
+find . -type l -name 'bazel-*' | xargs rm || :

tools/external_data/test/workspace_test.bzl

@@ -0,0 +1,22 @@
+# -*- python -*-
+
+# Include release options:
+# https://docs.bazel.build/versions/master/user-manual.html#bazel-releng
+# N.B. We do not need quotes because these arguments will be passed to
+# `eval "$@"`.
+CMD_DEFAULT = "bazel --bazelrc=/dev/null --batch test //..."
+
+def workspace_test(
+ name,
+ args = [CMD_DEFAULT],
+ data = []):
+ """Copies all contents under `*.runfiles/${workspace}/**` to a temporary
+ directory, then evalutates `args` in `bash` in the new temporary runfiles
+ workspace directory.
+ """
+ native.sh_test(
+ name = name,
+ srcs = ["@drake//tools/external_data/test:workspace_test.sh"],
+ args = args,
+ data = data,
+ )