From 34975f28b8fceedd45d8bd4d3fcdcd66c62c27b4 Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Wed, 26 Jan 2022 15:45:29 -0600 Subject: [PATCH] ci: update pebble metamorphic nightly to use bazel Closes #67335. Release note: None --- DEPS.bzl | 5 ++ build/bazelutil/bazel-generate.sh | 2 +- build/patches/README.md | 4 ++ .../com_github_cockroachdb_pebble.patch | 11 ++++ build/teamcity-nightly-pebble-metamorphic.sh | 58 ------------------- .../nightlies/pebble_nightly_metamorphic.sh | 40 +++++++++++++ .../pebble_nightly_metamorphic_impl.sh | 34 +++++++++++ pkg/cmd/github-post/main.go | 5 +- pkg/cmd/mirror/mirror.go | 14 ++++- 9 files changed, 109 insertions(+), 64 deletions(-) create mode 100644 build/patches/com_github_cockroachdb_pebble.patch delete mode 100755 build/teamcity-nightly-pebble-metamorphic.sh create mode 100755 build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh create mode 100755 build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh diff --git a/DEPS.bzl b/DEPS.bzl index 7d4875a6aadd..3984fa75f3c8 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1216,8 +1216,13 @@ def go_deps(): ) go_repository( name = "com_github_cockroachdb_pebble", + build_directives = ["gazelle:build_tags invariants"], build_file_proto_mode = "disable_global", importpath = "github.com/cockroachdb/pebble", + patch_args = ["-p1"], + patches = [ + "@cockroach//build/patches:com_github_cockroachdb_pebble.patch", + ], sha256 = "0018bcef357bf7bba06d5e3eb35277709b5fd98ee437924001531fa935d8c76d", strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220126162719-a5c1766b568a", urls = [ diff --git a/build/bazelutil/bazel-generate.sh b/build/bazelutil/bazel-generate.sh index feffee74b011..812c95b6bcdb 100755 --- a/build/bazelutil/bazel-generate.sh +++ b/build/bazelutil/bazel-generate.sh @@ -42,7 +42,7 @@ files_unchanged_from_upstream () { # be blocked by the existence of a file before the bazel command is # invoked. For now, this is left as an exercise for the user. -if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go'); then +if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/mirror -name BUILD.bazel -or -name '*.go') $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go') $(find ./build/patches -name '*.patch'); then echo "Skipping //pkg/cmd/mirror (relevant files are unchanged from upstream)." echo "Skipping //pkg/cmd/generate-staticcheck (relevant files are unchanged from upstream)." else diff --git a/build/patches/README.md b/build/patches/README.md index 0d79cbf12dd5..ac5bf3781547 100644 --- a/build/patches/README.md +++ b/build/patches/README.md @@ -10,6 +10,10 @@ Generally speaking, the patches here "fix" `BUILD` files, which were either originally present in the repo, or which will be generated by Gazelle. A couple examples: +* `com_github_cockroachdb_pebble.patch` adds the file + `internal/invariants/off.go` to the `go_library` for the `invariants` + package -- otherwise, Gazelle omits it, causing builds to fail unless you pass + the `invariants` build tag. * `com_github_gogo_protobuf.patch` adds a rule, `gogo_proto`, so that we can import `gogoproto/gogo.proto` in our own `.proto` files, while not updating the existing `gogoproto` rule that uses the pre-generated `.pb.go` file. diff --git a/build/patches/com_github_cockroachdb_pebble.patch b/build/patches/com_github_cockroachdb_pebble.patch new file mode 100644 index 000000000000..cfa82c80003e --- /dev/null +++ b/build/patches/com_github_cockroachdb_pebble.patch @@ -0,0 +1,11 @@ +diff -urN a/internal/invariants/BUILD.bazel b/internal/invariants/BUILD.bazel +--- a/internal/invariants/BUILD.bazel 1969-12-31 19:00:00.000000000 -0500 ++++ b/internal/invariants/BUILD.bazel 2000-01-01 00:00:00.000000000 -0000 +@@ -5,6 +5,7 @@ + srcs = [ + "finalizer_off.go", + "finalizer_on.go", ++ "off.go", + "on.go", + "race_off.go", + "race_on.go", diff --git a/build/teamcity-nightly-pebble-metamorphic.sh b/build/teamcity-nightly-pebble-metamorphic.sh deleted file mode 100755 index 3c88d6164d6e..000000000000 --- a/build/teamcity-nightly-pebble-metamorphic.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# -# This script is run by the Pebble Nightly Metamorphic - TeamCity build -# configuration. - -set -euo pipefail - -source "$(dirname "${0}")/teamcity-support.sh" - -run export BUILDER_HIDE_GOPATH_SRC=1 -run mkdir -p artifacts -definitely_ccache - -# Replace the vendored Pebble with the most current SHA, -# with test files too. -rm -fr vendor/github.com/cockroachdb/pebble -git clone https://github.com/cockroachdb/pebble vendor/github.com/cockroachdb/pebble - -PEBBLE_SHA=$(git -C vendor/github.com/cockroachdb/pebble rev-parse HEAD) -echo "Pebble SHA: $PEBBLE_SHA" - -env=( - "GITHUB_REPO=$GITHUB_REPO" - "GITHUB_API_TOKEN=$GITHUB_API_TOKEN" - "BUILD_VCS_NUMBER=$PEBBLE_SHA" - "TC_BUILD_ID=$TC_BUILD_ID" - "TC_SERVER_URL=$TC_SERVER_URL" - "TC_BUILD_BRANCH=$TC_BUILD_BRANCH" - "PKG=internal/metamorphic" - "STRESSFLAGS=-maxtime 3h -maxfails 1 -stderr -p 1" - "TZ=America/New_York" -) - -build/builder.sh env "${env[@]}" bash <<'EOF' -set -euxo pipefail -go install github.com/cockroachdb/stress -go install ./pkg/cmd/github-post -ARTIFACTS_DIR=`pwd`/artifacts/meta -mkdir -p $ARTIFACTS_DIR - -# We've set pipefail, so the exit status is going to come from stress if there -# are test failures. -# Use an `if` so that the `-e` option doesn't stop the script on error. -pushd ./vendor/github.com/cockroachdb/pebble/internal/metamorphic -if ! stdbuf -oL -eL \ - - # NB: If adjusting the metamorphic test flags below, be sure to also update - # pkg/cmd/github-post/main.go to ensure the GitHub issue poster includes the - # correct flags in the reproduction command. - go test -mod=vendor -exec "stress ${STRESSFLAGS}" -run 'TestMeta$$' \ - -timeout 0 -tags 'invariants' -test.v \ - -dir $ARTIFACTS_DIR -ops "uniform:5000-25000" 2>&1 | tee $ARTIFACTS_DIR/metamorphic.log; then - exit_status=${PIPESTATUS[0]} - go tool test2json -t -p "${PKG}" < $ARTIFACTS_DIR/metamorphic.log | github-post --formatter=pebble-metamorphic - exit $exit_status - fi -popd -EOF diff --git a/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh b/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh new file mode 100755 index 000000000000..a58fc6021991 --- /dev/null +++ b/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# +# This script is run by the Pebble Nightly Metamorphic - TeamCity build +# configuration. + +set -euo pipefail + +dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" + +source "$dir/teamcity-support.sh" # For $root +source "$dir/teamcity-bazel-support.sh" # For run_bazel + +mkdir -p artifacts + +# Pull in the latest version of Pebble from upstream. The benchmarks run +# against the tip of the 'master' branch. We do this by `go get`ting the +# latest version of the module, and then running `mirror` to update `DEPS.bzl` +# accordingly. +bazel run @go_sdk//:bin/go get github.com/cockroachdb/pebble@latest +NEW_DEPS_BZL_CONTENT=$(bazel run //pkg/cmd/mirror) +echo "$NEW_DEPS_BZL_CONTENT" > DEPS.bzl + +PEBBLE_SUM=$(grep 'version =' DEPS.bzl | cut -d'"' -f2) +echo "Pebble module sum: $PEBBLE_SUM" + +env=( + "GITHUB_REPO=$GITHUB_REPO" + "GITHUB_API_TOKEN=$GITHUB_API_TOKEN" + "BUILD_VCS_NUMBER=$PEBBLE_SUM" + "TC_BUILD_ID=$TC_BUILD_ID" + "TC_SERVER_URL=$TC_SERVER_URL" + "TC_BUILD_BRANCH=$TC_BUILD_BRANCH" + "PKG=internal/metamorphic" + "STRESSFLAGS=-maxtime 3h -maxfails 1 -stderr -p 1" + "TZ=America/New_York" +) + +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e BUILD_VCS_NUMBER=$PEBBLE_SUM -e GITHUB_API_TOKEN -e GITHUB_REPO -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL" \ + run_bazel build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh + diff --git a/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh b/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh new file mode 100755 index 000000000000..d09a050fd0c0 --- /dev/null +++ b/build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" +source "$dir/teamcity-bazel-support.sh" # For process_test_json + +set -euxo pipefail +ARTIFACTS_DIR=/artifacts/meta +mkdir -p $ARTIFACTS_DIR +GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt + +bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci + +BAZEL_BIN=$(bazel info bazel-bin --config ci) + +exit_status=0 +# NB: If adjusting the metamorphic test flags below, be sure to also update +# pkg/cmd/github-post/main.go to ensure the GitHub issue poster includes the +# correct flags in the reproduction command. +$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci test \ + @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test -- \ + --test_timeout=11000 '--test_filter=TestMeta$' \ + --define gotags=bazel,invariants \ + "--test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE" \ + --run_under "@com_github_cockroachdb_stress//:stress -bazel -shardable-artifacts 'GO_TEST_JSON_OUTPUT_FILE=cat,XML_OUTPUT_FILE=$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci merge-test-xmls' -maxtime 3h -maxfails 1 -stderr -p 1" \ + --test_arg -dir --test_arg $ARTIFACTS_DIR \ + --test_arg -ops --test_arg "uniform:5000-25000" \ + || exit_status=$? + +process_test_json \ + $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \ + $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \ + /artifacts $GO_TEST_JSON_OUTPUT_FILE $exit_status + +exit $exit_status diff --git a/pkg/cmd/github-post/main.go b/pkg/cmd/github-post/main.go index 2bc68962904f..ed2ff6087af9 100644 --- a/pkg/cmd/github-post/main.go +++ b/pkg/cmd/github-post/main.go @@ -541,8 +541,9 @@ func formatPebbleMetamorphicIssue( s = strings.TrimSpace(s) s = strings.TrimSpace(s[:strings.Index(s, "\n")]) - repro = fmt.Sprintf("go test -mod=vendor -tags 'invariants' -exec 'stress -p 1' "+ - `-timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed %s -ops "uniform:5000-25000"`, s) + repro = fmt.Sprintf("dev test @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test "+ + "-f TestMeta --stress --stress-args='-p 1' -- --define gotags=bazel,invariants --test_timeout=11000 "+ + `--test_arg -seed --test_arg %s --test_arg -ops --test_arg "uniform:5000-25000"`, s) } } return issues.UnitTestFormatter, issues.PostRequest{ diff --git a/pkg/cmd/mirror/mirror.go b/pkg/cmd/mirror/mirror.go index 4d4852c34118..44016db22992 100644 --- a/pkg/cmd/mirror/mirror.go +++ b/pkg/cmd/mirror/mirror.go @@ -386,6 +386,13 @@ func buildFileProtoModeForRepo(repoName string) string { return "disable_global" } +func dumpBuildDirectivesForRepo(repoName string) { + if repoName == "com_github_cockroachdb_pebble" { + fmt.Printf(` build_directives = ["gazelle:build_tags invariants"], +`) + } +} + func dumpBuildNamingConventionArgsForRepo(repoName string) { if repoName == "com_github_envoyproxy_protoc_gen_validate" || repoName == "com_github_grpc_ecosystem_grpc_gateway" { fmt.Printf(" build_naming_convention = \"go_default_library\",\n") @@ -448,9 +455,10 @@ def go_deps(): } fmt.Printf(` go_repository( name = "%s", - build_file_proto_mode = "%s", -`, repoName, buildFileProtoModeForRepo(repoName)) - +`, repoName) + dumpBuildDirectivesForRepo(repoName) + fmt.Printf(` build_file_proto_mode = "%s", +`, buildFileProtoModeForRepo(repoName)) dumpBuildNamingConventionArgsForRepo(repoName) expectedURL := formatURL(replaced.Path, replaced.Version) fmt.Printf(" importpath = \"%s\",\n", mod.Path)