ci: update pebble metamorphic nightly to use bazel

DEPS.bzl

@@ -1216,8 +1216,13 @@ def go_deps():
  )
  go_repository(
  name = "com_github_cockroachdb_pebble",
+ build_directives = ["gazelle:build_tags invariants"],
  build_file_proto_mode = "disable_global",
  importpath = "github.com/cockroachdb/pebble",
+ patch_args = ["-p1"],
+ patches = [
+ "@cockroach//build/patches:com_github_cockroachdb_pebble.patch",
+ ],
  sha256 = "0018bcef357bf7bba06d5e3eb35277709b5fd98ee437924001531fa935d8c76d",
  strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20220126162719-a5c1766b568a",
  urls = [

build/bazelutil/bazel-generate.sh

@@ -42,7 +42,7 @@ files_unchanged_from_upstream () {
 # be blocked by the existence of a file before the bazel command is
 # invoked. For now, this is left as an exercise for the user.
 
-if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go'); then
+if files_unchanged_from_upstream go.mod go.sum DEPS.bzl $(find ./pkg/cmd/mirror -name BUILD.bazel -or -name '*.go') $(find ./pkg/cmd/generate-staticcheck -name BUILD.bazel -or -name '*.go') $(find ./build/patches -name '*.patch'); then
  echo "Skipping //pkg/cmd/mirror (relevant files are unchanged from upstream)."
  echo "Skipping //pkg/cmd/generate-staticcheck (relevant files are unchanged from upstream)."
 else

build/patches/README.md

@@ -10,6 +10,10 @@ Generally speaking, the patches here "fix" `BUILD` files, which were either
 originally present in the repo, or which will be generated by Gazelle. A couple
 examples:
 
+* `com_github_cockroachdb_pebble.patch` adds the file
+ `internal/invariants/off.go` to the `go_library` for the `invariants`
+ package -- otherwise, Gazelle omits it, causing builds to fail unless you pass
+ the `invariants` build tag.
 * `com_github_gogo_protobuf.patch` adds a rule, `gogo_proto`, so that we can
  import `gogoproto/gogo.proto` in our own `.proto` files, while not updating
  the existing `gogoproto` rule that uses the pre-generated `.pb.go` file.

build/patches/com_github_cockroachdb_pebble.patch

@@ -0,0 +1,11 @@
+diff -urN a/internal/invariants/BUILD.bazel b/internal/invariants/BUILD.bazel
+--- a/internal/invariants/BUILD.bazel 1969-12-31 19:00:00.000000000 -0500
++++ b/internal/invariants/BUILD.bazel 2000-01-01 00:00:00.000000000 -0000
+@@ -5,6 +5,7 @@
+ srcs = [
+ "finalizer_off.go",
+ "finalizer_on.go",
++ "off.go",
+ "on.go",
+ "race_off.go",
+ "race_on.go",

build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# This script is run by the Pebble Nightly Metamorphic - TeamCity build
+# configuration.
+
+set -euo pipefail
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+
+source "$dir/teamcity-support.sh" # For $root
+source "$dir/teamcity-bazel-support.sh" # For run_bazel
+
+mkdir -p artifacts
+
+# Pull in the latest version of Pebble from upstream. The benchmarks run
+# against the tip of the 'master' branch. We do this by `go get`ting the
+# latest version of the module, and then running `mirror` to update `DEPS.bzl`
+# accordingly.
+bazel run @go_sdk//:bin/go get github.com/cockroachdb/pebble@latest
+NEW_DEPS_BZL_CONTENT=$(bazel run //pkg/cmd/mirror)
+echo "$NEW_DEPS_BZL_CONTENT" > DEPS.bzl
+
+PEBBLE_SUM=$(grep 'version =' DEPS.bzl | cut -d'"' -f2)
+echo "Pebble module sum: $PEBBLE_SUM"
+
+env=(
+ "GITHUB_REPO=$GITHUB_REPO"
+ "GITHUB_API_TOKEN=$GITHUB_API_TOKEN"
+ "BUILD_VCS_NUMBER=$PEBBLE_SUM"
+ "TC_BUILD_ID=$TC_BUILD_ID"
+ "TC_SERVER_URL=$TC_SERVER_URL"
+ "TC_BUILD_BRANCH=$TC_BUILD_BRANCH"
+ "PKG=internal/metamorphic"
+ "STRESSFLAGS=-maxtime 3h -maxfails 1 -stderr -p 1"
+ "TZ=America/New_York"
+)
+
+BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e BUILD_VCS_NUMBER=$PEBBLE_SUM -e GITHUB_API_TOKEN -e GITHUB_REPO -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL" \
+ run_bazel build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh
+

build/teamcity/cockroach/nightlies/pebble_nightly_metamorphic_impl.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh" # For process_test_json
+
+set -euxo pipefail
+ARTIFACTS_DIR=/artifacts/meta
+mkdir -p $ARTIFACTS_DIR
+GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
+
+BAZEL_BIN=$(bazel info bazel-bin --config ci)
+
+exit_status=0
+# NB: If adjusting the metamorphic test flags below, be sure to also update
+# pkg/cmd/github-post/main.go to ensure the GitHub issue poster includes the
+# correct flags in the reproduction command.
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci test \
+ @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test -- \
+ --test_timeout=11000 '--test_filter=TestMeta$' \
+ --define gotags=bazel,invariants \
+ "--test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE" \
+ --run_under "@com_github_cockroachdb_stress//:stress -bazel -shardable-artifacts 'GO_TEST_JSON_OUTPUT_FILE=cat,XML_OUTPUT_FILE=$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci merge-test-xmls' -maxtime 3h -maxfails 1 -stderr -p 1" \
+ --test_arg -dir --test_arg $ARTIFACTS_DIR \
+ --test_arg -ops --test_arg "uniform:5000-25000" \
+ || exit_status=$?
+
+process_test_json \
+ $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+ $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+ /artifacts $GO_TEST_JSON_OUTPUT_FILE $exit_status
+
+exit $exit_status

pkg/cmd/github-post/main.go

@@ -541,8 +541,9 @@ func formatPebbleMetamorphicIssue(
  s = strings.TrimSpace(s)
  s = strings.TrimSpace(s[:strings.Index(s, "\n")])
 
- repro = fmt.Sprintf("go test -mod=vendor -tags 'invariants' -exec 'stress -p 1' "+
- `-timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed %s -ops "uniform:5000-25000"`, s)
+ repro = fmt.Sprintf("dev test @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test "+
+ "-f TestMeta --stress --stress-args='-p 1' -- --define gotags=bazel,invariants --test_timeout=11000 "+
+ `--test_arg -seed --test_arg %s --test_arg -ops --test_arg "uniform:5000-25000"`, s)
  }
  }
  return issues.UnitTestFormatter, issues.PostRequest{

pkg/cmd/github-post/main_test.go

@@ -235,7 +235,7 @@ TestXXA - 1.00s
  testName: "TestMeta",
  title: "internal/metamorphic: TestMeta failed",
  message: "panic: induced panic",
- expRepro: `go test -mod=vendor -tags 'invariants' -exec 'stress -p 1' -timeout 0 -test.v -run TestMeta$ ./internal/metamorphic -seed 1600209371838097000 -ops "uniform:5000-25000"`,
+ expRepro: `dev test @com_github_cockroachdb_pebble//internal/metamorphic:metamorphic_test -f TestMeta --stress --stress-args='-p 1' -- --define gotags=bazel,invariants --test_timeout=11000 --test_arg -seed --test_arg 1600209371838097000 --test_arg -ops --test_arg "uniform:5000-25000"`,
  },
  },
  formatter: formatPebbleMetamorphicIssue,

pkg/cmd/mirror/mirror.go

@@ -386,6 +386,13 @@ func buildFileProtoModeForRepo(repoName string) string {
  return "disable_global"
 }
 
+func dumpBuildDirectivesForRepo(repoName string) {
+ if repoName == "com_github_cockroachdb_pebble" {
+ fmt.Printf(` build_directives = ["gazelle:build_tags invariants"],
+`)
+ }
+}
+
 func dumpBuildNamingConventionArgsForRepo(repoName string) {
  if repoName == "com_github_envoyproxy_protoc_gen_validate" || repoName == "com_github_grpc_ecosystem_grpc_gateway" {
  fmt.Printf(" build_naming_convention = \"go_default_library\",\n")
@@ -448,9 +455,10 @@ def go_deps():
  }
  fmt.Printf(` go_repository(
  name = "%s",
- build_file_proto_mode = "%s",
-`, repoName, buildFileProtoModeForRepo(repoName))
-
+`, repoName)
+ dumpBuildDirectivesForRepo(repoName)
+ fmt.Printf(` build_file_proto_mode = "%s",
+`, buildFileProtoModeForRepo(repoName))
  dumpBuildNamingConventionArgsForRepo(repoName)
  expectedURL := formatURL(replaced.Path, replaced.Version)
  fmt.Printf(" importpath = \"%s\",\n", mod.Path)