Merge #60537 #60539 #60610

60537: bazel: generate contents of `pkg/BUILD.bazel` in `make bazel-generate` r=rickystewart a=rickystewart

bazel: generate contents of `pkg/BUILD.bazel` in `make bazel-generate`

Previously, the `ALL_TESTS` list was hardcoded, leaving a possibility
that it would fall out of sync with the rest of tree. Now we use `bazel
query` to enumerate all the tests in the repo generate the contents of
`pkg/BUILD.bazel` accordingly. Since this is part of `make
bazel-generate`, CI should ensure that this never falls out of sync
again.

A couple tweaks to the build scripts were necessary to make this work:

- To avoid duplication, I moved the `make bazel-generate` logic to
  its own script at `build/bazelutil/bazel-generate.sh`.
- Refactored the Bazel image tag to one script in
  `build/teamcity-bazel-support.sh` so now there's only one place to
  update when a new builder image is created.
- In `build/teamcity-check.sh`, we were using `builder.sh` to run
  `make bazel-generate` in CI, but `builder.sh` [is known to work poorly
  w/ Bazel](#59224).
  Instead I directly use an appropriate `docker run`. This fixes some
  build errors that would occur otherwise.
    
Release note: None

60539: sql: add SHOW CREATE ALL TABLES command r=rafiss a=RichardJCai

sql: add SHOW CREATE ALL TABLES command to dump a database's create statement

Release note (sql change): SHOW CREATE ALL TABLES is a command
that allows the user to get the statements (CREATE/ALTER) to recreate a
the current database. The command returns a flat log of the create statements
followed by the alter statements for adding the constraints.
The commands are ordered topologically such that dependencies appear before
it's references. Alter statements follow create statements to guarantee
that the objects are all added before adding constraints.
This command is added to replace old dump logic.

Example output: 
```
SHOW CREATE ALL TABLES
----
CREATE TABLE public.parent (
  x INT8 NULL,
  y INT8 NULL,
  z INT8 NULL,
  UNIQUE INDEX parent_x_y_z_key (x ASC, y ASC, z ASC),
  UNIQUE INDEX parent_x_key (x ASC),
  FAMILY f1 (x, y, z, rowid)
);
CREATE TABLE public.full_test (
  x INT8 NULL,
  y INT8 NULL,
  z INT8 NULL,
  UNIQUE INDEX full_test_x_key (x ASC),
  FAMILY f1 (x, y, z, rowid)
);
CREATE SEQUENCE public.s MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1;
CREATE VIEW public.vx ("?column?") AS SELECT 1;
ALTER TABLE public.full_test ADD CONSTRAINT fk_x_ref_parent FOREIGN KEY (x, y, z) REFERENCES public.parent(x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE;
ALTER TABLE public.full_test ADD CONSTRAINT test_fk FOREIGN KEY (x) REFERENCES public.parent(x) ON DELETE CASCADE;
-- Validate foreign key constraints. These can fail if there was unvalidated data during the SHOW CREATE ALL TABLES
ALTER TABLE public.full_test VALIDATE CONSTRAINT fk_x_ref_parent;
ALTER TABLE public.full_test VALIDATE CONSTRAINT test_fk;
```

Resolves #53488

60610: cloud: update orchestrator configurations for v20.2.5 r=jlinder a=asubiotto

Release note: None

Co-authored-by: Ricky Stewart <ricky@cockroachlabs.com>
Co-authored-by: richardjcai <caioftherichard@gmail.com>
Co-authored-by: Alfonso Subiotto Marques <alfonso@cockroachlabs.com>

.gitattributes

@@ -1,3 +1,4 @@
 *.pb.* -diff
 *.eg.go -diff
 DEPS.bzl -diff
+pkg/BUILD.bazel -diff

Makefile

@@ -1772,8 +1772,7 @@ fuzz: bin/fuzz
 # Short hand to re-generate all bazel BUILD files.
 bazel-generate: ## Generate all bazel BUILD files.
 	@echo 'Generating DEPS.bzl and BUILD files using gazelle'
-	@bazel run //:gazelle -- update-repos -from_file=go.mod -build_file_proto_mode=disable_global -to_macro=DEPS.bzl%go_deps
-	@bazel run //:gazelle
+	./build/bazelutil/bazel-generate.sh
 
 # No need to include all the dependency files if the user is just
 # requesting help or cleanup.

build/README.md

@@ -108,7 +108,7 @@ The `bazelbuilder` image is used exclusively for performing builds using Bazel.
     docker image tag $IMAGE_HASH cockroachdb/bazel:$TAG
     docker image push cockroachdb/bazel:$TAG
 ```
-- Then, update `build/teamcity-bazel.sh` with the new tag and commit all your changes.
+- Then, update `build/teamcity-bazel-support.sh` with the new tag and commit all your changes.
 - Ensure the "Github CI (Optional)" job passes on your PR before merging.
 
 #  Dependencies

build/bazelutil/bazel-generate.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+bazel run //:gazelle -- update-repos -from_file=go.mod -build_file_proto_mode=disable_global -to_macro=DEPS.bzl%go_deps
+bazel run //:gazelle
+bazel run //pkg/cmd/generate-test-suites --run_under="cd $PWD && " > pkg/BUILD.bazel

build/teamcity-bazel-support.sh

@@ -0,0 +1 @@
+BAZEL_IMAGE=cockroachdb/bazel:20210201-174432

build/teamcity-bazel.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source "$(dirname "${0}")/teamcity-support.sh"
+source "$(dirname "${0}")/teamcity-bazel-support.sh"  # For BAZEL_IMAGE
 
 tc_prepare
 
@@ -18,5 +19,5 @@ docker run -i ${tty-} --rm --init \
        --workdir="/go/src/github.com/cockroachdb/cockroach" \
        -v "$root:/go/src/github.com/cockroachdb/cockroach:ro" \
        -v "$TMPDIR:/artifacts" \
-       cockroachdb/bazel:20210201-174432 bazelbuild.sh
+       $BAZEL_IMAGE bazelbuild.sh
 tc_end_block "Run Bazel build"

build/teamcity-check.sh

@@ -7,6 +7,7 @@ require_justification=0
 set -euo pipefail
 
 source "$(dirname "${0}")/teamcity-support.sh"
+source "$(dirname "${0}")/teamcity-bazel-support.sh"  # For BAZEL_IMAGE
 
 function check_clean() {
   # The workspace is clean iff `git status --porcelain` produces no output. Any
@@ -43,7 +44,11 @@ check_clean "Run \`make generate\` to automatically regenerate these."
 run build/builder.sh make buildshort &> artifacts/buildshort.log || (cat artifacts/buildshort.log && false)
 rm artifacts/buildshort.log
 check_clean "Run \`make buildshort\` to automatically regenerate these."
-run build/builder.sh make bazel-generate &> artifacts/buildshort.log || (cat artifacts/buildshort.log && false)
+# NB: $root is set by teamcity-support.sh.
+run docker run -i ${tty-} --rm --init \
+       --workdir="/go/src/github.com/cockroachdb/cockroach" \
+       -v "$root:/go/src/github.com/cockroachdb/cockroach" \
+       $BAZEL_IMAGE build/bazelutil/bazel-generate.sh &> artifacts/buildshort.log || (cat artifacts/buildshort.log && false)
 rm artifacts/buildshort.log
 check_clean "Run \`make bazel-generate\` to automatically regenerate these."
 tc_end_block "Ensure generated code is up-to-date"

cloud/kubernetes/bring-your-own-certs/client.yaml

@@ -19,7 +19,7 @@ spec:
   serviceAccountName: cockroachdb
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v20.2.4
+    image: cockroachdb/cockroach:v20.2.5
     # Keep a pod open indefinitely so kubectl exec can be used to get a shell to it
     # and run cockroach client commands, such as cockroach sql, cockroach node status, etc.
     command:

cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml

@@ -152,7 +152,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v20.2.4
+    image: cockroachdb/cockroach:v20.2.5
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/cockroachdb-statefulset-secure.yaml

@@ -194,7 +194,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/cockroachdb-statefulset.yaml

@@ -97,7 +97,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/multiregion/client-secure.yaml

@@ -8,7 +8,7 @@ spec:
   serviceAccountName: cockroachdb
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v20.2.4
+    image: cockroachdb/cockroach:v20.2.5
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/multiregion/cluster-init-secure.yaml

@@ -10,7 +10,7 @@ spec:
       serviceAccountName: cockroachdb
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/multiregion/cockroachdb-statefulset-secure.yaml

@@ -166,7 +166,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/multiregion/eks/cockroachdb-statefulset-secure-eks.yaml

@@ -184,7 +184,7 @@ spec:
           name: cockroach-env
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/performance/cockroachdb-daemonset-insecure.yaml

@@ -81,7 +81,7 @@ spec:
       hostNetwork: true
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: If you configured taints to give CockroachDB exclusive access to nodes, feel free
         # to remove the requests and limits sections. If you didn't, you'll need to change these to

cloud/kubernetes/performance/cockroachdb-daemonset-secure.yaml

@@ -197,7 +197,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: If you configured taints to give CockroachDB exclusive access to nodes, feel free
         # to remove the requests and limits sections. If you didn't, you'll need to change these to

cloud/kubernetes/performance/cockroachdb-statefulset-insecure.yaml

@@ -140,7 +140,7 @@ spec:
       - name: cockroachdb
         # NOTE: Always use the most recent version of CockroachDB for the best
         # performance and reliability.
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/performance/cockroachdb-statefulset-secure.yaml

@@ -231,7 +231,7 @@ spec:
       - name: cockroachdb
         # NOTE: Always use the most recent version of CockroachDB for the best
         # performance and reliability.
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         # TODO: Change these to appropriate values for the hardware that you're running. You can see
         # the resources that can be allocated on each of your Kubernetes nodes by running:

cloud/kubernetes/v1.6/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v20.2.4
+    image: cockroachdb/cockroach:v20.2.5
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/v1.6/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/v1.6/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/v1.6/cockroachdb-statefulset-secure.yaml

@@ -177,7 +177,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.6/cockroachdb-statefulset.yaml

@@ -80,7 +80,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.7/client-secure.yaml

@@ -31,7 +31,7 @@ spec:
       mountPath: /cockroach-certs
   containers:
   - name: cockroachdb-client
-    image: cockroachdb/cockroach:v20.2.4
+    image: cockroachdb/cockroach:v20.2.5
     imagePullPolicy: IfNotPresent
     volumeMounts:
     - name: client-certs

cloud/kubernetes/v1.7/cluster-init-secure.yaml

@@ -33,7 +33,7 @@ spec:
           mountPath: /cockroach-certs
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: client-certs

cloud/kubernetes/v1.7/cluster-init.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
       - name: cluster-init
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         command:
           - "/cockroach/cockroach"

cloud/kubernetes/v1.7/cockroachdb-statefulset-secure.yaml

@@ -189,7 +189,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

cloud/kubernetes/v1.7/cockroachdb-statefulset.yaml

@@ -92,7 +92,7 @@ spec:
               topologyKey: kubernetes.io/hostname
       containers:
       - name: cockroachdb
-        image: cockroachdb/cockroach:v20.2.4
+        image: cockroachdb/cockroach:v20.2.5
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 26257

docs/generated/sql/bnf/show_create_stmt.bnf

@@ -1,2 +1,3 @@
 show_create_stmt ::=
 	'SHOW' 'CREATE' object_name
+	| 'SHOW' 'CREATE' 'ALL' 'TABLES'

docs/generated/sql/bnf/stmt_block.bnf

@@ -602,6 +602,7 @@ show_constraints_stmt ::=
 
 show_create_stmt ::=
 	'SHOW' 'CREATE' table_name
+	| 'SHOW' 'CREATE' 'ALL' 'TABLES'
 
 show_csettings_stmt ::=
 	'SHOW' 'CLUSTER' 'SETTING' var_name

pkg/BUILD.bazel

@@ -1,6 +1,6 @@
+# Code generated by generate-test-suites, DO NOT EDIT.
 # gazelle:proto_strip_import_prefix /pkg
 
-# TODO(ricky): Add a lint to make sure this list stays up-to-date.
 ALL_TESTS = [
     "//pkg/acceptance:acceptance_test",
     "//pkg/base:base_test",
@@ -27,6 +27,7 @@ ALL_TESTS = [
     "//pkg/ccl/storageccl:storageccl_test",
     "//pkg/ccl/streamingccl/streamclient:streamclient_test",
     "//pkg/ccl/streamingccl/streamingest:streamingest_test",
+    "//pkg/ccl/streamingccl/streamingutils:streamingutils_test",
     "//pkg/ccl/utilccl/sampledataccl:sampledataccl_test",
     "//pkg/ccl/utilccl:utilccl_test",
     "//pkg/ccl/workloadccl/allccl:allccl_test",
@@ -153,6 +154,7 @@ ALL_TESTS = [
     "//pkg/sql/colexec/colbuilder:colbuilder_test",
     "//pkg/sql/colexec/execgen:execgen_test",
     "//pkg/sql/colexec:colexec_test",
+    "//pkg/sql/colexecbase/colexecerror:colexecerror_test",
     "//pkg/sql/colexecbase:colexecbase_test",
     "//pkg/sql/colflow/colrpc:colrpc_test",
     "//pkg/sql/colflow:colflow_test",
@@ -238,9 +240,10 @@ ALL_TESTS = [
     "//pkg/storage/metamorphic:metamorphic_test",
     "//pkg/storage:storage_test",
     "//pkg/testutils/keysutils:keysutils_test",
-    "//pkg/testutils/lint/passes/descriptormarshal:descriptormarshal_test",
     "//pkg/testutils/lint/passes/fmtsafe/testdata/src/a:a_test",
     "//pkg/testutils/lint/passes/fmtsafe:fmtsafe_test",
+    "//pkg/testutils/lint/passes/forbiddenmethod:descriptormarshal_test",
+    "//pkg/testutils/lint/passes/forbiddenmethod:forbiddenmethod_test",
     "//pkg/testutils/lint/passes/hash:hash_test",
     "//pkg/testutils/lint/passes/nocopy:nocopy_test",
     "//pkg/testutils/lint/passes/passesutil:passesutil_test",
@@ -327,8 +330,8 @@ ALL_TESTS = [
     "//pkg/workload:workload_test",
 ]
 
-# These suites run only the tests with the appropriate `size` (excepting those
-# tagged `broken_in_bazel`) [1]. Note that tests have a default timeout
+# These suites run only the tests with the appropriate "size" (excepting those
+# tagged "broken_in_bazel") [1]. Note that tests have a default timeout
 # depending on the size [2].
 
 # [1] https://docs.bazel.build/versions/master/be/general.html#test_suite

pkg/cmd/generate-test-suites/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "generate-test-suites_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/cmd/generate-test-suites",
+    visibility = ["//visibility:private"],
+)
+
+go_binary(
+    name = "generate-test-suites",
+    embed = [":generate-test-suites_lib"],
+    visibility = ["//visibility:public"],
+)