From 7bc9782c305ec7674de0aa579ae37e250c05ff3a Mon Sep 17 00:00:00 2001 From: Oliver Tan Date: Tue, 23 Feb 2021 15:16:26 +1100 Subject: [PATCH 1/7] sql: error if SURVIVE is set with no regions Release justification: bug fix to new functionality Release note: None --- pkg/sql/create_database.go | 7 +++++++ pkg/sql/logictest/testdata/logic_test/multiregion | 3 +++ 2 files changed, 10 insertions(+) diff --git a/pkg/sql/create_database.go b/pkg/sql/create_database.go index 5e887eaec592..7b0d0c65c6d8 100644 --- a/pkg/sql/create_database.go +++ b/pkg/sql/create_database.go @@ -86,6 +86,13 @@ func (p *planner) CreateDatabase(ctx context.Context, n *tree.CreateDatabase) (p ) } + if n.SurvivalGoal != tree.SurvivalGoalDefault && n.PrimaryRegion == "" { + return nil, pgerror.New( + pgcode.InvalidDatabaseDefinition, + "PRIMARY REGION must be specified when using SURVIVE", + ) + } + hasCreateDB, err := p.HasRoleOption(ctx, roleoption.CREATEDB) if err != nil { return nil, err diff --git a/pkg/sql/logictest/testdata/logic_test/multiregion b/pkg/sql/logictest/testdata/logic_test/multiregion index 2bbd0372eb7d..054ab3cd4119 100644 --- a/pkg/sql/logictest/testdata/logic_test/multiregion +++ b/pkg/sql/logictest/testdata/logic_test/multiregion @@ -261,6 +261,9 @@ DATABASE multi_region_test_survive_zone_failure_db ALTER DATABASE multi_region_ voter_constraints = '[+region=us-east-1]', lease_preferences = '[[+region=us-east-1]]' +statement error PRIMARY REGION must be specified when using SURVIVE +CREATE DATABASE no_primary_region SURVIVE ZONE FAILURE + statement error PRIMARY REGION must be specified if REGIONS are specified CREATE DATABASE no_primary_region_db REGIONS "ap-southeast-2" From 5519afb51c276551fe403d11b0231fa20403b1a4 Mon Sep 17 00:00:00 2001 From: Rail Aliiev Date: Mon, 22 Feb 2021 23:06:54 -0500 Subject: [PATCH 2/7] TeamCity: update build agents to use the latest sudo Release note: None --- build/packer/README.md | 5 +++-- build/packer/teamcity-agent.sh | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build/packer/README.md b/build/packer/README.md index d4c928629e77..d2b09b683eb2 100644 --- a/build/packer/README.md +++ b/build/packer/README.md @@ -9,7 +9,8 @@ Each `.json` file is a packer template. ```bash brew install packer ``` -2. Run: +2. Configure `gcloud` with your personal [User Application Default Credentials][gauth]. +3. Run: ```bash packer build .json ``` @@ -30,4 +31,4 @@ the `gcloud` tool or provide your Google Cloud JSON credentials in a [Packer]: https://www.packer.io -[gauth]: https://www.packer.io/docs/builders/googlecompute.html#running-without-a-compute-engine-service-account +[gauth]: https://www.packer.io/docs/builders/googlecompute#running-locally-on-your-workstation diff --git a/build/packer/teamcity-agent.sh b/build/packer/teamcity-agent.sh index 3a76b7e8bfd1..f0b199551cff 100644 --- a/build/packer/teamcity-agent.sh +++ b/build/packer/teamcity-agent.sh @@ -29,6 +29,9 @@ add-apt-repository ppa:longsleep/golang-backports add-apt-repository ppa:git-core/ppa apt-get update --yes +# Install the sudo version patched for CVE-2021-3156 +apt-get install --yes sudo + # Install the necessary dependencies. Keep this list small! apt-get install --yes \ docker-ce \ From 72692effa00ecd1f5decd548142412dc989ba872 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Mon, 22 Feb 2021 17:21:43 -0500 Subject: [PATCH 3/7] sql: add backup/restore tests for sequences This patch just adds tests for backup/restore with respect to sequences. Now that all new sequences are referenced by ID and can be renamed, we want to test that previously created sequences are not affected, and still follow the old rules (i.e. cannot be renamed and and are referenced by their names. In the future, we'll want to convert these old sequences to new sequences (i.e. can be renamed and are referenced by ID), but for now, we'll keep the old sequences as is. Release note (None): add backup/restore tests for sequences --- pkg/ccl/backupccl/BUILD.bazel | 1 + .../backupccl/restore_old_sequences_test.go | 104 ++++++++++++++++++ .../testdata/restore_old_sequences/create.sql | 17 +++ .../exports/v20.2.5/635573714396577793.sst | Bin 0 -> 925 bytes .../exports/v20.2.5/635573714396610561.sst | Bin 0 -> 925 bytes .../exports/v20.2.5/635573714396643329.sst | Bin 0 -> 926 bytes .../exports/v20.2.5/BACKUP-STATISTICS | 0 .../exports/v20.2.5/BACKUP_MANIFEST | Bin 0 -> 1085 bytes 8 files changed, 122 insertions(+) create mode 100644 pkg/ccl/backupccl/restore_old_sequences_test.go create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/create.sql create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396577793.sst create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396610561.sst create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396643329.sst create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP-STATISTICS create mode 100644 pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP_MANIFEST diff --git a/pkg/ccl/backupccl/BUILD.bazel b/pkg/ccl/backupccl/BUILD.bazel index 7de3fd69dc9a..b3d61212ca26 100644 --- a/pkg/ccl/backupccl/BUILD.bazel +++ b/pkg/ccl/backupccl/BUILD.bazel @@ -116,6 +116,7 @@ go_test( "main_test.go", "partitioned_backup_test.go", "restore_mid_schema_change_test.go", + "restore_old_sequences_test.go", "restore_old_versions_test.go", "show_test.go", "split_and_scatter_processor_test.go", diff --git a/pkg/ccl/backupccl/restore_old_sequences_test.go b/pkg/ccl/backupccl/restore_old_sequences_test.go new file mode 100644 index 000000000000..86ee498889e3 --- /dev/null +++ b/pkg/ccl/backupccl/restore_old_sequences_test.go @@ -0,0 +1,104 @@ +// Copyright 2021 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package backupccl + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/cockroachdb/cockroach/pkg/base" + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/stretchr/testify/require" +) + +// TestRestoreOldVersions ensures that we can successfully restore tables +// and databases exported by old version. +// +// The files being restored live in testdata and are all made from the same +// input SQL which lives in /create.sql. +// +// The SSTs were created via the following commands: +// +// VERSION=... +// roachprod create local +// roachprod wipe local +// roachprod stage local release ${VERSION} +// roachprod start local +// roachprod sql local:1 -- -e "$(cat pkg/ccl/backupccl/testdata/restore_old_sequences/create.sql)" +// roachprod sql local:1 -- -e "BACKUP DATABASE test TO 'nodelocal://1/backup'" +// # Then grab the backups and put the files into the appropriate +// # testdata directory. +// +func TestRestoreOldSequences(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + const ( + testdataBase = "testdata/restore_old_sequences" + exportDirs = testdataBase + "/exports" + ) + + t.Run("sequences-restore", func(t *testing.T) { + dirs, err := ioutil.ReadDir(exportDirs) + require.NoError(t, err) + for _, dir := range dirs { + require.True(t, dir.IsDir()) + exportDir, err := filepath.Abs(filepath.Join(exportDirs, dir.Name())) + require.NoError(t, err) + t.Run(dir.Name(), restoreOldSequencesTest(exportDir)) + } + }) +} + +func restoreOldSequencesTest(exportDir string) func(t *testing.T) { + return func(t *testing.T) { + params := base.TestServerArgs{} + const numAccounts = 1000 + _, _, sqlDB, dir, cleanup := backupRestoreTestSetupWithParams(t, singleNode, numAccounts, + InitManualReplication, base.TestClusterArgs{ServerArgs: params}) + defer cleanup() + err := os.Symlink(exportDir, filepath.Join(dir, "foo")) + require.NoError(t, err) + sqlDB.Exec(t, `CREATE DATABASE test`) + var unused string + var importedRows int + sqlDB.QueryRow(t, `RESTORE test.* FROM $1`, LocalFoo).Scan( + &unused, &unused, &unused, &importedRows, &unused, &unused, + ) + const totalRows = 3 + if importedRows != totalRows { + t.Fatalf("expected %d rows, got %d", totalRows, importedRows) + } + + // Verify that sequences created in older versions cannot be renamed, nor can the + // database they are referencing. + sqlDB.ExpectErr(t, + `pq: cannot rename relation "test.public.s" because view "t1" depends on it`, + `ALTER SEQUENCE test.s RENAME TO test.s2`) + sqlDB.ExpectErr(t, + `pq: cannot rename relation "test.public.t1_i_seq" because view "t1" depends on it`, + `ALTER SEQUENCE test.t1_i_seq RENAME TO test.t1_i_seq_new`) + sqlDB.ExpectErr(t, + `pq: cannot rename database because relation "test.public.t1" depends on relation "test.public.s"`, + `ALTER DATABASE test RENAME TO new_test`) + + sequenceResults := [][]string{ + {"1", "1"}, + {"2", "2"}, + } + + // Verify that tables with old sequences aren't corrupted. + //sqlDB.Exec(t, `SET database = test`) + //sqlDB.Exec(t, `USE test`) + sqlDB.Exec(t, `SET database = test; INSERT INTO test.t1 VALUES (default, default)`) + sqlDB.CheckQueryResults(t, `SELECT * FROM test.t1 ORDER BY i`, sequenceResults) + } +} diff --git a/pkg/ccl/backupccl/testdata/restore_old_sequences/create.sql b/pkg/ccl/backupccl/testdata/restore_old_sequences/create.sql new file mode 100644 index 000000000000..2df23d0553df --- /dev/null +++ b/pkg/ccl/backupccl/testdata/restore_old_sequences/create.sql @@ -0,0 +1,17 @@ +-- The below SQL is used to create the data that is then exported with BACKUP +-- for use in the RestoreOldSequences test. + +CREATE DATABASE test; + +SET database = test; + +-- t1 is a table with a SERIAL sequence and a DEFAULT value sequence. + +CREATE SEQUENCE s; + +SET serial_normalization = 'sql_sequence'; + +CREATE TABLE t1 (i SERIAL PRIMARY KEY, j INT NOT NULL DEFAULT nextval('test.public.s')); + +INSERT INTO t1 VALUES (default, default); + diff --git a/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396577793.sst b/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396577793.sst new file mode 100644 index 0000000000000000000000000000000000000000..459e0f9db7063fb7e0b81c676957161cb164aae7 GIT binary patch literal 925 zcmah|&2AGh5VqSi*`y(DDGjJXRSuQ7w5A0_4^=Px9}!gLfY8djJ4vj1?Zx&c*`7dL zig*JgehxeXCnPxX0&w98c!nNeyh#cd#GE|i`T4$YY|^fNetCRM+THCBm)Ab_>y(V` z`{01n8GkM#FW$X3q%re_Ed3fAxpaC5Kfk>t^FeHqh&jtrK056-69_eR$a=--_|Rfj zB=Nu^ZXscgM-rG&35=L4GVpCz7ingoD@LpuIc6s~WIYX!TI>EGw3SFw~ht-wqLr9)^){ zc5ie0+HOa}0Oa1)D?8VBc01N#Pv{kUTe}@(8YUukVN^Jw|`s43`|7iC!j67hnHbpivw{uHDjR zje>GU03H`8*7RT#dqSx`!2&H4u&!M6SfZ8MFNB(1vPM*D5&+q>D4Se?^z5oCK literal 0 HcmV?d00001 diff --git a/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396610561.sst b/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/635573714396610561.sst new file mode 100644 index 0000000000000000000000000000000000000000..c6f56856f928532877d98fd0ee7e890f809f605a GIT binary patch literal 925 zcmah|J8x4l6t??FZqksplm@9nRfb9|tj3048RJ4sd*v6c)spe2&lK`@Un7R^`*nqa)JlY<$>Rem|&DGPJkH zKcCWxa4sY5;{In+pExG-zlKIGp5A@`@!XS{D77MC-m#1iPPtF7Pw)b`~|n^!lt+RkH7=oZ`STWw?-3z_;jDLl9kIdopcA&gm4|A~VREJaAE zHK+w1CNVEFAW@A9?m2Vv5LRn56cl)r1z9>0Ek`gw{+g@UIUl1Mz`DvL!G~ws1hIlG zRi_TX20CG7(F`5xRyHN|CLIhrG6lNp_ekvwH8$g(jfzOLNQWB;;!4MQ4s3uxno~3Y zyU=XW1Fm|wD8~dS#0xq#0tAVgGy~fOW^%Ax3r7|vtEjr}DV2L0%~IXVSV^&5yQ7UN z1?{Z_JkCB1WJgi?Ki1x5*QzFhQBqQ&ayIiMPrr`Vlop228Def~z6?F)u4K|#27 zk9JYp9n;fdH&J)2$*+yCZkF}M>h)6SyJ7l2Ql3P(D&8+~1QA+|`bdP}Rf6v2-}w6a safh6p96#isyN`6h@KG9s0J9 zSad&%g|nNTwVNAl3Hu;7Z(Lu$wZ74|4tqkc++5vgBhx4qiHqXWf%TCC>(a=FF-xdF zaZsLR>QibBYJmGm%!v$0SfeR-oLO-Io!Tq~2@YjJmX?GULl_``#bsozi%<<tW=X7|%OvN$<@+&NA|G)V8^$@FQ`&+kV+zRk2d=X~Tp_Uz>4o6E(6qu(JLBGLc= literal 0 HcmV?d00001 diff --git a/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP-STATISTICS b/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP-STATISTICS new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP_MANIFEST b/pkg/ccl/backupccl/testdata/restore_old_sequences/exports/v20.2.5/BACKUP_MANIFEST new file mode 100644 index 0000000000000000000000000000000000000000..153818e5064835faa8cf6d2dec36b990c25d320d GIT binary patch literal 1085 zcmV-D1j73tiwFP!00000|J;&YY*S?z$DikIp@+kSLzP(?35Q<5VaZOcGQ(#ynTZ#Q!69xU8k2>n zfW(hU+)P(sn;U-IdT!2}=Y9Y0|9O7@mj)|M-MF^>k4qgd9Iv!fc5jBHIv-I$8H{Y zySJk))DmvbHctKSXvMYpyW50f^JiCnW};VbM=uw*{c^6a|EoDu23s#~x~}wG@a~h$ z6wROy0#NoVckvfL+2xYFWxti7?!||jIH8>u>&2#3!kPiAY4wNsBlhr$FP9y- z{HbTpk!N0U#y3B;Z^qdL%U2oa>o3i2UUjnP#P?mhd)g#3HF>08dDhW>irek>y2^ge zo3E}RZ)9(o-a1$Jt-pJFR@{>DRLr>%T~OFCYj1xfwdD6*+*H2d^Nwk|lm|{)K6uxg zeB#W&#r?UKy#p_8?H``6YQEJ=n+tlrz{vKQ^dx$MmZ&q*452C7oXcbi46zRVZRnt> z_SPNOJ4zvO=!Xzg9U}C8LSx>hurfz{H1*EM?>*aDIuMvmPiW9m^*V$03JMv&4Fm{3 zv<_GhU;|{K5tcw?C2h&(j5;mX#GnlvWra|>o1uUKJFvik03PteTG&D;XhwJ*p$(xO z;WeB{ffa0U7~vQ~C&D_|`1d0W$xZF0^OeKaU@sf9$YBxTJi-N>NU03i$Tw)CkU`tv zoo_3P97yI;bA1Rup}Ew-ZwSo&`{OCKBJ4u)#p^-_T1C4)^0*;>o`NtISZVC^NOaTo$Ze)=q94R86 z$NM3^hi(+&qlA8w(9~U6Huo6N|BEMy|NTjh%gZ;>3U&y9A43e+I|wKG{Rf}x+L8NW z#qlL!T-%NGr1cq*7hSw4TP+Qez)QShe^3%6(J4qyQDsHRr?`DC*};jb6j!vk>{UJS zq)UtodP0%o3CXKz9!-zO6RIl7?pUl!k$H(1*)CK?m DzIGgB literal 0 HcmV?d00001 From 9e2d1429f62eec69a6cb75c0eb0d85c74b38035e Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 23 Feb 2021 22:09:37 +0100 Subject: [PATCH 4/7] authors: add erikgrinaker Release note: none --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 8bf18e3c8818..6e80144b6774 100644 --- a/AUTHORS +++ b/AUTHORS @@ -119,6 +119,7 @@ embark Emmanuel Sales Emmanuel <@cockroachlabs.com> Ernie Cordeiro Eric Wang +Erik Grinaker Erik Trinh es-chow Evgeniy Vasilev From bf563bc996bbd0a528799e3580a9218b97507d16 Mon Sep 17 00:00:00 2001 From: Faizan Qazi Date: Tue, 23 Feb 2021 16:10:19 -0500 Subject: [PATCH 5/7] authors: add fqazi to authors Release note: None Release note (): --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 8bf18e3c8818..d4837c59e0f2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -123,6 +123,7 @@ Erik Trinh es-chow Evgeniy Vasilev fabio +Faizan Qazi fangwens Francis Bergin funkygao From 00e29817dfc88fc5d46c5f269cb5c681057c9607 Mon Sep 17 00:00:00 2001 From: Alex Barganier Date: Tue, 23 Feb 2021 16:09:06 -0500 Subject: [PATCH 6/7] authors: add abarganier to authors Release note: None --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 8bf18e3c8818..ab7fc9745df4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,6 +36,7 @@ Aditya Maru <@cockroachlabs.com> Aid Idrizović Ajaya Agrawal Alan Acosta +Alex Barganier Alex Gaynor Alex Lunev Alex Robinson <@cockroachlabs.com> From aa343ff05bc872dc4065b72d6d8ee3f51709facd Mon Sep 17 00:00:00 2001 From: Yahor Yuzefovich Date: Mon, 22 Feb 2021 23:09:37 -0800 Subject: [PATCH 7/7] colexec: remove duplication around bazel utility functions Previously, we had essentially duplicated `.bzl` files in two packages that need to generate code using `execgen`, but this can be avoided by defining a separate `.bzl` file as an extension used in both places. Release note: None --- pkg/sql/colexec/BUILD.bazel | 57 ++++++++++++- pkg/sql/colexec/COLEXEC.bzl | 85 ------------------- pkg/sql/colexec/colexecagg/BUILD.bazel | 31 ++++++- .../EXECGEN.bzl} | 41 +++------ 4 files changed, 93 insertions(+), 121 deletions(-) delete mode 100644 pkg/sql/colexec/COLEXEC.bzl rename pkg/sql/{colexec/colexecagg/COLEXECAGG.bzl => colexecbase/EXECGEN.bzl} (60%) diff --git a/pkg/sql/colexec/BUILD.bazel b/pkg/sql/colexec/BUILD.bazel index 7d0cc06ccfd6..7f2b208ddfd2 100644 --- a/pkg/sql/colexec/BUILD.bazel +++ b/pkg/sql/colexec/BUILD.bazel @@ -1,5 +1,5 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") -load(":COLEXEC.bzl", "eg_go_filegroup", "gen_eg_go_rules") +load("//pkg/sql/colexecbase:EXECGEN.bzl", "eg_go_filegroup", "gen_eg_go_rules") go_library( name = "colexec", @@ -211,8 +211,59 @@ go_test( ], ) +# Map between target name and relevant template. +targets = [ + ("and_or_projection.eg.go", "and_or_projection_tmpl.go"), + ("cast.eg.go", "cast_tmpl.go"), + ("const.eg.go", "const_tmpl.go"), + ("crossjoiner.eg.go", "crossjoiner_tmpl.go"), + ("default_cmp_expr.eg.go", "default_cmp_expr_tmpl.go"), + ("default_cmp_proj_ops.eg.go", "default_cmp_proj_ops_tmpl.go"), + ("default_cmp_sel_ops.eg.go", "default_cmp_sel_ops_tmpl.go"), + ("distinct.eg.go", "distinct_tmpl.go"), + ("hash_aggregator.eg.go", "hash_aggregator_tmpl.go"), + ("hash_utils.eg.go", "hash_utils_tmpl.go"), + ("hashjoiner.eg.go", "hashjoiner_tmpl.go"), + ("hashtable_distinct.eg.go", "hashtable_tmpl.go"), + ("hashtable_full_default.eg.go", "hashtable_tmpl.go"), + ("hashtable_full_deleting.eg.go", "hashtable_tmpl.go"), + ("is_null_ops.eg.go", "is_null_ops_tmpl.go"), + ("mergejoinbase.eg.go", "mergejoinbase_tmpl.go"), + ("mergejoiner_exceptall.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_fullouter.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_inner.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_intersectall.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_leftanti.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_leftouter.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_leftsemi.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_rightanti.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_rightouter.eg.go", "mergejoiner_tmpl.go"), + ("mergejoiner_rightsemi.eg.go", "mergejoiner_tmpl.go"), + ("ordered_synchronizer.eg.go", "ordered_synchronizer_tmpl.go"), + ("proj_const_left_ops.eg.go", "proj_const_ops_tmpl.go"), + ("proj_const_right_ops.eg.go", "proj_const_ops_tmpl.go"), + ("proj_like_ops.eg.go", "proj_const_ops_tmpl.go"), + ("proj_non_const_ops.eg.go", "proj_non_const_ops_tmpl.go"), + ("quicksort.eg.go", "quicksort_tmpl.go"), + ("rank.eg.go", "rank_tmpl.go"), + ("relative_rank.eg.go", "relative_rank_tmpl.go"), + ("row_number.eg.go", "row_number_tmpl.go"), + ("rowstovec.eg.go", "rowstovec_tmpl.go"), + ("select_in.eg.go", "select_in_tmpl.go"), + ("selection_ops.eg.go", "selection_ops_tmpl.go"), + ("sel_like_ops.eg.go", "selection_ops_tmpl.go"), + ("sort.eg.go", "sort_tmpl.go"), + ("substring.eg.go", "substring_tmpl.go"), + ("values_differ.eg.go", "values_differ_tmpl.go"), + ("vec_comparators.eg.go", "vec_comparators_tmpl.go"), + ("window_peer_grouper.eg.go", "window_peer_grouper_tmpl.go"), +] + # Define a file group for all the .eg.go targets. -eg_go_filegroup(name = "gen-exec") +eg_go_filegroup( + name = "gen-exec", + targets = targets, +) # Define gen rules for individual eg.go files. -gen_eg_go_rules() +gen_eg_go_rules(targets) diff --git a/pkg/sql/colexec/COLEXEC.bzl b/pkg/sql/colexec/COLEXEC.bzl deleted file mode 100644 index ede1a114bf43..000000000000 --- a/pkg/sql/colexec/COLEXEC.bzl +++ /dev/null @@ -1,85 +0,0 @@ -# Map between target name and relevant template. -targets = [ - ('and_or_projection.eg.go', 'and_or_projection_tmpl.go'), - ('cast.eg.go', 'cast_tmpl.go'), - ('const.eg.go', 'const_tmpl.go'), - ('crossjoiner.eg.go', 'crossjoiner_tmpl.go'), - ('default_cmp_expr.eg.go', 'default_cmp_expr_tmpl.go'), - ('default_cmp_proj_ops.eg.go', 'default_cmp_proj_ops_tmpl.go'), - ('default_cmp_sel_ops.eg.go', 'default_cmp_sel_ops_tmpl.go'), - ('distinct.eg.go', 'distinct_tmpl.go'), - ('hash_aggregator.eg.go', 'hash_aggregator_tmpl.go'), - ('hash_utils.eg.go', 'hash_utils_tmpl.go'), - ('hashjoiner.eg.go', 'hashjoiner_tmpl.go'), - ('hashtable_distinct.eg.go', 'hashtable_tmpl.go'), - ('hashtable_full_default.eg.go', 'hashtable_tmpl.go'), - ('hashtable_full_deleting.eg.go', 'hashtable_tmpl.go'), - ('is_null_ops.eg.go', 'is_null_ops_tmpl.go'), - ('mergejoinbase.eg.go', 'mergejoinbase_tmpl.go'), - ('mergejoiner_exceptall.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_fullouter.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_inner.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_intersectall.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_leftanti.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_leftouter.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_leftsemi.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_rightanti.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_rightouter.eg.go', 'mergejoiner_tmpl.go'), - ('mergejoiner_rightsemi.eg.go', 'mergejoiner_tmpl.go'), - ('ordered_synchronizer.eg.go', 'ordered_synchronizer_tmpl.go'), - ('proj_const_left_ops.eg.go', 'proj_const_ops_tmpl.go'), - ('proj_const_right_ops.eg.go', 'proj_const_ops_tmpl.go'), - ('proj_like_ops.eg.go', 'proj_const_ops_tmpl.go'), - ('proj_non_const_ops.eg.go', 'proj_non_const_ops_tmpl.go'), - ('quicksort.eg.go', 'quicksort_tmpl.go'), - ('rank.eg.go', 'rank_tmpl.go'), - ('relative_rank.eg.go', 'relative_rank_tmpl.go'), - ('row_number.eg.go', 'row_number_tmpl.go'), - ('rowstovec.eg.go', 'rowstovec_tmpl.go'), - ('select_in.eg.go', 'select_in_tmpl.go'), - ('selection_ops.eg.go', 'selection_ops_tmpl.go'), - ('sel_like_ops.eg.go', 'selection_ops_tmpl.go'), - ('sort.eg.go', 'sort_tmpl.go'), - ('substring.eg.go', 'substring_tmpl.go'), - ('values_differ.eg.go', 'values_differ_tmpl.go'), - ('vec_comparators.eg.go', 'vec_comparators_tmpl.go'), - ('window_peer_grouper.eg.go', 'window_peer_grouper_tmpl.go'), -] - -def rule_name_for(target): - # e.g. 'vec_comparators.eg.go' -> 'gen-vec-comparators' - return 'gen-{}'.format(target.replace('.eg.go', '').replace('_', '-')) - -# Define a file group for all the .eg.go targets. -def eg_go_filegroup(name): - native.filegroup( - name = name, - srcs = [':{}'.format(rule_name_for(target)) for target, _ in targets], - ) - -# Define gen rules for individual eg.go files. -def gen_eg_go_rules(): - # Define some aliases for ease of use. - native.alias( - name = "execgen", - actual = "//pkg/sql/colexec/execgen/cmd/execgen", - ) - native.alias( - name = "goimports", - actual = "@com_github_cockroachdb_gostdlib//x/tools/cmd/goimports", - ) - - for target, template in targets: - name = rule_name_for(target) - - native.genrule( - name = name, - srcs = [template], - outs = [target], - cmd = """ - $(location :execgen) -template $(SRCS) \ - -fmt=false pkg/sql/colexec/$@ > $@ - $(location :goimports) -w $@ - """, - tools = [":execgen", ":goimports"], - ) diff --git a/pkg/sql/colexec/colexecagg/BUILD.bazel b/pkg/sql/colexec/colexecagg/BUILD.bazel index 8973ffd486ee..26377a06e713 100644 --- a/pkg/sql/colexec/colexecagg/BUILD.bazel +++ b/pkg/sql/colexec/colexecagg/BUILD.bazel @@ -1,5 +1,5 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") -load(":COLEXECAGG.bzl", "eg_go_filegroup", "gen_eg_go_rules") +load("//pkg/sql/colexecbase:EXECGEN.bzl", "eg_go_filegroup", "gen_eg_go_rules") go_library( name = "colexecagg", @@ -30,8 +30,33 @@ go_library( ], ) +# Map between target name and relevant template. +targets = [ + ("hash_any_not_null_agg.eg.go", "any_not_null_agg_tmpl.go"), + ("hash_avg_agg.eg.go", "avg_agg_tmpl.go"), + ("hash_bool_and_or_agg.eg.go", "bool_and_or_agg_tmpl.go"), + ("hash_concat_agg.eg.go", "concat_agg_tmpl.go"), + ("hash_count_agg.eg.go", "count_agg_tmpl.go"), + ("hash_default_agg.eg.go", "default_agg_tmpl.go"), + ("hash_min_max_agg.eg.go", "min_max_agg_tmpl.go"), + ("hash_sum_agg.eg.go", "sum_agg_tmpl.go"), + ("hash_sum_int_agg.eg.go", "sum_agg_tmpl.go"), + ("ordered_any_not_null_agg.eg.go", "any_not_null_agg_tmpl.go"), + ("ordered_avg_agg.eg.go", "avg_agg_tmpl.go"), + ("ordered_bool_and_or_agg.eg.go", "bool_and_or_agg_tmpl.go"), + ("ordered_concat_agg.eg.go", "concat_agg_tmpl.go"), + ("ordered_count_agg.eg.go", "count_agg_tmpl.go"), + ("ordered_default_agg.eg.go", "default_agg_tmpl.go"), + ("ordered_min_max_agg.eg.go", "min_max_agg_tmpl.go"), + ("ordered_sum_agg.eg.go", "sum_agg_tmpl.go"), + ("ordered_sum_int_agg.eg.go", "sum_agg_tmpl.go"), +] + # Define a file group for all the .eg.go targets. -eg_go_filegroup(name = "gen-exec") +eg_go_filegroup( + name = "gen-exec", + targets = targets, +) # Define gen rules for individual eg.go files. -gen_eg_go_rules() +gen_eg_go_rules(targets) diff --git a/pkg/sql/colexec/colexecagg/COLEXECAGG.bzl b/pkg/sql/colexecbase/EXECGEN.bzl similarity index 60% rename from pkg/sql/colexec/colexecagg/COLEXECAGG.bzl rename to pkg/sql/colexecbase/EXECGEN.bzl index fbed45ccfaf4..869db5f96a14 100644 --- a/pkg/sql/colexec/colexecagg/COLEXECAGG.bzl +++ b/pkg/sql/colexecbase/EXECGEN.bzl @@ -1,41 +1,18 @@ -# NB: The helpers here were grafted from pkg/sql/colexec/COLEXEC.bzl. -# Future changes here may apply there. - -# Map between target name and relevant template. -targets = [ - ('hash_any_not_null_agg.eg.go', 'any_not_null_agg_tmpl.go'), - ('hash_avg_agg.eg.go', 'avg_agg_tmpl.go'), - ('hash_bool_and_or_agg.eg.go', 'bool_and_or_agg_tmpl.go'), - ('hash_concat_agg.eg.go', 'concat_agg_tmpl.go'), - ('hash_count_agg.eg.go', 'count_agg_tmpl.go'), - ('hash_default_agg.eg.go', 'default_agg_tmpl.go'), - ('hash_min_max_agg.eg.go', 'min_max_agg_tmpl.go'), - ('hash_sum_agg.eg.go', 'sum_agg_tmpl.go'), - ('hash_sum_int_agg.eg.go', 'sum_agg_tmpl.go'), - ('ordered_any_not_null_agg.eg.go', 'any_not_null_agg_tmpl.go'), - ('ordered_avg_agg.eg.go', 'avg_agg_tmpl.go'), - ('ordered_bool_and_or_agg.eg.go', 'bool_and_or_agg_tmpl.go'), - ('ordered_concat_agg.eg.go', 'concat_agg_tmpl.go'), - ('ordered_count_agg.eg.go', 'count_agg_tmpl.go'), - ('ordered_default_agg.eg.go', 'default_agg_tmpl.go'), - ('ordered_min_max_agg.eg.go', 'min_max_agg_tmpl.go'), - ('ordered_sum_agg.eg.go', 'sum_agg_tmpl.go'), - ('ordered_sum_int_agg.eg.go', 'sum_agg_tmpl.go'), -] - -def rule_name_for(target): - # e.g. 'vec_comparators.eg.go' -> 'gen-vec-comparators' - return 'gen-{}'.format(target.replace('.eg.go', '').replace('_', '-')) +# This file defines a couple of utility methods that are used to generate all +# code for the vectorized engine based on the templates. It should be loaded +# as an extension in relevant packages, and the two functions below should be +# invoked with the corresponding targets mapping (from .eg.go file to its +# _tmpl.go file). # Define a file group for all the .eg.go targets. -def eg_go_filegroup(name): +def eg_go_filegroup(name, targets): native.filegroup( name = name, srcs = [':{}'.format(rule_name_for(target)) for target, _ in targets], ) # Define gen rules for individual eg.go files. -def gen_eg_go_rules(): +def gen_eg_go_rules(targets): # Define some aliases for ease of use. native.alias( name = "execgen", @@ -71,3 +48,7 @@ def gen_eg_go_rules(): """, tools = [":execgen", ":goimports"], ) + +def rule_name_for(target): + # e.g. 'vec_comparators.eg.go' -> 'gen-vec-comparators' + return 'gen-{}'.format(target.replace('.eg.go', '').replace('_', '-'))