diff --git a/pkg/sql/backfill/backfill.go b/pkg/sql/backfill/backfill.go index 6038384b545c..030cab6a678a 100644 --- a/pkg/sql/backfill/backfill.go +++ b/pkg/sql/backfill/backfill.go @@ -253,9 +253,10 @@ func (cb *ColumnBackfiller) RunColumnBackfillChunk( ) (roachpb.Key, error) { // TODO(dan): Tighten up the bound on the requestedCols parameter to // makeRowUpdater. - requestedCols := make([]descpb.ColumnDescriptor, 0, len(tableDesc.Columns)+len(cb.added)) + requestedCols := make([]descpb.ColumnDescriptor, 0, len(tableDesc.Columns)+len(cb.added)+len(cb.dropped)) requestedCols = append(requestedCols, tableDesc.Columns...) requestedCols = append(requestedCols, cb.added...) + requestedCols = append(requestedCols, cb.dropped...) ru, err := row.MakeUpdater( ctx, txn, diff --git a/pkg/sql/logictest/testdata/logic_test/partial_index b/pkg/sql/logictest/testdata/logic_test/partial_index index c838197fdc9b..b3b5b152ec6a 100644 --- a/pkg/sql/logictest/testdata/logic_test/partial_index +++ b/pkg/sql/logictest/testdata/logic_test/partial_index @@ -1281,6 +1281,54 @@ SELECT * FROM inv_c@i WHERE j @> '{"x": "y"}' AND s IN ('foo', 'bar') ORDER BY k 1 {"num": 1, "x": "y"} foo 3 {"num": 3, "x": "y"} bar +# Updates and Upserts with fetch columns pruned. + +statement ok +CREATE TABLE prune ( + a INT PRIMARY KEY, + b INT, + c INT, + d INT, + INDEX idx (b) WHERE c > 0, + FAMILY (a), + FAMILY (b), + FAMILY (c), + FAMILY (d) +) + +statement ok +INSERT INTO prune (a, b, c, d) VALUES (1, 2, 3, 4) + +# Test that an update is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +UPDATE prune SET d = d + 1 WHERE a = 1 + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 5 + +# Test that an upsert is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +UPSERT INTO prune (a, d) VALUES (1, 6) + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 6 + +# Test that an upsert is successful when fetch columns b and c are pruned +# because an update to idx is not required. +statement ok +INSERT INTO prune (a, d) VALUES (1, 6) ON CONFLICT (a) DO UPDATE SET d = 7 + +query IIII rowsort +SELECT * FROM prune@idx WHERE c > 0 +---- +1 2 3 7 + # Regression tests for #52318. Mutations on partial indexes in the # DELETE_AND_WRITE_ONLY state should update the indexes correctly. subtest regression_52318 diff --git a/pkg/sql/opt/memo/expr.go b/pkg/sql/opt/memo/expr.go index d91af9257fae..5439b4fdbe39 100644 --- a/pkg/sql/opt/memo/expr.go +++ b/pkg/sql/opt/memo/expr.go @@ -658,7 +658,7 @@ func (s *ScanPrivate) IsLocking() bool { // index, nil is returned. func (s *ScanPrivate) PartialIndexPredicate(md *opt.Metadata) FiltersExpr { tabMeta := md.TableMeta(s.Table) - p, ok := tabMeta.PartialIndexPredicates[s.Index] + p, ok := tabMeta.PartialIndexPredicate(s.Index) if !ok { // The index is not a partial index. return nil diff --git a/pkg/sql/opt/memo/expr_format.go b/pkg/sql/opt/memo/expr_format.go index 38c8ac33ea1d..abcc749f2343 100644 --- a/pkg/sql/opt/memo/expr_format.go +++ b/pkg/sql/opt/memo/expr_format.go @@ -352,17 +352,18 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) { f.formatExpr(tab.ComputedCols[col], c.Child(f.ColumnString(col))) } } - if tab.PartialIndexPredicates != nil { + partialIndexPredicates := tab.PartialIndexPredicatesForFormattingOnly() + if partialIndexPredicates != nil { c := tp.Child("partial index predicates") - indexOrds := make([]cat.IndexOrdinal, 0, len(tab.PartialIndexPredicates)) - for ord := range tab.PartialIndexPredicates { + indexOrds := make([]cat.IndexOrdinal, 0, len(partialIndexPredicates)) + for ord := range partialIndexPredicates { indexOrds = append(indexOrds, ord) } sort.Ints(indexOrds) for _, ord := range indexOrds { name := string(tab.Table.Index(ord).Name()) f.Buffer.Reset() - f.formatScalarWithLabel(name, tab.PartialIndexPredicates[ord], c) + f.formatScalarWithLabel(name, partialIndexPredicates[ord], c) } } } diff --git a/pkg/sql/opt/memo/testdata/logprops/delete b/pkg/sql/opt/memo/testdata/logprops/delete index a499a2b85e91..013da35d6772 100644 --- a/pkg/sql/opt/memo/testdata/logprops/delete +++ b/pkg/sql/opt/memo/testdata/logprops/delete @@ -64,7 +64,6 @@ project ├── volatile, mutations ├── key: (5) ├── fd: ()-->(1), (5)-->(2-4) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── key: (12) @@ -107,7 +106,6 @@ project ├── volatile, mutations ├── key: () ├── fd: ()-->(1-5) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── cardinality: [0 - 1] @@ -148,7 +146,6 @@ project ├── volatile, mutations ├── key: (5) ├── fd: (2)==(3), (3)==(2), (5)-->(1-4) - ├── prune: (1-4) └── select ├── columns: a:8(int!null) b:9(int!null) c:10(int!null) d:11(int) rowid:12(int!null) e:13(int) crdb_internal_mvcc_timestamp:14(decimal) ├── key: (12) diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index 6a4ef3af62bd..51c644c37053 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -396,7 +396,7 @@ func (md *Metadata) AddTable(tab cat.Table, alias *tree.TableName) TableID { // ScalarExpr to new column IDs. It takes as arguments a ScalarExpr and a // mapping of old column IDs to new column IDs, and returns a new ScalarExpr. // This function is used when duplicating Constraints, ComputedCols, and -// PartialIndexPredicates. DuplicateTable requires this callback function, +// partialIndexPredicates. DuplicateTable requires this callback function, // rather than performing the remapping itself, because remapping column IDs // requires constructing new expressions with norm.Factory. The norm package // depends on opt, and cannot be imported here. @@ -450,9 +450,9 @@ func (md *Metadata) DuplicateTable( // Create new partial index predicate expressions by remapping the column // IDs in each ScalarExpr. var partialIndexPredicates map[cat.IndexOrdinal]ScalarExpr - if len(tabMeta.PartialIndexPredicates) > 0 { - partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr, len(tabMeta.PartialIndexPredicates)) - for idxOrd, e := range tabMeta.PartialIndexPredicates { + if len(tabMeta.partialIndexPredicates) > 0 { + partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr, len(tabMeta.partialIndexPredicates)) + for idxOrd, e := range tabMeta.partialIndexPredicates { partialIndexPredicates[idxOrd] = remapColumnIDs(e, colMap) } } @@ -464,7 +464,7 @@ func (md *Metadata) DuplicateTable( IgnoreForeignKeys: tabMeta.IgnoreForeignKeys, Constraints: constraints, ComputedCols: computedCols, - PartialIndexPredicates: partialIndexPredicates, + partialIndexPredicates: partialIndexPredicates, }) return newTabID diff --git a/pkg/sql/opt/metadata_test.go b/pkg/sql/opt/metadata_test.go index 6f98e25eff39..c1e4c3b45390 100644 --- a/pkg/sql/opt/metadata_test.go +++ b/pkg/sql/opt/metadata_test.go @@ -281,7 +281,7 @@ func TestIndexColumns(t *testing.T) { // TestDuplicateTable tests that we can extract a set of columns from an index ordinal. func TestDuplicateTable(t *testing.T) { cat := testcat.New() - _, err := cat.ExecuteDDL("CREATE TABLE a (b BOOL, b2 BOOL)") + _, err := cat.ExecuteDDL("CREATE TABLE a (b BOOL, b2 BOOL, INDEX (b2) WHERE b)") if err != nil { t.Fatal(err) } @@ -332,11 +332,12 @@ func TestDuplicateTable(t *testing.T) { t.Errorf("expected computed column to reference new column ID %d, got %d", dupB, col) } - if dupTabMeta.PartialIndexPredicates == nil || dupTabMeta.PartialIndexPredicates[1] == nil { + pred, isPartialIndex := dupTabMeta.PartialIndexPredicate(1) + if !isPartialIndex { t.Fatalf("expected partial index predicates to be duplicated") } - col = dupTabMeta.PartialIndexPredicates[1].(*memo.VariableExpr).Col + col = pred.(*memo.VariableExpr).Col if col == b { t.Errorf("expected partial index predicate to reference new column ID %d, got %d", dupB, col) } diff --git a/pkg/sql/opt/norm/BUILD.bazel b/pkg/sql/opt/norm/BUILD.bazel index 28af1cffb2d6..1355bdbd12fb 100644 --- a/pkg/sql/opt/norm/BUILD.bazel +++ b/pkg/sql/opt/norm/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "join_funcs.go", "limit_funcs.go", "list_sorter.go", + "mutation_funcs.go", "ordering_funcs.go", "project_builder.go", "project_funcs.go", diff --git a/pkg/sql/opt/norm/mutation_funcs.go b/pkg/sql/opt/norm/mutation_funcs.go new file mode 100644 index 000000000000..c1800fc60f33 --- /dev/null +++ b/pkg/sql/opt/norm/mutation_funcs.go @@ -0,0 +1,119 @@ +// Copyright 2020 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package norm + +import ( + "github.com/cockroachdb/cockroach/pkg/sql/opt" + "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" +) + +// SimplifiablePartialIndexProjectCols returns the set of projected partial +// index PUT and DEL columns with expressions that can be simplified to false. +// These projected expressions can only be simplified to false when an UPDATE +// mutates neither the associated index's columns nor the columns referenced in +// the partial index predicate. +func (c *CustomFuncs) SimplifiablePartialIndexProjectCols( + private *memo.MutationPrivate, + uniqueChecks memo.UniqueChecksExpr, + fkChecks memo.FKChecksExpr, + projections memo.ProjectionsExpr, +) opt.ColSet { + tabMeta := c.mem.Metadata().TableMeta(private.Table) + + // Determine the set of target table columns that need to be updated. Notice + // that we collect the target table column IDs, not the update column IDs. + var updateCols opt.ColSet + for ord, col := range private.UpdateCols { + if col != 0 { + updateCols.Add(tabMeta.MetaID.ColumnID(ord)) + } + } + + // Determine the set of columns needed for the mutation operator, excluding + // the partial index PUT and DEL columns. + neededMutationCols := c.neededMutationCols(private, uniqueChecks, fkChecks, false /* includePartialIndexCols */) + + // Determine the set of project columns that are already simplified to + // false. + var simplifiedProjectCols opt.ColSet + for i := range projections { + project := &projections[i] + if project.Element == memo.FalseSingleton { + simplifiedProjectCols.Add(project.Col) + } + } + + // Columns that are required by the mutation operator and columns that + // have already been simplified to false are ineligible to be simplified. + ineligibleCols := neededMutationCols.Union(simplifiedProjectCols) + + // ord is an ordinal into the mutation's PartialIndexPutCols and + // PartialIndexDelCols, which both have entries for each partial index + // defined on the table. + ord := -1 + var cols opt.ColSet + for i, n := 0, tabMeta.Table.DeletableIndexCount(); i < n; i++ { + pred, isPartialIndex := tabMeta.PartialIndexPredicate(i) + + // Skip non-partial indexes. + if !isPartialIndex { + continue + } + ord++ + + // If the columns being updated are part of the index or referenced in + // the partial index predicate, then updates to the index may be + // required. Therefore, the partial index PUT and DEL columns cannot be + // simplified. + // + // Note that we use the set of index columns where the virtual + // columns have been mapped to their source columns. Virtual columns + // are never part of the updated columns. Updates to source columns + // trigger index changes. + predFilters := *pred.(*memo.FiltersExpr) + indexAndPredCols := tabMeta.IndexColumnsMapVirtual(i) + indexAndPredCols.UnionWith(predFilters.OuterCols()) + if indexAndPredCols.Intersects(updateCols) { + continue + } + + // Add the projected PUT column if it is eligible to be simplified. + putCol := private.PartialIndexPutCols[ord] + if !ineligibleCols.Contains(putCol) { + cols.Add(putCol) + } + + // Add the projected DEL column if it is eligible to be simplified. + delCol := private.PartialIndexDelCols[ord] + if !ineligibleCols.Contains(delCol) { + cols.Add(delCol) + } + } + + return cols +} + +// SimplifyPartialIndexProjections returns a new projection expression with any +// projected column's expression simplified to false if the column exists in +// simplifiableCols. +func (c *CustomFuncs) SimplifyPartialIndexProjections( + projections memo.ProjectionsExpr, simplifiableCols opt.ColSet, +) memo.ProjectionsExpr { + simplified := make(memo.ProjectionsExpr, len(projections)) + for i := range projections { + if col := projections[i].Col; simplifiableCols.Contains(col) { + simplified[i] = c.f.ConstructProjectionsItem(memo.FalseSingleton, col) + } else { + simplified[i] = projections[i] + } + } + return simplified +} diff --git a/pkg/sql/opt/norm/prune_cols_funcs.go b/pkg/sql/opt/norm/prune_cols_funcs.go index ca0cf56b424b..c7aa2303aacf 100644 --- a/pkg/sql/opt/norm/prune_cols_funcs.go +++ b/pkg/sql/opt/norm/prune_cols_funcs.go @@ -44,6 +44,15 @@ func (c *CustomFuncs) NeededExplainCols(private *memo.ExplainPrivate) opt.ColSet // in turn trigger the PruneMutationInputCols rule. func (c *CustomFuncs) NeededMutationCols( private *memo.MutationPrivate, uniqueChecks memo.UniqueChecksExpr, fkChecks memo.FKChecksExpr, +) opt.ColSet { + return c.neededMutationCols(private, uniqueChecks, fkChecks, true /* includePartialIndexCols */) +} + +func (c *CustomFuncs) neededMutationCols( + private *memo.MutationPrivate, + uniqueChecks memo.UniqueChecksExpr, + fkChecks memo.FKChecksExpr, + includePartialIndexCols bool, ) opt.ColSet { var cols opt.ColSet @@ -60,8 +69,10 @@ func (c *CustomFuncs) NeededMutationCols( addCols(private.FetchCols) addCols(private.UpdateCols) addCols(private.CheckCols) - addCols(private.PartialIndexPutCols) - addCols(private.PartialIndexDelCols) + if includePartialIndexCols { + addCols(private.PartialIndexPutCols) + addCols(private.PartialIndexDelCols) + } addCols(private.ReturnCols) addCols(opt.OptionalColList(private.PassthroughCols)) if private.CanaryCol != 0 { @@ -89,17 +100,8 @@ func (c *CustomFuncs) NeededMutationCols( func (c *CustomFuncs) NeededMutationFetchCols( op opt.Operator, private *memo.MutationPrivate, ) opt.ColSet { - return neededMutationFetchCols(c.mem, op, private) -} - -// neededMutationFetchCols returns the set of columns needed by the given -// mutation operator. -func neededMutationFetchCols( - mem *memo.Memo, op opt.Operator, private *memo.MutationPrivate, -) opt.ColSet { - var cols opt.ColSet - tabMeta := mem.Metadata().TableMeta(private.Table) + tabMeta := c.mem.Metadata().TableMeta(private.Table) // familyCols returns the columns in the given family. familyCols := func(fam cat.Family) opt.ColSet { @@ -153,26 +155,26 @@ func neededMutationFetchCols( // Make sure to consider indexes that are being added or dropped. for i, n := 0, tabMeta.Table.DeletableIndexCount(); i < n; i++ { - // If the columns being updated are not part of the index and the - // index is not a partial index, then the update does not require - // changes to the index. Partial indexes may be updated (even when a - // column in the index is not changing) when rows that were not - // previously in the index must be added to the index because they - // now satisfy the partial index predicate. + // If the columns being updated are not part of the index, then the + // update does not require changes to the index. Partial indexes may + // be updated (even when a column in the index is not changing) when + // the predicate references columns that are being updated. For + // example, rows that were not previously in the index must be added + // to the index because they now satisfy the partial index + // predicate, requiring the index columns to be fetched. // // Note that we use the set of index columns where the virtual // columns have been mapped to their source columns. Virtual columns // are never part of the updated columns. Updates to source columns // trigger index changes. - // - // TODO(mgartner): Index columns are not necessary when neither the - // index columns nor the columns referenced in the partial index - // predicate are being updated. We should prune mutation fetch - // columns when this is the case, rather than always marking index - // columns of partial indexes as "needed". indexCols := tabMeta.IndexColumnsMapVirtual(i) - _, isPartialIndex := tabMeta.Table.Index(i).Predicate() - if !indexCols.Intersects(updateCols) && !isPartialIndex { + pred, isPartialIndex := tabMeta.PartialIndexPredicate(i) + indexAndPredCols := indexCols.Copy() + if isPartialIndex { + predFilters := *pred.(*memo.FiltersExpr) + indexAndPredCols.UnionWith(predFilters.OuterCols()) + } + if !indexAndPredCols.Intersects(updateCols) { continue } @@ -579,17 +581,6 @@ func DerivePruneCols(e memo.RelExpr) opt.ColSet { relProps.Rule.PruneCols.DifferenceWith(w.ScalarProps().OuterCols) } - case opt.UpdateOp, opt.UpsertOp, opt.DeleteOp: - // Find the columns that would need to be fetched, if no returning - // clause were present. - withoutReturningPrivate := *e.Private().(*memo.MutationPrivate) - withoutReturningPrivate.ReturnCols = opt.OptionalColList{} - neededCols := neededMutationFetchCols(e.Memo(), e.Op(), &withoutReturningPrivate) - - // Only the "free" RETURNING columns can be pruned away (i.e. the columns - // required by the mutation only because they're being returned). - relProps.Rule.PruneCols = relProps.OutputCols.Difference(neededCols) - case opt.WithOp: // WithOp passes through its input unchanged, so it has the same pruning // characteristics as its input. diff --git a/pkg/sql/opt/norm/reject_nulls_funcs.go b/pkg/sql/opt/norm/reject_nulls_funcs.go index 8ce83632baf1..b75eea8de5d9 100644 --- a/pkg/sql/opt/norm/reject_nulls_funcs.go +++ b/pkg/sql/opt/norm/reject_nulls_funcs.go @@ -331,9 +331,11 @@ func deriveScanRejectNullCols(in memo.RelExpr) opt.ColSet { scan := in.(*memo.ScanExpr) var rejectNullCols opt.ColSet - for _, pred := range md.TableMeta(scan.Table).PartialIndexPredicates { - predFilters := *pred.(*memo.FiltersExpr) - rejectNullCols.UnionWith(isNotNullCols(predFilters)) + for i, n := 0, md.Table(scan.Table).IndexCount(); i < n; i++ { + if pred, isPartialIndex := md.TableMeta(scan.Table).PartialIndexPredicate(i); isPartialIndex { + predFilters := *pred.(*memo.FiltersExpr) + rejectNullCols.UnionWith(isNotNullCols(predFilters)) + } } return rejectNullCols diff --git a/pkg/sql/opt/norm/rules/mutation.opt b/pkg/sql/opt/norm/rules/mutation.opt index e69de29bb2d1..e66c33278474 100644 --- a/pkg/sql/opt/norm/rules/mutation.opt +++ b/pkg/sql/opt/norm/rules/mutation.opt @@ -0,0 +1,38 @@ +# ============================================================================= +# mutation.opt contains normalization rules for the mutation operators. +# ============================================================================= + +# SimplifyPartialIndexProjections converts partial index PUT and DEL projected +# expressions to false when it is guaranteed that the mutation will not require +# changes to the associated partial index. These projected expressions can only +# be simplified to false when an UPDATE mutates neither the associated index's +# columns nor the columns referenced in the partial index predicate. +[SimplifyPartialIndexProjections, Normalize] +(Update + $project:(Project $input:* $projections:* $passthrough:*) + $uniqueChecks:* + $fkChecks:* + $mutationPrivate:* & + ^(ColsAreEmpty + $simplifiableCols:(SimplifiablePartialIndexProjectCols + $mutationPrivate + $uniqueChecks + $fkChecks + $projections + ) + ) +) +=> +(Update + (Project + $input + (SimplifyPartialIndexProjections + $projections + $simplifiableCols + ) + $passthrough + ) + $uniqueChecks + $fkChecks + $mutationPrivate +) diff --git a/pkg/sql/opt/norm/testdata/rules/mutation b/pkg/sql/opt/norm/testdata/rules/mutation new file mode 100644 index 000000000000..5c4962779360 --- /dev/null +++ b/pkg/sql/opt/norm/testdata/rules/mutation @@ -0,0 +1,258 @@ +# -------------------------------------------------- +# SimplifyPartialIndexProjections +# -------------------------------------------------- + +exec-ddl +CREATE TABLE t ( + k INT PRIMARY KEY, + a INT, + b INT, + c INT, + d INT, + e INT, + f INT, + g INT, + h BOOL, + INDEX (a), + INDEX (c) WHERE d > 1, + INDEX (e) WHERE f > 1 AND g > 1, + INDEX (b), + INDEX (d) WHERE c > 1 +) +---- + +# Simplify UPDATE partial index put/del column to false when the indexed columns +# and columns referenced in predicates are not mutating. +norm expect=SimplifyPartialIndexProjections +UPDATE t SET a = 2, b = 2 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── a_new:21 => a:2 + │ └── a_new:21 => b:3 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── partial index del columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22!null partial_index_put2:23!null partial_index_put3:24!null a_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-24) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── false [as=partial_index_put1:22] + ├── false [as=partial_index_put2:23] + ├── false [as=partial_index_put3:24] + └── 2 [as=a_new:21] + +# Simplify UPDATE partial index put/del column to false for second partial index +# only. +norm expect=SimplifyPartialIndexProjections +UPDATE t SET a = 2, d = 2 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── a_new:21 => a:2 + │ └── a_new:21 => d:5 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:24 partial_index_put3:25 + ├── partial index del columns: partial_index_del1:23 partial_index_put2:24 partial_index_put3:25 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22!null partial_index_del1:23 partial_index_put2:24!null partial_index_put3:25 a_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-25) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── true [as=partial_index_put1:22] + ├── d:15 > 1 [as=partial_index_del1:23, outer=(15)] + ├── false [as=partial_index_put2:24] + ├── c:14 > 1 [as=partial_index_put3:25, outer=(14)] + └── 2 [as=a_new:21] + +# Do not simplify partial index put/del column to false when the indexed columns +# are mutating. +norm expect-not=SimplifyPartialIndexProjections +UPDATE t SET c = 1, e = 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── c_new:21 => c:4 + │ └── c_new:21 => e:6 + ├── partial index put columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24 + ├── partial index del columns: partial_index_put1:22 partial_index_put2:23 partial_index_del3:25 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:22 partial_index_put2:23 partial_index_put3:24!null partial_index_del3:25 c_new:21!null k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-25) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── d:15 > 1 [as=partial_index_put1:22, outer=(15)] + ├── (f:17 > 1) AND (g:18 > 1) [as=partial_index_put2:23, outer=(17,18)] + ├── false [as=partial_index_put3:24] + ├── c:14 > 1 [as=partial_index_del3:25, outer=(14)] + └── 1 [as=c_new:21] + +# Do not simplify partial index put/del column to false when the columns +# referenced in partial index predicates are mutating. +norm expect-not=SimplifyPartialIndexProjections +UPDATE t SET d = d + 1, g = g + 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ ├── d_new:21 => d:5 + │ └── g_new:22 => g:8 + ├── partial index put columns: partial_index_put1:23 partial_index_put2:25 partial_index_put3:27 + ├── partial index del columns: partial_index_del1:24 partial_index_del2:26 partial_index_put3:27 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:23 partial_index_del1:24 partial_index_put2:25 partial_index_del2:26 partial_index_put3:27 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 d_new:21 g_new:22 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(11-19,21-27) + ├── project + │ ├── columns: d_new:21 g_new:22 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── immutable + │ ├── key: () + │ ├── fd: ()-->(11-19,21,22) + │ ├── select + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(11-19) + │ │ ├── scan t + │ │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ │ ├── partial index predicates + │ │ │ │ ├── secondary: filters + │ │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ │ ├── secondary: filters + │ │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ │ └── secondary: filters + │ │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ │ ├── key: (11) + │ │ │ └── fd: (11)-->(12-19) + │ │ └── filters + │ │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + │ └── projections + │ ├── d:15 + 1 [as=d_new:21, outer=(15), immutable] + │ └── g:18 + 1 [as=g_new:22, outer=(18), immutable] + └── projections + ├── d_new:21 > 1 [as=partial_index_put1:23, outer=(21)] + ├── d:15 > 1 [as=partial_index_del1:24, outer=(15)] + ├── (f:17 > 1) AND (g_new:22 > 1) [as=partial_index_put2:25, outer=(17,22)] + ├── (f:17 > 1) AND (g:18 > 1) [as=partial_index_del2:26, outer=(17,18)] + └── c:14 > 1 [as=partial_index_put3:27, outer=(14)] + +# Do not simplify partial index put/del column to false when it is also an +# update column (h_new). +norm +UPDATE t SET h = d > 1 WHERE k = 1 +---- +update t + ├── columns: + ├── fetch columns: k:11 a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── update-mapping: + │ └── h_new:21 => h:9 + ├── partial index put columns: h_new:21 partial_index_put2:22 partial_index_put3:23 + ├── partial index del columns: h_new:21 partial_index_put2:22 partial_index_put3:23 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put2:22!null partial_index_put3:23!null h_new:21 k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(11-19,21-23) + ├── select + │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(11-19) + │ ├── scan t + │ │ ├── columns: k:11!null a:12 b:13 c:14 d:15 e:16 f:17 g:18 h:19 + │ │ ├── partial index predicates + │ │ │ ├── secondary: filters + │ │ │ │ └── d:15 > 1 [outer=(15), constraints=(/15: [/2 - ]; tight)] + │ │ │ ├── secondary: filters + │ │ │ │ ├── f:17 > 1 [outer=(17), constraints=(/17: [/2 - ]; tight)] + │ │ │ │ └── g:18 > 1 [outer=(18), constraints=(/18: [/2 - ]; tight)] + │ │ │ └── secondary: filters + │ │ │ └── c:14 > 1 [outer=(14), constraints=(/14: [/2 - ]; tight)] + │ │ ├── key: (11) + │ │ └── fd: (11)-->(12-19) + │ └── filters + │ └── k:11 = 1 [outer=(11), constraints=(/11: [/1 - /1]; tight), fd=()-->(11)] + └── projections + ├── false [as=partial_index_put2:22] + ├── false [as=partial_index_put3:23] + └── d:15 > 1 [as=h_new:21, outer=(15)] diff --git a/pkg/sql/opt/norm/testdata/rules/prune_cols b/pkg/sql/opt/norm/testdata/rules/prune_cols index bde5476683f1..d07165b60127 100644 --- a/pkg/sql/opt/norm/testdata/rules/prune_cols +++ b/pkg/sql/opt/norm/testdata/rules/prune_cols @@ -48,9 +48,11 @@ CREATE TABLE partial_indexes ( a INT PRIMARY KEY, b INT, c STRING, + d INT, FAMILY (a), FAMILY (b), FAMILY (c), + FAMILY (d), INDEX (c) WHERE b > 1 ) ---- @@ -2058,51 +2060,404 @@ update computed ├── c_new:13 + 1 [as=column14:14, outer=(13), immutable] └── c_new:13 + 10 [as=column15:15, outer=(13), immutable] +# Prune UPDATE fetch columns when the partial index indexes the column but +# neither the column nor the columns referenced in the partial index predicate +# are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +UPDATE partial_indexes SET d = d + 1 WHERE a = 1 +---- +update partial_indexes + ├── columns: + ├── fetch columns: a:6 d:9 + ├── update-mapping: + │ └── d_new:11 => d:4 + ├── partial index put columns: partial_index_put1:12 + ├── partial index del columns: partial_index_put1:12 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:12!null d_new:11 a:6!null d:9 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(6,9,11,12) + ├── select + │ ├── columns: a:6!null d:9 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6,9) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null d:9 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(9) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + ├── false [as=partial_index_put1:12] + └── d:9 + 1 [as=d_new:11, outer=(9), immutable] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +UPDATE partial_indexes SET d = d + 1, b = 2 WHERE a = 1 +---- +update partial_indexes + ├── columns: + ├── fetch columns: a:6 b:7 c:8 d:9 + ├── update-mapping: + │ ├── b_new:12 => b:2 + │ └── d_new:11 => d:4 + ├── partial index put columns: partial_index_put1:13 + ├── partial index del columns: partial_index_del1:14 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:13!null partial_index_del1:14 d_new:11 b_new:12!null a:6!null b:7 c:8 d:9 + ├── cardinality: [0 - 1] + ├── immutable + ├── key: () + ├── fd: ()-->(6-9,11-14) + ├── select + │ ├── columns: a:6!null b:7 c:8 d:9 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-9) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null b:7 c:8 d:9 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(7-9) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + ├── true [as=partial_index_put1:13] + ├── b:7 > 1 [as=partial_index_del1:14, outer=(7)] + ├── d:9 + 1 [as=d_new:11, outer=(9), immutable] + └── 2 [as=b_new:12] + +# Prune UPSERT fetch columns when the partial index indexes the column but +# neither the column nor the columns referenced in the partial index predicate +# are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +UPSERT INTO partial_indexes (a, d) VALUES (1, 2) +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ └── column2:7 => d:4 + ├── partial index put columns: partial_index_put1:18 + ├── partial index del columns: partial_index_del1:19 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:18 partial_index_del1:19 column1:6!null column2:7!null column8:8 column9:9 a:10 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-10,13,18,19) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-11,13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, NULL, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10,11,13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11,13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE b:11 END > 1 [as=partial_index_put1:18, outer=(8,10,11)] + └── b:11 > 1 [as=partial_index_del1:19, outer=(11)] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +UPSERT INTO partial_indexes (a, b, d) VALUES (1, 2, 3) +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 b:11 c:12 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column2:7 => b:2 + │ ├── column9:9 => c:3 + │ └── column3:8 => d:4 + ├── update-mapping: + │ ├── column2:7 => b:2 + │ └── column3:8 => d:4 + ├── partial index put columns: partial_index_put1:17 + ├── partial index del columns: partial_index_del1:18 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:17!null partial_index_del1:18 column1:6!null column2:7!null column3:8!null column9:9 a:10 b:11 c:12 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-13,17,18) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column3:8!null column9:9 a:10 b:11 c:12 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column3:8!null column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, 3, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10-13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11-13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── column2:7 > 1 [as=partial_index_put1:17, outer=(7)] + └── b:11 > 1 [as=partial_index_del1:18, outer=(11)] + +# Prune INSERT ON CONFLICT DO UPDATE fetch columns when the partial index +# indexes the column but neither the column nor the columns referenced in the +# partial index predicate are mutating. +norm expect=(PruneMutationFetchCols,PruneMutationInputCols) +INSERT INTO partial_indexes (a, d) VALUES (1, 2) ON CONFLICT (a) DO UPDATE SET d = 3 +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ └── upsert_d:19 => d:4 + ├── partial index put columns: partial_index_put1:20 + ├── partial index del columns: partial_index_del1:21 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:20 partial_index_del1:21 upsert_d:19!null column1:6!null column2:7!null column8:8 column9:9 a:10 d:13 + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-10,13,19-21) + ├── left-join (cross) + │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 d:13 + │ ├── cardinality: [1 - 1] + │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ ├── key: () + │ ├── fd: ()-->(6-11,13) + │ ├── values + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ ├── cardinality: [1 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(6-9) + │ │ └── (1, 2, NULL, NULL) + │ ├── select + │ │ ├── columns: a:10!null b:11 d:13 + │ │ ├── cardinality: [0 - 1] + │ │ ├── key: () + │ │ ├── fd: ()-->(10,11,13) + │ │ ├── scan partial_indexes + │ │ │ ├── columns: a:10!null b:11 d:13 + │ │ │ ├── partial index predicates + │ │ │ │ └── secondary: filters + │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ ├── key: (10) + │ │ │ └── fd: (10)-->(11,13) + │ │ └── filters + │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ └── filters (true) + └── projections + ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE b:11 END > 1 [as=partial_index_put1:20, outer=(8,10,11)] + ├── b:11 > 1 [as=partial_index_del1:21, outer=(11)] + └── CASE WHEN a:10 IS NULL THEN column2:7 ELSE 3 END [as=upsert_d:19, outer=(7,10)] + +# Do not prune the indexed column c when a column in the partial index +# predicate, b, is being updated. +norm expect-not=PruneMutationFetchCols +INSERT INTO partial_indexes (a, d) VALUES (1, 2) ON CONFLICT (a) DO UPDATE SET b = 3, d = 4 +---- +upsert partial_indexes + ├── columns: + ├── arbiter indexes: primary + ├── canary column: a:10 + ├── fetch columns: a:10 b:11 c:12 d:13 + ├── insert-mapping: + │ ├── column1:6 => a:1 + │ ├── column8:8 => b:2 + │ ├── column9:9 => c:3 + │ └── column2:7 => d:4 + ├── update-mapping: + │ ├── upsert_b:18 => b:2 + │ └── upsert_d:20 => d:4 + ├── partial index put columns: partial_index_put1:21 + ├── partial index del columns: partial_index_del1:22 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_put1:21 partial_index_del1:22 column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 upsert_b:18 upsert_d:20!null + ├── cardinality: [1 - 1] + ├── key: () + ├── fd: ()-->(6-13,18,20-22) + ├── project + │ ├── columns: upsert_b:18 upsert_d:20!null column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 + │ ├── cardinality: [1 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-13,18,20) + │ ├── left-join (cross) + │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 a:10 b:11 c:12 d:13 + │ │ ├── cardinality: [1 - 1] + │ │ ├── multiplicity: left-rows(exactly-one), right-rows(exactly-one) + │ │ ├── key: () + │ │ ├── fd: ()-->(6-13) + │ │ ├── values + │ │ │ ├── columns: column1:6!null column2:7!null column8:8 column9:9 + │ │ │ ├── cardinality: [1 - 1] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(6-9) + │ │ │ └── (1, 2, NULL, NULL) + │ │ ├── select + │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ ├── cardinality: [0 - 1] + │ │ │ ├── key: () + │ │ │ ├── fd: ()-->(10-13) + │ │ │ ├── scan partial_indexes + │ │ │ │ ├── columns: a:10!null b:11 c:12 d:13 + │ │ │ │ ├── partial index predicates + │ │ │ │ │ └── secondary: filters + │ │ │ │ │ └── b:11 > 1 [outer=(11), constraints=(/11: [/2 - ]; tight)] + │ │ │ │ ├── key: (10) + │ │ │ │ └── fd: (10)-->(11-13) + │ │ │ └── filters + │ │ │ └── a:10 = 1 [outer=(10), constraints=(/10: [/1 - /1]; tight), fd=()-->(10)] + │ │ └── filters (true) + │ └── projections + │ ├── CASE WHEN a:10 IS NULL THEN column8:8 ELSE 3 END [as=upsert_b:18, outer=(8,10)] + │ └── CASE WHEN a:10 IS NULL THEN column2:7 ELSE 4 END [as=upsert_d:20, outer=(7,10)] + └── projections + ├── upsert_b:18 > 1 [as=partial_index_put1:21, outer=(18)] + └── b:11 > 1 [as=partial_index_del1:22, outer=(11)] + +# Do not prune DELETE fetch columns. +norm +DELETE FROM partial_indexes WHERE a = 1 +---- +delete partial_indexes + ├── columns: + ├── fetch columns: a:6 c:8 + ├── partial index del columns: partial_index_del1:11 + ├── cardinality: [0 - 0] + ├── volatile, mutations + └── project + ├── columns: partial_index_del1:11 a:6!null c:8 + ├── cardinality: [0 - 1] + ├── key: () + ├── fd: ()-->(6,8,11) + ├── select + │ ├── columns: a:6!null b:7 c:8 + │ ├── cardinality: [0 - 1] + │ ├── key: () + │ ├── fd: ()-->(6-8) + │ ├── scan partial_indexes + │ │ ├── columns: a:6!null b:7 c:8 + │ │ ├── partial index predicates + │ │ │ └── secondary: filters + │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ ├── key: (6) + │ │ └── fd: (6)-->(7,8) + │ └── filters + │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] + └── projections + └── b:7 > 1 [as=partial_index_del1:11, outer=(7)] + # Do not prune columns that are required for evaluating partial index # predicates. -norm expect-not=PruneMutationFetchCols +norm UPDATE partial_indexes SET b = b + 1 WHERE a = 1 ---- update partial_indexes ├── columns: - ├── fetch columns: a:5 b:6 c:7 + ├── fetch columns: a:6 b:7 c:8 ├── update-mapping: - │ └── b_new:9 => b:2 - ├── partial index put columns: partial_index_put1:10 - ├── partial index del columns: partial_index_del1:11 + │ └── b_new:11 => b:2 + ├── partial index put columns: partial_index_put1:12 + ├── partial index del columns: partial_index_del1:13 ├── cardinality: [0 - 0] ├── volatile, mutations └── project - ├── columns: partial_index_put1:10 partial_index_del1:11 a:5!null b:6 c:7 b_new:9 + ├── columns: partial_index_put1:12 partial_index_del1:13 a:6!null b:7 c:8 b_new:11 ├── cardinality: [0 - 1] ├── immutable ├── key: () - ├── fd: ()-->(5-7,9-11) + ├── fd: ()-->(6-8,11-13) ├── project - │ ├── columns: b_new:9 a:5!null b:6 c:7 + │ ├── columns: b_new:11 a:6!null b:7 c:8 │ ├── cardinality: [0 - 1] │ ├── immutable │ ├── key: () - │ ├── fd: ()-->(5-7,9) + │ ├── fd: ()-->(6-8,11) │ ├── select - │ │ ├── columns: a:5!null b:6 c:7 + │ │ ├── columns: a:6!null b:7 c:8 │ │ ├── cardinality: [0 - 1] │ │ ├── key: () - │ │ ├── fd: ()-->(5-7) + │ │ ├── fd: ()-->(6-8) │ │ ├── scan partial_indexes - │ │ │ ├── columns: a:5!null b:6 c:7 + │ │ │ ├── columns: a:6!null b:7 c:8 │ │ │ ├── partial index predicates │ │ │ │ └── secondary: filters - │ │ │ │ └── b:6 > 1 [outer=(6), constraints=(/6: [/2 - ]; tight)] - │ │ │ ├── key: (5) - │ │ │ └── fd: (5)-->(6,7) + │ │ │ │ └── b:7 > 1 [outer=(7), constraints=(/7: [/2 - ]; tight)] + │ │ │ ├── key: (6) + │ │ │ └── fd: (6)-->(7,8) │ │ └── filters - │ │ └── a:5 = 1 [outer=(5), constraints=(/5: [/1 - /1]; tight), fd=()-->(5)] + │ │ └── a:6 = 1 [outer=(6), constraints=(/6: [/1 - /1]; tight), fd=()-->(6)] │ └── projections - │ └── b:6 + 1 [as=b_new:9, outer=(6), immutable] + │ └── b:7 + 1 [as=b_new:11, outer=(7), immutable] └── projections - ├── b_new:9 > 1 [as=partial_index_put1:10, outer=(9)] - └── b:6 > 1 [as=partial_index_del1:11, outer=(6)] + ├── b_new:11 > 1 [as=partial_index_put1:12, outer=(11)] + └── b:7 > 1 [as=partial_index_del1:13, outer=(7)] # Prune secondary family column not needed for the update. norm expect=(PruneMutationFetchCols,PruneMutationInputCols) diff --git a/pkg/sql/opt/optbuilder/insert.go b/pkg/sql/opt/optbuilder/insert.go index aa929e258714..cdf095caad30 100644 --- a/pkg/sql/opt/optbuilder/insert.go +++ b/pkg/sql/opt/optbuilder/insert.go @@ -1061,6 +1061,11 @@ func (mb *mutationBuilder) buildUpsert(returning tree.ReturningExprs) { // Add any check constraint boolean columns to the input. mb.addCheckConstraintCols() + // Add the partial index predicate expressions to the table metadata. + // These expressions are used to prune fetch columns during + // normalization. + mb.b.addPartialIndexPredicatesForTable(mb.md.TableMeta(mb.tabID), nil /* scan */, true /* includeDeletable */) + // Project partial index PUT and DEL boolean columns. // // In some cases existing rows may not be fetched for an UPSERT (see diff --git a/pkg/sql/opt/optbuilder/partial_index.go b/pkg/sql/opt/optbuilder/partial_index.go index 9e78103943fb..c60cc64136c7 100644 --- a/pkg/sql/opt/optbuilder/partial_index.go +++ b/pkg/sql/opt/optbuilder/partial_index.go @@ -13,11 +13,93 @@ package optbuilder import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" + "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" "github.com/cockroachdb/errors" ) +// addPartialIndexPredicatesForTable finds all partial indexes in the table and +// adds their predicates to the table metadata (see +// TableMeta.partialIndexPredicates). The predicates are converted from strings +// to ScalarExprs here. +// +// The predicates are used as "known truths" about table data. Any predicates +// containing non-immutable operators are omitted. +// +// scan is an optional argument that is a Scan expression on the table. If scan +// outputs all the ordinary columns in the table, we avoid constructing a new +// scan. A scan and its logical properties are required in order to fully +// normalize the partial index predicates. +func (b *Builder) addPartialIndexPredicatesForTable( + tabMeta *opt.TableMeta, scan memo.RelExpr, includeDeletable bool, +) { + tab := tabMeta.Table + var numIndexes int + if includeDeletable { + numIndexes = tab.DeletableIndexCount() + } else { + numIndexes = tab.IndexCount() + } + + // Find the first partial index. + indexOrd := 0 + for ; indexOrd < numIndexes; indexOrd++ { + if _, ok := tab.Index(indexOrd).Predicate(); ok { + break + } + } + + // Return early if there are no partial indexes. Only partial indexes have + // predicates. + if indexOrd == numIndexes { + return + } + + // Construct a scan as the tableScope expr so that logical properties of the + // scan can be used to fully normalize the index predicate. + tableScope := b.allocScope() + tableScope.appendOrdinaryColumnsFromTable(tabMeta, &tabMeta.Alias) + + // If the optional scan argument was provided and it outputs all of the + // ordinary table columns, we use it as tableScope.expr. Otherwise, we must + // construct a new scan. Attaching a scan to tableScope.expr is required to + // fully normalize the partial index predicates with logical properties of + // the scan. + if scan != nil && tableScope.colSet().SubsetOf(scan.Relational().OutputCols) { + tableScope.expr = scan + } else { + tableScope.expr = b.factory.ConstructScan(&memo.ScanPrivate{ + Table: tabMeta.MetaID, + Cols: tableScope.colSet(), + }) + } + + // Skip to the first partial index we found above. + for ; indexOrd < numIndexes; indexOrd++ { + index := tab.Index(indexOrd) + pred, ok := index.Predicate() + + // If the index is not a partial index, do nothing. + if !ok { + continue + } + + expr, err := parser.ParseExpr(pred) + if err != nil { + panic(err) + } + + // Build the partial index predicate as a memo.FiltersExpr and add it + // to the table metadata. + predExpr, err := b.buildPartialIndexPredicate(tableScope, expr, "index predicate") + if err != nil { + panic(err) + } + tabMeta.AddPartialIndexPredicate(indexOrd, &predExpr) + } +} + // buildPartialIndexPredicate builds a memo.FiltersExpr from the given // tree.Expr. The expression must be of type Bool and it must be immutable. // Returns an error if any non-immutable operators are found. @@ -25,7 +107,7 @@ import ( // Note: This function should only be used to build partial index or arbiter // predicate expressions that have only a table's ordinary columns in scope and // that are not part of the relational expression tree. For example, this is -// used to populate the TableMeta.PartialIndexPredicates cache and for +// used to populate the partial index predicates map in TableMeta and for // determining arbiter indexes in UPSERT and INSERT ON CONFLICT mutations. But // it is not used for building synthesized mutation columns that determine // whether to issue PUT or DEL operations on a partial index for a mutated row; diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 6604913ce782..3a74a991df95 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -536,9 +536,10 @@ func (b *Builder) buildScan( outScope.expr = b.factory.ConstructScan(&private) // Add the partial indexes after constructing the scan so we can use the - // logical properties of the scan to fully normalize the index - // predicates. - b.addPartialIndexPredicatesForTable(tabMeta, outScope.expr) + // logical properties of the scan to fully normalize the index predicates. + // We don't need to add deletable partial index predicates in the context of + // a scan. + b.addPartialIndexPredicatesForTable(tabMeta, outScope.expr, false /* includeDeletable */) if !virtualColIDs.Empty() { // Project the expressions for the virtual columns (and pass through all @@ -688,80 +689,6 @@ func (b *Builder) addComputedColsForTable(tabMeta *opt.TableMeta) { } } -// addPartialIndexPredicatesForTable finds all partial indexes in the table and -// adds their predicates to the table metadata (see -// TableMeta.PartialIndexPredicates). The predicates are converted from strings -// to ScalarExprs here. -// -// The predicates are used as "known truths" about table data. Any predicates -// containing non-immutable operators are omitted. -// -// scan is an optional argument that is a Scan expression on the table. If scan -// outputs all the ordinary columns in the table, we avoid constructing a new -// scan. A scan and its logical properties are required in order to fully -// normalize the partial index predicates. -func (b *Builder) addPartialIndexPredicatesForTable(tabMeta *opt.TableMeta, scan memo.RelExpr) { - tab := tabMeta.Table - - // Find the first partial index. - numIndexes := tab.IndexCount() - indexOrd := 0 - for ; indexOrd < numIndexes; indexOrd++ { - if _, ok := tab.Index(indexOrd).Predicate(); ok { - break - } - } - - // Return early if there are no partial indexes. Only partial indexes have - // predicates. - if indexOrd == numIndexes { - return - } - - // Construct a scan as the tableScope expr so that logical properties of the - // scan can be used to fully normalize the index predicate. - tableScope := b.allocScope() - tableScope.appendOrdinaryColumnsFromTable(tabMeta, &tabMeta.Alias) - - // If the optional scan argument was provided and it outputs all of the - // ordinary table columns, we use it as tableScope.expr. Otherwise, we must - // construct a new scan. Attaching a scan to tableScope.expr is required to - // fully normalize the partial index predicates with logical properties of - // the scan. - if scan != nil && tableScope.colSet().SubsetOf(scan.Relational().OutputCols) { - tableScope.expr = scan - } else { - tableScope.expr = b.factory.ConstructScan(&memo.ScanPrivate{ - Table: tabMeta.MetaID, - Cols: tableScope.colSet(), - }) - } - - // Skip to the first partial index we found above. - for ; indexOrd < numIndexes; indexOrd++ { - index := tab.Index(indexOrd) - pred, ok := index.Predicate() - - // If the index is not a partial index, do nothing. - if !ok { - continue - } - - expr, err := parser.ParseExpr(pred) - if err != nil { - panic(err) - } - - // Build the partial index predicate as a memo.FiltersExpr and add it - // to the table metadata. - predExpr, err := b.buildPartialIndexPredicate(tableScope, expr, "index predicate") - if err != nil { - panic(err) - } - tabMeta.AddPartialIndexPredicate(indexOrd, &predExpr) - } -} - func (b *Builder) buildSequenceSelect( seq cat.Sequence, seqName *tree.TableName, inScope *scope, ) (outScope *scope) { diff --git a/pkg/sql/opt/optbuilder/update.go b/pkg/sql/opt/optbuilder/update.go index e91158ef185a..a38f3370a795 100644 --- a/pkg/sql/opt/optbuilder/update.go +++ b/pkg/sql/opt/optbuilder/update.go @@ -336,6 +336,11 @@ func (mb *mutationBuilder) buildUpdate(returning tree.ReturningExprs) { mb.addCheckConstraintCols() + // Add the partial index predicate expressions to the table metadata. + // These expressions are used to prune fetch columns during + // normalization. + mb.b.addPartialIndexPredicatesForTable(mb.md.TableMeta(mb.tabID), nil /* scan */, true /* includeDeletable */) + // Project partial index PUT and DEL boolean columns. mb.projectPartialIndexPutAndDelCols(preCheckScope, mb.fetchScope) diff --git a/pkg/sql/opt/table_meta.go b/pkg/sql/opt/table_meta.go index 35ada6385979..c9c67b92f768 100644 --- a/pkg/sql/opt/table_meta.go +++ b/pkg/sql/opt/table_meta.go @@ -156,11 +156,11 @@ type TableMeta struct { // Computed columns with non-immutable operators are omitted. ComputedCols map[ColumnID]ScalarExpr - // PartialIndexPredicates is a map from index ordinals on the table to + // partialIndexPredicates is a map from index ordinals on the table to // *FiltersExprs representing the predicate on the corresponding partial // index. If an index is not a partial index, it will not have an entry in // the map. - PartialIndexPredicates map[cat.IndexOrdinal]ScalarExpr + partialIndexPredicates map[cat.IndexOrdinal]ScalarExpr // anns annotates the table metadata with arbitrary data. anns [maxTableAnnIDCount]interface{} @@ -254,10 +254,36 @@ func (tm *TableMeta) AddComputedCol(colID ColumnID, computedCol ScalarExpr) { // AddPartialIndexPredicate adds a partial index predicate to the table's // metadata. func (tm *TableMeta) AddPartialIndexPredicate(ord cat.IndexOrdinal, pred ScalarExpr) { - if tm.PartialIndexPredicates == nil { - tm.PartialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr) + if tm.partialIndexPredicates == nil { + tm.partialIndexPredicates = make(map[cat.IndexOrdinal]ScalarExpr) } - tm.PartialIndexPredicates[ord] = pred + tm.partialIndexPredicates[ord] = pred +} + +// PartialIndexPredicate returns the given index's predicate scalar expression, +// if the index is a partial index. Returns ok=false if the index is not a +// partial index. Panics if the index is a partial index but a predicate scalar +// expression does not exist in the table metadata. +func (tm *TableMeta) PartialIndexPredicate(ord cat.IndexOrdinal) (pred ScalarExpr, ok bool) { + if _, isPartialIndex := tm.Table.Index(ord).Predicate(); !isPartialIndex { + return nil, false + } + pred, ok = tm.partialIndexPredicates[ord] + if !ok { + panic(errors.AssertionFailedf("partial index predicate does not exist in table metadata")) + } + return pred, true +} + +// PartialIndexPredicatesForFormattingOnly returns the partialIndexPredicates +// map. +// +// WARNING: The returned map is NOT a source-of-truth for determining if an +// index is a partial index. This function should only be used to show the +// partial index expressions that have been built for a table when formatting +// opt expressions. Use PartialIndexPredicate in all other cases. +func (tm *TableMeta) PartialIndexPredicatesForFormattingOnly() map[cat.IndexOrdinal]ScalarExpr { + return tm.partialIndexPredicates } // TableAnnotation returns the given annotation that is associated with the diff --git a/pkg/sql/opt/xform/scan_index_iter.go b/pkg/sql/opt/xform/scan_index_iter.go index 99aef8acde30..c694369c1aee 100644 --- a/pkg/sql/opt/xform/scan_index_iter.go +++ b/pkg/sql/opt/xform/scan_index_iter.go @@ -213,7 +213,7 @@ func (it *scanIndexIter) ForEachStartingAfter(ord int, f enumerateIndexFunc) { continue } - pred, isPartialIndex := it.tabMeta.PartialIndexPredicates[ord] + pred, isPartialIndex := it.tabMeta.PartialIndexPredicate(ord) // Skip over partial indexes if rejectPartialIndexes is set. if it.hasRejectFlags(rejectPartialIndexes) && isPartialIndex { diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 3a7eea0e7495..69c196d9ef9d 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -1655,7 +1655,7 @@ func (c *CustomFuncs) canMaybeConstrainIndexWithCols( // possible to generate an unconstrained partial index scan, which may // lead to better query plans. if _, isPartialIndex := index.Predicate(); isPartialIndex { - p, ok := tabMeta.PartialIndexPredicates[i] + p, ok := tabMeta.PartialIndexPredicate(i) if !ok { // A partial index predicate expression was not built for the // partial index. See Builder.buildScan for details on when this diff --git a/pkg/sql/row/updater.go b/pkg/sql/row/updater.go index 065480a37ed3..ebb451dce3d9 100644 --- a/pkg/sql/row/updater.go +++ b/pkg/sql/row/updater.go @@ -149,9 +149,6 @@ func MakeUpdater( } } - // Columns of the table to update, including those in delete/write-only state - tableCols := tableDesc.DeletableColumns() - var deleteOnlyIndexes []descpb.IndexDescriptor for _, idx := range tableDesc.DeleteOnlyIndexes() { if needsUpdate(idx) { @@ -172,6 +169,8 @@ func MakeUpdater( ru := Updater{ Helper: newRowHelper(codec, tableDesc, includeIndexes), DeleteHelper: deleteOnlyHelper, + FetchCols: requestedCols, + FetchColIDtoRowIndex: ColIDtoRowIndexFromCols(requestedCols), UpdateCols: updateCols, UpdateColIDtoRowIndex: updateColIDtoRowIndex, primaryKeyColChange: primaryKeyColChange, @@ -185,73 +184,13 @@ func MakeUpdater( // When changing the primary key, we delete the old values and reinsert // them, so request them all. var err error + tableCols := tableDesc.DeletableColumns() ru.rd = MakeDeleter(codec, tableDesc, tableCols) - ru.FetchCols = ru.rd.FetchCols - ru.FetchColIDtoRowIndex = ColIDtoRowIndexFromCols(ru.FetchCols) if ru.ri, err = MakeInserter( ctx, txn, codec, tableDesc, tableCols, alloc, ); err != nil { return Updater{}, err } - } else { - ru.FetchCols = requestedCols[:len(requestedCols):len(requestedCols)] - ru.FetchColIDtoRowIndex = ColIDtoRowIndexFromCols(ru.FetchCols) - - // maybeAddCol adds the provided column to ru.FetchCols and - // ru.FetchColIDtoRowIndex if it isn't already present. - maybeAddCol := func(colID descpb.ColumnID) error { - if _, ok := ru.FetchColIDtoRowIndex.Get(colID); !ok { - col, _, err := tableDesc.FindReadableColumnByID(colID) - if err != nil { - return err - } - ru.FetchColIDtoRowIndex.Set(col.ID, len(ru.FetchCols)) - ru.FetchCols = append(ru.FetchCols, *col) - } - return nil - } - - // Fetch all columns in the primary key so that we can construct the - // keys when writing out the new kvs to the primary index. - for _, colID := range tableDesc.GetPrimaryIndex().ColumnIDs { - if err := maybeAddCol(colID); err != nil { - return Updater{}, err - } - } - - // If any part of a column family is being updated, fetch all columns in - // that column family so that we can reconstruct the column family with - // the updated columns before writing it. - for i := range tableDesc.Families { - family := &tableDesc.Families[i] - familyBeingUpdated := false - for _, colID := range family.ColumnIDs { - if _, ok := ru.UpdateColIDtoRowIndex.Get(colID); ok { - familyBeingUpdated = true - break - } - } - if familyBeingUpdated { - for _, colID := range family.ColumnIDs { - if err := maybeAddCol(colID); err != nil { - return Updater{}, err - } - } - } - } - - // Fetch all columns from indices that are being update so that they can - // be used to create the new kv pairs for those indices. - for _, index := range includeIndexes { - if err := index.RunOverAllColumns(maybeAddCol); err != nil { - return Updater{}, err - } - } - for _, index := range deleteOnlyIndexes { - if err := index.RunOverAllColumns(maybeAddCol); err != nil { - return Updater{}, err - } - } } // If we are fetching from specific families, we might get