Merge #56337 #56350 #56445 #56578

56337: importccl: fixed import mysqldump crashing bug r=adityamaru,pbardea a=mokaixu

After the logic for parsing schemas for bundle formats
got moved from planning to execution phase, import mysqldump
commands caused panics from the walltime being 0.
This was because mysqldump' gets walltime from the job's evalCtx
which when retrieved in the execution phase, is 0 (vs.
the planning phase when evalCtx's walltime is nonzero).

To address this, the logic that initializes walltime was moved from
the execution to the planning phase. This ensures that we have
a consistent walltime that persists in the execution phase,
regardless of whether a job needs to restart/re-resume. This
walltime is then passed using importDetails to the execution phase
where it can be used. The methods responsible for import mysqldump
functionality now retrieve walltime from importDetails instead
of evalCtx.

Release note: None

Fixes #56284 

56350: sql: encode multi-column inverted index key spans r=mgartner a=mgartner

#### opt: fix assumptions about inverted columns of inverted indexes

This commit fixes some bugs caused by the assumption that the first
column of an inverted index is the inverted column. With the addition of
multi-column inverted indexes, the last column is the inverted column,
not the first.

Release note: None

#### sql: move inverted span encoding to span package

This commit moves the logic for generating `roachpb.Spans` from
`invertedexpr.InvertedSpans` from `pkg/sql/inverted_span_encoding.go` to
in`pkg/sql/span/span_builder.go`. Now, the logic for building both
inverted and non-inverted spans lives in the same place, in the
`span.Builder`.

Code in `pkg/sql/rowexec/inverted_joiner.go` also uses this new feature
of the span builder to create `roachpb.Spans` from a slice of
`SpanExpressionProto_Span`, reducing duplicate logic.

Release note: None

#### sql: encode multi-column inverted index key spans

This commit adds support for combining both a Scan's Constraint and
InvertedConstraint into key spans for scanning multi-column inverted
indexes. The Constraint is used to constrain the prefix columns, while
the InvertedConstraint is used to constrain the inverted column.

Release note: None


56445: ui: aggregate new statement stats on the client r=dhartunian a=dhartunian

Previously, we added new stats to the statements proto
without also ensuring that our aggregation of the same
fingerprint across multiple nodes averaged values for
those new fields.

Since our protobuf definitions allow null values, TypeScript
wasn't able to enforce the setting of these new fields
in the aggregation function.

Resolves #56432.

Release note (admin ui change): fixes a bug where the "Other
Execution Statistics" box in the Statement Details page would
be empty in situations where the same fingerprint had been
processed by multiple nodes.

56578: vendor: bump Pebble to 01067adc r=sumeerbhola a=sumeerbhola

01067adc db: add test for ingest idempotence
69de2263 internal/manifest: fix typo in comment for SeekGE
4e7ee20a db: fix Iterator direction switching behavior
7c3b8ea9 Merge pull request #983 from otan-cockroach/go1.15
d87c944f travis: upgrade to go 1.15 for testing

Release note: None

Co-authored-by: Monica Xu <monicax@cockroachlabs.com>
Co-authored-by: Marcus Gartner <marcus@cockroachlabs.com>
Co-authored-by: David Hartunian <davidh@cockroachlabs.com>
Co-authored-by: sumeerbhola <sumeer@cockroachlabs.com>

DEPS.bzl

@@ -440,8 +440,8 @@ def go_deps():
         name = "com_github_cockroachdb_pebble",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/cockroachdb/pebble",
-        sum = "h1:IWH+NluanHz5Emxz8yjJPmoS8fidJM7l0VkY0a10nFs=",
-        version = "v0.0.0-20201102015632-51fc020cf9c0",
+        sum = "h1:QFs0nqYhzdJeEvEIITMhH1UZN4CyZfPn16BCeW9TZEE=",
+        version = "v0.0.0-20201110222031-01067adc85a5",
     )
     go_repository(
         name = "com_github_cockroachdb_redact",

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55
 	github.com/cockroachdb/gostdlib v1.13.0
 	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f
-	github.com/cockroachdb/pebble v0.0.0-20201102015632-51fc020cf9c0
+	github.com/cockroachdb/pebble v0.0.0-20201110222031-01067adc85a5
 	github.com/cockroachdb/redact v1.0.8
 	github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd
 	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2

go.sum

@@ -162,8 +162,8 @@ github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249 h1:pZu
 github.com/cockroachdb/grpc-gateway v1.14.6-0.20200519165156-52697fc4a249/go.mod h1:UJ0EZAp832vCd54Wev9N1BMKEyvcZ5+IM0AwDrnlkEc=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/cockroachdb/pebble v0.0.0-20201102015632-51fc020cf9c0 h1:IWH+NluanHz5Emxz8yjJPmoS8fidJM7l0VkY0a10nFs=
-github.com/cockroachdb/pebble v0.0.0-20201102015632-51fc020cf9c0/go.mod h1:BbtTitvfmE0eZNcncJgJw5BlQhskTzgZgoISnY+8s6k=
+github.com/cockroachdb/pebble v0.0.0-20201110222031-01067adc85a5 h1:QFs0nqYhzdJeEvEIITMhH1UZN4CyZfPn16BCeW9TZEE=
+github.com/cockroachdb/pebble v0.0.0-20201110222031-01067adc85a5/go.mod h1:BbtTitvfmE0eZNcncJgJw5BlQhskTzgZgoISnY+8s6k=
 github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I=
 github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/redact v1.0.6 h1:W34uRRyNR4dlZFd0MibhNELsZSgMkl52uRV/tA1xToY=

pkg/ccl/importccl/import_stmt.go

@@ -1245,7 +1245,7 @@ func parseAndCreateBundleTableDescs(
 	switch format.Format {
 	case roachpb.IOFileFormat_Mysqldump:
 		evalCtx := &p.ExtendedEvalContext().EvalContext
-		tableDescs, err = readMysqlCreateTable(ctx, reader, evalCtx, p, defaultCSVTableID, parentID, tableName, fks, seqVals, owner)
+		tableDescs, err = readMysqlCreateTable(ctx, reader, evalCtx, p, defaultCSVTableID, parentID, tableName, fks, seqVals, owner, walltime)
 	case roachpb.IOFileFormat_PgDump:
 		evalCtx := &p.ExtendedEvalContext().EvalContext
 		tableDescs, err = readPostgresCreateTable(ctx, reader, evalCtx, p, tableName, parentID, walltime, fks, int(format.PgDump.MaxRowSize), owner)
@@ -1367,7 +1367,10 @@ func (r *importResumer) Resume(
 			}
 		}
 	}
-
+	// If details.Walltime is still 0, then it was not set during
+	// `prepareTableDescsForIngestion`. This indicates that we are in an IMPORT INTO,
+	// and that the walltime was not set in a previous run of IMPORT.
+	//
 	// In the case of importing into existing tables we must wait for all nodes
 	// to see the same version of the updated table descriptor, after which we
 	// shall chose a ts to import from.

pkg/ccl/importccl/read_import_mysql.go

@@ -291,6 +291,7 @@ func readMysqlCreateTable(
 	fks fkHandler,
 	seqVals map[descpb.ID]int64,
 	owner security.SQLUsername,
+	walltime int64,
 ) ([]*tabledesc.Mutable, error) {
 	match = lexbase.NormalizeName(match)
 	r := bufio.NewReaderSize(input, 1024*64)
@@ -322,7 +323,7 @@ func readMysqlCreateTable(
 				continue
 			}
 			id := descpb.ID(int(startingID) + len(ret))
-			tbl, moreFKs, err := mysqlTableToCockroach(ctx, evalCtx, p, parentID, id, name, i.TableSpec, fks, seqVals, owner)
+			tbl, moreFKs, err := mysqlTableToCockroach(ctx, evalCtx, p, parentID, id, name, i.TableSpec, fks, seqVals, owner, walltime)
 			if err != nil {
 				return nil, err
 			}
@@ -369,12 +370,13 @@ func mysqlTableToCockroach(
 	fks fkHandler,
 	seqVals map[descpb.ID]int64,
 	owner security.SQLUsername,
+	walltime int64,
 ) ([]*tabledesc.Mutable, []delayedFK, error) {
 	if in == nil {
 		return nil, nil, errors.Errorf("could not read definition for table %q (possible unsupported type?)", name)
 	}
 
-	time := hlc.Timestamp{WallTime: evalCtx.GetStmtTimestamp().UnixNano()}
+	time := hlc.Timestamp{WallTime: walltime}
 
 	const seqOpt = "auto_increment="
 	var seqName string

pkg/ccl/importccl/read_import_mysql_test.go

@@ -117,8 +117,8 @@ func readMysqlCreateFrom(
 		t.Fatal(err)
 	}
 	defer f.Close()
-
-	tbl, err := readMysqlCreateTable(context.Background(), f, testEvalCtx, nil, id, expectedParent, name, fks, map[descpb.ID]int64{}, security.RootUserName())
+	walltime := testEvalCtx.StmtTimestamp.UnixNano()
+	tbl, err := readMysqlCreateTable(context.Background(), f, testEvalCtx, nil, id, expectedParent, name, fks, map[descpb.ID]int64{}, security.RootUserName(), walltime)
 	if err != nil {
 		t.Fatal(err)
 	}