sql: v20.1.1: read table descriptor without ModificationTime with zero MVCC timestamp

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1699585911/?referrer=webhooks_plugin

Panic message:

structured.go:3496: read table descriptor for %q (%d.%d) without ModificationTime with zero MVCC timestamp | string; sqlbase.ID; sqlbase.ID

Stacktrace (expand for inline code snippets):

cockroach/pkg/sql/sqlbase/structured.go

Lines 3495 to 3497 in 6123c0c

	if t != nil {
	t.maybeSetTimeFromMVCCTimestamp(ts)
	}

in pkg/sql/sqlbase.(*Descriptor).Table

cockroach/pkg/sql/sqlbase/structured.go

Lines 407 to 409 in 6123c0c

	}
	table := desc.Table(ts)
	if table == nil {

in pkg/sql/sqlbase.getTableDescFromIDRaw

cockroach/pkg/sql/sqlbase/structured.go

Lines 988 to 990 in 6123c0c

	if _, ok := otherUnupgradedTables[ref.Table]; !ok {
	tbl, err := getTableDescFromIDRaw(ctx, protoGetter, ref.Table)
	if err != nil {

in pkg/sql/sqlbase.maybeUpgradeForeignKeyRepOnIndex

cockroach/pkg/sql/sqlbase/structured.go

Lines 955 to 957 in 6123c0c

	for i := range desc.Indexes {
	newChanged, err := maybeUpgradeForeignKeyRepOnIndex(
	ctx, protoGetter, otherUnupgradedTables, desc, &desc.Indexes[i], skipFKsWithNoMatchingTable,

in pkg/sql/sqlbase.(*TableDescriptor).MaybeUpgradeForeignKeyRepresentation

cockroach/pkg/sql/sqlbase/structured.go

Lines 892 to 894 in 6123c0c

	if protoGetter != nil {
	if _, err := desc.MaybeUpgradeForeignKeyRepresentation(ctx, protoGetter, false /* skipFKsWithNoMatchingTable*/); err != nil {
	return err

in pkg/sql/sqlbase.(*TableDescriptor).MaybeFillInDescriptor

cockroach/pkg/sql/sqlbase/structured.go

Lines 365 to 367 in 6123c0c

	if err := table.MaybeFillInDescriptor(ctx, protoGetter); err != nil {
	return nil, err

in pkg/sql/sqlbase.GetTableDescFromID

cockroach/pkg/sql/sqlbase/structured.go

Lines 421 to 423 in 6123c0c

	) (*MutableTableDescriptor, error) {
	table, err := GetTableDescFromID(ctx, protoGetter, id)
	if err != nil {

in pkg/sql/sqlbase.GetMutableTableDescFromID

cockroach/pkg/sql/table.go

Lines 480 to 482 in 6123c0c

	}
	return sqlbase.GetMutableTableDescFromID(ctx, txn, tableID)
	}

in pkg/sql.(*TableCollection).getMutableTableVersionByID

cockroach/pkg/sql/drop_table.go

Lines 431 to 433 in 6123c0c

	} else {
	lookup, err := p.Tables().getMutableTableVersionByID(ctx, ref.OriginTableID, p.txn)
	if err != nil {

in pkg/sql.(*planner).removeFKForBackReference

cockroach/pkg/sql/drop_table.go

Lines 261 to 263 in 6123c0c

	ref := &tableDesc.InboundFKs[i]
	if err := p.removeFKForBackReference(ctx, tableDesc, ref); err != nil {
	return droppedViews, err

in pkg/sql.(*planner).dropTableImpl

cockroach/pkg/sql/drop_database.go

Lines 191 to 193 in 6123c0c

	// TODO(knz): dependent dropped table names should be qualified here.
	cascadedObjects, err = p.dropTableImpl(ctx, desc, false /* queueJob */, "")
	}

in pkg/sql.(*dropDatabaseNode).startExec

cockroach/pkg/sql/plan.go

Lines 394 to 396 in 6123c0c

	}
	return n.startExec(params)
	},

in pkg/sql.startExec.func2

cockroach/pkg/sql/walk.go

Lines 143 to 145 in 6123c0c

	}
	v.err = v.observer.leaveNode(name, plan)
	}()

in pkg/sql.(*planVisitor).visitInternal.func1

cockroach/pkg/sql/walk.go

Lines 720 to 722 in 6123c0c

	}
	}

in pkg/sql.(*planVisitor).visitInternal

cockroach/pkg/sql/walk.go

Lines 110 to 112 in 6123c0c

	}
	v.visitInternal(plan, name)
	return plan

in pkg/sql.(*planVisitor).visit

cockroach/pkg/sql/walk.go

Lines 74 to 76 in 6123c0c

	v := makePlanVisitor(ctx, observer)
	v.visit(plan)
	return v.err

in pkg/sql.walkPlan

cockroach/pkg/sql/plan.go

Lines 397 to 399 in 6123c0c

	}
	return walkPlan(params.ctx, plan, o)
	}

in pkg/sql.startExec

cockroach/pkg/sql/plan_node_to_row_source.go

Lines 116 to 118 in 6123c0c

	// This starts all of the nodes below this node.
	if err := startExec(p.params, p.node); err != nil {
	p.MoveToDraining(err)

in pkg/sql.(*planNodeToRowSource).Start

cockroach/pkg/sql/execinfra/processorsbase.go

Lines 747 to 749 in 6123c0c

	}
	ctx = pb.self.Start(ctx)
	Run(ctx, pb.self, pb.Out.output)

in pkg/sql/execinfra.(*ProcessorBase).Run

cockroach/pkg/sql/flowinfra/flow.go

Lines 369 to 371 in 6123c0c

	}
	headProc.Run(ctx)
	return nil

in pkg/sql/flowinfra.(*FlowBase).Run

cockroach/pkg/sql/distsql_running.go

Lines 404 to 406 in 6123c0c

	// TODO(radu): this should go through the flow scheduler.
	if err := flow.Run(ctx, func() {}); err != nil {
	log.Fatalf(ctx, "unexpected error from syncFlow.Start(): %s "+

in pkg/sql.(*DistSQLPlanner).Run

cockroach/pkg/sql/distsql_running.go

Lines 1018 to 1020 in 6123c0c

	recv.expectedRowsRead = int64(physPlan.TotalEstimatedScannedRows)
	return dsp.Run(planCtx, txn, &physPlan, recv, evalCtx, nil /* finishedSetupFn */)
	}

in pkg/sql.(*DistSQLPlanner).PlanAndRun

cockroach/pkg/sql/conn_executor_exec.go

Lines 882 to 884 in 6123c0c

	// the planner whether or not to plan remote table readers.
	cleanup := ex.server.cfg.DistSQLPlanner.PlanAndRun(
	ctx, evalCtx, planCtx, planner.txn, planner.curPlan.plan, recv,

in pkg/sql.(*connExecutor).execWithDistSQLEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 775 to 777 in 6123c0c

	ex.sessionTracing.TraceExecStart(ctx, "distributed")
	bytesRead, rowsRead, err := ex.execWithDistSQLEngine(ctx, planner, stmt.AST.StatementType(), res, distributePlan, progAtomic)
	ex.sessionTracing.TraceExecEnd(ctx, res.Err(), res.RowsAffected())

in pkg/sql.(*connExecutor).dispatchToExecutionEngine

cockroach/pkg/sql/conn_executor_exec.go

Lines 480 to 482 in 6123c0c

	p.autoCommit = os.ImplicitTxn.Get() && !ex.server.cfg.TestingKnobs.DisableAutoCommit
	if err := ex.dispatchToExecutionEngine(ctx, p, res); err != nil {
	return nil, nil, err

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 95 to 97 in 6123c0c

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1367 to 1369 in 6123c0c

	stmtCtx := withStatement(ctx, ex.curStmt)
	ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
	if err != nil {

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1296 to 1298 in 6123c0c

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if err == io.EOF || err == errDrainingComplete {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 475 to 477 in 6123c0c

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 590 to 592 in 6123c0c

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1

pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.(*Descriptor).Table at line 3496
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.getTableDescFromIDRaw at line 408
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.maybeUpgradeForeignKeyRepOnIndex at line 989
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.(*TableDescriptor).MaybeUpgradeForeignKeyRepresentation at line 956
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.(*TableDescriptor).MaybeFillInDescriptor at line 893
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.GetTableDescFromID at line 366
pkg/sql/sqlbase/structured.go in pkg/sql/sqlbase.GetMutableTableDescFromID at line 422
pkg/sql/table.go in pkg/sql.(*TableCollection).getMutableTableVersionByID at line 481
pkg/sql/drop_table.go in pkg/sql.(*planner).removeFKForBackReference at line 432
pkg/sql/drop_table.go in pkg/sql.(*planner).dropTableImpl at line 262
pkg/sql/drop_database.go in pkg/sql.(*dropDatabaseNode).startExec at line 192
pkg/sql/plan.go in pkg/sql.startExec.func2 at line 395
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal.func1 at line 144
pkg/sql/walk.go in pkg/sql.(*planVisitor).visitInternal at line 721
pkg/sql/walk.go in pkg/sql.(*planVisitor).visit at line 111
pkg/sql/walk.go in pkg/sql.walkPlan at line 75
pkg/sql/plan.go in pkg/sql.startExec at line 398
pkg/sql/plan_node_to_row_source.go in pkg/sql.(*planNodeToRowSource).Start at line 117
pkg/sql/execinfra/processorsbase.go in pkg/sql/execinfra.(*ProcessorBase).Run at line 748
pkg/sql/flowinfra/flow.go in pkg/sql/flowinfra.(*FlowBase).Run at line 370
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).Run at line 405
pkg/sql/distsql_running.go in pkg/sql.(*DistSQLPlanner).PlanAndRun at line 1019
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execWithDistSQLEngine at line 883
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).dispatchToExecutionEngine at line 776
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 481
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 96
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1368
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1297
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 476
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 591

Tag	Value
Cockroach Release	v20.1.1
Cockroach SHA:	6123c0c
Platform	linux amd64
Distribution	CCL
Environment	v20.1.1
Command	start-single-node
Go Version	go1.13.9
# of CPUs	3
# of Goroutines	145

[yuzefovich]

Seems like a dup of #49266. cc @ajwerner

[ajwerner]

This is worrisome that we've now seen it a few times.

[thoszhang]

closing as a dupe of #49266