roachpb: v19.2.1: transaction unexpectedly finalized

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1358597309/?referrer=webhooks_plugin

Panic message:

errors.go:244: transaction unexpectedly finalized in (%T): %v | *roachpb.TransactionRetryError; *roachpb.Error

Stacktrace (expand for inline code snippets):

cockroach/pkg/roachpb/errors.go

Lines 243 to 245 in 53eef08

	}
	e.checkTxnStatusValid()
	}

in pkg/roachpb.(*Error).UpdateTxn

cockroach/pkg/roachpb/errors.go

Lines 225 to 227 in 53eef08

	e.UnexposedTxn = nil
	e.UpdateTxn(txn)
	}

in pkg/roachpb.(*Error).SetTxn

cockroach/pkg/roachpb/errors.go

Lines 121 to 123 in 53eef08

	e := NewError(err)
	e.SetTxn(txn)
	return e

in pkg/roachpb.NewErrorWithTxn

cockroach/pkg/kv/txn_interceptor_pipeliner.go

Lines 562 to 564 in 53eef08

	roachpb.RETRY_ASYNC_WRITE_FAILURE, fmt.Sprintf("missing intent on: %s", ime.Key))
	retryErr := roachpb.NewErrorWithTxn(err, pErr.GetTxn())
	retryErr.Index = pErr.Index

in pkg/kv.(*txnPipeliner).adjustError

cockroach/pkg/kv/txn_interceptor_pipeliner.go

Lines 214 to 216 in 53eef08

	if pErr != nil {
	return nil, tp.adjustError(ctx, ba, pErr)
	}

in pkg/kv.(*txnPipeliner).SendLocked

cockroach/pkg/kv/txn_interceptor_seq_num_allocator.go

Lines 88 to 90 in 53eef08

	return s.wrapped.SendLocked(ctx, ba)
	}

in pkg/kv.(*txnSeqNumAllocator).SendLocked

cockroach/pkg/kv/txn_interceptor_heartbeater.go

Lines 167 to 169 in 53eef08

	// Forward the batch through the wrapped lockedSender.
	return h.wrapped.SendLocked(ctx, ba)
	}

in pkg/kv.(*txnHeartbeater).SendLocked

cockroach/pkg/kv/txn_coord_sender.go

Lines 799 to 801 in 53eef08

	// Send the command through the txnInterceptor stack.
	br, pErr := tc.interceptorStack[0].SendLocked(ctx, ba)

in pkg/kv.(*TxnCoordSender).Send

cockroach/pkg/internal/client/db.go

Lines 753 to 755 in 53eef08

	tracing.AnnotateTrace()
	br, pErr := sender.Send(ctx, ba)
	if pErr != nil {

in pkg/internal/client.(*DB).sendUsingSender

cockroach/pkg/internal/client/txn.go

Lines 802 to 804 in 53eef08

	txn.mu.Unlock()
	br, pErr := txn.db.sendUsingSender(ctx, ba, sender)
	if pErr == nil {

in pkg/internal/client.(*Txn).Send

cockroach/pkg/internal/client/txn.go

Lines 501 to 503 in 53eef08

	ba.Add(endTxnReq(true /* commit */, txn.deadline(), txn.systemConfigTrigger))
	_, pErr := txn.Send(ctx, ba)
	if pErr == nil {

in pkg/internal/client.(*Txn).commit

cockroach/pkg/internal/client/txn.go

Lines 528 to 530 in 53eef08

	func (txn *Txn) Commit(ctx context.Context) error {
	return txn.commit(ctx)
	}

in pkg/internal/client.(*Txn).Commit

cockroach/pkg/sql/conn_executor_exec.go

Lines 580 to 582 in 53eef08

	if err := ex.state.mu.txn.Commit(ctx); err != nil {
	return ex.makeErrEvent(err, stmt)

in pkg/sql.(*connExecutor).commitSQLTransaction

cockroach/pkg/sql/conn_executor_exec.go

Lines 241 to 243 in 53eef08

	// ReleaseSavepoint is executed fully here; there's no plan for it.
	ev, payload := ex.commitSQLTransaction(ctx, stmt.AST)
	res.ResetStmtType((*tree.CommitTransaction)(nil))

in pkg/sql.(*connExecutor).execStmtInOpenState

cockroach/pkg/sql/conn_executor_exec.go

Lines 97 to 99 in 53eef08

	} else {
	ev, payload, err = ex.execStmtInOpenState(ctx, stmt, res, pinfo)
	}

in pkg/sql.(*connExecutor).execStmt

cockroach/pkg/sql/conn_executor.go

Lines 1242 to 1244 in 53eef08

	stmtCtx := withStatement(ctx, ex.curStmt)
	ev, payload, err = ex.execStmt(stmtCtx, curStmt, stmtRes, nil /* pinfo */)
	if err != nil {

in pkg/sql.(*connExecutor).execCmd

cockroach/pkg/sql/conn_executor.go

Lines 1171 to 1173 in 53eef08

	var err error
	if err = ex.execCmd(ex.Ctx()); err != nil {
	if err == io.EOF || err == errDrainingComplete {

in pkg/sql.(*connExecutor).run

cockroach/pkg/sql/conn_executor.go

Lines 443 to 445 in 53eef08

	}()
	return h.ex.run(ctx, s.pool, reserved, cancel)
	}

in pkg/sql.(*Server).ServeConn

cockroach/pkg/sql/pgwire/conn.go

Lines 583 to 585 in 53eef08

	reservedOwned = false // We're about to pass ownership away.
	retErr = sqlServer.ServeConn(ctx, connHandler, reserved, cancelConn)
	}()

in pkg/sql/pgwire.(*conn).processCommandsAsync.func1

pkg/roachpb/errors.go in pkg/roachpb.(*Error).UpdateTxn at line 244
pkg/roachpb/errors.go in pkg/roachpb.(*Error).SetTxn at line 226
pkg/roachpb/errors.go in pkg/roachpb.NewErrorWithTxn at line 122
pkg/kv/txn_interceptor_pipeliner.go in pkg/kv.(*txnPipeliner).adjustError at line 563
pkg/kv/txn_interceptor_pipeliner.go in pkg/kv.(*txnPipeliner).SendLocked at line 215
pkg/kv/txn_interceptor_seq_num_allocator.go in pkg/kv.(*txnSeqNumAllocator).SendLocked at line 89
pkg/kv/txn_interceptor_heartbeater.go in pkg/kv.(*txnHeartbeater).SendLocked at line 168
pkg/kv/txn_coord_sender.go in pkg/kv.(*TxnCoordSender).Send at line 800
pkg/internal/client/db.go in pkg/internal/client.(*DB).sendUsingSender at line 754
pkg/internal/client/txn.go in pkg/internal/client.(*Txn).Send at line 803
pkg/internal/client/txn.go in pkg/internal/client.(*Txn).commit at line 502
pkg/internal/client/txn.go in pkg/internal/client.(*Txn).Commit at line 529
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).commitSQLTransaction at line 581
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmtInOpenState at line 242
pkg/sql/conn_executor_exec.go in pkg/sql.(*connExecutor).execStmt at line 98
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).execCmd at line 1243
pkg/sql/conn_executor.go in pkg/sql.(*connExecutor).run at line 1172
pkg/sql/conn_executor.go in pkg/sql.(*Server).ServeConn at line 444
pkg/sql/pgwire/conn.go in pkg/sql/pgwire.(*conn).processCommandsAsync.func1 at line 584

Tag	Value
Cockroach Release	v19.2.1
Cockroach SHA:	53eef08
Platform	linux amd64
Distribution	CCL
Environment	v19.2.1
Command	server
Go Version	go1.12.12
# of CPUs	4
# of Goroutines	2186

[nvanbenschoten]

Sentry link gone and we haven't seen this since.