storage: v19.2.2: nil pointer in interval_btree

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/1809303514/?referrer=webhooks_plugin

Panic message:

panic.go:522: runtime.errorString: runtime error: invalid memory address or nil pointer dereference

Stacktrace (expand for inline code snippets):

/usr/local/go/src/runtime/panic.go#L521-L523 in runtime.gopanic

cockroach/pkg/storage/store_send.go

Lines 111 to 113 in 3cbd056

	// again due to undefined state.
	panic(r)
	}

in pkg/storage.(*Store).Send.func1
/usr/local/go/src/runtime/panic.go#L521-L523 in runtime.gopanic
/usr/local/go/src/runtime/panic.go#L81-L83 in runtime.panicmem
/usr/local/go/src/runtime/signal_unix.go#L389-L391 in runtime.sigpanic

cockroach/pkg/storage/spanlatch/interval_btree.go

Lines 380 to 382 in 3cbd056

	}
	if n.children[i].count >= maxLatches {
	splitLa, splitNode := mut(&n.children[i]).split(maxLatches / 2)

in pkg/storage/spanlatch.(*node).insert

cockroach/pkg/storage/spanlatch/interval_btree.go

Lines 702 to 704 in 3cbd056

	}
	if replaced, _ := mut(&t.root).insert(la); !replaced {
	t.length++

in pkg/storage/spanlatch.(*btree).Set

cockroach/pkg/storage/spanlatch/manager.go

Lines 289 to 291 in 3cbd056

	// Add writes directly to the write tree.
	sm.trees[spanset.SpanReadWrite].Set(latch)
	default:

in pkg/storage/spanlatch.(*Manager).insertLocked

cockroach/pkg/storage/spanlatch/manager.go

Lines 223 to 225 in 3cbd056

	snap := m.snapshotLocked(spans)
	m.insertLocked(lg)
	m.mu.Unlock()

in pkg/storage/spanlatch.(*Manager).sequence

cockroach/pkg/storage/spanlatch/manager.go

Lines 203 to 205 in 3cbd056

	) (*Guard, error) {
	lg, snap := m.sequence(spans, ts)
	defer snap.close()

in pkg/storage/spanlatch.(*Manager).Acquire

cockroach/pkg/storage/replica.go

Lines 1262 to 1264 in 3cbd056

	var err error
	lg, err = r.latchMgr.Acquire(ctx, spans, ba.Timestamp)
	if err != nil {

in pkg/storage.(*Replica).beginCmds

cockroach/pkg/storage/replica_write.go

Lines 89 to 91 in 3cbd056

	var err error
	ec, err = r.beginCmds(ctx, ba, spans)
	if err != nil {

in pkg/storage.(*Replica).executeWriteBatch

cockroach/pkg/storage/replica.go

Lines 576 to 578 in 3cbd056

	log.Event(ctx, "read-write path")
	br, pErr = r.executeWriteBatch(ctx, &ba)
	} else if isReadOnly {

in pkg/storage.(*Replica).sendWithRangeID

cockroach/pkg/storage/replica.go

Lines 524 to 526 in 3cbd056

	) (*roachpb.BatchResponse, *roachpb.Error) {
	return r.sendWithRangeID(ctx, r.RangeID, ba)
	}

in pkg/storage.(*Replica).Send

cockroach/pkg/storage/store_send.go

Lines 227 to 229 in 3cbd056

	}
	br, pErr = repl.Send(ctx, ba)
	if pErr == nil {

in pkg/storage.(*Store).Send

cockroach/pkg/storage/stores.go

Lines 180 to 182 in 3cbd056

	br, pErr := store.Send(ctx, ba)
	if br != nil && br.Error != nil {

in pkg/storage.(*Stores).Send

cockroach/pkg/server/node.go

Lines 924 to 926 in 3cbd056

	var pErr *roachpb.Error
	br, pErr = n.stores.Send(ctx, *args)
	if pErr != nil {

in pkg/server.(*Node).batchInternal.func1

cockroach/pkg/util/stop/stopper.go

Lines 299 to 301 in 3cbd056

	return f(ctx)
	}

in pkg/util/stop.(*Stopper).RunTaskWithErr

cockroach/pkg/server/node.go

Lines 912 to 914 in 3cbd056

	var br *roachpb.BatchResponse
	if err := n.stopper.RunTaskWithErr(ctx, "node.Node: batch", func(ctx context.Context) error {
	var finishSpan func(*roachpb.BatchResponse)

in pkg/server.(*Node).batchInternal

cockroach/pkg/server/node.go

Lines 950 to 952 in 3cbd056

	br, err := n.batchInternal(ctx, args)

in pkg/server.(*Node).Batch

cockroach/pkg/rpc/context.go

Lines 541 to 543 in 3cbd056

	) (*roachpb.BatchResponse, error) {
	return a.InternalServer.Batch(ctx, ba)
	}

in pkg/rpc.internalClientAdapter.Batch

cockroach/pkg/kv/transport.go

Lines 198 to 200 in 3cbd056

	}
	reply, err := iface.Batch(ctx, &ba)
	// If we queried a remote node, perform extra validation and

in pkg/kv.(*grpcTransport).sendBatch

cockroach/pkg/kv/transport.go

Lines 167 to 169 in 3cbd056

	ba.Replica = client.replica
	reply, err := gt.sendBatch(ctx, client.replica.NodeID, iface, ba)

in pkg/kv.(*grpcTransport).SendNext

cockroach/pkg/kv/dist_sender.go

Lines 1675 to 1677 in 3cbd056

	}
	br, err := transport.SendNext(ctx, ba)
	// maxSeenLeaseSequence tracks the maximum LeaseSequence seen in a

in pkg/kv.(*DistSender).sendToReplicas

cockroach/pkg/kv/dist_sender.go

Lines 435 to 437 in 3cbd056

	return ds.sendToReplicas(
	ctx,

in pkg/kv.(*DistSender).sendRPC

cockroach/pkg/kv/dist_sender.go

Lines 517 to 519 in 3cbd056

	class := rpc.ConnectionClassForKey(desc.RSpan().Key)
	br, err := ds.sendRPC(ctx, ba, class, desc.RangeID, replicas, cachedLeaseHolder, withCommit)
	if err != nil {

in pkg/kv.(*DistSender).sendSingleRange

cockroach/pkg/kv/dist_sender.go

Lines 1446 to 1448 in 3cbd056

	reply, pErr = ds.sendSingleRange(ctx, ba, desc, withCommit)

in pkg/kv.(*DistSender).sendPartialBatch

cockroach/pkg/kv/dist_sender.go

Lines 1364 to 1366 in 3cbd056

	ds.metrics.AsyncSentCount.Inc(1)
	responseCh <- ds.sendPartialBatch(
	ctx, ba, rs, desc, evictToken, withCommit, batchIdx, true, /* needsTruncate */

in pkg/kv.(*DistSender).sendPartialBatchAsync.func1

cockroach/pkg/util/stop/stopper.go

Lines 380 to 382 in 3cbd056

	f(ctx)
	}()

in pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1

/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 522
pkg/storage/store_send.go in pkg/storage.(*Store).Send.func1 at line 112
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 522
/usr/local/go/src/runtime/panic.go in runtime.panicmem at line 82
/usr/local/go/src/runtime/signal_unix.go in runtime.sigpanic at line 390
pkg/storage/spanlatch/interval_btree.go in pkg/storage/spanlatch.(*node).insert at line 381
pkg/storage/spanlatch/interval_btree.go in pkg/storage/spanlatch.(*btree).Set at line 703
pkg/storage/spanlatch/manager.go in pkg/storage/spanlatch.(*Manager).insertLocked at line 290
pkg/storage/spanlatch/manager.go in pkg/storage/spanlatch.(*Manager).sequence at line 224
pkg/storage/spanlatch/manager.go in pkg/storage/spanlatch.(*Manager).Acquire at line 204
pkg/storage/replica.go in pkg/storage.(*Replica).beginCmds at line 1263
pkg/storage/replica_write.go in pkg/storage.(*Replica).executeWriteBatch at line 90
pkg/storage/replica.go in pkg/storage.(*Replica).sendWithRangeID at line 577
pkg/storage/replica.go in pkg/storage.(*Replica).Send at line 525
pkg/storage/store_send.go in pkg/storage.(*Store).Send at line 228
pkg/storage/stores.go in pkg/storage.(*Stores).Send at line 181
pkg/server/node.go in pkg/server.(*Node).batchInternal.func1 at line 925
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunTaskWithErr at line 300
pkg/server/node.go in pkg/server.(*Node).batchInternal at line 913
pkg/server/node.go in pkg/server.(*Node).Batch at line 951
pkg/rpc/context.go in pkg/rpc.internalClientAdapter.Batch at line 542
pkg/kv/transport.go in pkg/kv.(*grpcTransport).sendBatch at line 199
pkg/kv/transport.go in pkg/kv.(*grpcTransport).SendNext at line 168
pkg/kv/dist_sender.go in pkg/kv.(*DistSender).sendToReplicas at line 1676
pkg/kv/dist_sender.go in pkg/kv.(*DistSender).sendRPC at line 436
pkg/kv/dist_sender.go in pkg/kv.(*DistSender).sendSingleRange at line 518
pkg/kv/dist_sender.go in pkg/kv.(*DistSender).sendPartialBatch at line 1447
pkg/kv/dist_sender.go in pkg/kv.(*DistSender).sendPartialBatchAsync.func1 at line 1365
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1 at line 381

Tag	Value
Cockroach Release	v19.2.2
Cockroach SHA:	3cbd056
Platform	linux amd64
Distribution	CCL
Environment	v19.2.2
Command	server
Go Version	go1.12.12
# of CPUs	32
# of Goroutines	386

[yuzefovich]

cc @nvanbenschoten

[nvanbenschoten]

This is coming from:

cockroach/pkg/storage/spanlatch/interval_btree.go

Line 381 in 3cbd056

if n.children[i].count >= maxLatches {

We already dereferenced n earlier in the function and this isn't a slice bounds error, so we must be failing to deference n.children[i]. This could be due to one of the following two reasons:

1. a leaf node lost its leaf flag and is being incorrectly considered an inner node.
2. an inner node's count is out of sync with its children array.

Given how few places manipulate leaf, I think the latter option is more likely.