diff --git a/pkg/ccl/multitenantccl/tenantcostserver/BUILD.bazel b/pkg/ccl/multitenantccl/tenantcostserver/BUILD.bazel index be0abdb9de97..998df5ac1d8e 100644 --- a/pkg/ccl/multitenantccl/tenantcostserver/BUILD.bazel +++ b/pkg/ccl/multitenantccl/tenantcostserver/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//pkg/sql/sem/tree", "//pkg/sql/sessiondata", "//pkg/util", + "//pkg/util/log", "//pkg/util/metric", "//pkg/util/metric/aggmetric", "//pkg/util/syncutil", diff --git a/pkg/ccl/multitenantccl/tenantcostserver/system_table.go b/pkg/ccl/multitenantccl/tenantcostserver/system_table.go index 8903a26d771a..086fe916f17f 100644 --- a/pkg/ccl/multitenantccl/tenantcostserver/system_table.go +++ b/pkg/ccl/multitenantccl/tenantcostserver/system_table.go @@ -24,6 +24,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sessiondata" "github.com/cockroachdb/cockroach/pkg/util" + "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/errors" ) @@ -392,16 +393,16 @@ func (h *sysTableHelper) accomodateNewInstance(tenant *tenantState, instance *in // maybeCheckInvariants checks the invariants for the system table with a random // probability and only if this is a test build. -func (h *sysTableHelper) maybeCheckInvariants() error { +func (h *sysTableHelper) maybeCheckInvariants(ctx context.Context) error { if util.CrdbTestBuild && rand.Intn(10) == 0 { - return h.checkInvariants() + return h.checkInvariants(ctx) } return nil } // checkInvariants reads all rows in the system table for the given tenant and // checks that the state is consistent. -func (h *sysTableHelper) checkInvariants() error { +func (h *sysTableHelper) checkInvariants(ctx context.Context) error { // Read the two rows for the per-tenant state (instance_id = 0) and the // per-instance state. rows, err := h.ex.QueryBufferedEx( @@ -430,7 +431,12 @@ func (h *sysTableHelper) checkInvariants() error { h.tenantID.ToUint64(), ) if err != nil { - return err + if h.ctx.Err() == nil { + log.Warningf(ctx, "checkInvariants query failed: %v", err) + } + // We don't want to cause a panic for a query error (which is expected + // during shutdown). + return nil } if len(rows) == 0 { return nil diff --git a/pkg/ccl/multitenantccl/tenantcostserver/token_bucket.go b/pkg/ccl/multitenantccl/tenantcostserver/token_bucket.go index 77dadc19f105..36a2fca5a609 100644 --- a/pkg/ccl/multitenantccl/tenantcostserver/token_bucket.go +++ b/pkg/ccl/multitenantccl/tenantcostserver/token_bucket.go @@ -79,7 +79,7 @@ func (s *instance) TokenBucketRequest( return err } - if err := h.maybeCheckInvariants(); err != nil { + if err := h.maybeCheckInvariants(ctx); err != nil { panic(err) } consumption = tenant.Consumption