sql: group statement statistics based on its transaction

fingerprintID

Previously, statement that are executed as part of different
transactions are grouped together resulting in one single
stats entry. This results in weird UX and makes it difficult
for developer to debug slow queries.
This commit groups statement statsitics based on its transaction
fingerprintIDs on top of the statement fingerprint ID.

Partially addresses cockroachdb#65106, cockroachdb#59205

Release note (bug fix): statement statsitics are now grouped by
the statement's corresponding transaction fingerprints.

pkg/roachpb/app_stats.go

@@ -19,6 +19,16 @@ import (
 // StmtFingerprintID is the type of a Statement's fingerprint ID.
 type StmtFingerprintID uint64
 
+// FingerprintID returns the FingerprintID of the StatementStatisticsKey.
+func (m *StatementStatisticsKey) FingerprintID() StmtFingerprintID {
+ return ConstructStatementFingerprintID(
+ m.Query,
+ m.Failed,
+ m.ImplicitTxn,
+ m.Database,
+ )
+}
+
 // ConstructStatementFingerprintID constructs an ID by hashing an anonymized query, its database
 // and failure status, and if it was part of an implicit txn. At the time of writing,
 // these are the axis' we use to bucket queries for stats collection

pkg/roachpb/app_stats.proto

@@ -186,6 +186,12 @@ message StatementStatisticsKey {
  optional bool full_scan = 8 [(gogoproto.nullable) = false];
  optional string database = 9 [(gogoproto.nullable) = false];
  optional uint64 plan_hash = 10 [(gogoproto.nullable) = false];
+
+ reserved 5;
+ optional uint64 transaction_fingerprint_id = 11
+ [(gogoproto.nullable) = false,
+ (gogoproto.customname) = "TransactionFingerprintID",
+ (gogoproto.casttype) = "TransactionFingerprintID"];
 }
 
 // CollectedStatementStatistics wraps collected timings and metadata for some

pkg/sql/BUILD.bazel

@@ -600,6 +600,7 @@ go_test(
  "//pkg/sql/sqlliveness",
  "//pkg/sql/sqlliveness/slinstance",
  "//pkg/sql/sqlliveness/slprovider",
+ "//pkg/sql/sqlstats",
  "//pkg/sql/sqltestutils",
  "//pkg/sql/stats",
  "//pkg/sql/stmtdiagnostics",

pkg/sql/conn_executor_exec.go

@@ -1897,7 +1897,15 @@ func (ex *connExecutor) recordTransactionStart() (
 
  onTxnFinish = func(ctx context.Context, ev txnEvent) {
  ex.phaseTimes.SetSessionPhaseTime(sessionphase.SessionEndExecTransaction, timeutil.Now())
- err := ex.recordTransaction(ctx, ev, implicit, txnStart)
+ transactionFingerprintID :=
+ roachpb.TransactionFingerprintID(ex.extraTxnState.transactionStatementsHash.Sum())
+ if !implicit {
+ ex.statsCollector.EndExplicitTransaction(
+ ctx,
+ transactionFingerprintID,
+ )
+ }
+ err := ex.recordTransaction(ctx, transactionFingerprintID, ev, implicit, txnStart)
  if err != nil {
  if log.V(1) {
  log.Warningf(ctx, "failed to record transaction stats: %s", err)
@@ -1921,11 +1929,20 @@ func (ex *connExecutor) recordTransactionStart() (
  ex.server.cfg.TestingKnobs.BeforeRestart(ex.Ctx(), ex.extraTxnState.autoRetryReason)
  }
  }
+
+ if !implicit {
+ ex.statsCollector.StartExplicitTransaction()
+ }
+
  return onTxnFinish, onTxnRestart
 }
 
 func (ex *connExecutor) recordTransaction(
- ctx context.Context, ev txnEvent, implicit bool, txnStart time.Time,
+ ctx context.Context,
+ transactionFingerprintID roachpb.TransactionFingerprintID,
+ ev txnEvent,
+ implicit bool,
+ txnStart time.Time,
 ) error {
  txnEnd := timeutil.Now()
  txnTime := txnEnd.Sub(txnStart)
@@ -1955,7 +1972,7 @@ func (ex *connExecutor) recordTransaction(
 
  return ex.statsCollector.RecordTransaction(
  ctx,
- roachpb.TransactionFingerprintID(ex.extraTxnState.transactionStatementsHash.Sum()),
+ transactionFingerprintID,
  recordedTxnStats,
  )
 }

pkg/sql/crdb_internal.go

@@ -5079,13 +5079,14 @@ var crdbInternalStmtStatsTable = virtualSchemaTable{
  `cluster-wide RPC-fanout.`,
  schema: `
 CREATE TABLE crdb_internal.statement_statistics (
- aggregated_ts TIMESTAMPTZ NOT NULL,
- fingerprint_id BYTES NOT NULL,
- plan_hash BYTES NOT NULL,
- app_name STRING NOT NULL,
- metadata JSONB NOT NULL,
- statistics JSONB NOT NULL,
- sampled_plan JSONB NOT NULL
+ aggregated_ts TIMESTAMPTZ NOT NULL,
+ fingerprint_id BYTES NOT NULL,
+ transaction_fingerprint_id BYTES NOT NULL,
+ plan_hash BYTES NOT NULL,
+ app_name STRING NOT NULL,
+ metadata JSONB NOT NULL,
+ statistics JSONB NOT NULL,
+ sampled_plan JSONB NOT NULL
 );`,
  generator: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) {
  // TODO(azhng): we want to eventually implement memory accounting within the
@@ -5119,7 +5120,7 @@ CREATE TABLE crdb_internal.statement_statistics (
  Knobs: execCfg.SQLStatsTestingKnobs,
  }, memSQLStats)
 
- row := make(tree.Datums, 7 /* number of columns for this virtual table */)
+ row := make(tree.Datums, 8 /* number of columns for this virtual table */)
  worker := func(pusher rowPusher) error {
  return sqlStats.IterateStatementStats(ctx, &sqlstats.IteratorOptions{
  SortedAppNames: true,
@@ -5134,6 +5135,9 @@ CREATE TABLE crdb_internal.statement_statistics (
  fingerprintID := tree.NewDBytes(
  tree.DBytes(sqlstatsutil.EncodeUint64ToBytes(uint64(statistics.ID))))
 
+ transactionFingerprintID := tree.NewDBytes(
+ tree.DBytes(sqlstatsutil.EncodeUint64ToBytes(uint64(statistics.Key.TransactionFingerprintID))))
+
  // TODO(azhng): properly update plan_hash value once we can expose it
  // from the optimizer.
  planHash := tree.NewDBytes(
@@ -5153,6 +5157,7 @@ CREATE TABLE crdb_internal.statement_statistics (
  row = append(row,
  aggregatedTs, // aggregated_ts
  fingerprintID, // fingerprint_id
+ transactionFingerprintID, // transaction_fingerprint_id
  planHash, // plan_hash
  tree.NewDString(statistics.Key.App), // app_name
  tree.NewDJSON(metadataJSON), // metadata