Merge cockroachdb#62006

62006: sql: add execution stats to node_statement_statistics r=yuzefovich a=asubiotto Release justification: low-risk, high-benefit change to existing functionality since this commit exposes pre-existing execution stats through the SQL shell. Release note (sql change): sampled execution stats are now available through node_statement_statistics. Closes cockroachdb#61637 Putting @yuzefovich as primary reviewer but @maryliag and @Azhng, please take a look and let me know if you have any questions. Co-authored-by: Alfonso Subiotto Marques <alfonso@cockroachlabs.com>
fqazi · Mar 16, 2021 · 98ae002 · 98ae002
2 parents db4f939 + 68d2e66
commit 98ae002
Show file tree

Hide file tree

Showing 4 changed files with 182 additions and 84 deletions.
diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go
@@ -810,6 +810,24 @@ func (t txnList) Less(i, j int) bool {
  return t[i] < t[j]
 }
 
+// execStatAvg is a helper for execution stats shown in virtual tables. Returns
+// NULL when the count is 0, or the mean of the given NumericStat.
+func execStatAvg(count int64, n roachpb.NumericStat) tree.Datum {
+ if count == 0 {
+ return tree.DNull
+ }
+ return tree.NewDFloat(tree.DFloat(n.Mean))
+}
+
+// execStatVar is a helper for execution stats shown in virtual tables. Returns
+// NULL when the count is 0, or the variance of the given NumericStat.
+func execStatVar(count int64, n roachpb.NumericStat) tree.Datum {
+ if count == 0 {
+ return tree.DNull
+ }
+ return tree.NewDFloat(tree.DFloat(n.GetVariance(count)))
+}
+
 var crdbInternalStmtStatsTable = virtualSchemaTable{
  comment: `statement statistics (in-memory, not durable; local node only). ` +
  `This table is wiped periodically (by default, at least every two hours)`,
@@ -840,6 +858,16 @@ CREATE TABLE crdb_internal.node_statement_statistics (
  bytes_read_var FLOAT NOT NULL,
  rows_read_avg FLOAT NOT NULL,
  rows_read_var FLOAT NOT NULL,
+ network_bytes_avg FLOAT,
+ network_bytes_var FLOAT,
+ network_msgs_avg FLOAT,
+ network_msgs_var FLOAT,
+ max_mem_usage_avg FLOAT,
+ max_mem_usage_var FLOAT,
+ max_disk_usage_avg FLOAT,
+ max_disk_usage_var FLOAT,
+ contention_time_avg FLOAT,
+ contention_time_var FLOAT,
  implicit_txn BOOL NOT NULL
 )`,
  populate: func(ctx context.Context, p *planner, _ *dbdesc.Immutable, addRow func(...tree.Datum) error) error {
@@ -908,32 +936,42 @@ CREATE TABLE crdb_internal.node_statement_statistics (
  flags = "!" + flags
  }
  err := addRow(
- tree.NewDInt(tree.DInt(nodeID)),
- tree.NewDString(appName),
- tree.NewDString(flags),
- tree.NewDString(stmtKey.anonymizedStmt),
- anonymized,
- tree.NewDInt(tree.DInt(s.mu.data.Count)),
- tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)),
- tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),
- errString,
- tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.GetVariance(s.mu.data.Count))),
- tree.MakeDBool(tree.DBool(stmtKey.implicitTxn)),
+ tree.NewDInt(tree.DInt(nodeID)), // node_id
+ tree.NewDString(appName), // application_name
+ tree.NewDString(flags), // flags
+ tree.NewDString(stmtKey.anonymizedStmt), // key
+ anonymized, // anonymized
+ tree.NewDInt(tree.DInt(s.mu.data.Count)), // count
+ tree.NewDInt(tree.DInt(s.mu.data.FirstAttemptCount)), // first_attempt_count
+ tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)), // max_retries
+ errString, // last_error
+ tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)), // rows_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))), // rows_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.Mean)), // parse_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.ParseLat.GetVariance(s.mu.data.Count))), // parse_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.Mean)), // plan_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.PlanLat.GetVariance(s.mu.data.Count))), // plan_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.Mean)), // run_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.RunLat.GetVariance(s.mu.data.Count))), // run_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)), // service_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))), // service_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.Mean)), // overhead_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.OverheadLat.GetVariance(s.mu.data.Count))), // overhead_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.Mean)), // bytes_read_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.BytesRead.GetVariance(s.mu.data.Count))), // bytes_read_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.Mean)), // rows_read_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.RowsRead.GetVariance(s.mu.data.Count))), // rows_read_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes), // network_bytes_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes), // network_bytes_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages), // network_msgs_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages), // network_msgs_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage), // max_mem_usage_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage), // max_mem_usage_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage), // max_disk_usage_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage), // max_disk_usage_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime), // contention_time_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime), // contention_time_var
+ tree.MakeDBool(tree.DBool(stmtKey.implicitTxn)), // implicit_txn
  )
  s.mu.Unlock()
  if err != nil {
@@ -953,20 +991,30 @@ var crdbInternalTransactionStatisticsTable = virtualSchemaTable{
  `This table is wiped periodically (by default, at least every two hours)`,
  schema: `
 CREATE TABLE crdb_internal.node_transaction_statistics (
- node_id INT NOT NULL,
- application_name STRING NOT NULL,
- key STRING,
- statement_ids STRING[],
- count INT,
- max_retries INT,
- service_lat_avg FLOAT NOT NULL,
- service_lat_var FLOAT NOT NULL,
- retry_lat_avg FLOAT NOT NULL,
- retry_lat_var FLOAT NOT NULL,
- commit_lat_avg FLOAT NOT NULL,
- commit_lat_var FLOAT NOT NULL,
- rows_read_avg FLOAT NOT NULL,
- rows_read_var FLOAT NOT NULL
+ node_id INT NOT NULL,
+ application_name STRING NOT NULL,
+ key STRING,
+ statement_ids STRING[],
+ count INT,
+ max_retries INT,
+ service_lat_avg FLOAT NOT NULL,
+ service_lat_var FLOAT NOT NULL,
+ retry_lat_avg FLOAT NOT NULL,
+ retry_lat_var FLOAT NOT NULL,
+ commit_lat_avg FLOAT NOT NULL,
+ commit_lat_var FLOAT NOT NULL,
+ rows_read_avg FLOAT NOT NULL,
+ rows_read_var FLOAT NOT NULL,
+ network_bytes_avg FLOAT,
+ network_bytes_var FLOAT,
+ network_msgs_avg FLOAT,
+ network_msgs_var FLOAT,
+ max_mem_usage_avg FLOAT,
+ max_mem_usage_var FLOAT,
+ max_disk_usage_avg FLOAT,
+ max_disk_usage_var FLOAT,
+ contention_time_avg FLOAT, 
+ contention_time_var FLOAT
 )
 `,
  populate: func(ctx context.Context, p *planner, _ *dbdesc.Immutable, addRow func(...tree.Datum) error) error {
@@ -1032,20 +1080,30 @@ CREATE TABLE crdb_internal.node_transaction_statistics (
  s.mu.Lock()
 
  err := addRow(
- tree.NewDInt(tree.DInt(nodeID)),
- tree.NewDString(appName),
- tree.NewDString(strconv.FormatUint(uint64(txnKey), 10)),
- stmtIDsDatum,
- tree.NewDInt(tree.DInt(s.mu.data.Count)),
- tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)),
- tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.GetVariance(s.mu.data.Count))),
- tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)),
- tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))),
+ tree.NewDInt(tree.DInt(nodeID)), // node_id
+ tree.NewDString(appName), // application_name
+ tree.NewDString(strconv.FormatUint(uint64(txnKey), 10)), // key
+ stmtIDsDatum, // statement_ids
+ tree.NewDInt(tree.DInt(s.mu.data.Count)), // count
+ tree.NewDInt(tree.DInt(s.mu.data.MaxRetries)), // max_retries
+ tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.Mean)), // service_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.ServiceLat.GetVariance(s.mu.data.Count))), // service_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.Mean)), // retry_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.RetryLat.GetVariance(s.mu.data.Count))), // retry_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.Mean)), // commit_lat_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.CommitLat.GetVariance(s.mu.data.Count))), // commit_lat_var
+ tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.Mean)), // rows_read_avg
+ tree.NewDFloat(tree.DFloat(s.mu.data.NumRows.GetVariance(s.mu.data.Count))), // rows_read_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes), // network_bytes_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkBytes), // network_bytes_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages), // network_msgs_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.NetworkMessages), // network_msgs_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage), // max_mem_usage_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxMemUsage), // max_mem_usage_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage), // max_disk_usage_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.MaxDiskUsage), // max_disk_usage_var
+ execStatAvg(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime), // contention_time_avg
+ execStatVar(s.mu.data.ExecStats.Count, s.mu.data.ExecStats.ContentionTime), // contention_time_var
  )
 
  s.mu.Unlock()

diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal
@@ -153,15 +153,15 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
 ----
 node_id table_id name parent_id expiration deleted
 
-query ITTTTIIITRRRRRRRRRRRRRRRRR colnames
+query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
 ----
-node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var implicit_txn
+node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn
 
-query ITTTIIRRRRRRRR colnames
+query ITTTIIRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
 ----
-node_id application_name key statement_ids count max_retries service_lat_avg service_lat_var retry_lat_avg retry_lat_var commit_lat_avg commit_lat_var rows_read_avg rows_read_var
+node_id application_name key statement_ids count max_retries service_lat_avg service_lat_var retry_lat_avg retry_lat_var commit_lat_avg commit_lat_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var
 
 query IITTTTTTT colnames
 SELECT * FROM crdb_internal.session_trace WHERE span_idx < 0

diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant b/pkg/sql/logictest/testdata/logic_test/crdb_internal_tenant
@@ -165,15 +165,15 @@ SELECT * FROM crdb_internal.leases WHERE node_id < 0
 ----
 node_id table_id name parent_id expiration deleted
 
-query ITTTTIIITRRRRRRRRRRRRRRRRR colnames
+query ITTTTIIITRRRRRRRRRRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_statement_statistics WHERE node_id < 0
 ----
-node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var implicit_txn
+node_id application_name flags key anonymized count first_attempt_count max_retries last_error rows_avg rows_var parse_lat_avg parse_lat_var plan_lat_avg plan_lat_var run_lat_avg run_lat_var service_lat_avg service_lat_var overhead_lat_avg overhead_lat_var bytes_read_avg bytes_read_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var implicit_txn
 
-query ITTTIIRRRRRRRR colnames
+query ITTTIIRRRRRRRRRRRRRRRRRR colnames
 SELECT * FROM crdb_internal.node_transaction_statistics WHERE node_id < 0
 ----
-node_id application_name key statement_ids count max_retries service_lat_avg service_lat_var retry_lat_avg retry_lat_var commit_lat_avg commit_lat_var rows_read_avg rows_read_var
+node_id application_name key statement_ids count max_retries service_lat_avg service_lat_var retry_lat_avg retry_lat_var commit_lat_avg commit_lat_var rows_read_avg rows_read_var network_bytes_avg network_bytes_var network_msgs_avg network_msgs_var max_mem_usage_avg max_mem_usage_var max_disk_usage_avg max_disk_usage_var contention_time_avg contention_time_var
 
 query IITTTTTTT colnames
 SELECT * FROM crdb_internal.session_trace WHERE span_idx < 0