feat(runner): add p95 to stats summaries

Agoric · Feb 2, 2022 · 1ca055b · 1ca055b
1 parent f98e081
commit 1ca055b
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 1 deletion.
diff --git a/runner/lib/stats/blocks.js b/runner/lib/stats/blocks.js
@@ -90,6 +90,7 @@ export const makeBlockStatsSummary = ({
   items,
   mins,
   maxes,
+  p95s,
 }) =>
   blockCount
     ? {
@@ -116,6 +117,19 @@ export const makeBlockStatsSummary = ({
         avgProcessingPercentage: percentageRounder(
           averages.processingPercentage / 100,
         ),
+        p95Lag: timeRounder(p95s.lag),
+        p95BlockDuration: timeRounder(p95s.blockDuration),
+        p95ChainBlockDuration: timeRounder(p95s.chainBlockDuration),
+        p95IdleTime: timeRounder(p95s.idleTime),
+        p95CosmosTime: timeRounder(p95s.cosmosTime),
+        p95SwingsetTime: timeRounder(p95s.swingsetTime),
+        p95ProcessingTime: timeRounder(p95s.processingTime),
+        p95Deliveries: timeRounder(p95s.deliveries),
+        p95Computrons: timeRounder(p95s.computrons),
+        p95SwingsetPercentage: percentageRounder(p95s.swingsetPercentage / 100),
+        p95ProcessingPercentage: percentageRounder(
+          p95s.processingPercentage / 100,
+        ),
       }
     : undefined;
 

diff --git a/runner/lib/stats/cycles.js b/runner/lib/stats/cycles.js
@@ -38,13 +38,19 @@ const rawCycleStatsInit = {
  * @param {import('./helpers.js').Summary<CycleStatsSumKeys>} sums
  * @returns {import('./types.js').CycleStatsSummary | undefined}
  */
-export const makeCycleStatsSummary = ({ weights: cycleCount, averages }) =>
+export const makeCycleStatsSummary = ({
+  weights: cycleCount,
+  averages,
+  p95s,
+}) =>
   cycleCount
     ? {
         cycleCount,
         cycleSuccessRate: percentageRounder(averages.success),
         avgBlockCount: rounder(averages.blockCount),
         avgDuration: rounder(averages.duration),
+        p95BlockCount: rounder(p95s.blockCount),
+        p95Duration: rounder(p95s.duration),
       }
     : undefined;
 

diff --git a/runner/lib/stats/helpers.d.ts b/runner/lib/stats/helpers.d.ts
@@ -95,6 +95,7 @@ export type Summary<K extends string> = {
   readonly totals: SumRecord<K>; // weighted
   readonly counts: SumRecord<K>; // weighted
   readonly averages: SumRecord<K>; // weighted
+  readonly p95s: SumRecord<K>;
 };
 
 type SummaryRecord = { readonly [P in string]: number | undefined };

diff --git a/runner/lib/stats/helpers.js b/runner/lib/stats/helpers.js
@@ -163,6 +163,7 @@ export const summarize = (data) => {
   const totals = /** @type {Record<string, number>} */ ({});
   const counts = /** @type {Record<string, number>} */ ({});
   const averages = /** @type {Record<string, number>} */ ({});
+  const p95s = /** @type {Record<string, number>} */ ({});
 
   for (const key of keys) {
     const sortedData = /** @type {Array<{values: Record<string, number>, weight?: number | undefined}>} */ (data.filter(
@@ -179,6 +180,21 @@ export const summarize = (data) => {
       counts[key] += weight;
     }
     averages[key] = totals[key] / counts[key];
+
+    if (
+      sortedData.length > 1 &&
+      sortedData.every(({ weight = 1 }) => weight === 1)
+    ) {
+      const rank = (95 * (sortedData.length - 1)) / 100;
+      const rankIndex = Math.floor(rank);
+      const basePercentile = sortedData[rankIndex].values[key];
+      const nextPercentile = sortedData[rankIndex + 1].values[key];
+      p95s[key] =
+        basePercentile +
+        (rankIndex - rankIndex) * (nextPercentile - basePercentile);
+    } else {
+      p95s[key] = NaN;
+    }
   }
 
   return harden({
@@ -190,5 +206,6 @@ export const summarize = (data) => {
     totals,
     counts,
     averages,
+    p95s,
   });
 };
diff --git a/runner/lib/stats/types.d.ts b/runner/lib/stats/types.d.ts
@@ -64,6 +64,17 @@ export type BlockStatsSummary = {
   readonly avgProcessingPercentage: number;
   readonly avgDeliveries: number;
   readonly avgComputrons: number;
+  readonly p95Lag: number;
+  readonly p95BlockDuration: number;
+  readonly p95ChainBlockDuration: number;
+  readonly p95IdleTime: number;
+  readonly p95CosmosTime: number;
+  readonly p95SwingsetTime: number;
+  readonly p95ProcessingTime: number;
+  readonly p95SwingsetPercentage: number;
+  readonly p95ProcessingPercentage: number;
+  readonly p95Deliveries: number;
+  readonly p95Computrons: number;
 };
 
 export interface CycleStatsInitData {
@@ -89,6 +100,8 @@ export type CycleStatsSummary = {
   readonly cycleCount: number;
   readonly avgBlockCount: number;
   readonly avgDuration: number;
+  readonly p95BlockCount: number | undefined;
+  readonly p95Duration: number | undefined;
   readonly cycleSuccessRate: number;
 };