Add platform (arch) for ExecuTorch dashboard (#6234)

Address part of pytorch/executorch#7986 The historical context here is that the older schema for benchmark database didn't have a separate columns for `device`, i.e. `cpu`, and `arch`, i.e. `x86_64`. So, they are bundle together into a single one called `deviceArch`. This PR splits this field into `device` and `arch` field behind the scene, but keeps the display name in the same format, i.e. ` Apple iPhone 15 (iOS 18.0)` to maintain the same UX, which allows me to add a new platform (arch) dropdown list for ExecuTorch. The list has only 3 options `Android` and `iOS` and `All Platforms`, ~~and only one can be selected at a time (this is the part that fixes pytorch/executorch#7986 This also applies lint to these queries and fixes a bug in which sqlfluff doesn't recognize the configuration file at `.sqlfluff` I found that https://mui.com/x/react-data-grid/pagination/ only allows up to 100 rows before enforcing pagination. Having more requires the paid pro version. ### Testing https://torchci-git-fork-huydhn-tweak-et-dashboard-fbopensource.vercel.app/benchmark/llms?repoName=pytorch%2Fexecutorch shows a new platform dropdown list
pytorch · Jan 31, 2025 · 1bc53d7 · 1bc53d7
1 parent 063abf8
commit 1bc53d7
Show file tree

Hide file tree

Showing 11 changed files with 93 additions and 48 deletions.
diff --git a/.lintrunner.toml b/.lintrunner.toml
@@ -360,7 +360,12 @@ is_formatter = true
 [[linter]]
 code = 'SQLFLUFF'
 # include_patterns = ['torchci/clickhouse_queries/**/*.sql']
-include_patterns = ['torchci/clickhouse_queries/workflow_load/query.sql']
+include_patterns = [
+    'torchci/clickhouse_queries/workflow_load/query.sql',
+    'torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql',
+    'torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql',
+    'torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql',
+]
 exclude_patterns = [
 ]
 command = [

diff --git a/tools/linter/adapters/sqlfluff_linter.py b/tools/linter/adapters/sqlfluff_linter.py
@@ -78,6 +78,8 @@ def check_file(
             [
                 "sqlfluff",
                 "format",
+                "--config",
+                os.path.join(os.getcwd(), ".sqlfluff"),
                 "--dialect",
                 "clickhouse",
                 tmp.name,

diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json
@@ -1,6 +1,7 @@
 {
   "params": {
-    "deviceArch": "String",
+    "arch": "String",
+    "device": "String",
     "dtypes": "Array(String)",
     "excludedMetrics": "Array(String)",
     "benchmarks": "Array(String)",
@@ -11,4 +12,4 @@
     "stopTime": "DateTime64(3)"
   },
   "tests": []
-}
+}
diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql
@@ -1,7 +1,7 @@
 -- This query is used to get the list of branches and commits used by different
 -- OSS CI benchmark experiments. This powers HUD benchmarks dashboards
-SELECT
-    DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS head_branch,
+SELECT DISTINCT
+    replaceOne(head_branch, 'refs/heads/', '') AS head_branch,
     head_sha,
     workflow_id AS id,
     toStartOfDay(fromUnixTimestamp(timestamp)) AS event_time
@@ -32,17 +32,15 @@ WHERE
         OR empty({excludedMetrics: Array(String) })
     )
     AND notEmpty(metric_name)
-    -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields
     AND (
-        CONCAT(
-            device,
-            ' (',
-            IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch),
-            ')'
-        ) = {deviceArch: String }
-        OR {deviceArch: String } = ''
+        startsWith({device: String }, device)
+        OR {device: String } = ''
     )
     AND notEmpty(device)
+    AND (
+        arch LIKE concat('%', {arch: String }, '%')
+        OR {arch: String } = ''
+    )
 ORDER BY
     head_branch,
     timestamp DESC
diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json
@@ -1,8 +1,9 @@
 {
   "params": {
+    "arch": "String",
     "branches": "Array(String)",
     "commits": "Array(String)",
-    "deviceArch": "String",
+    "device": "String",
     "dtypes": "Array(String)",
     "excludedMetrics": "Array(String)",
     "benchmarks": "Array(String)",
@@ -14,4 +15,4 @@
     "stopTime": "DateTime64(3)"
   },
   "tests": []
-}
+}
diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql
@@ -15,18 +15,18 @@ WITH benchmarks AS (
         o.benchmark.dtype AS dtype,
         IF(
             empty(o.runners),
-            tupleElement(o.benchmark, 'extra_info') [ 'device' ],
-            tupleElement(o.runners [ 1 ], 'name')
+            tupleElement(o.benchmark, 'extra_info')['device'],
+            tupleElement(o.runners[1], 'name')
         ) AS device,
         IF(
             empty(o.runners),
-            tupleElement(o.benchmark, 'extra_info') [ 'arch' ],
-            tupleElement(o.runners [ 1 ], 'type')
+            tupleElement(o.benchmark, 'extra_info')['arch'],
+            tupleElement(o.runners[1], 'type')
         ) AS arch,
         IF(
-            tupleElement(o.benchmark, 'extra_info') [ 'compile' ] = '',
+            tupleElement(o.benchmark, 'extra_info')['compile'] = '',
             'true',  -- Default to true
-            tupleElement(o.benchmark, 'extra_info') [ 'compile' ]
+            tupleElement(o.benchmark, 'extra_info')['compile']
         ) AS use_torch_compile,
         DATE_TRUNC(
             {granularity: String },
@@ -64,8 +64,9 @@ WITH benchmarks AS (
         )
         AND notEmpty(o.metric.name)
 )
-SELECT
-    DISTINCT workflow_id,
+
+SELECT DISTINCT
+    workflow_id,
     job_id,
     model,
     backend,
@@ -85,17 +86,15 @@ WHERE
         has({branches: Array(String) }, head_branch)
         OR empty({branches: Array(String) })
     )
-    -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields
     AND (
-        CONCAT(
-            device,
-            ' (',
-            IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch),
-            ')'
-        ) = {deviceArch: String }
-        OR {deviceArch: String } = ''
+        startsWith({device: String }, device)
+        OR {device: String } = ''
     )
     AND notEmpty(device)
+    AND (
+        arch LIKE concat('%', {arch: String }, '%')
+        OR {arch: String } = ''
+    )
 ORDER BY
     granularity_bucket DESC,
     workflow_id DESC,

diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json
@@ -1,6 +1,7 @@
 {
   "params": {
-    "deviceArch": "String",
+    "arch": "String",
+    "device": "String",
     "dtypes": "Array(String)",
     "excludedMetrics": "Array(String)",
     "benchmarks": "Array(String)",
@@ -11,4 +12,4 @@
     "stopTime": "DateTime64(3)"
   },
   "tests": []
-}
+}
diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql
@@ -1,6 +1,6 @@
 --- This query is used by HUD benchmarks dashboards to get the list of experiment names
-SELECT
-    DISTINCT benchmark_name AS benchmark,
+SELECT DISTINCT
+    benchmark_name AS benchmark,
     model_name AS model,
     model_backend AS backend,
     metric_name AS metric,
@@ -34,17 +34,15 @@ WHERE
         OR empty({excludedMetrics: Array(String) })
     )
     AND notEmpty(metric_name)
-    -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields
     AND (
-        CONCAT(
-            device,
-            ' (',
-            IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch),
-            ')'
-        ) = {deviceArch: String }
-        OR {deviceArch: String } = ''
+        startsWith({device: String }, device)
+        OR {device: String } = ''
     )
     AND notEmpty(device)
+    AND (
+        arch LIKE concat('%', {arch: String }, '%')
+        OR {arch: String } = ''
+    )
 ORDER BY
     benchmark,
     backend,

diff --git a/torchci/components/benchmark/llms/SummaryPanel.tsx b/torchci/components/benchmark/llms/SummaryPanel.tsx
@@ -23,6 +23,7 @@ export function SummaryPanel({
   modelName,
   backendName,
   metricNames,
+  archName,
   lPerfData,
   rPerfData,
 }: {
@@ -33,6 +34,7 @@ export function SummaryPanel({
   modelName: string;
   backendName: string;
   metricNames: string[];
+  archName: string;
   lPerfData: BranchAndCommitPerfData;
   rPerfData: BranchAndCommitPerfData;
 }) {
@@ -75,13 +77,15 @@ export function SummaryPanel({
           params.value.backend !== undefined
             ? `&backendName=${encodeURIComponent(params.value.backend)}`
             : "";
-        const deviceArch = `${params.value.device} (${params.value.arch})`;
+        const deviceName = `${params.value.device} (${params.value.arch})`;
 
         const url = `/benchmark/llms?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&repoName=${encodeURIComponent(
           repoName
         )}&modelName=${encodeURIComponent(
           model
-        )}${backend}${dtype}&deviceName=${encodeURIComponent(deviceArch)}`;
+        )}${backend}${dtype}&deviceName=${encodeURIComponent(
+          deviceName
+        )}&archName=${encodeURIComponent(archName)}`;
 
         return (
           <a href={url}>
@@ -215,17 +219,23 @@ export function SummaryPanel({
     ]
   );
 
+  // TODO (huydhn): Table bigger than 100 rows requires x-data-grid-pro
   return (
     <Grid2 container spacing={2} style={{ height: "100%" }}>
       <Grid2
         size={{ xs: 12, lg: 12 }}
-        height={data.length * ROW_HEIGHT + ROW_GAP}
+        height={
+          data.length > 99
+            ? 99 * ROW_HEIGHT
+            : data.length * ROW_HEIGHT + ROW_GAP
+        }
       >
         <TablePanelWithData
           title={"Models"}
           data={data}
           columns={columns}
           dataGridProps={{ getRowId: (el: any) => el.name }}
+          showFooter={true}
         />
       </Grid2>
     </Grid2>

diff --git a/torchci/components/benchmark/llms/common.tsx b/torchci/components/benchmark/llms/common.tsx
@@ -35,9 +35,15 @@ export const METRIC_DISPLAY_SHORT_HEADERS: { [k: string]: string } = {
   "compilation_time(s)": "CompTime",
 };
 export const DEFAULT_DEVICE_NAME = "All Devices";
+export const DEFAULT_ARCH_NAME = "All Platforms";
 export const DEFAULT_DTYPE_NAME = "All DType";
 export const DEFAULT_BACKEND_NAME = "All Backends";
 
+// Only used by ExecuTorch for now
+export const ARCH_NAMES: { [k: string]: string[] } = {
+  "pytorch/executorch": ["Android", "iOS"],
+};
+
 // Relative thresholds
 export const RELATIVE_THRESHOLD = 0.05;
 

diff --git a/torchci/pages/benchmark/llms.tsx b/torchci/pages/benchmark/llms.tsx
@@ -7,6 +7,8 @@ import {
   MAIN_BRANCH,
 } from "components/benchmark/common";
 import {
+  ARCH_NAMES,
+  DEFAULT_ARCH_NAME,
   DEFAULT_BACKEND_NAME,
   DEFAULT_DEVICE_NAME,
   DEFAULT_DTYPE_NAME,
@@ -45,6 +47,7 @@ function Report({
   backendName,
   dtypeName,
   deviceName,
+  archName,
   metricNames,
   lBranchAndCommit,
   rBranchAndCommit,
@@ -58,6 +61,7 @@ function Report({
   backendName: string;
   dtypeName: string;
   deviceName: string;
+  archName: string;
   metricNames: string[];
   lBranchAndCommit: BranchAndCommit;
   rBranchAndCommit: BranchAndCommit;
@@ -146,6 +150,7 @@ function Report({
         modelName={modelName}
         backendName={backendName}
         metricNames={metricNames}
+        archName={archName}
         lPerfData={{
           ...lBranchAndCommit,
           data: lDataWithSpeedup,
@@ -178,6 +183,7 @@ export default function Page() {
   const [backendName, setBackendName] = useState<string>(DEFAULT_BACKEND_NAME);
   const [dtypeName, setDTypeName] = useState<string>(DEFAULT_DTYPE_NAME);
   const [deviceName, setDeviceName] = useState<string>(DEFAULT_DEVICE_NAME);
+  const [archName, setArchName] = useState<string>(DEFAULT_ARCH_NAME);
 
   // Set the dropdown value what is in the param
   useEffect(() => {
@@ -231,6 +237,12 @@ export default function Page() {
       setDeviceName(deviceName);
     }
 
+    // Set the default arch to Android for ExecuTorch as it has only 2 options Android and iOS
+    const archName: string = (router.query.archName as string) ?? undefined;
+    if (archName !== undefined) {
+      setArchName(archName);
+    }
+
     const lBranch: string = (router.query.lBranch as string) ?? undefined;
     if (lBranch !== undefined) {
       setLBranch(lBranch);
@@ -260,7 +272,8 @@ export default function Page() {
 
   const queryName = "oss_ci_benchmark_names";
   const queryParams = {
-    deviceArch: deviceName === DEFAULT_DEVICE_NAME ? "" : deviceName,
+    arch: archName === DEFAULT_ARCH_NAME ? "" : archName,
+    device: deviceName === DEFAULT_DEVICE_NAME ? "" : deviceName,
     dtypes:
       dtypeName === DEFAULT_DTYPE_NAME
         ? []
@@ -326,7 +339,9 @@ export default function Page() {
             backendName
           )}&dtypeName=${encodeURIComponent(
             dtypeName
-          )}&deviceName=${encodeURIComponent(deviceName)}`}
+          )}&deviceName=${encodeURIComponent(
+            deviceName
+          )}&archName=${encodeURIComponent(archName)}`}
         />
       </Stack>
       <Stack direction="row" spacing={2} sx={{ mb: 2 }}>
@@ -365,6 +380,14 @@ export default function Page() {
             label={"DType"}
           />
         )}
+        {repoName === "pytorch/executorch" && (
+          <DTypePicker
+            dtype={archName}
+            setDType={setArchName}
+            dtypes={[DEFAULT_ARCH_NAME, ...ARCH_NAMES[repoName]]}
+            label={"Platform"}
+          />
+        )}
         <DTypePicker
           dtype={deviceName}
           setDType={setDeviceName}
@@ -407,6 +430,7 @@ export default function Page() {
         backendName={backendName}
         dtypeName={dtypeName}
         deviceName={deviceName}
+        archName={archName}
         metricNames={metricNames}
         lBranchAndCommit={{ branch: lBranch, commit: lCommit }}
         rBranchAndCommit={{ branch: rBranch, commit: rCommit }}