From 848fa6e9eb89be8955b8c8cf61bff2a6400e3a10 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 2 Oct 2025 10:40:04 -0700 Subject: [PATCH 01/19] add config --- .../v3/components/benchmarkList/BenchmarkCategoryCard.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx b/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx index 85c6e7b032..78f233f9fc 100644 --- a/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx @@ -15,6 +15,7 @@ import { Typography, } from "@mui/material"; import { Box } from "@mui/system"; +import { wrap } from "module"; import Link from "next/link"; import ReactMarkdown from "react-markdown"; From e1c081230d18de21e65b546fe50803c3229a96b3 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 8 Oct 2025 19:33:37 -0700 Subject: [PATCH 02/19] add config --- .../v3/components/benchmarkList/BenchmarkCategoryCard.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx b/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx index 78f233f9fc..85c6e7b032 100644 --- a/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkList/BenchmarkCategoryCard.tsx @@ -15,7 +15,6 @@ import { Typography, } from "@mui/material"; import { Box } from "@mui/system"; -import { wrap } from "module"; import Link from "next/link"; import ReactMarkdown from "react-markdown"; From da82a7b7b7a728be59524d001fae4d207a36e464 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 13 Oct 2025 18:25:37 -0700 Subject: [PATCH 03/19] add config --- .../list_commit_query/params.json | 14 ++ .../benchmark_v3/list_commit_query/query.sql | 34 +++ .../params.json | 3 +- .../benchmark/llms/LLMsBenchmarkPage.tsx | 2 + .../components/CommitWorkfowSelectSection.tsx | 2 +- .../components/SideBarMainSection.tsx | 2 +- .../BenchmarkReportFeatureNotification.tsx | 2 +- .../BenchmarkRegressionReportListWrapper.tsx | 4 +- .../RegressionReportViewWrapper.tsx | 2 +- .../fanout/defaultFanoutRenderContent.tsx | 2 +- .../v3/configs/teams/compilers/config.ts | 2 +- .../api_helper/backend/get_time_series_api.ts | 39 ++++ .../api_helper/backend/list_metadata_api.ts | 113 ++++++++++ .../defaultGetBenchmarkDataQueryBuilder.ts | 194 ++++++++++++++++++ .../defaultListMetadataQueryBuilder.ts | 66 ++++++ .../backend/queryBuilderUtils/queryBuilder.ts | 179 ++++++++++++++++ .../compilers/get_compiler_benchmark_data.ts | 60 +----- .../api_helper/compilers/helpers/general.ts | 6 - .../api_helper/compilers/helpers/type.ts | 5 + .../benchmark/api_helper/{apis => fe}/api.ts | 0 .../api_helper/{apis => fe}/hooks.ts | 0 .../lib/benchmark/api_helper/list_commits.ts | 112 ++++++++++ torchci/lib/benchmark/api_helper/type.ts | 24 +++ torchci/lib/benchmark/api_helper/utils.ts | 58 ++++++ torchci/pages/api/benchmark/list_commits.ts | 87 +------- torchci/pages/api/benchmark/list_metadata.ts | 34 +++ .../benchmark/regression/report/[id].tsx | 2 +- 27 files changed, 893 insertions(+), 155 deletions(-) create mode 100644 torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json create mode 100644 torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql create mode 100644 torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts create mode 100644 torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts create mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts create mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts create mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts create mode 100644 torchci/lib/benchmark/api_helper/compilers/helpers/type.ts rename torchci/lib/benchmark/api_helper/{apis => fe}/api.ts (100%) rename torchci/lib/benchmark/api_helper/{apis => fe}/hooks.ts (100%) create mode 100644 torchci/lib/benchmark/api_helper/list_commits.ts create mode 100644 torchci/pages/api/benchmark/list_metadata.ts diff --git a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json new file mode 100644 index 0000000000..047514f175 --- /dev/null +++ b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json @@ -0,0 +1,14 @@ +{ + "params": { + "repo": "String", + "benchmarkNames": "Array", + "branches": "Array", + "device": "String", + "arch": "Array(String)", + "dtype": "String", + "mode": "String", + "startTime": "DateTime64(3)", + "stopTime": "DateTime64(3)" + }, + "tests": [] +} diff --git a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql new file mode 100644 index 0000000000..f087f1f1bb --- /dev/null +++ b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql @@ -0,0 +1,34 @@ +SELECT + replaceOne(head_branch, 'refs/heads/', '') AS branch, + head_sha AS commit, + workflow_id, + toStartOfHour(min(fromUnixTimestamp(timestamp))) AS date +FROM + benchmark.oss_ci_benchmark_metadata +PREWHERE + timestamp >= toUnixTimestamp({startTime: DateTime64(3)}) + AND timestamp < toUnixTimestamp({stopTime: DateTime64(3)} +WHERE + repo = {repo: String } + AND ( + has( + {branches: Array(String)}, + replaceOne(head_branch, 'refs/heads/', '') + ) + OR empty({branches: Array(String)}) + ) + AND ( + has({benchmarkNames: Array(String) }, benchmark_name) + OR empty({benchmarkNames: Array(String) }) + ) + AND (benchmark_dtype = {dtype: String} OR empty({dtype: String})) + AND notEmpty(metric_name) + AND notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) +GROUP BY + branch, commit, workflow_id +ORDER BY + branch, date diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/params.json b/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/params.json index 858019bc52..98820faebd 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/params.json +++ b/torchci/clickhouse_queries/compilers_benchmark_api_commit_query/params.json @@ -7,8 +7,7 @@ "mode": "String", "startTime": "DateTime64(3)", "stopTime": "DateTime64(3)", - "suites": "Array(String)", - "workflowIds": "Array(Int64)" + "suites": "Array(String)" }, "tests": [] } diff --git a/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx b/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx index 56335ca289..a80201f8a3 100644 --- a/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx +++ b/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx @@ -139,6 +139,8 @@ const MainPage = ({ ); }, [router.query]); const queryParams = getLLMsBenchmarkPropsQueryParameter(props); + + console.log("elainewy query params", queryParams); const { data, error, isLoading } = useBenchmarkPropsData(queryParams); // an error occured while fetching the benchmark props data diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx index 2d19e42d9a..c4a965a9c7 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx @@ -2,7 +2,7 @@ import { Typography } from "@mui/material"; import { Stack } from "@mui/system"; import { QueryParameterConverterInputs } from "components/benchmark/v3/configs/utils/dataBindingRegistration"; import { UMDenseCommitDropdown } from "components/uiModules/UMDenseComponents"; -import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/apis/hooks"; +import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store"; import { useEffect, useState } from "react"; diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx index e8918881bd..44a5c509ca 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx @@ -8,7 +8,7 @@ import { CenteredLoader } from "components/common/LoadingIcon"; import { UMCopyLink } from "components/uiModules/UMCopyLink"; import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker"; import { UMDenseButtonLight } from "components/uiModules/UMDenseComponents"; -import dayjs from "dayjs"; +import dayjs from "dayjs";lib/benchmark/api_helper/fe/hooks import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/apis/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { useRouter } from "next/router"; diff --git a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureNotification.tsx b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureNotification.tsx index 221e71f158..0a0c7aecef 100644 --- a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureNotification.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureNotification.tsx @@ -1,7 +1,7 @@ import { Badge, IconButton, Tooltip } from "@mui/material"; import { Box } from "@mui/system"; import dayjs from "dayjs"; -import { useListBenchmarkRegressionReportsData } from "lib/benchmark/api_helper/apis/hooks"; +import { useListBenchmarkRegressionReportsData } from "lib/benchmark/api_helper/fe/hooks"; import { RiNotification2Fill } from "react-icons/ri"; import { BenchmarkReportFeatureSidePanel } from "./BenchmarkReportFeatureSidePanel"; import { diff --git a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/listView/BenchmarkRegressionReportListWrapper.tsx b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/listView/BenchmarkRegressionReportListWrapper.tsx index 63b98210b4..8fed6055f9 100644 --- a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/listView/BenchmarkRegressionReportListWrapper.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/listView/BenchmarkRegressionReportListWrapper.tsx @@ -1,7 +1,7 @@ import { Alert } from "@mui/material"; import LoadingPage from "components/common/LoadingPage"; -import { listBenchmarkRegressionReport } from "lib/benchmark/api_helper/apis/api"; -import { useListBenchmarkRegressionReportsData } from "lib/benchmark/api_helper/apis/hooks"; +import { listBenchmarkRegressionReport } from "lib/benchmark/api_helper/fe/api"; +import { useListBenchmarkRegressionReportsData } from "lib/benchmark/api_helper/fe/hooks"; import { useEffect, useState } from "react"; import RegressionReportList from "./RegressionReportListView"; diff --git a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/RegressionReportViewWrapper.tsx b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/RegressionReportViewWrapper.tsx index 2f70fa2eba..09af3c73cc 100644 --- a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/RegressionReportViewWrapper.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/RegressionReportViewWrapper.tsx @@ -1,7 +1,7 @@ import { Alert } from "@mui/material"; import { Box } from "@mui/system"; import LoadingPage from "components/common/LoadingPage"; -import { useGetBenchmarkRegressionReportData } from "lib/benchmark/api_helper/apis/hooks"; +import { useGetBenchmarkRegressionReportData } from "lib/benchmark/api_helper/fe/hooks"; import BenchmarkRegressionReportMetadataSection from "./BenchmarkRegressionReportMetadataSection"; import { RegressionReportDetail } from "./RegressionReportDetail"; diff --git a/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx b/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx index 5d2e063765..b9d49a1a2f 100644 --- a/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx @@ -4,7 +4,7 @@ import { HighlightStyles } from "components/benchmark/v3/components/common/highl import { getConfig } from "components/benchmark/v3/configs/configBook"; import { getFanoutRenderComponent } from "components/benchmark/v3/configs/utils/fanoutRegistration"; import LoadingPage from "components/common/LoadingPage"; -import { useBenchmarkData } from "lib/benchmark/api_helper/apis/hooks"; +import { useBenchmarkData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store"; import { useState } from "react"; diff --git a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts index 63788f9779..936c74d209 100644 --- a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts +++ b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts @@ -7,7 +7,7 @@ import { SUITES } from "components/benchmark/compilers/SuitePicker"; import { DEFAULT_MODE, MODES } from "components/benchmark/ModeAndDTypePicker"; import dayjs from "dayjs"; import utc from "dayjs/plugin/utc"; -import { REQUIRED_COMPLIER_LIST_COMMITS_KEYS } from "lib/benchmark/api_helper/compilers/helpers/general"; +import { REQUIRED_COMPLIER_LIST_COMMITS_KEYS } from "lib/benchmark/api_helper/compilers/helpers/type"; import { DISPLAY_NAMES_TO_COMPILER_NAMES } from "../../../../compilers/common"; import { BenchmarkUIConfig } from "../../configBook"; import { BenchmarkComparisonPolicyConfig } from "../../helpers/RegressionPolicy"; diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts new file mode 100644 index 0000000000..4506445c57 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts @@ -0,0 +1,39 @@ +import { getCompilerBenchmarkData } from "../compilers/get_compiler_benchmark_data"; +import { CompilerQueryType } from "../type"; +import { BenchmarkDataQuery } from "./queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder"; + +export async function getBenmarkTimeSeriesData( + request_name: string, + query_params: any, + formats: string[] = ["time_series"] +) { + switch (request_name) { + case "compiler_precompute": + return await getCompilerBenchmarkData( + query_params, + CompilerQueryType.PRECOMPUTE, + formats + ); + case "compiler": + return await getCompilerBenchmarkData( + query_params, + CompilerQueryType.GENERAL, + formats + ); + case "pytorch_operator_microbenchmak": + return await getBenchmarkData(query_params, formats, request_name); + + default: + throw new Error(`Unsupported request_name: ${request_name}`); + } +} + +export async function getBenchmarkData( + query_params: any, + formats: string[], + id: string +) { + const queryBuilder = new BenchmarkDataQuery(); + + const query = queryBuilder.applyQuery(query_params); +} diff --git a/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts b/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts new file mode 100644 index 0000000000..198f15b117 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts @@ -0,0 +1,113 @@ +import { + DEFAULT_BACKEND_NAME, + DEFAULT_DEVICE_NAME, + DEFAULT_DTYPE_NAME, + DEFAULT_MODE_NAME, +} from "lib/benchmark/llms/common"; +import ld from "lodash"; +import { BenchmarkMetadataItem, BenchmarkMetadataType } from "../type"; +import { BenchmarkMetadataQuery } from "./queryBuilderUtils/defaultListMetadataQueryBuilder"; +export async function listBenchmarkMetadata(queryParams: any, id: string) { + // fetch metadata from db + const db = await listBenchmarkMetadataFromDb(queryParams); + const result = groupBenchmarkMetadata(db, id); + return result; +} + +async function listBenchmarkMetadataFromDb(queryParams: any) { + const queryBuilder = new BenchmarkMetadataQuery(); + const results = queryBuilder.applyQuery(queryParams); + return results; +} + +/** + * grouping benchmark metadata into group list + */ +function groupBenchmarkMetadata(data: any, id: string = "") { + let groups = getDefaultBenchmarkMetadataGroup(data); + const additionalGroups = getCustomizedBenchmarkMetadataGroup(data, id); + return groups.concat(additionalGroups); +} + +function getCustomizedBenchmarkMetadataGroup(data: any, id: string) { + let items = []; + switch (id) { + case "pytorch_operator_microbenchmark": + const item = makeMetadataItem( + data, + "operator", + BenchmarkMetadataType.OperatornName, + "All Operators", + "Operator", + (r) => r.model.split("_")[0] ?? "unknown" + ); + if (item) { + items.push(item); + } + return items; + default: + return []; + } +} + +export function getDefaultBenchmarkMetadataGroup( + data: any[] +): BenchmarkMetadataItem[] { + return [ + makeMetadataItem( + data, + "backend", + BenchmarkMetadataType.BackendName, + DEFAULT_BACKEND_NAME, + "Backend" + ), + makeMetadataItem( + data, + "mode", + BenchmarkMetadataType.ModeName, + DEFAULT_MODE_NAME, + "Mode" + ), + makeMetadataItem( + data, + "dtype", + BenchmarkMetadataType.DtypeName, + DEFAULT_DTYPE_NAME, + "Dtype" + ), + makeMetadataItem( + data, + "device", + BenchmarkMetadataType.DeviceName, + DEFAULT_DEVICE_NAME, + "Device Name", + (r) => (r.device && r.arch ? `${r.device} (${r.arch})` : r.device) + ), + ].filter(Boolean) as BenchmarkMetadataItem[]; +} + +/** + * find distinct values from data and create metadata item + */ +function makeMetadataItem( + data: any[], + field: string, + type: BenchmarkMetadataType, + defaultValue: string, + labelName: string, + formatter?: (r: any) => string | undefined +): BenchmarkMetadataItem | null { + const values = ld.uniq( + data.map((r) => (formatter ? formatter(r) : r[field])).filter(Boolean) + ) as string[]; + + if (values.length === 0) { + return null; + } + + return { + type, + options: [defaultValue, ...values], + labelName, + }; +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts new file mode 100644 index 0000000000..f6e019d89b --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts @@ -0,0 +1,194 @@ +import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; + +/** + * Query to get benchmark data from the benchmark table + * for repo/benchmark specific data use: + * - addInnerExtraInfo(): if the field is also in the extra column as unique key + * - addInnerMetadataInfo(): if it's only metadata info and does not act as unique key + */ +export class BenchmarkDataQuery extends ExecutableQueryBase { + private _inner_query_builder: QueryBuilder; + private _main_query_builder: QueryBuilder; + + // must included in all select statement + private _required_metadata_info_statements: Map; + constructor() { + super(); + this._required_metadata_info_statements = new Map([ + [ + "timestamp", + "formatDateTime(fromUnixTimestamp(timestamp), '%Y-%m-%dT%H:%i:%sZ')", + ], + ]); + const metadata_infos = this._required_metadata_info_statements.forEach( + (statement, key) => { + return `${key},${statement}`; + } + ); + this._inner_query_builder = new QueryBuilder( + { + table: "benchmark.oss_ci_benchmark_v3", + select_exists: true, + where_exists: true, + // default select statement for customized query + select: [ + ["map()", "extra"], + [`map(${metadata_infos})`, "metadata_info"], + ], + prewhere: [ + "timestamp >= toUnixTimestamp({startTime: DateTime64(3) })", + "timestamp < toUnixTimestamp({endTime: DateTime64(3) })", + ], + }, + ` + SELECT + replaceOne(head_branch, 'refs/heads/', '') AS head_branch, + workflow_id AS workflow_id, + job_id AS job_id, + model.'name' AS model, + model.'backend' AS backend, + model.'origins' AS origins, + metric.'name' AS metric, + floor(arrayAvg(metric.'benchmark_values'), 2) AS value, + floor(toFloat64(metric.'target_value'), 2) AS target, + benchmark.'mode' AS mode, + benchmark.'dtype' AS dtype + IF( + empty(runners), + tupleElement(benchmark, 'extra_info')['device'], + tupleElement(runners[1], 'name') + ) AS device, + IF( + empty(runners), + tupleElement(benchmark, 'extra_info')['arch'], + tupleElement(runners[1], 'type') + ) AS arch, + DATE_TRUNC( + {granularity: String }, + fromUnixTimestamp(timestamp) + ) AS granularity_bucket + {{SELECT}} + FROM {{TABLE}} + {{PREWHERE}} + WHERE + repo = {repo: String } + AND notEmpty(metric.'name') + AND ( + has({commits: Array(String) }, head_sha) + OR empty({commits: Array(String) }) + ) + AND ( + benchmark.'name' in {benchmarks: Array(String) } + OR empty({benchmarks: Array(String) }) + ) + AND ( + has({models: Array(String) }, model.'name') + OR empty({models: Array(String) }) + ) + AND ( + has({backends: Array(String) }, model.'backend') + OR empty({backends: Array(String) }) + ) + AND ( + benchmark.'mode' = {mode: String } + OR {mode: String } = '' + ) + AND ( + has({dtypes: Array(String) }, benchmark.'dtype') + OR empty({dtypes: Array(String) }) + ) + {{WHERE}} + ` + ); + this._main_query_builder = new QueryBuilder( + { + table: "benchmarks", + select_exists: true, + where_exists: true, + }, + ` + SELECT DISTINCT + workflow_id, + job_id, + model, + backend, + origins, + metric, + actual, + target, + mode, + dtype, + device, + arch, + granularity_bucket, + extra, + metadata_info + {{SELECT}} + FROM {{TABLE}} + WHERE + ( + has({branches: Array(String) }, head_branch) + OR empty({branches: Array(String) }) + ) + notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) + {{WHERE}} + ORDER BY + granularity_bucket DESC, + workflow_id DESC, + backend, + model, + mode, + dtype, + device, + metric + ` + ); + } + + addInnerExtraInfo(extraInfoMapStatements: Map) { + const mapSelectItem = toQueryMapResult("extra", extraInfoMapStatements); + this._inner_query_builder.addSelect(mapSelectItem); + } + + addInnerMetadataInfo(metadataInfoMapStatements: Map) { + const mapSelectItem = toQueryMapResult( + "metadata_info", + metadataInfoMapStatements, + this._required_metadata_info_statements + ); + this._inner_query_builder.addSelect(mapSelectItem); + } + + build() { + const inner = this._inner_query_builder.build(); + const primary = this._main_query_builder.build(); + return ` + WITH benchmarks AS ( + ${inner} + ) + ${primary} + `; + } +} + +function toQueryMapResult( + resultName: string, + statements: Map, + additionalStatemetns?: Map +) { + let statement_items: string[] = []; + statements.forEach((statement, key) => { + statement_items.push(`${key},${statement}`); + }); + if (additionalStatemetns) { + additionalStatemetns.forEach((statement, key) => { + statement_items.push(`${key},${statement}`); + }); + } + const statement = statement_items.join(",\n "); + return [`map(${statement})`, resultName]; +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts new file mode 100644 index 0000000000..5d353f3f37 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts @@ -0,0 +1,66 @@ +import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; + +/** + * Query to get the list of metadata for a given benchmark name + */ +export class BenchmarkMetadataQuery extends ExecutableQueryBase { + private builder: QueryBuilder; + private _DEFAULT_QUERY_PARAMS = {}; + constructor() { + super(); + this.builder = new QueryBuilder({ + table: "benchmark.oss_ci_benchmark_metadata", + distinct: true, + select: [ + ["benchmark_name", "benchmark"], + ["model_name", "model"], + ["model_backend", "backend"], + ["metric_name", "metric"], + ["benchmark_dtype", "dtype"], + ["benchmark_mode", "mode"], + "device", + "arch", + ], + prewhere: [ + "timestamp >= toUnixTimestamp({startTime: DateTime64(3)})", + "timestamp < toUnixTimestamp({stopTime: DateTime64(3)})", + ], + where: [ + "repo = {repo: String}", + "benchmark_name = {benchmarkName: String}", + "notEmpty(metric_name)", + "notEmpty(device)", + ], + orderBy: [ + "benchmark", + "backend", + "model", + "metric", + "dtype", + "mode", + "device", + ], + }); + } + + build() { + return this.builder.build(); + } + + toQueryParams(inputs: any) { + this.validateInputs(inputs); + return { + ...this._DEFAULT_QUERY_PARAMS, + ...inputs, + }; + } + + validateInputs(inputs: any) { + if (!inputs.benchmarkName) { + throw new Error("No benchmark names provided"); + } + if (!inputs.repo) { + throw new Error("No repo provided"); + } + } +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts new file mode 100644 index 0000000000..01325bc544 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts @@ -0,0 +1,179 @@ +import { queryClickhouse } from "lib/clickhouse"; + +export type SelectItem = string | [field: string, alias: string]; + +interface BuildQueryClassOptions { + table: string; + orderBy?: string[]; // raw ORDER BY list + defaultSelect?: SelectItem[]; // default SELECT list + defaultWhere?: WhereItem[]; // default WHERE list +} + +type WhereItem = { + /** Unique key for merging/overriding (e.g. "repo", "has_benchmarkNames") */ + key: string; + /** Raw SQL fragment without "WHERE" and without leading AND */ + sql: string; +}; + +interface Defaults { + table: string; + distinct?: boolean; + select?: SelectItem[]; + prewhere?: string[]; + where?: string[]; + orderBy?: string[]; + selectAll?: boolean; + prewhere_exists?: boolean; + where_exists?: boolean; + order_by_exists?: boolean; + select_exists?: boolean; +} + +/** + * Dynamic SQL builder for clickhouse queries. + * special tokens: + * + * - {{DISTINCT}}: based on input customDistinct + * - {{SELECT}}: select items, comma-separated + * - {{TABLE}}: table + * - {{PREWHERE}}: prewhere []string + * - {{WHERE}}: where items, AND-separated + * {{ORDER_BY}}: order by items, comma-separated + * + * default template: + * + * SELECT {{DISTINCT}} + * {{SELECT}} + * FROM + * {{TABLE}}{{PREWHERE}}{{WHERE}}{{ORDER_BY}} + * + */ +export class QueryBuilder { + private template: string; + private defaults: Defaults; + + private extraSelect: SelectItem[] = []; + private extraWhere: string[] = []; + private customTable?: string; + private customDistinct?: boolean; + + constructor( + defaults: Defaults, + template = ` + SELECT {{DISTINCT}} + {{SELECT}} + FROM + {{TABLE}}{{PREWHERE}}{{WHERE}}{{ORDER_BY}} + `.trim() + ) { + this.defaults = defaults; + this.template = template; + } + addSelect(items: SelectItem[]) { + this.extraSelect.push(...items); + return this; + } + + /** Append WHERE clauses (exact-string dedupe). */ + addWhere(clauses: string[]) { + this.extraWhere.push(...clauses); + return this; + } + + /** Build final SQL string. */ + build(): string { + // SELECT: map by declared alias (right side); later entries override earlier ones. + const byField = new Map(); + const key = (s: SelectItem) => (Array.isArray(s) ? s[1] : s); + for (const it of this.defaults.select ?? []) byField.set(key(it), it); + for (const it of this.extraSelect) byField.set(key(it), it); + + let SELECT = [...byField.values()] + .map((it) => (Array.isArray(it) ? `${it[0]} AS ${it[1]}` : it)) + .join(",\n "); + + if (this.defaults.selectAll === true) { + SELECT = "*"; + } + if (SELECT && this.defaults.select_exists) { + SELECT = ",\n " + SELECT; + } + + // PREWHERE + const pre = (this.defaults.prewhere ?? []).filter(Boolean); + const prewherePreifx = this.defaults.prewhere_exists + ? "\n AND" + : "\nPREWHERE\n"; + const PREWHERE = pre.length + ? `${prewherePreifx} ${pre.join("\n AND ")}` + : ""; + + // WHERE (defaults + extras, dedup by exact string) + const whereSet = new Set(this.defaults.where ?? []); + for (const w of this.extraWhere) if (w && w.trim()) whereSet.add(w.trim()); + const whereArr = [...whereSet].filter(Boolean); + const wherePreifx = this.defaults.where_exists ? "\n AND" : "\nWHERE\n"; + const WHERE = whereArr.length + ? `${wherePreifx} ${whereArr.join("\n AND ")}` + : ""; + + // ORDER BY + const order = (this.defaults.orderBy ?? []).filter(Boolean); + const orderPreifx = this.defaults.order_by_exists + ? ",\n " + : "\nORDER BY\n"; + const ORDER_BY = order.length + ? `${orderPreifx} ${order.join(",\n ")}` + : ""; + + const TABLE = this.customTable ?? this.defaults.table; + const DISTINCT = + (this.customDistinct ?? this.defaults.distinct) === false + ? "" + : "DISTINCT"; + + return this.template + .replaceAll("{{DISTINCT}}", DISTINCT) + .replaceAll("{{SELECT}}", SELECT) + .replaceAll("{{TABLE}}", TABLE) + .replaceAll("{{PREWHERE}}", PREWHERE) + .replaceAll("{{WHERE}}", WHERE) + .replaceAll("{{ORDER_BY}}", ORDER_BY) + .trim(); + } +} + +export type QueryExecutor = ( + sql: string, + params: Record, + query_id?: string, + useQueryCache?: boolean +) => Promise; + +export interface BuildableQuery { + build(): string; // must return SQL string + toQueryParams(inputs: any, id?: string): Record; +} + +/** + * Base class for executable queries. + */ +export abstract class ExecutableQueryBase implements BuildableQuery { + abstract build(): string; + + /** Default: pass inputs through as params (override in subclasses if needed). */ + toQueryParams(inputs: any, id?: string): Record { + return inputs; + } + + /** Build SQL and execute via provided executor. */ + async applyQuery( + inputs: any, + executor: QueryExecutor = queryClickhouse + ): Promise { + const sql = this.build(); + const params = this.toQueryParams(inputs); + return executor(sql, params); + } +} diff --git a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts index e075d4013f..654f655954 100644 --- a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts @@ -1,4 +1,3 @@ -import dayjs from "dayjs"; import { queryClickhouseSaved } from "lib/clickhouse"; import { CommitResult, @@ -6,7 +5,7 @@ import { defaultGetTimeSeriesInputs, defaultListCommitsInputs, } from "../type"; -import { emptyTimeSeriesResponse } from "../utils"; +import { emptyTimeSeriesResponse, getCommitsWithSampling } from "../utils"; import { extractBackendSqlStyle, toApiArch, @@ -166,60 +165,3 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { } return rows; } - -function subsampleCommitsByDate(data: any[], maxCount: number | undefined) { - if (!maxCount) return { data, is_sampled: false }; - - if (data.length <= maxCount) - return { - data, - is_sampled: false, - }; - - // Sort by date ascending - const sorted = [...data].sort((a, b) => dayjs(a.date).diff(dayjs(b.date))); - - const first = sorted[0]; - const last = sorted[sorted.length - 1]; - - // Subsample the middle points evenly - const step = (sorted.length - 2) / (maxCount - 2); - const sampled = [first]; - - for (let i = 1; i < maxCount - 1; i++) { - const idx = Math.round(i * step); - sampled.push(sorted[idx]); - } - sampled.push(last); - - const sampling_info = { - origin: data.length, - result: sampled.length, - }; - return { - data: sampled, - origin: data, - is_sampled: true, - sampling_info, - }; -} - -async function getCommitsWithSampling( - tableName: string, - queryParams: any -): Promise { - const commit_results = await queryClickhouseSaved(tableName, queryParams); - let maxCount = undefined; - - // if subsampling is specified, use it - if (queryParams.sampling) { - maxCount = queryParams.sampling.max; - const res = subsampleCommitsByDate(commit_results, maxCount); - return res; - } - - return { - data: commit_results, - is_sampled: false, - }; -} diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts index 7ae288adf6..ad0a49b6a5 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts @@ -75,9 +75,3 @@ function getformat(data: any, format: string) { throw new Error("Invalid type"); } } - -export const REQUIRED_COMPLIER_LIST_COMMITS_KEYS = [ - "mode", - "dtype", - "deviceName", -] as const; diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/type.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/type.ts new file mode 100644 index 0000000000..4e246a7bc8 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/compilers/helpers/type.ts @@ -0,0 +1,5 @@ +export const REQUIRED_COMPLIER_LIST_COMMITS_KEYS = [ + "mode", + "dtype", + "deviceName", +] as const; diff --git a/torchci/lib/benchmark/api_helper/apis/api.ts b/torchci/lib/benchmark/api_helper/fe/api.ts similarity index 100% rename from torchci/lib/benchmark/api_helper/apis/api.ts rename to torchci/lib/benchmark/api_helper/fe/api.ts diff --git a/torchci/lib/benchmark/api_helper/apis/hooks.ts b/torchci/lib/benchmark/api_helper/fe/hooks.ts similarity index 100% rename from torchci/lib/benchmark/api_helper/apis/hooks.ts rename to torchci/lib/benchmark/api_helper/fe/hooks.ts diff --git a/torchci/lib/benchmark/api_helper/list_commits.ts b/torchci/lib/benchmark/api_helper/list_commits.ts new file mode 100644 index 0000000000..81568f917e --- /dev/null +++ b/torchci/lib/benchmark/api_helper/list_commits.ts @@ -0,0 +1,112 @@ +import { getCompilerCommits } from "./compilers/get_compiler_benchmark_data"; +import { CommitResult, defaultListCommitsInputs } from "./type"; +import { getCommitsWithSampling, groupByBenchmarkData } from "./utils"; + +const BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME = + "benchmark_v3/list_commit_query"; + +export async function listBenchmarkCommits( + id: string, + queryParams: any, + response_formats: [] +) { + const db = await getBenmarkCommits(id, queryParams); + if (!db) { + console.error("No data found for", id); + throw new Error(`No data found for ${id}`); + } + + // get all unique branches within the time range, + // if data is sampled, we get all branches from origin + // otherwise we list all branches from data + let all_branches: string[] = []; + + if (db.is_sampled) { + all_branches = [...new Set(db.origin?.map((c: any) => c.branch))]; + } else { + all_branches = [...new Set(db.data.map((c: any) => c.branch))]; + } + + const formats: string[] = + response_formats && response_formats.length != 0 + ? response_formats + : ["raw"]; + + // format data based on requested response formats, for instance if format is "branch", + // we group the data by branch and return the data for each branch + let result: any = {}; + formats.forEach((format) => { + const f = getFormat(db.data, format); + result[format] = f; + }); + + console.log( + "[API]list commits, response data: all_branches ", + all_branches.length + ); + + return { + metadata: { + branches: all_branches, + is_samplied: db.is_sampled, + sampling_info: db.sampling_info, + }, + data: result, + }; +} + +async function getBenmarkCommits( + request_name: string, + query_params: any +): Promise { + switch (request_name) { + case "compiler": + case "compiler_precompute": + return await getCompilerCommits(query_params); + default: + return await getCommits(query_params); + } +} + +function getFormat(data: any, format: string = "raw") { + console.log("[API]list commits, format data elaine: ", data.length); + switch (format) { + case "branch": + const branchgroup = groupByBenchmarkData(data, ["branch"], []); + branchgroup.forEach((branch: any) => { + branch["rows"] = branch.rows?.__ALL__?.data ?? []; + }); + return branchgroup; + case "raw": + return data; + default: + throw new Error(`Unsupported format: ${format}`); + } +} + +export async function getCommits(inputparams: any) { + if (!inputparams.repo) { + throw new Error("no repo provided in request"); + } + + if (!inputparams.benchmarkNames || inputparams.benchmarkName) { + throw new Error("no benchmarkNames || benchmarkName provided in request"); + } + + if (!inputparams.startTime || !inputparams.stopTime) { + throw new Error("no start/end time provided in request"); + } + + if (inputparams.benchmarkName) { + inputparams.benchmarkNames = [inputparams.benchmarkName]; + } + const queryParams = { + ...defaultListCommitsInputs, // base defaults + ...inputparams, // override with caller's values + }; + + return await getCommitsWithSampling( + BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME, + queryParams + ); +} diff --git a/torchci/lib/benchmark/api_helper/type.ts b/torchci/lib/benchmark/api_helper/type.ts index 7ed5de308f..29918a0104 100644 --- a/torchci/lib/benchmark/api_helper/type.ts +++ b/torchci/lib/benchmark/api_helper/type.ts @@ -83,3 +83,27 @@ export interface CommitResult { is_sampled?: boolean; sampling_info?: CommitSamplingInfo; } + +/** + * The enum type of benchmark dashboard dropgroup item + * this is used to render dropdowns dynamically in the LLMs Benchmark page. + * the field value must match the fields in LLMsBenchmarkProps + */ +export enum BenchmarkMetadataType { + ModelName = "modelName", + BackendName = "backendName", + ModeName = "modeName", + DtypeName = "dtypeName", + DeviceName = "deviceName", + ArchName = "archName", + OperatornName = "operatorName", + Qps = "qps", +} +/** + * The input item for benchmark metada item + */ +export interface BenchmarkMetadataItem { + type: BenchmarkMetadataType; + options: string[]; + labelName: string; +} diff --git a/torchci/lib/benchmark/api_helper/utils.ts b/torchci/lib/benchmark/api_helper/utils.ts index d11c480bf7..e9206dc5ec 100644 --- a/torchci/lib/benchmark/api_helper/utils.ts +++ b/torchci/lib/benchmark/api_helper/utils.ts @@ -1,5 +1,8 @@ // Utility to extract params from either GET or POST +import dayjs from "dayjs"; +import { queryClickhouseSaved } from "lib/clickhouse"; import { NextApiRequest } from "next"; +import { CommitResult } from "./type"; /** * Key-value map describing metadata for a group. @@ -330,3 +333,58 @@ export function parseTimestampTokenSeconds( export function badRequest(message: string, res: any) { return res.json({ error: message }, { status: 400 }); } + +function subsampleCommitsByDate(data: any[], maxCount: number | undefined) { + if (!maxCount) return { data, is_sampled: false }; + + if (data.length <= maxCount) + return { + data, + is_sampled: false, + }; + + // Sort by date ascending + const sorted = [...data].sort((a, b) => dayjs(a.date).diff(dayjs(b.date))); + + const first = sorted[0]; + const last = sorted[sorted.length - 1]; + + // Subsample the middle points evenly + const step = (sorted.length - 2) / (maxCount - 2); + const sampled = [first]; + + for (let i = 1; i < maxCount - 1; i++) { + const idx = Math.round(i * step); + sampled.push(sorted[idx]); + } + sampled.push(last); + + const sampling_info = { + origin: data.length, + result: sampled.length, + }; + return { + data: sampled, + origin: data, + is_sampled: true, + sampling_info, + }; +} + +export async function getCommitsWithSampling( + tableName: string, + queryParams: any +): Promise { + const commit_results = await queryClickhouseSaved(tableName, queryParams); + let maxCount = undefined; + // if subsampling is specified, use it + if (queryParams.sampling) { + maxCount = queryParams.sampling.max; + const res = subsampleCommitsByDate(commit_results, maxCount); + return res; + } + return { + data: commit_results, + is_sampled: false, + }; +} diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts index d1da368783..b4caa8296c 100644 --- a/torchci/pages/api/benchmark/list_commits.ts +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -1,9 +1,5 @@ -import { getCompilerCommits } from "lib/benchmark/api_helper/compilers/get_compiler_benchmark_data"; -import { CommitResult } from "lib/benchmark/api_helper/type"; -import { - groupByBenchmarkData, - readApiGetParams, -} from "lib/benchmark/api_helper/utils"; +import { listBenchmarkCommits } from "lib/benchmark/api_helper/list_commits"; +import { readApiGetParams } from "lib/benchmark/api_helper/utils"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( @@ -27,84 +23,17 @@ export default async function handler( ) { return res.status(400).json({ error: "Missing parameters" }); } + const { name, query_params, response_formats } = params; - // get time series data try { - const { name, query_params, response_formats } = params; - const db = await getBenmarkCommits(name, query_params); - if (!db) { - console.error("No data found for", name); - return res.status(404).json({ data: {} }); - } - - // get all unique branches within the time range, - // if data is sampled, we get all branches from origin - // otherwise we list all branches from data - let all_branches: string[] = []; - - if (db.is_sampled) { - all_branches = [...new Set(db.origin?.map((c) => c.branch))]; - } else { - all_branches = [...new Set(db.data.map((c) => c.branch))]; - } - - const formats: string[] = - response_formats && response_formats.length != 0 - ? response_formats - : ["raw"]; - - // format data based on requested response formats, for instance if format is "branch", - // we group the data by branch and return the data for each branch - let result: any = {}; - formats.forEach((format) => { - const f = getFormat(db.data, format); - result[format] = f; - }); - - console.log( - "[API]list commits, response data: all_branches ", - all_branches.length + const result = await listBenchmarkCommits( + name, + query_params, + response_formats ); - - return res.status(200).json({ - metadata: { - branches: all_branches, - is_samplied: db.is_sampled, - sampling_info: db.sampling_info, - }, - data: result, - }); + return res.status(200).json(result); } catch (err: any) { console.error("API error:", err.message); return res.status(400).json({ error: err.message }); } } - -async function getBenmarkCommits( - request_name: string, - query_params: any -): Promise { - switch (request_name) { - case "compiler": - case "compiler_precompute": - return await getCompilerCommits(query_params); - default: - throw new Error(`Unsupported request_name: ${request_name}`); - } -} - -function getFormat(data: any, format: string = "raw") { - console.log("[API]list commits, format data elaine: ", data.length); - switch (format) { - case "branch": - const branchgroup = groupByBenchmarkData(data, ["branch"], []); - branchgroup.forEach((branch: any) => { - branch["rows"] = branch.rows?.__ALL__?.data ?? []; - }); - return branchgroup; - case "raw": - return data; - default: - throw new Error(`Unsupported format: ${format}`); - } -} diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts new file mode 100644 index 0000000000..dcdd1c03a5 --- /dev/null +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -0,0 +1,34 @@ +import { listBenchmarkMetadata } from "lib/benchmark/api_helper/backend/list_metadata_api"; +import { readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { NextApiRequest, NextApiResponse } from "next"; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + if (req.method !== "GET" && req.method !== "POST") { + res.setHeader("Allow", "GET, POST"); + return res.status(405).json({ error: "Only GET and POST allowed" }); + } + const params = readApiGetParams(req); + + // validate params + if ( + !params || + !params.id || + !params.query_params || + Object.keys(params.query_params).length == 0 || + Object.keys(params).length === 0 + ) { + return res.status(400).json({ error: "Missing required parameters" }); + } + + console.log("[API] LIST_METRICS recieved params: ", params.query_params); + + try { + const groups = listBenchmarkMetadata(params.query_params, params.id); + return res.status(200).json(groups); + } catch (err: any) { + return res.status(500).json({ error: err.message }); + } +} diff --git a/torchci/pages/benchmark/regression/report/[id].tsx b/torchci/pages/benchmark/regression/report/[id].tsx index e94327b87a..c838f88ffc 100644 --- a/torchci/pages/benchmark/regression/report/[id].tsx +++ b/torchci/pages/benchmark/regression/report/[id].tsx @@ -4,7 +4,7 @@ import { BenchmarkReportFeatureSidePanel } from "components/benchmark/v3/compone import BenchmarkRegressionReportMetadataSection from "components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/BenchmarkRegressionReportMetadataSection"; import { RegressionReportDetail } from "components/benchmark/v3/components/dataRender/components/benchmarkRegressionReport/reportView/RegressionReportDetail"; import LoadingPage from "components/common/LoadingPage"; -import { useGetBenchmarkRegressionReportData } from "lib/benchmark/api_helper/apis/hooks"; +import { useGetBenchmarkRegressionReportData } from "lib/benchmark/api_helper/fe/hooks"; import { useRouter } from "next/router"; export default function Page() { From 6618009a3958e0fad7fa7c22d0f3610b0ab383cf Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Mon, 13 Oct 2025 18:30:06 -0700 Subject: [PATCH 04/19] add config --- .../components/SideBarMainSection.tsx | 4 +-- .../api_helper/backend/get_time_series_api.ts | 4 +-- .../api_helper/{ => backend}/list_commits.ts | 8 +++--- .../pages/api/benchmark/get_time_series.ts | 26 +------------------ torchci/pages/api/benchmark/list_commits.ts | 4 +-- 5 files changed, 11 insertions(+), 35 deletions(-) rename torchci/lib/benchmark/api_helper/{ => backend}/list_commits.ts (91%) diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx index 44a5c509ca..1ec29f5a07 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx @@ -8,8 +8,8 @@ import { CenteredLoader } from "components/common/LoadingIcon"; import { UMCopyLink } from "components/uiModules/UMCopyLink"; import { UMDateButtonPicker } from "components/uiModules/UMDateRangePicker"; import { UMDenseButtonLight } from "components/uiModules/UMDenseComponents"; -import dayjs from "dayjs";lib/benchmark/api_helper/fe/hooks -import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/apis/hooks"; +import dayjs from "dayjs"; +import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { useRouter } from "next/router"; import { useEffect, useRef, useState } from "react"; diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts index 4506445c57..3623adce5f 100644 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts @@ -34,6 +34,6 @@ export async function getBenchmarkData( id: string ) { const queryBuilder = new BenchmarkDataQuery(); - - const query = queryBuilder.applyQuery(query_params); + const result = queryBuilder.applyQuery(query_params); + return result; } diff --git a/torchci/lib/benchmark/api_helper/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts similarity index 91% rename from torchci/lib/benchmark/api_helper/list_commits.ts rename to torchci/lib/benchmark/api_helper/backend/list_commits.ts index 81568f917e..56e042937e 100644 --- a/torchci/lib/benchmark/api_helper/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -1,11 +1,11 @@ -import { getCompilerCommits } from "./compilers/get_compiler_benchmark_data"; -import { CommitResult, defaultListCommitsInputs } from "./type"; -import { getCommitsWithSampling, groupByBenchmarkData } from "./utils"; +import { getCompilerCommits } from "../compilers/get_compiler_benchmark_data"; +import { CommitResult, defaultListCommitsInputs } from "../type"; +import { getCommitsWithSampling, groupByBenchmarkData } from "../utils"; const BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME = "benchmark_v3/list_commit_query"; -export async function listBenchmarkCommits( +export async function listBenchmarkCommitsFromDb( id: string, queryParams: any, response_formats: [] diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index 54386c404a..961923a5c3 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -1,5 +1,4 @@ -import { getCompilerBenchmarkData } from "lib/benchmark/api_helper/compilers/get_compiler_benchmark_data"; -import { CompilerQueryType } from "lib/benchmark/api_helper/type"; +import { getBenmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/get_time_series_api"; import { readApiGetParams } from "lib/benchmark/api_helper/utils"; import type { NextApiRequest, NextApiResponse } from "next"; @@ -57,26 +56,3 @@ export default async function handler( return res.status(400).json({ error: err.message }); } } - -async function getBenmarkTimeSeriesData( - request_name: string, - query_params: any, - formats: string[] = ["time_series"] -) { - switch (request_name) { - case "compiler_precompute": - return await getCompilerBenchmarkData( - query_params, - CompilerQueryType.PRECOMPUTE, - formats - ); - case "compiler": - return await getCompilerBenchmarkData( - query_params, - CompilerQueryType.GENERAL, - formats - ); - default: - throw new Error(`Unsupported request_name: ${request_name}`); - } -} diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts index b4caa8296c..bbcb1a5ff4 100644 --- a/torchci/pages/api/benchmark/list_commits.ts +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -1,4 +1,4 @@ -import { listBenchmarkCommits } from "lib/benchmark/api_helper/list_commits"; +import { listBenchmarkCommitsFromDb } from "lib/benchmark/api_helper/backend/list_commits"; import { readApiGetParams } from "lib/benchmark/api_helper/utils"; import { NextApiRequest, NextApiResponse } from "next"; @@ -26,7 +26,7 @@ export default async function handler( const { name, query_params, response_formats } = params; try { - const result = await listBenchmarkCommits( + const result = await listBenchmarkCommitsFromDb( name, query_params, response_formats From 0883e55f45a7e24718f631aecd4196431280015d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 11:39:50 -0700 Subject: [PATCH 05/19] add config --- .../compilers_benchmark_api_query/params.json | 1 + .../compilers_benchmark_api_query/query.sql | 5 +- .../v3/configs/teams/compilers/config.ts | 2 +- .../api_helper/{ => backend/common}/type.ts | 23 ++- .../api_helper/{ => backend/common}/utils.ts | 49 ++++++ .../compilers/compiler_benchmark_data.ts} | 139 ++++++++++++------ .../{ => backend}/compilers/helpers/common.ts | 0 .../backend/compilers/helpers/general.ts | 54 +++++++ .../compilers/helpers/precompute.ts | 2 +- .../{ => backend}/compilers/helpers/type.ts | 0 .../api_helper/backend/get_time_series.ts | 35 +++++ .../api_helper/backend/get_time_series_api.ts | 39 ----- .../api_helper/backend/list_commits.ts | 6 +- .../api_helper/backend/list_metadata_api.ts | 2 +- .../api_helper/compilers/helpers/general.ts | 77 ---------- torchci/lib/benchmark/api_helper/fe/hooks.ts | 2 +- .../pages/api/benchmark/get_benchmark_data.ts | 0 .../get_regression_summary_report.ts | 2 +- .../pages/api/benchmark/get_time_series.ts | 6 +- torchci/pages/api/benchmark/list_commits.ts | 2 +- torchci/pages/api/benchmark/list_metadata.ts | 2 +- .../list_regression_summary_reports.ts | 2 +- 22 files changed, 274 insertions(+), 176 deletions(-) rename torchci/lib/benchmark/api_helper/{ => backend/common}/type.ts (84%) rename torchci/lib/benchmark/api_helper/{ => backend/common}/utils.ts (90%) rename torchci/lib/benchmark/api_helper/{compilers/get_compiler_benchmark_data.ts => backend/compilers/compiler_benchmark_data.ts} (50%) rename torchci/lib/benchmark/api_helper/{ => backend}/compilers/helpers/common.ts (100%) create mode 100644 torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts rename torchci/lib/benchmark/api_helper/{ => backend}/compilers/helpers/precompute.ts (99%) rename torchci/lib/benchmark/api_helper/{ => backend}/compilers/helpers/type.ts (100%) create mode 100644 torchci/lib/benchmark/api_helper/backend/get_time_series.ts delete mode 100644 torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts delete mode 100644 torchci/lib/benchmark/api_helper/compilers/helpers/general.ts create mode 100644 torchci/pages/api/benchmark/get_benchmark_data.ts diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json b/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json index 5f56edb418..d3f0977699 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/params.json @@ -3,6 +3,7 @@ "branches": "Array(String)", "commits": "Array(String)", "compilers": "Array(String)", + "workflows": "Array(UInt64)", "device": "String", "arch": "Array(String)", "dtype": "String", diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql index 5101c5fb9b..b00d304d2f 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql @@ -18,7 +18,10 @@ SELECT AS granularity_bucket FROM benchmark.oss_ci_benchmark_torchinductor WHERE - head_sha IN ({commits: Array(String)}) + ( + has ({workflows: Array(UInt64)}, workflow_id) + OR empty({workflows: Array(UInt64)}) + ) AND ( has( {branches: Array(String)}, diff --git a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts index 936c74d209..5be3d353bb 100644 --- a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts +++ b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts @@ -7,7 +7,7 @@ import { SUITES } from "components/benchmark/compilers/SuitePicker"; import { DEFAULT_MODE, MODES } from "components/benchmark/ModeAndDTypePicker"; import dayjs from "dayjs"; import utc from "dayjs/plugin/utc"; -import { REQUIRED_COMPLIER_LIST_COMMITS_KEYS } from "lib/benchmark/api_helper/compilers/helpers/type"; +import { REQUIRED_COMPLIER_LIST_COMMITS_KEYS } from "lib/benchmark/api_helper/backend/compilers/helpers/type"; import { DISPLAY_NAMES_TO_COMPILER_NAMES } from "../../../../compilers/common"; import { BenchmarkUIConfig } from "../../configBook"; import { BenchmarkComparisonPolicyConfig } from "../../helpers/RegressionPolicy"; diff --git a/torchci/lib/benchmark/api_helper/type.ts b/torchci/lib/benchmark/api_helper/backend/common/type.ts similarity index 84% rename from torchci/lib/benchmark/api_helper/type.ts rename to torchci/lib/benchmark/api_helper/backend/common/type.ts index 29918a0104..b8d1d2e570 100644 --- a/torchci/lib/benchmark/api_helper/type.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/type.ts @@ -3,9 +3,9 @@ export enum CompilerQueryType { GENERAL = "general", } -export const defaultGetTimeSeriesInputs: any = { +export const defaultCompilerGetTimeSeriesInputs: any = { models: [], - commits: [], + workflows:[], compilers: [], branches: [], device: "", @@ -18,6 +18,25 @@ export const defaultGetTimeSeriesInputs: any = { suites: [], }; + +/** + * The default input for compiler get benchmark data, + * must have worklfowid/commitId and branch + */ +export const defaultCompilerGetBenchmarkDataInputs: any = { + models: [], + compilers: [], + device: "", + arch: "", + dtype: "", + mode: "", + granularity: "hour", + startTime: "", + stopTime: "", + suites: [], +}; + + export const defaultListCommitsInputs: any = { branches: [], device: "", diff --git a/torchci/lib/benchmark/api_helper/utils.ts b/torchci/lib/benchmark/api_helper/backend/common/utils.ts similarity index 90% rename from torchci/lib/benchmark/api_helper/utils.ts rename to torchci/lib/benchmark/api_helper/backend/common/utils.ts index e9206dc5ec..14407421a0 100644 --- a/torchci/lib/benchmark/api_helper/utils.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/utils.ts @@ -388,3 +388,52 @@ export async function getCommitsWithSampling( is_sampled: false, }; } + + + +/** + * process general compiler data without precompute or aggregation + * @param rawData + * @param inputparams + * @param type + */ +export function toBenchmarkTimeSeriesReponseFormat( + rawData: any[], + config: any = {}, + formats: string[] = ["time_series"], +) { + const start_ts = new Date(rawData[0].granularity_bucket).getTime(); + const end_ts = new Date( + rawData[rawData.length - 1].granularity_bucket + ).getTime(); + + let formats_result: any = {}; + + formats.forEach((format) => { + const data = getformat(rawData, format, config); + formats_result[format] = data; + }); + return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); +} + +function getformat(data: any, format: string, config:any) { + switch (format) { + case "time_series": + return to_time_series_data( + data, + config[format].group_key, + config[format].sub_group_key, + ); + case "table": + return to_table( + data, + config[format].group_key, + config[format].sub_group_key, + ); + break; + case "raw": + return data; + default: + throw new Error("Invalid type"); + } +} diff --git a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts similarity index 50% rename from torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts rename to torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index 654f655954..f276d9d694 100644 --- a/torchci/lib/benchmark/api_helper/compilers/get_compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -2,39 +2,70 @@ import { queryClickhouseSaved } from "lib/clickhouse"; import { CommitResult, CompilerQueryType, - defaultGetTimeSeriesInputs, + defaultCompilerGetBenchmarkDataInputs, + defaultCompilerGetTimeSeriesInputs, defaultListCommitsInputs, -} from "../type"; -import { emptyTimeSeriesResponse, getCommitsWithSampling } from "../utils"; +} from "../common/type"; +import { emptyTimeSeriesResponse, getCommitsWithSampling } from "../common/utils"; import { extractBackendSqlStyle, toApiArch, toQueryArch, } from "./helpers/common"; -import { toGeneralCompilerData } from "./helpers/general"; import { toPrecomputeCompilerData } from "./helpers/precompute"; +import { toGeneralCompilerData } from "./helpers/general"; + //["x86_64","NVIDIA A10G","NVIDIA H100 80GB HBM3"] const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = "compilers_benchmark_api_commit_query"; -export async function getCompilerBenchmarkData( - inputparams: any, - type: CompilerQueryType = CompilerQueryType.PRECOMPUTE, - formats: string[] = ["time_series"] +/** + * backend method to get single compiler benchmark data + * must provide workflow and branch in inputParams + * @returns + */ +export async function getSingleCompilerBenchmarkData( + request_name: string, + inputParams: any, + formats: string[] = ["raw"], ) { - const rows = await getCompilerDataFromClickhouse(inputparams); + const queryParams = await getCompilerBenchmarkDataQueryParams(inputParams); + const rows = await fetchCompilerDataFromDb(queryParams); + if (rows.length === 0) { + return emptyTimeSeriesResponse(); + } + return toCompilerResponseFormat(rows, formats, request_name) +} + +/** + * backend method to get time series data + */ +export async function getCompilerBenchmarkTimeSeriesData(inputparams:any, type:CompilerQueryType, formats:string[]=["time_series"]){ + const queryParams = await getCompilerBenchmarkTimeRangeQueryParams(inputparams); + const rows = await fetchCompilerDataFromDb(queryParams); if (rows.length === 0) { return emptyTimeSeriesResponse(); } + return toCompilerResponseFormat(rows, formats, type) +} + - switch (type) { +/** + * return compiler benchmark data base on query type and formats + * @param data + * @param formats + * @param type + * @returns + */ +export function toCompilerResponseFormat(data:any[], formats:string[], type:string){ + switch (type) { case CompilerQueryType.PRECOMPUTE: - return toPrecomputeCompilerData(rows, formats); + return toPrecomputeCompilerData(data, formats); case CompilerQueryType.GENERAL: - return toGeneralCompilerData(rows, formats); + return toGeneralCompilerData(data, formats); default: throw new Error(`Invalid compiler query type, got ${type}`); } @@ -59,23 +90,46 @@ export async function getCompilerCommits( ); } -async function getCompilerDataFromClickhouse(inputparams: any): Promise { - const start = Date.now(); +async function getCompilerBenchmarkDataQueryParams( + inputparams: any +): Promise { + const queryParams = { + ...defaultCompilerGetBenchmarkDataInputs, // base defaults + ...inputparams, // override with caller's values + }; + const arch_list = toQueryArch(queryParams.device, queryParams.arch); + queryParams["arch"] = arch_list; + + if (!queryParams.workflow || !queryParams.branch) { + throw new Error("no workflow or branch provided in request for single data fetch"); + } + queryParams["workflows"] = [queryParams.workflow]; + queryParams["branches"] = [queryParams.branch]; + console.log("workflows provided in request", queryParams.workflows); + return queryParams; +} +/** + * get list of workflows based on start/end time and other filters + * + * @param inputparams + * @returns + */ +export async function getCompilerBenchmarkTimeRangeQueryParams(inputparams:any){ const queryParams = { - ...defaultGetTimeSeriesInputs, // base defaults + ...defaultCompilerGetTimeSeriesInputs, // base defaults ...inputparams, // override with caller's values }; const arch_list = toQueryArch(queryParams.device, queryParams.arch); queryParams["arch"] = arch_list; + // todo(elainewy): support lworkfow and rworkflow in the future for time range query + // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided - if (!queryParams.commits || queryParams.commits.length == 0) { - if (!queryParams.startTime || !queryParams.stopTime) { - console.log("no commits or start/end time provided in request"); - return []; - } + if (!queryParams.startTime || !queryParams.stopTime) { + throw new Error("(getCompilerBenchmarkTimeRangeQueryParams) no start/end time provided in request"); + } // get commits from clickhouse, if queryParams has samping config, use it // TODO(ELAINEWY): when use sampled commits to fetch list of commits for data fetching, @@ -88,36 +142,36 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { queryParams ); - // get unique commits - const unique_commits = [...new Set(commit_results.map((c) => c.commit))]; - if (unique_commits.length === 0) { - console.log("no commits found in clickhouse using", queryParams); - return []; - } - + const unique_workflows = [...new Set(commit_results.map((c) => c.workflow_id))]; console.log( - `no commits provided in request, searched unqiue commits based on - start/end time unique_commits: ${unique_commits.length}` + `no workflows provided in request, searched unqiue workflows based on + start/end time unique_workflows: ${unique_workflows.length}` ); if (commit_results.length > 0) { - queryParams["commits"] = unique_commits; + queryParams["workflows"] = unique_workflows; } else { - console.log(`no commits found in clickhouse using ${queryParams}`); + console.log(`no workflow found in clickhouse using ${queryParams}`); return []; } - } else { - console.log("commits provided in request", queryParams.commits); - } + return queryParams; +} - let rows = []; +/** + * + * @param queryParams + * @returns + */ +async function fetchCompilerDataFromDb(queryParams: any): Promise { + const start = Date.now(); + let rows: any[] = []; try { rows = await queryClickhouseSaved( COMPILER_BENCHMARK_TABLE_NAME, queryParams ); } catch (err: any) { - throw Error("(clickhouse query issue) ", err.message); + throw Error(`${COMPILER_BENCHMARK_TABLE_NAME}(clickhouse query issue) ${err.message}`); } const end = Date.now(); @@ -128,7 +182,6 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { } console.log("rows from clickhouse", rows[0], "total length", rows.length); - // extract backend from output in runtime instead of doing it in the query. since it's expensive for regex matching. // TODO(elainewy): we should add this as a column in the database for less runtime logics. rows.map((row) => { @@ -146,21 +199,21 @@ async function getCompilerDataFromClickhouse(inputparams: any): Promise { row["arch"] = toApiArch(row.device, row.arch); }); - if (inputparams.compilers && inputparams.compilers.length > 0) { + if (queryParams.compilers && queryParams.compilers.length > 0) { rows = rows.filter((row) => { - return inputparams.compilers.includes(row.backend); + return queryParams.compilers.includes(row.backend); }); } - if (inputparams.models && inputparams.models.length > 0) { + if (queryParams.models && queryParams.models.length > 0) { rows = rows.filter((row) => { - return inputparams.models.includes(row.model); + return queryParams.models.includes(row.model); }); } - if (inputparams.metrics && inputparams.metrics.length > 0) { + if (queryParams.metrics && queryParams.metrics.length > 0) { rows = rows.filter((row) => { - return inputparams.metrics.includes(row.metric); + return queryParams.metrics.includes(row.metric); }); } return rows; diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/common.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts similarity index 100% rename from torchci/lib/benchmark/api_helper/compilers/helpers/common.ts rename to torchci/lib/benchmark/api_helper/backend/compilers/helpers/common.ts diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts new file mode 100644 index 0000000000..49cb24b625 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts @@ -0,0 +1,54 @@ + +import { to_table, to_time_series_data, toBenchmarkTimeSeriesReponseFormat } from "../../common/utils"; + +const COMPILER_GENERAL_TS_GROUP_KEY = [ + "dtype", + "arch", + "device", + "suite", + "compiler", + "metric", + "mode", + "model", + "branch", +]; +const COMPILER_GENERAL_TS_SUB_GROUP_KEY = ["workflow_id"]; + +const COMPILER_GENERAL_TABLE_GROUP_KEY = [ + "dtype", + "arch", + "device", + "mode", + "workflow_id", + "branch", + "compiler", + "model", + "suite", +]; +const COMPILER_GENERAL_TABLE_SUB_GROUP_KEY = ["metric"]; + + +export const COMPILER_GROUP_MAP = { + "time_series": { + "group_key": COMPILER_GENERAL_TS_GROUP_KEY, + "sub_group_key": COMPILER_GENERAL_TS_SUB_GROUP_KEY, + }, + "table": { + "group_key": COMPILER_GENERAL_TABLE_GROUP_KEY, + "sub_group_key": COMPILER_GENERAL_TABLE_SUB_GROUP_KEY, + }, +} + +/** + * process general compiler data without precompute or aggregation + * @param rawData + * @param inputparams + * @param type + */ +export function toGeneralCompilerData( + rawData: any[], + formats: string[] = ["time_series"], +) { + const config = COMPILER_GROUP_MAP; + return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); +} diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts similarity index 99% rename from torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts rename to torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts index a70429d620..93a964dfef 100644 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/precompute.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts @@ -13,7 +13,7 @@ import { to_time_series_data, toTimeSeriesResponse, toWorkflowIdMap, -} from "../../utils"; +} from "../../common/utils"; import { toApiArch } from "./common"; const COMPILER_PRECOMPUTE_TS_GROUP_KEY = [ diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/type.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/type.ts similarity index 100% rename from torchci/lib/benchmark/api_helper/compilers/helpers/type.ts rename to torchci/lib/benchmark/api_helper/backend/compilers/helpers/type.ts diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts new file mode 100644 index 0000000000..35190d012f --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts @@ -0,0 +1,35 @@ +import { CompilerQueryType } from "./common/type"; +import { getCompilerBenchmarkTimeSeriesData } from "./compilers/compiler_benchmark_data"; +import { BenchmarkDataQuery } from "./queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder"; + +export async function getBenchmarkTimeSeriesData( + request_name: string, + query_params: any, + formats: string[] = ["time_series"], +) { + switch (request_name) { + case "compiler_precompute": + return await getCompilerBenchmarkTimeSeriesData( + query_params, + CompilerQueryType.PRECOMPUTE, + formats + ); + case "compiler": + return await getCompilerBenchmarkTimeSeriesData( + query_params, + CompilerQueryType.GENERAL, + formats + ); + case "pytorch_operator_microbenchmak": + return await getGenernalBenchmarkTimeSeries(query_params, formats, request_name); + default: + throw new Error(`Unsupported request_name: ${request_name}`); + } +} + +async function getGenernalBenchmarkTimeSeries(query_params: any, formats: string[], id: string) { + const queryBuilder = new BenchmarkDataQuery(); + const result = await queryBuilder.applyQuery(query_params); + + +} diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts deleted file mode 100644 index 3623adce5f..0000000000 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series_api.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { getCompilerBenchmarkData } from "../compilers/get_compiler_benchmark_data"; -import { CompilerQueryType } from "../type"; -import { BenchmarkDataQuery } from "./queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder"; - -export async function getBenmarkTimeSeriesData( - request_name: string, - query_params: any, - formats: string[] = ["time_series"] -) { - switch (request_name) { - case "compiler_precompute": - return await getCompilerBenchmarkData( - query_params, - CompilerQueryType.PRECOMPUTE, - formats - ); - case "compiler": - return await getCompilerBenchmarkData( - query_params, - CompilerQueryType.GENERAL, - formats - ); - case "pytorch_operator_microbenchmak": - return await getBenchmarkData(query_params, formats, request_name); - - default: - throw new Error(`Unsupported request_name: ${request_name}`); - } -} - -export async function getBenchmarkData( - query_params: any, - formats: string[], - id: string -) { - const queryBuilder = new BenchmarkDataQuery(); - const result = queryBuilder.applyQuery(query_params); - return result; -} diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index 56e042937e..ae1326d904 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -1,6 +1,6 @@ -import { getCompilerCommits } from "../compilers/get_compiler_benchmark_data"; -import { CommitResult, defaultListCommitsInputs } from "../type"; -import { getCommitsWithSampling, groupByBenchmarkData } from "../utils"; +import { getCompilerCommits } from "./compilers/compiler_benchmark_data"; +import { CommitResult, defaultListCommitsInputs } from "./common/type"; +import { getCommitsWithSampling, groupByBenchmarkData } from "./common/utils"; const BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME = "benchmark_v3/list_commit_query"; diff --git a/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts b/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts index 198f15b117..87861d4593 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts @@ -5,7 +5,7 @@ import { DEFAULT_MODE_NAME, } from "lib/benchmark/llms/common"; import ld from "lodash"; -import { BenchmarkMetadataItem, BenchmarkMetadataType } from "../type"; +import { BenchmarkMetadataItem, BenchmarkMetadataType } from "./common/type"; import { BenchmarkMetadataQuery } from "./queryBuilderUtils/defaultListMetadataQueryBuilder"; export async function listBenchmarkMetadata(queryParams: any, id: string) { // fetch metadata from db diff --git a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts deleted file mode 100644 index ad0a49b6a5..0000000000 --- a/torchci/lib/benchmark/api_helper/compilers/helpers/general.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { - to_table, - to_time_series_data, - toTimeSeriesResponse, -} from "../../utils"; - -const COMPILER_GENERAL_TS_GROUP_KEY = [ - "dtype", - "arch", - "device", - "suite", - "compiler", - "metric", - "mode", - "model", - "branch", -]; -const COMPILER_GENERAL_TS_SUB_GROUP_KEY = ["workflow_id"]; - -const COMPILER_GENERAL_TABLE_GROUP_KEY = [ - "dtype", - "arch", - "device", - "mode", - "workflow_id", - "branch", - "compiler", - "model", - "suite", -]; -const COMPILER_GENERAL_TABLE_SUB_GROUP_KEY = ["metric"]; - -/** - * process general compiler data without precompute or aggregation - * @param rawData - * @param inputparams - * @param type - */ -export function toGeneralCompilerData( - rawData: any[], - formats: string[] = ["time_series"] -) { - const start_ts = new Date(rawData[0].granularity_bucket).getTime(); - const end_ts = new Date( - rawData[rawData.length - 1].granularity_bucket - ).getTime(); - - let formats_result: any = {}; - - formats.forEach((format) => { - const data = getformat(rawData, format); - formats_result[format] = data; - }); - return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); -} - -function getformat(data: any, format: string) { - switch (format) { - case "time_series": - return to_time_series_data( - data, - COMPILER_GENERAL_TS_GROUP_KEY, - COMPILER_GENERAL_TS_SUB_GROUP_KEY - ); - case "table": - return to_table( - data, - COMPILER_GENERAL_TABLE_GROUP_KEY, - COMPILER_GENERAL_TABLE_SUB_GROUP_KEY - ); - break; - case "raw": - return data; - default: - throw new Error("Invalid type"); - } -} diff --git a/torchci/lib/benchmark/api_helper/fe/hooks.ts b/torchci/lib/benchmark/api_helper/fe/hooks.ts index 12579388d5..62f99b4657 100644 --- a/torchci/lib/benchmark/api_helper/fe/hooks.ts +++ b/torchci/lib/benchmark/api_helper/fe/hooks.ts @@ -1,5 +1,5 @@ import useSWR, { SWRConfiguration, SWRResponse } from "swr"; -import { BundleResult } from "../type"; +import { BundleResult } from "../backend/common/type"; import { getBenchmarkRegressionReport, listBenchmarkCommits, diff --git a/torchci/pages/api/benchmark/get_benchmark_data.ts b/torchci/pages/api/benchmark/get_benchmark_data.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/torchci/pages/api/benchmark/get_regression_summary_report.ts b/torchci/pages/api/benchmark/get_regression_summary_report.ts index ca7a49e18b..00e4bbf44a 100644 --- a/torchci/pages/api/benchmark/get_regression_summary_report.ts +++ b/torchci/pages/api/benchmark/get_regression_summary_report.ts @@ -1,4 +1,4 @@ -import { badRequest, readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { badRequest, readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; import { queryClickhouse } from "lib/clickhouse"; import { NextApiRequest, NextApiResponse } from "next"; import { toMiniReport } from "./list_regression_summary_reports"; diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index 961923a5c3..b3cfe977d3 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -1,5 +1,5 @@ -import { getBenmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/get_time_series_api"; -import { readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; +import { getBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/get_time_series"; import type { NextApiRequest, NextApiResponse } from "next"; /** @@ -49,7 +49,7 @@ export default async function handler( ? response_formats : ["time_series"]; - const data = await getBenmarkTimeSeriesData(name, query_params, formats); + const data = await getBenchmarkTimeSeriesData(name, query_params, formats); return res.status(200).json({ data }); } catch (err: any) { console.error("API error:", err.message); diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts index bbcb1a5ff4..c3dbf2c1ff 100644 --- a/torchci/pages/api/benchmark/list_commits.ts +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -1,5 +1,5 @@ import { listBenchmarkCommitsFromDb } from "lib/benchmark/api_helper/backend/list_commits"; -import { readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts index dcdd1c03a5..28e5d35c67 100644 --- a/torchci/pages/api/benchmark/list_metadata.ts +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -1,5 +1,5 @@ import { listBenchmarkMetadata } from "lib/benchmark/api_helper/backend/list_metadata_api"; -import { readApiGetParams } from "lib/benchmark/api_helper/utils"; +import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( diff --git a/torchci/pages/api/benchmark/list_regression_summary_reports.ts b/torchci/pages/api/benchmark/list_regression_summary_reports.ts index f9cb65da8d..757f54805b 100644 --- a/torchci/pages/api/benchmark/list_regression_summary_reports.ts +++ b/torchci/pages/api/benchmark/list_regression_summary_reports.ts @@ -2,7 +2,7 @@ import { badRequest, parseTimestampTokenSeconds, readApiGetParams, -} from "lib/benchmark/api_helper/utils"; +} from "lib/benchmark/api_helper/backend/common/utils"; import { queryClickhouse } from "lib/clickhouse"; import { NextApiRequest, NextApiResponse } from "next"; From 229b89d9221248ea0528add94290b9e65cdf15f4 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 11:46:38 -0700 Subject: [PATCH 06/19] add config --- .../api_helper/backend/common/type.ts | 4 +- .../api_helper/backend/common/utils.ts | 10 +- .../compilers/compiler_benchmark_data.ts | 92 +++++++++++-------- .../backend/compilers/helpers/general.ts | 20 ++-- .../api_helper/backend/get_time_series.ts | 18 ++-- .../api_helper/backend/list_commits.ts | 8 +- .../defaultGetBenchmarkDataQueryBuilder.ts | 4 +- .../pages/api/benchmark/get_benchmark_data.ts | 0 .../get_regression_summary_report.ts | 5 +- torchci/pages/api/benchmark/list_commits.ts | 2 +- torchci/pages/api/benchmark/list_metadata.ts | 2 +- 11 files changed, 94 insertions(+), 71 deletions(-) delete mode 100644 torchci/pages/api/benchmark/get_benchmark_data.ts diff --git a/torchci/lib/benchmark/api_helper/backend/common/type.ts b/torchci/lib/benchmark/api_helper/backend/common/type.ts index b8d1d2e570..d448e83095 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/type.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/type.ts @@ -5,7 +5,7 @@ export enum CompilerQueryType { export const defaultCompilerGetTimeSeriesInputs: any = { models: [], - workflows:[], + workflows: [], compilers: [], branches: [], device: "", @@ -18,7 +18,6 @@ export const defaultCompilerGetTimeSeriesInputs: any = { suites: [], }; - /** * The default input for compiler get benchmark data, * must have worklfowid/commitId and branch @@ -36,7 +35,6 @@ export const defaultCompilerGetBenchmarkDataInputs: any = { suites: [], }; - export const defaultListCommitsInputs: any = { branches: [], device: "", diff --git a/torchci/lib/benchmark/api_helper/backend/common/utils.ts b/torchci/lib/benchmark/api_helper/backend/common/utils.ts index 14407421a0..d3c2cebb80 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/utils.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/utils.ts @@ -389,8 +389,6 @@ export async function getCommitsWithSampling( }; } - - /** * process general compiler data without precompute or aggregation * @param rawData @@ -400,7 +398,7 @@ export async function getCommitsWithSampling( export function toBenchmarkTimeSeriesReponseFormat( rawData: any[], config: any = {}, - formats: string[] = ["time_series"], + formats: string[] = ["time_series"] ) { const start_ts = new Date(rawData[0].granularity_bucket).getTime(); const end_ts = new Date( @@ -416,19 +414,19 @@ export function toBenchmarkTimeSeriesReponseFormat( return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); } -function getformat(data: any, format: string, config:any) { +function getformat(data: any, format: string, config: any) { switch (format) { case "time_series": return to_time_series_data( data, config[format].group_key, - config[format].sub_group_key, + config[format].sub_group_key ); case "table": return to_table( data, config[format].group_key, - config[format].sub_group_key, + config[format].sub_group_key ); break; case "raw": diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index f276d9d694..2a44cd905c 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -6,15 +6,17 @@ import { defaultCompilerGetTimeSeriesInputs, defaultListCommitsInputs, } from "../common/type"; -import { emptyTimeSeriesResponse, getCommitsWithSampling } from "../common/utils"; +import { + emptyTimeSeriesResponse, + getCommitsWithSampling, +} from "../common/utils"; import { extractBackendSqlStyle, toApiArch, toQueryArch, } from "./helpers/common"; -import { toPrecomputeCompilerData } from "./helpers/precompute"; import { toGeneralCompilerData } from "./helpers/general"; - +import { toPrecomputeCompilerData } from "./helpers/precompute"; //["x86_64","NVIDIA A10G","NVIDIA H100 80GB HBM3"] const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; @@ -29,30 +31,34 @@ const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = export async function getSingleCompilerBenchmarkData( request_name: string, inputParams: any, - formats: string[] = ["raw"], + formats: string[] = ["raw"] ) { const queryParams = await getCompilerBenchmarkDataQueryParams(inputParams); const rows = await fetchCompilerDataFromDb(queryParams); if (rows.length === 0) { return emptyTimeSeriesResponse(); } - return toCompilerResponseFormat(rows, formats, request_name) + return toCompilerResponseFormat(rows, formats, request_name); } - /** * backend method to get time series data */ -export async function getCompilerBenchmarkTimeSeriesData(inputparams:any, type:CompilerQueryType, formats:string[]=["time_series"]){ - const queryParams = await getCompilerBenchmarkTimeRangeQueryParams(inputparams); +export async function getCompilerBenchmarkTimeSeriesData( + inputparams: any, + type: CompilerQueryType, + formats: string[] = ["time_series"] +) { + const queryParams = await getCompilerBenchmarkTimeRangeQueryParams( + inputparams + ); const rows = await fetchCompilerDataFromDb(queryParams); if (rows.length === 0) { return emptyTimeSeriesResponse(); } - return toCompilerResponseFormat(rows, formats, type) + return toCompilerResponseFormat(rows, formats, type); } - /** * return compiler benchmark data base on query type and formats * @param data @@ -60,8 +66,12 @@ export async function getCompilerBenchmarkTimeSeriesData(inputparams:any, type:C * @param type * @returns */ -export function toCompilerResponseFormat(data:any[], formats:string[], type:string){ - switch (type) { +export function toCompilerResponseFormat( + data: any[], + formats: string[], + type: string +) { + switch (type) { case CompilerQueryType.PRECOMPUTE: return toPrecomputeCompilerData(data, formats); case CompilerQueryType.GENERAL: @@ -101,7 +111,9 @@ async function getCompilerBenchmarkDataQueryParams( queryParams["arch"] = arch_list; if (!queryParams.workflow || !queryParams.branch) { - throw new Error("no workflow or branch provided in request for single data fetch"); + throw new Error( + "no workflow or branch provided in request for single data fetch" + ); } queryParams["workflows"] = [queryParams.workflow]; queryParams["branches"] = [queryParams.branch]; @@ -115,7 +127,9 @@ async function getCompilerBenchmarkDataQueryParams( * @param inputparams * @returns */ -export async function getCompilerBenchmarkTimeRangeQueryParams(inputparams:any){ +export async function getCompilerBenchmarkTimeRangeQueryParams( + inputparams: any +) { const queryParams = { ...defaultCompilerGetTimeSeriesInputs, // base defaults ...inputparams, // override with caller's values @@ -128,32 +142,36 @@ export async function getCompilerBenchmarkTimeRangeQueryParams(inputparams:any){ // use the startTime and endTime to fetch commits from clickhouse if commits field is not provided if (!queryParams.startTime || !queryParams.stopTime) { - throw new Error("(getCompilerBenchmarkTimeRangeQueryParams) no start/end time provided in request"); + throw new Error( + "(getCompilerBenchmarkTimeRangeQueryParams) no start/end time provided in request" + ); } - // get commits from clickhouse, if queryParams has samping config, use it - // TODO(ELAINEWY): when use sampled commits to fetch list of commits for data fetching, - // the result may contain more data than we expected. this is bc sometimes one commit - // can have multiple workflowid associated with it. we need to revisit this later. - // maybe we can use the workflowid to search for data instead of commit. - // if there is situation like this - const { data: commit_results } = await getCommitsWithSampling( - COMPILER_BENCHMARK_COMMITS_TABLE_NAME, - queryParams - ); + // get commits from clickhouse, if queryParams has samping config, use it + // TODO(ELAINEWY): when use sampled commits to fetch list of commits for data fetching, + // the result may contain more data than we expected. this is bc sometimes one commit + // can have multiple workflowid associated with it. we need to revisit this later. + // maybe we can use the workflowid to search for data instead of commit. + // if there is situation like this + const { data: commit_results } = await getCommitsWithSampling( + COMPILER_BENCHMARK_COMMITS_TABLE_NAME, + queryParams + ); - const unique_workflows = [...new Set(commit_results.map((c) => c.workflow_id))]; - console.log( - `no workflows provided in request, searched unqiue workflows based on + const unique_workflows = [ + ...new Set(commit_results.map((c) => c.workflow_id)), + ]; + console.log( + `no workflows provided in request, searched unqiue workflows based on start/end time unique_workflows: ${unique_workflows.length}` - ); + ); - if (commit_results.length > 0) { - queryParams["workflows"] = unique_workflows; - } else { - console.log(`no workflow found in clickhouse using ${queryParams}`); - return []; - } + if (commit_results.length > 0) { + queryParams["workflows"] = unique_workflows; + } else { + console.log(`no workflow found in clickhouse using ${queryParams}`); + return []; + } return queryParams; } @@ -171,7 +189,9 @@ async function fetchCompilerDataFromDb(queryParams: any): Promise { queryParams ); } catch (err: any) { - throw Error(`${COMPILER_BENCHMARK_TABLE_NAME}(clickhouse query issue) ${err.message}`); + throw Error( + `${COMPILER_BENCHMARK_TABLE_NAME}(clickhouse query issue) ${err.message}` + ); } const end = Date.now(); diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts index 49cb24b625..4634c49d62 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/general.ts @@ -1,5 +1,4 @@ - -import { to_table, to_time_series_data, toBenchmarkTimeSeriesReponseFormat } from "../../common/utils"; +import { toBenchmarkTimeSeriesReponseFormat } from "../../common/utils"; const COMPILER_GENERAL_TS_GROUP_KEY = [ "dtype", @@ -27,17 +26,16 @@ const COMPILER_GENERAL_TABLE_GROUP_KEY = [ ]; const COMPILER_GENERAL_TABLE_SUB_GROUP_KEY = ["metric"]; - export const COMPILER_GROUP_MAP = { - "time_series": { - "group_key": COMPILER_GENERAL_TS_GROUP_KEY, - "sub_group_key": COMPILER_GENERAL_TS_SUB_GROUP_KEY, + time_series: { + group_key: COMPILER_GENERAL_TS_GROUP_KEY, + sub_group_key: COMPILER_GENERAL_TS_SUB_GROUP_KEY, }, - "table": { - "group_key": COMPILER_GENERAL_TABLE_GROUP_KEY, - "sub_group_key": COMPILER_GENERAL_TABLE_SUB_GROUP_KEY, + table: { + group_key: COMPILER_GENERAL_TABLE_GROUP_KEY, + sub_group_key: COMPILER_GENERAL_TABLE_SUB_GROUP_KEY, }, -} +}; /** * process general compiler data without precompute or aggregation @@ -47,7 +45,7 @@ export const COMPILER_GROUP_MAP = { */ export function toGeneralCompilerData( rawData: any[], - formats: string[] = ["time_series"], + formats: string[] = ["time_series"] ) { const config = COMPILER_GROUP_MAP; return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts index 35190d012f..e75a77c29d 100644 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts @@ -5,7 +5,7 @@ import { BenchmarkDataQuery } from "./queryBuilderUtils/defaultGetBenchmarkDataQ export async function getBenchmarkTimeSeriesData( request_name: string, query_params: any, - formats: string[] = ["time_series"], + formats: string[] = ["time_series"] ) { switch (request_name) { case "compiler_precompute": @@ -20,16 +20,22 @@ export async function getBenchmarkTimeSeriesData( CompilerQueryType.GENERAL, formats ); - case "pytorch_operator_microbenchmak": - return await getGenernalBenchmarkTimeSeries(query_params, formats, request_name); + case "pytorch_operator_microbenchmark": + return await getGenernalBenchmarkTimeSeries( + query_params, + formats, + request_name + ); default: throw new Error(`Unsupported request_name: ${request_name}`); } } -async function getGenernalBenchmarkTimeSeries(query_params: any, formats: string[], id: string) { +async function getGenernalBenchmarkTimeSeries( + query_params: any, + formats: string[], + id: string +) { const queryBuilder = new BenchmarkDataQuery(); const result = await queryBuilder.applyQuery(query_params); - - } diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index ae1326d904..62cbd5fd2a 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -1,6 +1,6 @@ -import { getCompilerCommits } from "./compilers/compiler_benchmark_data"; import { CommitResult, defaultListCommitsInputs } from "./common/type"; import { getCommitsWithSampling, groupByBenchmarkData } from "./common/utils"; +import { getCompilerCommits } from "./compilers/compiler_benchmark_data"; const BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME = "benchmark_v3/list_commit_query"; @@ -8,7 +8,7 @@ const BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME = export async function listBenchmarkCommitsFromDb( id: string, queryParams: any, - response_formats: [] + responseFormats: [] ) { const db = await getBenmarkCommits(id, queryParams); if (!db) { @@ -28,8 +28,8 @@ export async function listBenchmarkCommitsFromDb( } const formats: string[] = - response_formats && response_formats.length != 0 - ? response_formats + responseFormats && responseFormats.length != 0 + ? responseFormats : ["raw"]; // format data based on requested response formats, for instance if format is "branch", diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts index f6e019d89b..6e0c14ca4f 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts @@ -52,7 +52,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { floor(arrayAvg(metric.'benchmark_values'), 2) AS value, floor(toFloat64(metric.'target_value'), 2) AS target, benchmark.'mode' AS mode, - benchmark.'dtype' AS dtype + benchmark.'dtype' AS dtype, IF( empty(runners), tupleElement(benchmark, 'extra_info')['device'], @@ -130,7 +130,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { has({branches: Array(String) }, head_branch) OR empty({branches: Array(String) }) ) - notEmpty(device) + AND notEmpty(device) AND ( arch LIKE concat('%', {arch: String }, '%') OR {arch: String } = '' diff --git a/torchci/pages/api/benchmark/get_benchmark_data.ts b/torchci/pages/api/benchmark/get_benchmark_data.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/torchci/pages/api/benchmark/get_regression_summary_report.ts b/torchci/pages/api/benchmark/get_regression_summary_report.ts index 00e4bbf44a..90176cf0ec 100644 --- a/torchci/pages/api/benchmark/get_regression_summary_report.ts +++ b/torchci/pages/api/benchmark/get_regression_summary_report.ts @@ -1,4 +1,7 @@ -import { badRequest, readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; +import { + badRequest, + readApiGetParams, +} from "lib/benchmark/api_helper/backend/common/utils"; import { queryClickhouse } from "lib/clickhouse"; import { NextApiRequest, NextApiResponse } from "next"; import { toMiniReport } from "./list_regression_summary_reports"; diff --git a/torchci/pages/api/benchmark/list_commits.ts b/torchci/pages/api/benchmark/list_commits.ts index c3dbf2c1ff..3e7aee07da 100644 --- a/torchci/pages/api/benchmark/list_commits.ts +++ b/torchci/pages/api/benchmark/list_commits.ts @@ -1,5 +1,5 @@ -import { listBenchmarkCommitsFromDb } from "lib/benchmark/api_helper/backend/list_commits"; import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; +import { listBenchmarkCommitsFromDb } from "lib/benchmark/api_helper/backend/list_commits"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts index 28e5d35c67..733a678057 100644 --- a/torchci/pages/api/benchmark/list_metadata.ts +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -1,5 +1,5 @@ -import { listBenchmarkMetadata } from "lib/benchmark/api_helper/backend/list_metadata_api"; import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; +import { listBenchmarkMetadata } from "lib/benchmark/api_helper/backend/list_metadata_api"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( From 25a6874e9105ccbab0a6fe2ceca3db58559d213b Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 14:01:14 -0700 Subject: [PATCH 07/19] add config --- .../api_helper/backend/get_time_series.ts | 5 +- .../api_helper/backend/list_commits.ts | 4 +- .../BenchmarkDataQueryBuilder.ts | 350 ++++++++++++++++++ .../defaultGetBenchmarkDataQueryBuilder.ts | 194 ---------- .../backend/queryBuilderUtils/queryBuilder.ts | 2 +- .../pages/api/benchmark/get_benchmark_data.ts | 0 6 files changed, 355 insertions(+), 200 deletions(-) create mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts delete mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts create mode 100644 torchci/pages/api/benchmark/get_benchmark_data.ts diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts index e75a77c29d..48c0ce1c0e 100644 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts @@ -1,6 +1,6 @@ import { CompilerQueryType } from "./common/type"; import { getCompilerBenchmarkTimeSeriesData } from "./compilers/compiler_benchmark_data"; -import { BenchmarkDataQuery } from "./queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder"; +import { getBenchmarkDataQuery } from "./queryBuilderUtils/BenchmarkDataQueryBuilder"; export async function getBenchmarkTimeSeriesData( request_name: string, @@ -36,6 +36,7 @@ async function getGenernalBenchmarkTimeSeries( formats: string[], id: string ) { - const queryBuilder = new BenchmarkDataQuery(); + const queryBuilder = getBenchmarkDataQuery(id); const result = await queryBuilder.applyQuery(query_params); + return queryBuilder.applyFormat(result, formats); } diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index 62cbd5fd2a..60c3559c4b 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -28,9 +28,7 @@ export async function listBenchmarkCommitsFromDb( } const formats: string[] = - responseFormats && responseFormats.length != 0 - ? responseFormats - : ["raw"]; + responseFormats && responseFormats.length != 0 ? responseFormats : ["raw"]; // format data based on requested response formats, for instance if format is "branch", // we group the data by branch and return the data for each branch diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts new file mode 100644 index 0000000000..c953c9bdca --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts @@ -0,0 +1,350 @@ +import { deepClone } from "@mui/x-data-grid/internals"; +import { toBenchmarkTimeSeriesReponseFormat } from "../common/utils"; +import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; + +const DEFAULT_TS_GROUP_KEY = [ + "dtype", + "arch", + "device", + "backend", + "metric", + "mode", + "model", + "branch", +]; +const DEFAULT_TS_SUBGROUP_KEY = ["workflow_id"]; + +const DEFAULT_TABLE_GROUP_KEY = [ + "workflow_id", + "dtype", + "arch", + "device", + "backend", + "mode", + "branch", + "model", +]; +const DEFAULT_TABLE_SUB_GROUP_KEY = ["metric"]; + +export interface BenchmarkGroupConfig { + group_key: string[]; + sub_group_key: string[]; +} + +export const DEFAULT_BENCHMARK_GROUP_MAP = { + time_series: { + group_key: DEFAULT_TS_GROUP_KEY, + sub_group_key: DEFAULT_TS_SUBGROUP_KEY, + }, + table: { + group_key: DEFAULT_TABLE_GROUP_KEY, + sub_group_key: DEFAULT_TABLE_SUB_GROUP_KEY, + }, +}; + +/** + * Query to get benchmark data from the benchmark table + * for repo/benchmark specific data use: + * - addInnerExtraInfo(): if the field is also in the extra column as unique key + * - addInnerMetadataInfo(): if it's only metadata info and does not act as unique key + */ +export class BenchmarkDataQuery extends ExecutableQueryBase { + private _inner_query_builder: QueryBuilder; + private _main_query_builder: QueryBuilder; + private _format_config: { [key: string]: BenchmarkGroupConfig } = deepClone( + DEFAULT_BENCHMARK_GROUP_MAP + ); + private _extra_keys = new Set(); + + // must included in all select statement + private _required_metadata_info_statements: Map; + constructor() { + super(); + this._required_metadata_info_statements = new Map([ + [ + "timestamp", + "formatDateTime(fromUnixTimestamp(timestamp), '%Y-%m-%dT%H:%i:%sZ')", + ], + ]); + + const metadata_info_select = toQueryMapResult( + "metadata_info", + this._required_metadata_info_statements + ); + + this._inner_query_builder = new QueryBuilder( + { + table: "benchmark.oss_ci_benchmark_v3 o", + select_exists: true, + where_exists: true, + // default select statement for customized query + select: [["map()", "extra"], metadata_info_select], + prewhere: [ + "o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) })", + "o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) })", + ], + }, + ` + SELECT + replaceOne(o.head_branch, 'refs/heads/', '') AS head_branch, + o.workflow_id AS workflow_id, + o.job_id AS job_id, + o.model.'name' AS model, + o.model.'backend' AS backend, + o.model.'origins' AS origins, + o.metric.'name' AS metric, + -- Arithmetic mean + floor(arrayAvg(o.metric.'benchmark_values'), 2) AS actual, + -- Geometric mean + floor(exp(arrayAvg(arrayMap(x -> log(x), o.metric.'benchmark_values'))), 2) AS actual_geomean, + floor(toFloat64(o.metric.'target_value'), 2) AS target, + o.benchmark.'mode' AS mode, + o.benchmark.'dtype' AS dtype, + o.benchmark.'extra_info' AS debugging_info, + IF( + empty(o.runners), + tupleElement(o.benchmark, 'extra_info')['device'], + tupleElement(o.runners[1], 'name') + ) AS device, + IF( + empty(o.runners), + tupleElement(o.benchmark, 'extra_info')['arch'], + tupleElement(o.runners[1], 'type') + ) AS arch, + DATE_TRUNC( + {granularity: String }, + fromUnixTimestamp(o.timestamp) + ) AS granularity_bucket + {{SELECT}} + FROM {{TABLE}} + {{PREWHERE}} + WHERE + o.repo = {repo: String } + AND ( + has({commits: Array(String) }, o.head_sha) + OR empty({commits: Array(String) }) + ) + AND ( + o.benchmark.'name' in {benchmarkNames: Array(String) } + OR empty({benchmarkNames: Array(String) }) + ) + AND ( + has({models: Array(String) }, o.model.'name') + OR empty({models: Array(String) }) + ) + AND ( + has({backends: Array(String) }, o.model.'backend') + OR empty({backends: Array(String) }) + ) + AND ( + o.benchmark.'mode' = {mode: String } + OR {mode: String } = '' + ) + AND ( + has({dtypes: Array(String) }, o.benchmark.'dtype') + OR empty({dtypes: Array(String) }) + ) + AND ( + NOT has({excludedMetrics: Array(String) }, o.metric.'name') + OR empty({excludedMetrics: Array(String) }) + ) + AND notEmpty(o.metric.'name') + {{WHERE}} + ` + ); + this._main_query_builder = new QueryBuilder( + { + table: "benchmarks", + select_exists: true, + where_exists: true, + }, + ` + SELECT DISTINCT + workflow_id, + job_id, + model, + backend, + origins, + metric, + actual, + target, + mode, + dtype, + device, + arch, + granularity_bucket, + extra, + metadata_info + {{SELECT}} + FROM {{TABLE}} + WHERE + ( + has({branches: Array(String) }, head_branch) + OR empty({branches: Array(String) }) + ) + AND notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) + {{WHERE}} + ORDER BY + granularity_bucket DESC, + workflow_id DESC, + backend, + model, + mode, + dtype, + device, + metric + ` + ); + } + + /** + * map of extra info statements to be added to the query + * notice this will override the default extra info statement, but will not override the required fields + * @param extraInfoMapStatements + */ + addExtraInfos(extraInfoMapStatements: Map) { + // store the extra keys for later use + this._extra_keys = new Set(extraInfoMapStatements.keys()); + + const mapSelectItem = toQueryMapResult("extra", extraInfoMapStatements); + this._inner_query_builder.addSelect([mapSelectItem]); + } + + /** + * map of extra info statements to be added to the query + * @param metadataInfoMapStatements + */ + addMetadataInfos(metadataInfoMapStatements: Map) { + const mapSelectItem = toQueryMapResult( + "metadata_info", + metadataInfoMapStatements, + this._required_metadata_info_statements + ); + this._inner_query_builder.addSelect([mapSelectItem]); + } + + /** + * + * @param rawData + * @param formats + * @returns + */ + toFormat(rawData: any[], formats: string[], includesAllExtraKey: boolean) { + const config = this._format_config; + if (includesAllExtraKey) { + config.time_series.group_key = [ + ...config.time_series.sub_group_key, + ...Array.from(this._extra_keys), + ]; + config.table.sub_group_key = [ + ...config.table.sub_group_key, + ...Array.from(this._extra_keys), + ]; + } + return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); + } + + build() { + const inner = this._inner_query_builder.build(); + const primary = this._main_query_builder.build(); + return ` + WITH benchmarks AS ( + ${inner} + ) + ${primary} + `; + } + + validateInputs(inputs: any) { + if (!inputs.benchmarkName && !inputs.benchmarkNames) { + throw new Error( + "Either benchmarkName or benchmarkNames must be provided" + ); + } + if (!inputs.repo) { + throw new Error("repo must be provided"); + } + } + + toQueryParams(inputs: any, id?: string): Record { + this.validateInputs(inputs); + const defaults = { + excludedMetrics: [], + commits: [], + arch: "", + mode: "", + models: [], + branches: [], + granularity: "hour", + backends: [], + dtypes: [], + }; + + if (inputs.benchmarkName) { + inputs.benchmarkNames = [inputs.benchmarkName]; + } + + const params = { ...defaults, ...inputs }; + return params; + } +} + +function toQueryMapResult( + resultName: string, + statements: Map, + additionalStatements?: Map +): [string, string] { + // Merge both maps into one array of [key, statement] + const allEntries: [string, string][] = [ + ...Array.from(statements.entries()), + ...(additionalStatements ? Array.from(additionalStatements.entries()) : []), + ]; + + // Build key-value pairs like `'key', value` + const pairs = allEntries + .filter(([key, stmt]) => key && stmt) + .map(([key, stmt]) => `'${key}', ${stmt}`) + .join(",\n "); + // Final SQL fragment: map('key1', value1, 'key2', value2, ...) + const sqlExpr = allEntries.length > 0 ? `map(${pairs})` : "map()"; + + return [sqlExpr, resultName]; +} + +export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase { + private _data_query: BenchmarkDataQuery; + constructor() { + super(); + this._data_query = new BenchmarkDataQuery(); + this._data_query.addExtraInfos( + new Map([ + [ + "operator_name", + `IF( + mapContains(tupleElement(o.benchmark, 'extra_info'), 'operator_name'), + tupleElement(o.benchmark, 'extra_info')['operator_name'], + '' + )`, + ], + ]) + ); + } + toQueryParams(inputs: any, id?: string): Record { + return this._data_query.toQueryParams(inputs, id); + } + build() { + return this._data_query.build(); + } +} + +export function getBenchmarkDataQuery(name: string) { + const MAP: Record = { + pytorch_operator_micro_benchmark: + new PytorchOperatorMicroBenchmarkDataQuery(), + }; + return MAP[name] ?? new BenchmarkDataQuery(); +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts deleted file mode 100644 index 6e0c14ca4f..0000000000 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultGetBenchmarkDataQueryBuilder.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; - -/** - * Query to get benchmark data from the benchmark table - * for repo/benchmark specific data use: - * - addInnerExtraInfo(): if the field is also in the extra column as unique key - * - addInnerMetadataInfo(): if it's only metadata info and does not act as unique key - */ -export class BenchmarkDataQuery extends ExecutableQueryBase { - private _inner_query_builder: QueryBuilder; - private _main_query_builder: QueryBuilder; - - // must included in all select statement - private _required_metadata_info_statements: Map; - constructor() { - super(); - this._required_metadata_info_statements = new Map([ - [ - "timestamp", - "formatDateTime(fromUnixTimestamp(timestamp), '%Y-%m-%dT%H:%i:%sZ')", - ], - ]); - const metadata_infos = this._required_metadata_info_statements.forEach( - (statement, key) => { - return `${key},${statement}`; - } - ); - this._inner_query_builder = new QueryBuilder( - { - table: "benchmark.oss_ci_benchmark_v3", - select_exists: true, - where_exists: true, - // default select statement for customized query - select: [ - ["map()", "extra"], - [`map(${metadata_infos})`, "metadata_info"], - ], - prewhere: [ - "timestamp >= toUnixTimestamp({startTime: DateTime64(3) })", - "timestamp < toUnixTimestamp({endTime: DateTime64(3) })", - ], - }, - ` - SELECT - replaceOne(head_branch, 'refs/heads/', '') AS head_branch, - workflow_id AS workflow_id, - job_id AS job_id, - model.'name' AS model, - model.'backend' AS backend, - model.'origins' AS origins, - metric.'name' AS metric, - floor(arrayAvg(metric.'benchmark_values'), 2) AS value, - floor(toFloat64(metric.'target_value'), 2) AS target, - benchmark.'mode' AS mode, - benchmark.'dtype' AS dtype, - IF( - empty(runners), - tupleElement(benchmark, 'extra_info')['device'], - tupleElement(runners[1], 'name') - ) AS device, - IF( - empty(runners), - tupleElement(benchmark, 'extra_info')['arch'], - tupleElement(runners[1], 'type') - ) AS arch, - DATE_TRUNC( - {granularity: String }, - fromUnixTimestamp(timestamp) - ) AS granularity_bucket - {{SELECT}} - FROM {{TABLE}} - {{PREWHERE}} - WHERE - repo = {repo: String } - AND notEmpty(metric.'name') - AND ( - has({commits: Array(String) }, head_sha) - OR empty({commits: Array(String) }) - ) - AND ( - benchmark.'name' in {benchmarks: Array(String) } - OR empty({benchmarks: Array(String) }) - ) - AND ( - has({models: Array(String) }, model.'name') - OR empty({models: Array(String) }) - ) - AND ( - has({backends: Array(String) }, model.'backend') - OR empty({backends: Array(String) }) - ) - AND ( - benchmark.'mode' = {mode: String } - OR {mode: String } = '' - ) - AND ( - has({dtypes: Array(String) }, benchmark.'dtype') - OR empty({dtypes: Array(String) }) - ) - {{WHERE}} - ` - ); - this._main_query_builder = new QueryBuilder( - { - table: "benchmarks", - select_exists: true, - where_exists: true, - }, - ` - SELECT DISTINCT - workflow_id, - job_id, - model, - backend, - origins, - metric, - actual, - target, - mode, - dtype, - device, - arch, - granularity_bucket, - extra, - metadata_info - {{SELECT}} - FROM {{TABLE}} - WHERE - ( - has({branches: Array(String) }, head_branch) - OR empty({branches: Array(String) }) - ) - AND notEmpty(device) - AND ( - arch LIKE concat('%', {arch: String }, '%') - OR {arch: String } = '' - ) - {{WHERE}} - ORDER BY - granularity_bucket DESC, - workflow_id DESC, - backend, - model, - mode, - dtype, - device, - metric - ` - ); - } - - addInnerExtraInfo(extraInfoMapStatements: Map) { - const mapSelectItem = toQueryMapResult("extra", extraInfoMapStatements); - this._inner_query_builder.addSelect(mapSelectItem); - } - - addInnerMetadataInfo(metadataInfoMapStatements: Map) { - const mapSelectItem = toQueryMapResult( - "metadata_info", - metadataInfoMapStatements, - this._required_metadata_info_statements - ); - this._inner_query_builder.addSelect(mapSelectItem); - } - - build() { - const inner = this._inner_query_builder.build(); - const primary = this._main_query_builder.build(); - return ` - WITH benchmarks AS ( - ${inner} - ) - ${primary} - `; - } -} - -function toQueryMapResult( - resultName: string, - statements: Map, - additionalStatemetns?: Map -) { - let statement_items: string[] = []; - statements.forEach((statement, key) => { - statement_items.push(`${key},${statement}`); - }); - if (additionalStatemetns) { - additionalStatemetns.forEach((statement, key) => { - statement_items.push(`${key},${statement}`); - }); - } - const statement = statement_items.join(",\n "); - return [`map(${statement})`, resultName]; -} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts index 01325bc544..085aa21f70 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts @@ -88,7 +88,6 @@ export class QueryBuilder { const key = (s: SelectItem) => (Array.isArray(s) ? s[1] : s); for (const it of this.defaults.select ?? []) byField.set(key(it), it); for (const it of this.extraSelect) byField.set(key(it), it); - let SELECT = [...byField.values()] .map((it) => (Array.isArray(it) ? `${it[0]} AS ${it[1]}` : it)) .join(",\n "); @@ -154,6 +153,7 @@ export type QueryExecutor = ( export interface BuildableQuery { build(): string; // must return SQL string toQueryParams(inputs: any, id?: string): Record; + applyQuery(inputs: any, executor?: QueryExecutor): Promise; } /** diff --git a/torchci/pages/api/benchmark/get_benchmark_data.ts b/torchci/pages/api/benchmark/get_benchmark_data.ts new file mode 100644 index 0000000000..e69de29bb2 From 6c08d9f524521d81b5d6d67ddbc18a3d103fca3c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 14:03:23 -0700 Subject: [PATCH 08/19] add config --- .../backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts index c953c9bdca..54d03cea9b 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts @@ -315,6 +315,9 @@ function toQueryMapResult( return [sqlExpr, resultName]; } +/** + * Builder to get PytorchOperatorMicroBenchmark + */ export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase { private _data_query: BenchmarkDataQuery; constructor() { @@ -333,6 +336,7 @@ export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase ]) ); } + toQueryParams(inputs: any, id?: string): Record { return this._data_query.toQueryParams(inputs, id); } From f489007c6a20ac03eb73783bcb4aa11d86b71bf6 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 14:05:42 -0700 Subject: [PATCH 09/19] add config --- .../queryBuilderUtils/BenchmarkDataQueryBuilder.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts index 54d03cea9b..53e07ebe13 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts @@ -42,11 +42,13 @@ export const DEFAULT_BENCHMARK_GROUP_MAP = { }, }; + /** - * Query to get benchmark data from the benchmark table + * QueryBuilder to get benchmark data from the benchmark table * for repo/benchmark specific data use: - * - addInnerExtraInfo(): if the field is also in the extra column as unique key - * - addInnerMetadataInfo(): if it's only metadata info and does not act as unique key + * - addExtraInfo(): if the field is also in the extra column as unique key + * - addMetadataInfo(): if it's only metadata info and does not act as unique key + * - toFormat(): to format the data to the desired format */ export class BenchmarkDataQuery extends ExecutableQueryBase { private _inner_query_builder: QueryBuilder; @@ -248,6 +250,10 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); } + getFormatConfig() { + return deepClone(this._format_config); + } + build() { const inner = this._inner_query_builder.build(); const primary = this._main_query_builder.build(); @@ -336,10 +342,10 @@ export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase ]) ); } - toQueryParams(inputs: any, id?: string): Record { return this._data_query.toQueryParams(inputs, id); } + build() { return this._data_query.build(); } From 55409eeca4d930878ae243b48707c9ac0fa8fe47 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 15:14:44 -0700 Subject: [PATCH 10/19] add config --- .../api_helper/backend/common/utils.ts | 14 ++-- .../api_helper/backend/get_time_series.ts | 9 +-- .../BenchmarkDataQueryBuilder.ts | 67 ++++++++++++++----- .../backend/queryBuilderUtils/queryBuilder.ts | 4 ++ 4 files changed, 65 insertions(+), 29 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/common/utils.ts b/torchci/lib/benchmark/api_helper/backend/common/utils.ts index d3c2cebb80..96c524d839 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/utils.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/utils.ts @@ -274,11 +274,6 @@ export function to_time_series_data( new Date(a.granularity_bucket).getTime() - new Date(b.granularity_bucket).getTime() ); - if (diffs.length > 0) { - console.log( - `we detected multiple datapoints for the same group keys ${diffs.length}` - ); - } return { group_info, num_of_dp: ts_list.length, @@ -287,6 +282,11 @@ export function to_time_series_data( data: ts_list, }; }); + if (diffs.length > 0) { + console.log( + `we detected multiple datapoints for the same group keys ${diffs.length}, peak first on \n ${JSON.stringify(diffs[0].key)}, \n Data1: ${JSON.stringify(diffs[0].data[0])}, Data:2 ${JSON.stringify(diffs[0].data[1])}` + ); + } return result; } @@ -408,13 +408,13 @@ export function toBenchmarkTimeSeriesReponseFormat( let formats_result: any = {}; formats.forEach((format) => { - const data = getformat(rawData, format, config); + const data = getBenchmarkDataformat(rawData, format, config); formats_result[format] = data; }); return toTimeSeriesResponse(formats_result, rawData.length, start_ts, end_ts); } -function getformat(data: any, format: string, config: any) { +function getBenchmarkDataformat(data: any, format: string, config: any) { switch (format) { case "time_series": return to_time_series_data( diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts index 48c0ce1c0e..7c28810c8d 100644 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts +++ b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts @@ -1,6 +1,6 @@ import { CompilerQueryType } from "./common/type"; import { getCompilerBenchmarkTimeSeriesData } from "./compilers/compiler_benchmark_data"; -import { getBenchmarkDataQuery } from "./queryBuilderUtils/BenchmarkDataQueryBuilder"; +import { getBenchmarkDataFetcher } from "./queryBuilderUtils/BenchmarkDataQueryBuilder"; export async function getBenchmarkTimeSeriesData( request_name: string, @@ -36,7 +36,8 @@ async function getGenernalBenchmarkTimeSeries( formats: string[], id: string ) { - const queryBuilder = getBenchmarkDataQuery(id); - const result = await queryBuilder.applyQuery(query_params); - return queryBuilder.applyFormat(result, formats); + + const fetcher = getBenchmarkDataFetcher(id); + const result = await fetcher.applyQuery(query_params); + return fetcher.applyFormat(result, formats); } diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts index 53e07ebe13..92871d9ec1 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts @@ -95,14 +95,10 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { o.model.'backend' AS backend, o.model.'origins' AS origins, o.metric.'name' AS metric, - -- Arithmetic mean floor(arrayAvg(o.metric.'benchmark_values'), 2) AS actual, - -- Geometric mean - floor(exp(arrayAvg(arrayMap(x -> log(x), o.metric.'benchmark_values'))), 2) AS actual_geomean, floor(toFloat64(o.metric.'target_value'), 2) AS target, o.benchmark.'mode' AS mode, o.benchmark.'dtype' AS dtype, - o.benchmark.'extra_info' AS debugging_info, IF( empty(o.runners), tupleElement(o.benchmark, 'extra_info')['device'], @@ -211,7 +207,6 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { addExtraInfos(extraInfoMapStatements: Map) { // store the extra keys for later use this._extra_keys = new Set(extraInfoMapStatements.keys()); - const mapSelectItem = toQueryMapResult("extra", extraInfoMapStatements); this._inner_query_builder.addSelect([mapSelectItem]); } @@ -235,21 +230,26 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { * @param formats * @returns */ - toFormat(rawData: any[], formats: string[], includesAllExtraKey: boolean) { + applyFormat(rawData: any[], formats: string[], includesAllExtraKey: boolean) { const config = this._format_config; if (includesAllExtraKey) { config.time_series.group_key = [ - ...config.time_series.sub_group_key, - ...Array.from(this._extra_keys), + ...config.time_series.group_key, + ...Array.from(this._extra_keys).map((key) => `extra.${key}`), ]; config.table.sub_group_key = [ ...config.table.sub_group_key, - ...Array.from(this._extra_keys), + ...Array.from(this._extra_keys).map((key) => `extra.${key}`), ]; } return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); } + // reset format config + setFormatConfig(config: { [key: string]: BenchmarkGroupConfig }) { + this._format_config = config; + } + getFormatConfig() { return deepClone(this._format_config); } @@ -299,6 +299,13 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { } } +/** + * helper function to convert a map of statements to a query map result + * e.g. map('key1', value1, 'key2', value2, ...) + * @param statements + * @param additionalStatements + * @returns + */ function toQueryMapResult( resultName: string, statements: Map, @@ -321,10 +328,27 @@ function toQueryMapResult( return [sqlExpr, resultName]; } + +/** + * Main function to get the query builder for a specific benchmark data + * if id not found, return default query builder + * @param name + * @returns + */ +export function getBenchmarkDataFetcher(name: string) { + const MAP: Record = { + pytorch_operator_microbenchmark: + new PytorchOperatorMicroBenchmarkDataFetcher(), + }; + return MAP[name] ?? new BenchmarkDataQuery(); +} + /** * Builder to get PytorchOperatorMicroBenchmark + * It inherits method from BenchmarkDataQuery + * */ -export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase { +export class PytorchOperatorMicroBenchmarkDataFetcher extends ExecutableQueryBase { private _data_query: BenchmarkDataQuery; constructor() { super(); @@ -339,22 +363,29 @@ export class PytorchOperatorMicroBenchmarkDataQuery extends ExecutableQueryBase '' )`, ], + [ + "use_compile", + `IF( + mapContains(tupleElement(o.benchmark, 'extra_info'), 'use_compile'), + tupleElement(o.benchmark, 'extra_info')['use_compile'], + '' + )`, + ] ]) ); } + + applyFormat(rawData: any[], formats: string[]) { + return this._data_query.applyFormat(rawData, formats, true); + } + toQueryParams(inputs: any, id?: string): Record { return this._data_query.toQueryParams(inputs, id); } build() { + // debugging + // console.log("build PytorchOperatorMicroBenchmarkDataQuery", this._data_query.build()); return this._data_query.build(); } } - -export function getBenchmarkDataQuery(name: string) { - const MAP: Record = { - pytorch_operator_micro_benchmark: - new PytorchOperatorMicroBenchmarkDataQuery(), - }; - return MAP[name] ?? new BenchmarkDataQuery(); -} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts index 085aa21f70..1ac1de90e8 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts @@ -158,6 +158,10 @@ export interface BuildableQuery { /** * Base class for executable queries. + * - build(): returns SQL string + * - toQueryParams(): returns params for SQL execution + * - applyQuery(): executes SQL via provided executor (default: queryClickhouse) + * */ export abstract class ExecutableQueryBase implements BuildableQuery { abstract build(): string; From d6c0be51e91938b30275d51c4e6cb2186ee0f637 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 15:21:12 -0700 Subject: [PATCH 11/19] add config --- .../BenchmarkDataQueryBuilder.ts | 17 ++++++++++------- .../backend/queryBuilderUtils/queryBuilder.ts | 5 ++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts index 92871d9ec1..97dc46b4e4 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts @@ -51,6 +51,8 @@ export const DEFAULT_BENCHMARK_GROUP_MAP = { * - toFormat(): to format the data to the desired format */ export class BenchmarkDataQuery extends ExecutableQueryBase { + private _EXTRA_KEY_FIELD_NAME = "extra_key"; + private _METADATA_INFO_FIELD_NAME = "metadata_info"; private _inner_query_builder: QueryBuilder; private _main_query_builder: QueryBuilder; private _format_config: { [key: string]: BenchmarkGroupConfig } = deepClone( @@ -70,7 +72,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { ]); const metadata_info_select = toQueryMapResult( - "metadata_info", + this._METADATA_INFO_FIELD_NAME, this._required_metadata_info_statements ); @@ -80,7 +82,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { select_exists: true, where_exists: true, // default select statement for customized query - select: [["map()", "extra"], metadata_info_select], + select: [["map()", this._EXTRA_KEY_FIELD_NAME], metadata_info_select], prewhere: [ "o.timestamp >= toUnixTimestamp({startTime: DateTime64(3) })", "o.timestamp < toUnixTimestamp({stopTime: DateTime64(3) })", @@ -171,7 +173,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { device, arch, granularity_bucket, - extra, + extra_key, metadata_info {{SELECT}} FROM {{TABLE}} @@ -207,7 +209,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { addExtraInfos(extraInfoMapStatements: Map) { // store the extra keys for later use this._extra_keys = new Set(extraInfoMapStatements.keys()); - const mapSelectItem = toQueryMapResult("extra", extraInfoMapStatements); + const mapSelectItem = toQueryMapResult(this._EXTRA_KEY_FIELD_NAME, extraInfoMapStatements); this._inner_query_builder.addSelect([mapSelectItem]); } @@ -217,7 +219,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { */ addMetadataInfos(metadataInfoMapStatements: Map) { const mapSelectItem = toQueryMapResult( - "metadata_info", + this._METADATA_INFO_FIELD_NAME, metadataInfoMapStatements, this._required_metadata_info_statements ); @@ -332,8 +334,7 @@ function toQueryMapResult( /** * Main function to get the query builder for a specific benchmark data * if id not found, return default query builder - * @param name - * @returns + * */ export function getBenchmarkDataFetcher(name: string) { const MAP: Record = { @@ -353,6 +354,8 @@ export class PytorchOperatorMicroBenchmarkDataFetcher extends ExecutableQueryBas constructor() { super(); this._data_query = new BenchmarkDataQuery(); + + // add extra info to the query this._data_query.addExtraInfos( new Map([ [ diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts index 1ac1de90e8..1709b7c525 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts @@ -171,7 +171,10 @@ export abstract class ExecutableQueryBase implements BuildableQuery { return inputs; } - /** Build SQL and execute via provided executor. */ + /** Build SQL and execute via provided executor. + * by default, queryClickhouse is used. and it calls toQueryParams() to get processed params. + * + */ async applyQuery( inputs: any, executor: QueryExecutor = queryClickhouse From c9efe81c0393ad06268218fbb942fe8a8cccb1b7 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 17:06:39 -0700 Subject: [PATCH 12/19] add config --- .../api_helper/backend/common/utils.ts | 17 +- .../backend/dataFetchers/fetchers.ts | 42 ++++ .../benchmarkDataQueryBuilder.ts} | 130 ++++++++----- .../listMetadataQueryBuilder.ts | 181 ++++++++++++++++++ .../queryBuilderUtils/queryBuilder.ts | 9 +- .../api_helper/backend/dataFetchers/type.ts | 13 ++ .../api_helper/backend/get_time_series.ts | 43 ----- .../api_helper/backend/list_metadata_api.ts | 113 ----------- .../defaultListMetadataQueryBuilder.ts | 66 ------- .../pages/api/benchmark/get_time_series.ts | 43 ++++- torchci/pages/api/benchmark/list_metadata.ts | 9 +- 11 files changed, 383 insertions(+), 283 deletions(-) create mode 100644 torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts rename torchci/lib/benchmark/api_helper/backend/{queryBuilderUtils/BenchmarkDataQueryBuilder.ts => dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts} (79%) create mode 100644 torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts rename torchci/lib/benchmark/api_helper/backend/{ => dataFetchers}/queryBuilderUtils/queryBuilder.ts (97%) create mode 100644 torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts delete mode 100644 torchci/lib/benchmark/api_helper/backend/get_time_series.ts delete mode 100644 torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts delete mode 100644 torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts diff --git a/torchci/lib/benchmark/api_helper/backend/common/utils.ts b/torchci/lib/benchmark/api_helper/backend/common/utils.ts index 96c524d839..78b34ced54 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/utils.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/utils.ts @@ -283,9 +283,15 @@ export function to_time_series_data( }; }); if (diffs.length > 0) { - console.log( - `we detected multiple datapoints for the same group keys ${diffs.length}, peak first on \n ${JSON.stringify(diffs[0].key)}, \n Data1: ${JSON.stringify(diffs[0].data[0])}, Data:2 ${JSON.stringify(diffs[0].data[1])}` - ); + console.log( + `we detected multiple datapoints for the same group keys ${ + diffs.length + }, peak first on \n ${JSON.stringify( + diffs[0].key + )}, \n Data1: ${JSON.stringify( + diffs[0].data[0] + )}, Data:2 ${JSON.stringify(diffs[0].data[1])}` + ); } return result; } @@ -400,7 +406,10 @@ export function toBenchmarkTimeSeriesReponseFormat( config: any = {}, formats: string[] = ["time_series"] ) { - const start_ts = new Date(rawData[0].granularity_bucket).getTime(); + if (rawData.length === 0) { + return emptyTimeSeriesResponse(); + } + const start_ts = new Date(rawData[0]?.granularity_bucket).getTime(); const end_ts = new Date( rawData[rawData.length - 1].granularity_bucket ).getTime(); diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts new file mode 100644 index 0000000000..605717909c --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/fetchers.ts @@ -0,0 +1,42 @@ +import { + BenchmarkDataQuery, + PytorchOperatorMicroBenchmarkDataFetcher, +} from "./queryBuilderUtils/benchmarkDataQueryBuilder"; +import { + BenchmarkMetadataQuery, + PytorchOperatorMicrobenchmarkMetadataFetcher, +} from "./queryBuilderUtils/listMetadataQueryBuilder"; +import { BenchmarkDataFetcher, BenchmarkMetadataFetcher } from "./type"; + +// Register benchmark data fetchers, this is mainly used in get_benchmark_data api and get_time_series api +const dataCtors: Record BenchmarkDataFetcher> = { + pytorch_operator_microbenchmark: PytorchOperatorMicroBenchmarkDataFetcher, + default: BenchmarkDataQuery, +}; + +// Register benchmark metadata fetchers. this is mainly used in list_metadata api +const metaCtors: Record BenchmarkMetadataFetcher> = { + pytorch_operator_microbenchmark: PytorchOperatorMicrobenchmarkMetadataFetcher, + default: BenchmarkMetadataQuery, +}; + +/** + * Main function to get the query builder for a specific benchmark data + * if id not found, return default query builder + * + */ +export function getBenchmarkDataFetcher(id: string): BenchmarkDataFetcher { + const Ctor = dataCtors[id] ?? dataCtors.default; + return new Ctor(); +} + +/** + * Main function to get the query builder for a specific benchmark data + * if id not found, return default query builder + */ +export function getListBenchmarkMetadataFetcher( + id: string +): BenchmarkMetadataFetcher { + const Ctor = metaCtors[id] ?? metaCtors.default; + return new Ctor(); +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts similarity index 79% rename from torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts rename to torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts index 97dc46b4e4..9200b48fa9 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/BenchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts @@ -1,5 +1,6 @@ import { deepClone } from "@mui/x-data-grid/internals"; -import { toBenchmarkTimeSeriesReponseFormat } from "../common/utils"; +import { toBenchmarkTimeSeriesReponseFormat } from "../../common/utils"; +import { BenchmarkDataFetcher } from "../type"; import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; const DEFAULT_TS_GROUP_KEY = [ @@ -16,6 +17,7 @@ const DEFAULT_TS_SUBGROUP_KEY = ["workflow_id"]; const DEFAULT_TABLE_GROUP_KEY = [ "workflow_id", + "commit", "dtype", "arch", "device", @@ -42,13 +44,14 @@ export const DEFAULT_BENCHMARK_GROUP_MAP = { }, }; - +// TODO(elainewy) apply listCommits first to get the completed data /** * QueryBuilder to get benchmark data from the benchmark table * for repo/benchmark specific data use: * - addExtraInfo(): if the field is also in the extra column as unique key * - addMetadataInfo(): if it's only metadata info and does not act as unique key - * - toFormat(): to format the data to the desired format + * - applyFormat(): to format the data to the desired format, default is time series + * */ export class BenchmarkDataQuery extends ExecutableQueryBase { private _EXTRA_KEY_FIELD_NAME = "extra_key"; @@ -60,6 +63,18 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { ); private _extra_keys = new Set(); + DEFAULT_PARAMS = { + excludedMetrics: [], + commits: [], + arch: "", + mode: "", + models: [], + branches: [], + granularity: "hour", + backends: [], + dtypes: [], + }; + // must included in all select statement private _required_metadata_info_statements: Map; constructor() { @@ -90,9 +105,10 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { }, ` SELECT - replaceOne(o.head_branch, 'refs/heads/', '') AS head_branch, + replaceOne(o.head_branch, 'refs/heads/', '') AS branch, o.workflow_id AS workflow_id, o.job_id AS job_id, + o.head_sha AS commit, o.model.'name' AS model, o.model.'backend' AS backend, o.model.'origins' AS origins, @@ -161,6 +177,8 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { ` SELECT DISTINCT workflow_id, + branch, + commit, job_id, model, backend, @@ -179,7 +197,7 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { FROM {{TABLE}} WHERE ( - has({branches: Array(String) }, head_branch) + has({branches: Array(String) }, branch) OR empty({branches: Array(String) }) ) AND notEmpty(device) @@ -209,7 +227,10 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { addExtraInfos(extraInfoMapStatements: Map) { // store the extra keys for later use this._extra_keys = new Set(extraInfoMapStatements.keys()); - const mapSelectItem = toQueryMapResult(this._EXTRA_KEY_FIELD_NAME, extraInfoMapStatements); + const mapSelectItem = toQueryMapResult( + this._EXTRA_KEY_FIELD_NAME, + extraInfoMapStatements + ); this._inner_query_builder.addSelect([mapSelectItem]); } @@ -226,22 +247,34 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { this._inner_query_builder.addSelect([mapSelectItem]); } + addInnerWhereStatements(statements: string[]) { + this._inner_query_builder.addWhere(statements); + } + /** * * @param rawData * @param formats * @returns */ - applyFormat(rawData: any[], formats: string[], includesAllExtraKey: boolean) { + applyFormat( + rawData: any[], + formats: string[], + includesAllExtraKey: boolean = true + ) { const config = this._format_config; if (includesAllExtraKey) { config.time_series.group_key = [ ...config.time_series.group_key, - ...Array.from(this._extra_keys).map((key) => `extra.${key}`), + ...Array.from(this._extra_keys).map( + (key) => `${this._EXTRA_KEY_FIELD_NAME}.${key}` + ), ]; config.table.sub_group_key = [ ...config.table.sub_group_key, - ...Array.from(this._extra_keys).map((key) => `extra.${key}`), + ...Array.from(this._extra_keys).map( + (key) => `${this._EXTRA_KEY_FIELD_NAME}.${key}` + ), ]; } return toBenchmarkTimeSeriesReponseFormat(rawData, config, formats); @@ -280,23 +313,12 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { toQueryParams(inputs: any, id?: string): Record { this.validateInputs(inputs); - const defaults = { - excludedMetrics: [], - commits: [], - arch: "", - mode: "", - models: [], - branches: [], - granularity: "hour", - backends: [], - dtypes: [], - }; if (inputs.benchmarkName) { inputs.benchmarkNames = [inputs.benchmarkName]; } - const params = { ...defaults, ...inputs }; + const params = { ...this.DEFAULT_PARAMS, ...inputs }; return params; } } @@ -330,26 +352,15 @@ function toQueryMapResult( return [sqlExpr, resultName]; } - -/** - * Main function to get the query builder for a specific benchmark data - * if id not found, return default query builder - * - */ -export function getBenchmarkDataFetcher(name: string) { - const MAP: Record = { - pytorch_operator_microbenchmark: - new PytorchOperatorMicroBenchmarkDataFetcher(), - }; - return MAP[name] ?? new BenchmarkDataQuery(); -} - /** * Builder to get PytorchOperatorMicroBenchmark * It inherits method from BenchmarkDataQuery * */ -export class PytorchOperatorMicroBenchmarkDataFetcher extends ExecutableQueryBase { +export class PytorchOperatorMicroBenchmarkDataFetcher + extends ExecutableQueryBase + implements BenchmarkDataFetcher +{ private _data_query: BenchmarkDataQuery; constructor() { super(); @@ -361,34 +372,53 @@ export class PytorchOperatorMicroBenchmarkDataFetcher extends ExecutableQueryBas [ "operator_name", `IF( - mapContains(tupleElement(o.benchmark, 'extra_info'), 'operator_name'), - tupleElement(o.benchmark, 'extra_info')['operator_name'], - '' - )`, + tupleElement(o.benchmark, 'extra_info')['operator_name'] = '', + arrayElement(splitByChar('_', tupleElement(o.model, 'name')), 1), + tupleElement(o.benchmark, 'extra_info')['operator_name'] + )`, ], [ "use_compile", - `IF( - mapContains(tupleElement(o.benchmark, 'extra_info'), 'use_compile'), - tupleElement(o.benchmark, 'extra_info')['use_compile'], - '' + `IF( + tupleElement(o.benchmark, 'extra_info')['use_compile'] = '', + 'false', + -- Default to true + tupleElement(o.benchmark, 'extra_info')['use_compile'] )`, - ] + ], ]) ); + this._data_query.addInnerWhereStatements([ + `( + {operatorName:String} = '' + OR startsWith(tupleElement(o.model, 'name'), {operatorName:String}) + ) + `, + ]); } - applyFormat(rawData: any[], formats: string[]) { - return this._data_query.applyFormat(rawData, formats, true); + applyFormat( + data: any[], + formats: string[], + includesAllExtraKey: boolean = true + ) { + return this._data_query.applyFormat(data, formats, includesAllExtraKey); } toQueryParams(inputs: any, id?: string): Record { - return this._data_query.toQueryParams(inputs, id); + const params = { + ...inputs, + operatorName: inputs.operatorName ?? "", + }; + return this._data_query.toQueryParams(params, id); } build() { - // debugging - // console.log("build PytorchOperatorMicroBenchmarkDataQuery", this._data_query.build()); + // debugging + console.log( + "build PytorchOperatorMicroBenchmarkDataQuery", + this._data_query.build() + ); return this._data_query.build(); } } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts new file mode 100644 index 0000000000..02f11fb1c1 --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts @@ -0,0 +1,181 @@ +import { + DEFAULT_BACKEND_NAME, + DEFAULT_DEVICE_NAME, + DEFAULT_DTYPE_NAME, + DEFAULT_MODE_NAME, +} from "lib/benchmark/llms/common"; +import ld from "lodash"; +import { + BenchmarkMetadataItem, + BenchmarkMetadataType, +} from "../../common/type"; +import { BenchmarkMetadataFetcher } from "../type"; +import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; + +/** + * Query to get the list of metadata for a given benchmark name + */ +export class BenchmarkMetadataQuery + extends ExecutableQueryBase + implements BenchmarkMetadataFetcher +{ + private builder: QueryBuilder; + private _DEFAULT_QUERY_PARAMS = {}; + constructor() { + super(); + this.builder = new QueryBuilder({ + table: "benchmark.oss_ci_benchmark_metadata", + distinct: true, + select: [ + ["benchmark_name", "benchmark"], + ["model_name", "model"], + ["model_backend", "backend"], + ["metric_name", "metric"], + ["benchmark_dtype", "dtype"], + ["benchmark_mode", "mode"], + "device", + "arch", + ], + prewhere: [ + "timestamp >= toUnixTimestamp({startTime: DateTime64(3)})", + "timestamp < toUnixTimestamp({stopTime: DateTime64(3)})", + ], + where: [ + "repo = {repo: String}", + "benchmark_name = {benchmarkName: String}", + "notEmpty(metric_name)", + "notEmpty(device)", + ], + orderBy: [ + "benchmark", + "backend", + "model", + "metric", + "dtype", + "mode", + "device", + ], + }); + } + + build() { + return this.builder.build(); + } + + toQueryParams(inputs: any) { + this.validateInputs(inputs); + return { + ...this._DEFAULT_QUERY_PARAMS, + ...inputs, + }; + } + + validateInputs(inputs: any) { + if (!inputs.benchmarkName) { + throw new Error("No benchmark names provided"); + } + if (!inputs.repo) { + throw new Error("No repo provided"); + } + } + + postProcess(data: any) { + return getDefaultBenchmarkMetadataGroup(data); + } +} + +export function getDefaultBenchmarkMetadataGroup( + data: any[] +): BenchmarkMetadataItem[] { + return [ + makeMetadataItem( + data, + "backend", + BenchmarkMetadataType.BackendName, + DEFAULT_BACKEND_NAME, + "Backend" + ), + makeMetadataItem( + data, + "mode", + BenchmarkMetadataType.ModeName, + DEFAULT_MODE_NAME, + "Mode" + ), + makeMetadataItem( + data, + "dtype", + BenchmarkMetadataType.DtypeName, + DEFAULT_DTYPE_NAME, + "Dtype" + ), + makeMetadataItem( + data, + "device", + BenchmarkMetadataType.DeviceName, + DEFAULT_DEVICE_NAME, + "Device Name", + (r) => (r.device && r.arch ? `${r.device} (${r.arch})` : r.device) + ), + ].filter(Boolean) as BenchmarkMetadataItem[]; +} + +/** + * find distinct values from data and create metadata item + */ +function makeMetadataItem( + data: any[], + field: string, + type: BenchmarkMetadataType, + defaultValue: string, + labelName: string, + formatter?: (r: any) => string | undefined +): BenchmarkMetadataItem | null { + const values = ld.uniq( + data.map((r) => (formatter ? formatter(r) : r[field])).filter(Boolean) + ) as string[]; + + if (values.length === 0) { + return null; + } + return { + type, + options: [defaultValue, ...values], + labelName, + }; +} + +export class PytorchOperatorMicrobenchmarkMetadataFetcher + extends ExecutableQueryBase + implements BenchmarkMetadataFetcher +{ + private _data_query: BenchmarkMetadataQuery; + + constructor() { + super(); + this._data_query = new BenchmarkMetadataQuery(); + } + + postProcess(data: any[]) { + let li = getDefaultBenchmarkMetadataGroup(data); + const item = makeMetadataItem( + data, + "operator", + BenchmarkMetadataType.OperatornName, + "All Operators", + "Operator", + (r) => r.model.split("_")[0] ?? "unknown" + ); + if (item) { + li.push(item); + } + } + + build() { + return this._data_query.build(); + } + + toQueryParams(inputs: any) { + return this._data_query.toQueryParams(inputs); + } +} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts similarity index 97% rename from torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts rename to torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts index 1709b7c525..a1f46ff5d6 100644 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/queryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/queryBuilder.ts @@ -167,14 +167,13 @@ export abstract class ExecutableQueryBase implements BuildableQuery { abstract build(): string; /** Default: pass inputs through as params (override in subclasses if needed). */ - toQueryParams(inputs: any, id?: string): Record { - return inputs; - } + abstract toQueryParams(inputs: any, id?: string): Record; - /** Build SQL and execute via provided executor. + /** + * Build SQL and execute via provided executor. * by default, queryClickhouse is used. and it calls toQueryParams() to get processed params. * - */ + */ async applyQuery( inputs: any, executor: QueryExecutor = queryClickhouse diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts new file mode 100644 index 0000000000..07641e7c9e --- /dev/null +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/type.ts @@ -0,0 +1,13 @@ +export interface BenchmarkDataFetcher { + applyFormat( + data: any[], + formats: string[], + includesAllExtraKey?: boolean + ): any; + applyQuery(inputs: string): Promise; +} + +export interface BenchmarkMetadataFetcher { + postProcess(data: any[]): any; + applyQuery(inputs: string): Promise; +} diff --git a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts b/torchci/lib/benchmark/api_helper/backend/get_time_series.ts deleted file mode 100644 index 7c28810c8d..0000000000 --- a/torchci/lib/benchmark/api_helper/backend/get_time_series.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { CompilerQueryType } from "./common/type"; -import { getCompilerBenchmarkTimeSeriesData } from "./compilers/compiler_benchmark_data"; -import { getBenchmarkDataFetcher } from "./queryBuilderUtils/BenchmarkDataQueryBuilder"; - -export async function getBenchmarkTimeSeriesData( - request_name: string, - query_params: any, - formats: string[] = ["time_series"] -) { - switch (request_name) { - case "compiler_precompute": - return await getCompilerBenchmarkTimeSeriesData( - query_params, - CompilerQueryType.PRECOMPUTE, - formats - ); - case "compiler": - return await getCompilerBenchmarkTimeSeriesData( - query_params, - CompilerQueryType.GENERAL, - formats - ); - case "pytorch_operator_microbenchmark": - return await getGenernalBenchmarkTimeSeries( - query_params, - formats, - request_name - ); - default: - throw new Error(`Unsupported request_name: ${request_name}`); - } -} - -async function getGenernalBenchmarkTimeSeries( - query_params: any, - formats: string[], - id: string -) { - - const fetcher = getBenchmarkDataFetcher(id); - const result = await fetcher.applyQuery(query_params); - return fetcher.applyFormat(result, formats); -} diff --git a/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts b/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts deleted file mode 100644 index 87861d4593..0000000000 --- a/torchci/lib/benchmark/api_helper/backend/list_metadata_api.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { - DEFAULT_BACKEND_NAME, - DEFAULT_DEVICE_NAME, - DEFAULT_DTYPE_NAME, - DEFAULT_MODE_NAME, -} from "lib/benchmark/llms/common"; -import ld from "lodash"; -import { BenchmarkMetadataItem, BenchmarkMetadataType } from "./common/type"; -import { BenchmarkMetadataQuery } from "./queryBuilderUtils/defaultListMetadataQueryBuilder"; -export async function listBenchmarkMetadata(queryParams: any, id: string) { - // fetch metadata from db - const db = await listBenchmarkMetadataFromDb(queryParams); - const result = groupBenchmarkMetadata(db, id); - return result; -} - -async function listBenchmarkMetadataFromDb(queryParams: any) { - const queryBuilder = new BenchmarkMetadataQuery(); - const results = queryBuilder.applyQuery(queryParams); - return results; -} - -/** - * grouping benchmark metadata into group list - */ -function groupBenchmarkMetadata(data: any, id: string = "") { - let groups = getDefaultBenchmarkMetadataGroup(data); - const additionalGroups = getCustomizedBenchmarkMetadataGroup(data, id); - return groups.concat(additionalGroups); -} - -function getCustomizedBenchmarkMetadataGroup(data: any, id: string) { - let items = []; - switch (id) { - case "pytorch_operator_microbenchmark": - const item = makeMetadataItem( - data, - "operator", - BenchmarkMetadataType.OperatornName, - "All Operators", - "Operator", - (r) => r.model.split("_")[0] ?? "unknown" - ); - if (item) { - items.push(item); - } - return items; - default: - return []; - } -} - -export function getDefaultBenchmarkMetadataGroup( - data: any[] -): BenchmarkMetadataItem[] { - return [ - makeMetadataItem( - data, - "backend", - BenchmarkMetadataType.BackendName, - DEFAULT_BACKEND_NAME, - "Backend" - ), - makeMetadataItem( - data, - "mode", - BenchmarkMetadataType.ModeName, - DEFAULT_MODE_NAME, - "Mode" - ), - makeMetadataItem( - data, - "dtype", - BenchmarkMetadataType.DtypeName, - DEFAULT_DTYPE_NAME, - "Dtype" - ), - makeMetadataItem( - data, - "device", - BenchmarkMetadataType.DeviceName, - DEFAULT_DEVICE_NAME, - "Device Name", - (r) => (r.device && r.arch ? `${r.device} (${r.arch})` : r.device) - ), - ].filter(Boolean) as BenchmarkMetadataItem[]; -} - -/** - * find distinct values from data and create metadata item - */ -function makeMetadataItem( - data: any[], - field: string, - type: BenchmarkMetadataType, - defaultValue: string, - labelName: string, - formatter?: (r: any) => string | undefined -): BenchmarkMetadataItem | null { - const values = ld.uniq( - data.map((r) => (formatter ? formatter(r) : r[field])).filter(Boolean) - ) as string[]; - - if (values.length === 0) { - return null; - } - - return { - type, - options: [defaultValue, ...values], - labelName, - }; -} diff --git a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts deleted file mode 100644 index 5d353f3f37..0000000000 --- a/torchci/lib/benchmark/api_helper/backend/queryBuilderUtils/defaultListMetadataQueryBuilder.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { ExecutableQueryBase, QueryBuilder } from "./queryBuilder"; - -/** - * Query to get the list of metadata for a given benchmark name - */ -export class BenchmarkMetadataQuery extends ExecutableQueryBase { - private builder: QueryBuilder; - private _DEFAULT_QUERY_PARAMS = {}; - constructor() { - super(); - this.builder = new QueryBuilder({ - table: "benchmark.oss_ci_benchmark_metadata", - distinct: true, - select: [ - ["benchmark_name", "benchmark"], - ["model_name", "model"], - ["model_backend", "backend"], - ["metric_name", "metric"], - ["benchmark_dtype", "dtype"], - ["benchmark_mode", "mode"], - "device", - "arch", - ], - prewhere: [ - "timestamp >= toUnixTimestamp({startTime: DateTime64(3)})", - "timestamp < toUnixTimestamp({stopTime: DateTime64(3)})", - ], - where: [ - "repo = {repo: String}", - "benchmark_name = {benchmarkName: String}", - "notEmpty(metric_name)", - "notEmpty(device)", - ], - orderBy: [ - "benchmark", - "backend", - "model", - "metric", - "dtype", - "mode", - "device", - ], - }); - } - - build() { - return this.builder.build(); - } - - toQueryParams(inputs: any) { - this.validateInputs(inputs); - return { - ...this._DEFAULT_QUERY_PARAMS, - ...inputs, - }; - } - - validateInputs(inputs: any) { - if (!inputs.benchmarkName) { - throw new Error("No benchmark names provided"); - } - if (!inputs.repo) { - throw new Error("No repo provided"); - } - } -} diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index b3cfe977d3..063223a83c 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -1,5 +1,7 @@ +import { CompilerQueryType } from "lib/benchmark/api_helper/backend/common/type"; import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; -import { getBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/get_time_series"; +import { getCompilerBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data"; +import { getBenchmarkDataFetcher } from "lib/benchmark/api_helper/backend/dataFetchers/fetchers"; import type { NextApiRequest, NextApiResponse } from "next"; /** @@ -56,3 +58,42 @@ export default async function handler( return res.status(400).json({ error: err.message }); } } + +async function getBenchmarkTimeSeriesData( + request_name: string, + query_params: any, + formats: string[] = ["time_series"] +) { + switch (request_name) { + case "compiler_precompute": + return await getCompilerBenchmarkTimeSeriesData( + query_params, + CompilerQueryType.PRECOMPUTE, + formats + ); + case "compiler": + return await getCompilerBenchmarkTimeSeriesData( + query_params, + CompilerQueryType.GENERAL, + formats + ); + case "pytorch_operator_microbenchmark": + return await getGenernalBenchmarkTimeSeries( + query_params, + formats, + request_name + ); + default: + throw new Error(`Unsupported request_name: ${request_name}`); + } +} + +async function getGenernalBenchmarkTimeSeries( + query_params: any, + formats: string[], + id: string +) { + const fetcher = getBenchmarkDataFetcher(id); + const result = await fetcher.applyQuery(query_params); + return fetcher.applyFormat(result, formats); +} diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts index 733a678057..73dca18cfc 100644 --- a/torchci/pages/api/benchmark/list_metadata.ts +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -1,5 +1,5 @@ import { readApiGetParams } from "lib/benchmark/api_helper/backend/common/utils"; -import { listBenchmarkMetadata } from "lib/benchmark/api_helper/backend/list_metadata_api"; +import { getListBenchmarkMetadataFetcher } from "lib/benchmark/api_helper/backend/dataFetchers/fetchers"; import { NextApiRequest, NextApiResponse } from "next"; export default async function handler( @@ -32,3 +32,10 @@ export default async function handler( return res.status(500).json({ error: err.message }); } } + +async function listBenchmarkMetadata(queryParams: any, id: string) { + // fetch metadata from db + const fetcher = getListBenchmarkMetadataFetcher(id); + const data = await fetcher.applyQuery(queryParams); + return fetcher.postProcess(data); +} From f5893106d2af5654990707a79fa4e103367565ed Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 17:12:32 -0700 Subject: [PATCH 13/19] add config --- torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx b/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx index a80201f8a3..29e4d90548 100644 --- a/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx +++ b/torchci/components/benchmark/llms/LLMsBenchmarkPage.tsx @@ -140,7 +140,6 @@ const MainPage = ({ }, [router.query]); const queryParams = getLLMsBenchmarkPropsQueryParameter(props); - console.log("elainewy query params", queryParams); const { data, error, isLoading } = useBenchmarkPropsData(queryParams); // an error occured while fetching the benchmark props data From f5b5c53be90c2af63c15e3011ad73ceadd0aef18 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 17:18:56 -0700 Subject: [PATCH 14/19] add config --- .../compilers_benchmark_api_query/query.sql | 5 +---- .../backend/compilers/compiler_benchmark_data.ts | 11 +++++++---- torchci/pages/api/benchmark/get_benchmark_data.ts | 0 3 files changed, 8 insertions(+), 8 deletions(-) delete mode 100644 torchci/pages/api/benchmark/get_benchmark_data.ts diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql index b00d304d2f..2dd34c1859 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql @@ -18,10 +18,7 @@ SELECT AS granularity_bucket FROM benchmark.oss_ci_benchmark_torchinductor WHERE - ( - has ({workflows: Array(UInt64)}, workflow_id) - OR empty({workflows: Array(UInt64)}) - ) + workflow_id IN ({workflows: Array(String)}) AND ( has( {branches: Array(String)}, diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index 2a44cd905c..3ca2126c90 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -23,17 +23,19 @@ const COMPILER_BENCHMARK_TABLE_NAME = "compilers_benchmark_api_query"; const COMPILER_BENCHMARK_COMMITS_TABLE_NAME = "compilers_benchmark_api_commit_query"; +// TODO(ELAINEWY): add GET BENCHMARK DATA API /** * backend method to get single compiler benchmark data * must provide workflow and branch in inputParams - * @returns */ export async function getSingleCompilerBenchmarkData( request_name: string, inputParams: any, formats: string[] = ["raw"] ) { - const queryParams = await getCompilerBenchmarkDataQueryParams(inputParams); + const queryParams = await getSingleCompilerBenchmarkDataQueryParams( + inputParams + ); const rows = await fetchCompilerDataFromDb(queryParams); if (rows.length === 0) { return emptyTimeSeriesResponse(); @@ -100,7 +102,8 @@ export async function getCompilerCommits( ); } -async function getCompilerBenchmarkDataQueryParams( +// TODO(ELAINEWY): add GET BENCHMARK DATA API +async function getSingleCompilerBenchmarkDataQueryParams( inputparams: any ): Promise { const queryParams = { @@ -117,7 +120,7 @@ async function getCompilerBenchmarkDataQueryParams( } queryParams["workflows"] = [queryParams.workflow]; queryParams["branches"] = [queryParams.branch]; - console.log("workflows provided in request", queryParams.workflows); + console.log("(getSingleCompilerBenchmarkDataQueryParams) workflows provided in request", queryParams.workflows); return queryParams; } diff --git a/torchci/pages/api/benchmark/get_benchmark_data.ts b/torchci/pages/api/benchmark/get_benchmark_data.ts deleted file mode 100644 index e69de29bb2..0000000000 From 12079fc4b946ecd5251adfcbd3fa7bc4216e6113 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 17:27:41 -0700 Subject: [PATCH 15/19] add config --- .../api_helper/backend/compilers/compiler_benchmark_data.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts index 3ca2126c90..420b403750 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/compiler_benchmark_data.ts @@ -120,7 +120,10 @@ async function getSingleCompilerBenchmarkDataQueryParams( } queryParams["workflows"] = [queryParams.workflow]; queryParams["branches"] = [queryParams.branch]; - console.log("(getSingleCompilerBenchmarkDataQueryParams) workflows provided in request", queryParams.workflows); + console.log( + "(getSingleCompilerBenchmarkDataQueryParams) workflows provided in request", + queryParams.workflows + ); return queryParams; } From 07b216bd2026354c06d4c73165b091b1b5abd71a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 19:47:56 -0700 Subject: [PATCH 16/19] add config --- .../benchmarkDataQueryBuilder.ts | 5 ----- .../listMetadataQueryBuilder.ts | 16 +++++++++++++++- torchci/pages/api/benchmark/list_metadata.ts | 9 ++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts index 9200b48fa9..14f9b7db84 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts @@ -414,11 +414,6 @@ export class PytorchOperatorMicroBenchmarkDataFetcher } build() { - // debugging - console.log( - "build PytorchOperatorMicroBenchmarkDataQuery", - this._data_query.build() - ); return this._data_query.build(); } } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts index 02f11fb1c1..28dd5da362 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts @@ -62,6 +62,10 @@ export class BenchmarkMetadataQuery return this.builder.build(); } + addWhere(where: string[]) { + this.builder.addWhere(where); + } + toQueryParams(inputs: any) { this.validateInputs(inputs); return { @@ -117,6 +121,13 @@ export function getDefaultBenchmarkMetadataGroup( "Device Name", (r) => (r.device && r.arch ? `${r.device} (${r.arch})` : r.device) ), + makeMetadataItem( + data, + "model", + BenchmarkMetadataType.ModelName, + "All models", + "Model Name" + ), ].filter(Boolean) as BenchmarkMetadataItem[]; } @@ -169,13 +180,16 @@ export class PytorchOperatorMicrobenchmarkMetadataFetcher if (item) { li.push(item); } + return li; } build() { + // console.log(this._data_query.build()); return this._data_query.build(); } toQueryParams(inputs: any) { - return this._data_query.toQueryParams(inputs); + const params = { ...inputs, operatorName: inputs.operatorName ?? "" }; + return this._data_query.toQueryParams(params); } } diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts index 73dca18cfc..73daed485d 100644 --- a/torchci/pages/api/benchmark/list_metadata.ts +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -23,11 +23,9 @@ export default async function handler( return res.status(400).json({ error: "Missing required parameters" }); } - console.log("[API] LIST_METRICS recieved params: ", params.query_params); - try { - const groups = listBenchmarkMetadata(params.query_params, params.id); - return res.status(200).json(groups); + const groups = await listBenchmarkMetadata(params.query_params, params.id); + return res.status(200).json({ data: groups }); } catch (err: any) { return res.status(500).json({ error: err.message }); } @@ -37,5 +35,6 @@ async function listBenchmarkMetadata(queryParams: any, id: string) { // fetch metadata from db const fetcher = getListBenchmarkMetadataFetcher(id); const data = await fetcher.applyQuery(queryParams); - return fetcher.postProcess(data); + const result = fetcher.postProcess(data); + return result; } From 5d10332d75fc5d76a3c0b7e6a156b739e18a54bb Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Tue, 14 Oct 2025 19:50:36 -0700 Subject: [PATCH 17/19] add config --- .../api_helper/backend/compilers/helpers/precompute.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts index 93a964dfef..746fe31b38 100644 --- a/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts +++ b/torchci/lib/benchmark/api_helper/backend/compilers/helpers/precompute.ts @@ -96,8 +96,8 @@ function postFetchProcess( commit_map: Map, metadata: any ) { - const start_ts = new Date(data[0].granularity_bucket).getTime(); - const end_ts = new Date(data[data.length - 1].granularity_bucket).getTime(); + const end_ts = new Date(data[0].granularity_bucket).getTime(); + const start_ts = new Date(data[data.length - 1].granularity_bucket).getTime(); data.map((row) => { row["commit"] = commit_map.get(row.workflow_id)?.commit; row["branch"] = commit_map.get(row.workflow_id)?.branch; From aec68f185541e43ffee82d589ea4f380b535bb2d Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 15 Oct 2025 09:37:03 -0700 Subject: [PATCH 18/19] add config --- .../clickhouse_queries/compilers_benchmark_api_query/query.sql | 2 +- torchci/lib/benchmark/api_helper/backend/list_commits.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql index 2dd34c1859..16707bc7b0 100644 --- a/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql +++ b/torchci/clickhouse_queries/compilers_benchmark_api_query/query.sql @@ -18,7 +18,7 @@ SELECT AS granularity_bucket FROM benchmark.oss_ci_benchmark_torchinductor WHERE - workflow_id IN ({workflows: Array(String)}) + workflow_id IN ({workflows: Array(UInt64)}) AND ( has( {branches: Array(String)}, diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index 60c3559c4b..efd60aef9c 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -67,7 +67,6 @@ async function getBenmarkCommits( } function getFormat(data: any, format: string = "raw") { - console.log("[API]list commits, format data elaine: ", data.length); switch (format) { case "branch": const branchgroup = groupByBenchmarkData(data, ["branch"], []); From 7d539bdfcb47cb64024aff205f545966d73c5d3a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 15 Oct 2025 19:23:52 -0700 Subject: [PATCH 19/19] add config --- .../list_commit_query/params.json | 12 +- .../benchmark_v3/list_commit_query/query.sql | 36 ++-- .../benchmarkSideBar/BenchmarkTopBar.tsx | 6 +- .../DefaultSideBarMetricsDropdowns.tsx | 3 - .../components/SideBarMainSection.tsx | 28 +++- .../BranchDropdown.tsx | 2 +- .../CommitWorkfowSelectSection.tsx | 16 +- .../filters/BenchmarkFilterDropdownGroup.tsx | 65 ++++++++ .../DefaultSideBarMetricsDropdowns.tsx | 72 ++++++++ .../{ => sampling}/SamplingInput.tsx | 0 .../{ => sampling}/SamplingSetting.tsx | 0 .../components/useUrlSync.tsx | 5 +- .../dataRender/auto/AutoComponents.tsx | 71 ++++++++ .../auto/defaultAutoRenderContent.tsx | 37 +++++ .../components/benchmarkTimeSeries/helper.tsx | 1 + .../fanout/defaultFanoutRenderContent.tsx | 13 +- .../benchmark/v3/configs/configBook.tsx | 129 -------------- .../v3/configs/helpers/RegressionPolicy.ts | 5 + .../v3/configs/teams/compilers/config.ts | 4 +- .../teams/default/default_dashboard_config.ts | 47 ++++++ .../v3/configs/utils/autoRegistration.tsx | 70 ++++++++ .../configs/utils/dataBindingRegistration.tsx | 21 ++- .../v3/pages/BenchmarkDashboardPage.tsx | 44 +++++ .../v3/pages/BenchmarkRegressionPage.tsx | 5 +- .../uiModules/UMDenseComponents.tsx | 27 ++- .../api_helper/backend/common/type.ts | 14 +- .../benchmarkDataQueryBuilder.ts | 31 +++- .../listMetadataQueryBuilder.ts | 71 ++++++-- .../api_helper/backend/list_commits.ts | 29 +++- torchci/lib/benchmark/api_helper/fe/api.ts | 29 ++++ torchci/lib/benchmark/api_helper/fe/hooks.ts | 114 ++++++++++++- torchci/lib/benchmark/llms/utils/llmUtils.ts | 1 + .../benchmark/store/benchmark_config_book.tsx | 157 ++++++++++++++++++ .../store/benchmark_regression_store.ts | 47 ++++++ .../pages/api/benchmark/get_time_series.ts | 2 - torchci/pages/api/benchmark/list_metadata.ts | 5 +- .../pages/benchmark/compilers_regression.tsx | 4 +- torchci/pages/benchmark/v3/dashboard/[id].tsx | 14 ++ torchci/tsconfig.json | 2 +- 39 files changed, 1013 insertions(+), 226 deletions(-) delete mode 100644 torchci/components/benchmark/v3/components/benchmarkSideBar/components/DefaultSideBarMetricsDropdowns.tsx rename torchci/components/benchmark/v3/components/benchmarkSideBar/components/{ => commitAndWorkflow}/BranchDropdown.tsx (96%) rename torchci/components/benchmark/v3/components/benchmarkSideBar/components/{ => commitAndWorkflow}/CommitWorkfowSelectSection.tsx (92%) create mode 100644 torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/BenchmarkFilterDropdownGroup.tsx create mode 100644 torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/DefaultSideBarMetricsDropdowns.tsx rename torchci/components/benchmark/v3/components/benchmarkSideBar/components/{ => sampling}/SamplingInput.tsx (100%) rename torchci/components/benchmark/v3/components/benchmarkSideBar/components/{ => sampling}/SamplingSetting.tsx (100%) create mode 100644 torchci/components/benchmark/v3/components/dataRender/auto/AutoComponents.tsx create mode 100644 torchci/components/benchmark/v3/components/dataRender/auto/defaultAutoRenderContent.tsx create mode 100644 torchci/components/benchmark/v3/configs/teams/default/default_dashboard_config.ts create mode 100644 torchci/components/benchmark/v3/configs/utils/autoRegistration.tsx create mode 100644 torchci/components/benchmark/v3/pages/BenchmarkDashboardPage.tsx create mode 100644 torchci/lib/benchmark/store/benchmark_config_book.tsx create mode 100644 torchci/pages/benchmark/v3/dashboard/[id].tsx diff --git a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json index 047514f175..35bf6c1481 100644 --- a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json +++ b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/params.json @@ -3,12 +3,14 @@ "repo": "String", "benchmarkNames": "Array", "branches": "Array", - "device": "String", - "arch": "Array(String)", - "dtype": "String", - "mode": "String", + "models": "Array(String)", "startTime": "DateTime64(3)", - "stopTime": "DateTime64(3)" + "stopTime": "DateTime64(3)", + "backends": "Array(String)", + "arch": "String", + "dtype": "String", + "device": "String", + "mode": "String" }, "tests": [] } diff --git a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql index f087f1f1bb..3c49e9c7fc 100644 --- a/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql +++ b/torchci/clickhouse_queries/benchmark_v3/list_commit_query/query.sql @@ -1,32 +1,40 @@ SELECT - replaceOne(head_branch, 'refs/heads/', '') AS branch, + replaceAll(head_branch, 'refs/heads/', '') AS branch, head_sha AS commit, workflow_id, toStartOfHour(min(fromUnixTimestamp(timestamp))) AS date -FROM - benchmark.oss_ci_benchmark_metadata +FROM benchmark.oss_ci_benchmark_metadata PREWHERE timestamp >= toUnixTimestamp({startTime: DateTime64(3)}) - AND timestamp < toUnixTimestamp({stopTime: DateTime64(3)} + AND timestamp < toUnixTimestamp({stopTime: DateTime64(3)}) -- ← closed ) WHERE - repo = {repo: String } + repo = {repo: String} AND ( - has( - {branches: Array(String)}, - replaceOne(head_branch, 'refs/heads/', '') - ) + has({branches: Array(String)}, replaceAll(head_branch, 'refs/heads/', '')) OR empty({branches: Array(String)}) ) + AND (benchmark_dtype = {dtype: String} OR empty({dtype: String})) AND ( - has({benchmarkNames: Array(String) }, benchmark_name) - OR empty({benchmarkNames: Array(String) }) + has({benchmarkNames: Array(String)}, benchmark_name) + OR empty({benchmarkNames: Array(String)}) ) - AND (benchmark_dtype = {dtype: String} OR empty({dtype: String})) AND notEmpty(metric_name) AND notEmpty(device) AND ( - arch LIKE concat('%', {arch: String }, '%') - OR {arch: String } = '' + arch LIKE concat('%', {arch: String}, '%') + OR {arch: String} = '' + ) + AND ( + has({models: Array(String) }, model_name) + OR empty({models: Array(String) }) + ) + AND ( + has({backends: Array(String) }, model_backend) + OR empty({backends: Array(String) }) + ) + AND ( + startsWith({device: String }, device) + OR {device: String } = '' ) GROUP BY branch, commit, workflow_id diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/BenchmarkTopBar.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/BenchmarkTopBar.tsx index 30071dfa03..aa91885db8 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/BenchmarkTopBar.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/BenchmarkTopBar.tsx @@ -1,15 +1,15 @@ import { Divider, Paper, Typography } from "@mui/material"; import { Stack } from "@mui/system"; -import { BenchmarkUI } from "../../configs/configBook"; import { BenchmarkReportFeatureNotification } from "../dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureNotification"; import { BenchmarkReportFeatureSidePanel } from "../dataRender/components/benchmarkRegressionReport/BenchmarkReportFeatureSidePanel"; -import { CommitWorflowSelectSection } from "./components/CommitWorkfowSelectSection"; +import { CommitWorflowSelectSection } from "./components/commitAndWorkflow/CommitWorkfowSelectSection"; +import { BenchmarkUIConfigHandler } from "lib/benchmark/store/benchmark_config_book"; export function BenchmarkTopBar({ config, title = "", }: { - config: BenchmarkUI; + config: BenchmarkUIConfigHandler; title?: string; }) { const reportFeature = diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/DefaultSideBarMetricsDropdowns.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/DefaultSideBarMetricsDropdowns.tsx deleted file mode 100644 index 5d69136924..0000000000 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/DefaultSideBarMetricsDropdowns.tsx +++ /dev/null @@ -1,3 +0,0 @@ -export default function DefaultMetricsDropdowns() { - return
Not supported yet: comming soon
; -} diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx index 1ec29f5a07..41b562ec6c 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SideBarMainSection.tsx @@ -13,11 +13,11 @@ import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { useRouter } from "next/router"; import { useEffect, useRef, useState } from "react"; -import { BenchmarkUIConfigBook } from "../../../configs/configBook"; import { DenseAlert } from "../../common/styledComponents"; -import { BranchDropdowns } from "./BranchDropdown"; -import { SamplingSetting } from "./SamplingSetting"; +import { BranchDropdowns } from "./commitAndWorkflow/BranchDropdown"; +import { SamplingSetting } from "./sampling/SamplingSetting"; import { useUrlStoreSync } from "./useUrlSync"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; const styles = { root: { @@ -68,13 +68,19 @@ export function SideBarMainSection() { // 1) Read benchmarkId (low-churn) to fetch config const benchmarkId = useDashboardSelector((s) => s.benchmarkId); - const config = BenchmarkUIConfigBook.instance.get(benchmarkId); - const dataBinding = - BenchmarkUIConfigBook.instance.getDataBinding(benchmarkId); - const required_filter_fields = config?.required_filter_fields ?? []; + + + const getConfig = useBenchmarkBook((s) => s.getConfig); + const config = getConfig(benchmarkId); + const dataBinding = config.dataBinding; + + + const required_filter_fields = config.raw?.required_filter_fields ?? []; // 2) One selector (with shallow inside useDashboardSelector) for the rest const { + repo, + benchmarkName, stagedTime, stagedFilters, stagedLbranch, @@ -118,6 +124,9 @@ export function SideBarMainSection() { lcommit: s.lcommit, rcommit: s.rcommit, + repo: s.repo, + benchmarkName: s.benchmarkName, + commitMainOptions: s.commitMainOptions, revertMainOptions: s.revertMainOptions, })); @@ -147,9 +156,10 @@ export function SideBarMainSection() { (enableSamplingSetting ? !!stagedMaxSampling : true) && required_filter_fields.every((k) => !!committedFilters[k]); - const params = BenchmarkUIConfigBook.instance - .getDataBinding(benchmarkId) + const params = dataBinding ?.toQueryParams({ + repo: repo, + benchmarkName: benchmarkName, timeRange: stagedTime, filters: stagedFilters, maxSampling: enableSamplingSetting ? stagedMaxSampling : undefined, diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/BranchDropdown.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/BranchDropdown.tsx similarity index 96% rename from torchci/components/benchmark/v3/components/benchmarkSideBar/components/BranchDropdown.tsx rename to torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/BranchDropdown.tsx index 1110c8666d..c8da8fe1c8 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/BranchDropdown.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/BranchDropdown.tsx @@ -1,6 +1,6 @@ import { Box } from "@mui/system"; import { UMDenseDropdown } from "components/uiModules/UMDenseComponents"; -import { DenseAlert } from "../../common/styledComponents"; +import { DenseAlert } from "../../../common/styledComponents"; type BranchDropdownsProps = { type: string; diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/CommitWorkfowSelectSection.tsx similarity index 92% rename from torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx rename to torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/CommitWorkfowSelectSection.tsx index c4a965a9c7..5e8d363e66 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/CommitWorkfowSelectSection.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/commitAndWorkflow/CommitWorkfowSelectSection.tsx @@ -6,7 +6,7 @@ import { useBenchmarkCommitsData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store"; import { useEffect, useState } from "react"; -import { BenchmarkUIConfigBook } from "../../../configs/configBook"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; /** * @@ -15,6 +15,8 @@ import { BenchmarkUIConfigBook } from "../../../configs/configBook"; */ export function CommitWorflowSelectSection() { const { + repo, + benchmarkName, benchmarkId, committedTime, committedFilters, @@ -32,6 +34,8 @@ export function CommitWorflowSelectSection() { committedFilters: s.committedFilters, committedMaxSampling: s.committedMaxSampling, enableSamplingSetting: s.enableSamplingSetting, + repo: s.repo, + benchmarkName: s.benchmarkName, lcommit: s.lcommit, rcommit: s.rcommit, committedLBranch: s.committedLbranch, @@ -43,10 +47,10 @@ export function CommitWorflowSelectSection() { const [leftList, setLeftList] = useState([]); const [rightList, setRightList] = useState([]); - const config = BenchmarkUIConfigBook.instance.get(benchmarkId); - const dataBinding = - BenchmarkUIConfigBook.instance.getDataBinding(benchmarkId); - const required_filter_fields = config?.required_filter_fields ?? []; + const getConfig = useBenchmarkBook((s) => s.getConfig); + const config = getConfig(benchmarkId); + const dataBinding = config.dataBinding + const required_filter_fields = config.raw?.required_filter_fields ?? []; const ready = !!committedTime?.start && @@ -67,6 +71,8 @@ export function CommitWorflowSelectSection() { // Convert to query params const params = dataBinding.toQueryParams({ + repo: repo, + benchmarkName: benchmarkName, branches, timeRange: committedTime, filters: committedFilters, diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/BenchmarkFilterDropdownGroup.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/BenchmarkFilterDropdownGroup.tsx new file mode 100644 index 0000000000..706f192a6b --- /dev/null +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/BenchmarkFilterDropdownGroup.tsx @@ -0,0 +1,65 @@ +import { Stack } from "@mui/system"; +import { DTypePicker } from "components/benchmark/ModeAndDTypePicker"; +import { UMDenseDropdown, UMDenseDropdownOption } from "components/uiModules/UMDenseComponents"; +import { DropdownGroupItem } from "lib/benchmark/llms/types/dashboardPickerTypes"; + +/** + * The enum type of benchmark dashboard dropgroup item + * this is used to render dropdowns dynamically in the LLMs Benchmark page. + * the field value must match the fields in LLMsBenchmarkProps + */ +export enum BenchmarkDropdownGroupItemType { + ModelName = "model", + BackendName = "backend", + ModeName = "modeN", + DtypeName = "dtype", + DeviceName = "deviceName", + ArchName = "arch", + Qps = "qps", +} + +/** + * The input item for benchmark dashboard dropdown + * @property DropdownGroupItemType enum type + * @property options the list of options in the dropdown + * @property labelName the label name of the dropdown + */ +export interface BenchmarkDropdownGroupItem { + type: BenchmarkDropdownGroupItemType; + options: (string | UMDenseDropdownOption)[] + labelName: string; +} + +export default function BenchmarkDropdownGroup({ + onChange, + props, + optionListMap, +}: { + onChange: (_key: string, _value: any) => void; + props: any; + optionListMap: BenchmarkDropdownGroupItem[]; +}) { + return ( + + {optionListMap.length > 1 && + optionListMap.map((option, index) => { + const type = option.type; + const olist = option.options; + if (!olist || olist.length <= 1) { + return null; + } + return ( + { + onChange(type, val); + }} + dtypes={olist} + label={option.labelName} + /> + ); + })} + + ); +} diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/DefaultSideBarMetricsDropdowns.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/DefaultSideBarMetricsDropdowns.tsx new file mode 100644 index 0000000000..a9861ee893 --- /dev/null +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/filters/DefaultSideBarMetricsDropdowns.tsx @@ -0,0 +1,72 @@ +import LoadingPage from "components/common/LoadingPage"; +import { useBenchmarkConfigBook, useListBenchmarkMetadata } from "lib/benchmark/api_helper/fe/hooks"; +import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; +import { RenderRawContent } from "../../../common/RawContentDialog"; +import { Box } from "@mui/system"; +import BenchmarkDropdownGroup from "./BenchmarkFilterDropdownGroup"; +import { Alert } from "@mui/material"; + +export default function DefaultMetricsDropdowns() { + const { + setStagedFilter, + repo, + benchmarkId, + benchmarkName, + stagedTime, + stagedFilters + } = useDashboardSelector((s) => ({ + setStagedFilter: s.setStagedFilter, + repo: s.repo, + benchmarkName: s.benchmarkName, + benchmarkId: s.benchmarkId, + stagedTime: s.stagedTime, + stagedFilters: s.stagedFilters, + })); + + const configHandler = useBenchmarkConfigBook(benchmarkId) + if (!configHandler) { + return + } + + const ready =!!stagedTime?.start && !!stagedTime?.end + // convert to the query params + const params = configHandler.dataBinding.toQueryParams({ + repo: repo, + benchmarkName: benchmarkName, + timeRange: stagedTime, + filters: {}, + }); + + const queryParams: any | null = ready ? params : null; + // fetch the bechmark data + + const { + data: resp, + isLoading: isLoading, + error: error, + } = useListBenchmarkMetadata(benchmarkId, queryParams); + + if (isLoading) { + return ; + } + + if (error) { + return DefaultMetricsDropdowns {error.message} + } + + const metadataList = resp?.data || []; + + return + + { + if (_key == "deviceName"){ + const v = _value.split("||"); + if (v.length === 2) { + setStagedFilter("device", v[0]); + setStagedFilter("arch", v[1]); + } + } + setStagedFilter(_key, _value); + }} props={stagedFilters} /> + +} diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SamplingInput.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/sampling/SamplingInput.tsx similarity index 100% rename from torchci/components/benchmark/v3/components/benchmarkSideBar/components/SamplingInput.tsx rename to torchci/components/benchmark/v3/components/benchmarkSideBar/components/sampling/SamplingInput.tsx diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/SamplingSetting.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/sampling/SamplingSetting.tsx similarity index 100% rename from torchci/components/benchmark/v3/components/benchmarkSideBar/components/SamplingSetting.tsx rename to torchci/components/benchmark/v3/components/benchmarkSideBar/components/sampling/SamplingSetting.tsx diff --git a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/useUrlSync.tsx b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/useUrlSync.tsx index 6362081e0f..09ae7e6082 100644 --- a/torchci/components/benchmark/v3/components/benchmarkSideBar/components/useUrlSync.tsx +++ b/torchci/components/benchmark/v3/components/benchmarkSideBar/components/useUrlSync.tsx @@ -57,10 +57,13 @@ export function useUrlStoreSync>( // prevent no-op replace and re-pushing the same thing if (nextSig === currSig || nextSig === lastPushedSigRef.current) return; + const [pathname] = router.asPath.split("?"); + + // briefly mark as syncing to avoid URL->store echo isApplyingUrlRef.current = true; router - .replace({ pathname: router.pathname, query: nextQueryObj }, undefined, { + .replace({ pathname: pathname, query: nextQueryObj }, undefined, { shallow: true, }) .finally(() => { diff --git a/torchci/components/benchmark/v3/components/dataRender/auto/AutoComponents.tsx b/torchci/components/benchmark/v3/components/dataRender/auto/AutoComponents.tsx new file mode 100644 index 0000000000..ef02cf6e72 --- /dev/null +++ b/torchci/components/benchmark/v3/components/dataRender/auto/AutoComponents.tsx @@ -0,0 +1,71 @@ +import { useBenchmarkCommittedContext, useBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/fe/hooks"; +import LoadingPage from "components/common/LoadingPage"; +import { Alert } from "@mui/material"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; +import { RenderRawContent } from "../../common/RawContentDialog"; + +export function AutoBenchmarkPairwiseComparisonTable() { + const ctx = useBenchmarkCommittedContext(); + + if(!ctx){ + return ; + } + + const branches = [ + ...new Set( + [ctx.committedLbranch, ctx.committedRbranch].filter((b) => b.length > 0) + ), + ]; + + const ready = + !!ctx.committedTime?.start && + !!ctx.committedTime?.end && + !!ctx.committedLbranch && + !!ctx.committedRbranch && + ctx.requiredFilters.every((k: string) => !!ctx.committedFilters[k]); + + const getConfig = useBenchmarkBook((s) => s.getConfig); + const config = getConfig(ctx.benchmarkId); + const dataBinding = config.dataBinding; + + // convert to the query params + const params = dataBinding.toQueryParams({ + repo: ctx.repo, + branches:branches, + benchmarkName: ctx.benchmarkName, + timeRange: ctx.committedTime, + filters: ctx.committedFilters, + maxSampling: ctx.committedMaxSampling, + }); + + + const queryParams: any | null = ready ? params : null; + // fetch the bechmark data + const { + data: resp, + isLoading, + error, + } = useBenchmarkTimeSeriesData(ctx.benchmarkId, queryParams, ["table"]); + + if (isLoading) { + return ; + } + + if (error) { + return (AutoBenchmarkPairwiseComparisonTable){error.message}; + } + if (!ctx.dataRender?.renders) { + return
no data render
; + } + if (!resp?.data?.data) { + return
no data
; + } + + const data = resp?.data?.data + return ( +
+ + +
+ ); +} diff --git a/torchci/components/benchmark/v3/components/dataRender/auto/defaultAutoRenderContent.tsx b/torchci/components/benchmark/v3/components/dataRender/auto/defaultAutoRenderContent.tsx new file mode 100644 index 0000000000..826e7cd919 --- /dev/null +++ b/torchci/components/benchmark/v3/components/dataRender/auto/defaultAutoRenderContent.tsx @@ -0,0 +1,37 @@ + +import { Box, Stack } from "@mui/system"; +import { HighlightStyles } from "components/benchmark/v3/components/common/highlight"; +import { getAutoRenderComponent } from "components/benchmark/v3/configs/utils/autoRegistration"; +import { Divider, Typography } from "@mui/material"; +import LoadingPage from "components/common/LoadingPage"; +import { useBenchmarkCommittedContext } from "lib/benchmark/api_helper/fe/hooks"; + +/** + * The default fanout component fetches pre-processed data for chart, + * table and other components in one api + * @returns + */ +export function DefaultAutoRenderContent() { + const ctx = useBenchmarkCommittedContext(); + if (!ctx) return ; + const autoUIConfigs = ctx.dataRender.renders; + + return ( + <> + + + + {ctx.config.raw.title} + + + {autoUIConfigs?.map((autoUIConfig, index) => { + const { Component} = getAutoRenderComponent(autoUIConfig); + return ( + + + + ); + })} + + ) +} diff --git a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx index 9d5f3fb6e4..65931ee894 100644 --- a/torchci/components/benchmark/v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/components/benchmarkTimeSeries/helper.tsx @@ -71,6 +71,7 @@ export type ComparisonTableConfig = { customizedConfirmDialog?: { type: string; id?: string; + scope?: string; }; }; diff --git a/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx b/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx index b9d49a1a2f..d44afb9644 100644 --- a/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx +++ b/torchci/components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent.tsx @@ -1,14 +1,14 @@ import { Alert, Divider, Typography } from "@mui/material"; import { Box, Stack } from "@mui/system"; import { HighlightStyles } from "components/benchmark/v3/components/common/highlight"; -import { getConfig } from "components/benchmark/v3/configs/configBook"; import { getFanoutRenderComponent } from "components/benchmark/v3/configs/utils/fanoutRegistration"; import LoadingPage from "components/common/LoadingPage"; -import { useBenchmarkData } from "lib/benchmark/api_helper/fe/hooks"; +import { useBenchmarkTimeSeriesData } from "lib/benchmark/api_helper/fe/hooks"; import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store"; import { useState } from "react"; import { ToggleSection, toToggleSectionId } from "../../common/ToggleSection"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; /** * The default fanout component fetches pre-processed data for chart, @@ -17,6 +17,8 @@ import { ToggleSection, toToggleSectionId } from "../../common/ToggleSection"; */ export function DefaultFanoutRenderContent() { const { + repo, + benchmarkName, benchmarkId, committedTime, committedFilters, @@ -28,6 +30,8 @@ export function DefaultFanoutRenderContent() { setLcommit, setRcommit, } = useDashboardSelector((s) => ({ + repo: s.repo, + benchmarkName: s.benchmarkName, benchmarkId: s.benchmarkId, committedTime: s.committedTime, committedFilters: s.committedFilters, @@ -40,6 +44,7 @@ export function DefaultFanoutRenderContent() { setRcommit: s.setRcommit, })); const [payload, setPayload] = useState(null); + const getConfig = useBenchmarkBook((s) => s.getConfig); const config = getConfig(benchmarkId); const requiredFilters = config.dataBinding.raw.required_filter_fields; const dataRender = config.raw.dataRender; @@ -84,6 +89,8 @@ export function DefaultFanoutRenderContent() { // convert to the query params const params = config.dataBinding.toQueryParams({ + repo, + benchmarkName: benchmarkName, timeRange: committedTime, branches, filters: committedFilters, @@ -96,7 +103,7 @@ export function DefaultFanoutRenderContent() { data: resp, isLoading, error, - } = useBenchmarkData(benchmarkId, queryParams); + } = useBenchmarkTimeSeriesData(benchmarkId, queryParams); if (isLoading) { return ; } diff --git a/torchci/components/benchmark/v3/configs/configBook.tsx b/torchci/components/benchmark/v3/configs/configBook.tsx index 7dc2e29fca..76405ab367 100644 --- a/torchci/components/benchmark/v3/configs/configBook.tsx +++ b/torchci/components/benchmark/v3/configs/configBook.tsx @@ -1,132 +1,3 @@ -import { DefaultFanoutRenderContent } from "../components/dataRender/fanout/defaultFanoutRenderContent"; -import { NotFoundComponent, resolveComponent } from "./configRegistration"; -import { - CompilerPrecomputeBenchmarkUIConfig, - COMPILTER_PRECOMPUTE_BENCHMARK_ID, -} from "./teams/compilers/config"; -import { - DataBinding, - DataBindingConfig, -} from "./utils/dataBindingRegistration"; - -export type UIRenderConfig = { - title?: string; // title of the component to render - id?: string; // id of the component to render - type: string; // type of the component to render - config: any; // config of the component to render -}; - -export type DataRenderOption = { - type: string; - api?: any; - id?: string; // id of the component to render, this is used when type is 'component' - sideRender?: { [key: string]: UIRenderConfig }; // this used to render side content, such as regression report access - renders?: UIRenderConfig[]; // this is used when type is predefined type such as 'default-fanout' -}; - -export type BenchmarkUIConfig = { - benchmarkId: string; - apiId: string; - title: string; - dataBinding: DataBindingConfig; // data binding config - dataRender?: DataRenderOption; // either binds a component or a converter function to render data - required_filter_fields?: readonly string[]; // required filter fields -}; - -export class BenchmarkUI { - private _benchmarkId: string; - private _config: BenchmarkUIConfig; - private _dataBinding: DataBinding; - - constructor(config: BenchmarkUIConfig) { - this._benchmarkId = config.benchmarkId; - this._config = config; - this._dataBinding = new DataBinding(config.dataBinding); - } - - get benchmarkId(): string { - return this._benchmarkId; - } - - get raw(): BenchmarkUIConfig { - return this._config; - } - - get dataBinding(): DataBinding { - return this._dataBinding; - } - - getDataRenderComponent = (): React.ComponentType => { - const dr = this._config.dataRender; - if (!dr || dr.type !== "component") return DefaultFanoutRenderContent; - - const Comp = resolveComponent(dr.id); - if (Comp) return Comp; - - // inline fallback component to satisfy the return type - const Missing: React.FC = () => ( - - ); - return Missing; - }; -} - -export class BenchmarkUIConfigBook { - private static _instance: BenchmarkUIConfigBook | null = null; - private readonly configs: Record; - - private constructor() { - this.configs = { - [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: CompilerPrecomputeBenchmarkUIConfig, - // add more configs here ... - }; - } - - /** Get the global singleton instance */ - static get instance(): BenchmarkUIConfigBook { - if (!this._instance) { - this._instance = new BenchmarkUIConfigBook(); - } - return this._instance; - } - - get(id: string): BenchmarkUIConfig | undefined { - const config = this.configs[id]; - return config; - } - - getConfigInstance(id: string): BenchmarkUI { - const config = this.get(id); - if (!config) { - throw new Error(`No config found for id ${id}`); - } - return new BenchmarkUI(config); - } - - listIds(): string[] { - return Object.keys(this.configs); - } - - listAll(): BenchmarkUIConfig[] { - return Object.values(this.configs); - } - - getDataBinding(id: string): DataBinding { - const config = this.get(id); - if (!config) { - throw new Error(`No config found for id ${id}, cannot get data binding`); - } - return new DataBinding(config.dataBinding); - } -} - -export function getBenchmarkBook(): BenchmarkUIConfigBook { - return BenchmarkUIConfigBook.instance; -} - -export function getConfig(id: string): BenchmarkUI { - return BenchmarkUIConfigBook.instance.getConfigInstance(id); -} const REPORT_ID_TO_BENCHMARK_ID_MAPPING: Record = { compiler_regression: "compiler_inductor", diff --git a/torchci/components/benchmark/v3/configs/helpers/RegressionPolicy.ts b/torchci/components/benchmark/v3/configs/helpers/RegressionPolicy.ts index 1f20be0097..b4651ba5a6 100644 --- a/torchci/components/benchmark/v3/configs/helpers/RegressionPolicy.ts +++ b/torchci/components/benchmark/v3/configs/helpers/RegressionPolicy.ts @@ -90,6 +90,11 @@ export type ComparisonResult = { // Evaluator // ------------------------------------------------------------------ +/** + * function to evaluate a comparison result based on benchmark comparison policy + * if policy is not provided, it will use the default policy instead, + * see getDefaultComparisonPolicy + */ export function evaluateComparison( target: string | undefined | null, oldValue: number | null | undefined, diff --git a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts index 5be3d353bb..6f015443a2 100644 --- a/torchci/components/benchmark/v3/configs/teams/compilers/config.ts +++ b/torchci/components/benchmark/v3/configs/teams/compilers/config.ts @@ -9,12 +9,12 @@ import dayjs from "dayjs"; import utc from "dayjs/plugin/utc"; import { REQUIRED_COMPLIER_LIST_COMMITS_KEYS } from "lib/benchmark/api_helper/backend/compilers/helpers/type"; import { DISPLAY_NAMES_TO_COMPILER_NAMES } from "../../../../compilers/common"; -import { BenchmarkUIConfig } from "../../configBook"; import { BenchmarkComparisonPolicyConfig } from "../../helpers/RegressionPolicy"; import { QueryParameterConverter, QueryParameterConverterInputs, } from "../../utils/dataBindingRegistration"; +import { BenchmarkUIConfig } from "lib/benchmark/store/benchmark_config_book"; dayjs.extend(utc); const PASSRATE_COMPARISON_POLICY: BenchmarkComparisonPolicyConfig = { @@ -169,6 +169,7 @@ export const COMPILTER_PRECOMPUTE_BENCHMARK_INITIAL = { export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = { benchmarkId: COMPILTER_PRECOMPUTE_BENCHMARK_ID, apiId: COMPILTER_PRECOMPUTE_BENCHMARK_ID, + type: "aggregate", title: "Compiler Inductor Regression Tracking", dataBinding: { initial: COMPILTER_PRECOMPUTE_BENCHMARK_INITIAL, @@ -257,6 +258,7 @@ export const CompilerPrecomputeBenchmarkUIConfig: BenchmarkUIConfig = { customizedConfirmDialog: { type: "component", id: "CompilerPrecomputeConfirmDialogContent", + scope: "comparison_value" }, targetField: "metric", comparisonPolicy: { diff --git a/torchci/components/benchmark/v3/configs/teams/default/default_dashboard_config.ts b/torchci/components/benchmark/v3/configs/teams/default/default_dashboard_config.ts new file mode 100644 index 0000000000..1fb63c717e --- /dev/null +++ b/torchci/components/benchmark/v3/configs/teams/default/default_dashboard_config.ts @@ -0,0 +1,47 @@ +import { RenderStaticContent } from "components/benchmark/v3/components/common/RawContentDialog"; +import dayjs from "dayjs"; + +export const DEFAULT_DASHBOARD_ID = "default-dashboard"; + +export const REQUIRED_COMPLIER_LIST_COMMITS_KEYS = ["mode","dtype","deviceName",] as const; + +// The initial config for the compiler benchmark regression page +export const DEFAULT_DASHBOARD_BENCHMARK_INITIAL= { + benchmarkId: DEFAULT_DASHBOARD_ID, + time: { + start: dayjs.utc().startOf("day").subtract(7, "day"), + end: dayjs.utc().endOf("day"), + }, + filters: { + // repo: "pytorch/pytorch", + // benchmarkName: "compiler" + }, + lbranch: "main", + rbranch: "main", +}; + +export const defaultDashboardBenchmarkUIConfig = { + benchmarkId: DEFAULT_DASHBOARD_ID, + apiId: DEFAULT_DASHBOARD_ID, + title: "Default dashboard", + dataBinding: { + initial: DEFAULT_DASHBOARD_BENCHMARK_INITIAL, + required_filter_fields: [], + }, + dataRender:{ + type: "auto", + renders:[ + { + type: "AutoBenchmarkPairwiseComparisonTable", + title: "Comparison Table", + filterByFieldValues:{ }, + config: { + tableConfig: { + nameKeys: ["compiler"], + enableDialog: true + }, + }, + }, + ] + } +} diff --git a/torchci/components/benchmark/v3/configs/utils/autoRegistration.tsx b/torchci/components/benchmark/v3/configs/utils/autoRegistration.tsx new file mode 100644 index 0000000000..035258f4fe --- /dev/null +++ b/torchci/components/benchmark/v3/configs/utils/autoRegistration.tsx @@ -0,0 +1,70 @@ +import { BenchmarkCommitMeta } from "lib/benchmark/store/benchmark_regression_store"; +import { AutoBenchmarkPairwiseComparisonTable } from "../../components/dataRender/auto/AutoComponents"; + + +export type AutoComponent = React.ComponentType; + +export type AutoComponentConfig = { + Component: AutoComponent; +}; + +/** ---------------- Fixed components (examples/placeholders) ---------------- */ +// Replace with your actual import + +const ErrorAutoComponent: AutoComponent = (config) => { + console.warn( + "Rendering default fallback auto component. Bad config:", + config + ); + return ( +
+ ⚠ Unknown auto component type. Please check config. +
+ ); +}; + +/** ---------------- Immutable class ---------------- */ +export class AutoComponentRegistry { + /** singleton instance */ + private static _instance: AutoComponentRegistry | null = null; + + /** read-only registry */ + readonly map: Readonly>; + + /** read-only fallback */ + readonly fallback: Readonly; + + private constructor() { + const registry: Record = { + AutoBenchmarkPairwiseComparisonTable: { + Component: AutoBenchmarkPairwiseComparisonTable, + }, + }; + this.map = Object.freeze({ ...registry }); + this.fallback = Object.freeze({ Component: ErrorAutoComponent }); + Object.freeze(this); // freeze the instance so it can't be mutated + } + + /** get the singleton */ + static get instance(): AutoComponentRegistry { + if (!this._instance) this._instance = new AutoComponentRegistry(); + return this._instance; + } + + /** lookup a config; fall back to default */ + get(type: string): AutoComponentConfig { + return this.map[type] ?? this.fallback; + } + + /** list all supported component types */ + listTypes(): string[] { + return Object.keys(this.map); + } +} + +/** ---------------- Helper function (optional) ---------------- */ +export function getAutoRenderComponent(config: { + type: string; +}): AutoComponentConfig { + return AutoComponentRegistry.instance.get(config.type); +} diff --git a/torchci/components/benchmark/v3/configs/utils/dataBindingRegistration.tsx b/torchci/components/benchmark/v3/configs/utils/dataBindingRegistration.tsx index c397fb383a..6d82cf1dda 100644 --- a/torchci/components/benchmark/v3/configs/utils/dataBindingRegistration.tsx +++ b/torchci/components/benchmark/v3/configs/utils/dataBindingRegistration.tsx @@ -1,8 +1,9 @@ import dayjs, { Dayjs } from "dayjs"; -import { TimeRange } from "lib/benchmark/store/benchmark_regression_store"; -import DefaultMetricsDropdowns from "../../components/benchmarkSideBar/components/DefaultSideBarMetricsDropdowns"; +import { BenchmarkCommitMeta, TimeRange } from "lib/benchmark/store/benchmark_regression_store"; +import DefaultMetricsDropdowns from "../../components/benchmarkSideBar/components/filters/DefaultSideBarMetricsDropdowns"; import { NotFoundComponent, resolveComponent } from "../configRegistration"; import { compilerQueryParameterConverter } from "../teams/compilers/config"; +import { arch } from "os"; export const MIN_SAMPLING_THRESHOLD = 2; export type DataBindingConfig = { @@ -25,8 +26,8 @@ export type BenchmarkUiParameters = { filters: Record; lbranch: string; rbranch: string; - lcommit?: string; - rcommit?: string; + lcommit?: BenchmarkCommitMeta; + rcommit?: BenchmarkCommitMeta; [key: string]: any; }; @@ -38,6 +39,8 @@ export type QueryParamsConfig = { /* ----------------------- Converter function signatures --------------------- */ export type QueryParameterConverterInputs = { timeRange: TimeRange; + benchmarkName: string; + repo: string; branches?: string[]; commits?: string[]; filters: Record; @@ -49,12 +52,22 @@ export type QueryParameterConverter = ( inputs: QueryParameterConverterInputs ) => any; +const DEFAULT_FILTERS ={ + dtype: "", + device: "", + arch: "", + mode: "", + backend: "", +} /* ---------------------------- Default converter ---------------------------- */ export const getDefaultDataConverter: QueryParameterConverter = (i) => { return { + ...DEFAULT_FILTERS, ...i.filters, branches: i.branches ?? [], commits: i.commits ?? [], + repo: i.repo, + benchmarkName: i.benchmarkName, startTime: dayjs.utc(i.timeRange.start).format("YYYY-MM-DDTHH:mm:ss"), stopTime: dayjs.utc(i.timeRange.end).format("YYYY-MM-DDTHH:mm:ss"), }; diff --git a/torchci/components/benchmark/v3/pages/BenchmarkDashboardPage.tsx b/torchci/components/benchmark/v3/pages/BenchmarkDashboardPage.tsx new file mode 100644 index 0000000000..02ffffce68 --- /dev/null +++ b/torchci/components/benchmark/v3/pages/BenchmarkDashboardPage.tsx @@ -0,0 +1,44 @@ +import { Box } from "@mui/system"; +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; +import { BenchmarkDashboardStoreProvider } from "lib/benchmark/store/benchmark_dashboard_provider"; +import BenchmarkSideBar from "../components/benchmarkSideBar/BenchmarkSideBar"; +import { BenchmarkTopBar } from "../components/benchmarkSideBar/BenchmarkTopBar"; +import { BenchmarkUIConfigHandler, useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; +import { useEffect, useState } from "react"; +import { useRouter } from "next/router"; +import LoadingPage from "components/common/LoadingPage"; +dayjs.extend(utc); + +export default function BenchmarkDashboardPage({ + benchmarkId, +}: { + benchmarkId: string; +}) { +const router = useRouter(); + const ensureConfig = useBenchmarkBook((s) => s.ensureConfig); + const [config, setConfig] = useState() + + useEffect(() => { + if (!router.isReady) return; + const configHandler = ensureConfig(benchmarkId, "dashboard", {}); + setConfig(configHandler); + }, [router.isReady, benchmarkId]); + + if (!config) return + + const Comp = config.getDataRenderComponent(); + return ( + + + + + + + + + + + + ); +} diff --git a/torchci/components/benchmark/v3/pages/BenchmarkRegressionPage.tsx b/torchci/components/benchmark/v3/pages/BenchmarkRegressionPage.tsx index c35aaf1778..5e6fdb1eb0 100644 --- a/torchci/components/benchmark/v3/pages/BenchmarkRegressionPage.tsx +++ b/torchci/components/benchmark/v3/pages/BenchmarkRegressionPage.tsx @@ -4,7 +4,8 @@ import utc from "dayjs/plugin/utc"; import { BenchmarkDashboardStoreProvider } from "lib/benchmark/store/benchmark_dashboard_provider"; import BenchmarkSideBar from "../components/benchmarkSideBar/BenchmarkSideBar"; import { BenchmarkTopBar } from "../components/benchmarkSideBar/BenchmarkTopBar"; -import { getConfig } from "../configs/configBook"; +import getConfig from "next/config"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; dayjs.extend(utc); export default function BenchmarkRegressionPage({ @@ -14,6 +15,8 @@ export default function BenchmarkRegressionPage({ benchmarkId: string; initial: any; }) { + + const getConfig = useBenchmarkBook((s) => s.getConfig); const config = getConfig(benchmarkId); // get dynamic componenet if any registered, otherwise use default diff --git a/torchci/components/uiModules/UMDenseComponents.tsx b/torchci/components/uiModules/UMDenseComponents.tsx index ce1610b539..6cb6f40955 100644 --- a/torchci/components/uiModules/UMDenseComponents.tsx +++ b/torchci/components/uiModules/UMDenseComponents.tsx @@ -64,10 +64,15 @@ const DENSE_SELECT_SX = { }, }; +export interface UMDenseDropdownOption { + value: string; + displayName?: string; +} + type Props = { dtype: string; setDType: (v: string) => void; - dtypes: string[]; + dtypes: (string | UMDenseDropdownOption)[] label: string; }; @@ -78,6 +83,7 @@ export const MODES: { [k: string]: string } = { inference: "bfloat16", }; + export const UMDenseDropdown: React.FC = ({ dtype, setDType, @@ -91,25 +97,30 @@ export const UMDenseDropdown: React.FC = ({ setDType(e.target.value); }; + const safeValue = dtype ?? ""; return ( - {label} + {label} ); diff --git a/torchci/lib/benchmark/api_helper/backend/common/type.ts b/torchci/lib/benchmark/api_helper/backend/common/type.ts index d448e83095..4f09a70302 100644 --- a/torchci/lib/benchmark/api_helper/backend/common/type.ts +++ b/torchci/lib/benchmark/api_helper/backend/common/type.ts @@ -107,12 +107,13 @@ export interface CommitResult { * the field value must match the fields in LLMsBenchmarkProps */ export enum BenchmarkMetadataType { - ModelName = "modelName", - BackendName = "backendName", - ModeName = "modeName", - DtypeName = "dtypeName", + ModelName = "model", + BackendName = "backend", + ModeName = "mode", + DtypeName = "dtype", DeviceName = "deviceName", - ArchName = "archName", + Device = "device", + ArchName = "arch", OperatornName = "operatorName", Qps = "qps", } @@ -121,6 +122,7 @@ export enum BenchmarkMetadataType { */ export interface BenchmarkMetadataItem { type: BenchmarkMetadataType; - options: string[]; + options: {displayName:string, value:string}[]; labelName: string; + initialValue?: string; } diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts index 14f9b7db84..92b980234d 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/benchmarkDataQueryBuilder.ts @@ -64,13 +64,14 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { private _extra_keys = new Set(); DEFAULT_PARAMS = { - excludedMetrics: [], - commits: [], arch: "", mode: "", + device: "", + granularity: "hour", + excludedMetrics: [], models: [], branches: [], - granularity: "hour", + commits: [], backends: [], dtypes: [], }; @@ -201,6 +202,10 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { OR empty({branches: Array(String) }) ) AND notEmpty(device) + AND ( + startsWith({device: String }, device) + OR {device: String } = '' + ) AND ( arch LIKE concat('%', {arch: String }, '%') OR {arch: String } = '' @@ -270,8 +275,8 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { (key) => `${this._EXTRA_KEY_FIELD_NAME}.${key}` ), ]; - config.table.sub_group_key = [ - ...config.table.sub_group_key, + config.table.group_key = [ + ...config.table.group_key, ...Array.from(this._extra_keys).map( (key) => `${this._EXTRA_KEY_FIELD_NAME}.${key}` ), @@ -314,9 +319,23 @@ export class BenchmarkDataQuery extends ExecutableQueryBase { toQueryParams(inputs: any, id?: string): Record { this.validateInputs(inputs); - if (inputs.benchmarkName) { + if (inputs.benchmarkName && !inputs.benchmarkNames) { inputs.benchmarkNames = [inputs.benchmarkName]; } + if (inputs.backend && !inputs.backends) { + inputs.backends = [inputs.backend]; + } + + if (inputs.dtype && !inputs.dtypes) { + inputs.dtypes = [inputs.dtype]; + } + + if (inputs.model && !inputs.models){ + inputs.models = [inputs.model]; + } + if (inputs.branch && !inputs.branches){ + inputs.branches = [inputs.branch]; + } const params = { ...this.DEFAULT_PARAMS, ...inputs }; return params; diff --git a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts index 28dd5da362..55a2030785 100644 --- a/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts +++ b/torchci/lib/benchmark/api_helper/backend/dataFetchers/queryBuilderUtils/listMetadataQueryBuilder.ts @@ -76,7 +76,7 @@ export class BenchmarkMetadataQuery validateInputs(inputs: any) { if (!inputs.benchmarkName) { - throw new Error("No benchmark names provided"); + throw new Error("No benchmark name provided"); } if (!inputs.repo) { throw new Error("No repo provided"); @@ -96,36 +96,52 @@ export function getDefaultBenchmarkMetadataGroup( data, "backend", BenchmarkMetadataType.BackendName, - DEFAULT_BACKEND_NAME, + {displayName: DEFAULT_BACKEND_NAME, value:""}, "Backend" ), makeMetadataItem( data, "mode", BenchmarkMetadataType.ModeName, - DEFAULT_MODE_NAME, + {displayName: DEFAULT_MODE_NAME, value:""}, "Mode" ), makeMetadataItem( data, "dtype", BenchmarkMetadataType.DtypeName, - DEFAULT_DTYPE_NAME, + {displayName: DEFAULT_DTYPE_NAME, value:""}, "Dtype" ), makeMetadataItem( data, "device", BenchmarkMetadataType.DeviceName, - DEFAULT_DEVICE_NAME, + {displayName: DEFAULT_DEVICE_NAME, value:""}, "Device Name", - (r) => (r.device && r.arch ? `${r.device} (${r.arch})` : r.device) + "", + (r: any) => { + if (r.device && r.arch) { + return { + value: `${r.device}||${r.arch}`, + displayName: `${r.device} (${r.arch})`, + }; + } + if (r.device) { + return { + value: r.device, + displayName: r.device, + }; + } + + return { value: "", displayName: "" }; + } ), makeMetadataItem( data, "model", BenchmarkMetadataType.ModelName, - "All models", + {displayName: "All Models", value:""}, "Model Name" ), ].filter(Boolean) as BenchmarkMetadataItem[]; @@ -138,13 +154,28 @@ function makeMetadataItem( data: any[], field: string, type: BenchmarkMetadataType, - defaultValue: string, + defaultValue: {value:string, displayName:string}, labelName: string, - formatter?: (r: any) => string | undefined + initialValue: string = "", + formatter?: (r: any) => { value: string; displayName: string } ): BenchmarkMetadataItem | null { - const values = ld.uniq( - data.map((r) => (formatter ? formatter(r) : r[field])).filter(Boolean) - ) as string[]; + const values = ld + .uniqBy( + data + .map((r) => { + // formatter must return { value, displayName } + const formatted = formatter + ? formatter(r) + : { + value: r[field], + displayName: r[field], + }; + if (!formatted?.displayName) return null; + return formatted; + }) + .filter(Boolean), + "value" + ) as { value: string; displayName: string }[]; if (values.length === 0) { return null; @@ -153,6 +184,7 @@ function makeMetadataItem( type, options: [defaultValue, ...values], labelName, + initialValue, }; } @@ -173,9 +205,20 @@ export class PytorchOperatorMicrobenchmarkMetadataFetcher data, "operator", BenchmarkMetadataType.OperatornName, - "All Operators", + {displayName: "All Operators", value:""}, "Operator", - (r) => r.model.split("_")[0] ?? "unknown" + "", + (r) => { + const splitted = r.model.split("_") + let value = undefined + if (splitted.length > 1) { + value = splitted[0] + } + return { + value: value, + displayName: value + } + } ); if (item) { li.push(item); diff --git a/torchci/lib/benchmark/api_helper/backend/list_commits.ts b/torchci/lib/benchmark/api_helper/backend/list_commits.ts index efd60aef9c..17b74f916e 100644 --- a/torchci/lib/benchmark/api_helper/backend/list_commits.ts +++ b/torchci/lib/benchmark/api_helper/backend/list_commits.ts @@ -81,12 +81,24 @@ function getFormat(data: any, format: string = "raw") { } } + +const defaultGetCommitsInputs: any = { + branches: [], + models:[], + backends: [], + device: "", + arch: [], + dtype: "", + mode: "", + +}; + export async function getCommits(inputparams: any) { if (!inputparams.repo) { throw new Error("no repo provided in request"); } - if (!inputparams.benchmarkNames || inputparams.benchmarkName) { + if (!inputparams.benchmarkNames && !inputparams.benchmarkName) { throw new Error("no benchmarkNames || benchmarkName provided in request"); } @@ -94,14 +106,25 @@ export async function getCommits(inputparams: any) { throw new Error("no start/end time provided in request"); } - if (inputparams.benchmarkName) { + if (inputparams.benchmarkName && !inputparams.benchmarkNames) { inputparams.benchmarkNames = [inputparams.benchmarkName]; } + + if (inputparams.model && !inputparams.models) { + inputparams.models = [inputparams.model]; + } + + if(inputparams.backend && !inputparams.backends) { + inputparams.backends = [inputparams.backend]; + } + const queryParams = { - ...defaultListCommitsInputs, // base defaults + ...defaultGetCommitsInputs, // base defaults ...inputparams, // override with caller's values }; + console.log("[API]list commits, defaultGetCommitsInputs query params: ", queryParams); + return await getCommitsWithSampling( BENCHMARK_DEFAULT_LIST_COMMITS_QUERY_NAME, queryParams diff --git a/torchci/lib/benchmark/api_helper/fe/api.ts b/torchci/lib/benchmark/api_helper/fe/api.ts index 34d2d428cc..30a60bca60 100644 --- a/torchci/lib/benchmark/api_helper/fe/api.ts +++ b/torchci/lib/benchmark/api_helper/fe/api.ts @@ -58,6 +58,35 @@ export async function postBenchmarkTimeSeriesFetcher( return res.json(); } + +export async function postBenchmarkMetadataFetcher( + name: string, + queryParams: Record +): Promise { + const body = { + name: name, + query_params: queryParams, + }; + const url = "/api/benchmark/list_metadata"; + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + let message = `Request failed with ${res.status}`; + try { + const payload = await res.json(); + if (payload?.error) message = payload.error; + } catch { + // if not valid JSON, ignore + } + throw new Error(message); + } + return res.json(); +} + export async function listBenchmarkRegressionReport( report_id: string, limit: number = 10, diff --git a/torchci/lib/benchmark/api_helper/fe/hooks.ts b/torchci/lib/benchmark/api_helper/fe/hooks.ts index 62f99b4657..26b88247fe 100644 --- a/torchci/lib/benchmark/api_helper/fe/hooks.ts +++ b/torchci/lib/benchmark/api_helper/fe/hooks.ts @@ -4,8 +4,13 @@ import { getBenchmarkRegressionReport, listBenchmarkCommits, listBenchmarkRegressionReport, + postBenchmarkMetadataFetcher, postBenchmarkTimeSeriesFetcher, } from "./api"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; +import { useDashboardSelector } from "lib/benchmark/store/benchmark_dashboard_provider"; +import { useMemo } from "react"; + export function useBenchmarkCommitsData( benchmarkId: string, @@ -32,15 +37,33 @@ export function useBenchmarkCommitsData( } // --- Hook wrapper --- -export function useBenchmarkData( - benchamrk_name: string, +export function useBenchmarkTimeSeriesData( + benchmark_id: string, queryParams: Record | null, formats: string[] = ["time_series", "table"] ): SWRResponse { + const shouldFetch = !!queryParams; return useApi( postBenchmarkTimeSeriesFetcher, - [benchamrk_name, formats, queryParams], + [benchmark_id, formats, queryParams], + { + revalidateOnFocus: false, + keepPreviousData: true, + dedupingInterval: 10_000, + }, + shouldFetch + ); +} + +export function useListBenchmarkMetadata( + benchmark_id: string, + queryParams: Record | null +): SWRResponse{ + const shouldFetch = !!queryParams; + return useApi( + postBenchmarkMetadataFetcher, + [benchmark_id, queryParams], { revalidateOnFocus: false, keepPreviousData: true, @@ -50,6 +73,7 @@ export function useBenchmarkData( ); } + export function useListBenchmarkRegressionReportsData( report_id: string, limit: number = 10, @@ -58,13 +82,16 @@ export function useListBenchmarkRegressionReportsData( return useApi(listBenchmarkRegressionReport, [report_id, limit], { refreshInterval: refreshInterval, // refresh every 12 hour revalidateOnFocus: false, + ...limitErrorRetrySWR }); } + export function useGetBenchmarkRegressionReportData(id: string): any { return useApi(getBenchmarkRegressionReport, [id], { refreshInterval: 12 * 60 * 60 * 1000, // refresh every 12 hour revalidateOnFocus: false, + ...limitErrorRetrySWR, }); } @@ -84,6 +111,85 @@ export function useApi( return useSWR( key, key ? ([, ...params]) => apiFunc(...params) : null, - options + { + ...limitErrorRetrySWR, + ...options, + } ); } + +/** Unified hook: collects dashboard + config info in one place + * this is a read-only hook, mainly used for UI components that + * only needs to read committed state +*/ +export function useBenchmarkCommittedContext() { + // read dashboard state + const { + repo, + benchmarkId, + benchmarkName, + committedTime, + committedFilters, + committedLbranch, + committedRbranch, + committedMaxSampling, + lcommit, + rcommit, + } = useDashboardSelector((s) => ({ + repo: s.repo, + benchmarkName: s.benchmarkName, + benchmarkId: s.benchmarkId, + committedTime: s.committedTime, + committedFilters: s.committedFilters, + committedLbranch: s.committedLbranch, + committedRbranch: s.committedRbranch, + committedMaxSampling: s.committedMaxSampling, + lcommit: s.lcommit, + rcommit: s.rcommit, + })); + + const configHandler = useBenchmarkConfigBook(benchmarkId); + if (!configHandler) return null + + const config = configHandler; + const requiredFilters = config.dataBinding?.raw?.required_filter_fields ?? []; + const dataRender = config?.raw?.dataRender ?? null; + + return { + repo, + benchmarkId, + benchmarkName, + committedTime, + committedFilters, + committedLbranch, + committedRbranch, + committedMaxSampling, + lcommit, + rcommit, + config, + configHandler, + requiredFilters, + dataRender, + }; +} + +// safely get config handler from benchmark book +export function useBenchmarkConfigBook(benchmarkId:string){ + const getConfig = useBenchmarkBook((s) => s.getConfig); + const configHandler = useMemo(() => { + if (!benchmarkId) return null; + try { + return getConfig(benchmarkId); + } catch { + return null; + } + }, [benchmarkId, getConfig]); + return configHandler; +} + +const limitErrorRetrySWR: SWRConfiguration={ + onErrorRetry: (error:any, key:any, config:any, revalidate, { retryCount }) => { + if (error.status === 404) return; + if (retryCount >= 2) return; + } + } diff --git a/torchci/lib/benchmark/llms/utils/llmUtils.ts b/torchci/lib/benchmark/llms/utils/llmUtils.ts index a72b22b619..7953fa4b19 100644 --- a/torchci/lib/benchmark/llms/utils/llmUtils.ts +++ b/torchci/lib/benchmark/llms/utils/llmUtils.ts @@ -36,6 +36,7 @@ export function useBenchmark( (queryParamsWithBranchAndCommit as { [key: string]: any })["commits"] = branchAndCommit.commit ? [branchAndCommit.commit] : []; + console.log("query params", queryParamsWithBranchAndCommit) const url = `/api/clickhouse/${queryName}?parameters=${encodeURIComponent( JSON.stringify(queryParamsWithBranchAndCommit) )}`; diff --git a/torchci/lib/benchmark/store/benchmark_config_book.tsx b/torchci/lib/benchmark/store/benchmark_config_book.tsx new file mode 100644 index 0000000000..4eadc57095 --- /dev/null +++ b/torchci/lib/benchmark/store/benchmark_config_book.tsx @@ -0,0 +1,157 @@ +import { DefaultAutoRenderContent } from "components/benchmark/v3/components/dataRender/auto/defaultAutoRenderContent"; +import { DefaultFanoutRenderContent } from "components/benchmark/v3/components/dataRender/fanout/defaultFanoutRenderContent"; +import { NotFoundComponent, resolveComponent } from "components/benchmark/v3/configs/configRegistration"; +import { CompilerPrecomputeBenchmarkUIConfig, COMPILTER_PRECOMPUTE_BENCHMARK_ID } from "components/benchmark/v3/configs/teams/compilers/config"; +import { defaultDashboardBenchmarkUIConfig } from "components/benchmark/v3/configs/teams/default/default_dashboard_config"; +import { DataBinding, DataBindingConfig } from "components/benchmark/v3/configs/utils/dataBindingRegistration"; +import { create } from "zustand"; + +export type BenchmarkUIConfig = { + benchmarkId: string; + apiId: string; + title: string; + type: string; // type of the component to render + dataBinding: DataBindingConfig; // data binding config + dataRender: DataRenderOption; // either binds a component or a converter function to render data + required_filter_fields?: readonly string[]; // required filter fields +}; + +export type UIRenderConfig = { + title?: string; // title of the component to render + id?: string; // id of the component to render + type: string; // type of the component to render + config: any; // config of the component to render +}; + +export type DataRenderOption = { + type: string; + api?: any; + id?: string; // id of the component to render, this is used when type is 'component' + sideRender?: { [key: string]: UIRenderConfig }; // this used to render side content, such as regression report access + renders?: UIRenderConfig[]; // this is used when type is predefined type such as 'default-fanout' +}; + + +/** + * A class to host a single benchmark UI config + */ +export class BenchmarkUIConfigHandler { + private _benchmarkId: string; + private _config: BenchmarkUIConfig; + private _dataBinding: DataBinding; + + constructor(config: BenchmarkUIConfig) { + this._benchmarkId = config.benchmarkId; + this._config = config; + this._dataBinding = new DataBinding(config.dataBinding); + } + + get benchmarkId(): string { + return this._benchmarkId; + } + + get raw(): BenchmarkUIConfig { + return this._config; + } + + get dataBinding(): DataBinding { + return this._dataBinding; + } + + /** + * Get DataRenderComponent, this fetches the skeleton to render the content of the benchmark + * - fanout: the component calls the api once and passes the data to the component + * - self-fetching: the component calls the api multiple times and maintain the state by themselves + * - customized: user defined component + * @returns the component to render the data + */ + getDataRenderComponent = (): React.ComponentType => { + const dr = this._config.dataRender; + if(!dr) { + throw new Error(`No data render config found for ${this._benchmarkId}, this is internal error, please report `); + } + switch (dr.type) { + case "fanout": + return DefaultFanoutRenderContent; + case "auto": + return DefaultAutoRenderContent; + case "customized": + const Comp = resolveComponent(dr.id); + if (Comp) return Comp; + default: + // inline fallback component to satisfy the return type + const Missing: React.FC = () => ( + + ); + return Missing; + } + }; +} + +type ConfigMap = Record; + +interface State { + predefined: ConfigMap; + temps: ConfigMap; + + initTempConfig: (id: string, type?: string, params?: Partial) => BenchmarkUIConfig; + ensureConfig: (id: string, type?: string, params?: Partial) => BenchmarkUIConfigHandler; + getConfig: (id: string) => BenchmarkUIConfigHandler; + listIds: () => string[]; +} + +const predefined: ConfigMap = { + [COMPILTER_PRECOMPUTE_BENCHMARK_ID]: CompilerPrecomputeBenchmarkUIConfig, +}; + +export const useBenchmarkBook = create()((set, get) => ({ + predefined, + temps: {}, + + initTempConfig: (id, type = "dashboard", params = {}) => { + const { temps } = get(); + let defaultConfig = defaultDashboardBenchmarkUIConfig + switch (type) { + case "dashboard": + defaultConfig = defaultDashboardBenchmarkUIConfig + break; + default: + throw new Error(`Unknown type: ${type}`); + } + const cfg: BenchmarkUIConfig = { + ...defaultDashboardBenchmarkUIConfig, + type, + benchmarkId: id, + apiId: params.apiId ?? id, + title: params.title ?? id, + dataBinding:{ + ...defaultConfig.dataBinding, + initial:{ + ...defaultConfig.dataBinding.initial, + benchmarkId: id, + + }, + } + }; + set({ temps: { ...temps, [id]: cfg } }); + return cfg; + }, + + ensureConfig: (id, type = "dashboard", params = {}) => { + const { predefined, temps, initTempConfig } = get(); + const cfg = predefined[id] ?? temps[id] ?? initTempConfig(id, type, params); + return new BenchmarkUIConfigHandler(cfg); + }, + + getConfig: (id) => { + const { predefined, temps } = get(); + const cfg = predefined[id] ?? temps[id]; + if (!cfg) throw new Error(`No config found for id: ${id}`); + return new BenchmarkUIConfigHandler(cfg); + }, + + listIds: () => { + const { predefined, temps } = get(); + return [...Object.keys(predefined), ...Object.keys(temps)]; + }, +})); diff --git a/torchci/lib/benchmark/store/benchmark_regression_store.ts b/torchci/lib/benchmark/store/benchmark_regression_store.ts index d388aa2d85..199c295cdf 100644 --- a/torchci/lib/benchmark/store/benchmark_regression_store.ts +++ b/torchci/lib/benchmark/store/benchmark_regression_store.ts @@ -14,6 +14,34 @@ export type BenchmarkCommitMeta = { index?: number; }; +/** + * BenchmarkIdMappingItem is a mapping from benchmarkId to repoName and benchmarkName + * benchmarkName is used to fetch the benchmark data from dv + */ +interface BenchmarkIdMappingItem{ + id: string; + repoName: string; + benchmarkName: string; +} + +const BENCHMARK_ID_MAPPING: Record ={ + "compiler_inductor":{ + id:"compiler_inductor", + repoName:"pytorch/pytorch", + benchmarkName:"compiler_inductor", + }, + "compiler_precompute":{ + id:"compiler_precompute", + repoName:"pytorch/pytorch", + benchmarkName:"compiler_precompute", + }, + "pytorch_operator_microbenchmark":{ + id:"pytorch_operator_microbenchmark", + repoName:"pytorch/pytorch", + benchmarkName:"PyTorch operator microbenchmark", + } +} + /** * Data model for BenchmarkDashboardState */ @@ -22,6 +50,9 @@ export interface BenchmarkDashboardState { stagedFilters: Record; stagedLbranch: string; stagedRbranch: string; + stagedLcommit: BenchmarkCommitMeta | null; + stagedRcommit: BenchmarkCommitMeta | null; + committedTime: TimeRange; committedFilters: Record; committedLbranch: string; @@ -37,6 +68,8 @@ export interface BenchmarkDashboardState { // may key to track of the benchamrk benchmarkId: string; + benchmarkName: string; + repo: string; lcommit: BenchmarkCommitMeta | null; rcommit: BenchmarkCommitMeta | null; @@ -47,6 +80,8 @@ export interface BenchmarkDashboardState { setStagedRbranch: (c: string) => void; setStagedFilter: (k: string, v: string) => void; setStagedFilters: (filters: Record) => void; + setStagedLcommit: (c: BenchmarkCommitMeta | null) => void; + setStagedRcommit: (c: BenchmarkCommitMeta | null) => void; commitMainOptions: () => void; revertMainOptions: () => void; @@ -89,8 +124,11 @@ export function createDashboardStore(initial: { rcommit?: BenchmarkCommitMeta | null; maxSampling?: number; }) { + const idItem = BENCHMARK_ID_MAPPING[initial.benchmarkId] return createWithEqualityFn()((set, get) => ({ benchmarkId: initial.benchmarkId, // <-- fixed name + benchmarkName : idItem? idItem.benchmarkName: initial.benchmarkId, // <-- fixed name + repo: idItem?.repoName? idItem.repoName : "pytorch/pytorch", // set only with initial config enableSamplingSetting: (initial.maxSampling ?? 0) > 0, @@ -108,6 +146,9 @@ export function createDashboardStore(initial: { stagedLbranch: initial.lbranch ?? "", stagedRbranch: initial.rbranch ?? "", + stagedLcommit: initial.lcommit ?? null, + stagedRcommit: initial.rcommit ?? null, + // committed committedTime: initial.time, committedFilters: initial.filters, @@ -130,6 +171,9 @@ export function createDashboardStore(initial: { setStagedLbranch: (c) => set({ stagedLbranch: c }), setStagedRbranch: (c) => set({ stagedRbranch: c }), + setStagedLcommit: (c) => set({ stagedLcommit: c }), + setStagedRcommit: (c) => set({ stagedRcommit: c }), + setStagedTime: (t) => set({ stagedTime: t }), setStagedFilter: (k, v) => set((s) => ({ stagedFilters: { ...s.stagedFilters, [k]: v } })), @@ -190,6 +234,9 @@ export function createDashboardStore(initial: { committedLbranch: next.lbranch ?? s.committedLbranch ?? "", committedRbranch: next.rbranch ?? s.committedRbranch ?? "", + stagedLcommit: next.lcommit !== undefined ? next.lcommit : s.stagedLcommit, + stagedRcommit: next.rcommit !== undefined ? next.rcommit : s.stagedRcommit, + lcommit: next.lcommit !== undefined ? next.lcommit : s.lcommit, rcommit: next.rcommit !== undefined ? next.rcommit : s.rcommit, }; diff --git a/torchci/pages/api/benchmark/get_time_series.ts b/torchci/pages/api/benchmark/get_time_series.ts index 063223a83c..53b671f1a0 100644 --- a/torchci/pages/api/benchmark/get_time_series.ts +++ b/torchci/pages/api/benchmark/get_time_series.ts @@ -31,7 +31,6 @@ export default async function handler( const params = readApiGetParams(req); console.log("[API]get_time_series, received request:", params); - // validate params if ( !params || @@ -41,7 +40,6 @@ export default async function handler( ) { return res.status(400).json({ error: "Missing parameters" }); } - // get time series data try { const { name, response_formats, query_params } = params; diff --git a/torchci/pages/api/benchmark/list_metadata.ts b/torchci/pages/api/benchmark/list_metadata.ts index 73daed485d..7de0bdf586 100644 --- a/torchci/pages/api/benchmark/list_metadata.ts +++ b/torchci/pages/api/benchmark/list_metadata.ts @@ -15,7 +15,7 @@ export default async function handler( // validate params if ( !params || - !params.id || + !params.name || !params.query_params || Object.keys(params.query_params).length == 0 || Object.keys(params).length === 0 @@ -23,8 +23,9 @@ export default async function handler( return res.status(400).json({ error: "Missing required parameters" }); } + console.log("[API]list metadata, received request:", params); try { - const groups = await listBenchmarkMetadata(params.query_params, params.id); + const groups = await listBenchmarkMetadata(params.query_params, params.name); return res.status(200).json({ data: groups }); } catch (err: any) { return res.status(500).json({ error: err.message }); diff --git a/torchci/pages/benchmark/compilers_regression.tsx b/torchci/pages/benchmark/compilers_regression.tsx index 1219cf7bc2..1087b882b4 100644 --- a/torchci/pages/benchmark/compilers_regression.tsx +++ b/torchci/pages/benchmark/compilers_regression.tsx @@ -1,8 +1,10 @@ -import { getConfig } from "components/benchmark/v3/configs/configBook"; import BenchmarkRegressionPage from "components/benchmark/v3/pages/BenchmarkRegressionPage"; +import { useBenchmarkBook } from "lib/benchmark/store/benchmark_config_book"; export default function Page() { const id = "compiler_precompute"; + + const getConfig = useBenchmarkBook((s) => s.getConfig); const config = getConfig(id); const dataBinding = config.dataBinding; return ( diff --git a/torchci/pages/benchmark/v3/dashboard/[id].tsx b/torchci/pages/benchmark/v3/dashboard/[id].tsx new file mode 100644 index 0000000000..4e4a2285b8 --- /dev/null +++ b/torchci/pages/benchmark/v3/dashboard/[id].tsx @@ -0,0 +1,14 @@ +import { useRouter } from "next/router"; +import { Alert, Grid, Link } from "@mui/material"; +import BenchmarkDashboardPage from "components/benchmark/v3/pages/BenchmarkDashboardPage"; + +export default function Page() { + const router = useRouter(); + const { id } = router.query; + if(!id){ + return Cannot find the page + } + + return + +} diff --git a/torchci/tsconfig.json b/torchci/tsconfig.json index 2dc1aee31c..3d411fba6c 100644 --- a/torchci/tsconfig.json +++ b/torchci/tsconfig.json @@ -30,7 +30,7 @@ "**/*.ts", "**/*.tsx", ".next/types/**/*.ts" -, "pages/benchmark/regression/[report_id]/regression_report" ], +, "pages/benchmark/regression/[report_id]/regression_report", "components/benchmark/v3/components/dataRender/components/benchmarkPairwise/BenchmarkPairwiseComparisonTable" ], "exclude": [ "node_modules" ]