Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions devops/scripts/benchmarks/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,23 +199,23 @@ def halfway_round(value: int, n: int):
regression = []

for test in target.results:
if test.name not in hist_avg:
if test.label not in hist_avg:
continue
# TODO compare command args which have an impact on performance
# (i.e. ignore --save-name): if command results are incomparable,
# skip the result.

delta = 1 - (
test.value / hist_avg[test.name].value
test.value / hist_avg[test.label].value
if test.lower_is_better
else hist_avg[test.name].value / test.value
else hist_avg[test.label].value / test.value
)

def perf_diff_entry() -> dict:
res = asdict(test)
res["delta"] = delta
res["hist_avg"] = hist_avg[test.name].value
res["avg_type"] = hist_avg[test.name].average_type
res["hist_avg"] = hist_avg[test.label].value
res["avg_type"] = hist_avg[test.label].average_type
return res

# Round to 2 decimal places: not going to fail a test on 0.001% over
Expand All @@ -226,7 +226,7 @@ def perf_diff_entry() -> dict:
regression.append(perf_diff_entry())

log.debug(
f"{test.name}: expect {hist_avg[test.name].value}, got {test.value}"
f"{test.label}: expect {hist_avg[test.label].value}, got {test.value}"
)

return improvement, regression
Expand Down
3 changes: 3 additions & 0 deletions devops/scripts/benchmarks/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,9 @@ def git_info_from_path(path: Path) -> (str, str):

# Get platform information
platform_info = get_platform_info()
if platform_info.gpu_info is None:
log.warning("GPU information detection failed.")
platform_info.gpu_info = []

return BenchmarkRun(
name=name,
Expand Down
12 changes: 6 additions & 6 deletions devops/scripts/benchmarks/html/scripts.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (C) 2024-2025 Intel Corporation
// Copyright (C) 2024-2026 Intel Corporation
// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
// See LICENSE.TXT
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Expand Down Expand Up @@ -1939,7 +1939,7 @@ function displaySelectedRunsPlatformInfo() {
.map(runName => {
const run = loadedBenchmarkRuns.find(r => r.name === runName);
if (run && run.platform) {
return { name: runName, platform: run.platform };
return { name: runName, platform: run.platform, date: run.date };
}
return null;
})
Expand All @@ -1959,17 +1959,17 @@ function displaySelectedRunsPlatformInfo() {
const platform = runData.platform;
const detailsContainer = document.createElement('div');
detailsContainer.className = 'platform-details-compact';
detailsContainer.innerHTML = createPlatformDetailsHTML(platform);
detailsContainer.innerHTML = createPlatformDetailsHTML(platform, runData.date);
runSection.appendChild(detailsContainer);
container.appendChild(runSection);
});
}

// Platform Information Functions

function createPlatformDetailsHTML(platform) {
const formattedTimestamp = platform.timestamp ?
new Date(platform.timestamp).toLocaleString('en-US', {
function createPlatformDetailsHTML(platform, run_date) {
const formattedTimestamp = run_date ?
new Date(run_date).toLocaleString('en-US', {
year: 'numeric',
month: 'short',
day: 'numeric',
Expand Down
13 changes: 12 additions & 1 deletion devops/scripts/benchmarks/output_html.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (C) 2024-2025 Intel Corporation
# Copyright (C) 2024-2026 Intel Corporation
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
# See LICENSE.TXT
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Expand Down Expand Up @@ -135,6 +135,17 @@ def generate_html(
# Sorted in reverse, such that runs are ordered from newest to oldest
current_runs.sort(key=lambda run: run.date or datetime.min, reverse=True)

# The solution below requires the above sort to happen (dashboard also uses the same order).
#
# Don't write "env" & "command" fields for all runs, but first 10 - it spams the output file.
# In dashboard we use only newest commands anyway. We actually pick command for each "label"
# from one of the newest runs; number 10 was picked arbitrarily as most likely all available
# benchmarks was within the last 10 runs (a few Baselines plus potentially a few custom runs).
for run in current_runs[10:]:
for result in run.results:
result.command = []
result.env = {}

# Create the comprehensive output object
output = BenchmarkOutput(
runs=current_runs,
Expand Down
7 changes: 2 additions & 5 deletions devops/scripts/benchmarks/tests/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@

DataJson = namedtuple("DataJson", ["runs", "metadata", "tags", "names"])
DataJsonRun = namedtuple("DataJsonRun", ["name", "results"])
DataJsonResult = namedtuple(
"DataJsonResult", ["name", "label", "suite", "value", "unit"]
)
DataJsonResult = namedtuple("DataJsonResult", ["label", "suite", "value", "unit"])
DataJsonMetatdata = namedtuple(
"DataJsonMetatdata",
[
Expand Down Expand Up @@ -112,7 +110,6 @@ def get_benchmark_output_data(self):
name=run["name"],
results=[
DataJsonResult(
name=r["name"],
label=r["label"],
suite=r["suite"],
value=r["value"],
Expand Down Expand Up @@ -167,7 +164,7 @@ def _checkGroup(
self.assertEqual(groupMetadata.type, "group")

def _checkResultsExist(self, caseName: str, out: DataJson):
self.assertIn(caseName, [r.name for r in out.runs[0].results])
self.assertIn(caseName, [r.label for r in out.runs[0].results])

def _checkExistsInProcessOutput(
self, proc: subprocess.CompletedProcess, expected: str
Expand Down
21 changes: 9 additions & 12 deletions devops/scripts/benchmarks/utils/platform.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
# Copyright (C) 2025 Intel Corporation
# Copyright (C) 2025-2026 Intel Corporation
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
# See LICENSE.TXT
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

import platform
import subprocess
import os
from datetime import datetime
from utils.result import Platform
from options import options

Expand Down Expand Up @@ -42,7 +41,11 @@ def get_project_clang_version(bin_dir):
[clang_path, "--version"], capture_output=True, text=True
)
if result.returncode == 0:
return result.stdout.split("\n")[0]
out = result.stdout.splitlines()
val = out[0]
if len(out) > 1:
val += " " + out[1]
return val
except (FileNotFoundError, subprocess.CalledProcessError):
pass

Expand Down Expand Up @@ -98,7 +101,6 @@ def get_compute_runtime_version_detailed():
def get_gpu_info():
"""Get GPU information including device list and driver version"""
gpu_list = []
gpu_count = 0
gpu_driver_version = "(unknown)"

# Get GPU info from lspci
Expand All @@ -116,11 +118,8 @@ def get_gpu_info():
if ": " in line:
gpu_name = line.split(": ", 1)[1]
gpu_list.append(gpu_name)

gpu_count = len(gpu_list)
except Exception:
gpu_list = ["Detection failed"]
gpu_count = 0
gpu_list = None

# Try to get GPU driver version
try:
Expand All @@ -146,7 +145,7 @@ def get_gpu_info():
except Exception:
pass

return gpu_list, gpu_count, gpu_driver_version
return gpu_list, gpu_driver_version


def get_platform_info() -> Platform:
Expand Down Expand Up @@ -178,7 +177,7 @@ def get_platform_info() -> Platform:
cpu_info = f"Detection failed: {str(e)}"

# Get GPU information
gpu_list, gpu_count, gpu_driver_version = get_gpu_info()
gpu_list, gpu_driver_version = get_gpu_info()

# Compiler versions - GCC from system, clang project-built
gcc_version = "gcc (unknown)"
Expand Down Expand Up @@ -230,12 +229,10 @@ def get_platform_info() -> Platform:
level_zero_version = f"{adapter_name} | level-zero (version unknown)"

return Platform(
timestamp=datetime.now().isoformat(),
os=os_info,
python=python_info,
cpu_count=cpu_count,
cpu_info=cpu_info,
gpu_count=gpu_count,
gpu_info=gpu_list,
gpu_driver_version=gpu_driver_version,
gcc_version=gcc_version,
Expand Down
5 changes: 1 addition & 4 deletions devops/scripts/benchmarks/utils/result.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (C) 2024-2025 Intel Corporation
# Copyright (C) 2024-2026 Intel Corporation
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
# See LICENSE.TXT
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Expand All @@ -11,12 +11,10 @@
@dataclass_json
@dataclass
class Platform:
timestamp: str = ""
os: str = ""
python: str = ""
cpu_count: int = 0
cpu_info: str = ""
gpu_count: int = 0
gpu_info: list[str] = field(default_factory=list)
gpu_driver_version: str = "" # Add GPU driver version
gcc_version: str = ""
Expand All @@ -39,7 +37,6 @@ class Result:
git_url: str = ""
git_hash: str = ""
# values below should not be set by the benchmark
name: str = ""
lower_is_better: bool = True
suite: str = "Unknown"

Expand Down
16 changes: 10 additions & 6 deletions devops/scripts/benchmarks/utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,17 @@ def run(
# order is important, we want provided sycl rt libraries to be first
if add_sycl:
sycl_bin_path = os.path.join(options.sycl, "bin")
env_vars["PATH"] = os.pathsep.join(
filter(None, [sycl_bin_path, env_vars.get("PATH", "")])
)
sycl_lib_path = os.path.join(options.sycl, "lib")
env_vars["LD_LIBRARY_PATH"] = os.pathsep.join(
filter(None, [sycl_lib_path, env_vars.get("LD_LIBRARY_PATH", "")])
)

# add them only if not already added
if sycl_bin_path not in env_vars.get("PATH", ""):
env_vars["PATH"] = os.pathsep.join(
filter(None, [sycl_bin_path, env_vars.get("PATH", "")])
)
if sycl_lib_path not in env_vars.get("LD_LIBRARY_PATH", ""):
env_vars["LD_LIBRARY_PATH"] = os.pathsep.join(
filter(None, [sycl_lib_path, env_vars.get("LD_LIBRARY_PATH", "")])
)

command_str = " ".join(command)
env_str = " ".join(f"{key}={value}" for key, value in env_vars.items())
Expand Down