Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions devops/scripts/benchmarks/CONTRIB.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ The suite is structured around four main components: Suites, Benchmarks, Results
* **Required Methods:**
* `setup()`: Initializes the benchmark (e.g., build, download data). Use `self.download()` for data dependencies. **Do not** perform setup in `__init__`.
* `run(env_vars)`: Executes the benchmark binary (use `self.run_bench()`) and returns a list of `Result` objects. Can be called multiple times, must produce consistent results.
* `teardown()`: Cleans up resources. Can be empty. No need to remove build artifacts or downloaded datasets.
* `name()`: Returns a unique identifier string for the benchmark across *all* suites. If a benchmark class is instantiated multiple times with different parameters (e.g., "Submit In Order", "Submit Out Of Order"), the `name()` must reflect this uniqueness.
* **Optional Methods:**
* `lower_is_better()`: Returns `True` if lower result values are better (default: `True`).
Expand Down Expand Up @@ -165,7 +164,7 @@ The benchmark suite generates an interactive HTML dashboard that visualizes `Res

## Adding New Benchmarks

1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`setup`, `run`, `teardown`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed.
1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`run`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

setup() is mentioned above as required method

2. **Add to Suite:**
* If adding to an existing category, modify the corresponding `Suite` class (e.g., `benches/compute.py`) to instantiate and return your new benchmark in its `benchmarks()` method.
* If creating a new category, create a new `Suite` class inheriting from `benches.base.Suite`. Implement `name()` and `benchmarks()`. Add necessary `setup()` if the suite requires shared setup. Add group metadata via `additional_metadata()` if needed.
Expand Down
72 changes: 34 additions & 38 deletions devops/scripts/benchmarks/benches/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,28 @@ def __init__(self, suite):
def name(self) -> str:
pass

@abstractmethod
def run(
self,
env_vars,
run_trace: TracingType = TracingType.NONE,
force_trace: bool = False,
) -> list[Result]:
"""Execute the benchmark with the given environment variables.

Args:
env_vars: Environment variables to use when running the benchmark.
run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
force_trace: If True, ignore the traceable() method and force tracing.

Returns:
A list of Result objects with the benchmark results.

Raises:
Exception: If the benchmark fails for any reason.
"""
pass

def display_name(self) -> str:
"""Returns a user-friendly name for display in charts.
By default returns the same as name(), but can be overridden.
Expand Down Expand Up @@ -87,44 +109,6 @@ def setup(self):
"""Extra setup steps to be performed before running the benchmark."""
pass

@abstractmethod
def teardown(self):
pass

@abstractmethod
def run(
self,
env_vars,
run_trace: TracingType = TracingType.NONE,
force_trace: bool = False,
) -> list[Result]:
"""Execute the benchmark with the given environment variables.

Args:
env_vars: Environment variables to use when running the benchmark.
run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
force_trace: If True, ignore the traceable() method and force tracing.

Returns:
A list of Result objects with the benchmark results.

Raises:
Exception: If the benchmark fails for any reason.
"""
pass

@staticmethod
def get_adapter_full_path():
for libs_dir_name in ["lib", "lib64"]:
adapter_path = os.path.join(
options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
)
if os.path.isfile(adapter_path):
return adapter_path
assert (
False
), f"could not find adapter file {adapter_path} (and in similar lib paths)"

def run_bench(
self,
command,
Expand Down Expand Up @@ -268,6 +252,18 @@ def get_metadata(self) -> dict[str, BenchmarkMetadata]:
)
}

@staticmethod
def get_adapter_full_path():
for libs_dir_name in ["lib", "lib64"]:
adapter_path = os.path.join(
options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
)
if os.path.isfile(adapter_path):
return adapter_path
assert (
False
), f"could not find adapter file {adapter_path} (and in similar lib paths)"


class Suite(ABC):
@abstractmethod
Expand Down
6 changes: 0 additions & 6 deletions devops/scripts/benchmarks/benches/benchdnn.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,6 @@ def setup(self) -> None:
timeout=60 * 20,
)

def teardown(self):
pass


class OneDnnBenchmark(Benchmark):
def __init__(self, suite, bench_driver, bench_name, bench_args, syclgraph=True):
Expand Down Expand Up @@ -207,6 +204,3 @@ def _extract_time(self, output):
if values:
return sum(values)
return 0.0

def teardown(self):
pass
Loading
Loading