[CI][Benchmarks] Automatically detect component versions in benchmark CI (#18339)

Previously, there was no way to figure out what version of
compute-runtime we were using. This PR:
- adds a way to detect compute-runtime version via looking up L0 patch
version in compute-runtime releases
- adds a way to pre-specify a compute-runtime version that should be
used before looking up L0 patch in compute-runtime releases
- additionally, adds a way to automatically detect SYCL version via
`__clang_version__`

Author: ianayl

devops/actions/run-tests/benchmark/action.yml

@@ -110,19 +110,6 @@ runs:
       pip install --user --break-system-packages -r ./devops/scripts/benchmarks/requirements.txt
       echo "-----"
 
-      # clang builds have git repo / commit hashes in their --version output,
-      # same goes for dpcpp. Obtain git repo / commit hash info this way:
-
-      # First line of --version is formatted 'clang version ... (<repo> <commit>)'
-      # thus we parse for (<repo> <commit>):
-      sycl_git_info="$(clang++ --version | head -n 1 | grep -oE '\([^ ]+ [a-f0-9]+\)$' | tr -d '()')"
-      if [ -z "$sycl_git_info" ]; then
-        echo "Error: Unable to deduce SYCL build source repo/commit: Are you sure dpcpp variable is in PATH?"
-        exit 1
-      fi
-      sycl_git_repo="$(printf "$sycl_git_info" | cut -d' ' -f1)"
-      sycl_git_commit="$(printf "$sycl_git_info" | cut -d' ' -f2)"
-
       # By default, the benchmark scripts forceload level_zero
       FORCELOAD_ADAPTER="${ONEAPI_DEVICE_SELECTOR%%:*}"
       echo "Adapter: $FORCELOAD_ADAPTER"
@@ -149,6 +136,12 @@ runs:
       SAVE_NAME="${SAVE_PREFIX}_${MACHINE_TYPE}_${SAVE_SUFFIX}"
       echo "SAVE_NAME=$SAVE_NAME" >> $GITHUB_ENV
       SAVE_TIMESTAMP="$(date -u +'%Y%m%d_%H%M%S')"  # Timestamps are in UTC time
+ 
+      # Cache the compute_runtime version from dependencies.json, but perform a
+      # check with L0 version before using it: This value is not guaranteed to
+      # accurately reflect the current compute_runtime version used, as the
+      # docker images are built nightly. 
+      export COMPUTE_RUNTIME_TAG_CACHE="$(cat ./devops/dependencies.json | jq -r .linux.compute_runtime.github_tag)"
 
       sycl-ls
       echo "-----"
@@ -163,8 +156,7 @@ runs:
         --output-dir "./llvm-ci-perf-results/" \
         --preset "$PRESET" \
         --timestamp-override "$SAVE_TIMESTAMP" \
-        --github-repo "$sycl_git_repo" \
-        --git-commit "$sycl_git_commit"
+        --detect-version sycl,compute_runtime
       echo "-----"
       python3 ./devops/scripts/benchmarks/compare.py to_hist \
         --name "$SAVE_NAME" \

devops/scripts/benchmarks/compare.py

@@ -145,6 +145,9 @@ def validate_benchmark_result(result: BenchmarkRun) -> bool:
 
                 def reset_aggregate() -> dict:
                     return {
+                        # TODO compare determine which command args have an
+                        # impact on perf results, and do not compare arg results
+                        # are incomparable
                         "command_args": set(test_run.command[1:]),
                         "aggregate": aggregator(starting_elements=[test_run.value]),
                     }
@@ -153,27 +156,7 @@ def reset_aggregate() -> dict:
                 if test_run.name not in average_aggregate:
                     average_aggregate[test_run.name] = reset_aggregate()
                 else:
-                    # Check that we are comparing runs with the same cmd args:
-                    if (
-                        set(test_run.command[1:])
-                        == average_aggregate[test_run.name]["command_args"]
-                    ):
-                        average_aggregate[test_run.name]["aggregate"].add(
-                            test_run.value
-                        )
-                    else:
-                        # If the command args used between runs are different,
-                        # discard old run data and prefer new command args
-                        #
-                        # This relies on the fact that paths from get_result_paths()
-                        # is sorted from older to newer
-                        print(
-                            f"Warning: Command args for {test_run.name} from {result_path} is different from prior runs."
-                        )
-                        print(
-                            "DISCARDING older data and OVERRIDING with data using new arg."
-                        )
-                        average_aggregate[test_run.name] = reset_aggregate()
+                    average_aggregate[test_run.name]["aggregate"].add(test_run.value)
 
         return {
             name: BenchmarkHistoricAverage(
@@ -217,9 +200,9 @@ def halfway_round(value: int, n: int):
         for test in target.results:
             if test.name not in hist_avg:
                 continue
-            if hist_avg[test.name].command_args != set(test.command[1:]):
-                print(f"Warning: skipped {test.name} due to command args mismatch.")
-                continue
+            # TODO compare command args which have an impact on performance
+            # (i.e. ignore --save-name): if command results are incomparable,
+            # skip the result.
 
             delta = 1 - (
                 test.value / hist_avg[test.name].value

devops/scripts/benchmarks/history.py

@@ -13,6 +13,8 @@
 from utils.utils import run
 from utils.validate import Validate
 
+from utils.detect_versions import DetectVersion
+
 
 class BenchmarkHistory:
     runs = []
@@ -94,9 +96,13 @@ def git_info_from_path(path: Path) -> (str, str):
             return git_hash, github_repo
 
         if options.git_commit_override is None or options.github_repo_override is None:
-            git_hash, github_repo = git_info_from_path(
-                os.path.dirname(os.path.abspath(__file__))
-            )
+            if options.detect_versions.sycl:
+                print(f"Auto-detecting sycl version...")
+                github_repo, git_hash = DetectVersion.instance().get_dpcpp_git_info()
+            else:
+                git_hash, github_repo = git_info_from_path(
+                    os.path.dirname(os.path.abspath(__file__))
+                )
         else:
             git_hash, github_repo = (
                 options.git_commit_override,
@@ -119,9 +125,19 @@ def git_info_from_path(path: Path) -> (str, str):
                 throw=ValueError("Illegal characters found in specified RUNNER_NAME."),
             )
 
-        compute_runtime = (
-            options.compute_runtime_tag if options.build_compute_runtime else ""
-        )
+        compute_runtime = None
+        if options.build_compute_runtime:
+            compute_runtime = options.compute_runtime_tag
+        elif options.detect_versions.compute_runtime:
+            print(f"Auto-detecting compute_runtime version...")
+            detect_res = DetectVersion.instance()
+            compute_runtime = detect_res.get_compute_runtime_ver()
+            if detect_res.get_compute_runtime_ver_cached() is None:
+                print(
+                    "Warning: Could not find compute_runtime version via github tags API."
+                )
+        else:
+            compute_runtime = "unknown"
 
         return BenchmarkRun(
             name=name,

devops/scripts/benchmarks/main.py

@@ -19,11 +19,13 @@
 from utils.utils import prepare_workdir
 from utils.compute_runtime import *
 from utils.validate import Validate
+from utils.detect_versions import DetectVersion
 from presets import enabled_suites, presets
 
 import argparse
 import re
 import statistics
+import os
 
 # Update this if you are changing the layout of the results files
 INTERNAL_WORKDIR_VERSION = "2.0"
@@ -506,7 +508,7 @@ def validate_and_parse_env_args(env_args):
         type=lambda ts: Validate.timestamp(
             ts,
             throw=argparse.ArgumentTypeError(
-                "Specified timestamp not in YYYYMMDD_HHMMSS format."
+                "Specified timestamp not in YYYYMMDD_HHMMSS format"
             ),
         ),
         help="Manually specify timestamp used in metadata",
@@ -517,7 +519,7 @@ def validate_and_parse_env_args(env_args):
         type=lambda gh_repo: Validate.github_repo(
             gh_repo,
             throw=argparse.ArgumentTypeError(
-                "Specified github repo not in <owner>/<repo> format."
+                "Specified github repo not in <owner>/<repo> format"
             ),
         ),
         help="Manually specify github repo metadata of component tested (e.g. SYCL, UMF)",
@@ -528,13 +530,32 @@ def validate_and_parse_env_args(env_args):
         type=lambda commit: Validate.commit_hash(
             commit,
             throw=argparse.ArgumentTypeError(
-                "Specified commit is not a valid commit hash."
+                "Specified commit is not a valid commit hash"
             ),
         ),
         help="Manually specify commit hash metadata of component tested (e.g. SYCL, UMF)",
         default=options.git_commit_override,
     )
 
+    parser.add_argument(
+        "--detect-version",
+        type=lambda components: Validate.on_re(
+            components,
+            r"[a-z_,]+",
+            throw=argparse.ArgumentTypeError(
+                "Specified --detect-version is not a comma-separated list"
+            ),
+        ),
+        help="Detect versions of components used: comma-separated list with choices from sycl,compute_runtime",
+        default=None,
+    )
+    parser.add_argument(
+        "--detect-version-cpp-path",
+        type=Path,
+        help="Location of detect_version.cpp used to query e.g. DPC++, L0",
+        default=None,
+    )
+
     args = parser.parse_args()
     additional_env_vars = validate_and_parse_env_args(args.env)
 
@@ -586,6 +607,28 @@ def validate_and_parse_env_args(env_args):
         options.github_repo_override = args.github_repo
         options.git_commit_override = args.git_commit
 
+    # Automatically detect versions:
+    if args.detect_version is not None:
+        detect_ver_path = args.detect_version_cpp_path
+        if detect_ver_path is None:
+            detect_ver_path = Path(
+                f"{os.path.dirname(__file__)}/utils/detect_versions.cpp"
+            )
+            if not detect_ver_path.is_file():
+                parser.error(
+                    f"Unable to find detect_versions.cpp at {detect_ver_path}, please specify --detect-version-cpp-path"
+                )
+        elif not detect_ver_path.is_file():
+            parser.error(f"Specified --detect-version-cpp-path is not a valid file")
+
+        enabled_components = args.detect_version.split(",")
+        options.detect_versions.sycl = "sycl" in enabled_components
+        options.detect_versions.compute_runtime = (
+            "compute_runtime" in enabled_components
+        )
+
+        detect_res = DetectVersion.init(detect_ver_path)
+
     benchmark_filter = re.compile(args.filter) if args.filter else None
 
     main(

devops/scripts/benchmarks/options.py

@@ -16,6 +16,31 @@ class MarkdownSize(Enum):
     FULL = "full"
 
 
+@dataclass
+class DetectVersionsOptions:
+    """
+    Options for automatic version detection
+    """
+
+    # Components to detect versions for:
+    sycl: bool = False
+    compute_runtime: bool = False
+    # umf: bool = False
+    # level_zero: bool = False
+
+    # Placeholder text, should automatic version detection fail: This text will
+    # only be used if automatic version detection for x component is explicitly
+    # specified.
+    not_found_placeholder = "unknown"  # None
+
+    # TODO unauthenticated users only get 60 API calls per hour: this will not
+    # work if we enable benchmark CI in precommit.
+    compute_runtime_tag_api: str = (
+        "https://api.github.com/repos/intel/compute-runtime/tags"
+    )
+    # Max amount of api calls permitted on each run of the benchmark scripts
+    max_api_calls = 4
+
 @dataclass
 class Options:
     workdir: str = None
@@ -64,5 +89,9 @@ class Options:
     github_repo_override: str = None
     git_commit_override: str = None
 
+    detect_versions: DetectVersionsOptions = field(
+        default_factory=DetectVersionsOptions
+    )
+
 
 options = Options()

devops/scripts/benchmarks/utils/detect_versions.cpp

@@ -0,0 +1,82 @@
+#include <cstdlib>
+#include <cstring>
+#include <iostream>
+#include <vector>
+
+#include <level_zero/ze_api.h>
+
+#define _assert(cond, msg)                                                     \
+  if (!(cond)) {                                                               \
+    std::cout << std::endl << "Error: " << msg << std::endl;                   \
+    exit(1);                                                                   \
+  }
+
+#define _success(res) res == ZE_RESULT_SUCCESS
+
+std::string query_dpcpp_ver() { return std::string(__clang_version__); }
+
+std::string query_l0_driver_ver() {
+  // Initialize L0 drivers:
+  ze_init_driver_type_desc_t driver_type = {};
+  driver_type.stype = ZE_STRUCTURE_TYPE_INIT_DRIVER_TYPE_DESC;
+  driver_type.flags = ZE_INIT_DRIVER_TYPE_FLAG_GPU;
+  driver_type.pNext = nullptr;
+
+  uint32_t driver_count = 0;
+  ze_result_t result = zeInitDrivers(&driver_count, nullptr, &driver_type);
+  _assert(_success(result), "Failed to initialize L0.");
+  _assert(driver_count > 0, "No L0 drivers available.");
+
+  std::vector<ze_driver_handle_t> drivers(driver_count);
+  result = zeInitDrivers(&driver_count, drivers.data(), &driver_type);
+  _assert(_success(result), "Could not fetch L0 drivers.");
+
+  // Check support for fetching driver version strings:
+  uint32_t ext_count = 0;
+  result = zeDriverGetExtensionProperties(drivers[0], &ext_count, nullptr);
+  _assert(_success(result), "Failed to obtain L0 extensions count.");
+  _assert(ext_count > 0, "No L0 extensions available.");
+
+  std::vector<ze_driver_extension_properties_t> extensions(ext_count);
+  result =
+      zeDriverGetExtensionProperties(drivers[0], &ext_count, extensions.data());
+  _assert(_success(result), "Failed to obtain L0 extensions.");
+  bool version_ext_support = false;
+  for (const auto &extension : extensions) {
+    // std::cout << extension.name << std::endl;
+    if (strcmp(extension.name, "ZE_intel_get_driver_version_string")) {
+      version_ext_support = true;
+    }
+  }
+  _assert(version_ext_support,
+          "ZE_intel_get_driver_version_string extension is not supported.");
+
+  // Fetch L0 driver version:
+  ze_result_t (*pfnGetDriverVersionFn)(ze_driver_handle_t, char *, size_t *);
+  result = zeDriverGetExtensionFunctionAddress(drivers[0],
+                                               "zeIntelGetDriverVersionString",
+                                               (void **)&pfnGetDriverVersionFn);
+  _assert(_success(result), "Failed to obtain GetDriverVersionString fn.");
+
+  size_t ver_str_len = 0;
+  result = pfnGetDriverVersionFn(drivers[0], nullptr, &ver_str_len);
+  _assert(_success(result), "Call to GetDriverVersionString failed.");
+
+  std::cout << "ver_str_len: " << ver_str_len << std::endl;
+  ver_str_len++; // ver_str_len does not account for '\0'
+  char *ver_str = (char *)calloc(ver_str_len, sizeof(char));
+  result = pfnGetDriverVersionFn(drivers[0], ver_str, &ver_str_len);
+  _assert(_success(result), "Failed to write driver version string.");
+
+  std::string res(ver_str);
+  free(ver_str);
+  return res;
+}
+
+int main() {
+  std::string dpcpp_ver = query_dpcpp_ver();
+  std::cout << "DPCPP_VER='" << dpcpp_ver << "'" << std::endl;
+
+  std::string l0_ver = query_l0_driver_ver();
+  std::cout << "L0_VER='" << l0_ver << "'" << std::endl;
+}