From 7e31dfdec846cc0ca55d5cfaffcee8ef68da66b3 Mon Sep 17 00:00:00 2001 From: Kareem Khazem Date: Mon, 18 Mar 2024 17:02:08 +0000 Subject: [PATCH] Expand ${var} in benchcomp variant `env` The values of environment variables in the benchcomp configuration file can now contain strings of the form '${var}'. Benchcomp will replace these strings with the value of the environment variable 'var'. This is intended to allow users to have several benchcomp variants, each of which differs only in the environment. This fixes #2981. --- docs/src/benchcomp-conf.md | 20 ++++++++ tools/benchcomp/benchcomp/entry/run.py | 44 ++++++++++++++++- tools/benchcomp/test/test_regression.py | 37 ++++++++++++++ tools/benchcomp/test/test_unit.py | 66 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 tools/benchcomp/test/test_unit.py diff --git a/docs/src/benchcomp-conf.md b/docs/src/benchcomp-conf.md index 77236d0917bf..de57ac831221 100644 --- a/docs/src/benchcomp-conf.md +++ b/docs/src/benchcomp-conf.md @@ -4,6 +4,26 @@ This page lists the different visualizations that are available. +## Variants + +A *variant* is a single invocation of a benchmark suite. Benchcomp runs several +variants, so that their performance can be compared later. A variant consists of +a command-line argument, working directory, and environment. Benchcomp invokes +the command using the operating system environment, updated with the keys and +values in `env`. If any values in `env` contain strings of the form `${var}`, +Benchcomp expands them to the value of the environment variable `$var`. + +```yaml +variants: + variant_1: + config: + command_line: echo "Hello, world" + directory: /tmp + env: + PATH: /my/local/directory:${PATH} +``` + + ## Built-in visualizations The following visualizations are available; these can be added to the `visualize` list of `benchcomp.yaml`. diff --git a/tools/benchcomp/benchcomp/entry/run.py b/tools/benchcomp/benchcomp/entry/run.py index a870e7e9a1b0..28457381da2b 100644 --- a/tools/benchcomp/benchcomp/entry/run.py +++ b/tools/benchcomp/benchcomp/entry/run.py @@ -13,6 +13,7 @@ import logging import os import pathlib +import re import shutil import subprocess import typing @@ -53,9 +54,10 @@ def __post_init__(self): else: self.working_copy = pathlib.Path(self.directory) + def __call__(self): - env = dict(os.environ) - env.update(self.env) + update_environment_with = _EnvironmentUpdater() + env = update_environment_with(self.env) if self.copy_benchmarks_dir: shutil.copytree( @@ -128,6 +130,44 @@ def __call__(self): tmp_symlink.rename(self.out_symlink) + +@dataclasses.dataclass +class _EnvironmentUpdater: + """Update the OS environment with keys and values containing variables + + When called, this class returns the operating environment updated with new + keys and values. The values can contain variables of the form '${var_name}'. + The class evaluates those variables using values already in the environment. + """ + + os_environment: dict = dataclasses.field( + default_factory=lambda : dict(os.environ)) + pattern: re.Pattern = re.compile(r"\$\{(\w+?)\}") + + + def _evaluate(self, key, value): + """Evaluate all ${var} in value using self.os_environment""" + old_value = value + + for variable in re.findall(self.pattern, value): + if variable not in self.os_environment: + logging.error( + "Couldn't evaluate ${%s} in the value '%s' for environment " + "variable '%s'. Ensure the environment variable $%s is set", + variable, old_value, key, variable) + sys.exit(1) + value = re.sub( + r"\$\{" + variable + "\}", self.os_environment[variable], value) + return value + + + def __call__(self, new_environment): + ret = dict(self.os_environment) + for key, value in new_environment.items(): + ret[key] = self._evaluate(key, value) + return ret + + def get_default_out_symlink(): return "latest" diff --git a/tools/benchcomp/test/test_regression.py b/tools/benchcomp/test/test_regression.py index 87df67a071cc..51f9dbb597db 100644 --- a/tools/benchcomp/test/test_regression.py +++ b/tools/benchcomp/test/test_regression.py @@ -646,6 +646,43 @@ def test_return_0_on_fail(self): result = yaml.safe_load(handle) + def test_env_expansion(self): + """Ensure that config parser expands '${}' in env key""" + + with tempfile.TemporaryDirectory() as tmp: + run_bc = Benchcomp({ + "variants": { + "env_set": { + "config": { + "command_line": 'echo "$__BENCHCOMP_ENV_VAR" > out', + "directory": tmp, + "env": {"__BENCHCOMP_ENV_VAR": "foo:${PATH}"} + } + }, + }, + "run": { + "suites": { + "suite_1": { + "parser": { + # The word 'bin' typically appears in $PATH, so + # check that what was echoed contains 'bin'. + "command": textwrap.dedent("""\ + grep bin out && grep '^foo:' out && echo '{ + "benchmarks": {}, + "metrics": {} + }' + """) + }, + "variants": ["env_set"] + } + } + }, + "visualize": [], + }) + run_bc() + self.assertEqual(run_bc.proc.returncode, 0, msg=run_bc.stderr) + + def test_env(self): """Ensure that benchcomp reads the 'env' key of variant config""" diff --git a/tools/benchcomp/test/test_unit.py b/tools/benchcomp/test/test_unit.py new file mode 100644 index 000000000000..12320116f217 --- /dev/null +++ b/tools/benchcomp/test/test_unit.py @@ -0,0 +1,66 @@ +# Copyright Kani Contributors +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Benchcomp regression testing suite. This suite uses Python's stdlib unittest +# module, but nevertheless actually runs the binary rather than running unit +# tests. + +import unittest +import uuid + +import benchcomp.entry.run + + + +class TestEnvironmentUpdater(unittest.TestCase): + def test_environment_construction(self): + """Test that the default constructor reads the OS environment""" + + update_environment = benchcomp.entry.run._EnvironmentUpdater() + environment = update_environment({}) + self.assertIn("PATH", environment) + + + def test_placeholder_construction(self): + """Test that the placeholder constructor reads the placeholder""" + + key, value = [str(uuid.uuid4()) for _ in range(2)] + update_environment = benchcomp.entry.run._EnvironmentUpdater({ + key: value, + }) + environment = update_environment({}) + self.assertIn(key, environment) + self.assertEqual(environment[key], value) + + + def test_environment_update(self): + """Test that the environment is updated""" + + key, value, update = [str(uuid.uuid4()) for _ in range(3)] + update_environment = benchcomp.entry.run._EnvironmentUpdater({ + key: value, + }) + environment = update_environment({ + key: update + }) + self.assertIn(key, environment) + self.assertEqual(environment[key], update) + + + def test_environment_update_variable(self): + """Test that the environment is updated""" + + old_env = { + "key1": str(uuid.uuid4()), + "key2": str(uuid.uuid4()), + } + + actual_update = "${key2}xxx${key1}" + expected_update = f"{old_env['key2']}xxx{old_env['key1']}" + + update_environment = benchcomp.entry.run._EnvironmentUpdater(old_env) + environment = update_environment({ + "key1": actual_update, + }) + self.assertIn("key1", environment) + self.assertEqual(environment["key1"], expected_update)