Merge pull request #1339 from buildtesters/sean_assert_le

Assert less than and equal status check

buildtest/builders/base.py

@@ -20,6 +20,7 @@
 from buildtest.buildsystem.checks import (
  assert_eq_check,
  assert_ge_check,
+ assert_le_check,
  assert_range_check,
  exists_check,
  is_dir_check,
@@ -961,6 +962,7 @@ def check_test_state(self):
  pbs_job_state_match = False
  lsf_job_state_match = False
  assert_ge_match = False
+ assert_le_match = False
  assert_eq_match = False
  assert_range_match = False
  assert_exists = False
@@ -994,6 +996,9 @@ def check_test_state(self):
  if self.status.get("assert_ge"):
  assert_ge_match = assert_ge_check(self)
 
+ if self.status.get("assert_le"):
+ assert_le_match = assert_le_check(self)
+
  if self.status.get("assert_eq"):
  assert_eq_match = assert_eq_check(self)
 
@@ -1020,6 +1025,7 @@ def check_test_state(self):
  lsf_job_state_match,
  runtime_match,
  assert_ge_match,
+ assert_le_match,
  assert_eq_match,
  assert_range_match,
  assert_exists,

buildtest/buildsystem/checks.py

@@ -258,6 +258,9 @@ def assert_ge_check(builder):
  One can specify multiple assert checks to check each metric with its reference value. When multiple items are specified, the operation is a logical AND and all checks
  must be ``True``.
 
+ Args:
+ builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
+
  Returns:
  bool: True or False for performance check ``assert_ge``
  """
@@ -316,6 +319,72 @@ def assert_ge_check(builder):
  return all(assert_check)
 
 
+def assert_le_check(builder):
+ """Perform check on assert less than and equal when ``assert_le`` is specified in buildspec. The return is a boolean value that determines if the check has passed.
+ One can specify multiple assert checks to check each metric with its reference value. When multiple items are specified, the operation is a logical AND and all checks
+ must be ``True``.
+
+ Args:
+ builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
+
+ Returns:
+ bool: True or False for performance check ``assert_le``
+ """
+
+ # a list containing booleans to evaluate reference check for each metric
+ assert_check = []
+
+ metric_names = list(builder.metadata["metrics"].keys())
+
+ # iterate over each metric in buildspec and determine reference check for each metric
+ for metric in builder.status["assert_le"]:
+ name = metric["name"]
+ ref_value = metric["ref"]
+
+ # if metric is not valid, then mark as False
+ if not builder.is_valid_metric(name):
+ msg = f"[blue]{builder}[/]: Unable to find metric: [red]{name}[/red]. List of valid metrics are the following: {metric_names}"
+ console.print(msg)
+ logger.warning(msg)
+ assert_check.append(False)
+ continue
+
+ metric_value = builder.metadata["metrics"][name]
+
+ # if metrics is empty string mark as False since we can't convert item to int or float
+ if builder.metadata["metrics"][name] == "":
+ assert_check.append(False)
+ continue
+
+ if builder.metrics[name]["type"] == "str":
+ msg = f"[blue]{builder}[/]: Unable to convert metric: [red]'{name}'[/red] for comparison. The type must be 'int' or 'float' but recieved [red]{builder.metrics[name]['type']}[/red]. "
+ console.print(msg)
+ logger.warning(msg)
+ assert_check.append(False)
+ continue
+
+ # convert metric value and reference value to int
+ conv_value, ref_value = convert_metrics(
+ metric_value=metric_value,
+ ref_value=ref_value,
+ dtype=builder.metrics[name]["type"],
+ )
+
+ console.print(
+ f"[blue]{builder}[/]: testing metric: {name} if {conv_value} <= {ref_value}"
+ )
+
+ # if there is a type mismatch then let's stop now before we do comparison
+ if (conv_value is None) or (ref_value is None):
+ assert_check.append(False)
+ continue
+
+ assert_check.append(conv_value <= ref_value)
+
+ # perform a logical AND on the list and return the boolean result
+ return all(assert_check)
+
+
 def assert_eq_check(builder):
  """This method is perform Assert Equality used when ``assert_eq`` property is specified
  in status check. This method will evaluate each metric value reference value and

buildtest/schemas/definitions.schema.json

@@ -238,6 +238,28 @@
  }
  }
  },
+ "assert_le": {
+ "type": "array",
+ "description": "Perform assertion of less than and equal (<=) with reference value",
+ "items": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "name",
+ "ref"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of metric to use for comparison"
+ },
+ "ref": {
+ "type": "number",
+ "description": "Specify reference value (int,float) for comparison"
+ }
+ }
+ }
+ },
  "assert_eq":
  {
  "description": "Perform assertion of equality (=) with reference value",

docs/buildspecs/performance_checks.rst

@@ -87,7 +87,7 @@ Let's run ``buildtest inspect query -o stream_test`` to retrieve the test detail
 .. dropdown:: ``buildtest inspect query -o stream_test``
 
  .. command-output:: buildtest inspect query -o stream_test
-
+ 
 Assert Equal
 ---------------
 

tests/builders/assert_le.yml

@@ -0,0 +1,46 @@
+buildspecs:
+ assert_le_example:
+ type: script
+ executor: generic.local.bash
+ description: Run stream test with metrics example using assert less than equal
+ env:
+ OMP_NUM_THREADS: 4
+ run: |
+ wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
+ gcc -openmp -o stream stream.c
+ ./stream
+ metrics:
+ copy:
+ type: float
+ regex:
+ exp: 'Copy:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ scale:
+ type: float
+ regex:
+ exp: 'Scale:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ add:
+ type: float
+ regex:
+ exp: 'Add:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ triad:
+ type: float
+ regex:
+ exp: 'Triad:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ status:
+ assert_le:
+ - name: copy
+ ref: 5000
+ - name: scale
+ ref: 5500
+ - name: add
+ ref: 6000
+ - name: triad
+ ref: 6500

tests/builders/test_builders.py

@@ -23,6 +23,16 @@ def test_assert_ge():
  cmd.build()
 
 
+def test_assert_le():
+
+ cmd = BuildTest(
+ buildspecs=[os.path.join(here, "assert_le.yml")],
+ buildtest_system=system,
+ configuration=config,
+ )
+ cmd.build()
+
+
 def test_assert_eq():
  cmd = BuildTest(
  buildspecs=[os.path.join(here, "assert_eq.yml")],

tutorials/perf_checks/assert_le.yml

@@ -0,0 +1,46 @@
+buildspecs:
+ assert_le_example:
+ type: script
+ executor: generic.local.bash
+ description: Run stream test with metrics example using assert less than equal
+ env:
+ OMP_NUM_THREADS: 4
+ run: |
+ wget https://raw.githubusercontent.com/jeffhammond/STREAM/master/stream.c
+ gcc -openmp -o stream stream.c
+ ./stream
+ metrics:
+ copy:
+ type: float
+ regex:
+ exp: 'Copy:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ scale:
+ type: float
+ regex:
+ exp: 'Scale:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ add:
+ type: float
+ regex:
+ exp: 'Add:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ triad:
+ type: float
+ regex:
+ exp: 'Triad:\s+(\S+)\s+.*'
+ stream: stdout
+ item: 1
+ status:
+ assert_le:
+ - name: copy
+ ref: 5000
+ - name: scale
+ ref: 5500
+ - name: add
+ ref: 6000
+ - name: triad
+ ref: 6500