support for logical AND in status check via 'mode'

buildtest/builders/base.py

@@ -231,8 +231,29 @@ def _set_metadata_values(self):
  self.metadata["compiler"] = None
 
  self.metadata["result"] = {"state": "N/A", "returncode": "-1", "runtime": 0}
- self.metadata["check"] = {"returncode": "N/A", "regex": "N/A", "runtime": "N/A"}
-
+ status_check_names = [
+ "regex",
+ "returncode",
+ "runtime",
+ "file_regex",
+ "slurm_job_state",
+ "pbs_job_state",
+ "lsf_job_state",
+ "assert_ge",
+ "assert_gt",
+ "assert_le",
+ "assert_lt",
+ "assert_eq",
+ "assert_ne",
+ "contains",
+ "not_contains",
+ "is_symlink",
+ "exists",
+ "is_dir",
+ "is_file",
+ "file_count",
+ ]
+ self.metadata["check"] = {name: None for name in status_check_names}
  self.metadata["metrics"] = {}
 
  # used to store job id from batch scheduler
@@ -985,128 +1006,94 @@ def check_test_state(self):
 
  # if status is defined in Buildspec, then check for returncode and regex
  if self.status:
- slurm_job_state_match = False
- pbs_job_state_match = False
- lsf_job_state_match = False
- assert_ge_match = False
- assert_le_match = False
- assert_gt_match = False
- assert_lt_match = False
- assert_eq_match = False
- assert_ne_match = False
- assert_range_match = False
- assert_contains_match = False
- assert_notcontains_match = False
- assert_is_symlink = False
- assert_exists = False
- assert_is_dir = False
- assert_is_file = False
- file_regex_match = False
- assert_file_count = False
 
- # returncode_match is boolean to check if reference returncode matches return code from test
- returncode_match = returncode_check(self)
+ # if 'state' property is specified explicitly honor this value regardless of what is calculated
+ if self.status.get("state"):
+ self.metadata["result"]["state"] = self.status["state"]
+ return
 
- # check regex against output or error stream based on regular expression
- # defined in status property. Return value is a boolean
- regex_match = regex_check(self)
+ # returncode_match is boolean to check if reference returncode matches return code from test
+ if self.status.get("returncode"):
+  self.metadata["check"]["returncode"] = returncode_check(self)
 
- runtime_match = runtime_check(self)
+ # check regex against output or error stream based on regular expression defined in status property. Return value is a boolean
+ if self.status.get("regex"):
+ self.metadata["check"]["regex"] = regex_check(self)
 
- self.metadata["check"]["regex"] = regex_match
- self.metadata["check"]["runtime"] = runtime_match
- self.metadata["check"]["returncode"] = returncode_match
+ if self.status.get("runtime"):
+ self.metadata["check"]["runtime"] = runtime_check(self)
 
  if self.status.get("file_regex"):
- file_regex_match = file_regex_check(self)
+ self.metadata["check"]["file_regex"] = file_regex_check(self)
 
  if self.status.get("slurm_job_state") and isinstance(self.job, SlurmJob):
- slurm_job_state_match = (
+ self.metadata["check"]["slurm_job_state"] = (
  self.status["slurm_job_state"] == self.job.state()
  )
 
  if self.status.get("pbs_job_state") and isinstance(self.job, PBSJob):
- pbs_job_state_match = self.status["pbs_job_state"] == self.job.state()
+ self.metadata["check"]["pbs_job_state"] = (
+ self.status["pbs_job_state"] == self.job.state()
+ )
 
  if self.status.get("lsf_job_state") and isinstance(self.job, LSFJob):
- lsf_job_state_match = self.status["lsf_job_state"] == self.job.state()
+ self.metadata["check"]["lsf_job_state"] = (
+ self.status["lsf_job_state"] == self.job.state()
+ )
 
  if self.status.get("assert_ge"):
- assert_ge_match = assert_ge_check(self)
+ self.metadata["check"]["assert_ge"] = assert_ge_check(self)
 
  if self.status.get("assert_le"):
- assert_le_match = assert_le_check(self)
+ self.metadata["check"]["assert_le"] = assert_le_check(self)
 
  if self.status.get("assert_gt"):
- assert_gt_match = assert_gt_check(self)
+ self.metadata["check"]["assert_gt"] = assert_gt_check(self)
 
  if self.status.get("assert_lt"):
- assert_lt_match = assert_lt_check(self)
+ self.metadata["check"]["assert_lt"] = assert_lt_check(self)
 
  if self.status.get("assert_eq"):
- assert_eq_match = assert_eq_check(self)
+ self.metadata["check"]["assert_eq"] = assert_eq_check(self)
 
  if self.status.get("assert_ne"):
- assert_ne_match = assert_ne_check(self)
+ self.metadata["check"]["assert_ne"] = assert_ne_check(self)
 
  if self.status.get("assert_range"):
- assert_range_match = assert_range_check(self)
+ self.metadata["check"]["assert_range"] = assert_range_check(self)
 
  if self.status.get("contains"):
- assert_contains_match = contains_check(self)
+ self.metadata["check"]["contains"] = contains_check(self)
 
  if self.status.get("not_contains"):
- assert_notcontains_match = notcontains_check(self)
+ self.metadata["check"]["not_contains"] = notcontains_check(self)
 
  if self.status.get("is_symlink"):
- assert_is_symlink = is_symlink_check(builder=self)
+ self.metadata["check"]["is_symlink"] = is_symlink_check(builder=self)
 
  if self.status.get("exists"):
- assert_exists = exists_check(builder=self)
+ self.metadata["check"]["exists"] = exists_check(builder=self)
 
  if self.status.get("is_dir"):
- assert_is_dir = is_dir_check(builder=self)
+ self.metadata["check"]["is_dir"] = is_dir_check(builder=self)
 
  if self.status.get("is_file"):
- assert_is_file = is_file_check(builder=self)
+ self.metadata["check"]["is_file"] = is_file_check(builder=self)
 
  if self.status.get("file_count"):
- assert_file_count = file_count_check(builder=self)
-
- # if any of checks is True we set the 'state' to PASS
- state = any(
- [
- returncode_match,
- regex_match,
- file_regex_match,
- slurm_job_state_match,
- pbs_job_state_match,
- lsf_job_state_match,
- runtime_match,
- assert_ge_match,
- assert_le_match,
- assert_gt_match,
- assert_lt_match,
- assert_eq_match,
- assert_ne_match,
- assert_range_match,
- assert_contains_match,
- assert_notcontains_match,
- assert_is_symlink,
- assert_exists,
- assert_is_dir,
- assert_is_file,
- assert_file_count,
- ]
- )
- if state:
- self.metadata["result"]["state"] = "PASS"
- else:
- self.metadata["result"]["state"] = "FAIL"
+ self.metadata["check"]["file_count"] = file_count_check(builder=self)
 
- # if 'state' property is specified explicitly honor this value regardless of what is calculated
- if self.status.get("state"):
- self.metadata["result"]["state"] = self.status["state"]
+ # filter out any None values from status check
+ status_checks = [
+ value for value in self.metadata["check"].values() if value is not None
+ ]
+
+ state = (
+ all(status_checks)
+ if self.status.get("mode") == "all"
+ else any(status_checks)
+ )
+ self.metadata["result"]["state"] = "PASS" if state else "FAIL"
 
  def _process_compiler_config(self):
  """This method is responsible for setting cc, fc, cxx class variables based

buildtest/buildsystem/checks.py

@@ -39,32 +39,26 @@ def returncode_check(builder):
  builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
  """
 
- returncode_match = False
-
- # if 'returncode' field set for 'status' check the returncode if its not set we return False
- if "returncode" in builder.status.keys():
- # returncode can be an integer or list of integers
- buildspec_returncode = builder.status["returncode"]
-
- # if buildspec returncode field is integer we convert to list for check
- if isinstance(buildspec_returncode, int):
- buildspec_returncode = [buildspec_returncode]
-
- logger.debug("Conducting Return Code check")
- logger.debug(
- "Status Return Code: %s Result Return Code: %s"
- % (
- buildspec_returncode,
- builder.metadata["result"]["returncode"],
- )
- )
- # checks if test returncode matches returncode specified in Buildspec and assign boolean to returncode_match
- returncode_match = (
- builder.metadata["result"]["returncode"] in buildspec_returncode
- )
- console.print(
- f"[blue]{builder}[/]: Checking returncode - {builder.metadata['result']['returncode']} is matched in list {buildspec_returncode}"
+ # returncode can be an integer or list of integers
+ buildspec_returncode = builder.status["returncode"]
+
+ # if buildspec returncode field is integer we convert to list for check
+ if isinstance(buildspec_returncode, int):
+ buildspec_returncode = [buildspec_returncode]
+
+ logger.debug("Conducting Return Code check")
+ logger.debug(
+ "Status Return Code: %s Result Return Code: %s"
+ % (
+ buildspec_returncode,
+ builder.metadata["result"]["returncode"],
  )
+ )
+ # checks if test returncode matches returncode specified in Buildspec and assign boolean to returncode_match
+ returncode_match = builder.metadata["result"]["returncode"] in buildspec_returncode
+ console.print(
+ f"[blue]{builder}[/]: Checking returncode - {builder.metadata['result']['returncode']} is matched in list {buildspec_returncode}"
+ )
 
  return returncode_match
 
@@ -77,9 +71,6 @@ def runtime_check(builder):
  builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
  """
 
- if not builder.status.get("runtime"):
- return False
-
  min_time = builder.status["runtime"].get("min") or 0
  max_time = builder.status["runtime"].get("max")
 
@@ -177,9 +168,6 @@ def regex_check(builder):
  bool: Returns True if their is a regex match otherwise returns False.
  """
 
- if not builder.status.get("regex"):
- return False
-
  file_stream = None
  if builder.status["regex"]["stream"] == "stdout":
  logger.debug(

buildtest/schemas/definitions.schema.json

@@ -254,7 +254,6 @@
  "EXIT"
  ]
  },
-
  "returncode": { "$ref": "#/definitions/returncode" },
  "regex": {
  "$ref": "#/definitions/regex",
@@ -573,6 +572,11 @@
  "state": {
  "$ref": "#/definitions/state",
  "description": "explicitly mark state of test regardless of status calculation"
+ },
+ "mode": {
+ "type": "string",
+ "description": "Determine how the status check is resolved, for instance it can be logical AND or OR",
+ "enum": ["any", "all"]
  }
  }
  },

buildtest/schemas/examples/script.schema.json/invalid/examples.yml

@@ -407,3 +407,11 @@ buildspecs:
  - dir: foo
  count: 1
  file_traverse_limit: 1000000
+ invalid_value_for_mode:
+ type: script
+ executor: generic.local.bash
+ description: The status mode must be 'any' or 'all'
+ run: exit 0
+ status:
+ returncode: 0
+ mode: '1'

buildtest/schemas/examples/script.schema.json/valid/examples.yml

@@ -703,3 +703,31 @@ buildspecs:
  - dir: foo
  count: 10
  file_traverse_limit: 20
+
+ status_logical_and:
+ type: script
+ executor: 'generic.local.bash'
+ description: 'Using logical AND to check status'
+ run: |
+ echo "This is a test"
+ exit 1
+ status:
+ mode: all
+ returncode: 1
+ regex:
+ stream: stdout
+ exp: 'This is a test'
+
+ status_logical_or:
+ type: script
+ executor: 'generic.local.bash'
+ description: 'Using logical OR to check status'
+ run: |
+ echo "This is a test"
+ exit 1
+ status:
+ mode: any
+ returncode: 0
+ regex:
+ stream: stdout
+ exp: 'This is a test'

docs/buildspecs/buildspec_overview.rst

@@ -245,7 +245,8 @@ Currently, we can match state based on the following:
  - :ref:`Performance Check <perf_checks>`
  - :ref:`Explicit Test Status <explicit_status>`
  - :ref:`File Checks <file_checks>`
- - :ref:`Symbolic Link Check <symlink_check>`
+ - :ref:`File Count <file_count>`
+
 
 .. _returncode:
 
@@ -588,6 +589,28 @@ We can try building this test by running the following:
 
  .. command-output:: buildtest build -b tutorials/test_status/file_count_file_traverse_limit.yml
 
+Status Mode
+~~~~~~~~~~~~~~
+
+By default, the status check performed by buildtest is a logical OR, where if any of the status check is True, then the test will
+PASS. However, if you want to change this behavior to logical AND, you can use the `mode` property. The valid values are
+``or``, ``and``. In the example below, we show two tests that illustrate the use of ``mode``.
+
+.. literalinclude:: ../tutorials/test_status/mode.yml
+ :language: yaml
+ :emphasize-lines: 10,24,25-28
+
+In the first test, we use ``mode: all`` which implies all status check are evaluated logical AND, we expect this test to PASS.
+In the second test, we use ``mode: any`` where status check are evalued as logical OR which is the default behavior. Note if ``mode``
+is not specified, it is equivalent to ``mode: any``. In second test, we expect this to pass because **regex** check will PASS however,
+the **returncode** check will fail. If we had changed this to ``mode: all`` then we would expect this test to fail.
+
+Shown below is the output of running this test.
+
+.. dropdown:: ``buildtest build -b tutorials/test_status/mode.yml``
+
+ .. command-output:: buildtest build -b tutorials/test_status/mode.yml
+
 Skipping test
 -------------