From dbb9fe1e86ec7de774273ff2d06c63ce043d983a Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 14:27:38 -0400 Subject: [PATCH 1/8] add support for property 'mode' under status check to allow for one to specify status check to be done as a 'or' 'and' which will be a logical OR/AND. add an example buildspec to demonstrate the example --- buildtest/schemas/definitions.schema.json | 6 ++++- tutorials/test_status/mode.yml | 28 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tutorials/test_status/mode.yml diff --git a/buildtest/schemas/definitions.schema.json b/buildtest/schemas/definitions.schema.json index 13dc3d564..746f2eb9d 100644 --- a/buildtest/schemas/definitions.schema.json +++ b/buildtest/schemas/definitions.schema.json @@ -254,7 +254,6 @@ "EXIT" ] }, - "returncode": { "$ref": "#/definitions/returncode" }, "regex": { "$ref": "#/definitions/regex", @@ -573,6 +572,11 @@ "state": { "$ref": "#/definitions/state", "description": "explicitly mark state of test regardless of status calculation" + }, + "mode": { + "type": "string", + "description": "Determine how the status check is resolved, for instance it can be logical AND or OR", + "enum": ["or", "and"] } } }, diff --git a/tutorials/test_status/mode.yml b/tutorials/test_status/mode.yml new file mode 100644 index 000000000..215c44970 --- /dev/null +++ b/tutorials/test_status/mode.yml @@ -0,0 +1,28 @@ +buildspecs: + status_logical_and: + type: script + executor: 'generic.local.bash' + description: 'Using logical AND to check status' + run: | + echo "This is a test" + exit 1 + status: + mode: and + returncode: 1 + regex: + stream: stdout + exp: 'This is a test' + + status_logical_or: + type: script + executor: 'generic.local.bash' + description: 'Using logical OR to check status' + run: | + echo "This is a test" + exit 1 + status: + mode: or + returncode: 0 + regex: + stream: stdout + exp: 'This is a test' \ No newline at end of file From 298cc2faf175b7a05c4a91901e9388dfcd7596b1 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 15:38:57 -0400 Subject: [PATCH 2/8] make use of 'self.metadata['check']' to keep track of all status checks. We assign all status check into a dictionary as a lookup table. Refactored the codebase for status check logic and added support for logical AND when using 'mode: and' --- buildtest/builders/base.py | 149 +++++++++++++++----------------- buildtest/buildsystem/checks.py | 50 ++++------- 2 files changed, 87 insertions(+), 112 deletions(-) diff --git a/buildtest/builders/base.py b/buildtest/builders/base.py index a57047522..ccf53d77e 100644 --- a/buildtest/builders/base.py +++ b/buildtest/builders/base.py @@ -231,8 +231,29 @@ def _set_metadata_values(self): self.metadata["compiler"] = None self.metadata["result"] = {"state": "N/A", "returncode": "-1", "runtime": 0} - self.metadata["check"] = {"returncode": "N/A", "regex": "N/A", "runtime": "N/A"} - + status_check_names = [ + "regex", + "returncode", + "runtime", + "file_regex", + "slurm_job_state", + "pbs_job_state", + "lsf_job_state", + "assert_ge", + "assert_gt", + "assert_le", + "assert_lt", + "assert_eq", + "assert_ne", + "contains", + "not_contains", + "is_symlink", + "exists", + "is_dir", + "is_file", + "file_count", + ] + self.metadata["check"] = {name: None for name in status_check_names} self.metadata["metrics"] = {} # used to store job id from batch scheduler @@ -985,128 +1006,94 @@ def check_test_state(self): # if status is defined in Buildspec, then check for returncode and regex if self.status: - slurm_job_state_match = False - pbs_job_state_match = False - lsf_job_state_match = False - assert_ge_match = False - assert_le_match = False - assert_gt_match = False - assert_lt_match = False - assert_eq_match = False - assert_ne_match = False - assert_range_match = False - assert_contains_match = False - assert_notcontains_match = False - assert_is_symlink = False - assert_exists = False - assert_is_dir = False - assert_is_file = False - file_regex_match = False - assert_file_count = False - # returncode_match is boolean to check if reference returncode matches return code from test - returncode_match = returncode_check(self) + # if 'state' property is specified explicitly honor this value regardless of what is calculated + if self.status.get("state"): + self.metadata["result"]["state"] = self.status["state"] + return - # check regex against output or error stream based on regular expression - # defined in status property. Return value is a boolean - regex_match = regex_check(self) + # returncode_match is boolean to check if reference returncode matches return code from test + if self.status.get("returncode"): + self.metadata["check"]["returncode"] = returncode_check(self) - runtime_match = runtime_check(self) + # check regex against output or error stream based on regular expression defined in status property. Return value is a boolean + if self.status.get("regex"): + self.metadata["check"]["regex"] = regex_check(self) - self.metadata["check"]["regex"] = regex_match - self.metadata["check"]["runtime"] = runtime_match - self.metadata["check"]["returncode"] = returncode_match + if self.status.get("runtime"): + self.metadata["check"]["runtime"] = runtime_check(self) if self.status.get("file_regex"): - file_regex_match = file_regex_check(self) + self.metadata["check"]["file_regex"] = file_regex_check(self) if self.status.get("slurm_job_state") and isinstance(self.job, SlurmJob): - slurm_job_state_match = ( + self.metadata["check"]["slurm_job_state"] = ( self.status["slurm_job_state"] == self.job.state() ) if self.status.get("pbs_job_state") and isinstance(self.job, PBSJob): - pbs_job_state_match = self.status["pbs_job_state"] == self.job.state() + self.metadata["check"]["pbs_job_state"] = ( + self.status["pbs_job_state"] == self.job.state() + ) if self.status.get("lsf_job_state") and isinstance(self.job, LSFJob): - lsf_job_state_match = self.status["lsf_job_state"] == self.job.state() + self.metadata["check"]["lsf_job_state"] = ( + self.status["lsf_job_state"] == self.job.state() + ) if self.status.get("assert_ge"): - assert_ge_match = assert_ge_check(self) + self.metadata["check"]["assert_ge"] = assert_ge_check(self) if self.status.get("assert_le"): - assert_le_match = assert_le_check(self) + self.metadata["check"]["assert_le"] = assert_le_check(self) if self.status.get("assert_gt"): - assert_gt_match = assert_gt_check(self) + self.metadata["check"]["assert_gt"] = assert_gt_check(self) if self.status.get("assert_lt"): - assert_lt_match = assert_lt_check(self) + self.metadata["check"]["assert_lt"] = assert_lt_check(self) if self.status.get("assert_eq"): - assert_eq_match = assert_eq_check(self) + self.metadata["check"]["assert_eq"] = assert_eq_check(self) if self.status.get("assert_ne"): - assert_ne_match = assert_ne_check(self) + self.metadata["check"]["assert_ne"] = assert_ne_check(self) if self.status.get("assert_range"): - assert_range_match = assert_range_check(self) + self.metadata["check"]["assert_range"] = assert_range_check(self) if self.status.get("contains"): - assert_contains_match = contains_check(self) + self.metadata["check"]["contains"] = contains_check(self) if self.status.get("not_contains"): - assert_notcontains_match = notcontains_check(self) + self.metadata["check"]["not_contains"] = notcontains_check(self) if self.status.get("is_symlink"): - assert_is_symlink = is_symlink_check(builder=self) + self.metadata["check"]["is_symlink"] = is_symlink_check(builder=self) if self.status.get("exists"): - assert_exists = exists_check(builder=self) + self.metadata["check"]["exists"] = exists_check(builder=self) if self.status.get("is_dir"): - assert_is_dir = is_dir_check(builder=self) + self.metadata["check"]["is_dir"] = is_dir_check(builder=self) if self.status.get("is_file"): - assert_is_file = is_file_check(builder=self) + self.metadata["check"]["is_file"] = is_file_check(builder=self) if self.status.get("file_count"): - assert_file_count = file_count_check(builder=self) - - # if any of checks is True we set the 'state' to PASS - state = any( - [ - returncode_match, - regex_match, - file_regex_match, - slurm_job_state_match, - pbs_job_state_match, - lsf_job_state_match, - runtime_match, - assert_ge_match, - assert_le_match, - assert_gt_match, - assert_lt_match, - assert_eq_match, - assert_ne_match, - assert_range_match, - assert_contains_match, - assert_notcontains_match, - assert_is_symlink, - assert_exists, - assert_is_dir, - assert_is_file, - assert_file_count, - ] - ) - if state: - self.metadata["result"]["state"] = "PASS" - else: - self.metadata["result"]["state"] = "FAIL" + self.metadata["check"]["file_count"] = file_count_check(builder=self) - # if 'state' property is specified explicitly honor this value regardless of what is calculated - if self.status.get("state"): - self.metadata["result"]["state"] = self.status["state"] + # filter out any None values from status check + status_checks = [ + value for value in self.metadata["check"].values() if value is not None + ] + + state = ( + all(status_checks) + if self.status.get("mode") == "and" + else any(status_checks) + ) + self.metadata["result"]["state"] = "PASS" if state else "FAIL" def _process_compiler_config(self): """This method is responsible for setting cc, fc, cxx class variables based diff --git a/buildtest/buildsystem/checks.py b/buildtest/buildsystem/checks.py index aa049846a..dcd867c1d 100644 --- a/buildtest/buildsystem/checks.py +++ b/buildtest/buildsystem/checks.py @@ -39,32 +39,26 @@ def returncode_check(builder): builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name """ - returncode_match = False - - # if 'returncode' field set for 'status' check the returncode if its not set we return False - if "returncode" in builder.status.keys(): - # returncode can be an integer or list of integers - buildspec_returncode = builder.status["returncode"] - - # if buildspec returncode field is integer we convert to list for check - if isinstance(buildspec_returncode, int): - buildspec_returncode = [buildspec_returncode] - - logger.debug("Conducting Return Code check") - logger.debug( - "Status Return Code: %s Result Return Code: %s" - % ( - buildspec_returncode, - builder.metadata["result"]["returncode"], - ) - ) - # checks if test returncode matches returncode specified in Buildspec and assign boolean to returncode_match - returncode_match = ( - builder.metadata["result"]["returncode"] in buildspec_returncode - ) - console.print( - f"[blue]{builder}[/]: Checking returncode - {builder.metadata['result']['returncode']} is matched in list {buildspec_returncode}" + # returncode can be an integer or list of integers + buildspec_returncode = builder.status["returncode"] + + # if buildspec returncode field is integer we convert to list for check + if isinstance(buildspec_returncode, int): + buildspec_returncode = [buildspec_returncode] + + logger.debug("Conducting Return Code check") + logger.debug( + "Status Return Code: %s Result Return Code: %s" + % ( + buildspec_returncode, + builder.metadata["result"]["returncode"], ) + ) + # checks if test returncode matches returncode specified in Buildspec and assign boolean to returncode_match + returncode_match = builder.metadata["result"]["returncode"] in buildspec_returncode + console.print( + f"[blue]{builder}[/]: Checking returncode - {builder.metadata['result']['returncode']} is matched in list {buildspec_returncode}" + ) return returncode_match @@ -77,9 +71,6 @@ def runtime_check(builder): builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name """ - if not builder.status.get("runtime"): - return False - min_time = builder.status["runtime"].get("min") or 0 max_time = builder.status["runtime"].get("max") @@ -177,9 +168,6 @@ def regex_check(builder): bool: Returns True if their is a regex match otherwise returns False. """ - if not builder.status.get("regex"): - return False - file_stream = None if builder.status["regex"]["stream"] == "stdout": logger.debug( From b2fcf63f1dcdfe8ecf9e33518de60b2a3d055ecc Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 15:44:07 -0400 Subject: [PATCH 3/8] add example buildspecs for testing 'mode' for valid and invalid use-case --- .../script.schema.json/invalid/examples.yml | 8 ++++++ .../script.schema.json/valid/examples.yml | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml index c8ae4a8ca..ffe8c8f6d 100644 --- a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml @@ -407,3 +407,11 @@ buildspecs: - dir: foo count: 1 file_traverse_limit: 1000000 + invalid_value_for_mode: + type: script + executor: generic.local.bash + description: The status mode must be 'or' or 'and' + run: exit 0 + status: + returncode: 0 + mode: 'any' \ No newline at end of file diff --git a/buildtest/schemas/examples/script.schema.json/valid/examples.yml b/buildtest/schemas/examples/script.schema.json/valid/examples.yml index afa83d94a..b0359f5ba 100644 --- a/buildtest/schemas/examples/script.schema.json/valid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/valid/examples.yml @@ -703,3 +703,31 @@ buildspecs: - dir: foo count: 10 file_traverse_limit: 20 + + status_logical_and: + type: script + executor: 'generic.local.bash' + description: 'Using logical AND to check status' + run: | + echo "This is a test" + exit 1 + status: + mode: and + returncode: 1 + regex: + stream: stdout + exp: 'This is a test' + + status_logical_or: + type: script + executor: 'generic.local.bash' + description: 'Using logical OR to check status' + run: | + echo "This is a test" + exit 1 + status: + mode: or + returncode: 0 + regex: + stream: stdout + exp: 'This is a test' \ No newline at end of file From 6da6b85b95f3328b552b9787cdb5c8847e626aff Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 15:54:07 -0400 Subject: [PATCH 4/8] fix issue with yamllint --- .../schemas/examples/script.schema.json/invalid/examples.yml | 2 +- .../schemas/examples/script.schema.json/valid/examples.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml index ffe8c8f6d..b90f8b571 100644 --- a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml @@ -414,4 +414,4 @@ buildspecs: run: exit 0 status: returncode: 0 - mode: 'any' \ No newline at end of file + mode: 'any' diff --git a/buildtest/schemas/examples/script.schema.json/valid/examples.yml b/buildtest/schemas/examples/script.schema.json/valid/examples.yml index b0359f5ba..2468cc241 100644 --- a/buildtest/schemas/examples/script.schema.json/valid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/valid/examples.yml @@ -730,4 +730,4 @@ buildspecs: returncode: 0 regex: stream: stdout - exp: 'This is a test' \ No newline at end of file + exp: 'This is a test' From d6e638df8e1f9c2b48f02fcb2304133f1b23aeca Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 16:07:43 -0400 Subject: [PATCH 5/8] add documentation section on status mode --- docs/buildspecs/buildspec_overview.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/buildspecs/buildspec_overview.rst b/docs/buildspecs/buildspec_overview.rst index df0f5e7c1..10f80167c 100644 --- a/docs/buildspecs/buildspec_overview.rst +++ b/docs/buildspecs/buildspec_overview.rst @@ -245,7 +245,8 @@ Currently, we can match state based on the following: - :ref:`Performance Check ` - :ref:`Explicit Test Status ` - :ref:`File Checks ` - - :ref:`Symbolic Link Check ` + - :ref:`File Count ` + .. _returncode: @@ -588,6 +589,28 @@ We can try building this test by running the following: .. command-output:: buildtest build -b tutorials/test_status/file_count_file_traverse_limit.yml +Status Mode +~~~~~~~~~~~~~~ + +By default, the status check performed by buildtest is a logical OR, where if any of the status check is True, then the test will +PASS. However, if you want to change this behavior to logical AND, you can use the `mode` property. The valid values are +``or``, ``and``. In the example below, we show two tests that illustrate the use of ``mode``. + +.. literalinclude:: ../tutorials/test_status/mode.yml + :language: yaml + :emphasize-lines: 10,24,25-28 + +In the first test, we use ``mode: and`` which implies all status check are evaluated logical AND, we expect this test to PASS. +In the second test, we use ``mode: or`` where status check are evalued as logical OR which is the default behavior. Note if ``mode`` +is not specified, it is equivalent to ``mode: or``. In second test, we expect this to pass because **regex** check will PASS however, +the **returncode** check will fail. If we had changed this to ``mode: and`` then we would expect this test to fail. + +Shown below is the output of running this test. + +.. dropdown:: ``buildtest build -b tutorials/test_status/mode.yml`` + + .. command-output:: buildtest build -b tutorials/test_status/mode.yml + Skipping test ------------- From 62b0df85c0acc38d641a52cf7b00e3e3b5bc10b9 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Wed, 22 Mar 2023 16:13:20 -0400 Subject: [PATCH 6/8] change the enum from 'or', 'and' to 'any' and 'all' for the mode. update this change in buildspec examples, documentation, implementation --- buildtest/builders/base.py | 2 +- buildtest/schemas/definitions.schema.json | 2 +- .../examples/script.schema.json/invalid/examples.yml | 4 ++-- .../examples/script.schema.json/valid/examples.yml | 4 ++-- docs/buildspecs/buildspec_overview.rst | 8 ++++---- tutorials/test_status/mode.yml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/buildtest/builders/base.py b/buildtest/builders/base.py index ccf53d77e..fcd540d9a 100644 --- a/buildtest/builders/base.py +++ b/buildtest/builders/base.py @@ -1090,7 +1090,7 @@ def check_test_state(self): state = ( all(status_checks) - if self.status.get("mode") == "and" + if self.status.get("mode") == "all" else any(status_checks) ) self.metadata["result"]["state"] = "PASS" if state else "FAIL" diff --git a/buildtest/schemas/definitions.schema.json b/buildtest/schemas/definitions.schema.json index 746f2eb9d..8494a1c04 100644 --- a/buildtest/schemas/definitions.schema.json +++ b/buildtest/schemas/definitions.schema.json @@ -576,7 +576,7 @@ "mode": { "type": "string", "description": "Determine how the status check is resolved, for instance it can be logical AND or OR", - "enum": ["or", "and"] + "enum": ["any", "all"] } } }, diff --git a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml index b90f8b571..d6452b7cd 100644 --- a/buildtest/schemas/examples/script.schema.json/invalid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/invalid/examples.yml @@ -410,8 +410,8 @@ buildspecs: invalid_value_for_mode: type: script executor: generic.local.bash - description: The status mode must be 'or' or 'and' + description: The status mode must be 'any' or 'all' run: exit 0 status: returncode: 0 - mode: 'any' + mode: '1' diff --git a/buildtest/schemas/examples/script.schema.json/valid/examples.yml b/buildtest/schemas/examples/script.schema.json/valid/examples.yml index 2468cc241..ec81d6f2f 100644 --- a/buildtest/schemas/examples/script.schema.json/valid/examples.yml +++ b/buildtest/schemas/examples/script.schema.json/valid/examples.yml @@ -712,7 +712,7 @@ buildspecs: echo "This is a test" exit 1 status: - mode: and + mode: all returncode: 1 regex: stream: stdout @@ -726,7 +726,7 @@ buildspecs: echo "This is a test" exit 1 status: - mode: or + mode: any returncode: 0 regex: stream: stdout diff --git a/docs/buildspecs/buildspec_overview.rst b/docs/buildspecs/buildspec_overview.rst index 10f80167c..cb14557cc 100644 --- a/docs/buildspecs/buildspec_overview.rst +++ b/docs/buildspecs/buildspec_overview.rst @@ -600,10 +600,10 @@ PASS. However, if you want to change this behavior to logical AND, you can use t :language: yaml :emphasize-lines: 10,24,25-28 -In the first test, we use ``mode: and`` which implies all status check are evaluated logical AND, we expect this test to PASS. -In the second test, we use ``mode: or`` where status check are evalued as logical OR which is the default behavior. Note if ``mode`` -is not specified, it is equivalent to ``mode: or``. In second test, we expect this to pass because **regex** check will PASS however, -the **returncode** check will fail. If we had changed this to ``mode: and`` then we would expect this test to fail. +In the first test, we use ``mode: all`` which implies all status check are evaluated logical AND, we expect this test to PASS. +In the second test, we use ``mode: any`` where status check are evalued as logical OR which is the default behavior. Note if ``mode`` +is not specified, it is equivalent to ``mode: any``. In second test, we expect this to pass because **regex** check will PASS however, +the **returncode** check will fail. If we had changed this to ``mode: all`` then we would expect this test to fail. Shown below is the output of running this test. diff --git a/tutorials/test_status/mode.yml b/tutorials/test_status/mode.yml index 215c44970..60e42cac8 100644 --- a/tutorials/test_status/mode.yml +++ b/tutorials/test_status/mode.yml @@ -7,7 +7,7 @@ buildspecs: echo "This is a test" exit 1 status: - mode: and + mode: all returncode: 1 regex: stream: stdout @@ -21,7 +21,7 @@ buildspecs: echo "This is a test" exit 1 status: - mode: or + mode: any returncode: 0 regex: stream: stdout From 63c5da3351d24416b3f5e1b3ac334f82e1fc3d5f Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 23 Mar 2023 11:02:50 -0400 Subject: [PATCH 7/8] remove comment --- buildtest/builders/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/buildtest/builders/base.py b/buildtest/builders/base.py index fcd540d9a..fce5abd8b 100644 --- a/buildtest/builders/base.py +++ b/buildtest/builders/base.py @@ -1012,7 +1012,6 @@ def check_test_state(self): self.metadata["result"]["state"] = self.status["state"] return - # returncode_match is boolean to check if reference returncode matches return code from test if self.status.get("returncode"): self.metadata["check"]["returncode"] = returncode_check(self) From 3e634d6a82edc59a4a373e0d3e55e3b1952a3981 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 23 Mar 2023 11:09:53 -0400 Subject: [PATCH 8/8] update few sentences in docs --- docs/buildspecs/buildspec_overview.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/buildspecs/buildspec_overview.rst b/docs/buildspecs/buildspec_overview.rst index cb14557cc..aa859e716 100644 --- a/docs/buildspecs/buildspec_overview.rst +++ b/docs/buildspecs/buildspec_overview.rst @@ -594,16 +594,17 @@ Status Mode By default, the status check performed by buildtest is a logical OR, where if any of the status check is True, then the test will PASS. However, if you want to change this behavior to logical AND, you can use the `mode` property. The valid values are -``or``, ``and``. In the example below, we show two tests that illustrate the use of ``mode``. +``any``, ``all``. In the example below, we have two tests that illustrate the use of ``mode``. .. literalinclude:: ../tutorials/test_status/mode.yml :language: yaml :emphasize-lines: 10,24,25-28 -In the first test, we use ``mode: all`` which implies all status check are evaluated logical AND, we expect this test to PASS. +The first test uses ``mode: all`` which implies all status check are evaluated as logical AND, we expect this test to PASS. In the second test, we use ``mode: any`` where status check are evalued as logical OR which is the default behavior. Note if ``mode`` is not specified, it is equivalent to ``mode: any``. In second test, we expect this to pass because **regex** check will PASS however, -the **returncode** check will fail. If we had changed this to ``mode: all`` then we would expect this test to fail. +the **returncode** check will fail due to mismatch in returncode. If we changed this to ``mode: all`` then we would expect this test +to fail. Shown below is the output of running this test.