Merge pull request #1730 from buildtesters/specify_regex_type

add support for regular expression types (re.search, re.match, re.fullmatch) in status check and metrics definition

buildtest/builders/base.py

@@ -949,8 +949,13 @@ def add_metrics(self):
  if regex:
  stream = regex.get("stream")
  content = self._output if stream == "stdout" else self._error
- match = re.search(regex["exp"], content, re.MULTILINE)
 
+ if regex.get("re") == "re.match":
+ match = re.match(regex["exp"], content, re.MULTILINE)
+ elif regex.get("re") == "re.fullmatch":
+ match = re.fullmatch(regex["exp"], content, re.MULTILINE)
+ else:
+ match = re.search(regex["exp"], content, re.MULTILINE)
  if match:
  try:
  self.metadata["metrics"][key] = match.group(

buildtest/buildsystem/checks.py

@@ -101,7 +101,6 @@ def file_regex_check(builder):
 
  Args:
  builder (buildtest.builders.base.BuilderBase): An instance of BuilderBase class used for printing the builder name
-
  Returns:
  bool: Returns True if there is a regex match otherwise returns False.
  """
@@ -110,6 +109,8 @@ def file_regex_check(builder):
 
  for file_check in builder.status["file_regex"]:
  fname = file_check["file"]
+ regex_type = file_check.get("re")
+ pattern = file_check["exp"]
  resolved_fname = resolve_path(fname)
  if not resolved_fname:
  msg = f"[blue]{builder}[/]: Unable to resolve file path: {fname}"
@@ -127,13 +128,22 @@ def file_regex_check(builder):
 
  # read file and apply regex
  content = read_file(resolved_fname)
- regex = re.search(file_check["exp"], content)
+ content = content.strip()
+ match = None
+
+ if regex_type == "re.match":
+ match = re.match(pattern, content, re.MULTILINE)
+ elif regex_type == "re.fullmatch":
+ match = re.fullmatch(pattern, content, re.MULTILINE)
+ else:
+ match = re.search(pattern, content, re.MULTILINE)
+
  console.print(
- f"[blue]{builder}[/]: Performing regex expression '{file_check['exp']}' on file {resolved_fname}"
+ f"[blue]{builder}[/]: Performing regex expression '{pattern}' on file {resolved_fname}"
  )
 
- if not regex:
- msg = f"[blue]{builder}[/]: Regular expression: '{file_check['exp']}' is not found in file: {resolved_fname}"
+ if not match:
+ msg = f"[blue]{builder}[/]: Regular expression: '{pattern}' not found in file: {resolved_fname}"
  logger.error(msg)
  console.print(msg, style="red")
  assert_file_regex.append(False)
@@ -166,6 +176,8 @@ def regex_check(builder):
  """
 
  file_stream = None
+ regex_type = builder.status["regex"].get("re")
+ pattern = builder.status["regex"]["exp"]
  if builder.status["regex"]["stream"] == "stdout":
  logger.debug(
  f"Detected regex stream 'stdout' so reading output file: {builder.metadata['outfile']}"
@@ -182,14 +194,20 @@ def regex_check(builder):
 
  file_stream = builder.metadata["errfile"]
 
- logger.debug(f"Applying re.search with exp: {builder.status['regex']['exp']}")
-
- regex = re.search(builder.status["regex"]["exp"], content)
+ logger.debug(f"Applying re.search with exp: {pattern}")
+ # remove any new lines
+ content = content.strip()
+ if regex_type == "re.match":
+ match = re.match(pattern, content, re.MULTILINE)
+ elif regex_type == "re.fullmatch":
+ match = re.fullmatch(pattern, content, re.MULTILINE)
+ else:
+ match = re.search(pattern, content, re.MULTILINE)
 
  console.print(
- f"[blue]{builder}[/]: performing regular expression - '{builder.status['regex']['exp']}' on file: {file_stream}"
+ f"[blue]{builder}[/]: performing regular expression - '{pattern}' on file: {file_stream}"
  )
- if not regex:
+ if not match:
  console.print(f"[blue]{builder}[/]: Regular Expression Match - [red]Failed![/]")
  return False
 

buildtest/schemas/definitions.schema.json

@@ -70,6 +70,20 @@
  "exp": {
  "type": "string",
  "description": "Specify a regular expression to run on the selected file name"
+ },
+ "item": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+ },
+ "re": {
+ "type": "string",
+ "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+ "enum": [
+ "re.search",
+ "re.match",
+ "re.fullmatch"
+ ]
  }
  }
  }
@@ -95,6 +109,15 @@
  "type": "integer",
  "minimum": 0,
  "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+ },
+ "re": {
+ "type": "string",
+ "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+ "enum": [
+ "re.search",
+ "re.match",
+ "re.fullmatch"
+ ]
  }
  }
  },
@@ -123,6 +146,15 @@
  "type": "integer",
  "minimum": 0,
  "description": "Specify the item number used to index element in `match.group() <https://docs.python.org/3/library/re.html#match-objects>`_"
+ },
+ "re": {
+ "type": "string",
+ "description": "Specify the regular expression type, it can be either re.search, re.match, or re.fullmatch. By default it uses re.search",
+ "enum": [
+ "re.search",
+ "re.match",
+ "re.fullmatch"
+ ]
  }
  }
  },

buildtest/schemas/examples/script.schema.json/invalid/invalid_re_value.yml

@@ -0,0 +1,11 @@
+buildspecs:
+ invalid_re_value:
+ type: script
+ executor: generic.local.bash
+ description: The "re" value is invalid
+ run: echo "world"
+ status:
+ regex:
+ stream: stdout
+ exp: "world$"
+ re: "search"

buildtest/schemas/examples/script.schema.json/valid/metrics_with_regex_type.yml

@@ -0,0 +1 @@
+../../../../../tutorials/metrics/metrics_with_regex_type.yml

buildtest/schemas/examples/script.schema.json/valid/specify_regex_type.yml

@@ -0,0 +1 @@
+../../../../../tutorials/test_status/specify_regex_type.yml

docs/writing_buildspecs/performance_checks.rst

@@ -52,6 +52,37 @@ join them together into a single string. Shown below is the metrics for the prev
  .. command-output:: buildtest report --filter buildspec=tutorials/metrics/metrics_regex.yml --format name,metrics
 
 
+Metrics with Regex Type via 're'
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Building on the previous example, we will use the ``re`` property specify the regular expression type to use. By default, buildtest will
+use `re.search <https://docs.python.org/3/library/re.html#re.search>`_ if **re** is not specified; however you can specify **re** to use `re.match <https://docs.python.org/3/library/re.html#re.match>`_,
+`re.fullmatch <https://docs.python.org/3/library/re.html#re.fullmatch>`_, or `re.search <https://docs.python.org/3/library/re.html#re.search>`_.
+
+In this example, we will define 4 metrics **hpcg_text**, **hpcg_result**, **hpcg_file_text**, **hpcg_file_result**. The first two
+metrics will capture from stdout using the ``regex`` property while the last two will capture from a file using the ``file_regex`` property.
+The ``re.match`` will be used to capture the text **HPCG result is VALID** and **HPCG result is INVALID** from stdout and file, whereas
+the ``re.search`` will be used to capture the test result **63.6515** and **28.1215** from stdout and file.
+
+Finally, we will use the comparison operator :ref:`assert_eq` to compare the metrics with reference value.
+
+.. literalinclude:: ../tutorials/metrics/metrics_with_regex_type.yml
+ :language: yaml
+ :emphasize-lines: 7-45
+
+Let's attempt to build this test
+
+.. dropdown:: ``buildtest build -b tutorials/metrics/metrics_with_regex_type.yml``
+
+ .. command-output:: buildtest build -b tutorials/metrics/metrics_with_regex_type.yml
+
+Upon completion, lets take a look at the metrics for this test, we can see this by running ``buildtest inspect query``
+which shows the name of captured metrics and its corresponding values.
+
+.. dropdown:: ``buildtest inspect query metric_regex_example_with_re``
+
+ .. command-output:: buildtest inspect query metric_regex_example_with_re
+
 Invalid Metrics
 ~~~~~~~~~~~~~~~~~
 

docs/writing_buildspecs/status_check.rst

@@ -87,3 +87,33 @@ We will simply try validating this buildspec and you will see the error message
 
  .. command-output:: buildtest buildspec validate -b tutorials/test_status/file_linecount_invalid.yml
  :returncode: 1
+
+.. _re:
+
+Using 're' property to specify regular expression type
+------------------------------------------------------
+
+The ``re`` property can be used to select the type of regular expression to use with :ref:`regex <regex>` or :ref:`file_regex <file_regex>`
+which can be `re.search <https://docs.python.org/3/library/re.html#re.search>`_, `re.match <https://docs.python.org/3/library/re.html#re.match>`_
+or `re.fullmatch <https://docs.python.org/3/library/re.html#re.fullmatch>`_. The ``re`` property is a string type which is used to select the
+regular expression function to use. In this next example, we will demonstrate the use of this feature with both ``regex`` and ``file_regex``.
+The ``re`` property is optional and if not specified it defaults to **re.search**.
+
+Since **re.search** will search for text at any position in the string, the first test ``re.search.stdout`` will match the
+string **is** with the output. In the second test ``re.match.stdout`` we use **re.match** which matches from beginning of
+string with input pattern **is** with output. We expect this match to **FAIL** since the output starts with **This is ...**.
+
+In the third test ``re.fullmatch.stdout`` we set ``re: re.fullmatch`` which will match the entire string with the pattern.
+We expect this match to **PASS** since the output and pattern are exactly the same. In the fourth test ``match_on_file_regex`` we have
+have three regular expression, one for each type **re.search**, **re.match** and **re.fullmatch**. All of these expressions will find a match
+and this test will **PASS**.
+
+.. literalinclude:: ../tutorials/test_status/specify_regex_type.yml
+ :language: yaml
+ :emphasize-lines: 6,8-11,17,19-22,28,30-33,39-41,43-52
+
+Let's try running this test example and see the generated output, all test should pass with exception of ``re.match.stdout``.
+
+.. dropdown:: ``buildtest build -b tutorials/test_status/specify_regex_type.yml``
+
+ .. command-output:: buildtest build -b tutorials/test_status/specify_regex_type.yml

tests/builders/metrics_with_regex_type.yml

@@ -0,0 +1 @@
+../../tutorials/metrics/metrics_with_regex_type.yml

tests/builders/specify_regex_type.yml

@@ -0,0 +1 @@
+../../tutorials/test_status/specify_regex_type.yml

tests/builders/test_builders.py

@@ -178,6 +178,16 @@ def test_file_regex():
  cmd.build()
 
 
+def test_regex_type():
+ """This test will perform status check with different regular expression type using ``re`` property that can be "re.match", "re.search", "re.fullmatch" """
+ cmd = BuildTest(
+ buildspecs=[os.path.join(here, "specify_regex_type.yml")],
+ buildtest_system=system,
+ configuration=config,
+ )
+ cmd.build()
+
+
 def test_runtime_check():
  """This test will perform status check with runtime"""
  cmd = BuildTest(
@@ -217,3 +227,13 @@ def test_file_linecount():
  )
  with pytest.raises(SystemExit):
  cmd.build()
+
+
+def test_metrics_with_regex_type():
+ """This test will perform status check with regular expression type and metrics"""
+ cmd = BuildTest(
+ buildspecs=[os.path.join(here, "metrics_with_regex_type.yml")],
+ buildtest_system=system,
+ configuration=config,
+ )
+ cmd.build()

tutorials/metrics/metrics_with_regex_type.yml

@@ -0,0 +1,45 @@
+buildspecs:
+ metric_regex_example_with_re:
+ executor: generic.local.bash
+ type: script
+ description: capture metric with different regex types
+ tags: tutorials
+ run: |
+ echo "HPCG result is VALID with a GFLOP/s rating of=63.6515"
+ echo "HPCG result is INVALID with a GFLOP/s rating of=28.1215" > hpcg.txt
+ metrics:
+ hpcg_result:
+ type: float
+ regex:
+ re: "re.search"
+ exp: '(\d+\.\d+)$'
+ stream: stdout
+ hpcg_text:
+ type: str
+ regex:
+ re: "re.match"
+ exp: '^HPCG result is VALID'
+ stream: stdout
+ hpcg_file_text:
+ type: str
+ file_regex:
+ re: "re.match"
+ exp: '^HPCG result is INVALID'
+ file: hpcg.txt
+ hpcg_file_result:
+ type: float
+ file_regex:
+ re: "re.search"
+ exp: '(\d+\.\d+)$'
+ file: hpcg.txt
+ status:
+ assert_eq:
+ comparisons:
+ - name: hpcg_text
+ ref: "HPCG result is VALID"
+ - name: hpcg_result
+ ref: 63.6515
+ - name: hpcg_file_text
+ ref: "HPCG result is INVALID"
+ - name: hpcg_file_result
+ ref: 28.1215

tutorials/test_status/specify_regex_type.yml

@@ -0,0 +1,52 @@
+buildspecs:
+ re.search.stdout:
+ executor: generic.local.bash
+ type: script
+ description: Test re.search on stdout
+ run: echo "This is a string"
+ status:
+ regex:
+ stream: stdout
+ exp: 'is'
+ re: "re.search"
+
+ re.match.stdout:
+ executor: generic.local.bash
+ type: script
+ description: Test re.match on stdout
+ run: echo "This is a string"
+ status:
+ regex:
+ stream: stdout
+ exp: 'is'
+ re: "re.match"
+
+ re.fullmatch.stdout:
+ executor: generic.local.bash
+ type: script
+ description: Test re.fullmatch on stdout
+ run: echo "This is a string"
+ status:
+ regex:
+ stream: stdout
+ exp: 'This is a string'
+ re: "re.fullmatch"
+
+ re.match_on_file_regex:
+ executor: generic.local.bash
+ type: script
+ description: Test re.match on file regex
+ run: |
+ echo "This is a string" > file.txt
+ echo "Hello World" > hello.txt
+ status:
+ file_regex:
+ - file: file.txt
+ exp: 'string'
+ re: "re.search"
+ - file: hello.txt
+ exp: 'Hello'
+ re: "re.match"
+ - file: hello.txt
+ exp: 'Hello World'
+ re: "re.fullmatch"