From cd785cd8e5519320affaf5f29419d26dc6379c47 Mon Sep 17 00:00:00 2001
From: Gerald Combs <gerald@wireshark.org>
Date: Thu, 28 Nov 2024 11:47:57 -0800
Subject: [PATCH] tools+GitLab CI: Add a test report wrapper

Add wrap-test.py, which runs a CI test command and adds it to a JUnit
report which can be consumed by GitLab CI.

Have the script "tee" output so that errors and warnings show up in CI
job output and in the merge request test report.
---
 .gitlab-ci.yml        |  87 ++++------------------------
 tools/wrap-ci-test.py | 129 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 139 insertions(+), 77 deletions(-)
 create mode 100755 tools/wrap-ci-test.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5cfaf6e6e06..b3fe25c8672 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -753,83 +753,16 @@ Commit Check:
     #   https://docs.gitlab.com/ee/ci/testing/unit_test_reports.html
     #   https://www.ibm.com/docs/en/developer-for-zos/14.2?topic=formats-junit-xml-format
     #
-    # Let's generate a unit test report, even if that means wondering which life choices
-    # led us to assemlbing an XML document from fragments in a shell script which is
-    # embedded in YAML.
-    - FAILURE_COUNT=0
-    - TESTCASES="<testcase class='commit_checks.num_commits' name='num_commits' time='1.0'"
-    - |
-      if [[ $NUM_COMMITS -gt 1 ]] ; then
-        TESTCASES+="><failure><![CDATA[ℹ️ This merge request has more than one commit. Please squash any trivial ones:
-
-        $(git log --oneline --no-decorate "${CI_COMMIT_SHA}~$NUM_COMMITS..${CI_COMMIT_SHA}")
-        ]]></failure></testcase>"
-        FAILURE_COUNT=$(( FAILURE_COUNT + 1 ))
-      else
-        TESTCASES+="/>"
-      fi
-    - PC_EXIT_CODE=0
-    - ANALYSIS_MESSAGE=$( ./tools/pre-commit "${CI_COMMIT_SHA}~$NUM_COMMITS" ) || PC_EXIT_CODE=1
-    - TESTCASES+="<testcase class='commit_checks.pre_commit' name='tools/pre-commit' time='1.0'"
-    - |
-      if [ -n "$ANALYSIS_MESSAGE" ] ; then
-        TESTCASES+="><failure><![CDATA[Pre-commit check results:
-
-        $ANALYSIS_MESSAGE
-        ]]></failure></testcase>"
-        FAILURE_COUNT=$(( FAILURE_COUNT + 1 ))
-      else
-        TESTCASES+="/>"
-      fi
-    - VC_EXIT_CODE=0
-    - ANALYSIS_MESSAGE=$( tools/validate-commit.py ) || VC_EXIT_CODE=1
-    - TESTCASES+="<testcase class='commit_checks.validate_commit' name='tools/validate-commit.py' time='1.0'"
-    - |
-      if [[ $VC_EXIT_CODE != 0 ]] ; then
-        TESTCASES+="><failure><![CDATA[Validate-commit check results:
-
-        $ANALYSIS_MESSAGE
-        ]]></failure></testcase>"
-        FAILURE_COUNT=$(( FAILURE_COUNT + 1 ))
-      else
-        TESTCASES+="/>"
-      fi
-    - LC_EXIT_CODE=0
-    - ANALYSIS_MESSAGE=$( python3 tools/checklicenses.py ) || LC_EXIT_CODE=1
-    - TESTCASES+="<testcase class='commit_checks.check_licenses' name='tools/checklicenses.py' time='1.0'"
-    - |
-      if [[ $LC_EXIT_CODE != 0 ]] ; then
-        TESTCASES+="><failure><![CDATA[License check failure:
-
-        $ANALYSIS_MESSAGE
-        ]]></failure></testcase>"
-        FAILURE_COUNT=$(( FAILURE_COUNT + 1 ))
-      else
-        TESTCASES+="/>"
-      fi
-    - HC_EXIT_CODE=0
-    - ANALYSIS_MESSAGE=$( python3 tools/check_help_urls.py ) || HC_EXIT_CODE=1
-    - TESTCASES+="<testcase class='commit_checks.check_help_urls' name='tools/check_help_urls.py' time='1.0'"
-    - |
-      if [[ $HC_EXIT_CODE != 0 ]] ; then
-        TESTCASES+="><failure><![CDATA[Help URL check failure:
-
-        $ANALYSIS_MESSAGE
-        ]]></failure></testcase>"
-        FAILURE_COUNT=$(( FAILURE_COUNT + 1 ))
-      else
-        TESTCASES+="/>"
-      fi
-    - |
-      cat > commit_checks.xml <<FIN
-      <?xml version="1.0" encoding="utf-8"?>
-      <testsuites><testsuite name="Commit Checks" errors="0" failures="$FAILURE_COUNT" skipped="0" tests="5" time="5">
-      $TESTCASES
-      </testsuite></testsuites>
-      FIN
-    # - cat commit_checks.xml
-    - grep '<failure' commit_checks.xml && xmllint --format --xpath '//*/failure' commit_checks.xml
-    - exit $(( PC_EXIT_CODE || VC_EXIT_CODE || LC_EXIT_CODE || HC_EXIT_CODE ))
+    # Let's generate a unit test report using tools/wrap-ci-test.py.
+    - REPORT_FILE=commit_checks.xml
+    - TEST_SUITE=commit_checks
+    - EXIT_CODE=0
+    - ./tools/wrap-ci-test.py --file $REPORT_FILE --suite $TEST_SUITE --case num_commits --command "[ $NUM_COMMITS -lt 2 ] || (echo 'ℹ️ This merge request has more than one commit. Please squash any trivial ones:\n'; git log --oneline --no-decorate --max-count=$NUM_COMMITS ; false)" || true
+    - for COMMIT in $( git log --max-count=$NUM_COMMITS --pretty=format:%h ) ; do ./tools/wrap-ci-test.py --file $REPORT_FILE --suite $TEST_SUITE --case tools/pre-commit/$COMMIT ./tools/pre-commit $COMMIT || EXIT_CODE=1 ; done
+    - ./tools/wrap-ci-test.py --file $REPORT_FILE --suite $TEST_SUITE --case tools/validate-commit ./tools/validate-commit.py $( git log --max-count=$NUM_COMMITS --pretty=format:%h ) || EXIT_CODE=1
+    - ./tools/wrap-ci-test.py --file $REPORT_FILE --suite $TEST_SUITE --case tools/checklicenses ./tools/checklicenses.py || EXIT_CODE=1
+    - ./tools/wrap-ci-test.py --file $REPORT_FILE --suite $TEST_SUITE --case tools/check_help_urls ./tools/check_help_urls.py || EXIT_CODE=1
+    - exit $EXIT_CODE
   artifacts:
     when: always
     paths:
diff --git a/tools/wrap-ci-test.py b/tools/wrap-ci-test.py
new file mode 100755
index 00000000000..e9403ba0e8b
--- /dev/null
+++ b/tools/wrap-ci-test.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+#
+# Add arbritrary commands to a GitLab CI compatible (JUnit) test report
+# SPDX-License-Identifier: MIT
+#
+# Usage:
+#   wrap-ci-test --file foo.xml --suite "Suite" --case "Name" --command "command"
+#   wrap-ci-test --file foo.xml --suite "Suite" --case "Name" command [args] ...
+
+# This script runs a command and adds it to a JUnit report which can then
+# be used as a GitLab CI test report:
+#
+#   https://docs.gitlab.com/ee/ci/testing/unit_test_reports.html
+#
+# Commands can be specified with the "--command" flag, which will run
+# in a subshell, or as a list of extra arguments, which will be run
+# directly.
+#
+# Command output will be "teed". Scrubbed versions will be added to the
+# report and unmodified versions will be printed to stdout and stderr.
+#
+# If the command exit code is nonzero it will be added to the report
+# as a failure.
+#
+# The wrapper will return the command exit code.
+
+# JUnit report information can be found at
+# https://github.com/testmoapp/junitxml
+# https://www.ibm.com/docs/en/developer-for-zos/14.2?topic=formats-junit-xml-format
+
+
+import argparse
+import html
+import time
+import pathlib
+import re
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+
+def main():
+    parser = argparse.ArgumentParser(usage='\n  %(prog)s [options] --command "command"\n  %(prog)s [options] command ...')
+    parser.add_argument('--file', required=True, type=pathlib.Path, help='The JUnit-compatible XML file')
+    parser.add_argument('--suite', required=True, help='The testsuite_el name')
+    parser.add_argument('--case', required=True, help='The testcase name')
+    parser.add_argument('--command', help='The command to run if no extra arguments are provided')
+
+    args, command_list = parser.parse_known_args()
+
+    if (args.command and len(command_list) > 0) or (args.command is None and len(command_list) == 0):
+        sys.stderr.write('Error: The command must be provided via the --command flag or extra arguments.\n')
+        sys.exit(1)
+
+    try:
+        tree = ET.parse(args.file)
+        testsuites_el = tree.getroot()
+    except FileNotFoundError:
+        testsuites_el = ET.Element('testsuites')
+        tree = ET.ElementTree(testsuites_el)
+    except ET.ParseError:
+        sys.stderr.write(f'Error: {args.file} is invalid.\n')
+        sys.exit(1)
+
+    suites_time = float(testsuites_el.get('time', 0.0))
+    suites_tests = int(testsuites_el.get('tests', 0)) + 1
+    suites_failures = int(testsuites_el.get('failures', 0))
+
+    testsuite_el = testsuites_el.find(f'./testsuite[@name="{args.suite}"]')
+    if testsuite_el is None:
+        testsuite_el = ET.Element('testsuite', attrib={'name': args.suite})
+        testsuites_el.append(testsuite_el)
+
+    suite_time = float(testsuite_el.get('time', 0.0))
+    suite_tests = int(testsuite_el.get('tests', 0)) + 1
+    suite_failures = int(testsuite_el.get('failures', 0))
+
+    testcase_el = ET.Element('testcase', attrib={'name': args.case})
+    testsuite_el.append(testcase_el)
+
+    if args.command:
+        proc_args = args.command
+        in_shell = True
+    else:
+        proc_args = command_list
+        in_shell = False
+
+    start_time = time.perf_counter()
+    proc = subprocess.run(proc_args, shell=in_shell, encoding='UTF-8', errors='replace', capture_output=True)
+    case_time = time.perf_counter() - start_time
+
+    testcase_el.set('time', f'{case_time}')
+    testsuite_el.set('time', f'{suite_time + case_time}')
+    testsuites_el.set('time', f'{suites_time + case_time}')
+
+    # XXX Try to interleave them?
+    sys.stdout.write(proc.stdout)
+    sys.stderr.write(proc.stderr)
+
+    # Remove ANSI control sequences and escape other invalid characters
+    # https://stackoverflow.com/a/14693789/82195
+    ansi_seq_re = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+    scrubbed_stdout = html.escape(ansi_seq_re.sub('', proc.stdout), quote=False)
+    scrubbed_stderr = html.escape(ansi_seq_re.sub('', proc.stderr), quote=False)
+
+    if proc.returncode != 0:
+        failure_el = ET.Element('failure')
+        failure_el.text = f'{scrubbed_stdout}{scrubbed_stderr}'
+        testcase_el.append(failure_el)
+        testsuite_el.set('failures', f'{suite_failures + 1}')
+        testsuites_el.set('failures', f'{suites_failures + 1}')
+    else:
+        system_out_el = ET.Element('system-out')
+        system_out_el.text = f'{scrubbed_stdout}'
+        testcase_el.append(system_out_el)
+        system_err_el = ET.Element('system-err')
+        system_err_el.text = f'{scrubbed_stderr}'
+        testcase_el.append(system_err_el)
+
+    testsuite_el.set('tests', f'{suite_tests}')
+    testsuites_el.set('tests', f'{suites_tests}')
+
+    tree.write(args.file, encoding='UTF-8', xml_declaration=True)
+
+    return proc.returncode
+
+if __name__ == '__main__':
+    sys.exit(main())