Refactoring test integ

bin/test/test_integ.sh

@@ -14,6 +14,5 @@ cd integration || exit
 ./integration.py
 ./test_curl_commands.sh 
 ./test_html_output.py tests_ok/*.html tests_failed/*.html
-./ad_hoc.sh
 ./report.py
 

contrib/windows/README.md

@@ -28,7 +28,7 @@ Install git, llvm, nsis and python3
 
 ```powershell
 choco install --confirm --no-progress git winlibs-llvm-free nsis
-choco install --confirm --no-progress python3 --version "3.8.0"
+choco install --confirm --no-progress python3 --version "3.9.13"
 python -m pip install --upgrade pip --quiet
 ```
 

integration/integration.py

@@ -4,6 +4,8 @@
 import test_lint
 import test_format
 import test_hurl
+import test_script
+import platform
 
 
 def get_files(glob_expr):
@@ -32,6 +34,13 @@ def main():
         + get_files("ssl/*.hurl")
         if accept(f)
     ]
+    # Run test scripts
+    extension = "ps1" if platform.system() == "Windows" else "sh"
+    script_files = get_files("tests_ok/*." + extension) + get_files(
+        "tests_failed/*." + extension
+    )
+    for f in sorted(script_files):
+        test_script.test(f)
 
     print("test integration ok!")
 

integration/test_script.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# Test script file.
+#
+import codecs
+import sys
+import subprocess
+import os
+import argparse
+from typing import Optional
+
+
+def decode_string(encoded: bytes) -> str:
+    """Decodes bytes to string from inferring encoding."""
+    if encoded.startswith(codecs.BOM_UTF8):
+        return encoded.decode("utf-8-sig")
+    elif encoded.startswith(codecs.BOM_UTF16):
+        encoded = encoded[len(codecs.BOM_UTF16) :]
+        return encoded.decode("utf-16")
+    else:
+        # No BOM to determine encoding, try utf-8
+        return encoded.decode("utf-8")
+
+
+def test(script_file: str):
+    """Runs a script, exit the process if there is an error.
+
+    Arguments:
+    script_file -- the script file to run
+    """
+    cmd = (
+        "powershell.exe -Command " + script_file + " ; exit $LASTEXITCODE"
+        if script_file.endswith("ps1")
+        else script_file
+    )
+    # subprocess.run(['powershell', '-Command', script_file + " ; exit $LASTEXITCODE"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    print(cmd)
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    # exit code
+    expected = get_expected_exit_code(script_file)
+    if result.returncode != expected:
+        print(">>> error in return code")
+        print(f"expected: {expected}  actual:{result.returncode}")
+        stderr = decode_string(result.stderr).strip()
+        if stderr != "":
+            print(stderr)
+        stdout = decode_string(result.stdout).strip()
+        if stdout != "":
+            sys.setdefaultencoding("utf-8")
+            sys.stdout.write(stdout + "\n")
+
+        sys.exit(1)
+
+    # stdout
+    expected = get_expected_stdout(script_file)
+    actual = result.stdout
+    if actual != expected:
+        print(">>> error in stdout")
+        print(f"actual: <{actual}>\nexpected: <{expected}>")
+        sys.exit(1)
+
+
+def get_expected_exit_code(script_file: str) -> int:
+    """Runs expected exit code for a given test script"""
+
+    f = os.path.splitext(script_file)[0] + ".exit"
+    if os.path.exists(f):
+        expected = int(open(f, encoding="utf-8").read().strip())
+    else:
+        expected = 0
+    return expected
+
+
+def get_expected_stdout(script_file: str) -> Optional[str]:
+    """Runs expected stdout for a given test script"""
+
+    f = os.path.splitext(script_file)[0] + ".out"
+    if os.path.exists(f):
+        value = open(f, "rb").read()
+    else:
+        value = None
+    return value
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("file", type=str, nargs="+", metavar="FILE")
+    args = parser.parse_args()
+    for script_file in args.file:
+        test(script_file=script_file)
+
+
+if __name__ == "__main__":
+    main()

integration/tests_failed/file_not_found.err

@@ -0,0 +1 @@
+error: hurl: cannot access 'does_not_exist.hurl': No such file or directory

integration/tests_failed/file_not_found.exit

@@ -0,0 +1 @@
+2

integration/tests_failed/file_not_found.ps1

@@ -0,0 +1,4 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl does_not_exist.hurl
+

integration/tests_failed/file_not_found.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl does_not_exist.hurl
+

integration/tests_ok/hurlfmt.out

@@ -0,0 +1 @@
+GET http://localhost:8000/hello

integration/tests_ok/hurlfmt.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -Eeuo pipefail
+echo 'GET http://localhost:8000/hello' | hurlfmt --color
+

integration/tests_ok/multiple.out

@@ -0,0 +1 @@
+Hello World!Hello World!

integration/tests_ok/multiple.ps1

@@ -0,0 +1,4 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = 'Stop'
+hurl tests_ok\hello.hurl tests_ok\hello.hurl
+

integration/tests_ok/multiple.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -Eeuo pipefail
+hurl tests_ok/hello.hurl tests_ok/hello.hurl
+

integration/tests_ok/stdin.out

@@ -0,0 +1 @@
+Hello World!

integration/tests_ok/stdin.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -Eeuo pipefail
+echo "GET http://localhost:8000/hello" | hurl
+
+