Review of failed tests logic and Integration Tests (#5)

* Introduced MojoTest without changes to the plugin

* Added test pipeline

* Added exports to the Test step

* Fixed bash code

* Checking now for exception or TEST_FAILED_PREFIX in stdout.

* Added Integration tests for warnings

* Updating version for testing

* Moving back the return code to 0.

* Removed raises and updated docstring.

* Moving warning return code back to 1

* Updated README.md

.github/workflows/test.yml

@@ -0,0 +1,42 @@
+name: Run Tests
+
+on: ["push"]
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Install dependencies
+        run: |
+          curl https://get.modular.com | MODULAR_AUTH=${{ secrets.MODULAR_AUTH }} sh -
+          modular auth ${{ secrets.MODULAR_AUTH }}
+          modular install mojo
+          pip install .
+      - name: Integration Tests
+        run: |
+          export MODULAR_HOME="/home/runner/.modular"
+          export PATH="/home/runner/.modular/pkg/packages.modular.com_mojo/bin:$PATH"
+
+          # Tests that do not fail
+          pytest example/tests/mod_b
+
+          # Tests that should fail
+          if pytest example/tests/mod_a ; then
+            echo "This tests should fail"
+            exit 1
+          fi
+
+          # Test should fail becuase I am passing options for checking warnings
+          rm -rf ~/.modular/.mojo_cache
+          if pytest -W error example/tests/test_warning.mojo ; then
+            echo "This tests should fail"
+            exit 1
+          fi
+
+          # Test should not fail becuase I am not checking for warning
+          rm -rf ~/.modular/.mojo_cache
+          pytest example/tests/test_warning.mojo
+

README.md

@@ -16,7 +16,7 @@ and exit codes.
   - Each test file must have a `main` entry point function.
   - Each `main` function may call arbitrary functions and methods in your Mojo package.
   - Each test item must print a line with it's test name, prefixed with `#` (hashtag, comment) character, ex: `# test name`.
-  - Failed tests must raise Mojo `Error`. A helper function for assertions is in `example/tests/util.mojo`.
+  - Failed tests should use one of the standard [testing assertions](https://docs.modular.com/mojo/stdlib/testing/testing.html). A helper struct for tests is available in `example/tests/util.mojo`. Tests will also fail if a Mojo `Error` is raised or if an unhandled exception occurs. Note that this will not collect subsequent tests.
 - Mojo compiler warnings may optionally be handled as test failures, using the `pytest -W error` mode.
 
 ## Usage

example/tests/mod_a/test_convert.mojo

@@ -1,14 +1,14 @@
 from example.mod_a.impl import maths, convert, convert_different
-from example.tests.util import assert_true
+from example.tests.util import MojoTest
 
 
 fn main() raises:
     test_convert()
 
 
 fn test_convert() raises:
-    print("# convert")
+    let test = MojoTest("convert")
     let x: Int = 42
     let expect = SIMD[DType.float64](42)
     let result = convert(x)
-    assert_true(result == expect, "convert unexpected result: " + String(result))
+    test.assert_true(result == expect, "convert unexpected result: " + String(result))

example/tests/mod_a/test_convert_different.mojo

@@ -1,14 +1,14 @@
 from example.mod_a.impl import maths, convert_different
-from example.tests.util import assert_true
+from example.tests.util import MojoTest
 
 
 fn main() raises:
     test_convert_different()
 
 
 fn test_convert_different() raises:
-    print("# convert_different")
+    let test = MojoTest("convert_different")
     let x: Int = 8
     let expect = SIMD[DType.float16](8)
     let result = convert_different(x)
-    assert_true(result == expect, "convert unexpected result: " + String(result))
+    test.assert_true(result == expect, "convert unexpected result: " + String(result))

example/tests/mod_a/test_maths.mojo

@@ -1,5 +1,5 @@
 from example.mod_a.impl import maths
-from example.tests.util import assert_true
+from example.tests.util import MojoTest
 
 
 fn main() raises:
@@ -9,22 +9,22 @@ fn main() raises:
 
 
 fn test_maths() raises:
-    print("# maths")
+    let test = MojoTest("maths")
     let result = maths(8)
     let expect = 64
-    assert_true(result == expect, "maths unexpected result: " + String(result))
+    test.assert_true(result == expect, "maths unexpected result: " + String(result))
 
 
 fn test_maths_more() raises:
-    print("# maths more")
+    let test = MojoTest("maths more")
     let result = maths(9)
     let expect = 81
-    assert_true(result == expect, "maths unexpected result: " + String(result))
+    test.assert_true(result == expect, "maths unexpected result: " + String(result))
 
 
 fn test_maths_lotsmore() raises:
     for i in range(40, 50):
-        print("# maths more: " + String(i))
+        let test = MojoTest("maths more: " + String(i))
         let result = maths(i)
         if result == i:
-            raise Error("bad maths: " + String(i))
+            test.assert_true(False, "bad maths: " + String(i))

example/tests/mod_b/test_greet.mojo

@@ -1,13 +1,13 @@
 from example.mod_b.impl import greet
-from example.tests.util import assert_true
+from example.tests.util import MojoTest
 
 
 fn main() raises:
     test_greet()
 
 
 fn test_greet() raises:
-    print("# greet result")
+    let test = MojoTest("greet result")
     let result = greet("guido")
     let expect = "Hey guido"
-    assert_true(result == expect, "greet unexpected result: " + String(result))
+    test.assert_true(result == expect, "greet unexpected result: " + String(result))

example/tests/test_warning.mojo

@@ -1,3 +1,6 @@
+from example.tests.util import MojoTest
+
+
 fn main() raises:
     test_warning_is_collected()
 
@@ -10,6 +13,6 @@ fn test_warning_is_collected():
 
     This can be captured by running `pytest -W error` (warning -> error mode).
     """
-    print("# compiler warning")
+    let test = MojoTest("compiler warning")
     var x = 100
     print(x)

example/tests/util.mojo

@@ -1,9 +1,20 @@
 import testing
 
 
-fn assert_true(cond: Bool, message: String) raises:
+@value
+struct MojoTest:
     """
-    Wraps testing.assert_true, raises Error on assertion failure.
+    A utility struct for testing.
     """
-    if not testing.assert_true(cond, message):
-        raise Error(message)
+
+    var test_name: String
+
+    fn __init__(inout self, test_name: String):
+        self.test_name = test_name
+        print("# " + test_name)
+
+    fn assert_true(self, cond: Bool, message: String):
+        """
+        Wraps testing.assert_true.
+        """
+        _ = testing.assert_true(cond, message)

pytest_mojo/plugin.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import Any
 
-from pytest import Package, File, Item
+from pytest import File, Item, Package
 
 MOJO_CMD = ["mojo", "run", "-I", "."]
 """
@@ -19,6 +19,11 @@
 By convention, a comment line (hashtag) signals the test item name.
 """
 
+TEST_FAILED_PREFIX = "ASSERT ERROR"
+"""
+This is the prefix used in Mojo assertions in the testing module
+"""
+
 
 def pytest_collect_file(parent: Package, file_path: Path) -> File | None:
     if file_path.suffix in (".mojo", ".🔥") and file_path.name.startswith(TEST_PREFIX):
@@ -66,7 +71,9 @@ def collect(self):
             if line.startswith(TEST_ITEM_PREFIX):
                 if cur_item is not None:
                     yield MojoTestItem.from_parent(
-                        self, name=cur_item, spec=dict(stdout=item_stdout, code=0)
+                        self,
+                        name=cur_item,
+                        spec=dict(stdout=item_stdout, code=0),
                     )
                 cur_item = line
                 item_stdout = []
@@ -83,11 +90,13 @@ def collect(self):
 
 class MojoTestItem(Item):
     def __init__(self, *, name: str, parent, spec: dict[str, Any], **kwargs):
-        super().__init__(name, parent, **kwargs)
+        super().__init__(name.removeprefix(TEST_ITEM_PREFIX), parent, **kwargs)
         self.spec = spec
 
     def runtest(self):
-        if self.spec["code"] != 0:
+        if self.spec.get("code") or any(
+            item.startswith(TEST_FAILED_PREFIX) for item in self.spec["stdout"]
+        ):
             raise MojoTestException(self, self.spec["stdout"][-1])
 
     def repr_failure(self, excinfo):

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="pytest-mojo",
-    version="0.1.0",
+    version="0.1.1",
     packages=find_packages(),
     entry_points={"pytest11": ["mojo = pytest_mojo.plugin"]},
     install_requires=["pytest"],