fix reports being generated over multiple PIDs

Since the scheduler launches multiple processes to run tests, the test
report functionality was always reporting the last batch of tests.

This commit adds IPC communication for the report class so that all
tests are properly reported.

Signed-off-by: Gabriel Ionescu <gbi@amazon.com>

tests/framework/report.py

@@ -227,24 +227,17 @@ def add_collected_items(self, items):
         for item in items:
             self._collected_items[item.nodeid] = Report.TestItem(item)
 
-    def catch_return(self, item):
+    @mpsing.ipcmethod
+    def set_return(self, nodeid, rval):
         """
-        Wrap around a pytest test function.
+        Set return value for a given test.
 
-        We do this because we want to somehow catch the return value of
-        functions to set expected test criteria and actual criteria.
+        This function is called over IPC and needs picklable
+        objects as params.
         """
-        # Save the original function
-        original_function = item.obj
-
-        def func_wrapper(*args, **kwargs):
-            # Set the return value of this test item as the return value
-            # of the function
-            self._collected_items[item.nodeid].set_return(
-                original_function(*args, **kwargs))
-
-        item.obj = func_wrapper
+        self._collected_items[nodeid].set_return(rval)
 
+    @mpsing.ipcmethod
     def finish_test_item(self, report):
         """Mark a test as finished and update the report."""
         self._collected_items[report.nodeid].finish(report)

tests/framework/scheduler.py

@@ -48,11 +48,11 @@ def __init__(self):
         `PytestScheduler.instance()` to get the scheduler object.
         """
         super().__init__()
-        self._mp_singletons = [self]
-        self.session = None
 
         # Initialize a report for this session
         self._report = treport.Report(defs.TEST_RESULTS_DIR / "report")
+        self._mp_singletons = [self, self._report]
+        self.session = None
 
     def register_mp_singleton(self, mp_singleton):
         """Register a multi-process singleton object.
@@ -124,7 +124,16 @@ def pytest_pyfunc_call(self, pyfuncitem):
         # Overwrite the function with a custom one to catch return values.
         # These are used to catch expected vs. actual values and used in
         # reporting
-        self._report.catch_return(pyfuncitem)
+        original_function = pyfuncitem.obj
+
+        def call_wrapper(*args, **kwargs):
+            # set_return is an IPC method and we can't use any params here
+            # so we just pass things that are picklable
+            self._report.set_return(
+                pyfuncitem.nodeid,
+                original_function(*args, **kwargs))
+
+        pyfuncitem.obj = call_wrapper
 
     def pytest_runtestloop(self, session):
         """Pytest hook. The main test scheduling and running loop.