Add gui system tests for operations and recon

.github/workflows/conda.yml

@@ -75,18 +75,18 @@ jobs:
       - name: GUI Tests System
         shell: bash -l {0}
         run: |
-          xvfb-run --auto-servernum python -m pytest -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov --run-system-tests
+          xvfb-run --auto-servernum python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov --run-system-tests
         timeout-minutes: 15
 
-      - name: GUI Tests Applitools
+      - name: GUI Tests Screenshots Applitools
         if: ${{ github.event_name == 'pull_request' }}
         shell: bash -l {0}
         env:
           APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
           APPLITOOLS_BATCH_ID: ${{ github.sha }}
           GITHUB_BRANCH_NAME: ${{ github.head_ref }}
         run: |
-          xvfb-run --auto-servernum python -m pytest -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov mantidimaging/eyes_tests
+          xvfb-run --auto-servernum python -m pytest -vs -rs -p no:xdist -p no:randomly -p no:repeat -p no:cov mantidimaging/eyes_tests
         timeout-minutes: 15
 
       - name: Coveralls

docs/developer_guide/testing.rst

@@ -36,8 +36,8 @@ Static analysis
 Mantid Imaging uses `mypy <http://mypy-lang.org/>`_, `flake8 <https://flake8.pycqa.org/>`_ and `yapf <https://github.com/google/yapf>`_ for static analysis and formatting. They are run by :code:`make check`, or can be run individually, e.g. :code:`make mypy`.
 
 
-GUI Testing
------------
+GUI screenshot testing
+----------------------
 
 Mantid Imaging uses `Applitools Eyes <https://applitools.com/products-eyes/>`_ for GUI approval testing. Screenshots of windows are uploaded and compared to known good baseline images. This is run in the github action on pull requests.
 
@@ -55,3 +55,18 @@ To run without a key or to prevent uploads, set ``APPLITOOLS_API_KEY`` to ``loca
 
     mkdir /tmp/gui_test
     APPLITOOLS_API_KEY=local APPLITOOLS_IMAGE_DIR=/tmp/gui_test xvfb-run --auto-servernum pytest -p no:xdist -p no:randomly -p no:repeat -p no:cov mantidimaging/eyes_tests
+
+GUI system tests
+----------------
+
+GUI system tests run work flows in Mantid Imaging in a 'realistic' way, where possible by using QTest methods to emulate mouse and keyboard actions. They use the same data files as the GUI screenshot tests. These take several minutes to run and so must be explicitly requested.
+
+.. code::
+
+    pytest -v --run-system-tests
+
+or in virtual X server xvfb-run
+
+.. code::
+
+    xvfb-run --auto-servernum pytest -v --run-system-tests

mantidimaging/gui/test/gui_system_base.py

@@ -3,12 +3,13 @@
 
 import os
 from pathlib import Path
+from typing import Callable, Optional
 import unittest
 from unittest import mock
 
 from PyQt5.QtCore import Qt, QTimer
 from PyQt5.QtTest import QTest
-from PyQt5.QtWidgets import QApplication, QMessageBox
+from PyQt5.QtWidgets import QApplication, QMessageBox, QInputDialog
 import pytest
 
 from mantidimaging.core.utility.version_check import versions
@@ -24,7 +25,6 @@
 
 SHOW_DELAY = 10  # Can be increased to watch tests
 SHORT_DELAY = 100
-LOAD_DELAY = 5000
 
 
 @pytest.mark.system
@@ -47,26 +47,62 @@ def tearDown(self) -> None:
     def _click_messageBox(cls, button_text: str):
         """Needs to be queued with QTimer.singleShot before triggering the message box"""
         for widget in cls.app.topLevelWidgets():
-            if isinstance(widget, QMessageBox):
+            if isinstance(widget, QMessageBox) and widget.isVisible():
                 for button in widget.buttons():
                     if button.text().replace("&", "") == button_text:
                         QTest.mouseClick(button, Qt.LeftButton)
                         return
                 button_texts = [button.text() for button in widget.buttons()]
                 raise ValueError(f"Could not find button '{button_text}' in {button_texts}")
 
+    @classmethod
+    def _click_InputDialog(cls, set_int: Optional[int] = None):
+        """Needs to be queued with QTimer.singleShot before triggering the message box"""
+        for widget in cls.app.topLevelWidgets():
+            if isinstance(widget, QInputDialog) and widget.isVisible():
+                if set_int:
+                    widget.setIntValue(set_int)
+                QTest.qWait(SHORT_DELAY)
+                widget.accept()
+
     def _close_welcome(self):
         self.main_window.welcome_window.view.close()
 
+    @classmethod
+    def _wait_until(cls, test_func: Callable[[], bool], delay=0.1, max_retry=100):
+        """
+        Repeat test_func every delay seconds until is becomes true. Or if max_retry is reached return false.
+        """
+        for _ in range(max_retry):
+            if test_func():
+                return True
+            QTest.qWait(delay * 1000)
+        raise RuntimeError("_wait_until reach max retries")
+
+    @classmethod
+    def _wait_for_widget_visible(cls, widget_type, delay=0.1, max_retry=100):
+        for _ in range(max_retry):
+            for widget in cls.app.topLevelWidgets():
+                if isinstance(widget, widget_type) and widget.isVisible():
+                    return True
+            QTest.qWait(delay * 1000)
+        raise RuntimeError("_wait_for_stack_selector reach max retries")
+
     @mock.patch("mantidimaging.gui.windows.load_dialog.view.MWLoadDialog.select_file")
     def _load_data_set(self, mocked_select_file):
         mocked_select_file.return_value = LOAD_SAMPLE
+        initial_stacks = len(self.main_window.presenter.model.get_all_stack_visualisers())
+
+        def test_func() -> bool:
+            current_stacks = len(self.main_window.presenter.model.get_all_stack_visualisers())
+            return (current_stacks - initial_stacks) >= 5
+
         self.main_window.actionLoadDataset.trigger()
         QTest.qWait(SHOW_DELAY)
         self.main_window.load_dialogue.presenter.notify(Notification.UPDATE_ALL_FIELDS)
         QTest.qWait(SHOW_DELAY)
         self.main_window.load_dialogue.accept()
-        QTest.qWait(LOAD_DELAY)
+        self._wait_until(test_func, max_retry=600)
 
     def _open_operations(self):
         self.main_window.actionFilters.trigger()

mantidimaging/gui/test/test_gui_system_loading.py

@@ -5,10 +5,9 @@
 from unittest import mock
 
 from PyQt5.QtCore import QTimer, QEventLoop
-from PyQt5.QtTest import QTest
 from PyQt5.QtWidgets import QApplication
 
-from mantidimaging.gui.test.gui_system_base import GuiSystemBase, SHORT_DELAY, LOAD_SAMPLE, LOAD_DELAY
+from mantidimaging.gui.test.gui_system_base import GuiSystemBase, SHORT_DELAY, LOAD_SAMPLE
 from mantidimaging.gui.widgets.stack_selector_dialog.stack_selector_dialog import StackSelectorDialog
 
 
@@ -20,11 +19,19 @@ def setUp(self) -> None:
     @mock.patch("mantidimaging.gui.windows.main.MainWindowView._get_file_name")
     def _load_images(self, mocked_select_file):
         mocked_select_file.return_value = LOAD_SAMPLE
+        initial_stacks = len(self.main_window.presenter.model.get_all_stack_visualisers())
+
         self.main_window.actionLoadImages.trigger()
-        QTest.qWait(LOAD_DELAY)
+
+        def test_func() -> bool:
+            current_stacks = len(self.main_window.presenter.model.get_all_stack_visualisers())
+            return (current_stacks - initial_stacks) >= 1
+
+        self._wait_until(test_func, max_retry=600)
 
     @classmethod
     def _click_stack_selector(cls):
+        cls._wait_for_widget_visible(StackSelectorDialog)
         for widget in cls.app.topLevelWidgets():
             if isinstance(widget, StackSelectorDialog):
                 for x in range(20):
@@ -40,15 +47,17 @@ def _click_stack_selector(cls):
     def test_load_180(self, mocked_select_file):
         path_180 = Path(LOAD_SAMPLE).parents[1] / "180deg" / "IMAT_Flower_180deg_000000.tif"
         mocked_select_file.return_value = path_180
+        self.assertEqual(len(self.main_window.presenter.get_all_stack_visualisers()), 0)
         self._load_images()
         stacks = self.main_window.presenter.get_all_stack_visualisers()
 
-        self.assertEqual(len(stacks), 1)
+        self.assertEqual(len(self.main_window.presenter.get_all_stack_visualisers()), 1)
         self.assertFalse(stacks[0].presenter.images.has_proj180deg())
 
-        QTimer.singleShot(SHORT_DELAY * 10, lambda: self._click_stack_selector())
+        QTimer.singleShot(SHORT_DELAY, lambda: self._click_stack_selector())
         self.main_window.actionLoad180deg.trigger()
-        QTest.qWait(LOAD_DELAY)
+
+        self._wait_until(lambda: len(self.main_window.presenter.get_all_stack_visualisers()) == 2)
 
         stacks_after = self.main_window.presenter.get_all_stack_visualisers()
         self.assertEqual(len(stacks_after), 2)

mantidimaging/gui/test/test_gui_system_operations.py

@@ -0,0 +1,151 @@
+# Copyright (C) 2021 ISIS Rutherford Appleton Laboratory UKRI
+# SPDX - License - Identifier: GPL-3.0-or-later
+
+from itertools import product
+from unittest import mock
+from uuid import UUID
+
+from parameterized import parameterized
+from PyQt5.QtTest import QTest
+from PyQt5.QtCore import Qt, QTimer
+from PyQt5.QtWidgets import QFormLayout, QLabel, QWidget
+
+from mantidimaging.gui.windows.stack_choice.presenter import StackChoicePresenter
+from mantidimaging.core.data import Images
+from mantidimaging.gui.test.gui_system_base import GuiSystemBase, SHOW_DELAY, SHORT_DELAY
+from mantidimaging.gui.windows.stack_choice.view import StackChoiceView
+from mantidimaging.gui.windows.operations.view import FiltersWindowView
+
+OP_LIST = [
+    ("Arithmetic", [["Multiply", "2"]]),
+    ("Circular Mask", []),
+    ("Clip Values", [["Clip Max", "10000"]]),
+    ("Crop Coordinates", [["ROI", "10,10,100,100"]]),
+    ("Divide", []),
+    ("Flat-fielding", []),
+    ("Gaussian", []),
+    ("Median", []),
+    # ("Monitor Normalisation", []),
+    ("NaN Removal", []),
+    ("Remove Outliers", []),
+    ("Rebin", []),
+    # ("Remove all stripes", []),
+    # ("Remove dead stripes", []),
+    # ("Remove large stripes", []),
+    # ("Stripe Removal", []),
+    # ("Remove stripes with filtering", []),
+    # ("Remove stripes with sorting and fitting", []),
+    ("Rescale", [["Max input", "10000"]]),
+    ("Ring Removal", []),
+    ("ROI Normalisation", []),
+    ("Rotate Stack", []),
+]
+
+
+class TestGuiSystemOperations(GuiSystemBase):
+    def setUp(self) -> None:
+        super().setUp()
+        self._close_welcome()
+        self._load_data_set()
+
+        self._open_operations()
+        self.assertIsNotNone(self.main_window.filters)
+        assert isinstance(self.main_window.filters, FiltersWindowView)  # for yapf
+        self.assertTrue(self.main_window.filters.isVisible())
+        self.op_window = self.main_window.filters
+
+    def tearDown(self) -> None:
+        self._close_stack_tabs()
+        super().tearDown()
+        self.assertFalse(self.main_window.isVisible())
+
+    @staticmethod
+    def _get_operation_parameter_widget(form: QFormLayout, param_name: str) -> QWidget:
+        for i in range(form.rowCount()):
+            label_item = form.itemAt(i * 2)
+            widget_item = form.itemAt(i * 2 + 1)
+
+            if label_item is not None and widget_item is not None:
+                label = label_item.widget()
+                assert isinstance(label, QLabel)
+                if label.text() == param_name:
+                    return widget_item.widget()
+
+        raise ValueError(f"Could not find '{param_name}' in form")
+
+    @classmethod
+    def _click_stack_selector(cls, keep_new: bool):
+        cls._wait_for_widget_visible(StackChoiceView)
+        QTest.qWait(SHOW_DELAY)
+        for widget in cls.app.topLevelWidgets():
+            if isinstance(widget, StackChoiceView):
+                if keep_new:
+                    QTest.mouseClick(widget.newDataButton, Qt.MouseButton.LeftButton)
+                else:
+                    QTest.mouseClick(widget.originalDataButton, Qt.MouseButton.LeftButton)
+
+    @parameterized.expand(OP_LIST)
+    def test_run_operation_stack(self, op_name, params):
+        QTest.qWait(SHOW_DELAY)
+        index = self.op_window.filterSelector.findText(op_name)
+        self.assertGreaterEqual(index, 0, f'Operation "{op_name}" not found in filterSelector')
+        self.op_window.filterSelector.setCurrentIndex(index)
+        QTest.qWait(SHOW_DELAY)
+
+        for param_name, param_value in params:
+            widget = self._get_operation_parameter_widget(self.op_window.filterPropertiesLayout, param_name)
+            widget.selectAll()
+            QTest.keyClicks(widget, param_value)
+            QTest.keyClick(widget, Qt.Key_Return)
+            QTest.qWait(SHOW_DELAY)
+
+        self.op_window.safeApply.setChecked(False)
+        QTest.mouseClick(self.op_window.applyButton, Qt.MouseButton.LeftButton)
+        QTest.qWait(SHORT_DELAY)
+        self._wait_until(lambda: self.op_window.presenter.filter_is_running is False, max_retry=600)
+
+        self.main_window.filters.close()
+        QTest.qWait(SHOW_DELAY)
+
+    @parameterized.expand(product(OP_LIST[:3], ["new", "original"]))
+    def test_run_operation_stack_safe(self, op_info, keep_stack):
+        op_name, params = op_info
+        print(f"test_run_operation_stack_safe {op_name=} {params=} {keep_stack=}")
+        QTest.qWait(SHOW_DELAY)
+        index = self.op_window.filterSelector.findText(op_name)
+        self.assertGreaterEqual(index, 0, f'Operation "{op_name}" not found in filterSelector')
+        self.op_window.filterSelector.setCurrentIndex(index)
+        QTest.qWait(SHOW_DELAY)
+
+        for param_name, param_value in params:
+            widget = self._get_operation_parameter_widget(self.op_window.filterPropertiesLayout, param_name)
+            widget.selectAll()
+            QTest.keyClicks(widget, param_value)
+            QTest.keyClick(widget, Qt.Key_Return)
+            QTest.qWait(SHOW_DELAY)
+
+        self.op_window.safeApply.setChecked(True)
+        QTest.qWait(SHOW_DELAY)
+
+        def mock_wait_for_stack_choice(self, new_stack: Images, stack_uuid: UUID):
+            print("mock_wait_for_stack_choice")
+            stack_choice = StackChoicePresenter(self.original_images_stack, new_stack, self, stack_uuid)
+            stack_choice.show()
+            QTest.qWait(SHOW_DELAY)
+            if keep_stack == "new":
+                QTest.mouseClick(stack_choice.view.newDataButton, Qt.MouseButton.LeftButton)
+            else:
+                QTest.mouseClick(stack_choice.view.originalDataButton, Qt.MouseButton.LeftButton)
+
+            return stack_choice.use_new_data
+
+        with mock.patch("mantidimaging.gui.windows.operations.presenter.FiltersWindowPresenter._wait_for_stack_choice",
+                        mock_wait_for_stack_choice):
+
+            QTimer.singleShot(SHORT_DELAY, lambda: self._click_messageBox("OK"))
+            QTest.mouseClick(self.op_window.applyButton, Qt.MouseButton.LeftButton)
+
+            QTest.qWait(SHORT_DELAY)
+            self._wait_until(lambda: self.op_window.presenter.filter_is_running is False)
+            self.main_window.filters.close()
+            QTest.qWait(SHOW_DELAY)