Address high CPU usage for some (GUI) simulations

Splits *UI rendering into three parts: ticker, progress and details.
Each has its own interval at which it renders, which is scaled based on
the number of realizations.

A simulation tracker has been implemented, and it now controls the flow
of information to both the GUI and CLI monitoring through events.

ert_gui/simulation/progress.py

@@ -66,10 +66,12 @@ def setIndeterminate(self, indeterminate):
         else:
             self.__timer.stop()
 
+    def get_indeterminate(self):
+        return self.__indeterminate
+
     def setIndeterminateColor(self, color):
         self.__indeterminate_color = color
 
-
     def paintEvent(self, paint_event):
         QFrame.paintEvent(self, paint_event)
         painter = QPainter(self)

ert_gui/simulation/run_dialog.py

@@ -2,7 +2,7 @@
 import sys
 
 try:
-    from PyQt4.QtCore import Qt, QTimer, QSize
+    from PyQt4.QtCore import Qt, QTimer, QSize, pyqtSignal, pyqtSlot
     from PyQt4.QtGui import (QDialog,
                              QVBoxLayout,
                              QLayout,
@@ -16,7 +16,7 @@
                              QStandardItem,
                              QWidget)
 except ImportError:
-    from PyQt5.QtCore import Qt, QTimer, QSize
+    from PyQt5.QtCore import Qt, QTimer, QSize, pyqtSignal, pyqtSlot
     from PyQt5.QtWidgets import (QDialog,
                                  QVBoxLayout,
                                  QLayout,
@@ -31,14 +31,22 @@
 
 from ert_gui.ertwidgets import resourceMovie, Legend
 from ert_gui.simulation import Progress, SimpleProgress, DetailedProgressWidget
-from ert_shared.models import BaseRunModel, SimulationsTracker
+from ert_shared.models import BaseRunModel
+from ert_shared.tracker.events import (TickEvent, GeneralEvent, DetailedEvent,
+                                       EndEvent)
+from ert_shared.tracker.factory import create_tracker
+from ert_shared.tracker.utils import format_running_time
+
 from ert_gui.tools.plot.plot_tool import PlotTool
 from res.job_queue import JobStatusType
 from ecl.util.util import BoolVector
 
+
 class RunDialog(QDialog):
 
-    def __init__(self, run_model, parent):
+    simulation_done = pyqtSignal(bool, str)
+
+    def __init__(self, run_model, arguments, parent):
         QDialog.__init__(self, parent)
         self.setWindowFlags(Qt.Window)
         self.setWindowFlags(self.windowFlags() & ~Qt.WindowContextHelpButtonHint)
@@ -53,8 +61,14 @@ def __init__(self, run_model, parent):
         if isinstance(run_model, BaseRunModel):
             ert = run_model.ert()
 
-        self.simulations_tracker = SimulationsTracker(model=run_model)
-        states = self.simulations_tracker.getStates()
+        self._simulations_argments = arguments
+
+        self.simulations_tracker = create_tracker(
+            run_model, qtimer_cls=QTimer,
+            event_handler=self._on_tracker_event,
+            num_realizations=arguments["active_realizations"].count())
+
+        states = self.simulations_tracker.get_states()
         self.state_colors = {state.name: state.color for state in states}
         self.state_colors['Success'] = self.state_colors["Finished"]
         self.state_colors['Failure'] = self.state_colors["Failed"]
@@ -150,29 +164,24 @@ def __init__(self, run_model, parent):
         self.done_button.clicked.connect(self.accept)
         self.restart_button.clicked.connect(self.restart_failed_realizations)
         self.show_details_button.clicked.connect(self.toggle_detailed_progress)
-
-        self.__updating = False
-        self.__update_queued = False
-        self.__simulation_started = False
-
-        self.__update_timer = QTimer(self)
-        self.__update_timer.setInterval(500)
-        self.__update_timer.timeout.connect(self.updateRunStatus)
-        self._simulations_argments = {}
+        self.simulation_done.connect(self._on_simulation_done)
 
     def reject(self):
         return
 
     def closeEvent(self, QCloseEvent):
-        if not self.checkIfRunFinished():
-            #Kill jobs if dialog is closed
+        self.simulations_tracker.stop()
+        if self._run_model.isFinished():
+            self.simulation_done.emit(self._run_model.hasRunFailed(),
+                                      self._run_model.getFailMessage())
+        else:
+            # Kill jobs if dialog is closed
             if self.killJobs() != QMessageBox.Yes:
                 QCloseEvent.ignore()
 
-    def startSimulation(self, arguments):
-
-        self._simulations_argments = arguments
+    def startSimulation(self):
         self._run_model.reset()
+        self.simulations_tracker.reset()
 
         def run():
             self._run_model.startSimulations( self._simulations_argments )
@@ -182,55 +191,7 @@ def run():
         simulation_thread.run = run
         simulation_thread.start()
 
-        self.__update_timer.start()
-
-
-    def checkIfRunFinished(self):
-        if self._run_model.isFinished():
-            self.hideKillAndShowDone()
-
-            if self._run_model.hasRunFailed():
-                error = self._run_model.getFailMessage()
-                QMessageBox.critical(self, "Simulations failed!", "The simulation failed with the following error:\n\n%s" % error)
-
-            return True
-        return False
-
-    def updateProgress(self):
-        self.simulations_tracker._update()
-
-        for state in self.simulations_tracker.getStates():
-            self.progress.updateState(state.state, 100.0 * state.count / state.total_count)
-            self.legends[state].updateLegend(state.name, state.count, state.total_count)
-
-    def updateRunStatus(self):
-        self.__status_label.setText(self._run_model.getPhaseName())
-
-        if self.checkIfRunFinished():
-            self.total_progress.setProgress(self._run_model.getProgress())
-            self.detailed_progress.set_progress(*self._run_model.getDetailedProgress())
-            self.updateProgress()
-            return
-
-        self.total_progress.setProgress(self._run_model.getProgress())
-
-        if self._run_model.isIndeterminate():
-            self.progress.setIndeterminate(True)
-            states = self.simulations_tracker.getStates()
-            for state in states:
-                self.legends[state].updateLegend(state.name, 0, 0)
-
-        else:
-            if self.detailed_progress and self.detailed_progress.isVisible():
-                self.detailed_progress.set_progress(*self._run_model.getDetailedProgress())
-            else:
-                self._run_model.updateDetailedProgress() #update information without rendering
-
-            self.progress.setIndeterminate(False)
-            self.updateProgress()
-
-        runtime = self._run_model.getRunningTime()
-        self.running_time.setText(SimulationsTracker.format_running_time(runtime))
+        self.simulations_tracker.track()
 
     def killJobs(self):
 
@@ -244,16 +205,46 @@ def killJobs(self):
                 self.reject()
         return kill_job
 
-
-    def hideKillAndShowDone(self):
-        self.__update_timer.stop()
+    @pyqtSlot(bool, str)
+    def _on_simulation_done(self, failed, failed_msg):
+        self.simulations_tracker.stop()
         self.processing_animation.hide()
         self.kill_button.setHidden(True)
         self.done_button.setHidden(False)
-        self.detailed_progress.set_progress(*self._run_model.getDetailedProgress())
-        self.restart_button.setVisible(self.has_failed_realizations() )
+        self.restart_button.setVisible(self.has_failed_realizations())
         self.restart_button.setEnabled(self._run_model.support_restart)
 
+        if failed:
+            QMessageBox.critical(self, "Simulations failed!",
+                                 "The simulation failed with the following " +
+                                 "error:\n\n{}".format(failed_msg))
+
+    @pyqtSlot(object)
+    def _on_tracker_event(self, event):
+        if isinstance(event, TickEvent):
+            self.running_time.setText(format_running_time(event.runtime))
+
+        if isinstance(event, GeneralEvent):
+            self.total_progress.setProgress(event.progress)
+            self.progress.setIndeterminate(event.indeterminate)
+
+            if event.indeterminate:
+                for state in event.sim_states:
+                    self.legends[state].updateLegend(state.name, 0, 0)
+            else:
+                for state in event.sim_states:
+                    self.progress.updateState(
+                        state.state, 100.0 * state.count / state.total_count)
+                    self.legends[state].updateLegend(
+                        state.name, state.count, state.total_count)
+
+        if isinstance(event, DetailedEvent):
+            if not self.progress.get_indeterminate():
+                self.detailed_progress.set_progress(event.details,
+                                                    event.iteration)
+
+        if isinstance(event, EndEvent):
+            self.simulation_done.emit(event.failed, event.failed_msg)
 
     def has_failed_realizations(self):
         completed = self._run_model.completed_realizations_mask
@@ -302,7 +293,7 @@ def restart_failed_realizations(self):
             self._simulations_argments['active_realizations'] = active_realizations
             self._simulations_argments['prev_successful_realizations'] = self._simulations_argments.get('prev_successful_realizations', 0)
             self._simulations_argments['prev_successful_realizations'] += self.count_successful_realizations()
-            self.startSimulation(self._simulations_argments)
+            self.startSimulation()
 
 
 

ert_gui/simulation/simulation_panel.py

@@ -110,8 +110,8 @@ def runSimulation(self):
         if start_simulations == QMessageBox.Yes:
             run_model = self.getCurrentSimulationModel()
             arguments = self.getSimulationArguments()
-            dialog = RunDialog(run_model(), self)
-            dialog.startSimulation( arguments )
+            dialog = RunDialog(run_model(), arguments, self)
+            dialog.startSimulation()
             dialog.exec_()
 
             ERT.emitErtChange() # simulations may have added new cases.

ert_shared/cli/main.py

@@ -8,7 +8,7 @@
 from ert_shared.cli.monitor import Monitor
 from ert_shared.cli.notifier import ErtCliNotifier
 from ert_shared.cli.workflow import execute_workflow
-from ert_shared.models import SimulationsTracker
+from ert_shared.tracker.factory import create_tracker
 from res.enkf import EnKFMain, ResConfig
 
 
@@ -36,7 +36,7 @@ def run_cli(args):
         )
         thread.start()
 
-        tracker = SimulationsTracker(model=model)
+        tracker = create_tracker(model, tick_interval=0, detailed_interval=0)
         monitor = Monitor(color_always=args.color_always)
 
         try:

ert_shared/cli/monitor.py

@@ -6,7 +6,9 @@
 from colors import color as ansi_color
 from console_progressbar import ProgressBar
 
-from ert_shared.models import SimulationStateStatus, SimulationsTracker
+from ert_shared.tracker.events import EndEvent, GeneralEvent
+from ert_shared.tracker.state import SimulationStateStatus
+from ert_shared.tracker.utils import format_running_time
 
 
 def _ansi_color(*args, **kwargs):
@@ -46,14 +48,16 @@ def _no_color(self, *args, **kwargs):
         return args[0]
 
     def monitor(self, tracker):
-        for update in tracker.track():
-            self._print_progress(update)
-
-        self._print_result(tracker.run_failed, tracker.failed_message)
-
-    def _get_legends(self, tracker):
+        for event in tracker.track():
+            if isinstance(event, GeneralEvent):
+                self._print_progress(event)
+            if isinstance(event, EndEvent):
+                self._print_result(event.failed, event.failed_msg)
+                return
+
+    def _get_legends(self, sim_states):
         legends = {}
-        for state in tracker.getStates():
+        for state in sim_states:
             legends[state] = "{}{:10} {:>10}".format(
                 self._colorize(self.dot, fg=state.color), state.name,
                 "{}/{}".format(state.count, state.total_count)
@@ -71,40 +75,35 @@ def _print_result(self, failed, failed_message):
                                  fg=SimulationStateStatus.COLOR_FINISHED),
                   file=self._out)
 
-    def _print_progress(self, tracker):
-        """Print a progress based on the information on a SimulationTracker
-        instance @tracker."""
-        if tracker.queue_size == 0:
-            # queue_size is 0, so no progress can be displayed
+    def _print_progress(self, event):
+        """Print a progress based on the information on a GeneralEvent."""
+        if event.indeterminate:
+            # indeterminate, no progress to be shown
             return
 
         prefix = """
     --> {phase_name}
 
     {current_phase}/{target}""".format(
-            phase_name=tracker.iteration_name,
-            current_phase=min(tracker.total_iterations,
-                              tracker.current_iteration + 1),
-            target=tracker.total_iterations,
+            phase_name=event.phase_name,
+            current_phase=min(event.total_phases,
+                              event.current_phase + 1),
+            target=event.total_phases,
         )
 
         statuses = ""
-        done = 0
-        legends = self._get_legends(tracker)
-        for state in tracker.getStates():
+        legends = self._get_legends(event.sim_states)
+        for state in event.sim_states:
             statuses += "    {}\n".format(legends[state])
-            if state.name == "Finished":
-                done = state.count
 
         suffix = """{runtime}
 
 {statuses}""".format(statuses=statuses,
-                     runtime=SimulationsTracker.format_running_time(
-                         tracker.runtime),)
+                     runtime=format_running_time(event.runtime))
 
         pb = ProgressBar(
-            total=tracker.queue_size, prefix=prefix, suffix=suffix, decimals=0,
+            total=100, prefix=prefix, suffix=suffix, decimals=0,
             length=self.bar_length, fill=self.filled_bar_char,
             zfill=self.empty_bar_char, file=self._out
         )
-        pb.print_progress_bar(done)
+        pb.print_progress_bar(event.progress * 100)

ert_shared/models/__init__.py

@@ -4,4 +4,3 @@
 from .ensemble_smoother import EnsembleSmoother
 from .iterated_ensemble_smoother import IteratedEnsembleSmoother
 from .multiple_data_assimilation import MultipleDataAssimilation
-from .simulations_tracker import SimulationsTracker, SimulationStateStatus