Report refactoring and hook

environment.yml

@@ -1,25 +1,26 @@
 name: floatcsep
 channels:
-    - conda-forge
-    - defaults
+  - conda-forge
+  - defaults
 dependencies:
-    - python <= 3.11
-    - numpy
-    - pycsep
-    - dateparser
-    - docker-py
-    - flake8
-    - gitpython
-    - h5py
-    - matplotlib
-    - pip
-    - pyshp
-    - pyyaml
-    - requests
-    - seaborn
-    - sphinx
-    - sphinx-autoapi
-    - sphinx-gallery
-    - sphinx-rtd-theme
-    - pytables
-    - xmltodict
+  - python <= 3.11
+  - numpy
+  - pycsep
+  - dateparser
+  - docker-py
+  - flake8
+  - gitpython
+  - h5py
+  - matplotlib
+  - pip
+  - pyshp
+  - pyyaml
+  - requests
+  - scipy
+  - seaborn
+  - sphinx
+  - sphinx-autoapi
+  - sphinx-gallery
+  - sphinx-rtd-theme
+  - pytables
+  - xmltodict

examples/case_g/config.yml

@@ -20,4 +20,4 @@ model_config: models.yml
 test_config: tests.yml
 
 postprocess:
-  plot_custom: plot_script.py:main
+  plot_custom: custom_plot_script.py:main

examples/case_h/config.yml

@@ -17,4 +17,8 @@ region_config:
 force_rerun: True
 catalog: catalog.csv
 model_config: models.yml
-test_config: tests.yml
+test_config: tests.yml
+
+postprocess:
+  plot_catalog: False
+  report: custom_report.py:main

examples/case_h/custom_report.py

@@ -0,0 +1,47 @@
+from floatcsep.report import MarkdownReport
+from floatcsep.utils import timewindow2str
+
+
+def main(experiment):
+
+    timewindow = experiment.timewindows[-1]
+    timestr = timewindow2str(timewindow)
+
+    report = MarkdownReport()
+    report.add_title(f"Experiment Report - {experiment.name}", "")
+    report.add_heading("Objectives", level=2)
+
+    objs = [
+        f"Comparison of ETAS, pyMock-Poisson and pyMock-NegativeBinomial models for the"
+        f"day after the Amatrice earthquake, for events with M>{min(experiment.magnitudes)}.",
+    ]
+    report.add_list(objs)
+
+    report.add_figure(
+        f"Input catalog",
+        [
+            experiment.registry.get_figure("main_catalog_map"),
+            experiment.registry.get_figure("main_catalog_time"),
+        ],
+        level=3,
+        ncols=1,
+        caption=f"Evaluation catalog of {experiment.start_date}. "
+        f"Earthquakes are filtered above Mw"
+        f" {min(experiment.magnitudes)}.",
+        add_ext=True,
+    )
+
+    # Include results from Experiment
+    test = experiment.tests[0]
+    for model in experiment.models:
+        fig_path = experiment.registry.get_figure(timestr, f"{test.name}_{model.name}")
+        report.add_figure(
+            f"{test.name}: {model.name}",
+            fig_path,
+            level=3,
+            caption="Catalog-based N-test",
+            add_ext=True,
+            width=200,
+        )
+
+    report.save(experiment.registry.abs(experiment.registry.run_dir))

floatcsep/cmd/main.py

@@ -2,10 +2,10 @@
 import logging
 
 from floatcsep import __version__
-from floatcsep.experiment import Experiment
+from floatcsep.experiment import Experiment, ExperimentComparison
 from floatcsep.logger import setup_logger, set_console_log_level
-from floatcsep.utils import ExperimentComparison
 from floatcsep.postprocess import plot_results, plot_forecasts, plot_catalogs, plot_custom
+from floatcsep.report import generate_report, reproducibility_report
 
 setup_logger()
 log = logging.getLogger("floatLogger")
@@ -34,7 +34,7 @@ def run(config, **kwargs):
     plot_results(experiment=exp)
     plot_custom(experiment=exp)
 
-    exp.generate_report()
+    generate_report(experiment=exp)
     exp.make_repr()
 
     log.info("Finalized")
@@ -54,7 +54,7 @@ def plot(config, **kwargs):
     plot_results(experiment=exp)
     plot_custom(experiment=exp)
 
-    exp.generate_report()
+    generate_report(experiment=exp)
 
     log.debug("")
 
@@ -76,6 +76,7 @@ def reproduce(config, **kwargs):
     comp = ExperimentComparison(original_exp, reproduced_exp)
     comp.compare_results()
 
+    reproducibility_report(exp_comparison=comp)
     log.info("Finalized")
     log.debug("")
 

floatcsep/environments.py

@@ -169,8 +169,7 @@ def create_environment(self, force=False):
                     ]
                 )
             log.info(f"\tSub-conda environment created: {self.env_name}")
-
-        self.install_dependencies()
+            self.install_dependencies()
 
     def env_exists(self) -> bool:
         """

floatcsep/experiment.py

@@ -1,4 +1,6 @@
 import datetime
+import filecmp
+import hashlib
 import logging
 import os
 import shutil
@@ -7,8 +9,9 @@
 
 import numpy
 import yaml
+import scipy
+
 
-from floatcsep import report
 from floatcsep.evaluation import Evaluation
 from floatcsep.logger import add_fhandler
 from floatcsep.model import Model, TimeDependentModel
@@ -559,13 +562,6 @@ def read_results(self, test: Evaluation, window: str) -> List:
 
         return test.read_results(window, self.models)
 
-    def generate_report(self) -> None:
-        """Creates a report summarizing the Experiment's results."""
-
-        log.info(f"Saving report into {self.registry.run_dir}")
-
-        report.generate_report(self)
-
     def make_repr(self):
 
         log.info("Creating reproducibility config file")
@@ -700,3 +696,144 @@ def from_yml(cls, config_yml: str, repr_dir=None, **kwargs):
                 kwargs.pop("logging")
 
         return cls(**_dict, **kwargs)
+
+
+class ExperimentComparison:
+
+    def __init__(self, original, reproduced, **kwargs):
+        """"""
+        self.original = original
+        self.reproduced = reproduced
+
+        self.num_results = {}
+        self.file_comp = {}
+
+    @staticmethod
+    def obs_diff(obs_orig, obs_repr):
+
+        return numpy.abs(
+            numpy.divide((numpy.array(obs_orig) - numpy.array(obs_repr)), numpy.array(obs_orig))
+        )
+
+    @staticmethod
+    def test_stat(test_orig, test_repr):
+
+        if isinstance(test_orig[0], str):
+            if not isinstance(test_orig[1], str):
+                stats = numpy.array(
+                    [0, numpy.divide((test_repr[1] - test_orig[1]), test_orig[1]), 0, 0]
+                )
+            else:
+                stats = None
+        else:
+            stats_orig = numpy.array(
+                [numpy.mean(test_orig), numpy.std(test_orig), scipy.stats.skew(test_orig)]
+            )
+            stats_repr = numpy.array(
+                [numpy.mean(test_repr), numpy.std(test_repr), scipy.stats.skew(test_repr)]
+            )
+
+            ks = scipy.stats.ks_2samp(test_orig, test_repr)
+            stats = [*numpy.divide(numpy.abs(stats_repr - stats_orig), stats_orig), ks.pvalue]
+        return stats
+
+    def get_results(self):
+
+        win_orig = timewindow2str(self.original.timewindows)
+        win_repr = timewindow2str(self.reproduced.timewindows)
+
+        tests_orig = self.original.tests
+        tests_repr = self.reproduced.tests
+
+        models_orig = [i.name for i in self.original.models]
+        models_repr = [i.name for i in self.reproduced.models]
+
+        results = dict.fromkeys([i.name for i in tests_orig])
+
+        for test in tests_orig:
+            if test.type in ["consistency", "comparative"]:
+                results[test.name] = dict.fromkeys(win_orig)
+                for tw in win_orig:
+                    results_orig = self.original.read_results(test, tw)
+                    results_repr = self.reproduced.read_results(test, tw)
+                    results[test.name][tw] = {
+                        models_orig[i]: {
+                            "observed_statistic": self.obs_diff(
+                                results_orig[i].observed_statistic,
+                                results_repr[i].observed_statistic,
+                            ),
+                            "test_statistic": self.test_stat(
+                                results_orig[i].test_distribution,
+                                results_repr[i].test_distribution,
+                            ),
+                        }
+                        for i in range(len(models_orig))
+                    }
+
+            else:
+                results_orig = self.original.read_results(test, win_orig[-1])
+                results_repr = self.reproduced.read_results(test, win_orig[-1])
+                results[test.name] = {
+                    models_orig[i]: {
+                        "observed_statistic": self.obs_diff(
+                            results_orig[i].observed_statistic,
+                            results_repr[i].observed_statistic,
+                        ),
+                        "test_statistic": self.test_stat(
+                            results_orig[i].test_distribution, results_repr[i].test_distribution
+                        ),
+                    }
+                    for i in range(len(models_orig))
+                }
+
+        return results
+
+    @staticmethod
+    def get_hash(filename):
+
+        with open(filename, "rb") as f:
+            bytes_file = f.read()
+            readable_hash = hashlib.sha256(bytes_file).hexdigest()
+        return readable_hash
+
+    def get_filecomp(self):
+
+        win_orig = timewindow2str(self.original.timewindows)
+        win_repr = timewindow2str(self.reproduced.timewindows)
+
+        tests_orig = self.original.tests
+        tests_repr = self.reproduced.tests
+
+        models_orig = [i.name for i in self.original.models]
+        models_repr = [i.name for i in self.reproduced.models]
+
+        results = dict.fromkeys([i.name for i in tests_orig])
+
+        for test in tests_orig:
+            if test.type in ["consistency", "comparative"]:
+                results[test.name] = dict.fromkeys(win_orig)
+                for tw in win_orig:
+                    results[test.name][tw] = dict.fromkeys(models_orig)
+                    for model in models_orig:
+                        orig_path = self.original.registry.get_result(tw, test, model)
+                        repr_path = self.reproduced.registry.get_result(tw, test, model)
+
+                        results[test.name][tw][model] = {
+                            "hash": (self.get_hash(orig_path) == self.get_hash(repr_path)),
+                            "byte2byte": filecmp.cmp(orig_path, repr_path),
+                        }
+            else:
+                results[test.name] = dict.fromkeys(models_orig)
+                for model in models_orig:
+                    orig_path = self.original.registry.get_result(win_orig[-1], test, model)
+                    repr_path = self.reproduced.registry.get_result(win_orig[-1], test, model)
+                    results[test.name][model] = {
+                        "hash": (self.get_hash(orig_path) == self.get_hash(repr_path)),
+                        "byte2byte": filecmp.cmp(orig_path, repr_path),
+                    }
+        return results
+
+    def compare_results(self):
+
+        self.num_results = self.get_results()
+        self.file_comp = self.get_filecomp()

floatcsep/model.py

@@ -309,7 +309,7 @@ def __init__(
             self.registry, model_class=self.__class__.__name__, **kwargs
         )
         self.build = kwargs.get("build", None)
-
+        self.force_build = kwargs.get("force_build", False)
         if self.func:
             self.environment = EnvironmentFactory.get_env(
                 self.build, self.name, self.registry.abs(model_path)
@@ -329,7 +329,7 @@ def stage(self, timewindows=None) -> None:
             self.get_source(self.zenodo_id, self.giturl, branch=self.repo_hash)
 
         if hasattr(self, "environment"):
-            self.environment.create_environment()
+            self.environment.create_environment(force=self.force_build)
 
         self.registry.build_tree(
             timewindows=timewindows,