Separate parsing log files from simulation run

.github/workflows/calibration.yaml

@@ -153,6 +153,7 @@ jobs:
           mkdir -p "${task_output_dir}"
 
           tlo scenario-run --output-dir "${task_output_dir}" --draw "${draw}" ${{ matrix.index }} "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/scenarios/long_run_all_diseases.py"
+          tlo parse-log ${task_output_dir}
         working-directory: "${{ needs.setup.outputs.worktree_path }}"
 
   # Do the postprocessing

src/scripts/dev/scenarios/playing_22.py

@@ -14,14 +14,13 @@ def __init__(self):
         super().__init__()
         self.seed = 655123742
         self.start_date = Date(2010, 1, 1)
-        self.end_date = Date(2011, 1, 1)
-        self.pop_size = 200
-        self.number_of_draws = 5
-        self.runs_per_draw = 1
+        self.end_date = Date(2013, 1, 1)
+        self.pop_size = 1000
+        self.number_of_draws = 3
+        self.runs_per_draw = 3
 
     def log_configuration(self):
         return {
-            # 'filename': 'my-scenario',
             'directory': './outputs',
             'custom_levels': {
                 '*': logging.INFO,
@@ -39,9 +38,8 @@ def modules(self):
 
     def draw_parameters(self, draw_number, rng):
         return {
-            'Lifestyle': {
-                'init_p_urban': rng.randint(10, 20) / 100.0,
-                'init_p_high_sugar': 0.52,
+            'Demography': {
+                'max_age_initial': [80, 90, 100][draw_number],
             },
         }
 

src/tlo/cli.py

@@ -4,6 +4,7 @@
 import json
 import math
 import os
+import pickle
 import tempfile
 from collections import defaultdict
 from pathlib import Path
@@ -22,6 +23,7 @@
 from azure.storage.fileshare import ShareClient, ShareDirectoryClient, ShareFileClient
 from git import Repo
 
+from tlo.analysis.utils import parse_log_file
 from tlo.scenario import SampleRunner, ScenarioLoader
 
 JOB_LABEL_PADDING = len("State transition time")
@@ -88,6 +90,21 @@ def scenario_run(scenario_file, draw_only, draw: tuple, output_dir=None, scenari
     else:
         runner.run()
 
+@cli.command()
+@click.argument("LOG_DIRECTORY", type=click.Path(exists=True))
+def parse_log(log_directory):
+    assert os.path.isdir(log_directory), f"{log_directory} must be a directory"
+
+    path = Path(log_directory)
+
+    log_files = list(path.glob("*.log"))
+    assert len(log_files) == 1, f"directory {log_directory} must contain exactly one file with extension .log"
+
+    outputs = parse_log_file(log_files[0])
+    for key, output in outputs.items():
+        if key.startswith("tlo."):
+            with open(path / f"{key}.pickle", "wb") as f:
+                pickle.dump(output, f)
 
 @cli.command()
 @click.argument("scenario_file", type=click.Path(exists=True))
@@ -237,6 +254,7 @@ def batch_submit(ctx, scenario_file, asserts_on, more_memory, keep_pool_alive, i
     pip install -r requirements/base.txt
     env | grep "^AZ_" | while read line; do echo "$line"; done
     {py_opt} tlo --config-file tlo.example.conf batch-run {azure_run_json} {working_dir} {{draw_number}} {{run_number}}
+    tlo --config-file tlo.example.conf parse-log {working_dir}/{{draw_number}}/{{run_number}}
     cp {task_dir}/std*.txt {working_dir}/{{draw_number}}/{{run_number}}/.
     gzip {working_dir}/{{draw_number}}/{{run_number}}/*.{gzip_pattern_match}
     cp -r {working_dir}/* {azure_directory}/.

src/tlo/scenario.py

@@ -63,7 +63,6 @@ def draw_parameters(self, draw_number, rng):
 import argparse
 import datetime
 import json
-import pickle
 from collections.abc import Iterable
 from itertools import product
 from pathlib import Path, PurePosixPath
@@ -72,7 +71,6 @@ def draw_parameters(self, draw_number, rng):
 import numpy as np
 
 from tlo import Date, Simulation, logging
-from tlo.analysis.utils import parse_log_file
 from tlo.util import str_to_pandas_date
 
 logger = logging.getLogger(__name__)
@@ -446,13 +444,6 @@ def run_sample_by_number(self, output_directory, draw_number, sample_number):
             sim.run_simulation_to(to_date=self.scenario.end_date)
             sim.finalise()
 
-        if sim.log_filepath is not None:
-            outputs = parse_log_file(sim.log_filepath)
-            for key, output in outputs.items():
-                if key.startswith("tlo."):
-                    with open(Path(log_config["directory"]) / f"{key}.pickle", "wb") as f:
-                        pickle.dump(output, f)
-
     def run(self):
         """Run all samples for the scenario. Used by `tlo scenario-run` to run the scenario locally"""
         log_config = self.scenario.get_log_config()