diff --git a/.github/workflows/calibration.yaml b/.github/workflows/calibration.yaml index af11fdcf83..b589faf014 100644 --- a/.github/workflows/calibration.yaml +++ b/.github/workflows/calibration.yaml @@ -153,6 +153,7 @@ jobs: mkdir -p "${task_output_dir}" tlo scenario-run --output-dir "${task_output_dir}" --draw "${draw}" ${{ matrix.index }} "${{ needs.setup.outputs.worktree_path }}/src/scripts/calibration_analyses/scenarios/long_run_all_diseases.py" + tlo parse-log ${task_output_dir} working-directory: "${{ needs.setup.outputs.worktree_path }}" # Do the postprocessing diff --git a/src/scripts/dev/scenarios/playing_22.py b/src/scripts/dev/scenarios/playing_22.py index 6357ddeff8..c1dabe1ec8 100644 --- a/src/scripts/dev/scenarios/playing_22.py +++ b/src/scripts/dev/scenarios/playing_22.py @@ -14,14 +14,13 @@ def __init__(self): super().__init__() self.seed = 655123742 self.start_date = Date(2010, 1, 1) - self.end_date = Date(2011, 1, 1) - self.pop_size = 200 - self.number_of_draws = 5 - self.runs_per_draw = 1 + self.end_date = Date(2013, 1, 1) + self.pop_size = 1000 + self.number_of_draws = 3 + self.runs_per_draw = 3 def log_configuration(self): return { - # 'filename': 'my-scenario', 'directory': './outputs', 'custom_levels': { '*': logging.INFO, @@ -39,9 +38,8 @@ def modules(self): def draw_parameters(self, draw_number, rng): return { - 'Lifestyle': { - 'init_p_urban': rng.randint(10, 20) / 100.0, - 'init_p_high_sugar': 0.52, + 'Demography': { + 'max_age_initial': [80, 90, 100][draw_number], }, } diff --git a/src/tlo/cli.py b/src/tlo/cli.py index 578c3a6619..6824dd1045 100644 --- a/src/tlo/cli.py +++ b/src/tlo/cli.py @@ -4,6 +4,7 @@ import json import math import os +import pickle import tempfile from collections import defaultdict from pathlib import Path @@ -22,6 +23,7 @@ from azure.storage.fileshare import ShareClient, ShareDirectoryClient, ShareFileClient from git import Repo +from tlo.analysis.utils import parse_log_file from tlo.scenario import SampleRunner, ScenarioLoader JOB_LABEL_PADDING = len("State transition time") @@ -88,6 +90,21 @@ def scenario_run(scenario_file, draw_only, draw: tuple, output_dir=None, scenari else: runner.run() +@cli.command() +@click.argument("LOG_DIRECTORY", type=click.Path(exists=True)) +def parse_log(log_directory): + assert os.path.isdir(log_directory), f"{log_directory} must be a directory" + + path = Path(log_directory) + + log_files = list(path.glob("*.log")) + assert len(log_files) == 1, f"directory {log_directory} must contain exactly one file with extension .log" + + outputs = parse_log_file(log_files[0]) + for key, output in outputs.items(): + if key.startswith("tlo."): + with open(path / f"{key}.pickle", "wb") as f: + pickle.dump(output, f) @cli.command() @click.argument("scenario_file", type=click.Path(exists=True)) @@ -237,6 +254,7 @@ def batch_submit(ctx, scenario_file, asserts_on, more_memory, keep_pool_alive, i pip install -r requirements/base.txt env | grep "^AZ_" | while read line; do echo "$line"; done {py_opt} tlo --config-file tlo.example.conf batch-run {azure_run_json} {working_dir} {{draw_number}} {{run_number}} + tlo --config-file tlo.example.conf parse-log {working_dir}/{{draw_number}}/{{run_number}} cp {task_dir}/std*.txt {working_dir}/{{draw_number}}/{{run_number}}/. gzip {working_dir}/{{draw_number}}/{{run_number}}/*.{gzip_pattern_match} cp -r {working_dir}/* {azure_directory}/. diff --git a/src/tlo/scenario.py b/src/tlo/scenario.py index f64325f9ec..d2b23c8646 100644 --- a/src/tlo/scenario.py +++ b/src/tlo/scenario.py @@ -63,7 +63,6 @@ def draw_parameters(self, draw_number, rng): import argparse import datetime import json -import pickle from collections.abc import Iterable from itertools import product from pathlib import Path, PurePosixPath @@ -72,7 +71,6 @@ def draw_parameters(self, draw_number, rng): import numpy as np from tlo import Date, Simulation, logging -from tlo.analysis.utils import parse_log_file from tlo.util import str_to_pandas_date logger = logging.getLogger(__name__) @@ -446,13 +444,6 @@ def run_sample_by_number(self, output_directory, draw_number, sample_number): sim.run_simulation_to(to_date=self.scenario.end_date) sim.finalise() - if sim.log_filepath is not None: - outputs = parse_log_file(sim.log_filepath) - for key, output in outputs.items(): - if key.startswith("tlo."): - with open(Path(log_config["directory"]) / f"{key}.pickle", "wb") as f: - pickle.dump(output, f) - def run(self): """Run all samples for the scenario. Used by `tlo scenario-run` to run the scenario locally""" log_config = self.scenario.get_log_config()