custom_log not behaving as expected (#1157)

Co-authored-by: mnjowe <emmanuelmnjowe@gmail.com> Co-authored-by: Asif Tamuri <tamuri@gmail.com>
UCL · May 27, 2024 · 6494277 · 6494277
1 parent de94ffa
commit 6494277
Show file tree

Hide file tree

Showing 11 changed files with 171 additions and 105 deletions.
diff --git a/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py b/src/scripts/breast_cancer_analyses/breast_cancer_analyses.py
@@ -50,8 +50,13 @@
 
 
 def run_sim(service_availability):
+    # configure logging
+    log_config = {
+        'filename': 'LogFile',
+        'directory': outputpath,
+    }
     # Establish the simulation object and set the seed
-    sim = Simulation(start_date=start_date, seed=0)
+    sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
     # Register the appropriate modules
     sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -71,14 +76,11 @@ def run_sim(service_availability):
                  breast_cancer.BreastCancer(resourcefilepath=resourcefilepath)
                  )
 
-    # Establish the logger
-    logfile = sim.configure_logging(filename="LogFile")
-
     # Run the simulation
     sim.make_initial_population(n=popsize)
     sim.simulate(end_date=end_date)
 
-    return logfile
+    return sim.log_filepath
 
 
 def get_summary_stats(logfile):

diff --git a/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/breast_cancer_analyses/calc_5y_survival_following_treatment.py
@@ -41,8 +41,18 @@
 end_date = Date(2025, 1, 1)
 popsize = 10000
 
+
+# Establish the logger and look at only demography
+log_config = {
+    'filename': 'LogFile',
+    'custom_levels': {
+        '*': logging.WARNING,  # <--
+        'tlo.methods.demography': logging.INFO
+                 }
+    }
+
 # Establish the simulation object and set the seed
-sim = Simulation(start_date=start_date, seed=0)
+sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
 # Register the appropriate modules
 sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -68,19 +78,12 @@
 bc_parameters["init_prop_treatment_status_breast_cancer"] = [0.0] * 4
 bc_parameters["init_prob_palliative_care"] = 0.0
 
-# Establish the logger and look at only demography
-custom_levels = {"*": logging.WARNING,  # <--
-                 "tlo.methods.demography": logging.INFO
-                 }
-logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)
-
-
 # Run the simulation
 sim.make_initial_population(n=popsize)
 sim.simulate(end_date=end_date)
 
 # Read the output:
-output = parse_log_file(logfile)
+output = parse_log_file(sim.log_filepath)
 
 # %% Analyse the output:
 

diff --git a/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/oesophagealcancer_analyses/calc_5y_survival_following_treatment.py
@@ -40,8 +40,17 @@
 end_date = Date(2080, 1, 1)
 popsize = 1000
 
+# Establish the logger and look at only demography
+log_config = {
+    'filename': 'LogFile',
+    'custom_levels': {
+        '*': logging.WARNING,  # <--
+        'tlo.methods.demography': logging.INFO
+                 }
+}
+
 # Establish the simulation object and set the seed
-sim = Simulation(start_date=start_date, seed=0)
+sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
 # Register the appropriate modules
 sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -66,19 +75,13 @@
 sim.modules['OesophagealCancer'].parameters["init_prop_treatment_status_oes_cancer"] = [0.0] * 6
 sim.modules['OesophagealCancer'].parameters["init_prob_palliative_care"] = 0.0
 
-# Establish the logger and look at only demography
-custom_levels = {"*": logging.WARNING,  # <--
-                 "tlo.methods.demography": logging.INFO
-                 }
-logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)
-
 
 # Run the simulation
 sim.make_initial_population(n=popsize)
 sim.simulate(end_date=end_date)
 
 # Read the output:
-output = parse_log_file(logfile)
+output = parse_log_file(sim.log_filepath)
 
 # %% Analyse the output:
 

diff --git a/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py b/src/scripts/oesophagealcancer_analyses/oesophagealcancer_analyses.py
@@ -48,8 +48,12 @@
 
 
 def run_sim(service_availability):
+    log_config = {
+        'filename': 'LogFile',
+        'directory': outputpath
+    }
     # Establish the simulation object and set the seed
-    sim = Simulation(start_date=start_date, seed=0)
+    sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
     # Register the appropriate modules
     sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -68,14 +72,11 @@ def run_sim(service_availability):
                  postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath)
                  )
 
-    # Establish the logger
-    logfile = sim.configure_logging(filename="LogFile")
-
     # Run the simulation
     sim.make_initial_population(n=popsize)
     sim.simulate(end_date=end_date)
 
-    return logfile
+    return sim.log_filepath
 
 
 def get_summary_stats(logfile):

diff --git a/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py b/src/scripts/other_adult_cancers_analyses/calc_5y_survival_following_treatment.py
@@ -41,8 +41,20 @@
 end_date = Date(2080, 1, 1)
 popsize = 1000
 
+# Establish the logger and look at only demography
+log_config = {
+    'filename': 'LogFile',
+    'directory': outputpath,
+    'custom_levels': {
+        '*': logging.WARNING,  # <--
+        'tlo.methods.demography': logging.INFO,
+        'tlo.methods.other_adult_cancers': logging.INFO,
+        'tlo.methods.healthsystem': logging.INFO,
+                 }
+}
+
 # Establish the simulation object and set the seed
-sim = Simulation(start_date=start_date, seed=0)
+sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
 # Register the appropriate modules
 sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -66,21 +78,12 @@
 # sim.modules['OtherAdultCancer'].parameters["init_prop_treatment_status_other_adult_cancer"] = [0.0] * 3
 # sim.modules['OtherAdultCancer'].parameters["init_prob_palliative_care"] = 0.0
 
-# Establish the logger and look at only demography
-custom_levels = {"*": logging.WARNING,  # <--
-                 "tlo.methods.demography": logging.INFO,
-                 "tlo.methods.other_adult_cancers": logging.INFO,
-                 'tlo.methods.healthsystem': logging.INFO,
-                 }
-logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)
-
-
 # Run the simulation
 sim.make_initial_population(n=popsize)
 sim.simulate(end_date=end_date)
 
 # Read the output:
-output = parse_log_file(logfile)
+output = parse_log_file(sim.log_filepath)
 
 # %% Analyse the output:
 

diff --git a/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py b/src/scripts/other_adult_cancers_analyses/other_adult_cancers_analyses.py
@@ -44,8 +44,12 @@
 
 
 def run_sim(allow_hsi):
+    log_config = {
+        'filename': 'LogFile',
+        'directory': outputpath
+    }
     # Establish the simulation object and set the seed
-    sim = Simulation(start_date=start_date, seed=0)
+    sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
     # Register the appropriate modules
     sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -60,14 +64,11 @@ def run_sim(allow_hsi):
                  other_adult_cancers.OtherAdultCancer(resourcefilepath=resourcefilepath),
                  )
 
-    # Establish the logger
-    logfile = sim.configure_logging(filename="LogFile")
-
     # Run the simulation
     sim.make_initial_population(n=popsize)
     sim.simulate(end_date=end_date)
 
-    return logfile
+    return sim.log_filepath
 
 
 def get_summary_stats(logfile):

diff --git a/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py b/src/scripts/prostate_cancer_analyses/calc_5y_survival_following_treatment.py
@@ -41,8 +41,17 @@
 end_date = Date(2030, 1, 1)
 popsize = 10000
 
+# Establish the logger and look at only demography
+log_config = {
+    'filename': 'LogFile',
+    'directory': outputpath,
+    'custom_levels': {"*": logging.WARNING,  # <--
+                      "tlo.methods.demography": logging.INFO
+                      }
+    }
+
 # Establish the simulation object and set the seed
-sim = Simulation(start_date=start_date, seed=0)
+sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
 # Register the appropriate modules
 sim.register(demography.Demography(resourcefilepath=resourcefilepath),
@@ -69,19 +78,13 @@
 # sim.modules['ProstateCancer'].parameters["init_prop_treatment_status_prostate_cancer"] = [0.0] * 3
 # sim.modules['ProstateCancer'].parameters["init_prob_palliative_care"] = 0.0
 
-# Establish the logger and look at only demography
-custom_levels = {"*": logging.WARNING,  # <--
-                 "tlo.methods.demography": logging.INFO
-                 }
-logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)
-
 
 # Run the simulation
 sim.make_initial_population(n=popsize)
 sim.simulate(end_date=end_date)
 
 # Read the output:
-output = parse_log_file(logfile)
+output = parse_log_file(sim.log_filepath)
 
 # %% Analyse the output:
 
@@ -106,16 +109,17 @@
 
 # calc % of those that were alive 5 years after starting treatment (not died of any cause):
 1 - (
-    len(cohort_treated.loc[cohort_treated['days_treatment_to_death'] < (5*365.25)]) /
-    len(cohort_treated)
+        len(cohort_treated.loc[cohort_treated['days_treatment_to_death'] < (5 * 365.25)]) /
+        len(cohort_treated)
 )  # 0.77
 
 # calc % of those that had not died of prostate cancer 5 years after starting treatment (could have died of another
 # cause):
 1 - (
-    len(
-        cohort_treated.loc[
-            (cohort_treated['cause'] == 'ProstateCancer') & (cohort_treated['days_treatment_to_death'] < (5*365.25))
-            ]
-    ) / len(cohort_treated)
-)   # 0.87
+        len(
+            cohort_treated.loc[
+                (cohort_treated['cause'] == 'ProstateCancer') & (
+                            cohort_treated['days_treatment_to_death'] < (5 * 365.25))
+                ]
+        ) / len(cohort_treated)
+)  # 0.87
diff --git a/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py b/src/scripts/prostate_cancer_analyses/prostate_cancer_analyses.py
@@ -50,8 +50,12 @@
 
 
 def run_sim(service_availability):
+    log_config = {
+        'filename': 'LogFile',
+        'directory': outputpath
+    }
     # Establish the simulation object and set the seed
-    sim = Simulation(start_date=start_date, seed=0)
+    sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
 
     # Register the appropriate modules
     sim.register(care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath),
@@ -71,14 +75,11 @@ def run_sim(service_availability):
                  postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath)
                  )
 
-    # Establish the logger
-    logfile = sim.configure_logging(filename="LogFile")
-
     # Run the simulation
     sim.make_initial_population(n=popsize)
     sim.simulate(end_date=end_date)
 
-    return logfile
+    return sim.log_filepath
 
 
 def get_summary_stats(logfile):
@@ -248,4 +249,4 @@ def get_cols_excl_none(allcols, stub):
 
 
 # ** 5-year survival following treatment
-# See sepaerate file
+# See separate file
diff --git a/src/scripts/schistosomiasis/schisto_analysis.py b/src/scripts/schistosomiasis/schisto_analysis.py
@@ -16,14 +16,24 @@
 
 
 def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True):
+    outputpath = Path("./outputs")  # folder for convenience of storing outputs
     # The resource files
     resourcefilepath = Path("./resources")
     start_date = Date(2010, 1, 1)
     end_date = Date(2011, 2, 1)
     popsize = popsize
 
+    # Sets all modules to WARNING threshold, then alters schisto to INFO
+    log_config = {
+        'filename': 'LogFile',
+        'directory': outputpath,
+        'custom_levels ': {"*": logging.WARNING,
+                           "tlo.methods.schisto": logging.INFO,
+                           }
+    }
+
     # Establish the simulation object
-    sim = Simulation(start_date=start_date)
+    sim = Simulation(start_date=start_date, log_config=log_config)
 
     # Register the appropriate modules
     sim.register(demography.Demography(resourcefilepath=resourcefilepath))
@@ -36,13 +46,6 @@ def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True):
     if mansoni:
         sim.register(schisto.Schisto_Mansoni(resourcefilepath=resourcefilepath, symptoms_and_HSI=False))
 
-    # Sets all modules to WARNING threshold, then alters schisto to INFO
-    custom_levels = {"*": logging.WARNING,
-                     "tlo.methods.schisto": logging.INFO,
-                     }
-    # configure logging after registering modules with custom levels
-    logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)
-
     # Run the simulation
     sim.seed_rngs(int(np.random.uniform(0, 1) * 0 + 1000))
     # initialise the population
@@ -51,12 +54,13 @@ def run_simulation(popsize=10000, haem=True, mansoni=True, mda_execute=True):
     # # start the simulation
     sim.simulate(end_date=end_date)
     # fh.flush()
-    output = parse_log_file(logfile)
+    output = parse_log_file(sim.log_filepath)
     return sim, output
 
 
 sim, output = run_simulation(popsize=10000, haem=True, mansoni=False, mda_execute=False)
 
+
 # ---------------------------------------------------------------------------------------------------------
 #   Saving the results - prevalence, mwb, dalys and parameters used
 # ---------------------------------------------------------------------------------------------------------
@@ -286,6 +290,8 @@ def plot_prev_high_infection_per_district(infection, high_inf_distr):
 districts_prevalence, districts_mwb, districts_high_inf_prev = get_values_per_district('Haematobium')
 expected_district_prevalence = get_expected_prevalence('Haematobium')
 plot_prevalence_per_district('Haematobium', districts_prevalence, expected_district_prevalence)
+
+
 # plot_mwb_per_district('Haematobium', districts_mwb)
 # plot_prev_high_infection_per_district('Haematobium', districts_high_inf_prev)
 
@@ -427,7 +433,6 @@ def plot_high_inf_years():
 plot_prevalent_years()
 plot_high_inf_years()
 
-
 # DALYS
 loger_daly = output['tlo.methods.healthburden']["DALYS"]
 loger_daly.drop(columns=['sex', 'YLL_Demography_Other'], inplace=True)