Convert excel files to csv files. Joe's modules

src/scripts/contraception/f_steril_use_2010vs2020.py

@@ -1,7 +1,7 @@
 """
 A helper script to see the numbers of women of reproductive age having female sterilisation per 5-years age categories +
 total, and the number of all women in the population in 2010 and 2020, to help to calibrate the intervention multipliers
-(saved in ResourceFile_Contraception.xlsx in the sheets Interventions_Pop & Interventions_PPFP).
+(saved in ResourceFile_Contraception folder in the Interventions_Pop & Interventions_PPFP CSV files).
 """
 from pathlib import Path
 

src/tlo/methods/care_of_women_during_pregnancy.py

@@ -12,6 +12,7 @@
 from tlo.methods.labour import LabourOnsetEvent
 from tlo.methods.malaria import HSI_MalariaIPTp
 from tlo.methods.tb import HSI_Tb_ScreeningAndRefer
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -176,8 +177,8 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare',
+                                            files=['parameter_values'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

src/tlo/methods/contraception.py

@@ -8,7 +8,7 @@
 from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging
 from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
 from tlo.methods.hsi_event import HSI_Event
-from tlo.util import random_date, sample_outcome, transition_states
+from tlo.util import random_date, read_csv_files, sample_outcome, transition_states
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -164,7 +164,7 @@ def read_parameters(self, data_folder):
         """Import the relevant sheets from the ResourceFile (excel workbook) and declare values for other parameters
         (CSV ResourceFile).
         """
-        workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', sheet_name=None)
+        workbook = read_csv_files(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception')
 
         # Import selected sheets from the workbook as the parameters
         sheet_names = [
@@ -1346,10 +1346,10 @@ def __init__(self, *args):
         super().__init__(name='Labour')
 
     def read_parameters(self, *args):
-        parameter_dataframe = pd.read_excel(self.sim.modules['Contraception'].resourcefilepath /
+        parameter_dataframe = read_csv_files(self.sim.modules['Contraception'].resourcefilepath /
                                             'contraception' /
-                                            'ResourceFile_Contraception.xlsx',
-                                            sheet_name='simplified_labour_parameters')
+                                            'ResourceFile_Contraception',
+                                            files=['simplified_labour_parameters'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

src/tlo/methods/labour.py

@@ -17,7 +17,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.postnatal_supervisor import PostnatalWeekOneMaternalEvent
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -620,9 +620,9 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth'
-                                                                          'Attendance.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth'
+                                                                          'Attendance',
+                                            files=['parameter_values'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

src/tlo/methods/newborn_outcomes.py

@@ -10,7 +10,7 @@
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.postnatal_supervisor import PostnatalWeekOneNeonatalEvent
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -311,8 +311,8 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
 
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes',
+                                            files=['parameter_values'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
         # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module

src/tlo/methods/postnatal_supervisor.py

@@ -9,6 +9,7 @@
 from tlo.methods import Metadata, postnatal_supervisor_lm, pregnancy_helper_functions
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_event import HSI_Event
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -230,8 +231,8 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor',
+                                            files=['parameter_values'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

src/tlo/methods/pregnancy_supervisor.py

@@ -27,7 +27,7 @@
 )
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -434,8 +434,8 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
         # load parameters from the resource file
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor',
+                                            files=['parameter_values'])
         self.load_parameters_from_dataframe(parameter_dataframe)
 
         # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module.

src/tlo/methods/simplified_births.py

@@ -11,6 +11,7 @@
 from tlo import DateOffset, Module, Parameter, Property, Types, logging
 from tlo.events import PopulationScopeEventMixin, RegularEvent
 from tlo.methods.contraception import get_medium_variant_asfr_from_wpp_resourcefile
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -111,7 +112,8 @@ def read_parameters(self, data_folder):
         self.parameters['months_between_pregnancy_and_delivery'] = 9
 
         # Breastfeeding status for newborns (importing from the Newborn resourcefile)
-        rf = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx')
+        # pylint: disable=E1101
+        rf = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes')
         param_as_string = rf.loc[rf.parameter_name == 'prob_breastfeeding_type']['value'].iloc[0]
         parameter = json.loads(param_as_string)[0]
         self.parameters['prob_breastfeeding_type'] = parameter

tests/test_contraception.py

@@ -12,6 +12,7 @@
 from tlo.methods import contraception, demography, enhanced_lifestyle, healthsystem, symptommanager
 from tlo.methods.contraception import HSI_Contraception_FamilyPlanningAppt
 from tlo.methods.hiv import DummyHivModule
+from tlo.util import read_csv_files
 
 
 def run_sim(tmpdir,
@@ -918,8 +919,8 @@ def test_input_probs_sum():
 
     # Import relevant sheets from the workbook
     resourcefilepath = Path(os.path.dirname(__file__)) / '../resources'
-    workbook = pd.read_excel(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx',
-                             sheet_name=None)
+    workbook = read_csv_files(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception',
+                             files=None)
     sheet_names = [
         'Initiation_ByMethod',
         'Interventions_Pop',