diff --git a/src/scripts/calibration_analyses/analysis_scripts/analysis_compare_appt_usage_real_and_simulation.py b/src/scripts/calibration_analyses/analysis_scripts/analysis_compare_appt_usage_real_and_simulation.py index ee3ea69cce..addd34c86b 100644 --- a/src/scripts/calibration_analyses/analysis_scripts/analysis_compare_appt_usage_real_and_simulation.py +++ b/src/scripts/calibration_analyses/analysis_scripts/analysis_compare_appt_usage_real_and_simulation.py @@ -83,7 +83,6 @@ def unpack_nested_dict_in_series(_raw: pd.Series): .loc[pd.to_datetime(_df['date']).between(*TARGET_PERIOD), 'Number_By_Appt_Type_Code_And_Level'] \ .pipe(unpack_nested_dict_in_series) \ .rename(columns=appt_dict, level=1) \ - .rename(columns={'1b': '2'}, level=0) \ .groupby(level=[0, 1], axis=1).sum() \ .mean(axis=0) # mean over each year (row) @@ -119,7 +118,6 @@ def unpack_nested_dict_in_series(_raw: pd.Series): .loc[pd.to_datetime(_df['date']).between(*TARGET_PERIOD), 'Number_By_Appt_Type_Code_And_Level'] \ .pipe(unpack_nested_dict_in_series) \ .rename(columns=appt_dict, level=1) \ - .rename(columns={'1b': '2'}, level=0) \ .groupby(level=[0, 1], axis=1).sum() \ .mean(axis=0) # mean over each year (row) @@ -213,6 +211,10 @@ def get_simulation_usage_with_confidence_interval(results_folder: Path) -> pd.Da model_output.drop(columns='name', inplace=True) model_output.reset_index(drop=True, inplace=True) + # drop dummy PharmDispensing for HCW paper + model_output = model_output.drop(index=model_output[model_output.appt_type == 'PharmDispensing'].index + ).reset_index(drop=True) + return model_output @@ -276,9 +278,10 @@ def get_real_usage(resourcefilepath, adjusted=True) -> pd.DataFrame: # get facility_level for each record real_usage = real_usage.merge(mfl[['Facility_ID', 'Facility_Level']], left_on='Facility_ID', right_on='Facility_ID') - # adjust annual MentalAll usage using annual reporting rates - if adjusted: - real_usage = adjust_real_usage_on_mentalall(real_usage) + # adjust annual MentalAll usage using annual reporting rates if needed + # for now not adjust it considering very low reporting rates and better match with Model usage + # if adjusted: + # real_usage = adjust_real_usage_on_mentalall(real_usage) # assign date to each record real_usage['date'] = pd.to_datetime({'year': real_usage['Year'], 'month': real_usage['Month'], 'day': 1}) @@ -302,7 +305,7 @@ def get_real_usage(resourcefilepath, adjusted=True) -> pd.DataFrame: annual_usage_by_level = pd.concat([totals_by_year.reset_index(), totals_by_year_TB.reset_index()], axis=0) # group levels 1b and 2 into 2 - annual_usage_by_level['Facility_Level'] = annual_usage_by_level['Facility_Level'].replace({'1b': '2'}) + # annual_usage_by_level['Facility_Level'] = annual_usage_by_level['Facility_Level'].replace({'1b': '2'}) annual_usage_by_level = annual_usage_by_level.groupby( ['Year', 'Appt_Type', 'Facility_Level'])['Usage'].sum().reset_index() @@ -413,10 +416,10 @@ def format_and_save(_fig, _ax, _name_of_plot): usage_all = usage_all / 1e6 # plot - name_of_plot = 'Model vs Data on average annual health service volume' + name_of_plot = 'Average annual health service volume on national level' fig, ax = plt.subplots() usage_all.plot(kind='bar', stacked=True, color=appt_color_dict, rot=0, ax=ax) - ax.set_ylabel('Health service volume in millions') + ax.set_ylabel('Number of visits') ax.set(xlabel=None) plt.legend(loc='center left', bbox_to_anchor=(1.0, 0.5), title='Appointment type', fontsize=9) plt.title(name_of_plot) @@ -445,15 +448,15 @@ def format_real_usage(): _rel_diff['upper_error'] = (_rel_diff['upper'] - _rel_diff['mean']) _asymmetric_error = [_rel_diff['lower_error'].values, _rel_diff['upper_error'].values] - _rel_diff = pd.DataFrame(_rel_diff['mean']) + _rel_diff = pd.DataFrame(_rel_diff['mean']).clip(lower=0.1, upper=10.0) return _rel_diff, _asymmetric_error rel_diff_real, err_real = format_rel_diff(adjusted=True) # plot - name_of_plot = 'Model vs Data usage per appointment type at all facility levels' \ - '\n[Model average annual 95% CI, Adjusted Data average annual]' + name_of_plot = 'Model vs Data on health service volume per appointment type' \ + '\n[Model average annual 95% CI, Data average annual]' fig, ax = plt.subplots() ax.errorbar(rel_diff_real.index.values, rel_diff_real['mean'].values, @@ -462,9 +465,9 @@ def format_real_usage(): for idx in rel_diff_real.index: if not pd.isna(rel_diff_real.loc[idx, 'mean']): ax.text(idx, - rel_diff_real.loc[idx, 'mean'] * (1 + 0.2), + rel_diff_real.loc[idx, 'mean'] * (1 + 0.3), round(rel_diff_real.loc[idx, 'mean'], 1), - ha='left', fontsize=8) + ha='center', fontsize=8) ax.axhline(1.0, color='r') format_and_save(fig, ax, name_of_plot) diff --git a/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py b/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py index cacde4b103..8f63a182f9 100644 --- a/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py +++ b/src/scripts/calibration_analyses/analysis_scripts/analysis_hsi_descriptions.py @@ -391,7 +391,7 @@ def get_share_of_time_used_for_each_officer_at_each_level(_df): ).set_index('Facility_ID') def find_level_for_facility(id): - return mfl.loc[id].Facility_Level if mfl.loc[id].Facility_Level != '1b' else '2' + return mfl.loc[id].Facility_Level color_for_level = {'0': 'blue', '1a': 'yellow', '1b': 'green', '2': 'grey', '3': 'orange', '4': 'black', '5': 'white'} @@ -422,7 +422,6 @@ def find_level_for_facility(id): capacity_unstacked_average = capacity_by_facility.unstack().mean() # levels = [find_level_for_facility(i) if i != 'All' else 'All' for i in capacity_unstacked_average.index] xpos_for_level = dict(zip((color_for_level.keys()), range(len(color_for_level)))) - xpos_for_level.update({'1b': 2, '2': 2, '3': 3, '4': 4, '5': 5}) for id, val in capacity_unstacked_average.items(): if id != 'All': _level = find_level_for_facility(id) diff --git a/src/scripts/healthsystem/descriptions_of_input_data/analysis_describe_healthsystem_capabilities.py b/src/scripts/healthsystem/descriptions_of_input_data/analysis_describe_healthsystem_capabilities.py index 8cfd59e745..f252f8ce55 100644 --- a/src/scripts/healthsystem/descriptions_of_input_data/analysis_describe_healthsystem_capabilities.py +++ b/src/scripts/healthsystem/descriptions_of_input_data/analysis_describe_healthsystem_capabilities.py @@ -1,53 +1,196 @@ """ -This file produces a nice plot of the capabilities of the healthsystem in terms of the hours available for -different cadres of healthcare workers. +This file produces histograms of the healthsystem capabilities \ +in terms of staff allocation and daily capabilities in minutes per cadre per facility level. """ -# %% - from pathlib import Path import pandas as pd from matplotlib import pyplot as plt +from matplotlib.ticker import ScalarFormatter -resourcefilepath = Path("./resources") +# Get the path of the folder that stores the data - three scenarios: actual, funded, funded_plus +workingpath = Path('./resources/healthsystem/human_resources') +wp_actual = workingpath / 'actual' +wp_funded_plus = workingpath / 'funded_plus' -# %% +# Define the path of output histograms - three scenarios: actual, funded, funded_plus +outputpath = Path('./outputs/healthsystem/human_resources/actual') +op_actual = outputpath / 'actual' +op_funded_plus = outputpath / 'funded_plus' -outputpath = Path("./outputs") # folder for convenience of storing outputs +# Read actual data +data = pd.read_csv(wp_actual / 'ResourceFile_Daily_Capabilities.csv') +# Read funded_plus data +data_funded_plus = pd.read_csv(wp_funded_plus / 'ResourceFile_Daily_Capabilities.csv') -data = pd.read_csv( - Path(resourcefilepath) / "ResourceFile_Daily_Capabilities.csv" -) -# [['Total_Minutes_Per_Day','Officer_Type','District']] +# ***for actual scenario*** +# MINUTES PER HEALTH OFFICER CATEGORY BY DISTRICT +data_districts = data.dropna(inplace=False) +dat = pd.DataFrame(data_districts.groupby(['District', 'Officer_Category'], as_index=False)['Total_Mins_Per_Day'].sum()) +dat['Total_Mins_Per_Day'] = dat['Total_Mins_Per_Day'] / 100000 +tab = dat.pivot(index='District', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day in 1e5', fontsize='large') +plt.xlabel('District', fontsize='large') -data = data.dropna() -# data['District'] = data['District'].fillna('National') +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') -# do some re-grouping to make a more manageable number of health cadres: -data['Officer_Type'] = data['Officer_Type'].replace('DCSA', 'CHW') -data['Officer_Type'] = data['Officer_Type'].replace(['Lab Officer', 'Lab Technician', 'Lab Assistant'], 'Lab Support') -data['Officer_Type'] = data['Officer_Type'].replace(['Radiographer', 'Radiography Technician'], 'Radiography') -data['Officer_Type'] = data['Officer_Type'].replace(['Nurse Officer', 'Nutrition Staff', 'Med. Assistant'], 'Nurse') -data['Officer_Type'] = data['Officer_Type'].replace('Nurse Midwife Technician', 'MidWife') -data['Officer_Type'] = data['Officer_Type'].replace(['Pharmacist', 'Pharm Technician', 'Pharm Assistant'], 'Pharmacy') -data['Officer_Type'] = data['Officer_Type'].replace(['Medical Officer / Specialist', 'Clinical Officer / Technician'], - 'Clinician') -data['Officer_Type'] = data['Officer_Type'].replace(['Dental Therapist'], 'Dentist') +plt.savefig(outputpath / 'health_officer_minutes_per_district.pdf', bbox_inches='tight') -# MINUTES PER HEALTH OFFICER TYPE BY DISTRICT: -dat = pd.DataFrame(data.groupby(['District', 'Officer_Type'], as_index=False)['Total_Minutes_Per_Day'].sum()) -tab = dat.pivot(index='District', columns='Officer_Type', values='Total_Minutes_Per_Day') -ax = tab.plot.bar(stacked=True) -plt.ylabel('Minutes per day') -plt.xlabel('District') +# STAFF COUNTS PER HEALTH OFFICER CATEGORY BY DISTRICT +data_districts = data.dropna(inplace=False) +dat = pd.DataFrame(data_districts.groupby(['District', 'Officer_Category'], as_index=False)['Staff_Count'].sum()) +dat['Staff_Count'] = dat['Staff_Count'] / 1000 +tab = dat.pivot(index='District', columns='Officer_Category', values='Staff_Count') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Staff counts in 1e3', fontsize='large') +plt.xlabel('District', fontsize='large') ax.legend(ncol=3, bbox_to_anchor=(0, 1), - loc='lower left', fontsize='small') + loc='lower left', fontsize='medium') -plt.savefig(outputpath / 'health_officer_minutes_per_district.pdf', bbox_inches='tight') -plt.show() +plt.savefig(outputpath / 'staff_allocation_per_district.pdf', bbox_inches='tight') + + +# MINUTES PER HEALTH OFFICER CATEGORY BY LEVEL +dat = pd.DataFrame(data.groupby(['Facility_Level', 'Officer_Category'], as_index=False)['Total_Mins_Per_Day'].sum()) +dat['Total_Mins_Per_Day'] = dat['Total_Mins_Per_Day'] / 100000 +tab = dat.pivot(index='Facility_Level', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +# ax = tab.plot.bar(stacked=True, log=True) +plt.ylabel('Average Total Minutes per Day in 1e5', fontsize='large') +plt.xlabel('Facility level', fontsize='large') + +ax.tick_params(axis='x', rotation=0) + +formatter = ScalarFormatter() +formatter.set_scientific(False) +ax.yaxis.set_major_formatter(formatter) + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_level.pdf', bbox_inches='tight') + +# STAFF COUNTS PER HEALTH OFFICER CATEGORY BY LEVEL +dat = pd.DataFrame(data.groupby(['Facility_Level', 'Officer_Category'], as_index=False)['Staff_Count'].sum()) +dat['Staff_Count'] = dat['Staff_Count'] / 1000 +tab = dat.pivot(index='Facility_Level', columns='Officer_Category', values='Staff_Count') +ax = tab.plot.bar(stacked=True, fontsize='medium') +# ax = tab.plot.bar(stacked=True, log=True) +plt.ylabel('Staff counts in 1e3', fontsize='large') +plt.xlabel('Facility level', fontsize='large') + +ax.tick_params(axis='x', rotation=0) + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'staff_allocation_per_level.pdf', bbox_inches='tight') + + +# MINUTES PER HEALTH OFFICER CATEGORY BY LEVEL + +# Level 0 +data_level = data.loc[data['Facility_Level'] == '0', :] +tab = data_level.pivot(index='District', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 0', fontsize='large') +plt.xlabel('District', fontsize='large') + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_0.pdf', bbox_inches='tight') + +# Level 1a +data_level = data.loc[data['Facility_Level'] == '1a', :] +tab = data_level.pivot(index='District', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 1a', fontsize='large') +plt.xlabel('District', fontsize='large') + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_1a.pdf', bbox_inches='tight') + +# Level 1b +data_level = data.loc[data['Facility_Level'] == '1b', :] +tab = data_level.pivot(index='District', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 1b', fontsize='large') +plt.xlabel('District', fontsize='large') + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_1b.pdf', bbox_inches='tight') + +# Level 2 +data_level = data.loc[data['Facility_Level'] == '2', :] +tab = data_level.pivot(index='District', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 2', fontsize='large') +plt.xlabel('District', fontsize='large') + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_2.pdf', bbox_inches='tight') + +# Level 3 +data_level = data.loc[data['Facility_Level'] == '3', :] +tab = data_level.pivot(index='Region', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 3', fontsize='large') +plt.xlabel('Regional Referral Hospital', fontsize='large') +ax.tick_params(axis='x', rotation=0) + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_3.pdf', bbox_inches='tight') + +# Level 4 +data_level = data.loc[data['Facility_Level'] == '4', :] +tab = data_level.pivot(index='Facility_Name', columns='Officer_Category', values='Total_Mins_Per_Day') +ax = tab.plot.bar(stacked=True, width=0.1, fontsize='medium') +plt.ylabel('Average Total Minutes per Day at Level 4', fontsize='large') +plt.xlabel('National resource hospital', fontsize='large') +ax.tick_params(axis='x', rotation=0) + +ax.legend(ncol=3, bbox_to_anchor=(0, 1), + loc='lower left', fontsize='medium') + +plt.savefig(outputpath / 'health_officer_minutes_per_district_level_4.pdf', bbox_inches='tight') + +# ***end of actual scenario*** + +# ***compare actual and funded_plus scenarios*** +total_actual = data.drop_duplicates().groupby(['Officer_Category']).agg( + {'Total_Mins_Per_Day': 'sum', 'Staff_Count': 'sum'}).reset_index() +total_actual['Total_Mins_Per_Year'] = total_actual['Total_Mins_Per_Day'] * 365.25 +total_actual['Scenario'] = 'Actual' +total_actual[['Abs_Change_Staff_Count', 'Rel_Change_Staff_Count', 'Abs_Change_Total_Mins', 'Rel_Change_Total_Mins']] = 0 + +total_funded_plus = data_funded_plus.drop_duplicates().groupby(['Officer_Category']).agg( + {'Total_Mins_Per_Day': 'sum', 'Staff_Count': 'sum'}).reset_index() +total_funded_plus['Total_Mins_Per_Year'] = total_funded_plus['Total_Mins_Per_Day'] * 365.25 +total_funded_plus['Scenario'] = 'Establishment' + +assert (total_actual.Officer_Category == total_funded_plus.Officer_Category).all() +total_funded_plus['Abs_Change_Staff_Count'] = total_funded_plus['Staff_Count'] - total_actual['Staff_Count'] +total_funded_plus['Rel_Change_Staff_Count'] = (total_funded_plus['Staff_Count'] - total_actual['Staff_Count'] + ) / total_actual['Staff_Count'] +total_funded_plus['Abs_Change_Total_Mins'] = (total_funded_plus['Total_Mins_Per_Year'] - + total_actual['Total_Mins_Per_Year']) +total_funded_plus['Rel_Change_Total_Mins'] = (total_funded_plus['Total_Mins_Per_Year'] - + total_actual['Total_Mins_Per_Year'] + ) / total_actual['Total_Mins_Per_Year'] -# %% +total = pd.concat([total_actual, total_funded_plus]).reset_index(drop=True) diff --git a/src/scripts/healthsystem/descriptions_of_input_data/analysis_sankey_coarse_officer_and_appt.ipynb b/src/scripts/healthsystem/descriptions_of_input_data/analysis_sankey_coarse_officer_and_appt.ipynb new file mode 100644 index 0000000000..6a874ce64d --- /dev/null +++ b/src/scripts/healthsystem/descriptions_of_input_data/analysis_sankey_coarse_officer_and_appt.ipynb @@ -0,0 +1,292 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "This file uses floweaver to generate Sankey diagrams that map coarse officers to appointments.\n", + "\n", + "Below is the instruction to run the file.\n", + "\n", + "### Install floweaver in Anaconda Prompt (if use Jupyter Notebook) / PyCharm Terminal:\n", + "\n", + "pip install floweaver\n", + "\n", + "pip install ipysankeywidget\n", + "\n", + "jupyter nbextension enable --py --sys-prefix ipysankeywidget\n", + "\n", + "jupyter notebook (to open jupyter notebook)\n", + "\n", + "### To display and save the output figures:\n", + "Select Start Jupyter Server from the Jupyter Actions Menu (lightbulb icon next to Run All cells icon)\n", + "\n", + "Open Event Log\n", + "\n", + "Open in Browser\n", + "\n", + "Find the script and run all cells" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "913300ab", + "metadata": { + "pycharm": { + "is_executing": true, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tlo\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "from ipysankeywidget import SankeyWidget\n", + "from matplotlib import pyplot as plt\n", + "from floweaver import *\n", + "from pathlib import Path\n", + "from ipywidgets import HBox, VBox\n", + "\n", + "# get the tlo path\n", + "tlopath = Path(tlo.__file__).parent.parent.parent\n", + "\n", + "# Get the path of current folder that stores the data\n", + "workingpath = tlopath / Path('resources/healthsystem/human_resources/definitions')\n", + "\n", + "# Define the path of output Sankeys\n", + "outputpath = tlopath / Path('outputs/healthsystem/human_resources/sankey_diagrams')\n", + "\n", + "# Read the data of appointment time table\n", + "appointment = pd.read_csv(workingpath / 'ResourceFile_Appt_Time_Table.csv')\n", + "\n", + "# Rename\n", + "appointment.loc[appointment['Officer_Category'] == 'Nursing_and_Midwifery',\n", + " 'Officer_Category'] = 'Nursing and Midwifery'\n", + "\n", + "# Read the data of appointment types table\n", + "appt_types = pd.read_csv(workingpath / 'ResourceFile_Appt_Types_Table.csv')\n", + "# Rename\n", + "appt_types.loc[appt_types['Appt_Cat'] == 'GENERAL_INPATIENT_AND_OUTPATIENT_CARE',\n", + " 'Appt_Cat'] = 'IPOP'\n", + "appt_types.loc[appt_types['Appt_Cat'] == 'Nutrition',\n", + " 'Appt_Cat'] = 'NUTRITION'\n", + "appt_types.loc[appt_types['Appt_Cat'] == 'Misc',\n", + " 'Appt_Cat'] = 'MISC'\n", + "appt_types.loc[appt_types['Appt_Cat'] == 'Mental_Health',\n", + " 'Appt_Cat'] = 'MENTAL'\n", + "\n", + "# Merge appt category to the time table\n", + "appointment = appointment.merge(appt_types[['Appt_Type_Code', 'Appt_Cat']],\n", + " on='Appt_Type_Code', how='left')\n", + "\n", + "# Add prefix 'Facility_Level'\n", + "appointment['Facility_Level'] = 'Facility_Level_' + appointment['Facility_Level'].astype(str)\n", + "\n", + "# Draw a diagram using appointment time table itself.\n", + "# Currentlt, we do not know how many appointments of a type at a level happened or \\\n", + "# how many minutes of an officer category at a level go to that appointment.\n", + "# We consider the mapping between officer categories and appointment types, \\\n", + "# which can be derived from the appointment time table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3e06de5", + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + }, + "outputs": [], + "source": [ + "# The flow maps 9 officer categories and 11 appt cateogories at all levels\n", + "flow_coarse_officer_appt = pd.DataFrame(\n", + " appointment.groupby(['Officer_Category', 'Appt_Cat', 'Facility_Level'],\n", + " dropna=False, sort=False).sum()\n", + ").reset_index()\n", + "# Drop column of minutes and add column 'value'\n", + "flow_coarse_officer_appt.drop(columns = ['Time_Taken_Mins'], inplace=True)\n", + "flow_coarse_officer_appt['value'] = 1\n", + "\n", + "# Add 'source' and 'target' columns\n", + "flow_coarse_officer_appt['source'] = 'Officer_Category'\n", + "flow_coarse_officer_appt['target'] = 'Appt_Cat'\n", + "\n", + "size = dict(width=800, height=800, margins=dict(left=180, right=180))\n", + "\n", + "# Different ways to order nodes\n", + "# Sorted\n", + "# partition_officer_cat = Partition.Simple('Officer_Category',\n", + "# np.unique(flow_coarse_officer_appt['Officer_Category']))\n", + "# Unsorted\n", + "# partition_officer_cat = Partition.Simple('Officer_Category',\n", + "# pd.unique(pd.Series(flow_coarse_officer_appt['Officer_Category'])))\n", + "# Fixed\n", + "partition_officer_cat = Partition.Simple('Officer_Category',\n", + " pd.array(['DCSA', 'Clinical', 'Nursing and Midwifery', 'Pharmacy',\n", + " 'Laboratory', 'Dental', 'Radiography', 'Mental']))\n", + "\n", + "# partition_appt_cat = Partition.Simple('Appt_Cat',\n", + "# np.unique(flow_coarse_officer_appt['Appt_Cat']))\n", + "partition_appt_cat = Partition.Simple('Appt_Cat',\n", + " pd.array(['ConWithDCSA', 'IPOP', 'RMNCH', 'MISC',\n", + " 'HIV', 'TB', 'NUTRITION', 'PharmDispensing', 'LABORATORY',\n", + " 'DENTAL', 'RADIOGRAPHY', 'MENTAL']))\n", + "\n", + "partition_facility_level = Partition.Simple('Facility_Level',\n", + " np.unique(flow_coarse_officer_appt['Facility_Level']))\n", + "\n", + "nodes = {\n", + " 'Officer': ProcessGroup(['Officer_Category'], partition_officer_cat),\n", + " 'Appt': ProcessGroup(['Appt_Cat'], partition_appt_cat),\n", + "}\n", + "\n", + "# Add nodes Waypoint\n", + "nodes['waypoint'] = Waypoint(partition_facility_level)\n", + "\n", + "bundles = [\n", + " Bundle('Officer', 'Appt', waypoints=['waypoint']),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Officer'], # left\n", + " ['waypoint'], # middle\n", + " ['Appt'], # right\n", + " ]\n", + "\n", + "# Set the color for each officer category\n", + "palette = {'Clinical': 'skyblue', 'Nursing and Midwifery': 'lightpink',\n", + " 'Pharmacy': 'khaki', 'Laboratory': 'cadetblue',\n", + " 'Radiography': 'yellowgreen', 'Dental': 'salmon',\n", + " 'Mental': 'mediumorchid', 'DCSA': 'royalblue'\n", + " }\n", + "\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_officer_cat)\n", + "\n", + "sankey_coarse_officer_and_coarse_appt = weave(sdd, flow_coarse_officer_appt,\n", + " palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_coarse_officer_and_coarse_appt.auto_save_png(outputpath /'Sankey_coarse_officer_and_coarse_appt.png')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe0021cb", + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "# The flow maps 9 officer categories and 51 appt types at an individaul level\n", + "flow_coarse_officer = pd.DataFrame(\n", + " appointment.groupby(['Officer_Category', 'Appt_Type_Code', 'Facility_Level'],\n", + " dropna=False, sort=False).sum()\n", + ").reset_index()\n", + "# As we do not care about the flow proportions, we add a 'value' columns with constant value 1.\n", + "flow_coarse_officer['value'] = 1\n", + "# Add 'source' and 'target' columns\n", + "flow_coarse_officer['source'] = 'Officer_Category'\n", + "flow_coarse_officer['target'] = 'Appt_Type_Code'\n", + "\n", + "def sankey_level_coarse_officer(level, h):\n", + " flow_coarse_officer_level = flow_coarse_officer.loc[flow_coarse_officer['Facility_Level'] == level, :].copy()\n", + " flow_coarse_officer_level.drop(columns = 'Facility_Level', inplace=True)\n", + " flow_coarse_officer_level.reset_index(drop=True, inplace=True)\n", + "\n", + " size = dict(width=800, height=h, margins=dict(left=180, right=180))\n", + "\n", + " partition_officer_cat = Partition.Simple('Officer_Category',\n", + " pd.array(['DCSA', 'Clinical', 'Nursing and Midwifery', 'Pharmacy',\n", + " 'Laboratory', 'Radiography', 'Dental', 'Mental']))\n", + " partition_appt_type = Partition.Simple('Appt_Type_Code', pd.unique(pd.Series(appt_types['Appt_Type_Code'])))\n", + "\n", + "\n", + " nodes = {\n", + " 'Officer': ProcessGroup(['Officer_Category'], partition_officer_cat),\n", + " 'Appt': ProcessGroup(['Appt_Type_Code'], partition_appt_type),\n", + " }\n", + "\n", + " bundles = [\n", + " Bundle('Officer', 'Appt'),\n", + " ]\n", + "\n", + " ordering = [\n", + " ['Officer'],\n", + " ['Appt'],\n", + " ]\n", + "\n", + " # Set the color for each officer category\n", + " palette = {'Clinical': 'skyblue', 'Nursing and Midwifery': 'lightpink',\n", + " 'Pharmacy': 'khaki', 'Laboratory': 'cadetblue',\n", + " 'Radiography': 'yellowgreen', 'Dental': 'salmon',\n", + " 'Mental': 'mediumorchid', 'DCSA': 'royalblue'\n", + " }\n", + "\n", + " sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_officer_cat) # color by officer cat\n", + "\n", + " return weave(sdd, flow_coarse_officer_level, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_coarse_officer_level_0 = sankey_level_coarse_officer('Facility_Level_0', 100)\n", + "sankey_coarse_officer_level_0.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_0.png')\n", + "\n", + "sankey_coarse_officer_level_1a = sankey_level_coarse_officer('Facility_Level_1a', 1200)\n", + "sankey_coarse_officer_level_1a.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_1a.png')\n", + "\n", + "sankey_coarse_officer_level_1b = sankey_level_coarse_officer('Facility_Level_1b', 1200)\n", + "sankey_coarse_officer_level_1b.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_1b.png')\n", + "\n", + "sankey_coarse_officer_level_2 = sankey_level_coarse_officer('Facility_Level_2', 1200)\n", + "sankey_coarse_officer_level_2.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_2.png')\n", + "\n", + "sankey_coarse_officer_level_3 = sankey_level_coarse_officer('Facility_Level_3', 1200)\n", + "sankey_coarse_officer_level_3.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_3.png')\n", + "\n", + "sankey_coarse_officer_level_4 = sankey_level_coarse_officer('Facility_Level_4', 200)\n", + "sankey_coarse_officer_level_4.auto_save_png(outputpath /'Sankey_coarse_officer_and_fine_appt_level_4.png')\n", + "\n", + "top_box = HBox([sankey_coarse_officer_level_0, sankey_coarse_officer_level_4])\n", + "mid_box = HBox([sankey_coarse_officer_level_1a, sankey_coarse_officer_level_1b])\n", + "bottom_box = HBox([sankey_coarse_officer_level_2, sankey_coarse_officer_level_3])\n", + "VBox([top_box, mid_box, bottom_box])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/src/scripts/healthsystem/hsi_in_typical_run/10_year_scale_run.py b/src/scripts/healthsystem/hsi_in_typical_run/10_year_scale_run.py new file mode 100644 index 0000000000..a4b4beeac0 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/10_year_scale_run.py @@ -0,0 +1,86 @@ +""" +This file defines a batch run of a large population for a long time with all disease modules and full use of HSIs +It's used for analysis of TLO implementation re. HCW and health services usage, for the paper on HCW. + +Run on the batch system using: +```tlo batch-submit src/scripts/healthsystem/hsi_in_typical_run/10_year_scale_run.py``` + +or locally using: + ```tlo scenario-run src/scripts/healthsystem/hsi_in_typical_run/10_year_scale_run.py``` + +""" +from pathlib import Path +from typing import Dict + +from tlo import Date, logging +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ScenarioSwitcher +from tlo.scenario import BaseScenario + + +class LongRun(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2020, 1, 1) + self.pop_size = 20_000 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 10 + + def log_configuration(self): + return { + 'filename': 'scale_run_for_hcw_analysis', + 'directory': Path('./outputs'), # <- (specified only for local running) + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem': logging.INFO, + 'tlo.methods.healthsystem.summary': logging.INFO, + } + } + + def modules(self): + return fullmodel(resourcefilepath=self.resources) + [ScenarioSwitcher(resourcefilepath=self.resources)] + + def draw_parameters(self, draw_number, rng): + return list(self._scenarios.values())[draw_number] + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" + + return { + "Status Quo": + mix_scenarios( + get_parameters_for_status_quo() + ), + + "Establishment HCW": + mix_scenarios( + get_parameters_for_status_quo(), + {'HealthSystem': {'use_funded_or_actual_staffing': 'funded_plus'}} + ), + + "Perfect Healthcare Seeking": + mix_scenarios( + get_parameters_for_status_quo(), + {'ScenarioSwitcher': {'max_healthsystem_function': False, 'max_healthcare_seeking': True}}, + ), + + "Establishment HCW + Perfect Healthcare Seeking": + mix_scenarios( + get_parameters_for_status_quo(), + {'HealthSystem': {'use_funded_or_actual_staffing': 'funded_plus'}}, + {'ScenarioSwitcher': {'max_healthsystem_function': False, 'max_healthcare_seeking': True}}, + ), + } + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) diff --git a/src/scripts/healthsystem/hsi_in_typical_run/analysis_hcw_usage_by_appt_and_by_disease.py b/src/scripts/healthsystem/hsi_in_typical_run/analysis_hcw_usage_by_appt_and_by_disease.py new file mode 100644 index 0000000000..79aba48f72 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/analysis_hcw_usage_by_appt_and_by_disease.py @@ -0,0 +1,288 @@ +import argparse +from collections import Counter +from pathlib import Path + +import matplotlib +import matplotlib.pyplot as plt +import pandas as pd + +from tlo import Date +from tlo.analysis.utils import ( + bin_hsi_event_details, + compute_mean_across_runs, + extract_results, + summarize, +) + +PREFIX_ON_FILENAME = '6' + +# Declare period for which the results will be generated (defined inclusively) +TARGET_PERIOD = (Date(2015, 1, 1), Date(2019, 12, 31)) + +# appointment dict to match model and data +appt_dict = {'Under5OPD': 'OPD', + 'Over5OPD': 'OPD', + 'AntenatalFirst': 'AntenatalTotal', + 'ANCSubsequent': 'AntenatalTotal', + 'NormalDelivery': 'Delivery', + 'CompDelivery': 'Delivery', + 'EstMedCom': 'EstAdult', + 'EstNonCom': 'EstAdult', + 'VCTPositive': 'VCTTests', + 'VCTNegative': 'VCTTests', + 'DentAccidEmerg': 'DentalAll', + 'DentSurg': 'DentalAll', + 'DentU5': 'DentalAll', + 'DentO5': 'DentalAll', + 'MentOPD': 'MentalAll', + 'MentClinic': 'MentalAll' + } + + +def get_annual_num_hsi_by_appt_and_level(results_folder: Path) -> pd.DataFrame: + """Return pd.DataFrame gives the (mean) simulated annual count of hsi + per treatment id per each appt type per level.""" + hsi_count = compute_mean_across_runs( + bin_hsi_event_details( + results_folder, + lambda event_details, count: sum( + [ + Counter({ + ( + event_details['treatment_id'], + appt_type, + event_details['facility_level'], + ): + count * appt_number + }) + for appt_type, appt_number in event_details["appt_footprint"] + ], + Counter() + ), + *TARGET_PERIOD, + True + ) + )[0] + + hsi_count = pd.DataFrame.from_dict(hsi_count, orient='index').reset_index().rename(columns={0: 'Count'}) + hsi_count[['Treatment_ID', 'Appt_Type_Code', 'Facility_Level']] = pd.DataFrame(hsi_count['index'].tolist(), + index=hsi_count.index) + # average annual count by treatment id, appt type and facility level + yr_count = TARGET_PERIOD[1].year - TARGET_PERIOD[0].year + 1 + hsi_count = hsi_count.groupby(['Treatment_ID', 'Appt_Type_Code', 'Facility_Level'])['Count'].sum()/yr_count + hsi_count = hsi_count.to_frame().reset_index() + + # drop dummy PharmDispensing for HCW paper results and plots + hsi_count = hsi_count.drop(index=hsi_count[hsi_count['Appt_Type_Code'] == 'PharmDispensing'].index) + + return hsi_count + + +def get_annual_hcw_time_used_with_confidence_interval(results_folder: Path, resourcefilepath: Path) -> pd.DataFrame: + """Return pd.DataFrame gives the (mean) simulated annual hcw time used per cadre across all levels, + with 95% confidence interval.""" + + def get_annual_hcw_time_used(_df) -> pd.Series: + """Get the annual hcw time used per cadre across all levels""" + + # get annual counts of appt per level + def unpack_nested_dict_in_series(_raw: pd.Series): + return pd.concat( + { + _idx: pd.DataFrame.from_dict(mydict) for _idx, mydict in _raw.items() + } + ).unstack().fillna(0.0).astype(int) + + annual_counts_of_appts_per_level = _df \ + .loc[pd.to_datetime(_df['date']).between(*TARGET_PERIOD), 'Number_By_Appt_Type_Code_And_Level'] \ + .pipe(unpack_nested_dict_in_series) \ + .groupby(level=[0, 1], axis=1).sum() \ + .mean(axis=0) \ + .to_frame().reset_index() \ + .rename(columns={'level_0': 'Facility_Level', 'level_1': 'Appt_Type_Code', 0: 'Count'}) \ + .pivot(index='Facility_Level', columns='Appt_Type_Code', values='Count') \ + .drop(columns='PharmDispensing') # do not include this dummy appt for HCW paper results and plots + + # get appt time definitions + appt_time = get_expected_appt_time(resourcefilepath) + + appts_def = set(appt_time.Appt_Type_Code) + appts_sim = set(annual_counts_of_appts_per_level.columns.values) + assert appts_sim.issubset(appts_def) + + # get hcw time used per cadre per level + _hcw_usage = appt_time.drop(index=appt_time[~appt_time.Appt_Type_Code.isin(appts_sim)].index).reset_index( + drop=True) + for idx in _hcw_usage.index: + fl = _hcw_usage.loc[idx, 'Facility_Level'] + appt = _hcw_usage.loc[idx, 'Appt_Type_Code'] + _hcw_usage.loc[idx, 'Total_Mins_Used_Per_Year'] = (_hcw_usage.loc[idx, 'Time_Taken_Mins'] * + annual_counts_of_appts_per_level.loc[fl, appt]) + + # get hcw time used per cadre + _hcw_usage = _hcw_usage.groupby(['Officer_Category'])['Total_Mins_Used_Per_Year'].sum() + + return _hcw_usage + + # get hcw time used per cadre with CI + hcw_usage = summarize( + extract_results( + results_folder, + module='tlo.methods.healthsystem.summary', + key='HSI_Event', + custom_generate_series=get_annual_hcw_time_used, + do_scaling=True + ), + only_mean=False, + collapse_columns=True, + ).unstack() + + # reformat + hcw_usage = hcw_usage.to_frame().reset_index() \ + .rename(columns={'stat': 'Value_Type', 0: 'Value'}) \ + .pivot(index='Officer_Category', columns='Value_Type', values='Value') + + return hcw_usage + + +def get_expected_appt_time(resourcefilepath) -> pd.DataFrame: + """This is to return the expected time requirements per appointment type per coarse cadre per facility level.""" + expected_appt_time = pd.read_csv( + resourcefilepath / 'healthsystem' / 'human_resources' / 'definitions' / 'ResourceFile_Appt_Time_Table.csv') + appt_type = pd.read_csv( + resourcefilepath / 'healthsystem' / 'human_resources' / 'definitions' / 'ResourceFile_Appt_Types_Table.csv') + expected_appt_time = expected_appt_time.merge( + appt_type[['Appt_Type_Code', 'Appt_Cat']], on='Appt_Type_Code', how='left') + # rename Appt_Cat + appt_cat = {'GENERAL_INPATIENT_AND_OUTPATIENT_CARE': 'IPOP', + 'Nutrition': 'NUTRITION', + 'Misc': 'MISC', + 'Mental_Health': 'MENTAL'} + expected_appt_time['Appt_Cat'] = expected_appt_time['Appt_Cat'].replace(appt_cat) + expected_appt_time.rename(columns={'Appt_Cat': 'Appt_Category'}, inplace=True) + + # modify time for dummy ConWithDCSA so that no overworking/underworking + expected_appt_time.loc[expected_appt_time['Appt_Category'] == 'ConWithDCSA', 'Time_Taken_Mins'] = 20.0 + + return expected_appt_time + + +def get_hcw_capability(resourcefilepath, hcwscenario='actual') -> pd.DataFrame: + """This is to return the annual hcw capabilities per cadre per facility level. + Argument hcwscenario can be actual, funded_plus.""" + hcw_capability = pd.read_csv( + resourcefilepath / 'healthsystem' / 'human_resources' / hcwscenario / 'ResourceFile_Daily_Capabilities.csv' + ) + hcw_capability = hcw_capability.groupby(['Facility_Level', 'Officer_Category'] + )['Total_Mins_Per_Day'].sum().reset_index() # todo: drop facility level 5 + hcw_capability['Total_Mins_Per_Year'] = hcw_capability['Total_Mins_Per_Day'] * 365.25 + hcw_capability.drop(columns='Total_Mins_Per_Day', inplace=True) + + return hcw_capability + + +def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None): + """Compare appointment usage from model output with real appointment usage. + The real appointment usage is collected from DHIS2 system and HIV Dept.""" + + make_graph_file_name = lambda stub: output_folder / f"{PREFIX_ON_FILENAME}_{stub}.png" # noqa: E731 + + # format data and plot bar chart for hcw working time per cadre + def format_hcw_usage(hcwscenario='actual'): + """format data for bar plot""" + # get hcw capability in actual or establishment (funded_plus) scenarios + hcw_capability = get_hcw_capability(resourcefilepath, hcwscenario=hcwscenario) \ + .groupby('Officer_Category')['Total_Mins_Per_Year'].sum().to_frame() \ + .rename(columns={'Total_Mins_Per_Year': 'capability'}) + + # calculate hcw time usage ratio against capability with CI + hcw_usage = get_annual_hcw_time_used_with_confidence_interval(results_folder, resourcefilepath) + hcw_usage_ratio = hcw_usage.join(hcw_capability) + hcw_usage_ratio.loc['All'] = hcw_usage_ratio.sum() + hcw_usage_ratio['mean'] = hcw_usage_ratio['mean'] / hcw_usage_ratio['capability'] + hcw_usage_ratio['lower'] = hcw_usage_ratio['lower'] / hcw_usage_ratio['capability'] + hcw_usage_ratio['upper'] = hcw_usage_ratio['upper'] / hcw_usage_ratio['capability'] + + hcw_usage_ratio['lower_error'] = (hcw_usage_ratio['mean'] - hcw_usage_ratio['lower']) + hcw_usage_ratio['upper_error'] = (hcw_usage_ratio['upper'] - hcw_usage_ratio['mean']) + + asymmetric_error = [hcw_usage_ratio['lower_error'].values, hcw_usage_ratio['upper_error'].values] + hcw_usage_ratio = pd.DataFrame(hcw_usage_ratio['mean']) \ + .clip(lower=0.1, upper=10.0) + + # reduce the mean ratio by 1.0, for the bar plot that starts from y=1.0 instead of y=0.0 + #hcw_usage_ratio['mean'] = hcw_usage_ratio['mean'] - 1.0 + + # rename cadre Nursing_and_Midwifery + hcw_usage_ratio.rename(index={'Nursing_and_Midwifery': 'Nursing and Midwifery'}, inplace=True) + + return hcw_usage_ratio, asymmetric_error + + hcw_usage_ratio_actual, error_actual = format_hcw_usage(hcwscenario='actual') + hcw_usage_ratio_establishment, error_establishment = format_hcw_usage(hcwscenario='funded_plus') + + name_of_plot = 'Simulated average annual working time (95% CI) vs Capability' + fig, ax = plt.subplots(figsize=(8, 5)) + hcw_usage_ratio_establishment.plot(kind='bar', yerr=error_establishment, width=0.4, + ax=ax, position=0, bottom=0.0, + legend=False, color='c') + hcw_usage_ratio_actual.plot(kind='bar', yerr=error_actual, width=0.4, + ax=ax, position=1, bottom=0.0, + legend=False, color='y') + ax.axhline(1.0, color='gray', linestyle='dashed') + ax.set_xlim(right=len(hcw_usage_ratio_establishment) - 0.3) + #ax.set_yscale('log') + ax.set_ylim(0, 11.5) + ax.set_yticks([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + ax.set_yticklabels(("0", "1", '2', '3', '4', '5', '6', '7', '8', '9', ">= 10")) + ax.set_ylabel('Simulated working time : Capability') + ax.set_xlabel('Cadre Category') + plt.xticks(rotation=60, ha='right') + #ax.xaxis.grid(True, which='major', linestyle='--') + #ax.yaxis.grid(True, which='both', linestyle='--') + ax.set_title(name_of_plot) + patch_establishment = matplotlib.patches.Patch(facecolor='c', label='Establishment capability') + patch_actual = matplotlib.patches.Patch(facecolor='y', label='Actual capability') + legend = plt.legend(handles=[patch_actual, patch_establishment], loc='center left', bbox_to_anchor=(1.0, 0.5)) + fig.add_artist(legend) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(',', '').replace('\n', '_').replace(' ', '_'))) + plt.show() + + # hcw usage per cadre per appt per hsi + hsi_count = get_annual_num_hsi_by_appt_and_level(results_folder) + + # first compare appts defined and appts in simulation/model + appt_time = get_expected_appt_time(resourcefilepath) + appts_def = set(appt_time.Appt_Type_Code) + appts_sim = set(hsi_count.Appt_Type_Code) + assert appts_sim.issubset(appts_def) + + # then calculate the hcw working time per treatment id, appt type and cadre + hcw_usage_hsi = appt_time.drop(index=appt_time[~appt_time.Appt_Type_Code.isin(appts_sim)].index + ).reset_index(drop=True) + hcw_usage_hsi = hsi_count.merge(hcw_usage_hsi, on=['Facility_Level', 'Appt_Type_Code'], how='left') + # save the data to draw sankey diagram of appt to hsi + hcw_usage_hsi.to_csv(output_folder / 'hsi_count_by_treatment_appt_level.csv', index=False) + hcw_usage_hsi['Total_Mins_Used_Per_Year'] = hcw_usage_hsi['Count'] * hcw_usage_hsi['Time_Taken_Mins'] + hcw_usage_hsi = hcw_usage_hsi.groupby(['Treatment_ID', 'Appt_Category', 'Officer_Category'] + )['Total_Mins_Used_Per_Year'].sum().reset_index() + + # rename Nursing_and_Midwifery + hcw_usage_hsi.Officer_Category = hcw_usage_hsi.Officer_Category.replace( + {'Nursing_and_Midwifery': 'Nursing and Midwifery'}) + + # save the data to draw sankey diagram of hcw time via appt to disease + hcw_usage_hsi.to_csv(output_folder/'hcw_working_time_per_hsi.csv', index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("results_folder", type=Path) + args = parser.parse_args() + + apply( + results_folder=args.results_folder, + output_folder=args.results_folder, + resourcefilepath=Path('./resources') + ) diff --git a/src/scripts/healthsystem/hsi_in_typical_run/analysis_hsi_in_typical_run.py b/src/scripts/healthsystem/hsi_in_typical_run/analysis_hsi_in_typical_run.py new file mode 100644 index 0000000000..37f0bcc781 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/analysis_hsi_in_typical_run.py @@ -0,0 +1,140 @@ +"""This file uses the run generated by `scenario_hsi_in_typical_run.py` to generate descriptions of the HSI that occur +in a typical run.""" + +# %% Declare the name of the file that specified the scenarios used in this run. +from pathlib import Path + +import matplotlib.pyplot as plt +# import numpy as np +import pandas as pd +from matplotlib.cm import get_cmap + +from tlo.analysis.utils import get_scenario_outputs, load_pickled_dataframes + +scenario_filename = 'long_run_all_diseases.py' + +# %% Declare usual paths: +outputspath = Path('./outputs/bshe@ic.ac.uk') +rfp = Path('./resources') + +# Find results folder (most recent run generated using that scenario_filename) +results_folder = get_scenario_outputs(scenario_filename, outputspath)[-1] +print(f"Results folder is: {results_folder}") + +# Declare path for output graphs from this script +make_graph_file_name = lambda stub: results_folder / f"{stub}.png" # noqa: E731 + +# %% Extract results +log = load_pickled_dataframes(results_folder)['tlo.methods.healthsystem'] # (There was only one draw and one run) + +# %% Plot: Fraction of Total Healthcare Worker Time Used + +cap = log['Capacity'] +cap["date"] = pd.to_datetime(cap["date"]) +cap = cap.set_index('date') + +frac_time_used = cap['Frac_Time_Used_Overall'] +frac_time_used_2014_2018 = frac_time_used.loc['2013-12-31':'2019-01-01'] +frac_time_used_2016 = frac_time_used.loc['2015-12-31':'2017-01-01'] + +# Plot: +frac_time_used_2014_2018.plot() +plt.title("Fraction of Total Healthcare Worker Time Used (year 2014-2018)") +plt.xlabel("Date") +plt.tight_layout() +plt.savefig(make_graph_file_name('HSI_Frac_time_used_2014_2018')) +plt.show() + +frac_time_used_2016.plot() +plt.title("Fraction of Total Healthcare Worker Time Used (year 2016)") +plt.xlabel("Date") +plt.tight_layout() +plt.savefig(make_graph_file_name('HSI_Frac_time_used_2016')) +plt.show() + +# %% Number of HSI: + +hsi = log['HSI_Event'] +hsi["date"] = pd.to_datetime(hsi["date"]) +hsi["month"] = hsi["date"].dt.month +hsi["year"] = hsi["date"].dt.year + +# Number of HSI that are taking place by originating module, by month +year = 2016 +hsi["Module"] = hsi["TREATMENT_ID"].str.split('_').apply(lambda x: x[0]) + +evs = hsi.loc[hsi.date.dt.year == year]\ + .groupby(by=['month', 'Module'])\ + .size().reset_index().rename(columns={0: 'count'})\ + .pivot_table(index='month', columns='Module', values='count', fill_value=0) + +# Plot: +# Use colormap tab20 so that each module has a unique color +color_tab20 = get_cmap('tab20_r') +evs.plot.bar(stacked=True, color=color_tab20.colors) +plt.title(f"HSI by Module, per Month (year {year})") +plt.ylabel('Total per month') +plt.tight_layout() +plt.legend(ncol=3, loc='center', fontsize='xx-small') +plt.savefig(make_graph_file_name('HSI_per_module_per_month')) +plt.show() + +# Plot the breakdown of all HSI, over all the years 2010-2021 +evs = pd.DataFrame(hsi.groupby(by=['Module']).size()) +# Calculate the fraction +evs[1] = 100*evs[0]/evs[0].sum() +color_tab20 = get_cmap('tab20_r') +patches, texts = plt.pie(evs[0], colors=color_tab20.colors) +labels = ['{0} - {1:1.2f} %'.format(i, j) for i, j in zip(evs.index, evs[1])] +# Sort legend +sort_legend = True +if sort_legend: + patches, labels, dummy = zip(*sorted(zip(patches, labels, evs[0]), + key=lambda x: x[2], + reverse=True)) +plt.legend(patches, labels, ncol=3, loc='lower center', fontsize='xx-small') +plt.title("HSI by Module (year 2010-2021)") +plt.tight_layout() +plt.savefig(make_graph_file_name('HSI_per_module')) +plt.show() + +# %% Demand for appointments + +num_hsi_by_treatment_id = hsi.groupby(hsi.TREATMENT_ID)['Number_By_Appt_Type_Code'].size() + +# find the appt footprint for each treatment_id +# in hsi, drop rows with empty 'Number_By_Appt_Type_Code' to avoid warnings of empty series +null_hsi_idx = hsi[hsi['Number_By_Appt_Type_Code'] == {}].index +hsi.drop(index=null_hsi_idx, inplace=True) +# generate the table +appts_by_treatment_id = pd.DataFrame({ + _treatment_id: pd.Series( + hsi.loc[hsi.TREATMENT_ID == _treatment_id, 'Number_By_Appt_Type_Code'].values[0] + ) for _treatment_id in num_hsi_by_treatment_id.index +}).fillna(0.0).T + +# the tricky one that omits many hsi events +appts_by_treatment_id_short = \ + hsi.set_index('TREATMENT_ID')['Number_By_Appt_Type_Code'].drop_duplicates().apply(pd.Series).fillna(0.0) + +# get the appointment usage per month year 2019 +hsi_2019 = hsi.loc[hsi.date.dt.year == 2019].copy() +M = range(1, 13) +D = {} +appt_usage_2019 = pd.DataFrame() +for m in M: + a = hsi_2019.loc[hsi_2019.month == m, 'Number_By_Appt_Type_Code'].apply(pd.Series) + D[m] = pd.DataFrame(columns=[m], data=a.sum(axis=0)) + appt_usage_2019 = appt_usage_2019.join(D[m], how='outer') +# save +# appt_usage_2019.to_csv(outputspath / 'appt_usage_2019.csv') + +# Possible issues: +# set(appts_by_treatment_id_short.columns)-set(appts_by_treatment_id.columns) +# the output is: {'ComDelivery', 'VCTPositive'}, not clear why the two appts are not in the table appts_by_treatment_id +# There are inconsistencies in appts_by_treatment_id_short: +# e.g. breastCancer_StartTreatment and oesophagealCancer_StartTreatment call for different appts, +# Labour_ReceivesComprehensiveEmergencyObstetricCare calls for non appt. + +# Plot... +# See the Sankey plot in analysis_sankey_appt_and_hsi.ipynb (in the same folder) diff --git a/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_and_hsi.ipynb b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_and_hsi.ipynb new file mode 100644 index 0000000000..1aec511003 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_and_hsi.ipynb @@ -0,0 +1,286 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": true, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "This file uses the run generated by `scenario_hsi_in_typical_run.py` and floweaver\n", + "to produce a Sankey diagram that maps appointments with HSI events.\n", + "\n", + "Below is the instruction to run the file.\n", + "\n", + "### Install floweaver in Anaconda Prompt (if use Jupyter Notebook) / PyCharm Terminal:\n", + "\n", + "pip install floweaver\n", + "\n", + "pip install ipysankeywidget\n", + "\n", + "jupyter nbextension enable --py --sys-prefix ipysankeywidget\n", + "\n", + "jupyter notebook (to open jupyter notebook)\n", + "\n", + "### To display and save the output figures:\n", + "Select Start Jupyter Server from the Jupyter Actions Menu (lightbulb icon next to Run All cells icon)\n", + "\n", + "Open Event Log\n", + "\n", + "Open in Browser\n", + "\n", + "Find the script and run all cells\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tlo\n", + "\n", + "from pathlib import Path\n", + "\n", + "from tlo.analysis.utils import get_scenario_outputs, load_pickled_dataframes\n", + "\n", + "import pandas as pd\n", + "\n", + "import numpy as np\n", + "\n", + "from matplotlib import pyplot as plt\n", + "\n", + "from ipysankeywidget import SankeyWidget\n", + "\n", + "from floweaver import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + }, + "outputs": [], + "source": [ + "# Declare the name of the file that specified the scenarios used in this run.\n", + "scenario_filename = 'scenario_hsi_in_typical_run.py'\n", + "\n", + "# Declare usual paths:\n", + "# Get the tlo path\n", + "tlopath = Path(tlo.__file__).parent.parent.parent\n", + "outputspath = tlopath / Path('outputs/bshe@ic.ac.uk')\n", + "\n", + "# Find results folder (most recent run generated using that scenario_filename)\n", + "results_folder = get_scenario_outputs(scenario_filename, outputspath)[-1]\n", + "print(f\"Results folder is: {results_folder}\")\n", + "\n", + "# Declare path for output graphs from this script\n", + "make_graph_file_name = lambda stub: results_folder / f\"{stub}.png\" # noqa: E731\n", + "\n", + "# Extract results\n", + "log = load_pickled_dataframes(results_folder)['tlo.methods.healthsystem'] # (There was only one draw and one run)\n", + "\n", + "# Number of HSI:\n", + "hsi = log['HSI_Event']\n", + "hsi[\"date\"] = pd.to_datetime(hsi[\"date\"])\n", + "hsi[\"month\"] = hsi[\"date\"].dt.month\n", + "hsi[\"Module\"] = hsi[\"TREATMENT_ID\"].str.split('_').apply(lambda x: x[0])\n", + "\n", + "# Demand for appointments\n", + "num_hsi_by_treatment_id = pd.DataFrame(hsi.groupby(hsi.TREATMENT_ID)['Number_By_Appt_Type_Code'].size())\n", + "num_hsi_by_treatment_id.rename(columns={'Number_By_Appt_Type_Code': 'Number_of_HSI'}, inplace=True)\n", + "# Note that some hsi events, e.g. VCTPositive, have zero number/frequency.\n", + "\n", + "# find the appt footprint for each treatment_id\n", + "# in hsi, drop rows with empty 'Number_By_Appt_Type_Code' to avoid warnings of empty series\n", + "null_hsi_idx = hsi[hsi['Number_By_Appt_Type_Code'] == {}].index\n", + "hsi.drop(index=null_hsi_idx, inplace=True)\n", + "\n", + "# generate the full table\n", + "appts_by_treatment_id_full =pd.DataFrame({\n", + " _treatment_id: pd.Series(hsi.loc[hsi.TREATMENT_ID == _treatment_id, 'Number_By_Appt_Type_Code'].values[0]) for _treatment_id in num_hsi_by_treatment_id.index\n", + "}).fillna(0.0).T\n", + "\n", + "# generate the short table (the tricky one that omits many hsi events)\n", + "appts_by_treatment_id_short = \\\n", + " hsi.set_index('TREATMENT_ID')['Number_By_Appt_Type_Code'].drop_duplicates().apply(pd.Series).fillna(0.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + }, + "outputs": [], + "source": [ + "# Sankey 0 that map appt to hsi considering only appt footprint for each hsi\n", + "\n", + "# Prepare the data: appt type and number per hsi\n", + "appt_and_hsi = appts_by_treatment_id_short.reset_index().copy()\n", + "appt_and_hsi.rename(columns={'index': 'TREATMENT_ID'}, inplace=True)\n", + "appt_and_hsi = pd.melt(appt_and_hsi, id_vars=['TREATMENT_ID'], value_vars=appt_and_hsi.columns[1:],\n", + " var_name='Appt_Type')\n", + "\n", + "# Define the flow\n", + "appt_and_hsi['source'] = 'Appt_Type'\n", + "appt_and_hsi['target'] = 'TREATMENT_ID'\n", + "\n", + "size = dict(width=1000, height=800, margins=dict(left=120, right=520))\n", + "\n", + "# Nodes in alphabetic order\n", + "# partition_appt_type = Partition.Simple('Appt_Type', np.unique(appt_and_hsi['Appt_Type']))\n", + "# partition_treatment_id = Partition.Simple('TREATMENT_ID', np.unique(appt_and_hsi['TREATMENT_ID']))\n", + "# if to keep the order in the dataframe\n", + "# partition_appt_type = Partition.Simple('Appt_Type', pd.unique(pd.Series(appt_and_hsi['Appt_Type'])))\n", + "# partition_treatment_id = Partition.Simple('TREATMENT_ID', pd.unique(pd.Series(appt_and_hsi['TREATMENT_ID'])))\n", + "# if to fix the oder of the nodes in the way we want\n", + "partition_appt_type = Partition.Simple('Appt_Type', pd.array([\n", + " 'IPAdmission', 'InpatientDays', 'Over5OPD', 'Under5OPD',\n", + " 'AntenatalFirst', 'ANCSubsequent', 'CompDelivery', 'NormalDelivery',\n", + " 'FamPlan', 'MajorSurg', 'ConWithDCSA',\n", + " 'MaleCirc', 'NewAdult', 'VCTNegative', 'VCTPositive']))\n", + "partition_treatment_id = Partition.Simple('TREATMENT_ID', pd.array([\n", + " 'Malaria_treatment_complicated_child', 'Malaria_IPTp',\n", + " 'Diarrhoea_Treatment_Inpatient', 'Depression_Antidepressant_Refill',\n", + " 'PostnatalSupervisor_NeonatalWardInpatientCare', 'CareOfWomenDuringPregnancy_FirstAntenatalCareContact',\n", + " 'CareOfWomenDuringPregnancy_AntenatalOutpatientManagementOfAnaemia',\n", + " 'CareOfWomenDuringPregnancy_PostAbortionCaseManagement', 'Labour_ReceivesSkilledBirthAttendanceDuringLabour',\n", + " 'Contraception_FamilyPlanningAppt', 'GenericEmergencyFirstApptAtFacilityLevel1',\n", + " 'GenericFirstApptAtFacilityLevel0', 'OesophagealCancer_StartTreatment', 'breastCancer_StartTreatment',\n", + " 'Hiv_Circumcision', 'Hiv_Treatment_InitiationOrContinuation', 'Hiv_TestAndRefer']))\n", + "\n", + "nodes = {\n", + " 'Appt': ProcessGroup(['Appt_Type'], partition_appt_type),\n", + " 'HSI': ProcessGroup(['TREATMENT_ID'], partition_treatment_id),\n", + "}\n", + "\n", + "bundles = [\n", + " Bundle('Appt', 'HSI'),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Appt'], # left\n", + " ['HSI'], # right\n", + " ]\n", + "\n", + "# Set the color for each appt type\n", + "palette = {'IPAdmission': 'lightsteelblue', 'InpatientDays': 'skyblue',\n", + " 'Over5OPD': 'cornflowerblue', 'Under5OPD': 'steelblue',\n", + " 'AntenatalFirst': 'plum', 'ANCSubsequent': 'hotpink',\n", + " 'CompDelivery': 'tomato', 'NormalDelivery': 'darksalmon',\n", + " 'FamPlan': 'gold', 'MajorSurg': 'orange', 'ConWithDCSA': 'mediumpurple',\n", + " 'MaleCirc': 'lightgreen', 'NewAdult': 'mediumseagreen',\n", + " 'VCTNegative': 'greenyellow', 'VCTPositive': 'olivedrab',\n", + " }\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_appt_type)\n", + "\n", + "# Generate and save Sankey\n", + "sankey_appt_and_hsi = weave(sdd, appt_and_hsi, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_appt_and_hsi.auto_save_png(make_graph_file_name('Sankey_appt_and_hsi'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "pycharm": { + "name": "#%%\n", + "is_executing": true + } + }, + "outputs": [], + "source": [ + "# Sankey 1 that maps appt to hsi considering appt footprint for each hsi and number of each hsi\n", + "\n", + "# Prepare the data: total number of appts per hsi for year 2010-2018\n", + "num_appt_by_hsi = appts_by_treatment_id_full.copy()\n", + "for event in num_appt_by_hsi.index:\n", + " num_appt_by_hsi.loc[event,:] = appts_by_treatment_id_full.loc[event,:] * num_hsi_by_treatment_id.loc[event, 'Number_of_HSI']\n", + "num_appt_by_hsi = num_appt_by_hsi.reset_index().copy()\n", + "num_appt_by_hsi.rename(columns={'index': 'TREATMENT_ID'}, inplace=True)\n", + "num_appt_by_hsi = pd.melt(num_appt_by_hsi, id_vars=['TREATMENT_ID'], value_vars=num_appt_by_hsi.columns[1:],\n", + " var_name='Appt_Type')\n", + "\n", + "# Define the flow\n", + "num_appt_by_hsi['source'] = 'Appt_Type'\n", + "num_appt_by_hsi['target'] = 'TREATMENT_ID'\n", + "\n", + "size = dict(width=1000, height=800, margins=dict(left=120, right=520))\n", + "\n", + "partition_appt_type = Partition.Simple('Appt_Type', np.unique(num_appt_by_hsi['Appt_Type']))\n", + "\n", + "partition_treatment_id = Partition.Simple('TREATMENT_ID', np.unique(num_appt_by_hsi['TREATMENT_ID']))\n", + "\n", + "nodes = {\n", + " 'Appt': ProcessGroup(['Appt_Type'], partition_appt_type),\n", + " 'HSI': ProcessGroup(['TREATMENT_ID'], partition_treatment_id),\n", + "}\n", + "\n", + "bundles = [\n", + " Bundle('Appt', 'HSI'),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Appt'], # left\n", + " ['HSI'], # right\n", + " ]\n", + "\n", + "# Set the color for each appt type\n", + "palette = {'IPAdmission': 'lightsteelblue', 'InpatientDays': 'skyblue',\n", + " 'Over5OPD': 'cornflowerblue', 'Under5OPD': 'steelblue',\n", + " 'AntenatalFirst': 'plum', 'ANCSubsequent': 'hotpink',\n", + " 'CompDelivery': 'tomato', 'NormalDelivery': 'darksalmon',\n", + " 'FamPlan': 'gold', 'MajorSurg': 'orange', 'ConWithDCSA': 'mediumpurple',\n", + " 'MaleCirc': 'lightgreen', 'NewAdult': 'mediumseagreen',\n", + " 'VCTNegative': 'greenyellow', 'VCTPositive': 'olivedrab',\n", + " }\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_appt_type)\n", + "\n", + "# Generate and save Sankey\n", + "sankey_num_appt_by_hsi = weave(sdd, num_appt_by_hsi, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_num_appt_by_hsi.auto_save_png(make_graph_file_name('Sankey_num_appt_by_hsi'))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_level_hsi.ipynb b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_level_hsi.ipynb new file mode 100644 index 0000000000..618ec331f7 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_appt_level_hsi.ipynb @@ -0,0 +1,422 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "collapsed": true, + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "This file uses the run generated by `10_year_scale_run.py` and floweaver\n", + "to produce a Sankey diagram that maps appointments with HSI events via facility levels.\n", + "\n", + "Below is the instruction to run the file.\n", + "\n", + "### Install floweaver in Anaconda Prompt (if use Jupyter Notebook) / PyCharm Terminal:\n", + "\n", + "pip install floweaver\n", + "\n", + "pip install ipysankeywidget\n", + "\n", + "jupyter nbextension enable --py --sys-prefix ipysankeywidget\n", + "\n", + "jupyter notebook (to open jupyter notebook, which should be installed first) \n", + "\n", + "### To display and save the output figures:\n", + "Select Start Jupyter Server from the Jupyter Actions Menu (lightbulb icon next to Run All cells icon) -> Open Event Log -> Open in Browser\n", + "Or \n", + "Type jupyter notebook in PyCharm Terminal\n", + "\n", + "Find the script and run all cells\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tlo\n", + "\n", + "from pathlib import Path\n", + "\n", + "from tlo.analysis.utils import get_scenario_outputs, load_pickled_dataframes\n", + "\n", + "import pandas as pd\n", + "\n", + "import numpy as np\n", + "\n", + "from matplotlib import pyplot as plt\n", + "\n", + "from ipysankeywidget import SankeyWidget\n", + "\n", + "from floweaver import *" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Results folder is: c:\\users\\jdbb1\\pycharmprojects\\tlomodel\\outputs\\bshe@ic.ac.uk\\scale_run_for_hcw_analysis-2023-08-20T102040Z_draw0\n" + ] + } + ], + "source": [ + "# Declare the name of the file that specified the scenarios used in this run.\n", + "scenario_filename = 'scale_run_for_hcw_analysis.py' # i.e., the 10_year_scale_run.py\n", + "\n", + "# Declare usual paths:\n", + "# Get the tlo path\n", + "tlopath = Path(tlo.__file__).parent.parent.parent\n", + "outputspath = tlopath / Path('outputs/bshe@ic.ac.uk')\n", + "\n", + "# Find results folder (most recent run generated using that scenario_filename)\n", + "f = -4 # -4: Actual + Default health care seeking\n", + "results_folder = get_scenario_outputs(scenario_filename, outputspath)[f]\n", + "print(f\"Results folder is: {results_folder}\")\n", + "\n", + "# Declare path for output graphs from this script\n", + "make_graph_file_name = lambda stub: results_folder / f\"{stub}.png\" # noqa: E731\n", + "\n", + "# Extract results\n", + "hsi = pd.read_csv(results_folder / 'hsi_count_by_treatment_appt_level.csv')\n", + "\n", + "# Format data\n", + "hsi = hsi[[\"Appt_Type_Code\", \"Facility_Level\", \"Treatment_ID\", \"Count\"]].drop_duplicates().reset_index(drop=True)\n", + "hsi['Facility_Level'] = 'Facility_Level_' + hsi['Facility_Level'].astype(str)\n", + "hsi['source'] = 'Appt_Type_Code'\n", + "hsi['target'] = 'Treatment_ID'\n", + "hsi['value'] = hsi['Count']\n", + "\n", + "# Format data alternatively\n", + "hsi_def = hsi.copy()\n", + "hsi_def['value'] = 1.0 # only show hsi definitions re. appt footprint and facility level, no hsi count\n", + "\n", + "hsi_all_levels = hsi.groupby(['Appt_Type_Code', 'Treatment_ID'])['Count'].sum().reset_index()\n", + "hsi_all_levels['source'] = 'Appt_Type_Code'\n", + "hsi_all_levels['target'] = 'Treatment_ID'\n", + "hsi_all_levels['value'] = hsi_all_levels['Count'] # hsi count per appt per level\n", + "\n", + "hsi_example = hsi[(hsi.Appt_Type_Code == 'Over5OPD')].reset_index(drop=True)\n", + "level_example = 'Facility_Level_1a'\n", + "appt_example = ['AntenatalFirst', 'FamPlan', 'IPAdmission', 'NormalDelivery', 'Over5OPD', 'VCTNegative', 'NewAdult', 'TBNew']\n", + "hsi_example = hsi[(hsi.Appt_Type_Code.isin(appt_example)) &\n", + " (hsi.Facility_Level == level_example) ].reset_index(drop=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "281afa32e1d940968322b5be1a998cb5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "SankeyWidget(groups=[{'id': 'Appt', 'type': 'process', 'title': '', 'nodes': ['Appt^ConWithDCSA', 'Appt^Under5…" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Format the flow\n", + "size = dict(width=1000, height=1600, margins=dict(left=150, right=520))\n", + "\n", + "# Nodes in alphabetic order\n", + "partition_appt_type = Partition.Simple('Appt_Type_Code', pd.array([\n", + " 'ConWithDCSA',\n", + " 'Under5OPD', 'Over5OPD', 'IPAdmission', 'InpatientDays',\n", + " 'AntenatalFirst', 'ANCSubsequent', 'FamPlan', 'EPI', \n", + " 'CompDelivery', 'NormalDelivery', 'Csection',\n", + " 'AccidentsandEmerg', 'MajorSurg', 'MinorSurg',\n", + " 'U5Malnutr',\n", + " 'MentOPD',\n", + " 'Mammography', 'DiagRadio', 'Tomography', 'LabMolec', 'LabTBMicro',\n", + " 'MaleCirc', 'Peds', 'VCTNegative', 'VCTPositive', 'NewAdult', 'EstNonCom',\n", + " 'TBNew', 'TBFollowUp']))\n", + "partition_treatment_id = Partition.Simple('Treatment_ID', np.unique(hsi['Treatment_ID']))\n", + "partition_facility_level = Partition.Simple('Facility_Level',np.unique(hsi['Facility_Level']))\n", + "\n", + "nodes = {\n", + " 'Appt': ProcessGroup(['Appt_Type_Code'], partition_appt_type),\n", + " 'HSI': ProcessGroup(['Treatment_ID'], partition_treatment_id),\n", + "}\n", + "\n", + "# Add nodes Waypoint\n", + "nodes['waypoint'] = Waypoint(partition_facility_level)\n", + "\n", + "bundles = [\n", + " Bundle('Appt', 'HSI', waypoints=['waypoint']),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Appt'], # left\n", + " ['waypoint'], # middle\n", + " ['HSI'], # right\n", + " ]\n", + "\n", + "\n", + "# Set the color for each appt type\n", + "palette = {'Under5OPD': 'lightpink', 'Over5OPD': 'lightpink',\n", + " 'IPAdmission': 'palevioletred', 'InpatientDays': 'mediumvioletred',\n", + " \n", + " 'AntenatalFirst': 'green', 'ANCSubsequent': 'green',\n", + " 'FamPlan': 'darkseagreen', 'EPI': 'paleturquoise', \n", + " 'CompDelivery': 'limegreen', 'NormalDelivery': 'limegreen', 'Csection': 'springgreen',\n", + " \n", + " 'AccidentsandEmerg': 'darkorange', 'MajorSurg': 'orange', 'MinorSurg': 'gold',\n", + " \n", + " 'ConWithDCSA': 'violet',\n", + " \n", + " 'U5Malnutr': 'orchid',\n", + " \n", + " 'MentOPD': 'darkgrey',\n", + " \n", + " 'Mammography': 'lightgrey', 'DiagRadio': 'lightgrey', 'Tomography': 'lightgrey', \n", + " 'LabMolec': 'gainsboro', 'LabTBMicro': 'gainsboro',\n", + " \n", + " 'MaleCirc': 'mediumslateblue', 'Peds': 'lightskyblue', \n", + " 'VCTNegative': 'lightsteelblue', 'VCTPositive': 'lightsteelblue', \n", + " 'NewAdult': 'cornflowerblue', 'EstNonCom': 'royalblue',\n", + " \n", + " 'TBNew': 'yellow', 'TBFollowUp': 'yellow'}\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_appt_type)\n", + "\n", + "# Generate and save Sankey\n", + "sankey_appt_level_hsi = weave(sdd, hsi_def, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_appt_level_hsi.auto_save_png(make_graph_file_name('Sankey_appt_level_hsi'))" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "00de864cca414bf1b9af1e91f324e834", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "SankeyWidget(groups=[{'id': 'Appt', 'type': 'process', 'title': '', 'nodes': ['Appt^AntenatalFirst', 'Appt^Fam…" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Format the flow alt 1\n", + "# Format the flow\n", + "size = dict(width=1000, height=1000, margins=dict(left=150, right=520))\n", + "\n", + "# Nodes in alphabetic order\n", + "partition_appt_type = Partition.Simple('Appt_Type_Code', pd.array([\n", + " 'AntenatalFirst', 'FamPlan', \n", + " 'VCTNegative', 'NewAdult', 'Over5OPD', 'IPAdmission', 'NormalDelivery', 'TBNew']))\n", + "partition_treatment_id = Partition.Simple('Treatment_ID', pd.array(\n", + " ['AntenatalCare_Outpatient', 'Contraception_Routine', \n", + " 'Hiv_Prevention_Infant', 'Hiv_Prevention_Prep', 'Hiv_Test', 'Hiv_Treatment',\n", + " 'AntenatalCare_FollowUp', \n", + " 'CardioMetabolicDisorders_Prevention_CommunityTestingForHypertension', 'Depression_Treatment',\n", + " 'Malaria_Prevention_Iptp', 'Malaria_Test', 'Malaria_Treatment',\n", + " 'Measles_Treatment', 'PostnatalCare_Maternal', 'Schisto_Treatment',\n", + " 'Tb_Prevention_Ipt', 'Tb_Test_Screening', \n", + " 'Alri_Pneumonia_Treatment_Inpatient_Followup', 'AntenatalCare_Inpatient', 'Diarrhoea_Treatment_Inpatient',\n", + " 'DeliveryCare_Basic', 'Tb_Treatment']))\n", + "partition_facility_level = Partition.Simple('Facility_Level',np.unique(hsi_example['Facility_Level']))\n", + "\n", + "nodes = {\n", + " 'Appt': ProcessGroup(['Appt_Type_Code'], partition_appt_type),\n", + " 'HSI': ProcessGroup(['Treatment_ID'], partition_treatment_id),\n", + "}\n", + "\n", + "# Add nodes Waypoint\n", + "nodes['waypoint'] = Waypoint(partition_facility_level)\n", + "\n", + "bundles = [\n", + " Bundle('Appt', 'HSI', waypoints=['waypoint']),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Appt'], # left\n", + " ['waypoint'], # middle\n", + " ['HSI'], # right\n", + " ]\n", + "\n", + "\n", + "# Set the color for each appt type\n", + "# Set the color for each appt type\n", + "palette = {'AntenatalFirst': 'green', 'FamPlan': 'darkseagreen',\n", + " 'VCTNegative': 'lightsteelblue', 'NewAdult': 'cornflowerblue',\n", + " 'Over5OPD': 'lightpink','IPAdmission': 'palevioletred',\n", + " 'NormalDelivery': 'limegreen', 'TBNew': 'yellow'}\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_appt_type)\n", + "\n", + "# Generate and save Sankey\n", + "sankey_appt_level_hsi = weave(sdd, hsi_example, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_appt_level_hsi.auto_save_png(make_graph_file_name('Sankey_appt_level_hsi_example'))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "31629b7c852e4ab5a2862085ff8f6212", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "SankeyWidget(groups=[{'id': 'Appt', 'type': 'process', 'title': '', 'nodes': ['Appt^ConWithDCSA', 'Appt^Under5…" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Format the flow alt 2\n", + "size = dict(width=1000, height=2000, margins=dict(left=150, right=520))\n", + "\n", + "# Nodes in alphabetic order\n", + "partition_appt_type = Partition.Simple('Appt_Type_Code', pd.array([\n", + " 'ConWithDCSA',\n", + " 'Under5OPD', 'Over5OPD', 'IPAdmission', 'InpatientDays',\n", + " 'AntenatalFirst', 'ANCSubsequent', 'FamPlan', 'EPI', \n", + " 'CompDelivery', 'NormalDelivery', 'Csection',\n", + " 'AccidentsandEmerg', 'MajorSurg', 'MinorSurg',\n", + " 'U5Malnutr',\n", + " 'MentOPD',\n", + " 'Mammography', 'DiagRadio', 'Tomography', 'LabMolec', 'LabTBMicro',\n", + " 'MaleCirc', 'Peds', 'VCTNegative', 'VCTPositive', 'NewAdult', 'EstNonCom',\n", + " 'TBNew', 'TBFollowUp']))\n", + "partition_treatment_id = Partition.Simple('Treatment_ID', np.unique(hsi_all_levels['Treatment_ID']))\n", + "\n", + "nodes = {\n", + " 'Appt': ProcessGroup(['Appt_Type_Code'], partition_appt_type),\n", + " 'HSI': ProcessGroup(['Treatment_ID'], partition_treatment_id),\n", + "}\n", + "\n", + "\n", + "bundles = [\n", + " Bundle('Appt', 'HSI'),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Appt'], # left\n", + " ['HSI'], # right\n", + " ]\n", + "\n", + "\n", + "# Set the color for each appt type\n", + "palette = {'Under5OPD': 'lightpink', 'Over5OPD': 'lightpink',\n", + " 'IPAdmission': 'palevioletred', 'InpatientDays': 'mediumvioletred',\n", + " \n", + " 'AntenatalFirst': 'green', 'ANCSubsequent': 'green',\n", + " 'FamPlan': 'darkseagreen', 'EPI': 'paleturquoise', \n", + " 'CompDelivery': 'limegreen', 'NormalDelivery': 'limegreen', 'Csection': 'springgreen',\n", + " \n", + " 'AccidentsandEmerg': 'darkorange', 'MajorSurg': 'orange', 'MinorSurg': 'gold',\n", + " \n", + " 'ConWithDCSA': 'violet',\n", + " \n", + " 'U5Malnutr': 'orchid',\n", + " \n", + " 'MentOPD': 'darkgrey',\n", + " \n", + " 'Mammography': 'lightgrey', 'DiagRadio': 'lightgrey', 'Tomography': 'lightgrey', \n", + " 'LabMolec': 'gainsboro', 'LabTBMicro': 'gainsboro',\n", + " \n", + " 'MaleCirc': 'mediumslateblue', 'Peds': 'lightskyblue', \n", + " 'VCTNegative': 'lightsteelblue', 'VCTPositive': 'lightsteelblue', \n", + " 'NewAdult': 'cornflowerblue', 'EstNonCom': 'royalblue',\n", + " \n", + " 'TBNew': 'yellow', 'TBFollowUp': 'yellow'}\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_appt_type)\n", + "\n", + "# Generate and save Sankey\n", + "sankey_appt_level_hsi = weave(sdd, hsi_all_levels, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "sankey_appt_level_hsi.auto_save_png(make_graph_file_name('Sankey_appt_hsi'))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_hcwtime_appt_hsi.ipynb b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_hcwtime_appt_hsi.ipynb new file mode 100644 index 0000000000..4609694f42 --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/analysis_sankey_hcwtime_appt_hsi.ipynb @@ -0,0 +1,209 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# This file uses the run generated by `10_year_scale_run.py` and floweaver\n", + "to produce a Sankey diagram that maps hcw working time per cadre, appointments and disease modules.\n", + "\n", + "Below is the instruction to run the file.\n", + "\n", + "### Install floweaver in Anaconda Prompt (if use Jupyter Notebook) / PyCharm Terminal:\n", + "\n", + "pip install floweaver\n", + "\n", + "pip install ipysankeywidget\n", + "\n", + "jupyter nbextension enable --py --sys-prefix widgetsnbextension\n", + "\n", + "jupyter nbextension enable --py --sys-prefix ipysankeywidget\n", + "\n", + "jupyter notebook (to open jupyter notebook)\n", + "\n", + "### To display and save the output figures:\n", + "Select Start Jupyter Server from the Jupyter Actions Menu (lightbulb icon next to Run All cells icon)\n", + "\n", + "Open Event Log\n", + "\n", + "Open in Browser\n", + "\n", + "Find the script and run all cells\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import tlo\n", + "\n", + "from pathlib import Path\n", + "\n", + "from tlo.analysis.utils import get_scenario_outputs, load_pickled_dataframes\n", + "\n", + "import pandas as pd\n", + "\n", + "import numpy as np\n", + "\n", + "from matplotlib import pyplot as plt\n", + "\n", + "from ipysankeywidget import SankeyWidget\n", + "\n", + "from floweaver import *" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Results folder is: c:\\users\\jdbb1\\pycharmprojects\\tlomodel\\outputs\\bshe@ic.ac.uk\\scale_run_for_hcw_analysis-2023-08-20T102040Z_draw2\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0ac92b252f14078b47b92cdbb790d4e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "SankeyWidget(groups=[{'id': 'Officer', 'type': 'process', 'title': '', 'nodes': ['Officer^DCSA', 'Officer^Clin…" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Declare the name of the file that specified the scenarios used in this run.\n", + "scenario_filename = 'scale_run_for_hcw_analysis.py' # i.e., the 10_year_scale_run.py\n", + "\n", + "# Declare usual paths:\n", + "# Get the tlo path\n", + "tlopath = Path(tlo.__file__).parent.parent.parent\n", + "outputspath = tlopath / Path('outputs/bshe@ic.ac.uk')\n", + "\n", + "# Find results folder (most recent run generated using that scenario_filename)\n", + "f = -2 # -4: Actual + Default health care seeking, -2: Actual + Maximal health care seeking\n", + "results_folder = get_scenario_outputs(scenario_filename, outputspath)[f]\n", + "print(f\"Results folder is: {results_folder}\")\n", + "\n", + "# Sankey diagram height scale factor\n", + "if (f == -4) or (f == -3):\n", + " sankey_scale = 1\n", + "elif (f == -2) or (f == -1):\n", + " sankey_scale = 7144638826.033691 / 2206856635.0910654 # the total working time of file -2 / ... of file -4\n", + " \n", + "# Declare path for output graphs from this script\n", + "make_graph_file_name = lambda stub: results_folder / f\"{stub}.png\" # noqa: E731\n", + "\n", + "# Extract results\n", + "hcw_time = pd.read_csv(results_folder / 'hcw_working_time_per_hsi.csv')\n", + "\n", + "# Format data for flow\n", + "hcw_time['Module'] = hcw_time['Treatment_ID'].str.split('_').apply(lambda x: x[0])\n", + "hcw_time = hcw_time.groupby(['Officer_Category', 'Appt_Category', 'Module'])['Total_Mins_Used_Per_Year'].sum().reset_index()\n", + "\n", + "hcw_time['source'] = 'Officer_Category'\n", + "hcw_time['target'] = 'Module'\n", + "hcw_time['value'] = hcw_time['Total_Mins_Used_Per_Year']\n", + "\n", + "# Format the flow\n", + "\n", + "partition_officer_cat = Partition.Simple('Officer_Category',\n", + " pd.array(['DCSA', 'Clinical', 'Nursing and Midwifery', 'Pharmacy',\n", + " 'Laboratory', 'Radiography', 'Mental']))\n", + "partition_module = Partition.Simple('Module',\n", + " np.unique(hcw_time['Module']))\n", + "partition_appt_cat = Partition.Simple('Appt_Category',\n", + " pd.array(['ConWithDCSA', 'IPOP', 'RMNCH', 'MISC',\n", + " 'HIV', 'TB', 'NUTRITION', 'LABORATORY',\n", + " 'RADIOGRAPHY', 'MENTAL']))\n", + "\n", + "\n", + "nodes = {\n", + " 'Officer': ProcessGroup(['Officer_Category'], partition_officer_cat),\n", + " 'Module': ProcessGroup(['Module'], partition_module),\n", + "}\n", + "\n", + "# Add nodes Waypoint\n", + "nodes['waypoint'] = Waypoint(partition_appt_cat)\n", + "\n", + "bundles = [\n", + " Bundle('Officer', 'Module', waypoints=['waypoint']),\n", + "]\n", + "\n", + "ordering = [\n", + " ['Officer'], # left\n", + " ['waypoint'], # middle\n", + " ['Module'], # right\n", + " ]\n", + "\n", + "# Set the color for each officer category\n", + "palette = {'Clinical': 'skyblue', 'Nursing and Midwifery': 'lightpink',\n", + " 'Pharmacy': 'khaki', 'Laboratory': 'cadetblue',\n", + " 'Radiography': 'yellowgreen',\n", + " 'Mental': 'mediumorchid', 'DCSA': 'royalblue'\n", + " }\n", + "\n", + "# Set the size for the Sankey\n", + "size = dict(width=800, height=550*sankey_scale, margins=dict(left=180, right=180))\n", + "\n", + "# Sankey diagram definition (SDD)\n", + "sdd = SankeyDefinition(nodes, bundles, ordering, flow_partition=partition_officer_cat)\n", + "\n", + "hcw_time_flow = weave(sdd, hcw_time, palette=palette, measures='value').to_widget(**size)\n", + "\n", + "hcw_time_flow.auto_save_png(results_folder /'Sankey_hcw_time_flow.png')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/src/scripts/healthsystem/hsi_in_typical_run/scenario_hsi_in_typical_run.py b/src/scripts/healthsystem/hsi_in_typical_run/scenario_hsi_in_typical_run.py new file mode 100644 index 0000000000..489b1e6f0a --- /dev/null +++ b/src/scripts/healthsystem/hsi_in_typical_run/scenario_hsi_in_typical_run.py @@ -0,0 +1,124 @@ +""" +This file is used to capture the HSI that are run during a typical simulation, 2010-2021. It defines a large population + with all disease modules registered and an unconstrained (mode_appt_constraints=0) HealthSystem. + +Run on the remote batch system using: + ```tlo batch-submit src/scripts/healthsystem/hsi_in_typical_run/scenario_hsi_in_typical_run.py``` + +... or locally using: + ```tlo scenario-run src/scripts/healthsystem/hsi_in_typical_run/scenario_hsi_in_typical_run.py``` +""" + +from tlo import Date, logging +from tlo.methods import ( + alri, + bladder_cancer, + breast_cancer, + cardio_metabolic_disorders, + care_of_women_during_pregnancy, + contraception, + demography, + depression, + diarrhoea, + enhanced_lifestyle, + epi, + epilepsy, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + labour, + malaria, + measles, + newborn_outcomes, + oesophagealcancer, + other_adult_cancers, + postnatal_supervisor, + pregnancy_supervisor, + prostate_cancer, + stunting, + symptommanager, + wasting, +) +from tlo.scenario import BaseScenario + + +class LongRun(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2020, 1, 1) # looking at the usage from 2010 to 2019 + self.pop_size = 20_000 + self.number_of_draws = 1 + self.runs_per_draw = 1 + + def log_configuration(self): + return { + 'filename': 'scenario_hsi_in_typical_run', + 'directory': './outputs', + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.healthsystem': logging.INFO, + 'tlo.methods.demography': logging.INFO, + } + } + + def modules(self): + return [ + # Core Modules + demography.Demography(resourcefilepath=self.resources), + enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), + symptommanager.SymptomManager(resourcefilepath=self.resources, spurious_symptoms=False), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources), + healthburden.HealthBurden(resourcefilepath=self.resources), + + # Representations of the Healthcare System + healthsystem.HealthSystem(resourcefilepath=self.resources, mode_appt_constraints=0), + epi.Epi(resourcefilepath=self.resources), + + # - Contraception, Pregnancy and Labour + contraception.Contraception(resourcefilepath=self.resources, use_healthsystem=True), + pregnancy_supervisor.PregnancySupervisor(resourcefilepath=self.resources), + care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=self.resources), + labour.Labour(resourcefilepath=self.resources), + newborn_outcomes.NewbornOutcomes(resourcefilepath=self.resources), + postnatal_supervisor.PostnatalSupervisor(resourcefilepath=self.resources), + + # - Conditions of Early Childhood + diarrhoea.Diarrhoea(resourcefilepath=self.resources), + alri.Alri(resourcefilepath=self.resources), + stunting.Stunting(resourcefilepath=self.resources), + wasting.Wasting(resourcefilepath=self.resources), + + # - Communicable Diseases + hiv.Hiv(resourcefilepath=self.resources), + malaria.Malaria(resourcefilepath=self.resources), + measles.Measles(resourcefilepath=self.resources), + + # - Non-Communicable Conditions + # -- Cancers + bladder_cancer.BladderCancer(resourcefilepath=self.resources), + breast_cancer.BreastCancer(resourcefilepath=self.resources), + oesophagealcancer.OesophagealCancer(resourcefilepath=self.resources), + other_adult_cancers.OtherAdultCancer(resourcefilepath=self.resources), + prostate_cancer.ProstateCancer(resourcefilepath=self.resources), + + # -- Cardiometabolic Disorders + cardio_metabolic_disorders.CardioMetabolicDisorders(resourcefilepath=self.resources), + + # -- Injuries (Forthcoming) + + # -- Other Non-Communicable Conditions + depression.Depression(resourcefilepath=self.resources), + epilepsy.Epilepsy(resourcefilepath=self.resources), + ] + + def draw_parameters(self, draw_number, rng): + pass + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 5db4e4f054..fa19cc7ca5 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -345,7 +345,7 @@ def initialise(self): health_system = self.sim.modules['HealthSystem'] # Over-write ACCEPTED_FACILITY_LEVEL to to redirect all '1b' appointments to '2' - self.ACCEPTED_FACILITY_LEVEL = adjust_facility_level_to_merge_1b_and_2(self.ACCEPTED_FACILITY_LEVEL) + # self.ACCEPTED_FACILITY_LEVEL = adjust_facility_level_to_merge_1b_and_2(self.ACCEPTED_FACILITY_LEVEL) if not isinstance(self.target, tlo.population.Population): self.facility_info = health_system.get_facility_info(self) @@ -821,8 +821,7 @@ def pre_initialise_population(self): # Initialise the Consumables class self.consumables = Consumables( - data=self.update_consumables_availability_to_represent_merging_of_levels_1b_and_2( - self.parameters['availability_estimates']), + data=self.parameters['availability_estimates'], rng=rng_for_consumables, availability=self.get_cons_availability() ) @@ -1044,8 +1043,7 @@ def format_daily_capabilities(self, use_funded_or_actual_staffing: str) -> pd.Se """ # Get the capabilities data imported (according to the specified underlying assumptions). - capabilities = pool_capabilities_at_levels_1b_and_2( - self.parameters[f'Daily_Capabilities_{use_funded_or_actual_staffing}']) + capabilities = self.parameters[f'Daily_Capabilities_{use_funded_or_actual_staffing}'] capabilities = capabilities.rename(columns={'Officer_Category': 'Officer_Type_Code'}) # neaten # Create dataframe containing background information about facility and officer types @@ -1860,7 +1858,7 @@ def log_current_capabilities_and_usage(self): # Compute Fraction of Time For Each Officer and level officer = [_f.rsplit('Officer_')[1] for _f in comparison.index] level = [self._facility_by_facility_id[int(_fac_id)].level for _fac_id in facility_id] - level = list(map(lambda x: x.replace('1b', '2'), level)) + # level = list(map(lambda x: x.replace('1b', '2'), level)) summary_by_officer = comparison.groupby(by=[officer, level])[['Total_Minutes_Per_Day', 'Minutes_Used']].sum() summary_by_officer['Fraction_Time_Used'] = ( summary_by_officer['Minutes_Used'] / summary_by_officer['Total_Minutes_Per_Day'] diff --git a/tests/test_alri.py b/tests/test_alri.py index a98d2f277c..0601eb6075 100644 --- a/tests/test_alri.py +++ b/tests/test_alri.py @@ -1101,9 +1101,9 @@ def test_treatment_pathway_if_all_consumables_severe_case(seed, tmpdir): # If the child is older than 2 months (classification will be `danger_signs_pneumonia`). # - If Treatments Works --> No follow-up assert [ - ('FirstAttendance_Emergency', '2'), # <-- these would all be '1b' if levels '1b' and '2' are separate - ('Alri_Pneumonia_Treatment_Outpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient', '2'), + ('FirstAttendance_Emergency', '1b'), + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient', '1b'), ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='all'), incident_case_event=AlriIncidentCase_Lethal_DangerSigns_Pneumonia, treatment_effect='perfectly_effective', @@ -1112,10 +1112,10 @@ def test_treatment_pathway_if_all_consumables_severe_case(seed, tmpdir): # - If Treatment Does Not Work --> One follow-up as an inpatient. assert [ - ('FirstAttendance_Emergency', '2'), # <-- these would all be '1b' if levels '1b' and '2' are separate - ('Alri_Pneumonia_Treatment_Outpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient_Followup', '2') + ('FirstAttendance_Emergency', '1b'), + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient_Followup', '1b') ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='all'), incident_case_event=AlriIncidentCase_Lethal_DangerSigns_Pneumonia, treatment_effect='perfectly_ineffective', @@ -1125,9 +1125,9 @@ def test_treatment_pathway_if_all_consumables_severe_case(seed, tmpdir): # If the child is younger than 2 months # - If Treatments Works --> No follow-up assert [ - ('FirstAttendance_Emergency', '2'), # <-- these would all be '1b' if levels '1b' and '2' are separate - ('Alri_Pneumonia_Treatment_Outpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient', '2'), + ('FirstAttendance_Emergency', '1b'), + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient', '1b'), ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='all'), incident_case_event=AlriIncidentCase_Lethal_DangerSigns_Pneumonia, age_of_person_under_2_months=True, @@ -1136,10 +1136,10 @@ def test_treatment_pathway_if_all_consumables_severe_case(seed, tmpdir): # - If Treatment Does Not Work --> One follow-up as an inpatient. assert [ - ('FirstAttendance_Emergency', '2'), # <-- these would all be '1b' if levels '1b' and '2' are separate - ('Alri_Pneumonia_Treatment_Outpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient', '2'), - ('Alri_Pneumonia_Treatment_Inpatient_Followup', '2'), + ('FirstAttendance_Emergency', '1b'), + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient_Followup', '1b'), ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='all'), incident_case_event=AlriIncidentCase_Lethal_DangerSigns_Pneumonia, age_of_person_under_2_months=True, @@ -1156,8 +1156,7 @@ def test_treatment_pathway_if_no_consumables_mild_case(seed, tmpdir): ('FirstAttendance_NonEmergency', '0'), ('Alri_Pneumonia_Treatment_Outpatient', '0'), ('Alri_Pneumonia_Treatment_Outpatient', '1a'), # <-- referral due to lack of consumables - # ('Alri_Pneumonia_Treatment_Outpatient', '1b'), # <-- referral due to lack of consumables - # (would occur if levels '1b' and '2' are separate) + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), # <-- referral due to lack of consumables ('Alri_Pneumonia_Treatment_Outpatient', '2'), # <-- referral due to lack of consumables ('Alri_Pneumonia_Treatment_Inpatient_Followup', '2'), # <-- follow-up because treatment not successful ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='none'), @@ -1171,9 +1170,9 @@ def test_treatment_pathway_if_no_consumables_severe_case(seed, tmpdir): # Severe case and not available consumables --> successive referrals up to level 2, following emergency # appointment, plus follow-up appointment because treatment was not successful. assert [ - ('FirstAttendance_Emergency', '2'), - ('Alri_Pneumonia_Treatment_Outpatient', '2'), - # ('Alri_Pneumonia_Treatment_Inpatient', '1b'), # <-- would occur if levels '1b' and '2' are separate + ('FirstAttendance_Emergency', '1b'), + ('Alri_Pneumonia_Treatment_Outpatient', '1b'), + ('Alri_Pneumonia_Treatment_Inpatient', '1b'), ('Alri_Pneumonia_Treatment_Inpatient', '2'), # <-- referral due to lack of consumables ('Alri_Pneumonia_Treatment_Inpatient_Followup', '2'), # <-- follow-up because treatment not successful ] == generate_hsi_sequence(sim=get_sim(seed=seed, tmpdir=tmpdir, cons_available='none'), diff --git a/tests/test_healthcareseeking.py b/tests/test_healthcareseeking.py index a53006166c..e85ee4226c 100644 --- a/tests/test_healthcareseeking.py +++ b/tests/test_healthcareseeking.py @@ -1579,19 +1579,17 @@ def on_birth(self, mother, child): assert {'1a': 1.0} == get_events_scheduled_following_hcs_poll( prob_non_emergency_care_seeking_by_level=[0.0, 1.0, 0.0, 0.0]) - # 100% chance that non-emergency-appointment is at level ('1b') {occurs at level labelled as '2' with merge of - # levels '1b & 2') - assert {'2': 1.0} == get_events_scheduled_following_hcs_poll( + # 100% chance that non-emergency-appointment is at level ('1b') + assert {'1b': 1.0} == get_events_scheduled_following_hcs_poll( prob_non_emergency_care_seeking_by_level=[0.0, 0.0, 1.0, 0.0]) - # 100% chance that non-emergency-appointment is at level ('2') - assert {'2': 1.0} == get_events_scheduled_following_hcs_poll( - prob_non_emergency_care_seeking_by_level=[0.0, 0.0, 0.0, 1.0]) - - # A mixture of 0 / 1a / (1b) / 2 - props = get_events_scheduled_following_hcs_poll(prob_non_emergency_care_seeking_by_level=[0.25, 0.25, 0.25, 0.25]) - assert ('0' in set(props.keys())) and ('1a' in set(props.keys())) and ('2' in set(props.keys())) - assert all(np.array(list(props.values())) > 0) + # A mixture of 0 / 1a / 1b / 2 + props = np.array(list( + get_events_scheduled_following_hcs_poll( + prob_non_emergency_care_seeking_by_level=[0.25, 0.25, 0.25, 0.25]).values() + )) + assert 4 == len(props) + assert all(props > 0) def test_custom_function_is_equivalent_to_linear_model(seed): diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index ee9c64d22b..e9b7d6678c 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -2047,19 +2047,19 @@ def check_appt_works(district, level, appt_type) -> Tuple: # (If they can happen at either, then this test will make it look like they are happening at both!) # The file on the HSI expected not to run should show such appointments as not happening at either '1b' or '2'. # .... work out which appointment cannot happen at either '1b' or '2' - _levels_at_which_appts_dont_run = appts_not_run.groupby( - by=['use_funded_or_actual_staffing', 'appt_type', 'district'])['level'].sum() - _levels_at_which_appts_dont_run = _levels_at_which_appts_dont_run.drop( - _levels_at_which_appts_dont_run.index[_levels_at_which_appts_dont_run.isin(['1b', '2'])] - ) - appts_not_run = _levels_at_which_appts_dont_run.reset_index().dropna() - appts_not_run['level'] = appts_not_run['level'].replace({'21b': '2'}) # ... label such appointments for level '2' - # ... reproduce that block labelled for level '1b' - appts_not_run_level2 = appts_not_run.loc[appts_not_run.level == '2'].copy() - appts_not_run_level2['level'] = '1b' - appts_not_run = pd.concat([appts_not_run, appts_not_run_level2]) - # ... re-order columns to suit. - appts_not_run = appts_not_run[['use_funded_or_actual_staffing', 'level', 'appt_type', 'district']] + # _levels_at_which_appts_dont_run = appts_not_run.groupby( + # by=['use_funded_or_actual_staffing', 'appt_type', 'district'])['level'].sum() + # _levels_at_which_appts_dont_run = _levels_at_which_appts_dont_run.drop( + # _levels_at_which_appts_dont_run.index[_levels_at_which_appts_dont_run.isin(['1b', '2'])] + # ) + # appts_not_run = _levels_at_which_appts_dont_run.reset_index().dropna() + # appts_not_run['level'] = appts_not_run['level'].replace({'21b': '2'}) # ... label such appointments for level '2' + # # ... reproduce that block labelled for level '1b' + # appts_not_run_level2 = appts_not_run.loc[appts_not_run.level == '2'].copy() + # appts_not_run_level2['level'] = '1b' + # appts_not_run = pd.concat([appts_not_run, appts_not_run_level2]) + # # ... re-order columns to suit. + # appts_not_run = appts_not_run[['use_funded_or_actual_staffing', 'level', 'appt_type', 'district']] # reformat the 'district' info at levels 3 and 4 in results to map with appts_not_run file for convenience districts_per_region = mfl[['District', 'Region']].drop_duplicates().dropna(axis='index', how='any').set_index(