From dbff470b51cde44beeefdae3575d52e0c19964bc Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:00:09 +0100 Subject: [PATCH 001/103] Investigate analysis of events at sim level --- src/tlo/simulation.py | 9 +++++++++ tests/test_rti.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 219b1b8a6f..a641909ed1 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -231,6 +231,15 @@ def simulate(self, *, end_date): if date >= end_date: self.date = end_date break + + #if event.target != self.population: + # print("Event: ", event) + + if event.module == self.modules['RTI']: + print("RTI event ", event) + print(" target ", event.target) + if event.target != self.population: + self.population.props.at[event.tar] self.fire_single_event(event, date) # The simulation has ended. diff --git a/tests/test_rti.py b/tests/test_rti.py index 0e231fb4af..99243b988e 100644 --- a/tests/test_rti.py +++ b/tests/test_rti.py @@ -25,6 +25,17 @@ end_date = Date(2012, 1, 1) popsize = 1000 +@pytest.mark.slow +def test_data_harvesting(seed): + """ + This test runs a simulation with a functioning health system with full service availability and no set + constraints + """ + # create sim object + sim = create_basic_rti_sim(popsize, seed) + # run simulation + sim.simulate(end_date=end_date) + exit(-1) def check_dtypes(simulation): # check types of columns in dataframe, check they are the same, list those that aren't @@ -65,6 +76,7 @@ def test_run(seed): check_dtypes(sim) + @pytest.mark.slow def test_all_injuries_run(seed): """ From 6a29cee74dde35c140e0a71e3e8425725389724a Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 19 Apr 2024 13:27:11 +0100 Subject: [PATCH 002/103] update contraception.py logging to capture df for new pregnancies --- .../full_model_long_run_cohort.py | 35 +++++++++++++++++++ src/tlo/methods/contraception.py | 6 ++++ 2 files changed, 41 insertions(+) create mode 100644 src/scripts/maternal_perinatal_analyses/scenario_files/full_model_long_run_cohort.py diff --git a/src/scripts/maternal_perinatal_analyses/scenario_files/full_model_long_run_cohort.py b/src/scripts/maternal_perinatal_analyses/scenario_files/full_model_long_run_cohort.py new file mode 100644 index 0000000000..b0a5072bc7 --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/scenario_files/full_model_long_run_cohort.py @@ -0,0 +1,35 @@ +from tlo import Date, logging +from tlo.methods.fullmodel import fullmodel + +from tlo.scenario import BaseScenario + + +class FullModelRunForCohort(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 537184 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2025, 1, 1) + self.pop_size = 200_000 + self.number_of_draws = 1 + self.runs_per_draw = 1 + + def log_configuration(self): + return { + 'filename': 'fullmodel_200k_cohort', 'directory': './outputs', + "custom_levels": { + "*": logging.WARNING, + "tlo.methods.contraception": logging.DEBUG, + } + } + + def modules(self): + return fullmodel(resourcefilepath=self.resources) + + def draw_parameters(self, draw_number, rng): + return {} + + +if __name__ == '__main__': + from tlo.cli import scenario_run + scenario_run([__file__]) diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 67d6684fce..ad61e1aeb5 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1068,6 +1068,12 @@ def set_new_pregnancy(self, women_id: list): }, description='pregnancy following the failure of contraceptive method') + person = df.loc[w] + + logger.debug(key='properties_of_pregnant_person', + data=person.to_dict(), + description='values of all properties at the time of pregnancy for newwly pregnany persons') + class ContraceptionLoggingEvent(RegularEvent, PopulationScopeEventMixin): def __init__(self, module): From e23faaf5c531a89eeeced1d20115e33e2308bb8d Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 31 May 2024 11:32:16 +0100 Subject: [PATCH 003/103] create cohort module and cohort resource file. updated demography logging due to error when no males in population --- resources/ResourceFile_PregnancyCohort.xlsx | 3 + src/tlo/methods/demography.py | 4 +- src/tlo/methods/mnh_cohort_module.py | 257 ++++++++++++++++++++ tests/test_mnh_cohort.py | 54 ++++ 4 files changed, 316 insertions(+), 2 deletions(-) create mode 100644 resources/ResourceFile_PregnancyCohort.xlsx create mode 100644 src/tlo/methods/mnh_cohort_module.py create mode 100644 tests/test_mnh_cohort.py diff --git a/resources/ResourceFile_PregnancyCohort.xlsx b/resources/ResourceFile_PregnancyCohort.xlsx new file mode 100644 index 0000000000..bd4b0af086 --- /dev/null +++ b/resources/ResourceFile_PregnancyCohort.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f41deeb8fd44fda6cc451a7e23390c98bba449e704ccd62e258caaf2e6e6606 +size 18595782 diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index f3d3e3a8c4..3e99dd7653 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -781,8 +781,8 @@ def apply(self, population): logger.info( key='population', data={'total': sum(sex_count), - 'male': sex_count['M'], - 'female': sex_count['F'] + 'male': sex_count['M'] if 'M' in sex_count.index else 0, + 'female': sex_count['F'] if 'F' in sex_count.index else 0, }) # (nb. if you groupby both sex and age_range, you weirdly lose categories where size==0, so diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py new file mode 100644 index 0000000000..da8c2e8598 --- /dev/null +++ b/src/tlo/methods/mnh_cohort_module.py @@ -0,0 +1,257 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +from tlo import DateOffset, Module, Parameter, Property, Types, logging +from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent +from tlo.methods import Metadata +from tlo.methods.causes import Cause +from tlo.methods.hsi_event import HSI_Event + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +# --------------------------------------------------------------------------------------------------------- +# MODULE DEFINITIONS +# --------------------------------------------------------------------------------------------------------- + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +class MaternalNewbornHealthCohort(Module): + """ + + """ + + # INIT_DEPENDENCIES = {'Demography'} + # + # OPTIONAL_INIT_DEPENDENCIES = {''} + # + # ADDITIONAL_DEPENDENCIES = {''} + + # Declare Metadata (this is for a typical 'Disease Module') + METADATA = { + Metadata.DISEASE_MODULE, + Metadata.USES_SYMPTOMMANAGER, + Metadata.USES_HEALTHSYSTEM, + Metadata.USES_HEALTHBURDEN + } + CAUSES_OF_DEATH = {} + CAUSES_OF_DISABILITY = {} + PARAMETERS = {} + PROPERTIES = {} + + def __init__(self, name=None, resourcefilepath=None): + # NB. Parameters passed to the module can be inserted in the __init__ definition. + + super().__init__(name) + self.resourcefilepath = resourcefilepath + + def read_parameters(self, data_folder): + """Read parameter values from file, if required. + To access files use: Path(self.resourcefilepath) / file_name + """ + pass + + def initialise_population(self, population): + """Set our property values for the initial population. + + This method is called by the simulation when creating the initial population, and is + responsible for assigning initial values, for every individual, of those properties + 'owned' by this module, i.e. those declared in the PROPERTIES dictionary above. + + :param population: the population of individuals + """ + pass + + def initialise_simulation(self, sim): + """Get ready for simulation start. + + This method is called just before the main simulation loop begins, and after all + modules have read their parameters and the initial population has been created. + It is a good place to add initial events to the event queue. + + """ + # cohort_prop_df = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancyCohort.xlsx') + # from tlo.analysis.utils import parse_log_file + # cohort_prop_df = parse_log_file( + # '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', + # level=logging.DEBUG)['tlo.methods.contraception'] + # + # cohort_prop_df_for_pop = cohort_prop_df.loc[0:(len(self.sim.population.props))-1] + # self.sim.population.props = cohort_prop_df_for_pop + # + # # todo: pd.NaT values are not carried over when excel file was created. + # # df = self.sim.population.props + # # population = df.loc[df.is_alive] + # # for column in df.columns: + # # df.loc[population.index, column] = cohort_prop_df_for_pop[column] + # + # df = self.sim.population.props + # population = df.loc[df.is_alive] + # df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date + # df.loc[population.index, 'co_contraception'] = "not_using" + + + def on_birth(self, mother_id, child_id): + """Initialise our properties for a newborn individual. + + This is called by the simulation whenever a new person is born. + + :param mother_id: the mother for this child + :param child_id: the new child + """ + raise NotImplementedError + + def report_daly_values(self): + """ + This must send back a pd.Series or pd.DataFrame that reports on the average daly-weights that have been + experienced by persons in the previous month. Only rows for alive-persons must be returned. + If multiple causes in CAUSES_OF_DISABILITY are defined, a pd.DataFrame must be returned with a column + corresponding to each cause (but if only one cause in CAUSES_OF_DISABILITY is defined, the pd.Series does not + need to be given a specific name). + + To return a value of 0.0 (fully health) for everyone, use: + df = self.sim.population.props + return pd.Series(index=df.index[df.is_alive],data=0.0) + """ + raise NotImplementedError + + def on_hsi_alert(self, person_id, treatment_id): + """ + This is called whenever there is an HSI event commissioned by one of the other disease modules. + """ + + raise NotImplementedError + + +# --------------------------------------------------------------------------------------------------------- +# DISEASE MODULE EVENTS +# +# These are the events which drive the simulation of the disease. It may be a regular event that updates +# the status of all the population of subsections of it at one time. There may also be a set of events +# that represent disease events for particular persons. +# --------------------------------------------------------------------------------------------------------- + +class Skeleton_Event(RegularEvent, PopulationScopeEventMixin): + """A skeleton class for an event + + Regular events automatically reschedule themselves at a fixed frequency, + and thus implement discrete timestep type behaviour. The frequency is + specified when calling the base class constructor in our __init__ method. + """ + + def __init__(self, module): + """One line summary here + + We need to pass the frequency at which we want to occur to the base class + constructor using super(). We also pass the module that created this event, + so that random number generators can be scoped per-module. + + :param module: the module that created this event + """ + super().__init__(module, frequency=DateOffset(months=1)) + assert isinstance(module, MaternalNewbornHealthCohort) + + def apply(self, population): + """Apply this event to the population. + + :param population: the current population + """ + raise NotImplementedError + + +# --------------------------------------------------------------------------------------------------------- +# LOGGING EVENTS +# +# Put the logging events here. There should be a regular logger outputting current states of the +# population. There may also be a logging event that is driven by particular events. +# --------------------------------------------------------------------------------------------------------- + +class Skeleton_LoggingEvent(RegularEvent, PopulationScopeEventMixin): + def __init__(self, module): + """Produce a summary of the numbers of people with respect to the action of this module. + This is a regular event that can output current states of people or cumulative events since last logging event. + """ + + # run this event every year + self.repeat = 12 + super().__init__(module, frequency=DateOffset(months=self.repeat)) + assert isinstance(module, MaternalNewbornHealthCohort) + + def apply(self, population): + # Make some summary statistics + + dict_to_output = { + 'Metric_One': 1.0, + 'Metric_Two': 2.0 + } + + logger.info(key='summary_12m', data=dict_to_output) + + +# --------------------------------------------------------------------------------------------------------- +# HEALTH SYSTEM INTERACTION EVENTS +# +# Here are all the different Health System Interactions Events that this module will use. +# --------------------------------------------------------------------------------------------------------- + +class HSI_Skeleton_Example_Interaction(HSI_Event, IndividualScopeEventMixin): + """This is a Health System Interaction Event. An interaction with the healthsystem are encapsulated in events + like this. + It must begin HSI__Description + """ + + def __init__(self, module, person_id): + super().__init__(module, person_id=person_id) + assert isinstance(module, MaternalNewbornHealthCohort) + + # Define the appointments types needed + the_appt_footprint = self.make_appt_footprint({'Over5OPD': 1}) # This requires one adult out-patient appt. + + # Define the facilities at which this event can occur (only one is allowed) + # Choose from: list(pd.unique(self.sim.modules['HealthSystem'].parameters['Facilities_For_Each_District'] + # ['Facility_Level'])) + the_accepted_facility_level = 0 + + # Define the necessary information for an HSI + self.TREATMENT_ID = 'Skeleton_Example_Interaction' # This must begin with the module name + self.EXPECTED_APPT_FOOTPRINT = the_appt_footprint + self.ACCEPTED_FACILITY_LEVEL = the_accepted_facility_level + self.ALERT_OTHER_DISEASES = [] + + def apply(self, person_id, squeeze_factor): + """ + Do the action that take place in this health system interaction, in light of prevailing conditions in the + healthcare system: + + * squeeze_factor (an argument provided to the event) indicates the extent to which this HSI_Event is being + run in the context of an over-burdened healthcare facility. + * bed_days_allocated_to_this_event (a property of the event) indicates the number and types of bed-days + that have been allocated to this event. + + Can return an updated APPT_FOOTPRINT if this differs from the declaration in self.EXPECTED_APPT_FOOTPRINT + + To request consumables use - self.get_all_consumables(item_codes=my_item_codes) + """ + pass + + def did_not_run(self): + """ + Do any action that is necessary each time when the health system interaction is not run. + This is called each day that the HSI is 'due' but not run due to insufficient health system capabilities. + Return False to cause this HSI event not to be rescheduled and to therefore never be run. + (Returning nothing or True will cause this event to be rescheduled for the next day.) + """ + pass + + def never_ran(self): + """ + Do any action that is necessary when it is clear that the HSI event will never be run. This will occur if it + has not run and the simulation date has passed the date 'tclose' by which the event was scheduled to have + occurred. + Do not return anything. + """ + pass diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py new file mode 100644 index 0000000000..a47fd7753e --- /dev/null +++ b/tests/test_mnh_cohort.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from tlo import DAYS_IN_MONTH, DAYS_IN_YEAR, Date, Simulation +from tlo.analysis.utils import parse_log_file +from tlo.lm import LinearModel, LinearModelType +from tlo.methods import mnh_cohort_module +from tlo.methods.fullmodel import fullmodel + + +# The resource files +try: + resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' +except NameError: + # running interactively + resourcefilepath = Path('./resources') + +start_date = Date(2010, 1, 1) + + +def check_dtypes(simulation): + # check types of columns + df = simulation.population.props + orig = simulation.population.new_row + assert (df.dtypes == orig.dtypes).all() + + +def register_modules(sim): + """Defines sim variable and registers all modules that can be called when running the full suite of pregnancy + modules""" + + sim.register(*fullmodel(resourcefilepath=resourcefilepath), + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), + ) + +def test_run_sim_with_mnh_cohort(tmpdir, seed): + sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) + + register_modules(sim) + sim.make_initial_population(n=1000) + sim.simulate(end_date=Date(2011, 1, 1)) + check_dtypes(sim) + +# def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): +# sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) +# +# register_modules(sim) +# sim.make_initial_population(n=1000) +# sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) +# # to do: check properties!! From 460f870542c2451d806a95ebe58142c6fa3c2ebd Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 31 May 2024 15:04:21 +0100 Subject: [PATCH 004/103] updates to mnh module --- src/tlo/methods/mnh_cohort_module.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index da8c2e8598..487e1e8f57 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -75,13 +75,15 @@ def initialise_simulation(self, sim): """ # cohort_prop_df = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancyCohort.xlsx') - # from tlo.analysis.utils import parse_log_file - # cohort_prop_df = parse_log_file( - # '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', - # level=logging.DEBUG)['tlo.methods.contraception'] - # - # cohort_prop_df_for_pop = cohort_prop_df.loc[0:(len(self.sim.population.props))-1] - # self.sim.population.props = cohort_prop_df_for_pop + from tlo.analysis.utils import parse_log_file + t = parse_log_file( + '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', + level=logging.DEBUG)['tlo.methods.contraception'] + + cohort_prop_df2 = t['properties_of_pregnant_person'] + cohort_prop_df_for_pop = cohort_prop_df2.loc[0:(len(self.sim.population.props))-1] + self.sim.population.props = cohort_prop_df_for_pop + # # # todo: pd.NaT values are not carried over when excel file was created. # # df = self.sim.population.props @@ -89,10 +91,10 @@ def initialise_simulation(self, sim): # # for column in df.columns: # # df.loc[population.index, column] = cohort_prop_df_for_pop[column] # - # df = self.sim.population.props - # population = df.loc[df.is_alive] - # df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date - # df.loc[population.index, 'co_contraception'] = "not_using" + df = self.sim.population.props + population = df.loc[df.is_alive] + df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date + df.loc[population.index, 'co_contraception'] = "not_using" def on_birth(self, mother_id, child_id): From 878695d60693228164e693b4322d0f164f909f0f Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 10 Jun 2024 12:57:28 +0100 Subject: [PATCH 005/103] updates to mnh module --- src/tlo/methods/mnh_cohort_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 487e1e8f57..e96b20ed27 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -83,9 +83,10 @@ def initialise_simulation(self, sim): cohort_prop_df2 = t['properties_of_pregnant_person'] cohort_prop_df_for_pop = cohort_prop_df2.loc[0:(len(self.sim.population.props))-1] self.sim.population.props = cohort_prop_df_for_pop - + self.sim.population.props.index.name = 'person' # # # todo: pd.NaT values are not carried over when excel file was created. + # todo replace with pickles # # df = self.sim.population.props # # population = df.loc[df.is_alive] # # for column in df.columns: From 12a56a7249c385d3251d38929ce7b5e1a09a5552 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 26 Jun 2024 12:27:33 +0100 Subject: [PATCH 006/103] update to cohort and util.py --- .../cohort_interventions_scenario.py | 48 ++++ src/tlo/methods/mnh_cohort_module.py | 208 +++++------------- src/tlo/util.py | 9 +- tests/test_mnh_cohort.py | 5 +- 4 files changed, 108 insertions(+), 162 deletions(-) create mode 100644 src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py new file mode 100644 index 0000000000..15c722b687 --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -0,0 +1,48 @@ +from tlo import Date, logging +from tlo.methods import mnh_cohort_module +from tlo.methods.fullmodel import fullmodel +from tlo.scenario import BaseScenario + + +class BaselineScenario(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 661184 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2031, 1, 1) + self.pop_size = 250_000 + self.number_of_draws = 1 + self.runs_per_draw = 20 + + def log_configuration(self): + return { + 'filename': 'intervention_scenario_test', 'directory': './outputs', + "custom_levels": { + "*": logging.WARNING, + "tlo.methods.demography": logging.INFO, + "tlo.methods.demography.detail": logging.INFO, + "tlo.methods.contraception": logging.INFO, + "tlo.methods.healthsystem.summary": logging.INFO, + "tlo.methods.healthburden": logging.INFO, + "tlo.methods.labour": logging.INFO, + "tlo.methods.labour.detail": logging.INFO, + "tlo.methods.newborn_outcomes": logging.INFO, + "tlo.methods.care_of_women_during_pregnancy": logging.INFO, + "tlo.methods.pregnancy_supervisor": logging.INFO, + "tlo.methods.postnatal_supervisor": logging.INFO, + } + } + + def modules(self): + return [*fullmodel(resourcefilepath=self.resources), + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=self.resources)] + + def draw_parameters(self, draw_number, rng): + return { + 'MaternalNewbornHealthCohort': {'blocked_intervention': 'screening (direct)'}, + } + + +if __name__ == '__main__': + from tlo.cli import scenario_run + scenario_run([__file__]) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index e96b20ed27..61dddc3c0c 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -3,10 +3,8 @@ import numpy as np import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata -from tlo.methods.causes import Cause -from tlo.methods.hsi_event import HSI_Event +from tlo.analysis.utils import parse_log_file logger = logging.getLogger(__name__) @@ -64,7 +62,31 @@ def initialise_population(self, population): :param population: the population of individuals """ - pass + + # TODO: CURRENT ISSUE - INDIVIDUALS IN THE POPULATION ARE SCHEDULED HSIs IN INITIALISE_POP/SIM BY OTHER MODULES, + # THEIR PROPERTIES ARE THEN OVER WRITTEN BY THIS MODULE AND ITS CRASHING HSIs + + log_file = parse_log_file( + '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/' + 'fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', + level=logging.DEBUG)['tlo.methods.contraception'] + + all_pregnancies = log_file['properties_of_pregnant_person'].loc[ + log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) + all_pregnancies.index = [x for x in range(len(all_pregnancies))] + + preg_pop = all_pregnancies.loc[0:(len(self.sim.population.props))-1] + + props_dtypes = self.sim.population.props.dtypes + preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) + preg_pop_final.index.name = 'person' + + self.sim.population.props = preg_pop_final + + df = self.sim.population.props + population = df.loc[df.is_alive] + df.loc[population.index, 'date_of_last_pregnancy'] = self.sim.start_date + df.loc[population.index, 'co_contraception'] = "not_using" def initialise_simulation(self, sim): """Get ready for simulation start. @@ -74,29 +96,31 @@ def initialise_simulation(self, sim): It is a good place to add initial events to the event queue. """ - # cohort_prop_df = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancyCohort.xlsx') - from tlo.analysis.utils import parse_log_file - t = parse_log_file( - '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', - level=logging.DEBUG)['tlo.methods.contraception'] - - cohort_prop_df2 = t['properties_of_pregnant_person'] - cohort_prop_df_for_pop = cohort_prop_df2.loc[0:(len(self.sim.population.props))-1] - self.sim.population.props = cohort_prop_df_for_pop - self.sim.population.props.index.name = 'person' + pass + # # cohort_prop_df = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancyCohort.xlsx') + # from tlo.analysis.utils import parse_log_file, load_pickled_dataframes, get_scenario_outputs # - # # todo: pd.NaT values are not carried over when excel file was created. - # todo replace with pickles - # # df = self.sim.population.props - # # population = df.loc[df.is_alive] - # # for column in df.columns: - # # df.loc[population.index, column] = cohort_prop_df_for_pop[column] + # log_file = parse_log_file( + # '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/' + # 'fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', + # level=logging.DEBUG)['tlo.methods.contraception'] # - df = self.sim.population.props - population = df.loc[df.is_alive] - df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date - df.loc[population.index, 'co_contraception'] = "not_using" - + # all_pregnancies = log_file['properties_of_pregnant_person'].loc[ + # log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) + # all_pregnancies.index = [x for x in range(len(all_pregnancies))] + # + # preg_pop = all_pregnancies.loc[0:(len(self.sim.population.props))-1] + # + # props_dtypes = self.sim.population.props.dtypes + # preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) + # preg_pop_final.index.name = 'person' + # + # self.sim.population.props = preg_pop_final + # + # df = self.sim.population.props + # population = df.loc[df.is_alive] + # df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date + # df.loc[population.index, 'co_contraception'] = "not_using" def on_birth(self, mother_id, child_id): """Initialise our properties for a newborn individual. @@ -106,7 +130,7 @@ def on_birth(self, mother_id, child_id): :param mother_id: the mother for this child :param child_id: the new child """ - raise NotImplementedError + pass def report_daly_values(self): """ @@ -120,141 +144,11 @@ def report_daly_values(self): df = self.sim.population.props return pd.Series(index=df.index[df.is_alive],data=0.0) """ - raise NotImplementedError + pass def on_hsi_alert(self, person_id, treatment_id): """ This is called whenever there is an HSI event commissioned by one of the other disease modules. """ - raise NotImplementedError - - -# --------------------------------------------------------------------------------------------------------- -# DISEASE MODULE EVENTS -# -# These are the events which drive the simulation of the disease. It may be a regular event that updates -# the status of all the population of subsections of it at one time. There may also be a set of events -# that represent disease events for particular persons. -# --------------------------------------------------------------------------------------------------------- - -class Skeleton_Event(RegularEvent, PopulationScopeEventMixin): - """A skeleton class for an event - - Regular events automatically reschedule themselves at a fixed frequency, - and thus implement discrete timestep type behaviour. The frequency is - specified when calling the base class constructor in our __init__ method. - """ - - def __init__(self, module): - """One line summary here - - We need to pass the frequency at which we want to occur to the base class - constructor using super(). We also pass the module that created this event, - so that random number generators can be scoped per-module. - - :param module: the module that created this event - """ - super().__init__(module, frequency=DateOffset(months=1)) - assert isinstance(module, MaternalNewbornHealthCohort) - - def apply(self, population): - """Apply this event to the population. - - :param population: the current population - """ - raise NotImplementedError - - -# --------------------------------------------------------------------------------------------------------- -# LOGGING EVENTS -# -# Put the logging events here. There should be a regular logger outputting current states of the -# population. There may also be a logging event that is driven by particular events. -# --------------------------------------------------------------------------------------------------------- - -class Skeleton_LoggingEvent(RegularEvent, PopulationScopeEventMixin): - def __init__(self, module): - """Produce a summary of the numbers of people with respect to the action of this module. - This is a regular event that can output current states of people or cumulative events since last logging event. - """ - - # run this event every year - self.repeat = 12 - super().__init__(module, frequency=DateOffset(months=self.repeat)) - assert isinstance(module, MaternalNewbornHealthCohort) - - def apply(self, population): - # Make some summary statistics - - dict_to_output = { - 'Metric_One': 1.0, - 'Metric_Two': 2.0 - } - - logger.info(key='summary_12m', data=dict_to_output) - - -# --------------------------------------------------------------------------------------------------------- -# HEALTH SYSTEM INTERACTION EVENTS -# -# Here are all the different Health System Interactions Events that this module will use. -# --------------------------------------------------------------------------------------------------------- - -class HSI_Skeleton_Example_Interaction(HSI_Event, IndividualScopeEventMixin): - """This is a Health System Interaction Event. An interaction with the healthsystem are encapsulated in events - like this. - It must begin HSI__Description - """ - - def __init__(self, module, person_id): - super().__init__(module, person_id=person_id) - assert isinstance(module, MaternalNewbornHealthCohort) - - # Define the appointments types needed - the_appt_footprint = self.make_appt_footprint({'Over5OPD': 1}) # This requires one adult out-patient appt. - - # Define the facilities at which this event can occur (only one is allowed) - # Choose from: list(pd.unique(self.sim.modules['HealthSystem'].parameters['Facilities_For_Each_District'] - # ['Facility_Level'])) - the_accepted_facility_level = 0 - - # Define the necessary information for an HSI - self.TREATMENT_ID = 'Skeleton_Example_Interaction' # This must begin with the module name - self.EXPECTED_APPT_FOOTPRINT = the_appt_footprint - self.ACCEPTED_FACILITY_LEVEL = the_accepted_facility_level - self.ALERT_OTHER_DISEASES = [] - - def apply(self, person_id, squeeze_factor): - """ - Do the action that take place in this health system interaction, in light of prevailing conditions in the - healthcare system: - - * squeeze_factor (an argument provided to the event) indicates the extent to which this HSI_Event is being - run in the context of an over-burdened healthcare facility. - * bed_days_allocated_to_this_event (a property of the event) indicates the number and types of bed-days - that have been allocated to this event. - - Can return an updated APPT_FOOTPRINT if this differs from the declaration in self.EXPECTED_APPT_FOOTPRINT - - To request consumables use - self.get_all_consumables(item_codes=my_item_codes) - """ - pass - - def did_not_run(self): - """ - Do any action that is necessary each time when the health system interaction is not run. - This is called each day that the HSI is 'due' but not run due to insufficient health system capabilities. - Return False to cause this HSI event not to be rescheduled and to therefore never be run. - (Returning nothing or True will cause this event to be rescheduled for the next day.) - """ - pass - - def never_ran(self): - """ - Do any action that is necessary when it is clear that the HSI event will never be run. This will occur if it - has not run and the simulation date has passed the date 'tclose' by which the event was scheduled to have - occurred. - Do not return anything. - """ pass diff --git a/src/tlo/util.py b/src/tlo/util.py index cafd04f738..5b6e18fbd9 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -112,10 +112,13 @@ def sample_outcome(probs: pd.DataFrame, rng: np.random.RandomState): cumsum = _probs.cumsum(axis=1) draws = pd.Series(rng.rand(len(cumsum)), index=cumsum.index) y = cumsum.gt(draws, axis=0) - outcome = y.idxmax(axis=1) + if not y.empty: + outcome = y.idxmax(axis=1) + # return as a dict of form {person_id: outcome} only in those cases where the outcome is one of the events. + return outcome.loc[outcome != '_'].to_dict() - # return as a dict of form {person_id: outcome} only in those cases where the outcome is one of the events. - return outcome.loc[outcome != '_'].to_dict() + else: + return dict() BitsetDType = Property.PANDAS_TYPE_MAP[Types.BITSET] diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index a47fd7753e..521f525c61 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -33,8 +33,9 @@ def register_modules(sim): """Defines sim variable and registers all modules that can be called when running the full suite of pregnancy modules""" - sim.register(*fullmodel(resourcefilepath=resourcefilepath), - mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), + sim.register(mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), + *fullmodel(resourcefilepath=resourcefilepath), + ) def test_run_sim_with_mnh_cohort(tmpdir, seed): From 973cbff7c339e5bef7418ca24bf4b69edc988554 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 24 Jul 2024 08:16:03 +0100 Subject: [PATCH 007/103] update to cohort and util.py --- tests/test_mnh_cohort.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 521f525c61..bc2f14d1bb 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -1,13 +1,7 @@ import os from pathlib import Path -import numpy as np -import pandas as pd -import pytest - -from tlo import DAYS_IN_MONTH, DAYS_IN_YEAR, Date, Simulation -from tlo.analysis.utils import parse_log_file -from tlo.lm import LinearModel, LinearModelType +from tlo import Date, Simulation, logging from tlo.methods import mnh_cohort_module from tlo.methods.fullmodel import fullmodel @@ -39,7 +33,8 @@ def register_modules(sim): ) def test_run_sim_with_mnh_cohort(tmpdir, seed): - sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) + sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ + "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) sim.make_initial_population(n=1000) From 05098f78668a5317667d58cbda882a364a031277 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:26:39 +0200 Subject: [PATCH 008/103] Final data-printing set-up --- src/tlo/methods/demography.py | 7 ++- src/tlo/methods/healthsystem.py | 18 ++++++ src/tlo/methods/hiv.py | 67 ++++++++++++++++++---- src/tlo/methods/tb.py | 99 +++++++++++++++++++++++++-------- src/tlo/simulation.py | 82 ++++++++++++++++++++++++--- 5 files changed, 226 insertions(+), 47 deletions(-) diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index e58f3895f4..6b2578fd44 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -315,9 +315,10 @@ def initialise_simulation(self, sim): # Launch the repeating event that will store statistics about the population structure sim.schedule_event(DemographyLoggingEvent(self), sim.date) - # Create (and store pointer to) the OtherDeathPoll and schedule first occurrence immediately - self.other_death_poll = OtherDeathPoll(self) - sim.schedule_event(self.other_death_poll, sim.date) + if sim.generate_data is False: + # Create (and store pointer to) the OtherDeathPoll and schedule first occurrence immediately + self.other_death_poll = OtherDeathPoll(self) + sim.schedule_event(self.other_death_poll, sim.date) # Log the initial population scaling-factor (to the logger of this module and that of `tlo.methods.population`) for _logger in (logger, logger_scale_factor): diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 181c08f5aa..6e251e636c 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2033,8 +2033,26 @@ def run_individual_level_events_in_mode_0_or_1(self, assert event.facility_info is not None, \ f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." + go_ahead = False + if (event.module == self.sim.modules['Tb'] or event.module == self.sim.modules['Hiv']): + go_ahead = True + row = self.sim.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = self.sim.date + row['when'] = 'Before' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + # Run the HSI event (allowing it to return an updated appt_footprint) actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + + if go_ahead: + row = self.sim.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = self.sim.date + row['when'] = 'After' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) # Check if the HSI event returned updated appt_footprint if actual_appt_footprint is not None: diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index d6455cc861..8e0d337fc1 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -631,11 +631,12 @@ def initialise_population(self, population): df.loc[df.is_alive, "hv_date_treated"] = pd.NaT df.loc[df.is_alive, "hv_date_last_ART"] = pd.NaT - # Launch sub-routines for allocating the right number of people into each category - self.initialise_baseline_prevalence(population) # allocate baseline prevalence + if self.sim.generate_data is False: + # Launch sub-routines for allocating the right number of people into each category + self.initialise_baseline_prevalence(population) # allocate baseline prevalence - self.initialise_baseline_art(population) # allocate baseline art coverage - self.initialise_baseline_tested(population) # allocate baseline testing coverage + self.initialise_baseline_art(population) # allocate baseline art coverage + self.initialise_baseline_tested(population) # allocate baseline testing coverage def initialise_baseline_prevalence(self, population): """ @@ -905,10 +906,16 @@ def initialise_simulation(self, sim): df = sim.population.props p = self.parameters - # 1) Schedule the Main HIV Regular Polling Event - sim.schedule_event( - HivRegularPollingEvent(self), sim.date + DateOffset(days=0) - ) + if self.sim.generate_data: + print("Should be generating data") + sim.schedule_event( + HivPollingEventForDataGeneration(self), sim.date + DateOffset(days=0) + ) + else: + # 1) Schedule the Main HIV Regular Polling Event + sim.schedule_event( + HivRegularPollingEvent(self), sim.date + DateOffset(days=0) + ) # 2) Schedule the Logging Event sim.schedule_event(HivLoggingEvent(self), sim.date + DateOffset(years=1)) @@ -1662,6 +1669,37 @@ def do_at_generic_first_appt( # Main Polling Event # --------------------------------------------------------------------------- +class HivPollingEventForDataGeneration(RegularEvent, PopulationScopeEventMixin): + """ The HIV Polling Events for Data Generation + * Ensures that + """ + + def __init__(self, module): + super().__init__( + module, frequency=DateOffset(years=120) + ) # repeats every 12 months, but this can be changed + + def apply(self, population): + + df = population.props + + # Make everyone who is alive and not infected (no-one should be) susceptible + susc_idx = df.loc[ + df.is_alive + & ~df.hv_inf + ].index + + n_susceptible = len(susc_idx) + print("Number of individuals susceptible", n_susceptible) + # Schedule the date of infection for each new infection: + for i in susc_idx: + date_of_infection = self.sim.date + pd.DateOffset( + # Ensure that individual will be infected before end of sim + days=self.module.rng.randint(0, 365*(int(self.sim.end_date.year - self.sim.date.year)+1)) + ) + self.sim.schedule_event( + HivInfectionEvent(self.module, i), date_of_infection + ) class HivRegularPollingEvent(RegularEvent, PopulationScopeEventMixin): """ The HIV Regular Polling Events @@ -1683,6 +1721,7 @@ def apply(self, population): fraction_of_year_between_polls = self.frequency.months / 12 beta = p["beta"] * fraction_of_year_between_polls + # ----------------------------------- HORIZONTAL TRANSMISSION ----------------------------------- def horizontal_transmission(to_sex, from_sex): # Count current number of alive 15-80 year-olds at risk of transmission @@ -1758,6 +1797,7 @@ def horizontal_transmission(to_sex, from_sex): HivInfectionEvent(self.module, idx), date_of_infection ) + # ----------------------------------- SPONTANEOUS TESTING ----------------------------------- def spontaneous_testing(current_year): @@ -1861,11 +1901,12 @@ def vmmc_for_child(): priority=0, ) - # Horizontal transmission: Male --> Female - horizontal_transmission(from_sex="M", to_sex="F") + if self.sim.generate_data is False: + # Horizontal transmission: Male --> Female + horizontal_transmission(from_sex="M", to_sex="F") - # Horizontal transmission: Female --> Male - horizontal_transmission(from_sex="F", to_sex="M") + # Horizontal transmission: Female --> Male + horizontal_transmission(from_sex="F", to_sex="M") # testing # if year later than 2020, set testing rates to those reported in 2020 @@ -1882,6 +1923,8 @@ def vmmc_for_child(): vmmc_for_child() + + # --------------------------------------------------------------------------- # Natural History Events # --------------------------------------------------------------------------- diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 623ee2e483..cd79ae22a5 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -833,28 +833,29 @@ def initialise_population(self, population): df["tb_date_ipt"] = pd.NaT # # ------------------ infection status ------------------ # - # WHO estimates of active TB for 2010 to get infected initial population - # don't need to scale or include treated proportion as no-one on treatment yet - inc_estimates = p["who_incidence_estimates"] - incidence_year = (inc_estimates.loc[ - (inc_estimates.year == self.sim.date.year), "incidence_per_100k" - ].values[0]) / 100_000 - - incidence_year = incidence_year * p["scaling_factor_WHO"] - - self.assign_active_tb( - population, - strain="ds", - incidence=incidence_year) - - self.assign_active_tb( - population, - strain="mdr", - incidence=incidence_year * p['prop_mdr2010']) - - self.send_for_screening_general( - population - ) # send some baseline population for screening + if self.sim.generate_data is False: + # WHO estimates of active TB for 2010 to get infected initial population + # don't need to scale or include treated proportion as no-one on treatment yet + inc_estimates = p["who_incidence_estimates"] + incidence_year = (inc_estimates.loc[ + (inc_estimates.year == self.sim.date.year), "incidence_per_100k" + ].values[0]) / 100_000 + + incidence_year = incidence_year * p["scaling_factor_WHO"] + + self.assign_active_tb( + population, + strain="ds", + incidence=incidence_year) + + self.assign_active_tb( + population, + strain="mdr", + incidence=incidence_year * p['prop_mdr2010']) + + self.send_for_screening_general( + population + ) # send some baseline population for screening def initialise_simulation(self, sim): """ @@ -867,7 +868,11 @@ def initialise_simulation(self, sim): sim.schedule_event(TbActiveEvent(self), sim.date) sim.schedule_event(TbRegularEvents(self), sim.date) sim.schedule_event(TbSelfCureEvent(self), sim.date) - sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) + + if sim.generate_data is False: + sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) + else: + sim.schedule_event(TbActiveCasePollGenerateData(self), sim.date + DateOffset(days=0)) # 2) log at the end of the year # Optional: Schedule the scale-up of programs @@ -1366,6 +1371,53 @@ def is_subset(col_for_set, col_for_subset): # # TB infection event # # --------------------------------------------------------------------------- +class TbActiveCasePollGenerateData(RegularEvent, PopulationScopeEventMixin): + """The Tb Regular Poll Event for Data Generation for assigning active infections + * selects everyone to develop an active infection and schedules onset of active tb + sometime during the simulation + """ + + def __init__(self, module): + super().__init__(module, frequency=DateOffset(years=120)) + + def apply(self, population): + + df = population.props + now = self.sim.date + rng = self.module.rng + # Make everyone who is alive and not infected (no-one should be) susceptible + susc_idx = df.loc[ + df.is_alive + & (df.tb_inf != "active") + ].index + + n_susceptible = len(susc_idx) + + middle_index = len(susc_idx) // 2 + + # Will equally split two strains among the population + list_ds = susc_idx[:middle_index] + list_mdr = susc_idx[middle_index:] + + # schedule onset of active tb. This will be equivalent to the "Onset", so it + # doesn't matter how long after we have decided which infection this is. + for person_id in list_ds: + date_progression = now + pd.DateOffset( + # At some point during their lifetime, this person will develop TB + days=self.module.rng.randint(0, 365*(int(self.sim.end_date.year - self.sim.date.year)+1)) + ) + # set date of active tb - properties will be updated at TbActiveEvent poll daily + df.at[person_id, "tb_scheduled_date_active"] = date_progression + df.at[person_id, "tb_strain"] = "ds" + + for person_id in list_mdr: + date_progression = now + pd.DateOffset( + days=rng.randint(0, 365*int(self.sim.end_date.year - self.sim.start_date.year + 1)) + ) + # set date of active tb - properties will be updated at TbActiveEvent poll daily + df.at[person_id, "tb_scheduled_date_active"] = date_progression + df.at[person_id, "tb_strain"] = "mdr" + class TbActiveCasePoll(RegularEvent, PopulationScopeEventMixin): """The Tb Regular Poll Event for assigning active infections @@ -1439,7 +1491,6 @@ def apply(self, population): self.module.update_parameters_for_program_scaleup() - class TbActiveEvent(RegularEvent, PopulationScopeEventMixin): """ * check for those with dates of active tb onset within last time-period diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 5b4e2fff4c..f0c8d6f09f 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -7,7 +7,7 @@ from collections import OrderedDict from pathlib import Path from typing import Dict, Optional, Union - +import pandas as pd import numpy as np from tlo import Date, Population, logging @@ -63,9 +63,11 @@ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = Non self.date = self.start_date = start_date self.modules = OrderedDict() self.event_queue = EventQueue() + self.generate_data = None self.end_date = None self.output_file = None self.population: Optional[Population] = None + self.event_chains: Optinoal[Population] = None self.show_progress_bar = show_progress_bar self.resourcefilepath = resourcefilepath @@ -209,6 +211,8 @@ def make_initial_population(self, *, n): module.initialise_population(self.population) logger.debug(key='debug', data=f'{module.name}.initialise_population() {time.time() - start1} s') + self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) + end = time.time() logger.info(key='info', data=f'make_initial_population() {end - start} s') @@ -221,7 +225,14 @@ def simulate(self, *, end_date): """ start = time.time() self.end_date = end_date # store the end_date so that others can reference it + self.generate_data = True # for now ensure we're always aiming to print data + + f = open('output.txt', mode='a') + #df_event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) + # Reorder columns to place the new columns at the front + pd.set_option('display.max_columns', None) + print(self.event_chains.columns) for module in self.modules.values(): module.initialise_simulation(self) @@ -250,17 +261,72 @@ def simulate(self, *, end_date): if date >= end_date: self.date = end_date + self.event_chains.to_csv('output.csv', index=False) break - + #if event.target != self.population: # print("Event: ", event) - - if event.module == self.modules['RTI']: - print("RTI event ", event) - print(" target ", event.target) - if event.target != self.population: - self.population.props.at[event.tar] + go_ahead = False + df_before = [] + + # Only print events relevant to modules of interest + # Do not want to compare before/after in births because it may expand the pop dataframe + print_output = True + if print_output: + if (event.module == self.modules['Tb'] or event.module == self.modules['Hiv']) and 'TbActiveCasePollGenerateData' not in str(event) and 'HivPollingEventForDataGeneration' not in str(event) and "SimplifiedBirthsPoll" not in str(event) and "AgeUpdateEvent" not in str(event) and "HealthSystemScheduler" not in str(event): + #if 'TbActiveCasePollGenerateData' not in str(event) and 'HivPollingEventForDataGeneration' not in str(event) and "SimplifiedBirthsPoll" not in str(event) and "AgeUpdateEvent" not in str(event): + go_ahead = True + if event.target != self.population: + row = self.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = date + row['when'] = 'Before' + self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) + else: + df_before = self.population.props.copy() + self.fire_single_event(event, date) + + if print_output: + if go_ahead == True: + if event.target != self.population: + row = self.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = date + row['when'] = 'After' + self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) + else: + df_after = self.population.props.copy() + # if not df_before.columns.equals(df_after.columns): + # print("Number of columns in pop dataframe", len(self.population.props.columns)) + # print("Before", df_before.columns) + # print("After", df_after.columns#) + # exit(-1) + # if not df_before.index.equals(df_after.index): + # print("Number of indices in pop dataframe", len(self.population.props.index)) + # print("----> ", event) + # print("Before", df_before.index#) + # print("After", df_after.index) + # exit(-1) + + change = df_before.compare(df_after) + if ~change.empty: + indices = change.index + new_rows_before = df_before.loc[indices] + new_rows_before['person_ID'] = new_rows_before.index + new_rows_before['event'] = event + new_rows_before['event_date'] = date + new_rows_before['when'] = 'Before' + new_rows_after = df_after.loc[indices] + new_rows_after['person_ID'] = new_rows_after.index + new_rows_after['event'] = event + new_rows_after['event_date'] = date + new_rows_after['when'] = 'After' + + self.event_chains = pd.concat([self.event_chains,new_rows_before], ignore_index=True) + self.event_chains = pd.concat([self.event_chains,new_rows_after], ignore_index=True) # The simulation has ended. if self.show_progress_bar: From 16c071c6220edcc20b539f346625f628e5e8c4c5 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 2 Oct 2024 12:37:38 +0200 Subject: [PATCH 009/103] Print event chains --- src/tlo/methods/demography.py | 2 +- src/tlo/methods/healthsystem.py | 8 ++-- src/tlo/methods/hiv.py | 6 +-- src/tlo/methods/tb.py | 4 +- src/tlo/simulation.py | 47 +++++++++--------- tests/test_data_generation.py | 85 +++++++++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 35 deletions(-) create mode 100644 tests/test_data_generation.py diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index 6b2578fd44..4f19af6d55 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -315,7 +315,7 @@ def initialise_simulation(self, sim): # Launch the repeating event that will store statistics about the population structure sim.schedule_event(DemographyLoggingEvent(self), sim.date) - if sim.generate_data is False: + if sim.generate_event_chains is False: # Create (and store pointer to) the OtherDeathPoll and schedule first occurrence immediately self.other_death_poll = OtherDeathPoll(self) sim.schedule_event(self.other_death_poll, sim.date) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 6e251e636c..203ca10985 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2033,9 +2033,9 @@ def run_individual_level_events_in_mode_0_or_1(self, assert event.facility_info is not None, \ f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." - go_ahead = False - if (event.module == self.sim.modules['Tb'] or event.module == self.sim.modules['Hiv']): - go_ahead = True + print_chains = False + if event.module in self.sim.generate_event_chains_modules_of_interest and all(sub not in str(event) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True row = self.sim.population.props.iloc[[event.target]] row['person_ID'] = event.target row['event'] = event @@ -2046,7 +2046,7 @@ def run_individual_level_events_in_mode_0_or_1(self, # Run the HSI event (allowing it to return an updated appt_footprint) actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) - if go_ahead: + if print_chains: row = self.sim.population.props.iloc[[event.target]] row['person_ID'] = event.target row['event'] = event diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 8e0d337fc1..36b1a4bd6e 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -631,7 +631,7 @@ def initialise_population(self, population): df.loc[df.is_alive, "hv_date_treated"] = pd.NaT df.loc[df.is_alive, "hv_date_last_ART"] = pd.NaT - if self.sim.generate_data is False: + if self.sim.generate_event_chains is False: # Launch sub-routines for allocating the right number of people into each category self.initialise_baseline_prevalence(population) # allocate baseline prevalence @@ -906,7 +906,7 @@ def initialise_simulation(self, sim): df = sim.population.props p = self.parameters - if self.sim.generate_data: + if self.sim.generate_event_chains: print("Should be generating data") sim.schedule_event( HivPollingEventForDataGeneration(self), sim.date + DateOffset(days=0) @@ -1901,7 +1901,7 @@ def vmmc_for_child(): priority=0, ) - if self.sim.generate_data is False: + if self.sim.generate_event_chains is False: # Horizontal transmission: Male --> Female horizontal_transmission(from_sex="M", to_sex="F") diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index cd79ae22a5..57ccd97368 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -833,7 +833,7 @@ def initialise_population(self, population): df["tb_date_ipt"] = pd.NaT # # ------------------ infection status ------------------ # - if self.sim.generate_data is False: + if self.sim.generate_event_chains is False: # WHO estimates of active TB for 2010 to get infected initial population # don't need to scale or include treated proportion as no-one on treatment yet inc_estimates = p["who_incidence_estimates"] @@ -869,7 +869,7 @@ def initialise_simulation(self, sim): sim.schedule_event(TbRegularEvents(self), sim.date) sim.schedule_event(TbSelfCureEvent(self), sim.date) - if sim.generate_data is False: + if sim.generate_event_chains is False: sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) else: sim.schedule_event(TbActiveCasePollGenerateData(self), sim.date + DateOffset(days=0)) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index f0c8d6f09f..d055d6e367 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -63,7 +63,9 @@ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = Non self.date = self.start_date = start_date self.modules = OrderedDict() self.event_queue = EventQueue() - self.generate_data = None + self.generate_event_chains = None + self.generate_event_chains_modules_of_interest = [] + self.generate_event_chains_ignore_events = [] self.end_date = None self.output_file = None self.population: Optional[Population] = None @@ -216,7 +218,7 @@ def make_initial_population(self, *, n): end = time.time() logger.info(key='info', data=f'make_initial_population() {end - start} s') - def simulate(self, *, end_date): + def simulate(self, *, end_date, generate_event_chains = False): """Simulation until the given end date :param end_date: when to stop simulating. Only events strictly before this @@ -225,7 +227,11 @@ def simulate(self, *, end_date): """ start = time.time() self.end_date = end_date # store the end_date so that others can reference it - self.generate_data = True # for now ensure we're always aiming to print data + self.generate_event_chains = generate_event_chains # for now ensure we're always aiming to print data + if self.generate_event_chains: + # For now keep these fixed, eventually they will be input from user + self.generate_event_chains_modules_of_interest = [self.modules['Tb'], self.modules['Hiv'], self.modules['CardioMetabolicDisorders']] + self.generate_event_chains_ignore_events = ['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] f = open('output.txt', mode='a') #df_event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) @@ -264,17 +270,13 @@ def simulate(self, *, end_date): self.event_chains.to_csv('output.csv', index=False) break - #if event.target != self.population: - # print("Event: ", event) - go_ahead = False + + print_chains = False df_before = [] - # Only print events relevant to modules of interest - # Do not want to compare before/after in births because it may expand the pop dataframe - print_output = True - if print_output: - if (event.module == self.modules['Tb'] or event.module == self.modules['Hiv']) and 'TbActiveCasePollGenerateData' not in str(event) and 'HivPollingEventForDataGeneration' not in str(event) and "SimplifiedBirthsPoll" not in str(event) and "AgeUpdateEvent" not in str(event) and "HealthSystemScheduler" not in str(event): - #if 'TbActiveCasePollGenerateData' not in str(event) and 'HivPollingEventForDataGeneration' not in str(event) and "SimplifiedBirthsPoll" not in str(event) and "AgeUpdateEvent" not in str(event): + if self.generate_event_chains: + # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore + if (event.module in self.generate_event_chains_modules_of_interest) and all(sub not in str(event) for sub in self.generate_event_chains_ignore_events): go_ahead = True if event.target != self.population: row = self.population.props.iloc[[event.target]] @@ -288,7 +290,7 @@ def simulate(self, *, end_date): self.fire_single_event(event, date) - if print_output: + if go_ahead: if go_ahead == True: if event.target != self.population: row = self.population.props.iloc[[event.target]] @@ -299,18 +301,6 @@ def simulate(self, *, end_date): self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) else: df_after = self.population.props.copy() - # if not df_before.columns.equals(df_after.columns): - # print("Number of columns in pop dataframe", len(self.population.props.columns)) - # print("Before", df_before.columns) - # print("After", df_after.columns#) - # exit(-1) - # if not df_before.index.equals(df_after.index): - # print("Number of indices in pop dataframe", len(self.population.props.index)) - # print("----> ", event) - # print("Before", df_before.index#) - # print("After", df_after.index) - # exit(-1) - change = df_before.compare(df_after) if ~change.empty: indices = change.index @@ -385,6 +375,13 @@ def do_birth(self, mother_id): child_id = self.population.do_birth() for module in self.modules.values(): module.on_birth(mother_id, child_id) + if self.generate_event_chains: + row = self.population.props.iloc[[child_id]] + row['person_ID'] = child_id + row['event'] = 'Birth' + row['event_date'] = self.date + row['when'] = 'After' + self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) return child_id def find_events_for_person(self, person_id: int): diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py new file mode 100644 index 0000000000..1f6333bbfe --- /dev/null +++ b/tests/test_data_generation.py @@ -0,0 +1,85 @@ +import os +from pathlib import Path + +import pandas as pd +import pytest + +from tlo import Date, Simulation +from tlo.methods import ( + care_of_women_during_pregnancy, + demography, + depression, + enhanced_lifestyle, + epi, + epilepsy, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + cardio_metabolic_disorders, + labour, + newborn_outcomes, + postnatal_supervisor, + pregnancy_helper_functions, + pregnancy_supervisor, + depression, + tb, + contraception, +# simplified_births, + symptommanager, +) +from tlo.methods.hsi_generic_first_appts import HSI_GenericEmergencyFirstAppt + +# create simulation parameters +start_date = Date(2010, 1, 1) +end_date = Date(2015, 1, 1) +popsize = 100 + +@pytest.mark.slow +def test_data_harvesting(seed): + """ + This test runs a simulation to print all individual events of specific individuals + """ + + module_of_interest = 'Hiv' + # create sim object + sim = create_basic_sim(popsize, seed) + + dependencies_list = sim.modules[module_of_interest].ADDITIONAL_DEPENDENCIES.union(sim.modules[module_of_interest].INIT_DEPENDENCIES) + + # Check that all dependencies are included + for dep in dependencies_list: + if dep not in sim.modules: + print("WARNING: dependency ", dep, "not included") + exit(-1) + + # run simulation + sim.simulate(end_date=end_date, generate_event_chains = True) + + +def create_basic_sim(population_size, seed): + # create the basic outline of an rti simulation object + sim = Simulation(start_date=start_date, seed=seed) + resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' + sim.register(demography.Demography(resourcefilepath=resourcefilepath), + contraception.Contraception(resourcefilepath=resourcefilepath), + enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), + healthburden.HealthBurden(resourcefilepath=resourcefilepath), + symptommanager.SymptomManager(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath, service_availability=['*']), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), + epi.Epi(resourcefilepath=resourcefilepath), + hiv.Hiv(resourcefilepath=resourcefilepath), + tb.Tb(resourcefilepath=resourcefilepath), + cardio_metabolic_disorders.CardioMetabolicDisorders(resourcefilepath=resourcefilepath), + depression.Depression(resourcefilepath=resourcefilepath), + newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), + pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), + care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), + labour.Labour(resourcefilepath=resourcefilepath), + postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), + ) + + sim.make_initial_population(n=population_size) + return sim + From ba81487a3fa003e2f10206e435a1d64f170f14e3 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:08:50 +0200 Subject: [PATCH 010/103] Add chains in mode 2 too and clean up in simuation --- src/tlo/methods/healthsystem.py | 40 ++++++++++++++++++------ src/tlo/simulation.py | 55 ++++++++++++++++----------------- 2 files changed, 58 insertions(+), 37 deletions(-) diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 203ca10985..54cb976b26 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2034,18 +2034,20 @@ def run_individual_level_events_in_mode_0_or_1(self, f"Cannot run HSI {event.TREATMENT_ID} without facility_info being defined." print_chains = False - if event.module in self.sim.generate_event_chains_modules_of_interest and all(sub not in str(event) for sub in self.sim.generate_event_chains_ignore_events): - print_chains = True - row = self.sim.population.props.iloc[[event.target]] - row['person_ID'] = event.target - row['event'] = event - row['event_date'] = self.sim.date - row['when'] = 'Before' - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + if self.sim.generate_event_chains: + if event.module in self.sim.generate_event_chains_modules_of_interest and all(sub not in str(event) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True + row = self.sim.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = self.sim.date + row['when'] = 'Before' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) # Run the HSI event (allowing it to return an updated appt_footprint) actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + # Print individual info after event if print_chains: row = self.sim.population.props.iloc[[event.target]] row['person_ID'] = event.target @@ -2445,8 +2447,28 @@ def process_events_mode_2(self, hold_over: List[HSIEventQueueItem]) -> None: # Expected appt footprint before running event _appt_footprint_before_running = event.EXPECTED_APPT_FOOTPRINT - # Run event & get actual footprint + + print_chains = False + if self.sim.generate_event_chains: + if event.module in self.sim.generate_event_chains_modules_of_interest and all(sub not in str(event) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True + row = self.sim.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = self.sim.date + row['when'] = 'Before' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + + # Run the HSI event (allowing it to return an updated appt_footprint) actual_appt_footprint = event.run(squeeze_factor=squeeze_factor) + + if print_chains: + row = self.sim.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = self.sim.date + row['when'] = 'After' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) # Check if the HSI event returned updated_appt_footprint, and if so adjust original_call if actual_appt_footprint is not None: diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index d055d6e367..616e159453 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -277,7 +277,7 @@ def simulate(self, *, end_date, generate_event_chains = False): if self.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore if (event.module in self.generate_event_chains_modules_of_interest) and all(sub not in str(event) for sub in self.generate_event_chains_ignore_events): - go_ahead = True + print_chains = True if event.target != self.population: row = self.population.props.iloc[[event.target]] row['person_ID'] = event.target @@ -290,33 +290,32 @@ def simulate(self, *, end_date, generate_event_chains = False): self.fire_single_event(event, date) - if go_ahead: - if go_ahead == True: - if event.target != self.population: - row = self.population.props.iloc[[event.target]] - row['person_ID'] = event.target - row['event'] = event - row['event_date'] = date - row['when'] = 'After' - self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) - else: - df_after = self.population.props.copy() - change = df_before.compare(df_after) - if ~change.empty: - indices = change.index - new_rows_before = df_before.loc[indices] - new_rows_before['person_ID'] = new_rows_before.index - new_rows_before['event'] = event - new_rows_before['event_date'] = date - new_rows_before['when'] = 'Before' - new_rows_after = df_after.loc[indices] - new_rows_after['person_ID'] = new_rows_after.index - new_rows_after['event'] = event - new_rows_after['event_date'] = date - new_rows_after['when'] = 'After' - - self.event_chains = pd.concat([self.event_chains,new_rows_before], ignore_index=True) - self.event_chains = pd.concat([self.event_chains,new_rows_after], ignore_index=True) + if print_chains: + if event.target != self.population: + row = self.population.props.iloc[[event.target]] + row['person_ID'] = event.target + row['event'] = event + row['event_date'] = date + row['when'] = 'After' + self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) + else: + df_after = self.population.props.copy() + change = df_before.compare(df_after) + if ~change.empty: + indices = change.index + new_rows_before = df_before.loc[indices] + new_rows_before['person_ID'] = new_rows_before.index + new_rows_before['event'] = event + new_rows_before['event_date'] = date + new_rows_before['when'] = 'Before' + new_rows_after = df_after.loc[indices] + new_rows_after['person_ID'] = new_rows_after.index + new_rows_after['event'] = event + new_rows_after['event_date'] = date + new_rows_after['when'] = 'After' + + self.event_chains = pd.concat([self.event_chains,new_rows_before], ignore_index=True) + self.event_chains = pd.concat([self.event_chains,new_rows_after], ignore_index=True) # The simulation has ended. if self.show_progress_bar: From 06246c479110502a3777f027c1d498edf49e3c60 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 2 Oct 2024 16:06:57 +0100 Subject: [PATCH 011/103] update to cohort --- src/tlo/methods/mnh_cohort_module.py | 41 ++++++++----------------- tests/test_mnh_cohort.py | 45 ++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 61dddc3c0c..826930d161 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -5,6 +5,7 @@ from tlo import DateOffset, Module, Parameter, Property, Types, logging from tlo.methods import Metadata from tlo.analysis.utils import parse_log_file +from tlo.events import Event, IndividualScopeEventMixin logger = logging.getLogger(__name__) @@ -63,9 +64,6 @@ def initialise_population(self, population): :param population: the population of individuals """ - # TODO: CURRENT ISSUE - INDIVIDUALS IN THE POPULATION ARE SCHEDULED HSIs IN INITIALISE_POP/SIM BY OTHER MODULES, - # THEIR PROPERTIES ARE THEN OVER WRITTEN BY THIS MODULE AND ITS CRASHING HSIs - log_file = parse_log_file( '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/' 'fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', @@ -88,6 +86,7 @@ def initialise_population(self, population): df.loc[population.index, 'date_of_last_pregnancy'] = self.sim.start_date df.loc[population.index, 'co_contraception'] = "not_using" + def initialise_simulation(self, sim): """Get ready for simulation start. @@ -96,31 +95,17 @@ def initialise_simulation(self, sim): It is a good place to add initial events to the event queue. """ - pass - # # cohort_prop_df = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancyCohort.xlsx') - # from tlo.analysis.utils import parse_log_file, load_pickled_dataframes, get_scenario_outputs - # - # log_file = parse_log_file( - # '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/' - # 'fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', - # level=logging.DEBUG)['tlo.methods.contraception'] - # - # all_pregnancies = log_file['properties_of_pregnant_person'].loc[ - # log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) - # all_pregnancies.index = [x for x in range(len(all_pregnancies))] - # - # preg_pop = all_pregnancies.loc[0:(len(self.sim.population.props))-1] - # - # props_dtypes = self.sim.population.props.dtypes - # preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) - # preg_pop_final.index.name = 'person' - # - # self.sim.population.props = preg_pop_final - # - # df = self.sim.population.props - # population = df.loc[df.is_alive] - # df.loc[population.index, 'date_of_last_pregnancy'] = sim.start_date - # df.loc[population.index, 'co_contraception'] = "not_using" + df = self.sim.population.props + + sim.modules['HealthSystem'].HSI_EVENT_QUEUE.clear() + + for item in self.sim.event_queue.queue: + if isinstance(item[3], IndividualScopeEventMixin): + self.sim.event_queue.queue.remove(item) + + for person in df.index: + self.sim.modules['Labour'].set_date_of_labour(person) + def on_birth(self, mother_id, child_id): """Initialise our properties for a newborn individual. diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index bc2f14d1bb..8cee5e76d0 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -4,7 +4,7 @@ from tlo import Date, Simulation, logging from tlo.methods import mnh_cohort_module from tlo.methods.fullmodel import fullmodel - +from tlo.analysis.utils import parse_log_file # The resource files try: @@ -16,30 +16,49 @@ start_date = Date(2010, 1, 1) -def check_dtypes(simulation): - # check types of columns - df = simulation.population.props - orig = simulation.population.new_row - assert (df.dtypes == orig.dtypes).all() - - def register_modules(sim): """Defines sim variable and registers all modules that can be called when running the full suite of pregnancy modules""" - sim.register(mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), - *fullmodel(resourcefilepath=resourcefilepath), - + sim.register(*fullmodel(resourcefilepath=resourcefilepath), + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), ) + def test_run_sim_with_mnh_cohort(tmpdir, seed): sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) sim.make_initial_population(n=1000) - sim.simulate(end_date=Date(2011, 1, 1)) - check_dtypes(sim) + sim.simulate(end_date=Date(2024, 1, 1)) + + output= parse_log_file(sim.log_filepath) + live_births = len(output['tlo.methods.demography']['on_birth']) + + deaths_df = output['tlo.methods.demography']['death'] + prop_deaths_df = ['tlo.methods.demography.detail']['properties_of_deceased_persons'] + + dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] + init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' + 'chronic_ischemic_hd|ever_heart_attack|' + 'chronic_kidney_disease') | + (prop_deaths_df['cause_of_death'] == 'TB'))] + + hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] + + indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) + + # TOTAL_DEATHS + mmr = (len(dir_mat_deaths) + indir_mat_deaths) / live_births * 100_000 + + + + # df = sim.population.props + # orig = sim.population.new_row + # assert (df.dtypes == orig.dtypes).all() # def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): # sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) From 86d130a1af311a833274965678a05bdcc1fe329e Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 2 Oct 2024 16:21:38 +0100 Subject: [PATCH 012/103] update to cohort --- tests/test_mnh_cohort.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 8cee5e76d0..4502393967 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -13,7 +13,7 @@ # running interactively resourcefilepath = Path('./resources') -start_date = Date(2010, 1, 1) +start_date = Date(2024, 1, 1) def register_modules(sim): @@ -31,7 +31,7 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): register_modules(sim) sim.make_initial_population(n=1000) - sim.simulate(end_date=Date(2024, 1, 1)) + sim.simulate(end_date=Date(2025, 1, 1)) output= parse_log_file(sim.log_filepath) live_births = len(output['tlo.methods.demography']['on_birth']) From 3a90ab79c223203b63405afecfdc87adece59ec7 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 4 Oct 2024 09:22:24 +0100 Subject: [PATCH 013/103] small updates to prevent scheduling of events in the past when sim starts past 2010 --- src/tlo/methods/epi.py | 6 +++++- src/tlo/methods/labour.py | 5 +++-- src/tlo/methods/mnh_cohort_module.py | 7 ++++--- src/tlo/methods/pregnancy_supervisor.py | 12 +++++++++--- src/tlo/methods/schisto.py | 3 ++- tests/test_mnh_cohort.py | 15 ++++++++++++--- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/tlo/methods/epi.py b/src/tlo/methods/epi.py index 1cdf8a1612..f39bb44c63 100644 --- a/src/tlo/methods/epi.py +++ b/src/tlo/methods/epi.py @@ -181,8 +181,12 @@ def initialise_simulation(self, sim): # add an event to log to screen sim.schedule_event(EpiLoggingEvent(self), sim.date + DateOffset(years=1)) + # TODO: check with Tara shes happy with this (could come in as its own PR) # HPV vaccine given from 2018 onwards - sim.schedule_event(HpvScheduleEvent(self), Date(2018, 1, 1)) + if self.sim.date.year < 2018: + sim.schedule_event(HpvScheduleEvent(self), Date(2018, 1, 1)) + else: + sim.schedule_event(HpvScheduleEvent(self), Date(self.sim.date.year, 1, 1)) # Look up item codes for consumables self.get_item_codes() diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 35081b7d27..92725cbbe4 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -878,8 +878,9 @@ def initialise_simulation(self, sim): sim.schedule_event(LabourLoggingEvent(self), sim.date + DateOffset(days=1)) # Schedule analysis event - sim.schedule_event(LabourAndPostnatalCareAnalysisEvent(self), - Date(self.current_parameters['analysis_year'], 1, 1)) + if self.sim.date.year <= self.current_parameters['analysis_year']: + sim.schedule_event(LabourAndPostnatalCareAnalysisEvent(self), + Date(self.current_parameters['analysis_year'], 1, 1)) # This list contains all the women who are currently in labour and is used for checks/testing self.women_in_labour = [] diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 826930d161..8430d3aba7 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -99,9 +99,10 @@ def initialise_simulation(self, sim): sim.modules['HealthSystem'].HSI_EVENT_QUEUE.clear() - for item in self.sim.event_queue.queue: - if isinstance(item[3], IndividualScopeEventMixin): - self.sim.event_queue.queue.remove(item) + updated_event_queue = [item for item in self.sim.event_queue.queue + if not isinstance(item[3], IndividualScopeEventMixin)] + + self.sim.event_queue.queue = updated_event_queue for person in df.index: self.sim.modules['Labour'].set_date_of_labour(person) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7dd8819ab6..97e98858df 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -523,13 +523,19 @@ def initialise_simulation(self, sim): sim.date + DateOffset(years=1)) # ...and register and schedule the parameter update event - sim.schedule_event(ParameterUpdateEvent(self), - Date(2015, 1, 1)) + if self.sim.date.year < 2015: + sim.schedule_event(ParameterUpdateEvent(self), + Date(2015, 1, 1)) + else: + sim.schedule_event(ParameterUpdateEvent(self), + Date(self.sim.date.year, 1, 1)) # ... and finally register and schedule the parameter override event. This is used in analysis scripts to change # key parameters after the simulation 'burn in' period. The event is schedule to run even when analysis is not # conducted but no changes to parameters can be made. - sim.schedule_event(PregnancyAnalysisEvent(self), Date(params['analysis_year'], 1, 1)) + + if self.sim.date.year <= params['analysis_year']: + sim.schedule_event(PregnancyAnalysisEvent(self), Date(params['analysis_year'], 1, 1)) # ==================================== LINEAR MODEL EQUATIONS ================================================= # Next we scale linear models according to distribution of predictors in the dataframe at baseline diff --git a/src/tlo/methods/schisto.py b/src/tlo/methods/schisto.py index 0e9735286a..f2a4253fad 100644 --- a/src/tlo/methods/schisto.py +++ b/src/tlo/methods/schisto.py @@ -170,7 +170,8 @@ def initialise_simulation(self, sim): sim.schedule_event(SchistoLoggingEvent(self), sim.date) # Schedule MDA events - if self.mda_execute: + # TODO: check with Tara shes happy with this (could come in as its own PR) + if self.mda_execute and self.sim.date.year == 2010: self._schedule_mda_events() def on_birth(self, mother_id, child_id): diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 4502393967..35b662b3fc 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -30,14 +30,14 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=1000) + sim.make_initial_population(n=5000) sim.simulate(end_date=Date(2025, 1, 1)) output= parse_log_file(sim.log_filepath) live_births = len(output['tlo.methods.demography']['on_birth']) deaths_df = output['tlo.methods.demography']['death'] - prop_deaths_df = ['tlo.methods.demography.detail']['properties_of_deceased_persons'] + prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & @@ -50,9 +50,18 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) + total_deaths = len(dir_mat_deaths) + indir_mat_deaths # TOTAL_DEATHS - mmr = (len(dir_mat_deaths) + indir_mat_deaths) / live_births * 100_000 + mmr = (total_deaths / live_births) * 100_000 + + print(f'The MMR for this simulation is {mmr}') + print(f'The maternal deaths for this simulation (unscaled) are {total_deaths}') + print(f'The total maternal deaths for this simulation (scaled) are ' + f'{total_deaths * output["tlo.methods.population"]["scaling_factor"]["scaling_factor"].values[0]}') + + maternal_dalys = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() + print(f'The maternal DALYs for this simulation (unscaled) are {maternal_dalys}') From 930973e4b9135159a70a0964bdb7c1556efa799a Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 4 Oct 2024 14:06:51 +0100 Subject: [PATCH 014/103] new calibration file. prevent new pregnancies during cohort run. --- .../local_run_cohort_calibration.py | 111 ++++++++++++++++++ src/tlo/methods/mnh_cohort_module.py | 8 +- 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py new file mode 100644 index 0000000000..c22549d3cf --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py @@ -0,0 +1,111 @@ +import os +from pathlib import Path + +import pandas as pd + +from tlo import Date, Simulation, logging +from tlo.methods import mnh_cohort_module +from tlo.methods.fullmodel import fullmodel +from tlo.analysis.utils import parse_log_file + + +resourcefilepath = Path('./resources') +outputpath = Path("./outputs/cohort_testing") # folder for convenience of storing outputs +population_size = 2000 + +sim = Simulation(start_date=Date(2024, 1, 1), + seed=123, + log_config={"filename": "log_cohort_calibration", + "custom_levels": {"*": logging.DEBUG}, + "directory": outputpath}) + +sim.register(*fullmodel(resourcefilepath=resourcefilepath), + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath)) + +sim.make_initial_population(n=population_size) +sim.simulate(end_date=Date(2025, 1, 1)) + +output = parse_log_file(sim.log_filepath) + +# output = parse_log_file( +# '/Users/j_collins/PycharmProjects/TLOmodel/outputs/log_cohort_calibration__2024-10-04T101535.log') + +# Make output dataframe +results = pd.DataFrame(columns=['model', 'data', 'source'], + index= ['deaths', + 'MMR', + 'DALYs', + 'twins', + 'ectopic', + 'abortion', + 'miscarriage', + 'syphilis', + 'anaemia_an', + 'anaemia_pn' + 'gdm', + 'PROM', + 'pre_eclampsia', + 'gest-htn', + 'severe_gest-htn', + 'severe pre-eclampsia', + 'eclampsia', + 'praevia', + 'abruption', + 'aph', + 'OL', + 'UR', + 'sepsis', + 'PPH']) + +# total_pregnancies = population_size +total_pregnancies = 2000 + len(output['tlo.methods.contraception']['pregnancy']) +total_births = len(output['tlo.methods.demography']['on_birth']) +prop_live_births = (total_births/total_pregnancies) * 100 + +# Mortality/DALY +deaths_df = output['tlo.methods.demography']['death'] +prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] + +dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] +init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' + 'chronic_ischemic_hd|ever_heart_attack|' + 'chronic_kidney_disease') | + (prop_deaths_df['cause_of_death'] == 'TB'))] + +hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] + +indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) +total_deaths = len(dir_mat_deaths) + indir_mat_deaths + +# TOTAL_DEATHS +results.at['deaths', 'model'] = total_deaths +results.at['MMR', 'model'] = (total_deaths / total_births) * 100_000 +results.at['DALYs', 'model'] = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() + +# Maternal conditions +an_comps = output['tlo.methods.pregnancy_supervisor']['maternal_complication'] +la_comps = output['tlo.methods.labour']['maternal_complication'] +pn_comps = output['tlo.methods.postnatal_supervisor']['maternal_complication'] + +total_completed_pregnancies = (len(an_comps.loc[an_comps['type'] == 'ectopic_unruptured']) + + len(an_comps.loc[an_comps['type'] == 'induced_abortion']) + + len(an_comps.loc[an_comps['type'] == 'spontaneous_abortion']) + + total_births + + len(output['tlo.methods.pregnancy_supervisor']['antenatal_stillbirth']) + + len(output['tlo.methods.labour']['intrapartum_stillbirth'])) + +# Twins (todo) + +# Ectopic +results.at['ectopic', 'model'] = (len(an_comps.loc[an_comps['type'] == 'ectopic_unruptured']) / total_pregnancies) * 1000 +results.at['ectopic', 'data'] = 10.0 +results.at['ectopic', 'source'] = 'Panelli et al.' + +# Abortion + + +# Miscarriage + +# Health system diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 8430d3aba7..4007d11a31 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -97,17 +97,23 @@ def initialise_simulation(self, sim): """ df = self.sim.population.props + # Clear HSI queue for events scheduled during initialisation sim.modules['HealthSystem'].HSI_EVENT_QUEUE.clear() + # Clear HSI queue for events scheduled during initialisation updated_event_queue = [item for item in self.sim.event_queue.queue if not isinstance(item[3], IndividualScopeEventMixin)] self.sim.event_queue.queue = updated_event_queue + # Prevent additional pregnancies from occurring + self.sim.modules['Contraception'].processed_params['p_pregnancy_with_contraception_per_month'].iloc[:] = 0 + self.sim.modules['Contraception'].processed_params['p_pregnancy_no_contraception_per_month'].iloc[:] = 0 + + # Set labour date for cohort women for person in df.index: self.sim.modules['Labour'].set_date_of_labour(person) - def on_birth(self, mother_id, child_id): """Initialise our properties for a newborn individual. From b1c907c12bfa54621983415b560381d1737afc9a Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 7 Oct 2024 09:36:06 +0200 Subject: [PATCH 015/103] Fix issue with tests by ensuring standard Polling and infection is maintained is generate_event_chains is None --- src/tlo/methods/hiv.py | 6 +++--- src/tlo/methods/hsi_event.py | 14 ++++++++------ src/tlo/methods/tb.py | 10 ++++++---- src/tlo/simulation.py | 4 +++- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 36b1a4bd6e..391cf587a8 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -631,7 +631,7 @@ def initialise_population(self, population): df.loc[df.is_alive, "hv_date_treated"] = pd.NaT df.loc[df.is_alive, "hv_date_last_ART"] = pd.NaT - if self.sim.generate_event_chains is False: + if self.sim.generate_event_chains is False or self.sim.generate_event_chains is None or self.sim.generate_event_chains_overwrite_epi is False: # Launch sub-routines for allocating the right number of people into each category self.initialise_baseline_prevalence(population) # allocate baseline prevalence @@ -906,7 +906,7 @@ def initialise_simulation(self, sim): df = sim.population.props p = self.parameters - if self.sim.generate_event_chains: + if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi: print("Should be generating data") sim.schedule_event( HivPollingEventForDataGeneration(self), sim.date + DateOffset(days=0) @@ -1901,7 +1901,7 @@ def vmmc_for_child(): priority=0, ) - if self.sim.generate_event_chains is False: + if self.sim.generate_event_chains is False or self.sim.generate_event_chains is None or self.sim.generate_event_chains_overwrite_epi is False: # Horizontal transmission: Male --> Female horizontal_transmission(from_sex="M", to_sex="F") diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 470794bcdd..785f27b7a6 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -193,10 +193,12 @@ def run(self, squeeze_factor): print_chains = False df_before = [] - + if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and + if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True if self.target != self.sim.population: row = self.sim.population.props.iloc[[self.target]] @@ -204,7 +206,7 @@ def run(self, squeeze_factor): row['event'] = self row['event_date'] = self.sim.date row['when'] = 'Before' - self.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: df_before = self.sim.population.props.copy() @@ -219,7 +221,7 @@ def run(self, squeeze_factor): row['event'] = self row['event_date'] = self.sim.date row['when'] = 'After' - self.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: df_after = self.sim.population.props.copy() change = df_before.compare(df_after) @@ -236,8 +238,8 @@ def run(self, squeeze_factor): new_rows_after['event_date'] = self.sim.date new_rows_after['when'] = 'After' - self.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) - self.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) + self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) + self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) return updated_appt_footprint def get_consumables( diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 57ccd97368..4c170944d2 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -832,8 +832,9 @@ def initialise_population(self, population): df["tb_on_ipt"] = False df["tb_date_ipt"] = pd.NaT + # # ------------------ infection status ------------------ # - if self.sim.generate_event_chains is False: + if self.sim.generate_event_chains is False or self.sim.generate_event_chains is None: # WHO estimates of active TB for 2010 to get infected initial population # don't need to scale or include treated proportion as no-one on treatment yet inc_estimates = p["who_incidence_estimates"] @@ -869,10 +870,11 @@ def initialise_simulation(self, sim): sim.schedule_event(TbRegularEvents(self), sim.date) sim.schedule_event(TbSelfCureEvent(self), sim.date) - if sim.generate_event_chains is False: - sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) - else: + if sim.generate_event_chains is True and sim.generate_event_chains_overwrite_epi is True: sim.schedule_event(TbActiveCasePollGenerateData(self), sim.date + DateOffset(days=0)) + else: + sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) + # 2) log at the end of the year # Optional: Schedule the scale-up of programs diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 794bfef98e..4aff23c9d7 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -105,6 +105,7 @@ def __init__( self.modules = OrderedDict() self.event_queue = EventQueue() self.generate_event_chains = None + self.generate_event_chains_overwrite_epi = None self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] self.end_date = None @@ -298,10 +299,11 @@ def initialise(self, *, end_date: Date, generate_event_chains) -> None: self.end_date = end_date # store the end_date so that others can reference it self.generate_event_chains = generate_event_chains # for now ensure we're always aiming to print data + self.generate_event_chains_overwrite_epi = False if self.generate_event_chains: # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules['Tb'], self.modules['Hiv'], self.modules['CardioMetabolicDisorders']] - self.generate_event_chains_ignore_events = ['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] #df_event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) From f1354c5cc6ca1ad0edb8f830954655f9989b6ad1 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 08:54:59 +0100 Subject: [PATCH 016/103] update scenario for azure test --- .../cohort_interventions_scenario.py | 13 ++++++------- .../cohort_analysis/local_run_cohort_calibration.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 15c722b687..30b5ebfc09 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -7,16 +7,16 @@ class BaselineScenario(BaseScenario): def __init__(self): super().__init__() - self.seed = 661184 - self.start_date = Date(2010, 1, 1) - self.end_date = Date(2031, 1, 1) - self.pop_size = 250_000 + self.seed = 537184 + self.start_date = Date(2024, 1, 1) + self.end_date = Date(2025, 1, 1) + self.pop_size = 5000 self.number_of_draws = 1 - self.runs_per_draw = 20 + self.runs_per_draw = 10 def log_configuration(self): return { - 'filename': 'intervention_scenario_test', 'directory': './outputs', + 'filename': 'cohort_test', 'directory': './outputs', "custom_levels": { "*": logging.WARNING, "tlo.methods.demography": logging.INFO, @@ -39,7 +39,6 @@ def modules(self): def draw_parameters(self, draw_number, rng): return { - 'MaternalNewbornHealthCohort': {'blocked_intervention': 'screening (direct)'}, } diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py index c22549d3cf..a93cdfb61a 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/local_run_cohort_calibration.py @@ -14,7 +14,7 @@ population_size = 2000 sim = Simulation(start_date=Date(2024, 1, 1), - seed=123, + seed=456, log_config={"filename": "log_cohort_calibration", "custom_levels": {"*": logging.DEBUG}, "directory": outputpath}) @@ -58,7 +58,7 @@ 'PPH']) # total_pregnancies = population_size -total_pregnancies = 2000 + len(output['tlo.methods.contraception']['pregnancy']) +total_pregnancies = 2000 total_births = len(output['tlo.methods.demography']['on_birth']) prop_live_births = (total_births/total_pregnancies) * 100 @@ -89,13 +89,18 @@ la_comps = output['tlo.methods.labour']['maternal_complication'] pn_comps = output['tlo.methods.postnatal_supervisor']['maternal_complication'] +twin_births = len(output['tlo.methods.newborn_outcomes']['twin_birth']) + total_completed_pregnancies = (len(an_comps.loc[an_comps['type'] == 'ectopic_unruptured']) + len(an_comps.loc[an_comps['type'] == 'induced_abortion']) + len(an_comps.loc[an_comps['type'] == 'spontaneous_abortion']) + - total_births + + (total_births - twin_births) + len(output['tlo.methods.pregnancy_supervisor']['antenatal_stillbirth']) + len(output['tlo.methods.labour']['intrapartum_stillbirth'])) +print(total_completed_pregnancies) # this value may be less than the starting population size due to antenatal +# maternal deaths + # Twins (todo) # Ectopic From 3c0d4856b181c73ed85e99564dc8b31d6aff4c0e Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 09:02:41 +0100 Subject: [PATCH 017/103] update scenario for azure test --- src/tlo/methods/mnh_cohort_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 4007d11a31..0bbae2c854 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -63,10 +63,10 @@ def initialise_population(self, population): :param population: the population of individuals """ - + # todo: we need to think of a better way to do this log_file = parse_log_file( - '/Users/j_collins/PycharmProjects/TLOmodel/outputs/sejjj49@ucl.ac.uk/' - 'fullmodel_200k_cohort-2024-04-24T072206Z/0/0/fullmodel_200k_cohort__2024-04-24T072516.log', + '/Users/j_collins/PycharmProjects/TLOmodel/resources/maternal cohort/' + 'fullmodel_200k_cohort__2024-04-24T072516.log', level=logging.DEBUG)['tlo.methods.contraception'] all_pregnancies = log_file['properties_of_pregnant_person'].loc[ From d51ff89bab69e0fb057042123aa31993d433a40f Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 11:42:51 +0100 Subject: [PATCH 018/103] make cohort df come from excel --- ...rceFile_All2024PregnanciesCohortModel.xlsx | 3 +++ src/tlo/methods/mnh_cohort_module.py | 27 ++++++++++++------- tests/test_mnh_cohort.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) create mode 100644 resources/maternal cohort/ResourceFile_All2024PregnanciesCohortModel.xlsx diff --git a/resources/maternal cohort/ResourceFile_All2024PregnanciesCohortModel.xlsx b/resources/maternal cohort/ResourceFile_All2024PregnanciesCohortModel.xlsx new file mode 100644 index 0000000000..500fcfd169 --- /dev/null +++ b/resources/maternal cohort/ResourceFile_All2024PregnanciesCohortModel.xlsx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39435a18741e06862f5c0b29d47c331c2c15d22f2954d204c5cf9e3fd80f20bc +size 20542399 diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 0bbae2c854..f5e19c6767 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -63,22 +63,31 @@ def initialise_population(self, population): :param population: the population of individuals """ - # todo: we need to think of a better way to do this - log_file = parse_log_file( - '/Users/j_collins/PycharmProjects/TLOmodel/resources/maternal cohort/' - 'fullmodel_200k_cohort__2024-04-24T072516.log', - level=logging.DEBUG)['tlo.methods.contraception'] - all_pregnancies = log_file['properties_of_pregnant_person'].loc[ - log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) - all_pregnancies.index = [x for x in range(len(all_pregnancies))] + # log_file = parse_log_file( + # '/Users/j_collins/PycharmProjects/TLOmodel/resources/maternal cohort/' + # 'fullmodel_200k_cohort__2024-04-24T072516.log', + # level=logging.DEBUG)['tlo.methods.contraception'] + # + # all_pregnancies = log_file['properties_of_pregnant_person'].loc[ + # log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) + # all_pregnancies.index = [x for x in range(len(all_pregnancies))] - preg_pop = all_pregnancies.loc[0:(len(self.sim.population.props))-1] + all_preg_df = pd.read_excel(Path(f'{self.resourcefilepath}/maternal cohort') / + 'ResourceFile_All2024PregnanciesCohortModel.xlsx') + preg_pop = all_preg_df.loc[0:(len(self.sim.population.props))-1] props_dtypes = self.sim.population.props.dtypes + preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) preg_pop_final.index.name = 'person' + + for column in ['rt_injuries_for_minor_surgery', 'rt_injuries_for_major_surgery', + 'rt_injuries_to_heal_with_time', 'rt_injuries_for_open_fracture_treatment', + 'rt_injuries_left_untreated', 'rt_injuries_to_cast']: + preg_pop_final[column] = [[] for _ in range(len(preg_pop_final))] + self.sim.population.props = preg_pop_final df = self.sim.population.props diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 35b662b3fc..df9bc58e15 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -30,7 +30,7 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=5000) + sim.make_initial_population(n=2500) sim.simulate(end_date=Date(2025, 1, 1)) output= parse_log_file(sim.log_filepath) From 0c7aa4169b8def292aa93f8f95d3a94e80110419 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 13:38:49 +0100 Subject: [PATCH 019/103] created mnh_outcome_logger to eventually replace logging for complication incidence --- src/tlo/methods/labour.py | 19 +++ src/tlo/methods/newborn_outcomes.py | 12 ++ src/tlo/methods/postnatal_supervisor.py | 14 ++ src/tlo/methods/pregnancy_helper_functions.py | 78 +++++++++++ src/tlo/methods/pregnancy_supervisor.py | 126 +++++++++++++----- 5 files changed, 215 insertions(+), 34 deletions(-) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 35081b7d27..02fcd65630 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1286,6 +1286,8 @@ def set_intrapartum_complications(self, individual_id, complication): 'type': f'{complication}', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[complication] += 1 + if complication == 'obstruction_cpd': mni[individual_id]['cpd'] = True @@ -1296,6 +1298,8 @@ def set_intrapartum_complications(self, individual_id, complication): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'placental_abruption', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['placental_abruption'] += 1 + elif complication == 'antepartum_haem': random_choice = self.rng.choice(['mild_moderate', 'severe'], @@ -1308,6 +1312,8 @@ def set_intrapartum_complications(self, individual_id, complication): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'mild_mod_antepartum_haemorrhage', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += 1 + else: pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'severe_aph_onset', @@ -1315,6 +1321,7 @@ def set_intrapartum_complications(self, individual_id, complication): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'severe_antepartum_haemorrhage', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_antepartum_haemorrhage'] += 1 elif complication == 'sepsis_chorioamnionitis': df.at[individual_id, 'la_sepsis'] = True @@ -1324,6 +1331,7 @@ def set_intrapartum_complications(self, individual_id, complication): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'sepsis', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_intrapartum'] += 1 elif complication == 'uterine_rupture': df.at[individual_id, 'la_uterine_rupture'] = True @@ -1332,6 +1340,7 @@ def set_intrapartum_complications(self, individual_id, complication): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'uterine_rupture', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['uterine_rupture'] += 1 def set_postpartum_complications(self, individual_id, complication): """ @@ -1387,6 +1396,7 @@ def set_postpartum_complications(self, individual_id, complication): logger_pn.info(key='maternal_complication', data={'person': individual_id, 'type': f'{complication}', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[complication] += 1 # Store mni variables used during treatment if complication == 'pph_uterine_atony': @@ -2427,6 +2437,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'early_preterm_labour', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_preterm_labour'] += 1 elif params['list_limits_for_defining_term_status'][4] <= gestational_age_in_days <= params['list_limits' '_for_defining' @@ -2438,6 +2449,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'late_preterm_labour', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['late_preterm_labour'] += 1 elif gestational_age_in_days >= params['list_limits_for_defining_term_status'][6]: @@ -2446,6 +2458,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'post_term_labour', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['post_term_labour'] += 1 labour_state = mni[individual_id]['labour_state'] @@ -2615,6 +2628,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'obstructed_labour', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['obstructed_labour'] += 1 # And we determine if any existing hypertensive disorders would worsen self.module.progression_of_hypertensive_disorders(individual_id, property_prefix='ps') @@ -2731,6 +2745,7 @@ def apply(self, individual_id): if df.at[individual_id, 'la_intrapartum_still_birth'] or mni[individual_id]['single_twin_still_birth']: logger.info(key='intrapartum_stillbirth', data={'mother_id': individual_id, 'date_of_ip_stillbirth': self.sim.date}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['intrapartum_stillbirth'] += 1 # Reset property if individual_id in mni: @@ -2821,10 +2836,13 @@ def apply(self, mother_id): logger_pn.info(key='maternal_complication', data={'person': mother_id, 'type': 'sepsis_postnatal', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 + if df.at[mother_id, 'la_postpartum_haem']: logger_pn.info(key='maternal_complication', data={'person': mother_id, 'type': 'primary_postpartum_haemorrhage', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['primary_postpartum_haemorrhage'] += 1 self.module.progression_of_hypertensive_disorders(mother_id, property_prefix='pn') @@ -2980,6 +2998,7 @@ def apply(self, person_id, squeeze_factor): logger.info(key='maternal_complication', data={'person': person_id, 'type': 'obstructed_labour', 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['obstructed_labour'] += 1 # ======================================= COMPLICATION MANAGEMENT ========================== # Next, women in labour are assessed for complications and treatment delivered if a need is identified and diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 3691bc6003..b0e8d461bd 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -525,26 +525,31 @@ def apply_risk_of_congenital_anomaly(self, child_id): self.congeintal_anomalies.set(child_id, 'heart') logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'congenital_heart_anomaly'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['congenital_heart_anomaly'] += 1 if self.rng.random_sample() < params['prob_limb_musc_skeletal_anomaly']: self.congeintal_anomalies.set(child_id, 'limb_musc_skeletal') logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'limb_or_musculoskeletal_anomaly'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['limb_or_musculoskeletal_anomaly'] += 1 if self.rng.random_sample() < params['prob_urogenital_anomaly']: self.congeintal_anomalies.set(child_id, 'urogenital') logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'urogenital_anomaly'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['urogenital_anomaly'] += 1 if self.rng.random_sample() < params['prob_digestive_anomaly']: self.congeintal_anomalies.set(child_id, 'digestive') logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'digestive_anomaly'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['digestive_anomaly'] += 1 if self.rng.random_sample() < params['prob_other_anomaly']: self.congeintal_anomalies.set(child_id, 'other') logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'other_anomaly'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['other_anomaly'] += 1 def apply_risk_of_neonatal_infection_and_sepsis(self, child_id): """ @@ -561,6 +566,7 @@ def apply_risk_of_neonatal_infection_and_sepsis(self, child_id): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'early_onset_sepsis'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_onset_sepsis'] += 1 def apply_risk_of_encephalopathy(self, child_id, timing): """ @@ -594,6 +600,7 @@ def apply_risk_of_encephalopathy(self, child_id, timing): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': f'{df.at[child_id, "nb_encephalopathy"]}'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{df.at[child_id, "nb_encephalopathy"]}'] += 1 # Check all encephalopathy cases receive a grade if df.at[child_id, 'nb_encephalopathy'] == 'none': @@ -619,6 +626,7 @@ def apply_risk_of_preterm_respiratory_distress_syndrome(self, child_id): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'respiratory_distress_syndrome'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['respiratory_distress_syndrome'] += 1 def apply_risk_of_not_breathing_at_birth(self, child_id): """ @@ -641,6 +649,7 @@ def apply_risk_of_not_breathing_at_birth(self, child_id): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'not_breathing_at_birth'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['not_breathing_at_birth'] += 1 def scheduled_week_one_postnatal_event(self, individual_id): """ @@ -1158,13 +1167,16 @@ def on_birth(self, mother_id, child_id): (df.at[child_id, 'nb_low_birth_weight_status'] == 'very_low_birth_weight') or\ (df.at[child_id, 'nb_low_birth_weight_status'] == 'extremely_low_birth_weight'): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'low_birth_weight'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['low_birth_weight'] += 1 elif df.at[child_id, 'nb_low_birth_weight_status'] == 'macrosomia': logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'macrosomia'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['macrosomia'] += 1 df.at[child_id, 'nb_size_for_gestational_age'] = mni[mother_id]['birth_size'] if df.at[child_id, 'nb_size_for_gestational_age'] == 'small_for_gestational_age': logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'small_for_gestational_age'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['small_for_gestational_age'] += 1 df.at[child_id, 'nb_early_init_breastfeeding'] = False df.at[child_id, 'nb_breastfeeding_status'] = 'none' diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 25bce6013f..428be94aaa 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -358,6 +358,7 @@ def further_on_birth_postnatal_supervisor(self, mother_id): logger.info(key='maternal_complication', data={'person': mother_id, 'type': f'{fistula_type}_fistula', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{fistula_type}_fistula'] += 1 # Determine if she will seek care for repair care_seeking_for_repair = self.pn_linear_models[ @@ -503,6 +504,7 @@ def onset(eq): logger.info(key='maternal_complication', data={'person': person, 'type': 'sepsis', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 # ------------------------------------ SECONDARY PPH ---------------------------------------------------------- # Next we determine if any women will experience postnatal bleeding @@ -518,6 +520,7 @@ def onset(eq): logger.info(key='maternal_complication', data={'person': person, 'type': 'secondary_postpartum_haemorrhage', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['secondary_postpartum_haemorrhage'] += 1 # --------------------------------------------- ANAEMIA -------------------------------------------------- # We apply a risk of anaemia developing in this week, and determine its severity @@ -597,6 +600,7 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': person, 'type': disease, 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[disease] += 1 if disease == 'severe_pre_eclamp': mni[person]['new_onset_spe'] = True @@ -642,6 +646,7 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': person, 'type': 'mild_pre_eclamp', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_pre_eclamp'] += 1 # -------------------------------- RISK OF GESTATIONAL HYPERTENSION -------------------------------------- gest_hypertension = self.apply_linear_model( @@ -655,6 +660,7 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': person, 'type': 'mild_gest_htn', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 # -------------------------------- RISK OF DEATH SEVERE HYPERTENSION ------------------------------------------ # Risk of death is applied to women with severe hypertensive disease @@ -747,6 +753,7 @@ def apply_risk_of_neonatal_complications_in_week_one(self, child_id, mother_id): logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'early_onset_sepsis'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_onset_sepsis'] += 1 def set_postnatal_complications_neonates(self, upper_and_lower_day_limits): """ @@ -774,6 +781,7 @@ def set_postnatal_complications_neonates(self, upper_and_lower_day_limits): logger.info(key='newborn_complication', data={'newborn': person, 'type': 'late_onset_sepsis'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['late_onset_sepsis'] += 1 # Then we determine if care will be sought for newly septic newborns care_seeking = pd.Series( @@ -1030,6 +1038,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'sepsis', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 # Sepsis secondary to endometritis is stored within the mni as it is used as a predictor in a linear model if endo_result: @@ -1049,6 +1058,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'secondary_postpartum_haemorrhage', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['secondary_postpartum_haemorrhage'] += 1 # ------------------------------------------------ NEW ONSET ANAEMIA ------------------------------------------ # And then risk of developing anaemia... @@ -1117,6 +1127,8 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': person, 'type': disease, 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[disease] += 1 + if disease == 'severe_pre_eclamp': mni[person]['new_onset_spe'] = True @@ -1139,6 +1151,7 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'mild_pre_eclamp', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_pre_eclamp'] += 1 else: risk_gh_after_pregnancy = self.module.pn_linear_models['gest_htn_pn'].predict(df.loc[[ @@ -1149,6 +1162,7 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'mild_gest_htn', 'timing': 'postnatal'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 # ====================================== POSTNATAL CHECK ================================================== # Import the HSI which represents postnatal care diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 8f7faa0503..4866d78ce7 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -8,6 +8,84 @@ from tlo import logging +def generate_mnh_outcome_counter(): + outcome_list = ['spontaneous_abortion', + 'induced_abortion', + 'complicated_spontaneous_abortion', + 'complicated_induced_abortion', + 'induced_abortion_injury', + 'induced_abortion_sepsis', + 'induced_abortion_haemorrhage', + 'induced_abortion_other_comp', + 'spontaneous_abortion_sepsis', + 'spontaneous_abortion_haemorrhage', + 'spontaneous_abortion_other_comp', + 'gest_diab', + 'clinical_chorioamnionitis', + 'antenatal_stillbirth', + 'ectopic_unruptured', + 'multiple_pregnancy', + 'placenta_praevia', + 'ectopic_ruptured', + 'syphilis', + 'placental_abruption', + 'severe_antepartum_haemorrhage', + 'mild_mod_antepartum_haemorrhage', + 'PROM' + 'obstruction_cpd', + 'obstruction_malpos_malpres', + 'obstruction_other', + 'obstructed_labour', + 'sepsis_intrapartum', + 'uterine_rupture', 'early_preterm_labour', 'late_preterm_labour', 'post_term_labour', + 'intrapartum_stillbirth', + 'spontaneous_abortion', + 'induced_abortion', + 'complicated_spontaneous_abortion', + 'complicated_induced_abortion', + 'induced_abortion_injury', + 'induced_abortion_sepsis', + 'induced_abortion_haemorrhage', + 'induced_abortion_other_comp', + 'spontaneous_abortion_sepsis', + 'spontaneous_abortion_haemorrhage', + 'spontaneous_abortion_other_comp', + 'gest_diab', + 'mild_pre_eclamp', + 'mild_gest_htn' + 'severe_pre_eclamp', + 'eclampsia', + 'severe_gest_htn', + 'placental_abruption', + 'severe_antepartum_haemorrhage', + 'mild_mod_antepartum_haemorrhage', + 'PROM' + 'clinical_chorioamnionitis', + 'antenatal_stillbirth', + 'ectopic_unruptured', + 'multiple_pregnancy', + 'placenta_praevia', + 'ectopic_ruptured', + 'syphilis', + 'obstruction_cpd', + 'obstruction_malpos_malpres', + 'obstruction_other', + 'obstructed_labour', + 'sepsis_intrapartum', + 'uterine_rupture', 'early_preterm_labour', 'late_preterm_labour', 'post_term_labour', + 'intrapartum_stillbirth', 'vesicovaginal_fistula', 'rectovaginal_fistula', + 'secondary_postpartum_haemorrhage', 'pph_uterine_atony', 'pph_retained_placenta', 'pph_other', + 'primary_postpartum_haemorrhage', 'sepsis_endometritis', 'sepsis_urinary_tract', + 'sepsis_skin_soft_tissue', 'sepsis_postnatal', 'congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', + 'urogenital_anomaly', + 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', 'severe_enceph', + 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', 'macrosomia', + 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis', + ] + + mnh_outcome_counter = {k: 0 for k in outcome_list} + + return mnh_outcome_counter def get_list_of_items(self, item_list): """ diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7dd8819ab6..e82eb71fad 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -62,6 +62,9 @@ def __init__(self, name=None, resourcefilepath=None): # This variable will store a Bitset handler for the property ps_abortion_complications self.abortion_complications = None + # Finally we create a dictionary to capture the frequency of key outcomes for logging + self.mnh_outcome_counter = pregnancy_helper_functions.generate_mnh_outcome_counter() + INIT_DEPENDENCIES = {'Demography'} OPTIONAL_INIT_DEPENDENCIES = {'HealthBurden', 'Malaria', 'CardioMetabolicDisorders', 'Hiv'} @@ -970,6 +973,7 @@ def do_after_abortion(self, individual_id, type_abortion): logger.info(key='maternal_complication', data={'person': individual_id, 'type': f'{type_abortion}', 'timing': 'antenatal'}) + self.mnh_outcome_counter[type_abortion] += 1 # This function officially ends a pregnancy through the contraception module (updates 'is_pregnant' and # determines post pregnancy contraception) @@ -1000,6 +1004,8 @@ def do_after_abortion(self, individual_id, type_abortion): 'type': f'complicated_{type_abortion}', 'timing': 'antenatal'}) + self.mnh_outcome_counter[f'complicated_{type_abortion}'] += 1 + self.apply_risk_of_abortion_complications(individual_id, f'{type_abortion}') def apply_risk_of_abortion_complications(self, individual_id, cause): @@ -1019,6 +1025,7 @@ def apply_risk_of_abortion_complications(self, individual_id, cause): logger.info(key='maternal_complication', data={'person': individual_id, 'type': f'{cause}_injury', 'timing': 'antenatal'}) + self.mnh_outcome_counter[f'{cause}_injury'] += 1 if self.rng.random_sample() < params['prob_haemorrhage_post_abortion']: self.abortion_complications.set([individual_id], 'haemorrhage') @@ -1027,6 +1034,7 @@ def apply_risk_of_abortion_complications(self, individual_id, cause): logger.info(key='maternal_complication', data={'person': individual_id, 'type': f'{cause}_haemorrhage', 'timing': 'antenatal'}) + self.mnh_outcome_counter[f'{cause}_haemorrhage'] += 1 if self.rng.random_sample() < params['prob_sepsis_post_abortion']: self.abortion_complications.set([individual_id], 'sepsis') @@ -1035,12 +1043,14 @@ def apply_risk_of_abortion_complications(self, individual_id, cause): logger.info(key='maternal_complication', data={'person': individual_id, 'type': f'{cause}_sepsis', 'timing': 'antenatal'}) + self.mnh_outcome_counter[f'{cause}_sepsis'] += 1 if not self.abortion_complications.has_any([individual_id], 'sepsis', 'haemorrhage', 'injury', first=True): self.abortion_complications.set([individual_id], 'other') logger.info(key='maternal_complication', data={'person': individual_id, 'type': f'{cause}_other_comp', 'timing': 'antenatal'}) + self.mnh_outcome_counter[f'{cause}_other_comp'] += 1 # We assume only women with complicated abortions will experience disability pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'abortion_onset', self.sim.date) @@ -1103,6 +1113,8 @@ def apply_risk_of_gestational_diabetes(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'gest_diab', 'timing': 'antenatal'}) + self.mnh_outcome_counter['gest_diab'] += 1 + def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): """ @@ -1127,6 +1139,7 @@ def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'mild_pre_eclamp', 'timing': 'antenatal'}) + self.mnh_outcome_counter['mild_pre_eclamp'] += 1 # -------------------------------- RISK OF GESTATIONAL HYPERTENSION -------------------------------------- # For women who dont develop pre-eclampsia during this month, we apply a risk of gestational hypertension @@ -1141,6 +1154,7 @@ def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'mild_gest_htn', 'timing': 'antenatal'}) + self.mnh_outcome_counter['mild_gest_htn'] += 1 def apply_risk_of_progression_of_hypertension(self, gestation_of_interest): """ @@ -1198,6 +1212,8 @@ def log_new_progressed_cases(disease): logger.info(key='maternal_complication', data={'person': person, 'type': disease, 'timing': 'antenatal'}) + self.mnh_outcome_counter[disease] +=1 + if disease == 'severe_pre_eclamp': self.mother_and_newborn_info[person]['new_onset_spe'] = True @@ -1273,6 +1289,7 @@ def apply_risk_of_placental_abruption(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'placental_abruption', 'timing': 'antenatal'}) + self.mnh_outcome_counter['placental_abruption'] += 1 def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): """ @@ -1314,6 +1331,7 @@ def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'severe_antepartum_haemorrhage', 'timing': 'antenatal'}) + self.mnh_outcome_counter['severe_antepartum_haemorrhage'] += 1 non_severe_women = (df.loc[antepartum_haemorrhage.loc[antepartum_haemorrhage].index, 'ps_antepartum_haemorrhage'] != 'severe') @@ -1326,6 +1344,8 @@ def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): logger.info(key='maternal_complication', data={'person': person, 'type': 'mild_mod_antepartum_haemorrhage', 'timing': 'antenatal'}) + self.mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += 1 + def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gestation_of_interest): """ @@ -1354,6 +1374,7 @@ def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gest logger.info(key='maternal_complication', data={'person': person, 'type': 'PROM', 'timing': 'antenatal'}) + self.mnh_outcome_counter['PROM'] += 1 # Determine if those with PROM will develop infection prior to care seeking infection = pd.Series(self.rng.random_sample(len(prom.loc[prom])) < params['prob_chorioamnionitis'], @@ -1369,6 +1390,7 @@ def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gest logger.info(key='maternal_complication', data={'person': person, 'type': 'clinical_chorioamnionitis', 'timing': 'antenatal'}) + self.mnh_outcome_counter['clinical_chorioamnionitis'] += 1 def apply_risk_of_preterm_labour(self, gestation_of_interest): """ @@ -1433,6 +1455,7 @@ def update_variables_post_still_birth_for_data_frame(self, women): self.sim.modules['Contraception'].end_pregnancy(person) mni[person]['delete_mni'] = True logger.info(key='antenatal_stillbirth', data={'mother': person}) + self.mnh_outcome_counter['antenatal_stillbirth'] += 1 # Call functions across the modules to ensure properties are rest self.sim.modules['Labour'].reset_due_date(id_or_index=women.index, new_due_date=pd.NaT) @@ -1455,6 +1478,7 @@ def update_variables_post_still_birth_for_individual(self, individual_id): mni[individual_id]['delete_mni'] = True logger.info(key='antenatal_stillbirth', data={'mother': individual_id}) + self.mnh_outcome_counter['antenatal_stillbirth'] += 1 # Reset pregnancy and schedule possible update of contraception self.sim.modules['Contraception'].end_pregnancy(individual_id) @@ -1768,6 +1792,7 @@ def apply(self, population): logger.info(key='maternal_complication', data={'person': person, 'type': 'ectopic_unruptured', 'timing': 'antenatal'}) + self.module.mnh_outcome_counter['ectopic_unruptured'] += 1 self.sim.schedule_event(EctopicPregnancyEvent(self.module, person), (self.sim.date + pd.Timedelta(days=7 * 3 + self.module.rng.randint(0, 7 * 2)))) @@ -1787,6 +1812,7 @@ def apply(self, population): logger.info(key='maternal_complication', data={'person': person, 'type': 'multiple_pregnancy', 'timing': 'antenatal'}) + self.module.mnh_outcome_counter['multiple_pregnancy'] += 1 # -----------------------------APPLYING RISK OF PLACENTA PRAEVIA ------------------------------------------- # Next,we apply a one off risk of placenta praevia (placenta will grow to cover the cervix either partially or @@ -1801,6 +1827,7 @@ def apply(self, population): logger.info(key='maternal_complication', data={'person': person, 'type': 'placenta_praevia', 'timing': 'antenatal'}) + self.module.mnh_outcome_counter['placenta_praevia'] += 1 # ------------------------- APPLYING RISK OF SYPHILIS INFECTION DURING PREGNANCY --------------------------- # Finally apply risk that syphilis will develop during pregnancy @@ -1987,6 +2014,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'ectopic_ruptured', 'timing': 'antenatal'}) + self.module.mnh_outcome_counter['ectopic_ruptured'] += 1 # Set the variable df.at[individual_id, 'ps_ectopic_pregnancy'] = 'ruptured' @@ -2097,7 +2125,7 @@ def apply(self, individual_id): logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'syphilis', 'timing': 'antenatal'}) - + self.module.mnh_outcome_counter['syphilis'] += 1 class ParameterUpdateEvent(Event, PopulationScopeEventMixin): """This is ParameterUpdateEvent. It is scheduled to occur once on 2015 to update parameters being used by the @@ -2236,36 +2264,66 @@ def __init__(self, module): def apply(self, population): df = self.sim.population.props - women_reproductive_age = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - (df.age_years < 50))]) - pregnant_at_year_end = len(df.index[df.is_alive & df.is_pregnant]) - women_with_previous_sa = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - (df.age_years < 50) & df.ps_prev_spont_abortion)]) - women_with_previous_pe = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - (df.age_years < 50) & df.ps_prev_pre_eclamp)]) - women_with_hysterectomy = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - (df.age_years < 50) & df.la_has_had_hysterectomy)]) - - yearly_prev_sa = (women_with_previous_sa / women_reproductive_age) * 100 - yearly_prev_pe = (women_with_previous_pe / women_reproductive_age) * 100 - yearly_prev_hysterectomy = (women_with_hysterectomy / women_reproductive_age) * 100 - - parity_list = list() - for parity in [0, 1, 2, 3, 4, 5]: - if parity < 5: - par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & - (df.la_parity == parity))]) - else: - par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & - (df.la_parity >= parity))]) - - yearly_prev = (par / women_reproductive_age) * 100 - parity_list.append(yearly_prev) - - logger.info(key='preg_info', - data={'women_repro_age': women_reproductive_age, - 'women_pregnant': pregnant_at_year_end, - 'prev_sa': yearly_prev_sa, - 'prev_pe': yearly_prev_pe, - 'hysterectomy': yearly_prev_hysterectomy, - 'parity': parity_list}) + # Complication incidence + # Denominators + yrly_live_births = len(df.index[(df.date_of_birth.year == self.sim.date.year)]) + yrly_pregnancies = len(df.index[(df.date_of_last_pregnancy.year == self.sim.date.year)]) + yrly_comp_pregnancies = [] + + # # Lets only do purely antenatal stuff here, purely labour stuff in labour and everything else in postnatal/newborn + # + # ectopic_incidence = (self.module.mnh_outcome_counter[''] / yrly_pregnancies) * 1000 + # abortion_incidence = + # miscarriage_incidence = + + + + + # logger.info(key='an_comp_incidence', + # data={'women_repro_age': women_reproductive_age, + # 'women_pregnant': pregnant_at_year_end, + # 'prev_sa': yearly_prev_sa, + # 'prev_pe': yearly_prev_pe, + # 'hysterectomy': yearly_prev_hysterectomy, + # 'parity': parity_list} + # + # + # + # + # + # + # women_reproductive_age = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & + # (df.age_years < 50))]) + # pregnant_at_year_end = len(df.index[df.is_alive & df.is_pregnant]) + # women_with_previous_sa = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & + # (df.age_years < 50) & df.ps_prev_spont_abortion)]) + # women_with_previous_pe = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & + # (df.age_years < 50) & df.ps_prev_pre_eclamp)]) + # women_with_hysterectomy = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & + # (df.age_years < 50) & df.la_has_had_hysterectomy)]) + # + # yearly_prev_sa = (women_with_previous_sa / women_reproductive_age) * 100 + # yearly_prev_pe = (women_with_previous_pe / women_reproductive_age) * 100 + # yearly_prev_hysterectomy = (women_with_hysterectomy / women_reproductive_age) * 100 + # + # parity_list = list() + # for parity in [0, 1, 2, 3, 4, 5]: + # if parity < 5: + # par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & + # (df.la_parity == parity))]) + # else: + # par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & + # (df.la_parity >= parity))]) + # + # yearly_prev = (par / women_reproductive_age) * 100 + # parity_list.append(yearly_prev) + # + # logger.info(key='preg_info', + # data={'women_repro_age': women_reproductive_age, + # 'women_pregnant': pregnant_at_year_end, + # 'prev_sa': yearly_prev_sa, + # 'prev_pe': yearly_prev_pe, + # 'hysterectomy': yearly_prev_hysterectomy, + # 'parity': parity_list}) + + self.module.mnh_outcome_counter = {k:0 for k in self.module.outcome_list} From 87d6f1f98e91ed1309564c33f51b906599d7ec3c Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 14:02:04 +0100 Subject: [PATCH 020/103] created mnh_outcome_logger to eventually replace logging for complication incidence --- src/tlo/methods/pregnancy_helper_functions.py | 98 +++++-------------- src/tlo/methods/pregnancy_supervisor.py | 8 +- 2 files changed, 29 insertions(+), 77 deletions(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 4866d78ce7..0eb75480ab 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -9,79 +9,31 @@ from tlo import logging def generate_mnh_outcome_counter(): - outcome_list = ['spontaneous_abortion', - 'induced_abortion', - 'complicated_spontaneous_abortion', - 'complicated_induced_abortion', - 'induced_abortion_injury', - 'induced_abortion_sepsis', - 'induced_abortion_haemorrhage', - 'induced_abortion_other_comp', - 'spontaneous_abortion_sepsis', - 'spontaneous_abortion_haemorrhage', - 'spontaneous_abortion_other_comp', - 'gest_diab', - 'clinical_chorioamnionitis', - 'antenatal_stillbirth', - 'ectopic_unruptured', - 'multiple_pregnancy', - 'placenta_praevia', - 'ectopic_ruptured', - 'syphilis', - 'placental_abruption', - 'severe_antepartum_haemorrhage', - 'mild_mod_antepartum_haemorrhage', - 'PROM' - 'obstruction_cpd', - 'obstruction_malpos_malpres', - 'obstruction_other', - 'obstructed_labour', - 'sepsis_intrapartum', - 'uterine_rupture', 'early_preterm_labour', 'late_preterm_labour', 'post_term_labour', - 'intrapartum_stillbirth', - 'spontaneous_abortion', - 'induced_abortion', - 'complicated_spontaneous_abortion', - 'complicated_induced_abortion', - 'induced_abortion_injury', - 'induced_abortion_sepsis', - 'induced_abortion_haemorrhage', - 'induced_abortion_other_comp', - 'spontaneous_abortion_sepsis', - 'spontaneous_abortion_haemorrhage', - 'spontaneous_abortion_other_comp', - 'gest_diab', - 'mild_pre_eclamp', - 'mild_gest_htn' - 'severe_pre_eclamp', - 'eclampsia', - 'severe_gest_htn', - 'placental_abruption', - 'severe_antepartum_haemorrhage', - 'mild_mod_antepartum_haemorrhage', - 'PROM' - 'clinical_chorioamnionitis', - 'antenatal_stillbirth', - 'ectopic_unruptured', - 'multiple_pregnancy', - 'placenta_praevia', - 'ectopic_ruptured', - 'syphilis', - 'obstruction_cpd', - 'obstruction_malpos_malpres', - 'obstruction_other', - 'obstructed_labour', - 'sepsis_intrapartum', - 'uterine_rupture', 'early_preterm_labour', 'late_preterm_labour', 'post_term_labour', - 'intrapartum_stillbirth', 'vesicovaginal_fistula', 'rectovaginal_fistula', - 'secondary_postpartum_haemorrhage', 'pph_uterine_atony', 'pph_retained_placenta', 'pph_other', - 'primary_postpartum_haemorrhage', 'sepsis_endometritis', 'sepsis_urinary_tract', - 'sepsis_skin_soft_tissue', 'sepsis_postnatal', 'congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', - 'urogenital_anomaly', - 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', 'severe_enceph', - 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', 'macrosomia', - 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis', - ] + outcome_list = [ # early/abortive outcomes + 'ectopic_unruptured', 'ectopic_ruptured','multiple_pregnancy', 'placenta_praevia', + 'spontaneous_abortion', 'induced_abortion', 'complicated_spontaneous_abortion', + 'complicated_induced_abortion', 'induced_abortion_injury', 'induced_abortion_sepsis', + 'induced_abortion_haemorrhage','induced_abortion_other_comp','spontaneous_abortion_sepsis', + 'spontaneous_abortion_haemorrhage', 'spontaneous_abortion_other_comp', + + # antenatal onset outcomes + 'gest_diab', 'mild_pre_eclamp', 'mild_gest_htn','severe_pre_eclamp', 'eclampsia','severe_gest_htn', + 'syphilis', 'PROM', 'clinical_chorioamnionitis', 'placental_abruption', + 'mild_mod_antepartum_haemorrhage','severe_antepartum_haemorrhage', 'antenatal_stillbirth', + + # intrapartum/postpartum onset outcomes + 'obstruction_cpd', 'obstruction_malpos_malpres', 'obstruction_other','obstructed_labour', + 'uterine_rupture','sepsis_intrapartum','sepsis_endometritis', 'sepsis_urinary_tract', + 'sepsis_skin_soft_tissue', 'sepsis_postnatal', 'intrapartum_stillbirth', 'early_preterm_labour', + 'late_preterm_labour', 'post_term_labour', 'pph_uterine_atony', 'pph_retained_placenta', + 'pph_other', 'primary_postpartum_haemorrhage', 'secondary_postpartum_haemorrhage', + 'vesicovaginal_fistula', 'rectovaginal_fistula', + + # newborn outcomes + 'congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', 'urogenital_anomaly', + 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', + 'severe_enceph', 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', + 'macrosomia', 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis'] mnh_outcome_counter = {k: 0 for k in outcome_list} diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index e82eb71fad..8a13d62164 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2266,9 +2266,9 @@ def apply(self, population): # Complication incidence # Denominators - yrly_live_births = len(df.index[(df.date_of_birth.year == self.sim.date.year)]) - yrly_pregnancies = len(df.index[(df.date_of_last_pregnancy.year == self.sim.date.year)]) - yrly_comp_pregnancies = [] + # yrly_live_births = len(df.index[(df.date_of_birth.year == self.sim.date.year)]) + # yrly_pregnancies = len(df.index[(df.date_of_last_pregnancy.year == self.sim.date.year)]) + # yrly_comp_pregnancies = [] # # Lets only do purely antenatal stuff here, purely labour stuff in labour and everything else in postnatal/newborn # @@ -2326,4 +2326,4 @@ def apply(self, population): # 'hysterectomy': yearly_prev_hysterectomy, # 'parity': parity_list}) - self.module.mnh_outcome_counter = {k:0 for k in self.module.outcome_list} + # self.module.mnh_outcome_counter = {k:0 for k in self.module.outcome_list} From cfb4264f0133fccbc0a82a6c9d3f51479d19038f Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:51:37 +0200 Subject: [PATCH 021/103] Switch iloc for loc --- src/tlo/events.py | 5 ++--- src/tlo/methods/hsi_event.py | 4 ++-- src/tlo/simulation.py | 9 ++++++--- tests/test_data_generation.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 78b828091d..a50832a58d 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -74,7 +74,7 @@ def run(self): if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): print_chains = True if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target row['event'] = self row['event_date'] = self.sim.date @@ -83,13 +83,12 @@ def run(self): else: df_before = self.sim.population.props.copy() - self.apply(self.target) self.post_apply_hook() if print_chains: if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target row['event'] = self row['event_date'] = self.sim.date diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 785f27b7a6..cffeb32992 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -201,7 +201,7 @@ def run(self, squeeze_factor): print_chains = True if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target row['event'] = self row['event_date'] = self.sim.date @@ -216,7 +216,7 @@ def run(self, squeeze_factor): if print_chains: if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target row['event'] = self row['event_date'] = self.sim.date diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 4aff23c9d7..42a2a288d3 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -298,14 +298,17 @@ def initialise(self, *, end_date: Date, generate_event_chains) -> None: self.date = self.start_date self.end_date = end_date # store the end_date so that others can reference it - self.generate_event_chains = generate_event_chains # for now ensure we're always aiming to print data - self.generate_event_chains_overwrite_epi = False + self.generate_event_chains = generate_event_chains if self.generate_event_chains: + # Eventually this can be made an option + self.generate_event_chains_overwrite_epi = True # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules['Tb'], self.modules['Hiv'], self.modules['CardioMetabolicDisorders']] self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] + else: + # If not using to print chains, cannot ignore epi + self.generate_event_chains_overwrite_epi = False - #df_event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) # Reorder columns to place the new columns at the front pd.set_option('display.max_columns', None) diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py index 1f6333bbfe..8dd92513f9 100644 --- a/tests/test_data_generation.py +++ b/tests/test_data_generation.py @@ -32,7 +32,7 @@ # create simulation parameters start_date = Date(2010, 1, 1) -end_date = Date(2015, 1, 1) +end_date = Date(2014, 1, 1) popsize = 100 @pytest.mark.slow From e0327de6b6f850ac871a2308271f6863333f173e Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:55:57 +0200 Subject: [PATCH 022/103] Change syntax of if statement --- src/tlo/events.py | 2 +- src/tlo/methods/hsi_event.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index a50832a58d..2eef87ba3f 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -71,7 +71,7 @@ def run(self): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersect(str(self)): print_chains = True if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index cffeb32992..805c9584fb 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -196,9 +196,7 @@ def run(self, squeeze_factor): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and - if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - + if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersect(str(self)): print_chains = True if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] From cdd98420615dba156a37e36ef484058821008963 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 7 Oct 2024 16:16:05 +0100 Subject: [PATCH 023/103] added first logging to pregnancy supervisor --- src/tlo/methods/pregnancy_helper_functions.py | 3 +- src/tlo/methods/pregnancy_supervisor.py | 71 ++++++++++++++++--- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 0eb75480ab..93607873df 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -37,7 +37,8 @@ def generate_mnh_outcome_counter(): mnh_outcome_counter = {k: 0 for k in outcome_list} - return mnh_outcome_counter + return {'counter': mnh_outcome_counter, + 'outcomes': outcome_list} def get_list_of_items(self, item_list): """ diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 8a13d62164..34f52e6271 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -63,7 +63,8 @@ def __init__(self, name=None, resourcefilepath=None): self.abortion_complications = None # Finally we create a dictionary to capture the frequency of key outcomes for logging - self.mnh_outcome_counter = pregnancy_helper_functions.generate_mnh_outcome_counter() + mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() + self.mnh_outcome_counter = mnh_oc['counter'] INIT_DEPENDENCIES = {'Demography'} @@ -2263,18 +2264,66 @@ def __init__(self, module): def apply(self, population): df = self.sim.population.props + counter = self.module.mnh_outcome_counter # Complication incidence # Denominators - # yrly_live_births = len(df.index[(df.date_of_birth.year == self.sim.date.year)]) - # yrly_pregnancies = len(df.index[(df.date_of_last_pregnancy.year == self.sim.date.year)]) - # yrly_comp_pregnancies = [] + yrly_live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) + yrly_pregnancies =len(df[df['date_of_last_pregnancy'].dt.year == self.sim.date.year - 1]) - # # Lets only do purely antenatal stuff here, purely labour stuff in labour and everything else in postnatal/newborn - # - # ectopic_incidence = (self.module.mnh_outcome_counter[''] / yrly_pregnancies) * 1000 - # abortion_incidence = - # miscarriage_incidence = + yrly_comp_pregnancies = (counter['ectopic_unruptured'] + counter['spontaneous_abortion'] + + counter['induced_abortion'] + counter['antenatal_stillbirth'] + + counter['intrapartum_stillbirth'] + yrly_live_births) + + yrly_total_births = yrly_live_births + counter['antenatal_stillbirth'] + counter['intrapartum_stillbirth'] + + logger.info(key='yrl_counter_dict', + data=counter) + + # MATERNAL COMPLICATIONS + logger.info(key='mat_comp_incidence', + data= {k:(counter[k]/denom) * 1000 for k, denom in + zip(['ectopic_unruptured', 'induced_abortion', 'spontaneous_abortion', + 'placenta_praevia', 'gest_diab', 'syphilis'], + [yrly_pregnancies, yrly_comp_pregnancies, yrly_comp_pregnancies, yrly_pregnancies, + yrly_comp_pregnancies, yrly_comp_pregnancies,])}) + + logger.info(key='mat_comp_incidence', + data={k: (counter[k] / yrly_live_births) * 1000 for k in + ['obstructed_labour', 'uterine_rupture', 'sepsis_intrapartum', + 'mild_mod_antepartum_haemorrhage', 'severe_antepartum_haemorrhage', + 'mild_pre_eclamp', 'mild_gest_htn', 'mild_gest_htn', 'severe_pre_eclamp', + 'severe_gest_htn', 'eclampsia', 'sepsis_postnatal', 'primary_postpartum_haemorrhage', + 'secondary_postpartum_haemorrhage', 'vesicovaginal_fistula', 'rectovaginal_fistula']}) + + logger.info(key='mat_comp_incidence', + data={'antepartum_haemorrhage': ((counter['mild_mod_antepartum_haemorrhage'] + + counter['severe_antepartum_haemorrhage']) / + yrly_live_births) * 1000}) + + logger.info(key='mat_comp_incidence', + data={k: (counter[k] / yrly_live_births) * 100 for k in + ['early_preterm_labour', 'late_preterm_labour', 'post_term_labour']}) + + # NEWBORN COMPLICATIONS + logger.info(key='nb_incidence', + data={k: (counter[k] / yrly_live_births) * 1000 for k in + ['congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', 'urogenital_anomaly', + 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', 'severe_enceph', + 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', 'macrosomia', + 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis']}) + + + neonatal_deaths = len(df[(df['date_of_death'].dt.year == self.sim.date.year - 1) & (df['age_days'] <= 28)]) + direct_maternal_deaths = [] + + # DIRECT MATERNAL DEATHS, NEWBORN DEATHS AND STILLBIRTH + logger.info(key='deaths_and_stillbirths', + data={'antenatal_stillbirth': (counter['antenatal_stillbirth'] / yrly_total_births) * 1000, + 'intrapartum_stillbirth': (counter['intrapartum_stillbirth'] / yrly_total_births) * 1000, + 'neonatal_deaths': neonatal_deaths, + 'nmr' : (neonatal_deaths/yrly_live_births) * 1000, + 'direct_maternal_deaths': []}) @@ -2326,4 +2375,6 @@ def apply(self, population): # 'hysterectomy': yearly_prev_hysterectomy, # 'parity': parity_list}) - # self.module.mnh_outcome_counter = {k:0 for k in self.module.outcome_list} + mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() + outcome_list = mnh_oc['outcomes'] + self.module.mnh_outcome_counter = {k:0 for k in outcome_list} From b9a193dafd32a8544eb8cc463175433cbccdab86 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 10:23:36 +0100 Subject: [PATCH 024/103] finalising logging event --- src/tlo/methods/labour.py | 4 + src/tlo/methods/newborn_outcomes.py | 1 + src/tlo/methods/postnatal_supervisor.py | 8 + src/tlo/methods/pregnancy_helper_functions.py | 11 +- src/tlo/methods/pregnancy_supervisor.py | 164 ++++++++---------- 5 files changed, 91 insertions(+), 97 deletions(-) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 02fcd65630..21bc602d01 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1529,9 +1529,12 @@ def apply_risk_of_early_postpartum_death(self, individual_id): # If a cause is returned death is scheduled if potential_cause_of_death: pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, originating_module=self.sim.modules['Labour']) + # If she hasn't died from any complications, we reset some key properties that resolve after risk of death # has been applied else: @@ -2700,6 +2703,7 @@ def apply(self, individual_id): # If a cause is returned death is scheduled if potential_cause_of_death: pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, originating_module=self.sim.modules['Labour']) diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index b0e8d461bd..4019544bd4 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -1030,6 +1030,7 @@ def link_twins(self, child_one, child_two, mother_id): 'date_of_delivery': self.sim.date} logger.info(key='twin_birth', data=twin_birth, description='A record of each birth of twin pairs') + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['twin_birth'] += 1 # Finally we log the second live birth and add another to the womans parity df.at[mother_id, 'la_parity'] += 1 diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 428be94aaa..ef81b3b8a5 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -838,6 +838,7 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in if potential_cause_of_death: mni[individual_id]['didnt_seek_care'] = True pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, originating_module=self.sim.modules['PostnatalSupervisor']) del mni[individual_id] @@ -950,6 +951,13 @@ def apply(self, population): # Set mni[person]['delete_mni'] to True meaning after the next DALY event each womans MNI dict is deleted for person in week_8_postnatal_women.loc[week_8_postnatal_women].index: mni[person]['delete_mni'] = True + + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['six_week_survivors'] += 1 + + if df.at[person, 'pn_anaemia_following_pregnancy'] != 'none': + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ + f'pn_anaemia_{df.at[person, "pn_anaemia_following_pregnancy"]}'] += 1 + logger.info(key='total_mat_pnc_visits', data={'mother': person, 'visits': df.at[person, 'la_pn_checks_maternal'], 'anaemia': df.at[person, 'pn_anaemia_following_pregnancy']}) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 93607873df..ad3237328f 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -10,13 +10,14 @@ def generate_mnh_outcome_counter(): outcome_list = [ # early/abortive outcomes - 'ectopic_unruptured', 'ectopic_ruptured','multiple_pregnancy', 'placenta_praevia', + 'ectopic_unruptured', 'ectopic_ruptured','multiple_pregnancy', 'twin_birth', 'placenta_praevia', 'spontaneous_abortion', 'induced_abortion', 'complicated_spontaneous_abortion', 'complicated_induced_abortion', 'induced_abortion_injury', 'induced_abortion_sepsis', 'induced_abortion_haemorrhage','induced_abortion_other_comp','spontaneous_abortion_sepsis', 'spontaneous_abortion_haemorrhage', 'spontaneous_abortion_other_comp', # antenatal onset outcomes + 'an_anaemia_mild', 'an_anaemia_moderate', 'an_anaemia_severe', 'gest_diab', 'mild_pre_eclamp', 'mild_gest_htn','severe_pre_eclamp', 'eclampsia','severe_gest_htn', 'syphilis', 'PROM', 'clinical_chorioamnionitis', 'placental_abruption', 'mild_mod_antepartum_haemorrhage','severe_antepartum_haemorrhage', 'antenatal_stillbirth', @@ -27,13 +28,17 @@ def generate_mnh_outcome_counter(): 'sepsis_skin_soft_tissue', 'sepsis_postnatal', 'intrapartum_stillbirth', 'early_preterm_labour', 'late_preterm_labour', 'post_term_labour', 'pph_uterine_atony', 'pph_retained_placenta', 'pph_other', 'primary_postpartum_haemorrhage', 'secondary_postpartum_haemorrhage', - 'vesicovaginal_fistula', 'rectovaginal_fistula', + 'vesicovaginal_fistula', 'rectovaginal_fistula', 'pn_anaemia_mild', 'pn_anaemia_moderate', + 'pn_anaemia_severe', # newborn outcomes 'congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', 'urogenital_anomaly', 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', 'severe_enceph', 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', - 'macrosomia', 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis'] + 'macrosomia', 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis', + + 'direct_mat_death', 'six_week_survivors' + ] mnh_outcome_counter = {k: 0 for k in outcome_list} diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 34f52e6271..671a6af07a 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -658,6 +658,9 @@ def further_on_birth_pregnancy_supervisor(self, mother_id): df.at[mother_id, 'ps_date_of_anc1'] = pd.NaT # And store her anaemia status to calculate the prevalence of anaemia on birth + if df.at[mother_id, 'ps_anaemia_in_pregnancy'] != 'none': + self.mnh_outcome_counter[f'an_anaemia_{df.at[mother_id, "ps_anaemia_in_pregnancy"]}'] += 1 + logger.info(key='conditions_on_birth', data={'mother': mother_id, 'anaemia_status': df.at[mother_id, 'ps_anaemia_in_pregnancy'], 'gdm_status': df.at[mother_id, 'ps_gest_diab'], @@ -1609,6 +1612,7 @@ def apply_risk_of_death_from_monthly_complications(self, individual_id): pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, originating_module=self.sim.modules['PregnancySupervisor']) + self.mnh_outcome_counter['direct_mat_death'] += 1 del mni[individual_id] # If not we reset variables and the woman survives @@ -2264,116 +2268,88 @@ def __init__(self, module): def apply(self, population): df = self.sim.population.props - counter = self.module.mnh_outcome_counter + c = self.module.mnh_outcome_counter - # Complication incidence - # Denominators - yrly_live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) - yrly_pregnancies =len(df[df['date_of_last_pregnancy'].dt.year == self.sim.date.year - 1]) + logger.info(key='yrly_counter_dict', data=c) - yrly_comp_pregnancies = (counter['ectopic_unruptured'] + counter['spontaneous_abortion'] + - counter['induced_abortion'] + counter['antenatal_stillbirth'] + - counter['intrapartum_stillbirth'] + yrly_live_births) + def rate (count, denom, multiplier): + return (count/denom) * multiplier - yrly_total_births = yrly_live_births + counter['antenatal_stillbirth'] + counter['intrapartum_stillbirth'] + # DENOMINATORS + live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) - logger.info(key='yrl_counter_dict', - data=counter) + pregnancies =len(df[df['date_of_last_pregnancy'].dt.year == self.sim.date.year - 1]) - # MATERNAL COMPLICATIONS - logger.info(key='mat_comp_incidence', - data= {k:(counter[k]/denom) * 1000 for k, denom in - zip(['ectopic_unruptured', 'induced_abortion', 'spontaneous_abortion', - 'placenta_praevia', 'gest_diab', 'syphilis'], - [yrly_pregnancies, yrly_comp_pregnancies, yrly_comp_pregnancies, yrly_pregnancies, - yrly_comp_pregnancies, yrly_comp_pregnancies,])}) + comp_pregnancies = (c['ectopic_unruptured'] + c['spontaneous_abortion'] + + c['induced_abortion'] + c['antenatal_stillbirth'] + + c['intrapartum_stillbirth'] + live_births) - logger.info(key='mat_comp_incidence', - data={k: (counter[k] / yrly_live_births) * 1000 for k in - ['obstructed_labour', 'uterine_rupture', 'sepsis_intrapartum', - 'mild_mod_antepartum_haemorrhage', 'severe_antepartum_haemorrhage', - 'mild_pre_eclamp', 'mild_gest_htn', 'mild_gest_htn', 'severe_pre_eclamp', - 'severe_gest_htn', 'eclampsia', 'sepsis_postnatal', 'primary_postpartum_haemorrhage', - 'secondary_postpartum_haemorrhage', 'vesicovaginal_fistula', 'rectovaginal_fistula']}) + deliveries = live_births - c['twin_birth'] - logger.info(key='mat_comp_incidence', - data={'antepartum_haemorrhage': ((counter['mild_mod_antepartum_haemorrhage'] + - counter['severe_antepartum_haemorrhage']) / - yrly_live_births) * 1000}) + total_births = live_births + c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] + + # MATERNAL COMPLICATION INCIDENCE + total_an_anaemia_cases = c['an_anaemia_mild'] + c['an_anaemia_moderate'] + c['an_anaemia_severe'] + total_pn_anaemia_cases = c['pn_anaemia_mild'] + c['pn_anaemia_moderate'] + c['pn_anaemia_severe'] + total_preterm_birth = c['early_preterm_labour'] + c['late_preterm_labour'] + total_aph = c['mild_mod_antepartum_haemorrhage'] + c['severe_antepartum_haemorrhage'] + total_sepsis = c['clinical_chorioamnionitis'] + c['sepsis_intrapartum'] + c['sepsis_postnatal'] + total_pph = c['primary_postpartum_haemorrhage'] + c['secondary_postpartum_haemorrhage'] + total_fistula = c['vesicovaginal_fistula'] + c['rectovaginal_fistula'] + total_neo_sepsis = c['early_onset_sepsis'] + c['late_onset_sepsis'] + total_neo_enceph = c['mild_enceph'] + c['moderate_enceph'] + c['severe_enceph'] + total_neo_resp_conds = c['respiratory_distress_syndrome'] + c['not_breathing_at_birth'] + total_neo_enceph + total_cba = (c['congenital_heart_anomaly'] + c['limb_or_musculoskeletal_anomaly'] + + c['urogenital_anomaly'] + c['digestive_anomaly'] + c['other_anomaly']) logger.info(key='mat_comp_incidence', - data={k: (counter[k] / yrly_live_births) * 100 for k in - ['early_preterm_labour', 'late_preterm_labour', 'post_term_labour']}) + data={'an_anaemia': rate(total_an_anaemia_cases, live_births, 100), + 'ectopic_unruptured' : rate(c['ectopic_unruptured'], pregnancies, 1000), + 'induced_abortion' : rate(c['induced_abortion'], comp_pregnancies, 1000), + 'spontaneous_abortion': rate(c['spontaneous_abortion'], comp_pregnancies, 1000), + 'placenta_praevia': rate(c['placenta_praevia'], pregnancies, 1000), + 'gest_diab': rate(c['gest_diab'], comp_pregnancies, 1000), + 'PROM': rate(c['PROM'], comp_pregnancies, 1000), + 'preterm_birth': rate(total_preterm_birth, live_births, 100), + 'antepartum_haem': rate(total_aph, live_births, 1000), + 'obstructed_labour': rate(c['obstructed_labour'], live_births, 1000), + 'uterine_rupture': rate(c['uterine_rupture'], live_births, 1000), + 'sepsis': rate(total_sepsis, live_births, 1000), + 'mild_pre_eclamp': rate(c['mild_pre_eclamp'], live_births, 1000), + 'mild_gest_htn': rate(c['mild_gest_htn'], live_births, 1000), + 'severe_pre_eclamp': rate(c['severe_pre_eclamp'], live_births, 1000), + 'severe_gest_htn': rate(c['severe_gest_htn'], live_births, 1000), + 'eclampsia': rate(c['eclampsia'], live_births, 1000), + 'postpartum_haem': rate(total_pph, live_births, 1000), + 'fistula': rate(total_fistula, live_births, 1000), + 'pn_anaemia': rate(total_pn_anaemia_cases, c['six_week_survivors'], 100)}) # NEWBORN COMPLICATIONS - logger.info(key='nb_incidence', - data={k: (counter[k] / yrly_live_births) * 1000 for k in - ['congenital_heart_anomaly', 'limb_or_musculoskeletal_anomaly', 'urogenital_anomaly', - 'digestive_anomaly', 'other_anomaly', 'mild_enceph', 'moderate_enceph', 'severe_enceph', - 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', 'macrosomia', - 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis']}) - + logger.info(key='nb_comp_incidence', + data={'twin_birth': rate(c['twin_birth'], deliveries, 100), + 'nb_sepsis': rate(total_neo_sepsis, live_births, 1000), + 'nb_enceph': rate(total_neo_enceph, live_births, 1000), + 'nb_resp_diff': rate(total_neo_resp_conds, live_births, 100), + 'nb_cba': rate(total_cba, live_births, 1000), + 'nb_rds': rate(c['respiratory_distress_syndrome'], total_preterm_birth, 1000), + 'nb_lbw': rate(c['low_birth_weight'], live_births, 100), + 'nb_macrosomia': rate(c['macrosomia'], live_births, 100), + 'nb_sga': rate(c['small_for_gestational_age'], live_births, 100)}) + # DIRECT MATERNAL DEATHS, NEWBORN DEATHS AND STILLBIRTH neonatal_deaths = len(df[(df['date_of_death'].dt.year == self.sim.date.year - 1) & (df['age_days'] <= 28)]) - direct_maternal_deaths = [] + stillbirths = c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] - # DIRECT MATERNAL DEATHS, NEWBORN DEATHS AND STILLBIRTH logger.info(key='deaths_and_stillbirths', - data={'antenatal_stillbirth': (counter['antenatal_stillbirth'] / yrly_total_births) * 1000, - 'intrapartum_stillbirth': (counter['intrapartum_stillbirth'] / yrly_total_births) * 1000, + data={'antenatal_sbr': rate(c['antenatal_stillbirth'], total_births, 1000), + 'intrapartum_sbr': rate(c['intrapartum_stillbirth'], total_births, 1000), + 'total_stillbirths': stillbirths, + 'sbr': rate(stillbirths, total_births, 1000), 'neonatal_deaths': neonatal_deaths, - 'nmr' : (neonatal_deaths/yrly_live_births) * 1000, - 'direct_maternal_deaths': []}) - - - - - # logger.info(key='an_comp_incidence', - # data={'women_repro_age': women_reproductive_age, - # 'women_pregnant': pregnant_at_year_end, - # 'prev_sa': yearly_prev_sa, - # 'prev_pe': yearly_prev_pe, - # 'hysterectomy': yearly_prev_hysterectomy, - # 'parity': parity_list} - # - # - # - # - # - # - # women_reproductive_age = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - # (df.age_years < 50))]) - # pregnant_at_year_end = len(df.index[df.is_alive & df.is_pregnant]) - # women_with_previous_sa = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - # (df.age_years < 50) & df.ps_prev_spont_abortion)]) - # women_with_previous_pe = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - # (df.age_years < 50) & df.ps_prev_pre_eclamp)]) - # women_with_hysterectomy = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & - # (df.age_years < 50) & df.la_has_had_hysterectomy)]) - # - # yearly_prev_sa = (women_with_previous_sa / women_reproductive_age) * 100 - # yearly_prev_pe = (women_with_previous_pe / women_reproductive_age) * 100 - # yearly_prev_hysterectomy = (women_with_hysterectomy / women_reproductive_age) * 100 - # - # parity_list = list() - # for parity in [0, 1, 2, 3, 4, 5]: - # if parity < 5: - # par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & - # (df.la_parity == parity))]) - # else: - # par = len(df.index[(df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50) & - # (df.la_parity >= parity))]) - # - # yearly_prev = (par / women_reproductive_age) * 100 - # parity_list.append(yearly_prev) - # - # logger.info(key='preg_info', - # data={'women_repro_age': women_reproductive_age, - # 'women_pregnant': pregnant_at_year_end, - # 'prev_sa': yearly_prev_sa, - # 'prev_pe': yearly_prev_pe, - # 'hysterectomy': yearly_prev_hysterectomy, - # 'parity': parity_list}) + 'nmr' : rate(neonatal_deaths, live_births, 1000), + 'direct_maternal_deaths': c['direct_mat_death'], + 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000), + }) mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() outcome_list = mnh_oc['outcomes'] From ada05e089db947e31b5e85531bde429e839deb07 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 12:58:21 +0100 Subject: [PATCH 025/103] add health system logging --- .../methods/care_of_women_during_pregnancy.py | 9 ++++++- src/tlo/methods/labour.py | 3 +++ src/tlo/methods/postnatal_supervisor.py | 18 +++++++++++-- src/tlo/methods/pregnancy_helper_functions.py | 9 ++++++- src/tlo/methods/pregnancy_supervisor.py | 25 +++++++++++++++++-- 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 69ce038299..b2df6c777b 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -484,8 +484,10 @@ def further_on_birth_care_of_women_in_pregnancy(self, mother_id): if df.at[mother_id, 'is_alive']: + anc_count = df.at[mother_id, "ac_total_anc_visits_current_pregnancy"] + # run a check at birth to make sure no women exceed 8 visits - if df.at[mother_id, 'ac_total_anc_visits_current_pregnancy'] > 9: + if anc_count > 9: logger.info(key='error', data=f'Mother {mother_id} attended >8 ANC visits during her pregnancy') # We log the total number of ANC contacts a woman has undergone at the time of birth via this dictionary @@ -494,6 +496,11 @@ def further_on_birth_care_of_women_in_pregnancy(self, mother_id): else: ga_anc_one = 0.0 + if anc_count > 8: + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['anc8+'] += 1 + else: + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'anc{anc_count}'] += 1 + total_anc_visit_count = {'person_id': mother_id, 'total_anc': df.at[mother_id, 'ac_total_anc_visits_current_pregnancy'], 'ga_anc_one': ga_anc_one} diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 21bc602d01..d84c41c1e0 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1060,6 +1060,9 @@ def further_on_birth_labour(self, mother_id): 'facility_type': str(mni[mother_id]['delivery_setting']), 'mode': mni[mother_id]['mode_of_delivery']}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ + f'{str(mni[mother_id]["delivery_setting"])}_delivery'] += 1 + # Store only live births to a mother parity if not df.at[mother_id, 'la_intrapartum_still_birth']: df.at[mother_id, 'la_parity'] += 1 # Only live births contribute to parity diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index ef81b3b8a5..5ba48a0e24 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -895,6 +895,7 @@ def apply(self, population): df = population.props store_dalys_in_mni = pregnancy_helper_functions.store_dalys_in_mni mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + mnh_counter = self.sim.modules['PregnancySupervisor'].mnh_outcome_counter # ================================ UPDATING LENGTH OF POSTPARTUM PERIOD IN WEEKS ============================ # Here we update how far into the postpartum period each woman who has recently delivered is @@ -950,13 +951,19 @@ def apply(self, population): # Set mni[person]['delete_mni'] to True meaning after the next DALY event each womans MNI dict is deleted for person in week_8_postnatal_women.loc[week_8_postnatal_women].index: + pn_checks = df.at[person, 'la_pn_checks_maternal'] + mni[person]['delete_mni'] = True self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['six_week_survivors'] += 1 if df.at[person, 'pn_anaemia_following_pregnancy'] != 'none': - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ - f'pn_anaemia_{df.at[person, "pn_anaemia_following_pregnancy"]}'] += 1 + mnh_counter[f'pn_anaemia_{df.at[person, "pn_anaemia_following_pregnancy"]}'] += 1 + + if pn_checks >= 3: + mnh_counter['m_pnc3+'] += 1 + else: + mnh_counter[f'm_pnc{pn_checks}'] += 1 logger.info(key='total_mat_pnc_visits', data={'mother': person, 'visits': df.at[person, 'la_pn_checks_maternal'], @@ -977,6 +984,13 @@ def apply(self, population): self.sim.start_date) for person in week_5_postnatal_neonates.loc[week_5_postnatal_neonates].index: + pn_checks = df.at[person, 'nb_pnc_check'] + + if pn_checks >= 3: + mnh_counter['n_pnc3+'] += 1 + else: + mnh_counter[f'n_pnc{pn_checks}'] += 1 + self.sim.modules['NewbornOutcomes'].set_disability_status(person) logger.info(key='total_neo_pnc_visits', data={'child': person, 'visits': df.at[person, 'nb_pnc_check']}) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index ad3237328f..3353f86fc8 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -37,7 +37,13 @@ def generate_mnh_outcome_counter(): 'severe_enceph', 'respiratory_distress_syndrome', 'not_breathing_at_birth', 'low_birth_weight', 'macrosomia', 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis', - 'direct_mat_death', 'six_week_survivors' + # death outcomes + 'direct_mat_death', 'six_week_survivors', + + # service coverage outcomes + 'anc0', 'anc1', 'anc2', 'anc3', 'anc4', 'anc5', 'anc6', 'anc7', 'anc8', 'anc8+', + 'home_birth_delivery', 'hospital_delivery', 'health_centre_delivery', + 'm_pnc0', 'm_pnc1', 'm_pnc2', 'm_pnc3+', 'n_pnc0', 'n_pnc1', 'n_pnc2', 'n_pnc3+', ] mnh_outcome_counter = {k: 0 for k in outcome_list} @@ -386,6 +392,7 @@ def calculate_risk_of_death_from_causes(self, risks): cause_of_death = self.rng.choice(list(risks.keys()), p=probs) # Return the primary cause of death so that it can be passed to the demography function + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause_of_death}_death'] += 1 return cause_of_death else: # Return false if death will not occur diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 671a6af07a..520474222d 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2348,8 +2348,29 @@ def rate (count, denom, multiplier): 'neonatal_deaths': neonatal_deaths, 'nmr' : rate(neonatal_deaths, live_births, 1000), 'direct_maternal_deaths': c['direct_mat_death'], - 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000), - }) + 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000)}) + + anc1 = sum(c[f'anc{i}'] for i in range(1, 9)) + c['anc8+'] + anc4 = sum(c[f'anc{i}'] for i in range(4, 9))+ c['anc8+'] + anc8 = c['anc8'] + c['anc8+'] + + m_pnc1 = sum(c[f'm_pnc{i}'] for i in range(1, 3)) + c['m_pnc3+'] + n_pnc1 = sum(c[f'm_pnc{i}'] for i in range(1, 3)) + c['m_pnc3+'] + + # HEALTH SERVICE COVERAGE + logger.info(key='service_coverage', + data={'anc1+': rate(anc1 , total_births, 100), + 'anc4+': rate(anc4, total_births, 100), + 'anc8+': rate(anc8, total_births, 100), + + 'fd_rate': rate(anc1 , total_births, 100), + 'hb_rate': rate(c['home_birth_delivery'] , total_births, 100), + 'hc_rate': rate(c['health_centre_delivery'] , total_births, 100), + 'hp_rate': rate(c['hospital_delivery'] , total_births, 100), + + 'm_pnc1+': rate(m_pnc1, total_births, 1000), + 'n_pnc1+': rate(n_pnc1, total_births, 1000)}) + mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() outcome_list = mnh_oc['outcomes'] From bfac2451d4822ce885f8d526ace23182374196a2 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 13:27:39 +0100 Subject: [PATCH 026/103] old logging removed --- .../methods/care_of_women_during_pregnancy.py | 16 ++- src/tlo/methods/labour.py | 59 ++-------- src/tlo/methods/newborn_outcomes.py | 24 ----- src/tlo/methods/postnatal_supervisor.py | 53 +-------- src/tlo/methods/pregnancy_helper_functions.py | 1 + src/tlo/methods/pregnancy_supervisor.py | 101 ++---------------- 6 files changed, 27 insertions(+), 227 deletions(-) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index b2df6c777b..d1d52f076a 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -490,22 +490,18 @@ def further_on_birth_care_of_women_in_pregnancy(self, mother_id): if anc_count > 9: logger.info(key='error', data=f'Mother {mother_id} attended >8 ANC visits during her pregnancy') - # We log the total number of ANC contacts a woman has undergone at the time of birth via this dictionary - if 'ga_anc_one' in mni[mother_id]: - ga_anc_one = float(mni[mother_id]['ga_anc_one']) - else: - ga_anc_one = 0.0 - if anc_count > 8: self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['anc8+'] += 1 else: self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'anc{anc_count}'] += 1 - total_anc_visit_count = {'person_id': mother_id, - 'total_anc': df.at[mother_id, 'ac_total_anc_visits_current_pregnancy'], - 'ga_anc_one': ga_anc_one} + # We log the gestational age at first ANC + if 'ga_anc_one' in mni[mother_id]: + ga_anc_one = float(mni[mother_id]['ga_anc_one']) + else: + ga_anc_one = 0.0 - logger.info(key='anc_count_on_birth', data=total_anc_visit_count, + logger.info(key='ga_at_anc1', data={'person_id': mother_id, 'ga_anc_one': ga_anc_one}, description='A dictionary containing the number of ANC visits each woman has on birth') def on_hsi_alert(self, person_id, treatment_id): diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index d84c41c1e0..3d97aa23dd 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1056,9 +1056,7 @@ def further_on_birth_labour(self, mother_id): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info # log delivery setting - logger.info(key='delivery_setting_and_mode', data={'mother': mother_id, - 'facility_type': str(mni[mother_id]['delivery_setting']), - 'mode': mni[mother_id]['mode_of_delivery']}) + logger.info(key='delivery_mode', data={'mother': mother_id, 'mode': mni[mother_id]['mode_of_delivery']}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ f'{str(mni[mother_id]["delivery_setting"])}_delivery'] += 1 @@ -1285,10 +1283,6 @@ def set_intrapartum_complications(self, individual_id, complication): pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'obstructed_labour_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{complication}', - 'timing': 'intrapartum'}) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[complication] += 1 if complication == 'obstruction_cpd': @@ -1298,11 +1292,8 @@ def set_intrapartum_complications(self, individual_id, complication): # to labour) elif complication == 'placental_abruption': df.at[individual_id, 'la_placental_abruption'] = True - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'placental_abruption', - 'timing': 'intrapartum'}) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['placental_abruption'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['placental_abruption'] += 1 elif complication == 'antepartum_haem': random_choice = self.rng.choice(['mild_moderate', 'severe'], @@ -1312,18 +1303,13 @@ def set_intrapartum_complications(self, individual_id, complication): if random_choice != 'severe': pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'mild_mod_aph_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'mild_mod_antepartum_haemorrhage', - 'timing': 'intrapartum'}) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += 1 else: pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'severe_aph_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'severe_antepartum_haemorrhage', - 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_antepartum_haemorrhage'] += 1 elif complication == 'sepsis_chorioamnionitis': @@ -1331,18 +1317,14 @@ def set_intrapartum_complications(self, individual_id, complication): mni[individual_id]['chorio_in_preg'] = True pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'sepsis_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'sepsis', - 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_intrapartum'] += 1 elif complication == 'uterine_rupture': df.at[individual_id, 'la_uterine_rupture'] = True pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, f'{complication}_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'uterine_rupture', - 'timing': 'intrapartum'}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['uterine_rupture'] += 1 def set_postpartum_complications(self, individual_id, complication): @@ -1396,9 +1378,6 @@ def set_postpartum_complications(self, individual_id, complication): # Set primary complication to true df.at[individual_id, 'la_postpartum_haem'] = True - logger_pn.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{complication}', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[complication] += 1 # Store mni variables used during treatment @@ -2439,10 +2418,6 @@ def apply(self, individual_id): 'defining_term_status'][3]: mni[individual_id]['labour_state'] = 'early_preterm_labour' - - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'early_preterm_labour', - 'timing': 'intrapartum'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_preterm_labour'] += 1 elif params['list_limits_for_defining_term_status'][4] <= gestational_age_in_days <= params['list_limits' @@ -2451,19 +2426,11 @@ def apply(self, individual_id): 'status'][5]: mni[individual_id]['labour_state'] = 'late_preterm_labour' - - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'late_preterm_labour', - 'timing': 'intrapartum'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['late_preterm_labour'] += 1 elif gestational_age_in_days >= params['list_limits_for_defining_term_status'][6]: mni[individual_id]['labour_state'] = 'postterm_labour' - - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'post_term_labour', - 'timing': 'intrapartum'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['post_term_labour'] += 1 labour_state = mni[individual_id]['labour_state'] @@ -2631,9 +2598,6 @@ def apply(self, individual_id): self.module.set_intrapartum_complications(individual_id, complication=complication) if df.at[individual_id, 'la_obstructed_labour']: - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'obstructed_labour', - 'timing': 'intrapartum'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['obstructed_labour'] += 1 # And we determine if any existing hypertensive disorders would worsen @@ -2750,8 +2714,6 @@ def apply(self, individual_id): del mni[individual_id] if df.at[individual_id, 'la_intrapartum_still_birth'] or mni[individual_id]['single_twin_still_birth']: - logger.info(key='intrapartum_stillbirth', data={'mother_id': individual_id, - 'date_of_ip_stillbirth': self.sim.date}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['intrapartum_stillbirth'] += 1 # Reset property @@ -2840,15 +2802,9 @@ def apply(self, mother_id): self.module.set_postpartum_complications(mother_id, complication=complication) if df.at[mother_id, 'la_sepsis_pp']: - logger_pn.info(key='maternal_complication', data={'person': mother_id, - 'type': 'sepsis_postnatal', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 if df.at[mother_id, 'la_postpartum_haem']: - logger_pn.info(key='maternal_complication', data={'person': mother_id, - 'type': 'primary_postpartum_haemorrhage', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['primary_postpartum_haemorrhage'] += 1 self.module.progression_of_hypertensive_disorders(mother_id, property_prefix='pn') @@ -3002,9 +2958,6 @@ def apply(self, person_id, squeeze_factor): self.module.progression_of_hypertensive_disorders(person_id, property_prefix='ps') if df.at[person_id, 'la_obstructed_labour']: - logger.info(key='maternal_complication', data={'person': person_id, - 'type': 'obstructed_labour', - 'timing': 'intrapartum'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['obstructed_labour'] += 1 # ======================================= COMPLICATION MANAGEMENT ========================== diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 4019544bd4..e8f32ed7af 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -523,32 +523,22 @@ def apply_risk_of_congenital_anomaly(self, child_id): if self.rng.random_sample() < params['prob_congenital_heart_anomaly']: self.congeintal_anomalies.set(child_id, 'heart') - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'congenital_heart_anomaly'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['congenital_heart_anomaly'] += 1 if self.rng.random_sample() < params['prob_limb_musc_skeletal_anomaly']: self.congeintal_anomalies.set(child_id, 'limb_musc_skeletal') - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'limb_or_musculoskeletal_anomaly'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['limb_or_musculoskeletal_anomaly'] += 1 if self.rng.random_sample() < params['prob_urogenital_anomaly']: self.congeintal_anomalies.set(child_id, 'urogenital') - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'urogenital_anomaly'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['urogenital_anomaly'] += 1 if self.rng.random_sample() < params['prob_digestive_anomaly']: self.congeintal_anomalies.set(child_id, 'digestive') - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'digestive_anomaly'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['digestive_anomaly'] += 1 if self.rng.random_sample() < params['prob_other_anomaly']: self.congeintal_anomalies.set(child_id, 'other') - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'other_anomaly'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['other_anomaly'] += 1 def apply_risk_of_neonatal_infection_and_sepsis(self, child_id): @@ -563,9 +553,6 @@ def apply_risk_of_neonatal_infection_and_sepsis(self, child_id): # The linear model calculates the individuals probability of early_onset_neonatal_sepsis if self.eval(self.nb_linear_models['early_onset_neonatal_sepsis'], child_id): df.at[child_id, 'nb_early_onset_neonatal_sepsis'] = True - - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'early_onset_sepsis'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_onset_sepsis'] += 1 def apply_risk_of_encephalopathy(self, child_id, timing): @@ -598,8 +585,6 @@ def apply_risk_of_encephalopathy(self, child_id, timing): else: df.at[child_id, 'nb_encephalopathy'] = 'severe_enceph' - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': f'{df.at[child_id, "nb_encephalopathy"]}'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{df.at[child_id, "nb_encephalopathy"]}'] += 1 # Check all encephalopathy cases receive a grade @@ -623,9 +608,6 @@ def apply_risk_of_preterm_respiratory_distress_syndrome(self, child_id): # Use the linear model to calculate individual risk and make changes if self.eval(self.nb_linear_models['rds_preterm'], child_id): df.at[child_id, 'nb_preterm_respiratory_distress'] = True - - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'respiratory_distress_syndrome'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['respiratory_distress_syndrome'] += 1 def apply_risk_of_not_breathing_at_birth(self, child_id): @@ -646,9 +628,6 @@ def apply_risk_of_not_breathing_at_birth(self, child_id): # explicitly modelled elif self.rng.random_sample() < params['prob_failure_to_transition']: df.at[child_id, 'nb_not_breathing_at_birth'] = True - - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'not_breathing_at_birth'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['not_breathing_at_birth'] += 1 def scheduled_week_one_postnatal_event(self, individual_id): @@ -1167,16 +1146,13 @@ def on_birth(self, mother_id, child_id): if (df.at[child_id, 'nb_low_birth_weight_status'] == 'low_birth_weight') or\ (df.at[child_id, 'nb_low_birth_weight_status'] == 'very_low_birth_weight') or\ (df.at[child_id, 'nb_low_birth_weight_status'] == 'extremely_low_birth_weight'): - logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'low_birth_weight'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['low_birth_weight'] += 1 elif df.at[child_id, 'nb_low_birth_weight_status'] == 'macrosomia': - logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'macrosomia'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['macrosomia'] += 1 df.at[child_id, 'nb_size_for_gestational_age'] = mni[mother_id]['birth_size'] if df.at[child_id, 'nb_size_for_gestational_age'] == 'small_for_gestational_age': - logger.info(key='newborn_complication', data={'newborn': child_id, 'type': 'small_for_gestational_age'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['small_for_gestational_age'] += 1 df.at[child_id, 'nb_early_init_breastfeeding'] = False diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 5ba48a0e24..34c31fca9e 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -354,10 +354,6 @@ def further_on_birth_postnatal_supervisor(self, mother_id): # Store the onset weight for daly calculations store_dalys_in_mni(mother_id, mni, f'{fistula_type}_fistula_onset', self.sim.date) - - logger.info(key='maternal_complication', data={'person': mother_id, - 'type': f'{fistula_type}_fistula', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{fistula_type}_fistula'] += 1 # Determine if she will seek care for repair @@ -501,9 +497,6 @@ def onset(eq): for person in new_sepsis.loc[new_sepsis].index: store_dalys_in_mni(person, mni, 'sepsis_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': person, - 'type': 'sepsis', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 # ------------------------------------ SECONDARY PPH ---------------------------------------------------------- @@ -516,10 +509,6 @@ def onset(eq): for person in onset_pph.loc[onset_pph].index: store_dalys_in_mni(person, mni, 'secondary_pph_onset', self.sim.date) - - logger.info(key='maternal_complication', data={'person': person, - 'type': 'secondary_postpartum_haemorrhage', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['secondary_postpartum_haemorrhage'] += 1 # --------------------------------------------- ANAEMIA -------------------------------------------------- @@ -597,9 +586,6 @@ def log_new_progressed_cases(disease): mni=mni, mni_variable='eclampsia_onset', date=self.sim.date) for person in new_onset_disease.index: - logger.info(key='maternal_complication', data={'person': person, - 'type': disease, - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[disease] += 1 if disease == 'severe_pre_eclamp': @@ -641,12 +627,8 @@ def log_new_progressed_cases(disease): df.loc[pre_eclampsia.loc[pre_eclampsia].index, 'ps_prev_pre_eclamp'] = True df.loc[pre_eclampsia.loc[pre_eclampsia].index, 'pn_htn_disorders'] = 'mild_pre_eclamp' - - for person in pre_eclampsia.loc[pre_eclampsia].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'mild_pre_eclamp', - 'timing': 'postnatal'}) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_pre_eclamp'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ + 'mild_pre_eclamp'] += (len(pre_eclampsia.loc[pre_eclampsia].index)) # -------------------------------- RISK OF GESTATIONAL HYPERTENSION -------------------------------------- gest_hypertension = self.apply_linear_model( @@ -655,12 +637,8 @@ def log_new_progressed_cases(disease): (df['pn_htn_disorders'] == 'none')]) df.loc[gest_hypertension.loc[gest_hypertension].index, 'pn_htn_disorders'] = 'gest_htn' - - for person in gest_hypertension.loc[gest_hypertension].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'mild_gest_htn', - 'timing': 'postnatal'}) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[ + 'mild_gest_htn'] += len(gest_hypertension.loc[gest_hypertension].index) # -------------------------------- RISK OF DEATH SEVERE HYPERTENSION ------------------------------------------ # Risk of death is applied to women with severe hypertensive disease @@ -750,9 +728,6 @@ def apply_risk_of_neonatal_complications_in_week_one(self, child_id, mother_id): if self.rng.random_sample() < risk_eons: df.at[child_id, 'pn_sepsis_early_neonatal'] = True self.sim.modules['NewbornOutcomes'].newborn_care_info[child_id]['sepsis_postnatal'] = True - - logger.info(key='newborn_complication', data={'newborn': child_id, - 'type': 'early_onset_sepsis'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['early_onset_sepsis'] += 1 def set_postnatal_complications_neonates(self, upper_and_lower_day_limits): @@ -778,9 +753,6 @@ def set_postnatal_complications_neonates(self, upper_and_lower_day_limits): for person in onset_sepsis.loc[onset_sepsis].index: if person in nci: nci[person]['sepsis_postnatal'] = True - - logger.info(key='newborn_complication', data={'newborn': person, - 'type': 'late_onset_sepsis'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['late_onset_sepsis'] += 1 # Then we determine if care will be sought for newly septic newborns @@ -1057,9 +1029,6 @@ def apply(self, individual_id): if endo_result or ut_result or ssti_result: df.at[individual_id, 'pn_sepsis_late_postpartum'] = True store_dalys_in_mni(individual_id, mni, 'sepsis_onset', self.sim.date) - - logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'sepsis', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['sepsis_postnatal'] += 1 # Sepsis secondary to endometritis is stored within the mni as it is used as a predictor in a linear model @@ -1076,10 +1045,6 @@ def apply(self, individual_id): if risk_secondary_pph > self.module.rng.random_sample(): df.at[individual_id, 'pn_postpartum_haem_secondary'] = True store_dalys_in_mni(individual_id, mni, 'secondary_pph_onset', self.sim.date) - - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'secondary_postpartum_haemorrhage', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['secondary_postpartum_haemorrhage'] += 1 # ------------------------------------------------ NEW ONSET ANAEMIA ------------------------------------------ @@ -1146,9 +1111,6 @@ def log_new_progressed_cases(disease): if not new_onset_disease.empty: for person in new_onset_disease.index: - logger.info(key='maternal_complication', data={'person': person, - 'type': disease, - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[disease] += 1 if disease == 'severe_pre_eclamp': @@ -1170,9 +1132,6 @@ def log_new_progressed_cases(disease): if risk_pe_after_pregnancy > self.module.rng.random_sample(): df.at[individual_id, 'pn_htn_disorders'] = 'mild_pre_eclamp' df.at[individual_id, 'ps_prev_pre_eclamp'] = True - - logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'mild_pre_eclamp', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_pre_eclamp'] += 1 else: @@ -1180,10 +1139,6 @@ def log_new_progressed_cases(disease): individual_id]])[individual_id] if risk_gh_after_pregnancy > self.module.rng.random_sample(): df.at[individual_id, 'pn_htn_disorders'] = 'gest_htn' - - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'mild_gest_htn', - 'timing': 'postnatal'}) self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 # ====================================== POSTNATAL CHECK ================================================== diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 3353f86fc8..446ad0a433 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -8,6 +8,7 @@ from tlo import logging + def generate_mnh_outcome_counter(): outcome_list = [ # early/abortive outcomes 'ectopic_unruptured', 'ectopic_ruptured','multiple_pregnancy', 'twin_birth', 'placenta_praevia', diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 520474222d..ff3295a5c2 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -661,11 +661,6 @@ def further_on_birth_pregnancy_supervisor(self, mother_id): if df.at[mother_id, 'ps_anaemia_in_pregnancy'] != 'none': self.mnh_outcome_counter[f'an_anaemia_{df.at[mother_id, "ps_anaemia_in_pregnancy"]}'] += 1 - logger.info(key='conditions_on_birth', data={'mother': mother_id, - 'anaemia_status': df.at[mother_id, 'ps_anaemia_in_pregnancy'], - 'gdm_status': df.at[mother_id, 'ps_gest_diab'], - 'htn_status': df.at[mother_id, 'ps_htn_disorders']}) - # We currently assume that hyperglycemia due to gestational diabetes resolves following birth if df.at[mother_id, 'ps_gest_diab'] != 'none': df.at[mother_id, 'ps_gest_diab'] = 'none' @@ -974,9 +969,6 @@ def do_after_abortion(self, individual_id, type_abortion): params = self.current_parameters # Log the pregnancy loss - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{type_abortion}', - 'timing': 'antenatal'}) self.mnh_outcome_counter[type_abortion] += 1 # This function officially ends a pregnancy through the contraception module (updates 'is_pregnant' and @@ -1004,10 +996,6 @@ def do_after_abortion(self, individual_id, type_abortion): risk_of_complications = params['prob_complicated_ia'] if self.rng.random_sample() < risk_of_complications: - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'complicated_{type_abortion}', - 'timing': 'antenatal'}) - self.mnh_outcome_counter[f'complicated_{type_abortion}'] += 1 self.apply_risk_of_abortion_complications(individual_id, f'{type_abortion}') @@ -1026,34 +1014,22 @@ def apply_risk_of_abortion_complications(self, individual_id, cause): if cause == 'induced_abortion': if self.rng.random_sample() < params['prob_injury_post_abortion']: self.abortion_complications.set([individual_id], 'injury') - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{cause}_injury', - 'timing': 'antenatal'}) self.mnh_outcome_counter[f'{cause}_injury'] += 1 if self.rng.random_sample() < params['prob_haemorrhage_post_abortion']: self.abortion_complications.set([individual_id], 'haemorrhage') pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'abortion_haem_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{cause}_haemorrhage', - 'timing': 'antenatal'}) self.mnh_outcome_counter[f'{cause}_haemorrhage'] += 1 if self.rng.random_sample() < params['prob_sepsis_post_abortion']: self.abortion_complications.set([individual_id], 'sepsis') pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'abortion_sep_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{cause}_sepsis', - 'timing': 'antenatal'}) self.mnh_outcome_counter[f'{cause}_sepsis'] += 1 if not self.abortion_complications.has_any([individual_id], 'sepsis', 'haemorrhage', 'injury', first=True): self.abortion_complications.set([individual_id], 'other') - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': f'{cause}_other_comp', - 'timing': 'antenatal'}) self.mnh_outcome_counter[f'{cause}_other_comp'] += 1 # We assume only women with complicated abortions will experience disability @@ -1113,11 +1089,7 @@ def apply_risk_of_gestational_diabetes(self, gestation_of_interest): df.loc[gest_diab.loc[gest_diab].index, 'ps_gest_diab'] = 'uncontrolled' df.loc[gest_diab.loc[gest_diab].index, 'ps_prev_gest_diab'] = True - for person in gest_diab.loc[gest_diab].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'gest_diab', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['gest_diab'] += 1 + self.mnh_outcome_counter['gest_diab'] += len(gest_diab.loc[gest_diab].index) def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): @@ -1139,11 +1111,7 @@ def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): df.loc[pre_eclampsia.loc[pre_eclampsia].index, 'ps_prev_pre_eclamp'] = True df.loc[pre_eclampsia.loc[pre_eclampsia].index, 'ps_htn_disorders'] = 'mild_pre_eclamp' - for person in pre_eclampsia.loc[pre_eclampsia].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'mild_pre_eclamp', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['mild_pre_eclamp'] += 1 + self.mnh_outcome_counter['mild_pre_eclamp'] += len(pre_eclampsia.loc[pre_eclampsia].index) # -------------------------------- RISK OF GESTATIONAL HYPERTENSION -------------------------------------- # For women who dont develop pre-eclampsia during this month, we apply a risk of gestational hypertension @@ -1154,11 +1122,7 @@ def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): df.loc[gest_hypertension.loc[gest_hypertension].index, 'ps_htn_disorders'] = 'gest_htn' - for person in gest_hypertension.loc[gest_hypertension].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'mild_gest_htn', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['mild_gest_htn'] += 1 + self.mnh_outcome_counter['mild_gest_htn'] += len(gest_hypertension.loc[gest_hypertension].inde) def apply_risk_of_progression_of_hypertension(self, gestation_of_interest): """ @@ -1213,9 +1177,6 @@ def log_new_progressed_cases(disease): # And log all of the new onset cases of any hypertensive disease for person in new_onset_disease.index: - logger.info(key='maternal_complication', data={'person': person, - 'type': disease, - 'timing': 'antenatal'}) self.mnh_outcome_counter[disease] +=1 if disease == 'severe_pre_eclamp': @@ -1289,11 +1250,7 @@ def apply_risk_of_placental_abruption(self, gestation_of_interest): ~df['la_currently_in_labour']]) df.loc[placenta_abruption.loc[placenta_abruption].index, 'ps_placental_abruption'] = True - for person in placenta_abruption.loc[placenta_abruption].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'placental_abruption', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['placental_abruption'] += 1 + self.mnh_outcome_counter['placental_abruption'] += len(placenta_abruption.loc[placenta_abruption].index) def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): """ @@ -1331,11 +1288,7 @@ def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): severe_women.loc[severe_women].index.to_series().apply( pregnancy_helper_functions.store_dalys_in_mni, mni=mni, mni_variable='severe_aph_onset', date=self.sim.date) - for person in severe_women.loc[severe_women].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'severe_antepartum_haemorrhage', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['severe_antepartum_haemorrhage'] += 1 + self.mnh_outcome_counter['severe_antepartum_haemorrhage'] += len(severe_women.loc[severe_women].index) non_severe_women = (df.loc[antepartum_haemorrhage.loc[antepartum_haemorrhage].index, 'ps_antepartum_haemorrhage'] != 'severe') @@ -1344,12 +1297,7 @@ def apply_risk_of_antepartum_haemorrhage(self, gestation_of_interest): pregnancy_helper_functions.store_dalys_in_mni, mni=mni, mni_variable='mild_mod_aph_onset', date=self.sim.date) - for person in non_severe_women.loc[non_severe_women].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'mild_mod_antepartum_haemorrhage', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += 1 - + self.mnh_outcome_counter['mild_mod_antepartum_haemorrhage'] += len(non_severe_women.loc[non_severe_women].index) def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gestation_of_interest): """ @@ -1373,12 +1321,7 @@ def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gest # We allow women to seek care for PROM df.loc[prom.loc[prom].index, 'ps_emergency_event'] = True - - for person in prom.loc[prom].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'PROM', - 'timing': 'antenatal'}) - self.mnh_outcome_counter['PROM'] += 1 + self.mnh_outcome_counter['PROM'] += len(prom.loc[prom].index) # Determine if those with PROM will develop infection prior to care seeking infection = pd.Series(self.rng.random_sample(len(prom.loc[prom])) < params['prob_chorioamnionitis'], @@ -1391,9 +1334,6 @@ def apply_risk_of_premature_rupture_of_membranes_and_chorioamnionitis(self, gest for person in infection.loc[infection].index: self.mother_and_newborn_info[person]['chorio_in_preg'] = True - logger.info(key='maternal_complication', data={'person': person, - 'type': 'clinical_chorioamnionitis', - 'timing': 'antenatal'}) self.mnh_outcome_counter['clinical_chorioamnionitis'] += 1 def apply_risk_of_preterm_labour(self, gestation_of_interest): @@ -1458,7 +1398,6 @@ def update_variables_post_still_birth_for_data_frame(self, women): for person in women.index: self.sim.modules['Contraception'].end_pregnancy(person) mni[person]['delete_mni'] = True - logger.info(key='antenatal_stillbirth', data={'mother': person}) self.mnh_outcome_counter['antenatal_stillbirth'] += 1 # Call functions across the modules to ensure properties are rest @@ -1481,7 +1420,6 @@ def update_variables_post_still_birth_for_individual(self, individual_id): df.at[individual_id, 'ps_prev_stillbirth'] = True mni[individual_id]['delete_mni'] = True - logger.info(key='antenatal_stillbirth', data={'mother': individual_id}) self.mnh_outcome_counter['antenatal_stillbirth'] += 1 # Reset pregnancy and schedule possible update of contraception @@ -1794,9 +1732,6 @@ def apply(self, population): # For women whose pregnancy is ectopic we scheduled them to the EctopicPregnancyEvent in between 3-5 weeks # (this simulates time period prior to which symptoms onset- and may trigger care seeking) for person in ectopic_risk.loc[ectopic_risk].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'ectopic_unruptured', - 'timing': 'antenatal'}) self.module.mnh_outcome_counter['ectopic_unruptured'] += 1 self.sim.schedule_event(EctopicPregnancyEvent(self.module, person), @@ -1812,12 +1747,7 @@ def apply(self, population): < params['prob_multiples'], index=multiple_risk.loc[multiple_risk].index) df.loc[multiples.loc[multiples].index, 'ps_multiple_pregnancy'] = True - - for person in multiples.loc[multiples].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'multiple_pregnancy', - 'timing': 'antenatal'}) - self.module.mnh_outcome_counter['multiple_pregnancy'] += 1 + self.module.mnh_outcome_counter['multiple_pregnancy'] += len(multiples.loc[multiples].index) # -----------------------------APPLYING RISK OF PLACENTA PRAEVIA ------------------------------------------- # Next,we apply a one off risk of placenta praevia (placenta will grow to cover the cervix either partially or @@ -1827,12 +1757,7 @@ def apply(self, population): df.loc[new_pregnancy & (df['ps_ectopic_pregnancy'] == 'none')]) df.loc[placenta_praevia.loc[placenta_praevia].index, 'ps_placenta_praevia'] = True - - for person in placenta_praevia.loc[placenta_praevia].index: - logger.info(key='maternal_complication', data={'person': person, - 'type': 'placenta_praevia', - 'timing': 'antenatal'}) - self.module.mnh_outcome_counter['placenta_praevia'] += 1 + self.module.mnh_outcome_counter['placenta_praevia'] += len(placenta_praevia.loc[placenta_praevia].index) # ------------------------- APPLYING RISK OF SYPHILIS INFECTION DURING PREGNANCY --------------------------- # Finally apply risk that syphilis will develop during pregnancy @@ -2016,9 +1941,6 @@ def apply(self, individual_id): if not df.at[individual_id, 'is_alive'] or (df.at[individual_id, 'ps_ectopic_pregnancy'] != 'not_ruptured'): return - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'ectopic_ruptured', - 'timing': 'antenatal'}) self.module.mnh_outcome_counter['ectopic_ruptured'] += 1 # Set the variable @@ -2127,9 +2049,7 @@ def apply(self, individual_id): return df.at[individual_id, 'ps_syphilis'] = True - logger.info(key='maternal_complication', data={'person': individual_id, - 'type': 'syphilis', - 'timing': 'antenatal'}) + self.module.mnh_outcome_counter['syphilis'] += 1 class ParameterUpdateEvent(Event, PopulationScopeEventMixin): @@ -2371,7 +2291,6 @@ def rate (count, denom, multiplier): 'm_pnc1+': rate(m_pnc1, total_births, 1000), 'n_pnc1+': rate(n_pnc1, total_births, 1000)}) - mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() outcome_list = mnh_oc['outcomes'] self.module.mnh_outcome_counter = {k:0 for k in outcome_list} From 905091b6ed1ad32f25bf9e454b3af116821cb6c2 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 13:35:41 +0100 Subject: [PATCH 027/103] fix --- src/tlo/methods/pregnancy_supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index ff3295a5c2..20c91fe4ef 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -1122,7 +1122,7 @@ def apply_risk_of_hypertensive_disorders(self, gestation_of_interest): df.loc[gest_hypertension.loc[gest_hypertension].index, 'ps_htn_disorders'] = 'gest_htn' - self.mnh_outcome_counter['mild_gest_htn'] += len(gest_hypertension.loc[gest_hypertension].inde) + self.mnh_outcome_counter['mild_gest_htn'] += len(gest_hypertension.loc[gest_hypertension].index) def apply_risk_of_progression_of_hypertension(self, gestation_of_interest): """ From 5a04b444bb38c1c39b46ca462c6a9b6333e7d195 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 14:06:34 +0100 Subject: [PATCH 028/103] fix --- src/tlo/methods/pregnancy_supervisor.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 20c91fe4ef..4185bd935a 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2190,11 +2190,6 @@ def apply(self, population): df = self.sim.population.props c = self.module.mnh_outcome_counter - logger.info(key='yrly_counter_dict', data=c) - - def rate (count, denom, multiplier): - return (count/denom) * multiplier - # DENOMINATORS live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) @@ -2208,7 +2203,15 @@ def rate (count, denom, multiplier): total_births = live_births + c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] + if (live_births == 0) or (pregnancies == 0): + return + # MATERNAL COMPLICATION INCIDENCE + logger.info(key='yearly_counter_dict', data=c) + + def rate (count, denom, multiplier): + return (count/denom) * multiplier + total_an_anaemia_cases = c['an_anaemia_mild'] + c['an_anaemia_moderate'] + c['an_anaemia_severe'] total_pn_anaemia_cases = c['pn_anaemia_mild'] + c['pn_anaemia_moderate'] + c['pn_anaemia_severe'] total_preterm_birth = c['early_preterm_labour'] + c['late_preterm_labour'] From c62d5df215023617e8acc4adbf2951f61313c196 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 14:26:45 +0100 Subject: [PATCH 029/103] fix --- src/tlo/methods/pregnancy_supervisor.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 4185bd935a..408b5c7847 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2192,19 +2192,17 @@ def apply(self, population): # DENOMINATORS live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) - pregnancies =len(df[df['date_of_last_pregnancy'].dt.year == self.sim.date.year - 1]) - comp_pregnancies = (c['ectopic_unruptured'] + c['spontaneous_abortion'] + c['induced_abortion'] + c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] + live_births) - deliveries = live_births - c['twin_birth'] - total_births = live_births + c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] + total_preterm_birth = c['early_preterm_labour'] + c['late_preterm_labour'] - if (live_births == 0) or (pregnancies == 0): - return + for denom in [live_births, pregnancies, comp_pregnancies, deliveries, total_births, total_preterm_birth]: + if denom == 0: + return # MATERNAL COMPLICATION INCIDENCE logger.info(key='yearly_counter_dict', data=c) @@ -2214,7 +2212,6 @@ def rate (count, denom, multiplier): total_an_anaemia_cases = c['an_anaemia_mild'] + c['an_anaemia_moderate'] + c['an_anaemia_severe'] total_pn_anaemia_cases = c['pn_anaemia_mild'] + c['pn_anaemia_moderate'] + c['pn_anaemia_severe'] - total_preterm_birth = c['early_preterm_labour'] + c['late_preterm_labour'] total_aph = c['mild_mod_antepartum_haemorrhage'] + c['severe_antepartum_haemorrhage'] total_sepsis = c['clinical_chorioamnionitis'] + c['sepsis_intrapartum'] + c['sepsis_postnatal'] total_pph = c['primary_postpartum_haemorrhage'] + c['secondary_postpartum_haemorrhage'] From d8423e1b93130e6af2935306e11f09ab5e6cdf56 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 14:32:11 +0100 Subject: [PATCH 030/103] comments for clarity --- src/tlo/methods/pregnancy_supervisor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 408b5c7847..4f2e05dfcb 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2191,6 +2191,7 @@ def apply(self, population): c = self.module.mnh_outcome_counter # DENOMINATORS + # Define denominators used to calculate rates, cancel the event if any are 0 to prevent division by 0 errors live_births = len(df[(df['date_of_birth'].dt.year == self.sim.date.year - 1) & (df['mother_id'] >= 0)]) pregnancies =len(df[df['date_of_last_pregnancy'].dt.year == self.sim.date.year - 1]) comp_pregnancies = (c['ectopic_unruptured'] + c['spontaneous_abortion'] + @@ -2205,8 +2206,10 @@ def apply(self, population): return # MATERNAL COMPLICATION INCIDENCE - logger.info(key='yearly_counter_dict', data=c) + # Log the yearly dictionary (allows for analyses with outcomes not used in this event) + logger.info(key='yearly_mnh_counter_dict', data=c) + # Calculate and store rates of key maternal and neonatal complications def rate (count, denom, multiplier): return (count/denom) * multiplier @@ -2257,6 +2260,7 @@ def rate (count, denom, multiplier): 'nb_sga': rate(c['small_for_gestational_age'], live_births, 100)}) # DIRECT MATERNAL DEATHS, NEWBORN DEATHS AND STILLBIRTH + # Calculate and store rates of maternal and newborn death and stillbirth neonatal_deaths = len(df[(df['date_of_death'].dt.year == self.sim.date.year - 1) & (df['age_days'] <= 28)]) stillbirths = c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] @@ -2270,6 +2274,7 @@ def rate (count, denom, multiplier): 'direct_maternal_deaths': c['direct_mat_death'], 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000)}) + # Finally log coverage of key health services anc1 = sum(c[f'anc{i}'] for i in range(1, 9)) + c['anc8+'] anc4 = sum(c[f'anc{i}'] for i in range(4, 9))+ c['anc8+'] anc8 = c['anc8'] + c['anc8+'] @@ -2291,6 +2296,7 @@ def rate (count, denom, multiplier): 'm_pnc1+': rate(m_pnc1, total_births, 1000), 'n_pnc1+': rate(n_pnc1, total_births, 1000)}) + # Reset the dictionary so all values = 0 mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() outcome_list = mnh_oc['outcomes'] self.module.mnh_outcome_counter = {k:0 for k in outcome_list} From cd3bd347a3b21f20aa3fa9499a804118d8df0c47 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 14:58:32 +0100 Subject: [PATCH 031/103] improved commenting --- .../cohort_interventions_scenario.py | 2 +- src/tlo/methods/mnh_cohort_module.py | 61 ++++--------------- 2 files changed, 14 insertions(+), 49 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 30b5ebfc09..11cac4deb7 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -9,7 +9,7 @@ def __init__(self): super().__init__() self.seed = 537184 self.start_date = Date(2024, 1, 1) - self.end_date = Date(2025, 1, 1) + self.end_date = Date(2025, 1, 2) self.pop_size = 5000 self.number_of_draws = 1 self.runs_per_draw = 10 diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index f5e19c6767..582855df5d 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -11,47 +11,30 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -# --------------------------------------------------------------------------------------------------------- -# MODULE DEFINITIONS -# --------------------------------------------------------------------------------------------------------- - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - class MaternalNewbornHealthCohort(Module): """ - + When registered this module overrides the population data frame with a cohort of pregnant women. Cohort properties + are sourced from a long run of the full model in which the properties of all newly pregnant women per year were + logged. The cohort represents women in 2024. The maximum population size is 13,000. """ - # INIT_DEPENDENCIES = {'Demography'} - # - # OPTIONAL_INIT_DEPENDENCIES = {''} - # - # ADDITIONAL_DEPENDENCIES = {''} - - # Declare Metadata (this is for a typical 'Disease Module') METADATA = { Metadata.DISEASE_MODULE, Metadata.USES_SYMPTOMMANAGER, Metadata.USES_HEALTHSYSTEM, Metadata.USES_HEALTHBURDEN } + CAUSES_OF_DEATH = {} CAUSES_OF_DISABILITY = {} PARAMETERS = {} PROPERTIES = {} def __init__(self, name=None, resourcefilepath=None): - # NB. Parameters passed to the module can be inserted in the __init__ definition. - super().__init__(name) self.resourcefilepath = resourcefilepath def read_parameters(self, data_folder): - """Read parameter values from file, if required. - To access files use: Path(self.resourcefilepath) / file_name - """ pass def initialise_population(self, population): @@ -73,23 +56,28 @@ def initialise_population(self, population): # log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) # all_pregnancies.index = [x for x in range(len(all_pregnancies))] + # Read in excel sheet with cohort all_preg_df = pd.read_excel(Path(f'{self.resourcefilepath}/maternal cohort') / 'ResourceFile_All2024PregnanciesCohortModel.xlsx') + + # Only select rows equal to the desired population size preg_pop = all_preg_df.loc[0:(len(self.sim.population.props))-1] + # Set the dtypes and index of the cohort dataframe props_dtypes = self.sim.population.props.dtypes - preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) preg_pop_final.index.name = 'person' - + # For the below columns we manually overwrite the dtypes for column in ['rt_injuries_for_minor_surgery', 'rt_injuries_for_major_surgery', 'rt_injuries_to_heal_with_time', 'rt_injuries_for_open_fracture_treatment', 'rt_injuries_left_untreated', 'rt_injuries_to_cast']: preg_pop_final[column] = [[] for _ in range(len(preg_pop_final))] + # Set the population.props dataframe to the new cohort self.sim.population.props = preg_pop_final + # Update key pregnancy properties df = self.sim.population.props population = df.loc[df.is_alive] df.loc[population.index, 'date_of_last_pregnancy'] = self.sim.start_date @@ -109,13 +97,12 @@ def initialise_simulation(self, sim): # Clear HSI queue for events scheduled during initialisation sim.modules['HealthSystem'].HSI_EVENT_QUEUE.clear() - # Clear HSI queue for events scheduled during initialisation + # Clear the individual event queue for events scheduled during initialisation updated_event_queue = [item for item in self.sim.event_queue.queue if not isinstance(item[3], IndividualScopeEventMixin)] - self.sim.event_queue.queue = updated_event_queue - # Prevent additional pregnancies from occurring + # Prevent additional pregnancies from occurring during the cohort tun self.sim.modules['Contraception'].processed_params['p_pregnancy_with_contraception_per_month'].iloc[:] = 0 self.sim.modules['Contraception'].processed_params['p_pregnancy_no_contraception_per_month'].iloc[:] = 0 @@ -124,32 +111,10 @@ def initialise_simulation(self, sim): self.sim.modules['Labour'].set_date_of_labour(person) def on_birth(self, mother_id, child_id): - """Initialise our properties for a newborn individual. - - This is called by the simulation whenever a new person is born. - - :param mother_id: the mother for this child - :param child_id: the new child - """ pass def report_daly_values(self): - """ - This must send back a pd.Series or pd.DataFrame that reports on the average daly-weights that have been - experienced by persons in the previous month. Only rows for alive-persons must be returned. - If multiple causes in CAUSES_OF_DISABILITY are defined, a pd.DataFrame must be returned with a column - corresponding to each cause (but if only one cause in CAUSES_OF_DISABILITY is defined, the pd.Series does not - need to be given a specific name). - - To return a value of 0.0 (fully health) for everyone, use: - df = self.sim.population.props - return pd.Series(index=df.index[df.is_alive],data=0.0) - """ pass def on_hsi_alert(self, person_id, treatment_id): - """ - This is called whenever there is an HSI event commissioned by one of the other disease modules. - """ - pass From a6b12fe5d200c736043150d81a3097835cc18763 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 8 Oct 2024 15:33:09 +0100 Subject: [PATCH 032/103] removed counting which wasnt needed --- src/tlo/methods/pregnancy_helper_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 446ad0a433..3ae5b5b264 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -393,7 +393,6 @@ def calculate_risk_of_death_from_causes(self, risks): cause_of_death = self.rng.choice(list(risks.keys()), p=probs) # Return the primary cause of death so that it can be passed to the demography function - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause_of_death}_death'] += 1 return cause_of_death else: # Return false if death will not occur From fceee02e68722e29314c3d9efe35983709a78deb Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 9 Oct 2024 09:27:54 +0100 Subject: [PATCH 033/103] Change syntax of if statement and print string of event --- src/tlo/events.py | 6 +++--- src/tlo/methods/hsi_event.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 2eef87ba3f..2a7871c2c8 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -71,12 +71,12 @@ def run(self): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersect(str(self)): + if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): print_chains = True if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target - row['event'] = self + row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) @@ -90,7 +90,7 @@ def run(self): if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target - row['event'] = self + row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'After' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 805c9584fb..ea9066bc8b 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -196,12 +196,12 @@ def run(self, squeeze_factor): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersect(str(self)): + if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): print_chains = True if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target - row['event'] = self + row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) @@ -216,7 +216,7 @@ def run(self, squeeze_factor): if self.target != self.sim.population: row = self.sim.population.props.loc[[self.target]] row['person_ID'] = self.target - row['event'] = self + row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'After' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) From 5ca2314b4eb04cb14909fe835ab4e98f87688668 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 9 Oct 2024 10:30:13 +0100 Subject: [PATCH 034/103] fixes to pregnancy_supervisor test --- tests/test_pregnancy_supervisor.py | 33 +++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/test_pregnancy_supervisor.py b/tests/test_pregnancy_supervisor.py index 31adcf03b5..4a1aafe0b7 100644 --- a/tests/test_pregnancy_supervisor.py +++ b/tests/test_pregnancy_supervisor.py @@ -123,8 +123,11 @@ def test_run_core_modules_normal_allocation_of_pregnancy(seed, tmpdir): # check that no errors have been logged during the simulation run output = parse_log_file(sim.log_filepath) - assert 'error' not in output['tlo.methods.pregnancy_supervisor'] - assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] + if 'tlo.methods.pregnancy_supervisor' in output: + assert 'error' not in output['tlo.methods.pregnancy_supervisor'] + + if 'tlo.methods.care_of_women_during_pregnancy' in output: + assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] @pytest.mark.slow @@ -136,13 +139,16 @@ def test_run_core_modules_high_volumes_of_pregnancy(seed, tmpdir): register_modules(sim) sim.make_initial_population(n=5000) set_all_women_as_pregnant_and_reset_baseline_parity(sim) - sim.simulate(end_date=Date(2011, 1, 1)) + sim.simulate(end_date=Date(2011, 1, 2)) check_dtypes(sim) # check that no errors have been logged during the simulation run output = parse_log_file(sim.log_filepath) - assert 'error' not in output['tlo.methods.pregnancy_supervisor'] - assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] + if 'tlo.methods.pregnancy_supervisor' in output: + assert 'error' not in output['tlo.methods.pregnancy_supervisor'] + + if 'tlo.methods.care_of_women_during_pregnancy' in output: + assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] @pytest.mark.slow @@ -172,14 +178,15 @@ def test_run_core_modules_high_volumes_of_pregnancy_hsis_cant_run(seed, tmpdir): sim.make_initial_population(n=5000) set_all_women_as_pregnant_and_reset_baseline_parity(sim) - sim.simulate(end_date=Date(2011, 1, 1)) + sim.simulate(end_date=Date(2011, 1, 2)) check_dtypes(sim) # check that no errors have been logged during the simulation run output = parse_log_file(sim.log_filepath) for module in ['pregnancy_supervisor', 'care_of_women_during_pregnancy', 'labour', 'postnatal_supervisor', 'newborn_outcomes']: - assert 'error' not in output[f'tlo.methods.{module}'] + if module in output: + assert 'error' not in output[f'tlo.methods.{module}'] @pytest.mark.slow @@ -220,13 +227,16 @@ def test_run_with_all_referenced_modules_registered(seed, tmpdir): sim.make_initial_population(n=5000) set_all_women_as_pregnant_and_reset_baseline_parity(sim) # keep high volume of pregnancy to increase risk of error - sim.simulate(end_date=Date(2011, 1, 1)) + sim.simulate(end_date=Date(2011, 1, 2)) check_dtypes(sim) # check that no errors have been logged during the simulation run output = parse_log_file(sim.log_filepath) - assert 'error' not in output['tlo.methods.pregnancy_supervisor'] - assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] + if 'tlo.methods.pregnancy_supervisor' in output: + assert 'error' not in output['tlo.methods.pregnancy_supervisor'] + + if 'tlo.methods.care_of_women_during_pregnancy' in output: + assert 'error' not in output['tlo.methods.care_of_women_during_pregnancy'] def test_store_dalys_in_mni_function_and_daly_calculations(seed): @@ -1082,6 +1092,9 @@ def test_pregnancy_supervisor_gdm(seed): pregnancy_helper_functions.update_mni_dictionary(sim.modules['PregnancySupervisor'], woman) pregnancy_helper_functions.update_mni_dictionary(sim.modules['Labour'], woman) + sim.modules['PregnancySupervisor'].mother_and_newborn_info[woman]['delivery_setting'] = 'home_birth' + + # Run pregnancy supervisor event pregnancy_sup = pregnancy_supervisor.PregnancySupervisorEvent(module=sim.modules['PregnancySupervisor']) sim.date = sim.date + pd.DateOffset(weeks=20) From ba22a9904bcc1a389225098d903d5027e2879937 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 9 Oct 2024 14:04:31 +0100 Subject: [PATCH 035/103] fix logic in HIV causing crashes --- src/tlo/methods/hiv.py | 2 +- src/tlo/methods/labour.py | 2 +- tests/test_mnh_cohort.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index d6455cc861..4c956de29c 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -1504,7 +1504,7 @@ def per_capita_testing_rate(self): df = self.sim.population.props # get number of tests performed in last time period - if self.sim.date.year == 2011: + if self.sim.date.year == (self.sim.start_date.year + 1): number_tests_new = df.hv_number_tests.sum() previous_test_numbers = 0 diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index debe2b7e83..28c1684db0 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -875,7 +875,7 @@ def initialise_simulation(self, sim): self.get_and_store_labour_item_codes() # We set the LoggingEvent to run on the last day of each year to produce statistics for that year - sim.schedule_event(LabourLoggingEvent(self), sim.date + DateOffset(days=1)) + sim.schedule_event(LabourLoggingEvent(self), sim.date + DateOffset(years=1)) # Schedule analysis event if self.sim.date.year <= self.current_parameters['analysis_year']: diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index df9bc58e15..c73f8d7588 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -31,7 +31,7 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): register_modules(sim) sim.make_initial_population(n=2500) - sim.simulate(end_date=Date(2025, 1, 1)) + sim.simulate(end_date=Date(2025, 1, 2)) output= parse_log_file(sim.log_filepath) live_births = len(output['tlo.methods.demography']['on_birth']) From 011a0017b427f2342d10c76439ab0c785625cde5 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 09:14:46 +0100 Subject: [PATCH 036/103] updates to tests and new dummy analysis script --- .../dummy_cohort_azure_calib.py | 28 ++++++ tests/test_mnh_cohort.py | 90 +++++++++++-------- 2 files changed, 80 insertions(+), 38 deletions(-) create mode 100644 src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py new file mode 100644 index 0000000000..257c2f00e8 --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -0,0 +1,28 @@ +import os + +import pandas as pd + +from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize + +outputspath = './outputs/sejjj49@ucl.ac.uk/' +scenario_filename = 'cohort_test-2024-10-09T130546Z' + +results_folder = get_scenario_outputs(scenario_filename, outputspath)[-1] + +def get_data_frames(key): + def sort_df(_df): + _x = _df.drop(columns=['date'], inplace=False) + return _x.iloc[0] + + results_df = summarize (extract_results( + results_folder, + module="tlo.methods.pregnancy_supervisor", + key=key, + custom_generate_series=sort_df, + do_scaling=False + )) + + return results_df + +results = {k:get_data_frames(k) for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', + 'service_coverage']} diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index c73f8d7588..184ef4b947 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -1,4 +1,7 @@ import os + +import pandas as pd + from pathlib import Path from tlo import Date, Simulation, logging @@ -21,51 +24,62 @@ def register_modules(sim): modules""" sim.register(*fullmodel(resourcefilepath=resourcefilepath), - mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath), - ) - - -def test_run_sim_with_mnh_cohort(tmpdir, seed): - sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ - "*": logging.DEBUG},"directory": tmpdir}) - - register_modules(sim) - sim.make_initial_population(n=2500) - sim.simulate(end_date=Date(2025, 1, 2)) + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath)) - output= parse_log_file(sim.log_filepath) - live_births = len(output['tlo.methods.demography']['on_birth']) - - deaths_df = output['tlo.methods.demography']['death'] - prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] - - dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] - init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & - (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' - 'chronic_ischemic_hd|ever_heart_attack|' - 'chronic_kidney_disease') | - (prop_deaths_df['cause_of_death'] == 'TB'))] - - hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & - (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] - - indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) - total_deaths = len(dir_mat_deaths) + indir_mat_deaths +# def test_run_sim_with_mnh_cohort(tmpdir, seed): +# sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ +# "*": logging.DEBUG},"directory": tmpdir}) +# +# register_modules(sim) +# sim.make_initial_population(n=2500) +# sim.simulate(end_date=Date(2025, 1, 2)) +# +# output= parse_log_file(sim.log_filepath) +# live_births = len(output['tlo.methods.demography']['on_birth']) +# +# deaths_df = output['tlo.methods.demography']['death'] +# prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] +# +# dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] +# init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & +# (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' +# 'chronic_ischemic_hd|ever_heart_attack|' +# 'chronic_kidney_disease') | +# (prop_deaths_df['cause_of_death'] == 'TB'))] +# +# hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & +# (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] +# +# indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) +# total_deaths = len(dir_mat_deaths) + indir_mat_deaths +# +# # TOTAL_DEATHS +# mmr = (total_deaths / live_births) * 100_000 +# +# print(f'The MMR for this simulation is {mmr}') +# print(f'The maternal deaths for this simulation (unscaled) are {total_deaths}') +# print(f'The total maternal deaths for this simulation (scaled) are ' +# f'{total_deaths * output["tlo.methods.population"]["scaling_factor"]["scaling_factor"].values[0]}') +# +# maternal_dalys = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() +# print(f'The maternal DALYs for this simulation (unscaled) are {maternal_dalys}') - # TOTAL_DEATHS - mmr = (total_deaths / live_births) * 100_000 - print(f'The MMR for this simulation is {mmr}') - print(f'The maternal deaths for this simulation (unscaled) are {total_deaths}') - print(f'The total maternal deaths for this simulation (scaled) are ' - f'{total_deaths * output["tlo.methods.population"]["scaling_factor"]["scaling_factor"].values[0]}') +def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): + sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) - maternal_dalys = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() - print(f'The maternal DALYs for this simulation (unscaled) are {maternal_dalys}') + register_modules(sim) + sim.make_initial_population(n=1000) + sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) + df = sim.population.props + pop = df.loc[df.is_alive] + assert (df.loc[pop.index, 'sex'] == 'F').all() + assert df.loc[pop.index, 'is_pregnant'].all() + assert not (pd.isnull(df.loc[pop.index, 'la_due_date_current_pregnancy'])).all() + assert (df.loc[pop.index, 'co_contraception'] == 'not_using').all() - # df = sim.population.props # orig = sim.population.new_row # assert (df.dtypes == orig.dtypes).all() From 09a216f6b058b23d51cc20ebd3b5aa8618e891a3 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 09:31:32 +0100 Subject: [PATCH 037/103] fix denom error --- src/tlo/methods/pregnancy_supervisor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 4f2e05dfcb..648cde5a59 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2293,8 +2293,8 @@ def rate (count, denom, multiplier): 'hc_rate': rate(c['health_centre_delivery'] , total_births, 100), 'hp_rate': rate(c['hospital_delivery'] , total_births, 100), - 'm_pnc1+': rate(m_pnc1, total_births, 1000), - 'n_pnc1+': rate(n_pnc1, total_births, 1000)}) + 'm_pnc1+': rate(m_pnc1, total_births, 100), + 'n_pnc1+': rate(n_pnc1, total_births, 100)}) # Reset the dictionary so all values = 0 mnh_oc = pregnancy_helper_functions.generate_mnh_outcome_counter() From 03161e0b15306d4302db589b36cd49bd77376495 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 09:40:50 +0100 Subject: [PATCH 038/103] fix indentation error --- src/tlo/methods/postnatal_supervisor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 34c31fca9e..c8cbbcc748 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -1139,7 +1139,7 @@ def log_new_progressed_cases(disease): individual_id]])[individual_id] if risk_gh_after_pregnancy > self.module.rng.random_sample(): df.at[individual_id, 'pn_htn_disorders'] = 'gest_htn' - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['mild_gest_htn'] += 1 # ====================================== POSTNATAL CHECK ================================================== # Import the HSI which represents postnatal care From fd239bcd789908861ec15934ab8a3adabcd3fe77 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 13:40:14 +0100 Subject: [PATCH 039/103] fix indentation error --- .../dummy_cohort_azure_calib.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 257c2f00e8..621bb9516f 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -26,3 +26,25 @@ def sort_df(_df): results = {k:get_data_frames(k) for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage']} + + +import matplotlib.pyplot as plt +import numpy as np +# Sample data +mmr_data = { + 'int_1': [(235, 250, 265), (335, 350, 365)], + 'int_2': [(170, 195, 200), (290, 305, 320)], + 'int_3': [(280, 295, 310), (295 ,310, 325)], + 'int_4': [(165, 180, 195), (385, 400, 415)] +} +# Plotting +fig, ax = plt.subplots() +for key, intervals in mmr_data.items(): + for idx, (lower, mean, upper) in enumerate(intervals): + x = np.arange(len(mmr_data)) * len(intervals) + idx + ax.plot(x, mean, 'o', label=f'{key}' if idx == 0 else "") + ax.fill_between([x, x], [lower, lower], [upper, upper], alpha=0.2) +ax.set_xticks(np.arange(len(mmr_data)) * len(intervals) + 0.5) +ax.set_xticklabels(mmr_data.keys()) +plt.legend() +plt.show() From ae57c4031d9d07258bfecf589891cb531bc2b3ff Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 14:19:18 +0100 Subject: [PATCH 040/103] revert comit --- .../cohort_analysis/cohort_interventions_scenario.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 11cac4deb7..5a64abf546 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -34,7 +34,8 @@ def log_configuration(self): } def modules(self): - return [*fullmodel(resourcefilepath=self.resources), + return [*fullmodel(resourcefilepath=self.resources, + module_kwargs={'Schisto': {'mda_execute': False}}), mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=self.resources)] def draw_parameters(self, draw_number, rng): From de3ee93e0065c29c570da7b625e436332eaef1d4 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 10 Oct 2024 14:24:21 +0100 Subject: [PATCH 041/103] added missing logging --- src/tlo/methods/labour.py | 16 ++++------------ src/tlo/methods/postnatal_supervisor.py | 10 ---------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 3d97aa23dd..3692c691a8 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1446,9 +1446,7 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): pregnancy_helper_functions.store_dalys_in_mni(individual_id, mni, 'eclampsia_onset', self.sim.date) - current_log.info(key='maternal_complication', data={'person': individual_id, - 'type': 'eclampsia', - 'timing': timing}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['eclampsia'] +=1 # Or from mild to severe gestational hypertension, risk reduced by treatment if df.at[individual_id, f'{property_prefix}_htn_disorders'] == 'gest_htn': @@ -1461,9 +1459,7 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): if risk_prog_gh_sgh > self.rng.random_sample(): df.at[individual_id, f'{property_prefix}_htn_disorders'] = 'severe_gest_htn' - current_log.info(key='maternal_complication', data={'person': individual_id, - 'type': 'severe_gest_htn', - 'timing': timing}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_gest_htn'] +=1 # Or from severe gestational hypertension to severe pre-eclampsia... if df.at[individual_id, f'{property_prefix}_htn_disorders'] == 'severe_gest_htn': @@ -1471,9 +1467,7 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): df.at[individual_id, f'{property_prefix}_htn_disorders'] = 'severe_pre_eclamp' mni[individual_id]['new_onset_spe'] = True - current_log.info(key='maternal_complication', data={'person': individual_id, - 'type': 'severe_pre_eclamp', - 'timing': timing}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_pre_eclamp'] +=1 # Or from mild pre-eclampsia to severe pre-eclampsia... if df.at[individual_id, f'{property_prefix}_htn_disorders'] == 'mild_pre_eclamp': @@ -1481,9 +1475,7 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): df.at[individual_id, f'{property_prefix}_htn_disorders'] = 'severe_pre_eclamp' mni[individual_id]['new_onset_spe'] = True - current_log.info(key='maternal_complication', data={'person': individual_id, - 'type': 'severe_pre_eclamp', - 'timing': timing}) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_pre_eclamp'] +=1 def apply_risk_of_early_postpartum_death(self, individual_id): """ diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index c8cbbcc748..183877f4ed 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -525,11 +525,6 @@ def onset(eq): store_dalys_in_mni(person, mni, f'{df.at[person, "pn_anaemia_following_pregnancy"]}_anaemia_pp_onset', self.sim.date) - logger.info(key='maternal_complication', data={'person': person, - 'type': f'{df.at[person, "pn_anaemia_following_pregnancy"]}' - f'_anaemia', - 'timing': 'postnatal'}) - # --------------------------------------- HYPERTENSION ------------------------------------------ # For women who are still experiencing a hypertensive disorder of pregnancy we determine if that will now # resolve @@ -1062,11 +1057,6 @@ def apply(self, individual_id): store_dalys_in_mni(individual_id, mni, f'{df.at[individual_id, "pn_anaemia_following_pregnancy"]}_' f'anaemia_pp_onset', self.sim.date) - logger.info(key='maternal_complication', - data={'person': individual_id, - 'type': f'{df.at[individual_id, "pn_anaemia_following_pregnancy"]}_anaemia', - 'timing': 'postnatal'}) - # -------------------------------------------- HYPERTENSION ----------------------------------------------- # For women who remain hypertensive after delivery we apply a probability that this will resolve in the # first week after birth From eaeae626a4b37c024db38abf82bdb7c2e723ffe2 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:45:41 +0100 Subject: [PATCH 042/103] Focus on rti and print footprint --- src/tlo/events.py | 16 +++++++++++++--- src/tlo/methods/hsi_event.py | 36 ++++++++++++++++------------------- src/tlo/methods/rti.py | 8 ++++++-- src/tlo/simulation.py | 6 +++--- tests/test_data_generation.py | 31 ++++++++++++++++-------------- 5 files changed, 55 insertions(+), 42 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 2a7871c2c8..76e1b9a117 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -71,14 +71,19 @@ def run(self): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and not + #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True if self.target != self.sim.population: - row = self.sim.population.props.loc[[self.target]] + row = self.sim.population.props.iloc[[self.target]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' + row['appt_footprint'] = 'N/A' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: df_before = self.sim.population.props.copy() @@ -88,11 +93,12 @@ def run(self): if print_chains: if self.target != self.sim.population: - row = self.sim.population.props.loc[[self.target]] + row = self.sim.population.props.iloc[[self.target]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'After' + row['appt_footprint'] = 'N/A' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: df_after = self.sim.population.props.copy() @@ -104,11 +110,15 @@ def run(self): new_rows_before['event'] = self new_rows_before['event_date'] = self.sim.date new_rows_before['when'] = 'Before' + new_rows_before['appt_footprint'] = 'N/A' + new_rows_after = df_after.loc[indices] new_rows_after['person_ID'] = new_rows_after.index new_rows_after['event'] = self new_rows_after['event_date'] = self.sim.date new_rows_after['when'] = 'After' + new_rows_after['appt_footprint'] = 'N/A' + self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index ea9066bc8b..f8e8738543 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -196,14 +196,19 @@ def run(self, squeeze_factor): if self.sim.generate_event_chains: # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - if (self.module in self.sim.generate_event_chains_modules_of_interest) and not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and not + #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): +# if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + print_chains = True if self.target != self.sim.population: - row = self.sim.population.props.loc[[self.target]] + row = self.sim.population.props.iloc[[self.target]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' + row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: df_before = self.sim.population.props.copy() @@ -212,32 +217,23 @@ def run(self, squeeze_factor): self.post_apply_hook() self._run_after_hsi_event() + footprint = self.EXPECTED_APPT_FOOTPRINT + if updated_appt_footprint is not None: + footprint = updated_appt_footprint + if print_chains: if self.target != self.sim.population: - row = self.sim.population.props.loc[[self.target]] + row = self.sim.population.props.iloc[[self.target]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'After' + row['appt_footprint'] = str(footprint) self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: - df_after = self.sim.population.props.copy() - change = df_before.compare(df_after) - if ~change.empty: - indices = change.index - new_rows_before = df_before.loc[indices] - new_rows_before['person_ID'] = new_rows_before.index - new_rows_before['event'] = self - new_rows_before['event_date'] = self.sim.date - new_rows_before['when'] = 'Before' - new_rows_after = df_after.loc[indices] - new_rows_after['person_ID'] = new_rows_after.index - new_rows_after['event'] = self - new_rows_after['event_date'] = self.sim.date - new_rows_after['when'] = 'After' - - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) + print("Error, I shouldn't be here") + exit(-1) + return updated_appt_footprint def get_consumables( diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 18c1987483..1c12e7162b 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -2776,7 +2776,7 @@ class RTIPollingEvent(RegularEvent, PopulationScopeEventMixin): def __init__(self, module): """Schedule to take place every month """ - super().__init__(module, frequency=DateOffset(months=1)) + super().__init__(module, frequency=DateOffset(months=1000)) p = module.parameters # Parameters which transition the model between states self.base_1m_prob_rti = (p['base_rate_injrti'] / 12) @@ -2864,9 +2864,13 @@ def apply(self, population): .when('.between(70,79)', self.rr_injrti_age7079), Predictor('li_ex_alc').when(True, self.rr_injrti_excessalcohol) ) - pred = eq.predict(df.loc[rt_current_non_ind]) + if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi is True: + pred = 1 + else: + pred = eq.predict(df.loc[rt_current_non_ind]) random_draw_in_rti = self.module.rng.random_sample(size=len(rt_current_non_ind)) selected_for_rti = rt_current_non_ind[pred > random_draw_in_rti] + # Update to say they have been involved in a rti df.loc[selected_for_rti, 'rt_road_traffic_inc'] = True # Set the date that people were injured to now diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 42a2a288d3..a8ecf14cc6 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -281,7 +281,7 @@ def make_initial_population(self, *, n: int) -> None: data=f"{module.name}.initialise_population() {time.time() - start1} s", ) - self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when']) + self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint']) end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") @@ -303,8 +303,8 @@ def initialise(self, *, end_date: Date, generate_event_chains) -> None: # Eventually this can be made an option self.generate_event_chains_overwrite_epi = True # For now keep these fixed, eventually they will be input from user - self.generate_event_chains_modules_of_interest = [self.modules['Tb'], self.modules['Hiv'], self.modules['CardioMetabolicDisorders']] - self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] + self.generate_event_chains_modules_of_interest = [self.modules['RTI']] + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] else: # If not using to print chains, cannot ignore epi self.generate_event_chains_overwrite_epi = False diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py index 8dd92513f9..af3c4f0ae9 100644 --- a/tests/test_data_generation.py +++ b/tests/test_data_generation.py @@ -25,15 +25,16 @@ depression, tb, contraception, -# simplified_births, + simplified_births, + rti, symptommanager, ) from tlo.methods.hsi_generic_first_appts import HSI_GenericEmergencyFirstAppt # create simulation parameters start_date = Date(2010, 1, 1) -end_date = Date(2014, 1, 1) -popsize = 100 +end_date = Date(2012, 1, 1) +popsize = 200 @pytest.mark.slow def test_data_harvesting(seed): @@ -41,7 +42,7 @@ def test_data_harvesting(seed): This test runs a simulation to print all individual events of specific individuals """ - module_of_interest = 'Hiv' + module_of_interest = 'RTI' # create sim object sim = create_basic_sim(popsize, seed) @@ -55,29 +56,31 @@ def test_data_harvesting(seed): # run simulation sim.simulate(end_date=end_date, generate_event_chains = True) - + exit(-1) def create_basic_sim(population_size, seed): # create the basic outline of an rti simulation object sim = Simulation(start_date=start_date, seed=seed) resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' sim.register(demography.Demography(resourcefilepath=resourcefilepath), - contraception.Contraception(resourcefilepath=resourcefilepath), + # contraception.Contraception(resourcefilepath=resourcefilepath), enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), healthburden.HealthBurden(resourcefilepath=resourcefilepath), symptommanager.SymptomManager(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath, service_availability=['*']), + rti.RTI(resourcefilepath=resourcefilepath), healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), - epi.Epi(resourcefilepath=resourcefilepath), - hiv.Hiv(resourcefilepath=resourcefilepath), - tb.Tb(resourcefilepath=resourcefilepath), + simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + # epi.Epi(resourcefilepath=resourcefilepath), + # hiv.Hiv(resourcefilepath=resourcefilepath), + # tb.Tb(resourcefilepath=resourcefilepath), cardio_metabolic_disorders.CardioMetabolicDisorders(resourcefilepath=resourcefilepath), depression.Depression(resourcefilepath=resourcefilepath), - newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), - pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), - care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), - labour.Labour(resourcefilepath=resourcefilepath), - postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), + # newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), + # pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), + # care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), + # labour.Labour(resourcefilepath=resourcefilepath), + #postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), ) sim.make_initial_population(n=population_size) From c7bd9d058cea79fad0f8471830766f5c335a7df1 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:57:21 +0100 Subject: [PATCH 043/103] Only store change in individual properties, not entire property row. Log changes to logger. --- src/tlo/events.py | 204 ++++++++++++++++++++++++++-------- src/tlo/methods/hsi_event.py | 134 ++++++++++++++++------ src/tlo/simulation.py | 2 +- tests/test_data_generation.py | 22 ++-- 4 files changed, 268 insertions(+), 94 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 76e1b9a117..436a01a97c 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -4,13 +4,20 @@ from enum import Enum from typing import TYPE_CHECKING -from tlo import DateOffset +from tlo import DateOffset, logging if TYPE_CHECKING: from tlo import Simulation import pandas as pd +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +logger_summary = logging.getLogger(f"{__name__}.summary") +logger_summary.setLevel(logging.INFO) + +debug_chains = True class Priority(Enum): """Enumeration for the Priority, which is used in sorting the events in the simulation queue.""" @@ -62,66 +69,167 @@ def apply(self, target): :param target: the target of the event """ raise NotImplementedError - - def run(self): - """Make the event happen.""" + def compare_population_dataframe(self,df_before, df_after): + """ This function compares the population dataframe before/after a population-wide event has occurred. + It allows us to identify the individuals for which this event led to a significant (i.e. property) change, and to store the properties which have changed as a result of it. """ + + # Create a mask of where values are different + diff_mask = (df_before != df_after) & ~(df_before.isna() & df_after.isna()) + + # Create an empty list to store changes for each of the individuals + chain_links = {} + + # Loop through each row of the mask + for idx, row in diff_mask.iterrows(): + changed_cols = row.index[row].tolist() + + if changed_cols: # Proceed only if there are changes in the row + + # Create a dictionary for this person + # First add event info + link_info = { + #'person_ID': idx, + 'event': str(self), + 'event_date': self.sim.date, + } + + # Store the new values from df_after for the changed columns + for col in changed_cols: + link_info[col] = df_after.at[idx, col] + + + # Append the event and changes to the individual key + chain_links = {idx : link_info} + + return chain_links + + def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame]: + """ This function checks whether this event should be logged as part of the event chains, and if so stored required information before the event has occurred. """ + + # Initialise these variables print_chains = False df_before = [] + row_before = pd.Series() - if self.sim.generate_event_chains: - # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and not - #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - - print_chains = True - if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and .. + if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + + # Will eventually use this once I can actually GET THE NAME OF THE SELF + #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + + print_chains = True + + # Target is single individual + if self.target != self.sim.population: + # Save row for comparison after event has occurred + row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + + if debug_chains: + # Print entire row + row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' - row['appt_footprint'] = 'N/A' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) - else: - df_before = self.sim.population.props.copy() - - self.apply(self.target) - self.post_apply_hook() + else: + # This will be a population-wide event. In order to find individuals for which this led to + # a meaningful change, make a copy of the pop dataframe before the event has occurred. + df_before = self.sim.population.props.copy() + + return print_chains, row_before, df_before + + def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> dict: + """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ + + chain_links = {} + if print_chains: + + # Target is single individual if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] - row['person_ID'] = self.target - row['event'] = str(self) - row['event_date'] = self.sim.date - row['when'] = 'After' - row['appt_footprint'] = 'N/A' - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) + + # Create and store event for this individual + link_info = { + #'person_ID' : self.target, + 'event' : str(self), + 'event_date' : self.sim.date, + } + # Store property changes as a result of the event for this individual + for key in row_before.index: + if row_before[key] != row_after[key]: # Note: used fillna previously + link_info[key] = row_after[key] + + chain_links = {self.target : link_info} + + if debug_chains: + # Print entire row + row = self.sim.population.props.loc[[abs(self.target)]] # Use abs to avoid potentil issue with direct births + row['person_ID'] = self.target + row['event'] = str(self) + row['event_date'] = self.sim.date + row['when'] = 'After' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + else: - df_after = self.sim.population.props.copy() - change = df_before.compare(df_after) - if ~change.empty: - indices = change.index - new_rows_before = df_before.loc[indices] - new_rows_before['person_ID'] = new_rows_before.index - new_rows_before['event'] = self - new_rows_before['event_date'] = self.sim.date - new_rows_before['when'] = 'Before' - new_rows_before['appt_footprint'] = 'N/A' - - new_rows_after = df_after.loc[indices] - new_rows_after['person_ID'] = new_rows_after.index - new_rows_after['event'] = self - new_rows_after['event_date'] = self.sim.date - new_rows_after['when'] = 'After' - new_rows_after['appt_footprint'] = 'N/A' - - - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) + # Target is entire population. Identify individuals for which properties have changed + # and store their changes. + + # Population frame after event + df_after = self.sim.population.props + + # Create and store the event and dictionary of changes for affected individuals + chain_links = self.compare_population_dataframe(df_before, df_after) + + if debug_chains: + # Or print entire rows + change = df_before.compare(df_after) + if not change.empty: + indices = change.index + new_rows_before = df_before.loc[indices] + new_rows_before['person_ID'] = new_rows_before.index + new_rows_before['event'] = self + new_rows_before['event_date'] = self.sim.date + new_rows_before['when'] = 'Before' + + new_rows_after = df_after.loc[indices] + new_rows_after['person_ID'] = new_rows_after.index + new_rows_after['event'] = self + new_rows_after['event_date'] = self.sim.date + new_rows_after['when'] = 'After' + + self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) + self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) + + return chain_links + + def run(self): + """Make the event happen.""" + + # Collect relevant information before event takes place + if self.sim.generate_event_chains: + print_chains, row_before, df_before = self.store_chains_to_do_before_event() + + self.apply(self.target) + self.post_apply_hook() + + # Collect event info + meaningful property changes of individuals. Combined, these will constitute a 'link' + # in the individual's event chain. + if self.sim.generate_event_chains: + chain_links = self.store_chains_to_do_after_event(print_chains, row_before, df_before) + + # Log chain_links here + if len(chain_links)>0: + logger.info(key='event_chains', + data= chain_links, + description='Links forming chains of events for simulated individuals') + + #print("Chain events ", chain_links) + class RegularEvent(Event): diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index f8e8738543..1c727f014b 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -16,12 +16,19 @@ from tlo import Module, Simulation from tlo.methods.healthsystem import HealthSystem +# Pointing to the logger in events +logger_chains = logging.getLogger("tlo.methods.event") +logger_chains.setLevel(logging.INFO) + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) logger_summary = logging.getLogger(f"{__name__}.summary") logger_summary.setLevel(logging.INFO) +debug_chains = True + + # Declare the level which will be used to represent the merging of levels '1b' and '2' LABEL_FOR_MERGED_FACILITY_LEVELS_1B_AND_2 = "2" @@ -187,54 +194,113 @@ def _run_after_hsi_event(self) -> None: item_codes=self._EQUIPMENT, facility_id=self.facility_info.id ) - - def run(self, squeeze_factor): - """Make the event happen.""" + + def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: + """ This function checks whether this event should be logged as part of the event chains, and if so stored required information before the event has occurred. """ + # Initialise these variables print_chains = False - df_before = [] - - if self.sim.generate_event_chains: - # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and not - #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): -# if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - - print_chains = True - if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] - row['person_ID'] = self.target - row['event'] = str(self) - row['event_date'] = self.sim.date - row['when'] = 'Before' - row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) - else: - df_before = self.sim.population.props.copy() - - updated_appt_footprint = self.apply(self.target, squeeze_factor) - self.post_apply_hook() - self._run_after_hsi_event() + row_before = pd.Series() - footprint = self.EXPECTED_APPT_FOOTPRINT - if updated_appt_footprint is not None: - footprint = updated_appt_footprint + # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore + # if (self.module in self.sim.generate_event_chains_modules_of_interest) and + if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): - if print_chains: + # Will eventually use this once I can actually GET THE NAME OF THE SELF + # if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + if self.target != self.sim.population: - row = self.sim.population.props.iloc[[self.target]] + + # In the case of HSI events, only individual events should exist and therefore be logged + print_chains = True + + # Save row for comparison after event has occurred + row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + + row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target row['event'] = str(self) row['event_date'] = self.sim.date - row['when'] = 'After' - row['appt_footprint'] = str(footprint) + row['when'] = 'Before' + row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) + row['level'] = self.facility_info.level self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + else: + # Many of our HealthSystem implementations rely on the assumption that print("Error, I shouldn't be here") exit(-1) + + return print_chains, row_before + + def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> dict: + """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ + if print_chains: + # For HSI event, this will only ever occur for individual events + + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) + + # Create and store dictionary of changes. Note that person_ID, event, event_date, appt_foot, and level + # will be stored regardless of whether individual experienced property changes. + + # Add event details + link_info = { + 'event' : str(self), + 'event_date' : self.sim.date, + 'appt_footprint' : str(footprint), + 'level' : self.facility_info.level, + } + + # Add changes to properties + for key in row_before.index: + if row_before[key] != row_after[key]: # Note: used fillna previously + link_info[key] = row_after[key] + + chain_links = {self.target : link_info} + + # Print entire row + row = self.sim.population.props.loc[[abs(self.target)]] + row['person_ID'] = self.target + row['event'] = str(self) + row['event_date'] = self.sim.date + row['when'] = 'After' + row['appt_footprint'] = footprint + row['level'] = self.facility_info.level + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + + return chain_links + + + def run(self, squeeze_factor): + """Make the event happen.""" + + + if self.sim.generate_event_chains: + print_chains, row_before = self.store_chains_to_do_before_event() + + footprint = self.EXPECTED_APPT_FOOTPRINT + updated_appt_footprint = self.apply(self.target, squeeze_factor) + self.post_apply_hook() + self._run_after_hsi_event() + + + if self.sim.generate_event_chains: + + # If the footprint has been updated when the event ran, change it here + if updated_appt_footprint is not None: + footprint = updated_appt_footprint + + chain_links = self.store_chains_to_do_after_event(print_chains, row_before, str(footprint)) + + if len(chain_links)>0: + logger_chains.info(key='event_chains', + data = chain_links, + description='Links forming chains of events for simulated individuals') + #print(chain_links) + return updated_appt_footprint + def get_consumables( self, diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index a8ecf14cc6..20b3a4898f 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -281,7 +281,7 @@ def make_initial_population(self, *, n: int) -> None: data=f"{module.name}.initialise_population() {time.time() - start1} s", ) - self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint']) + self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint'] + ['level']) end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py index af3c4f0ae9..39f2b022aa 100644 --- a/tests/test_data_generation.py +++ b/tests/test_data_generation.py @@ -33,7 +33,7 @@ # create simulation parameters start_date = Date(2010, 1, 1) -end_date = Date(2012, 1, 1) +end_date = Date(2011, 1, 1) popsize = 200 @pytest.mark.slow @@ -63,24 +63,24 @@ def create_basic_sim(population_size, seed): sim = Simulation(start_date=start_date, seed=seed) resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' sim.register(demography.Demography(resourcefilepath=resourcefilepath), - # contraception.Contraception(resourcefilepath=resourcefilepath), + contraception.Contraception(resourcefilepath=resourcefilepath), enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), healthburden.HealthBurden(resourcefilepath=resourcefilepath), symptommanager.SymptomManager(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=resourcefilepath, service_availability=['*']), rti.RTI(resourcefilepath=resourcefilepath), healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), - simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), - # epi.Epi(resourcefilepath=resourcefilepath), - # hiv.Hiv(resourcefilepath=resourcefilepath), - # tb.Tb(resourcefilepath=resourcefilepath), + # simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + epi.Epi(resourcefilepath=resourcefilepath), + hiv.Hiv(resourcefilepath=resourcefilepath), + tb.Tb(resourcefilepath=resourcefilepath), cardio_metabolic_disorders.CardioMetabolicDisorders(resourcefilepath=resourcefilepath), depression.Depression(resourcefilepath=resourcefilepath), - # newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), - # pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), - # care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), - # labour.Labour(resourcefilepath=resourcefilepath), - #postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), + newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), + pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), + care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), + labour.Labour(resourcefilepath=resourcefilepath), + postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), ) sim.make_initial_population(n=population_size) From 769aaeca44aaedc324bd3da2f5f338bb47e02106 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:03:22 +0100 Subject: [PATCH 044/103] Style fixes --- src/tlo/methods/tb.py | 2 +- src/tlo/simulation.py | 4 ++-- tests/test_data_generation.py | 5 ----- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 4c170944d2..9dc05ff301 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -1393,7 +1393,7 @@ def apply(self, population): & (df.tb_inf != "active") ].index - n_susceptible = len(susc_idx) + len(susc_idx) middle_index = len(susc_idx) // 2 diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 20b3a4898f..75dfa76429 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -8,7 +8,7 @@ import time from collections import OrderedDict from pathlib import Path -from typing import Dict, Optional, Union +from typing import Optional from typing import TYPE_CHECKING, Optional import pandas as pd @@ -374,7 +374,7 @@ def run_simulation_to(self, *, to_date: Date) -> None: :param to_date: Date to simulate up to but not including - must be before or equal to simulation end date specified in call to :py:meth:`initialise`. """ - f = open('output.txt', mode='a') + open('output.txt', mode='a') if not self._initialised: msg = "Simulation must be initialised before calling run_simulation_to" diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py index 39f2b022aa..c94618a77d 100644 --- a/tests/test_data_generation.py +++ b/tests/test_data_generation.py @@ -1,7 +1,6 @@ import os from pathlib import Path -import pandas as pd import pytest from tlo import Date, Simulation @@ -11,7 +10,6 @@ depression, enhanced_lifestyle, epi, - epilepsy, healthburden, healthseekingbehaviour, healthsystem, @@ -20,16 +18,13 @@ labour, newborn_outcomes, postnatal_supervisor, - pregnancy_helper_functions, pregnancy_supervisor, depression, tb, contraception, - simplified_births, rti, symptommanager, ) -from tlo.methods.hsi_generic_first_appts import HSI_GenericEmergencyFirstAppt # create simulation parameters start_date = Date(2010, 1, 1) From 757cee36b0ae611f1f7ae31d25799fc0d6e7daa1 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Sun, 13 Oct 2024 11:15:17 +0100 Subject: [PATCH 045/103] Include printing of individual properties at the beginning and at birth, label what is only used for ddebugging and will be later removed --- src/tlo/events.py | 5 +++-- src/tlo/methods/hsi_event.py | 7 ++++--- src/tlo/methods/rti.py | 2 +- src/tlo/simulation.py | 28 ++++++++++++++++++++++++++++ tests/test_data_generation.py | 5 ++--- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 436a01a97c..03bf7c72fa 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -98,7 +98,6 @@ def compare_population_dataframe(self,df_before, df_after): for col in changed_cols: link_info[col] = df_after.at[idx, col] - # Append the event and changes to the individual key chain_links = {idx : link_info} @@ -127,7 +126,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) if debug_chains: - # Print entire row + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target row['event'] = str(self) @@ -166,6 +165,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> chain_links = {self.target : link_info} + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. if debug_chains: # Print entire row row = self.sim.population.props.loc[[abs(self.target)]] # Use abs to avoid potentil issue with direct births @@ -185,6 +185,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> # Create and store the event and dictionary of changes for affected individuals chain_links = self.compare_population_dataframe(df_before, df_after) + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. if debug_chains: # Or print entire rows change = df_before.compare(df_after) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 1c727f014b..0c3bc16072 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -217,6 +217,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target row['event'] = str(self) @@ -228,8 +229,8 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: else: # Many of our HealthSystem implementations rely on the assumption that - print("Error, I shouldn't be here") - exit(-1) + raise RuntimeError("Cannot have population-wide HSI events") + return print_chains, row_before @@ -258,7 +259,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> chain_links = {self.target : link_info} - # Print entire row + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target row['event'] = str(self) diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 1c12e7162b..3642365976 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -2865,7 +2865,7 @@ def apply(self, population): Predictor('li_ex_alc').when(True, self.rr_injrti_excessalcohol) ) if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi is True: - pred = 1 + pred = 1.0 else: pred = eq.predict(df.loc[rt_current_non_ind]) random_draw_in_rti = self.module.rng.random_sample(size=len(rt_current_non_ind)) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 75dfa76429..582fb4ba1c 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -37,6 +37,9 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +logger_chains = logging.getLogger("tlo.methods.event") +logger_chains.setLevel(logging.INFO) + class SimulationPreviouslyInitialisedError(Exception): """Exception raised when trying to initialise an already initialised simulation.""" @@ -111,6 +114,8 @@ def __init__( self.end_date = None self.output_file = None self.population: Optional[Population] = None + + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. self.event_chains: Optinoal[Population] = None self.show_progress_bar = show_progress_bar @@ -281,7 +286,16 @@ def make_initial_population(self, *, n: int) -> None: data=f"{module.name}.initialise_population() {time.time() - start1} s", ) + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint'] + ['level']) + + # When logging events for each individual to reconstruct chains, only the changes in individual properties will be logged. + # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. + if self.generate_event_chains: + pop_dict = self.population.props.to_dict(orient='index') + logger_chains.info(key='event_chains', + data = pop_dict, + description='Links forming chains of events for simulated individuals') end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") @@ -392,6 +406,8 @@ def run_simulation_to(self, *, to_date: Date) -> None: self._update_progress_bar(progress_bar, date) self.fire_single_event(event, date) self.date = to_date + + # TO BE REMOVED: this is currently only used for debugging, will be removed from final PR. self.event_chains.to_csv('output.csv', index=False) if self.show_progress_bar: @@ -449,13 +465,25 @@ def do_birth(self, mother_id: int) -> int: child_id = self.population.do_birth() for module in self.modules.values(): module.on_birth(mother_id, child_id) + if self.generate_event_chains: + # When individual is born, store their initial properties to provide a starting point to the chain of property + # changes that this individual will undergo as a result of events taking place. + prop_dict = self.population.props.loc[child_id].to_dict() + + child_dict = {child_id : prop_dict} + logger_chains.info(key='event_chains', + data = child_dict, + description='Links forming chains of events for simulated individuals') + + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.population.props.iloc[[child_id]] row['person_ID'] = child_id row['event'] = 'Birth' row['event_date'] = self.date row['when'] = 'After' self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) + return child_id def find_events_for_person(self, person_id: int) -> list[tuple[Date, Event]]: diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py index c94618a77d..d9885c1fab 100644 --- a/tests/test_data_generation.py +++ b/tests/test_data_generation.py @@ -28,8 +28,8 @@ # create simulation parameters start_date = Date(2010, 1, 1) -end_date = Date(2011, 1, 1) -popsize = 200 +end_date = Date(2012, 1, 1) +popsize = 100 @pytest.mark.slow def test_data_harvesting(seed): @@ -51,7 +51,6 @@ def test_data_harvesting(seed): # run simulation sim.simulate(end_date=end_date, generate_event_chains = True) - exit(-1) def create_basic_sim(population_size, seed): # create the basic outline of an rti simulation object From 7c3bf259250ddb9d3f39cba31d97222fc3cbe782 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 14 Oct 2024 16:31:31 +0100 Subject: [PATCH 046/103] work on cohort --- .../ResourceFile_PregnancySupervisor.xlsx | 4 +- .../methods/care_of_women_during_pregnancy.py | 588 +++++++++++------- src/tlo/methods/pregnancy_helper_functions.py | 84 +++ src/tlo/methods/pregnancy_supervisor.py | 14 +- tests/test_mnh_cohort.py | 74 +-- 5 files changed, 508 insertions(+), 256 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index 3ee3406d7a..b7b42c445d 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fbd0974016b374a3286e379ef3cb1f297307ef410880c3f902e9439194a6d37d -size 22225 +oid sha256:f03864369f37f0a3d03deffe3b60f1f029172540fd692b1ece19b693979bab95 +size 23759 diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index d1d52f076a..44712b4ea0 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -281,7 +281,7 @@ def get_and_store_pregnancy_item_codes(self): # ---------------------------------- IRON AND FOLIC ACID ------------------------------------------------------ # Dose changes at run time self.item_codes_preg_consumables['iron_folic_acid'] = \ - {ic('Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg'): 1} # TODO: update con requested here + {ic('Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg'): 1} # --------------------------------- BALANCED ENERGY AND PROTEIN ---------------------------------------------- # Dose changes at run time @@ -728,45 +728,63 @@ def screening_interventions_delivered_at_every_contact(self, hsi_event): params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - hypertension_diagnosed = False - proteinuria_diagnosed = False - - # Delivery of the intervention is conditioned on a random draw against a probability that the intervention - # would be delivered (used to calibrate to SPA data - acts as proxy for clinical quality) - if self.rng.random_sample() < params['prob_intervention_delivered_urine_ds']: - - # check consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, cons=self.item_codes_preg_consumables['urine_dipstick'], opt_cons=None) - - # If the intervention will be delivered the dx_manager runs, returning True if the consumables are - # available and the test detects protein in the urine - if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='urine_dipstick_protein', hsi_event=hsi_event): - - # We use a temporary variable to store if proteinuria is detected - proteinuria_diagnosed = True - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'dipstick'}) - - # The process is repeated for blood pressure monitoring - if self.rng.random_sample() < params['prob_intervention_delivered_bp']: - hsi_event.add_equipment({'Sphygmomanometer'}) - - if self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='blood_pressure_measurement', - hsi_event=hsi_event): - hypertension_diagnosed = True - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'bp_measurement'}) - - if not df.at[person_id, 'ac_gest_htn_on_treatment'] and\ - (df.at[person_id, 'ps_htn_disorders'] != 'none') and pd.isnull(mni[person_id]['hypertension' - '_onset']): - - # We store date of onset to calculate dalys- only women who are aware of diagnosis experience DALYs - # (see daly weight for hypertension) - pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'hypertension_onset', self.sim.date) - - # If either high blood pressure or proteinuria are detected (or both) we assume this woman needs to be admitted - # for further treatment following this ANC contact + proteinuria_diagnosed = pregnancy_helper_functions.check_int_deliverable( + self, int_name='urine_dipstick', + hsi_event=hsi_event, + q_param=[params['prob_intervention_delivered_urine_ds']], + cons=self.item_codes_preg_consumables['urine_dipstick'], + dx_test='urine_dipstick_protein') + + hypertension_diagnosed = pregnancy_helper_functions.check_int_deliverable( + self, int_name='bp_measurement', + hsi_event=hsi_event, + q_param=[params['prob_intervention_delivered_bp']], + cons=None, + equipment={'Sphygmomanometer'}, + dx_test='blood_pressure_measurement') + + if hypertension_diagnosed and not df.at[person_id, 'ac_gest_htn_on_treatment'] and \ + (df.at[person_id, 'ps_htn_disorders'] != 'none') and pd.isnull(mni[person_id]['hypertension_onset']): + # We store date of onset to calculate dalys- only women who are aware of diagnosis experience DALYs + # (see daly weight for hypertension) + pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'hypertension_onset', self.sim.date) + + # # Delivery of the intervention is conditioned on a random draw against a probability that the intervention + # # would be delivered (used to calibrate to SPA data - acts as proxy for clinical quality) + # if self.rng.random_sample() < params['prob_intervention_delivered_urine_ds']: + # + # # check consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, cons=self.item_codes_preg_consumables['urine_dipstick'], opt_cons=None) + # + # # If the intervention will be delivered the dx_manager runs, returning True if the consumables are + # # available and the test detects protein in the urine + # if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + # dx_tests_to_run='urine_dipstick_protein', hsi_event=hsi_event): + # + # # We use a temporary variable to store if proteinuria is detected + # proteinuria_diagnosed = True + # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'dipstick'}) + # + # # The process is repeated for blood pressure monitoring + # if self.rng.random_sample() < params['prob_intervention_delivered_bp']: + # hsi_event.add_equipment({'Sphygmomanometer'}) + # + # if self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='blood_pressure_measurement', + # hsi_event=hsi_event): + # hypertension_diagnosed = True + # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'bp_measurement'}) + # + # if not df.at[person_id, 'ac_gest_htn_on_treatment'] and\ + # (df.at[person_id, 'ps_htn_disorders'] != 'none') and pd.isnull(mni[person_id]['hypertension' + # '_onset']): + # + # # We store date of onset to calculate dalys- only women who are aware of diagnosis experience DALYs + # # (see daly weight for hypertension) + # pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'hypertension_onset', self.sim.date) + # + # # If either high blood pressure or proteinuria are detected (or both) we assume this woman needs to be admitted + # # for further treatment following this ANC contact # Only women who are not on treatment OR are determined to have severe disease whilst on treatment are admitted if hypertension_diagnosed or proteinuria_diagnosed: @@ -798,10 +816,18 @@ def iron_and_folic_acid_supplementation(self, hsi_event): days = self.get_approx_days_of_pregnancy(person_id) updated_cons = {k: v*(days*2) for (k, v) in self.item_codes_preg_consumables['iron_folic_acid'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, cons=updated_cons, opt_cons=None) + iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='iron_folic_acid', + hsi_event=hsi_event, + q_param=None, + cons=updated_cons) - if avail: + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, cons=updated_cons, opt_cons=None) + # + # if avail: + + if iron_folic_acid_delivered: logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'iron_folic_acid'}) # Importantly, only women who will be adherent to iron will experience the benefits of the @@ -835,11 +861,16 @@ def balance_energy_and_protein_supplementation(self, hsi_event): updated_cons = {k: v*days for (k, v) in self.item_codes_preg_consumables['balanced_energy_protein'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, cons=updated_cons, opt_cons=None) + bep_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='protein_supplement', hsi_event=hsi_event, q_param=None, cons=updated_cons) + + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, cons=updated_cons, opt_cons=None) + # + # # And she is deemed to be at risk (i.e. BMI < 18) she is started on supplements + # if avail and (df.at[person_id, 'li_bmi'] == 1): - # And she is deemed to be at risk (i.e. BMI < 18) she is started on supplements - if avail and (df.at[person_id, 'li_bmi'] == 1): + if bep_delivered and (df.at[person_id, 'li_bmi'] == 1): df.at[person_id, 'ac_receiving_bep_supplements'] = True logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'b_e_p'}) @@ -901,10 +932,15 @@ def calcium_supplementation(self, hsi_event): updated_cons = {k: v * days for (k, v) in self.item_codes_preg_consumables['calcium'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, cons=updated_cons, opt_cons=None) + calcium_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='calcium_supplement', hsi_event=hsi_event, q_param=None, cons=updated_cons) - if avail: + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, cons=updated_cons, opt_cons=None) + # + # if avail: + + if calcium_delivered: df.at[person_id, 'ac_receiving_calcium_supplements'] = True logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'calcium'}) @@ -923,16 +959,23 @@ def point_of_care_hb_testing(self, hsi_event): logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'hb_screen'}) - # Run check against probability of testing being delivered - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + hb_test_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='hb_test', hsi_event=hsi_event, q_param=None, cons=self.item_codes_preg_consumables['hb_test'], - opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - - # We run the test through the dx_manager and if a woman has anaemia and its detected she will be admitted - # for further care - if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='point_of_care_hb_test', - hsi_event=hsi_event): + opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], + dx_test='point_of_care_hb_test') + + # # Run check against probability of testing being delivered + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['hb_test'], + # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) + + # # We run the test through the dx_manager and if a woman has anaemia and its detected she will be admitted + # # for further care + # if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='point_of_care_hb_test', + # hsi_event=hsi_event): + if hb_test_delivered: df.at[person_id, 'ac_to_be_admitted'] = True def albendazole_administration(self, hsi_event): @@ -996,30 +1039,47 @@ def syphilis_screening_and_treatment(self, hsi_event): if not self.check_intervention_should_run_and_update_mni(person_id, 'syph_1', 'syph_2'): return - # See if she will receive testing - if self.rng.random_sample() < params['prob_intervention_delivered_syph_test']: - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_test'}) - - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, - cons=self.item_codes_preg_consumables['syphilis_test'], - opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - - test = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='blood_test_syphilis', hsi_event=hsi_event) - - # If the testing occurs and detects syphilis she will get treatment (if consumables are available) - if avail and test: - - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, - cons=self.item_codes_preg_consumables['syphilis_treatment'], - opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) + syph_test_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='syphilis_test', hsi_event=hsi_event, + q_param=[params['prob_intervention_delivered_syph_test']], + cons=self.item_codes_preg_consumables['syphilis_test'], + opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], + dx_test='blood_test_syphilis') + + + # # See if she will receive testing + # if self.rng.random_sample() < params['prob_intervention_delivered_syph_test']: + # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_test'}) + # + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['syphilis_test'], + # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) + # + # test = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + # dx_tests_to_run='blood_test_syphilis', hsi_event=hsi_event) + + # # If the testing occurs and detects syphilis she will get treatment (if consumables are available) + # if avail and test: + + if syph_test_delivered: + + syph_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='syphilis_treatment', hsi_event=hsi_event, + q_param=None, + cons=self.item_codes_preg_consumables['syphilis_treatment'], + opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - if avail: - # We assume that treatment is 100% effective at curing infection - df.at[person_id, 'ps_syphilis'] = False - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_treat'}) + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['syphilis_treatment'], + # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) + # + # # if avail: + if syph_treatment_delivered: + # We assume that treatment is 100% effective at curing infection + df.at[person_id, 'ps_syphilis'] = False + logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_treat'}) def hiv_testing(self, hsi_event): """ @@ -1075,34 +1135,45 @@ def gdm_screening(self, hsi_event): if df.at[person_id, 'li_bmi'] >= 4 or df.at[person_id, 'ps_prev_gest_diab'] or df.at[person_id, 'ps_prev_stillbirth']: - # If they are available, the test is conducted - if self.rng.random_sample() < params['prob_intervention_delivered_gdm_test']: - - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, - cons=self.item_codes_preg_consumables['gdm_test'], - opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - - # If the test accurately detects a woman has gestational diabetes the consumables are recorded and - # she is referred for treatment - if avail: - hsi_event.add_equipment({'Glucometer'}) - - if ( - self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event) - ): - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'}) - mni[person_id]['anc_ints'].append('gdm_screen') + gdm_screening_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='gdm_test', hsi_event=hsi_event, + q_param=[params['prob_intervention_delivered_gdm_test']], + cons=self.item_codes_preg_consumables['gdm_test'], + opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], + dx_test='blood_test_glucose', + equipment={'Glucometer'}) + + + # # If they are available, the test is conducted + # if self.rng.random_sample() < params['prob_intervention_delivered_gdm_test']: + # + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['gdm_test'], + # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) + # + # # If the test accurately detects a woman has gestational diabetes the consumables are recorded and + # # she is referred for treatment + # if avail: + # hsi_event.add_equipment({'Glucometer'}) + # + # if ( + # self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + # dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event) + # ): + # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'}) + # mni[person_id]['anc_ints'].append('gdm_screen') # We assume women with a positive GDM screen will be admitted (if they are not already receiving # outpatient care) - if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none': - # Store onset after diagnosis as daly weight is tied to diagnosis - pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset', - self.sim.date) - df.at[person_id, 'ac_to_be_admitted'] = True + if gdm_screening_delivered: + if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none': + + # Store onset after diagnosis as daly weight is tied to diagnosis + pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset', + self.sim.date) + df.at[person_id, 'ac_to_be_admitted'] = True def interventions_delivered_each_visit_from_anc2(self, hsi_event): """This function contains a collection of interventions that are delivered to women every time they attend ANC @@ -1247,24 +1318,34 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event): """ df = self.sim.population.props params = self.current_parameters + l_params = self.sim.modules['Labour'].current_parameters store_dalys_in_mni = pregnancy_helper_functions.store_dalys_in_mni mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - # Check for consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # Check for consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['blood_transfusion'], + # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) + # + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], + # sf='blood_tran', + # hsi_event=hsi_event) + # + # # If the blood is available we assume the intervention can be delivered + # if avail and sf_check: + + blood_transfusion_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='blood_transfusion', hsi_event=hsi_event, + q_param=[l_params['prob_hcw_avail_blood_tran'], l_params['mean_hcw_competence_hp']], cons=self.item_codes_preg_consumables['blood_transfusion'], - opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) - - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - sf='blood_tran', - hsi_event=hsi_event) + opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], + equipment={'Drip stand', 'Infusion pump'}) - # If the blood is available we assume the intervention can be delivered - if avail and sf_check: + if blood_transfusion_delivered: pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # If the woman is receiving blood due to anaemia we apply a probability that a transfusion of 2 units # RBCs will correct this woman's severe anaemia @@ -1287,11 +1368,16 @@ def initiate_maintenance_anti_hypertensive_treatment(self, individual_id, hsi_ev updated_cons = {k: v * days for (k, v) in self.item_codes_preg_consumables['oral_antihypertensives'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, cons=updated_cons, opt_cons=None) + oral_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='oral_antihypertensives', hsi_event=hsi_event, + q_param=None, cons=updated_cons) + # + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, cons=updated_cons, opt_cons=None) + # # If the consumables are available then the woman is started on treatment + # if avail: - # If the consumables are available then the woman is started on treatment - if avail: + if oral_anti_htns_delivered: df.at[individual_id, 'ac_gest_htn_on_treatment'] = True def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event): @@ -1305,16 +1391,23 @@ def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event): """ df = self.sim.population.props - # Define the consumables and check their availability - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, - cons=self.item_codes_preg_consumables['iv_antihypertensives'], - opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) - - # If they are available then the woman is started on treatment - if avail: + # # Define the consumables and check their availability + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['iv_antihypertensives'], + # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) + # + # # If they are available then the woman is started on treatment + # if avail: + + iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='oral_antihypertensives', hsi_event=hsi_event, + q_param=None, cons=self.item_codes_preg_consumables['iv_antihypertensives'], + opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'], equipment={'Drip stand', 'Infusion pump'}) + + if iv_anti_htns_delivered: pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event) - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # We assume women treated with antihypertensives would no longer be severely hypertensive- meaning they # are not at risk of death from severe gestational hypertension in the PregnancySupervisor event @@ -1338,22 +1431,32 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve :param hsi_event: HSI event in which the function has been called """ df = self.sim.population.props - - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + l_params = self.sim.modules['Labour'].current_parameters + + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['magnesium_sulfate'], + # opt_cons=self.item_codes_preg_consumables['eclampsia_management_optional']) + # + # # check HCW will deliver intervention + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], + # sf='anticonvulsant', + # hsi_event=hsi_event) + # + # # If available deliver the treatment + # if avail and sf_check: + + mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='mgso4', hsi_event=hsi_event, + q_param=[l_params['prob_hcw_avail_anticonvulsant'], l_params['mean_hcw_competence_hp']], cons=self.item_codes_preg_consumables['magnesium_sulfate'], - opt_cons=self.item_codes_preg_consumables['eclampsia_management_optional']) - - # check HCW will deliver intervention - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - sf='anticonvulsant', - hsi_event=hsi_event) + opt_cons=self.item_codes_preg_consumables['eclampsia_management_optional'], + equipment={'Drip stand', 'Infusion pump'}) - # If available deliver the treatment - if avail and sf_check: + if mag_sulph_delivered: df.at[individual_id, 'ac_mag_sulph_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def antibiotics_for_prom(self, individual_id, hsi_event): """ @@ -1363,20 +1466,29 @@ def antibiotics_for_prom(self, individual_id, hsi_event): :param hsi_event: HSI event in which the function has been called """ df = self.sim.population.props - - # check consumables and whether HCW are available to deliver the intervention - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + l_params = self.sim.modules['Labour'].current_parameters + + # # check consumables and whether HCW are available to deliver the intervention + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_preg_consumables['abx_for_prom'], + # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) + # + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], + # sf='iv_abx', + # hsi_event=hsi_event) + # + # if avail and sf_check: + abx_prom_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='abx_for_prom', hsi_event=hsi_event, + q_param=[l_params['prob_hcw_avail_iv_abx'], l_params['mean_hcw_competence_hp']], cons=self.item_codes_preg_consumables['abx_for_prom'], - opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) + opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'], + equipment={'Drip stand', 'Infusion pump'}) - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - sf='iv_abx', - hsi_event=hsi_event) - - if avail and sf_check: + if abx_prom_delivered: df.at[individual_id, 'ac_received_abx_for_prom'] = True - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def ectopic_pregnancy_treatment_doesnt_run(self, hsi_event): """ @@ -2522,11 +2634,17 @@ def schedule_gdm_event_and_checkup(): updated_cons = {k: v * days for (k, v) in self.module.item_codes_preg_consumables['oral_diabetic_treatment'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, cons=updated_cons, opt_cons=None) + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, cons=updated_cons, opt_cons=None) + # + # # If the meds are available women are started on that treatment + # if avail: + + gdm_orals_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='gdm_treatment_orals', hsi_event=self, + q_param=None, cons=updated_cons) - # If the meds are available women are started on that treatment - if avail: + if gdm_orals_delivered: df.at[person_id, 'ac_gest_diab_on_treatment'] = 'orals' # Assume new treatment is effective in controlling blood glucose on initiation @@ -2546,10 +2664,16 @@ def schedule_gdm_event_and_checkup(): updated_cons = {k: v * required_vials for (k, v) in self.module.item_codes_preg_consumables['insulin_treatment'].items()} - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, cons=updated_cons, opt_cons=None) + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, cons=updated_cons, opt_cons=None) + # + # if avail: - if avail: + gdm_insulin_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='gdm_treatment_insulin', hsi_event=self, + q_param=None, cons=updated_cons) + + if gdm_insulin_delivered: df.at[person_id, 'ac_gest_diab_on_treatment'] = 'insulin' df.at[person_id, 'ps_gest_diab'] = 'controlled' @@ -2584,65 +2708,91 @@ def apply(self, person_id, squeeze_factor): df = self.sim.population.props mother = df.loc[person_id] abortion_complications = self.sim.modules['PregnancySupervisor'].abortion_complications + l_params = self.sim.modules['Labour'].current_parameters if not mother.is_alive or not abortion_complications.has_any([person_id], 'sepsis', 'haemorrhage', 'injury', 'other', first=True): return - # Request baseline PAC consumables - baseline_cons = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_preg_consumables['post_abortion_care_core'], - opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_optional']) - - # Check HCW availability to deliver surgical removal of retained products - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - sf='retained_prod', - hsi_event=self) - - # Add used equipment if intervention can happen - if baseline_cons and sf_check: - self.add_equipment({'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) + pac_cons = self.module.item_codes_preg_consumables['post_abortion_care_core'] + pac_opt_cons = self.module.item_codes_preg_consumables['post_abortion_care_optional'] - # Then we determine if a woman gets treatment for her complication depending on availability of the baseline - # consumables (misoprostol) or a HCW who can conduct MVA/DC (we dont model equipment) and additional - # consumables for management of her specific complication if abortion_complications.has_any([person_id], 'sepsis', first=True): + pac_cons.update(self.module.item_codes_preg_consumables['post_abortion_care_sepsis_core']) + pac_opt_cons.update(self.module.item_codes_preg_consumables['post_abortion_care_sepsis_optional']) - cons_for_sepsis_pac = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_core'], - opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_optional']) - - if cons_for_sepsis_pac and (baseline_cons or sf_check): - df.at[person_id, 'ac_received_post_abortion_care'] = True - - elif abortion_complications.has_any([person_id], 'haemorrhage', first=True): - cons_for_haemorrhage = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_preg_consumables['blood_transfusion'], - opt_cons=self.module.item_codes_preg_consumables['iv_drug_equipment']) + if abortion_complications.has_any([person_id], 'haemorrhage', first=True): + pac_cons.update(self.module.item_codes_preg_consumables['blood_transfusion']) + pac_opt_cons.update(self.module.item_codes_preg_consumables['iv_drug_equipment']) - cons_for_shock = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], - opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) + if abortion_complications.has_any([person_id], 'injury', first=True): + pac_cons.update(self.module.item_codes_preg_consumables['post_abortion_care_shock']) + pac_opt_cons.update(self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) - if cons_for_haemorrhage and cons_for_shock and (baseline_cons or sf_check): - df.at[person_id, 'ac_received_post_abortion_care'] = True + pac_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='post_abortion_care_core', hsi_event=self, + q_param=[l_params['prob_hcw_avail_retained_prod'], l_params['mean_hcw_competence_hp']], + cons=pac_cons, + opt_cons=pac_opt_cons, + equipment={'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) - elif abortion_complications.has_any([person_id], 'injury', first=True): - cons_for_shock = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], - opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) - - if cons_for_shock and (baseline_cons or sf_check): - df.at[person_id, 'ac_received_post_abortion_care'] = True - - elif abortion_complications.has_any([person_id], 'other', first=True) and (baseline_cons or sf_check): + if pac_delivered: df.at[person_id, 'ac_received_post_abortion_care'] = True + # # Request baseline PAC consumables + # baseline_cons = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['post_abortion_care_core'], + # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_optional']) + # + # # Check HCW availability to deliver surgical removal of retained products + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], + # sf='retained_prod', + # hsi_event=self) + # + # # Add used equipment if intervention can happen + # if baseline_cons and sf_check: + # self.add_equipment({'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) + # + # # Then we determine if a woman gets treatment for her complication depending on availability of the baseline + # # consumables (misoprostol) or a HCW who can conduct MVA/DC (we dont model equipment) and additional + # # consumables for management of her specific complication + # if abortion_complications.has_any([person_id], 'sepsis', first=True): + # + # cons_for_sepsis_pac = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_core'], + # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_optional']) + # + # if cons_for_sepsis_pac and (baseline_cons or sf_check): + # df.at[person_id, 'ac_received_post_abortion_care'] = True + # + # elif abortion_complications.has_any([person_id], 'haemorrhage', first=True): + # cons_for_haemorrhage = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['blood_transfusion'], + # opt_cons=self.module.item_codes_preg_consumables['iv_drug_equipment']) + # + # cons_for_shock = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], + # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) + # + # if cons_for_haemorrhage and cons_for_shock and (baseline_cons or sf_check): + # df.at[person_id, 'ac_received_post_abortion_care'] = True + # + # elif abortion_complications.has_any([person_id], 'injury', first=True): + # cons_for_shock = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], + # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) + # + # if cons_for_shock and (baseline_cons or sf_check): + # df.at[person_id, 'ac_received_post_abortion_care'] = True + # + # elif abortion_complications.has_any([person_id], 'other', first=True) and (baseline_cons or sf_check): + # df.at[person_id, 'ac_received_post_abortion_care'] = True + if df.at[person_id, 'ac_received_post_abortion_care']: pregnancy_helper_functions.log_met_need(self.module, 'pac', self) @@ -2674,21 +2824,31 @@ def __init__(self, module, person_id): def apply(self, person_id, squeeze_factor): df = self.sim.population.props mother = df.loc[person_id] + l_params = self.sim.modules['Labour'].current_parameters if not mother.is_alive or (mother.ps_ectopic_pregnancy == 'none'): return - # We define the required consumables and check their availability - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, + ep_surg_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='ectopic_pregnancy_treatment', hsi_event=self, + q_param=[l_params['prob_hcw_avail_surg'], l_params['mean_hcw_competence_hp']], cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_core'], - opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional']) - - # If they are available then treatment can go ahead - if avail: + opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional'], + equipment={'Laparotomy Set'}) + + # # We define the required consumables and check their availability + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_core'], + # opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional']) + # + # # If they are available then treatment can go ahead + # if avail: + + if ep_surg_delivered: self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person_id]['delete_mni'] = True pregnancy_helper_functions.log_met_need(self.module, 'ep_case_mang', self) - self.add_equipment({'Laparotomy Set'}) + # self.add_equipment({'Laparotomy Set'}) # For women who have sought care after they have experienced rupture we use this treatment variable to # reduce risk of death (women who present prior to rupture do not pass through the death event as we assume diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 3ae5b5b264..dd3dbce209 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -63,6 +63,90 @@ def get_list_of_items(self, item_list): return codes +def check_int_deliverable(self, int_name, hsi_event, q_param, cons, opt_cons=None, equipment=None, dx_test=None): + """ + This function is called to determine if an intervention within the MNH modules can be delivered to an individual + during a given HSI. This applied to all MNH interventions. If analyses are being conducted in which the probability + of intervention delivery should be set explicitly, this is achieved during this function. Otherwise, probability of + intervention delivery is determined by any module-level quality parameters, consumable availability, and + (if applicable) the results of any dx_tests. Equipment is also declared. + + TODO: add to newborn interventions + TODO: what about interventions outside mnh modules + +: param self: module + param int_name: items for code look up + param hsi_event: module + param q_param: items for code look up + param cons: module + param opt_cons: items for code look up + param equipment: module + param dx_test: items for code look up + """ + + df = self.sim.population.props + individual_id = hsi_event.target + int_avail_df = self.sim.modules['PregnancySupervisor'].current_parameters['intervention_availability'] + + # Firstly, we determine if an analysis is currently being conducted during which the probability of intervention + # delivery is being overridden + # To do: replace this parameter + if self.sim.modules['PregnancySupervisor'].current_parameters['ps_analysis_in_progress']: + + # If so, we determine if this intervention will be delivered given the set probability of delivery. + can_int_run_analysis = self.rng.random_sample() < int_avail_df.at[int_name, 'availability'] + + # The intervention has no effect + if not can_int_run_analysis: + return False + + else: + # The intervention will have an effect. If this is an intervention which leads to an outcome dependent on + # correct identification of a condition through a dx_test we account for that here. + if dx_test is not None: + test = self.sim.modules['HealthSystem'].dx_manager.dx_tests[dx_test] + + if test[0].target_categories is None and (df.at[individual_id, test[0].property]): + return True + + elif ((test[0].target_categories is not None) and + (df.at[individual_id, test[0].property] in test[0].target_categories)): + return True + + else: + return False + + else: + return True + + else: + + # If analysis is not being conducted, intervention delivery is dependent on quality parameters, consumable + # availability and dx_test results + quality = False + consumables = False + test = False + + if all([self.rng.random_sample() < value for value in q_param]) or (q_param is None): + quality = True + + if equipment is not None: + hsi_event.add_equipment({equipment}) + + if ((hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons if not None else [])) or + (cons is None)): + consumables = True + + if (self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run=dx_test, hsi_event=hsi_event) + or (dx_test is None)): + test = True + + if quality and consumables and test: + return True + + else: + return False + def return_cons_avail(self, hsi_event, cons, opt_cons): """ diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 2868dd425d..34ac27a916 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -389,6 +389,10 @@ def __init__(self, name=None, resourcefilepath=None): 'sens_analysis_max': Parameter( Types.BOOL, 'Signals within the analysis event and code that sensitivity analysis is being undertaken in ' 'which the maximum coverage of ANC is enforced'), + + 'intervention_availability': Parameter( + Types.DATA_FRAME, ''), + } PROPERTIES = { @@ -437,10 +441,14 @@ def __init__(self, name=None, resourcefilepath=None): } def read_parameters(self, data_folder): + p = self.parameters + # load parameters from the resource file - parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx', - sheet_name='parameter_values') - self.load_parameters_from_dataframe(parameter_dataframe) + workbook = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx', + sheet_name=None) + self.load_parameters_from_dataframe(workbook["parameter_values"]) + workbook['intervention_availability'].set_index('intervention', inplace=True) + p['intervention_availability'] = workbook['intervention_availability'] # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module. # Currently this module calculates and reports all DALY weights from all maternal modules diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 184ef4b947..efb37029bb 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -26,43 +26,43 @@ def register_modules(sim): sim.register(*fullmodel(resourcefilepath=resourcefilepath), mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath)) -# def test_run_sim_with_mnh_cohort(tmpdir, seed): -# sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ -# "*": logging.DEBUG},"directory": tmpdir}) -# -# register_modules(sim) -# sim.make_initial_population(n=2500) -# sim.simulate(end_date=Date(2025, 1, 2)) -# -# output= parse_log_file(sim.log_filepath) -# live_births = len(output['tlo.methods.demography']['on_birth']) -# -# deaths_df = output['tlo.methods.demography']['death'] -# prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] -# -# dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] -# init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & -# (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' -# 'chronic_ischemic_hd|ever_heart_attack|' -# 'chronic_kidney_disease') | -# (prop_deaths_df['cause_of_death'] == 'TB'))] -# -# hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & -# (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] -# -# indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) -# total_deaths = len(dir_mat_deaths) + indir_mat_deaths -# -# # TOTAL_DEATHS -# mmr = (total_deaths / live_births) * 100_000 -# -# print(f'The MMR for this simulation is {mmr}') -# print(f'The maternal deaths for this simulation (unscaled) are {total_deaths}') -# print(f'The total maternal deaths for this simulation (scaled) are ' -# f'{total_deaths * output["tlo.methods.population"]["scaling_factor"]["scaling_factor"].values[0]}') -# -# maternal_dalys = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() -# print(f'The maternal DALYs for this simulation (unscaled) are {maternal_dalys}') +def test_run_sim_with_mnh_cohort(tmpdir, seed): + sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ + "*": logging.DEBUG},"directory": tmpdir}) + + register_modules(sim) + sim.make_initial_population(n=2000) + sim.simulate(end_date=Date(2025, 1, 2)) + + output= parse_log_file(sim.log_filepath) + live_births = len(output['tlo.methods.demography']['on_birth']) + + deaths_df = output['tlo.methods.demography']['death'] + prop_deaths_df = output['tlo.methods.demography.detail']['properties_of_deceased_persons'] + + dir_mat_deaths = deaths_df.loc[(deaths_df['label'] == 'Maternal Disorders')] + init_indir_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('Malaria|Suicide|ever_stroke|diabetes|' + 'chronic_ischemic_hd|ever_heart_attack|' + 'chronic_kidney_disease') | + (prop_deaths_df['cause_of_death'] == 'TB'))] + + hiv_mat_deaths = prop_deaths_df.loc[(prop_deaths_df['is_pregnant'] | prop_deaths_df['la_is_postpartum']) & + (prop_deaths_df['cause_of_death'].str.contains('AIDS_non_TB|AIDS_TB'))] + + indir_mat_deaths = len(init_indir_mat_deaths) + (len(hiv_mat_deaths) * 0.3) + total_deaths = len(dir_mat_deaths) + indir_mat_deaths + + # TOTAL_DEATHS + mmr = (total_deaths / live_births) * 100_000 + + print(f'The MMR for this simulation is {mmr}') + print(f'The maternal deaths for this simulation (unscaled) are {total_deaths}') + print(f'The total maternal deaths for this simulation (scaled) are ' + f'{total_deaths * output["tlo.methods.population"]["scaling_factor"]["scaling_factor"].values[0]}') + + maternal_dalys = output['tlo.methods.healthburden']['dalys_stacked']['Maternal Disorders'].sum() + print(f'The maternal DALYs for this simulation (unscaled) are {maternal_dalys}') def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): From 53ed3bbd8ce6fd68e00d2775d79b0c97369652d4 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 15 Oct 2024 12:45:50 +0100 Subject: [PATCH 047/103] work on cohort --- ...urceFile_LabourSkilledBirthAttendance.xlsx | 4 +- .../ResourceFile_PregnancySupervisor.xlsx | 4 +- .../methods/care_of_women_during_pregnancy.py | 24 +- src/tlo/methods/labour.py | 526 +++++++++++------- src/tlo/methods/newborn_outcomes.py | 61 +- src/tlo/methods/pregnancy_helper_functions.py | 21 +- tests/test_mnh_cohort.py | 2 +- 7 files changed, 399 insertions(+), 243 deletions(-) diff --git a/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx b/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx index 9d61093db8..6ae417a123 100644 --- a/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx +++ b/resources/ResourceFile_LabourSkilledBirthAttendance.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ebd3aff1d77254f7cd7bb1dc019e2e6a86700dcd6009cecd11a70da4b85a3c93 -size 31777 +oid sha256:4bc6e58d3ebfadb58552a58fe7a0592195d28cb7d29fcff3c7c2859f2f3ea559 +size 31686 diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index b7b42c445d..9dcd06e6f1 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f03864369f37f0a3d03deffe3b60f1f029172540fd692b1ece19b693979bab95 -size 23759 +oid sha256:cbc67135d0dce5eb308649937ed4395ac435b8da222a1341cc6f3f5cf2a15e99 +size 23856 diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 44712b4ea0..1b458156e4 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -1288,20 +1288,26 @@ def full_blood_count_testing(self, hsi_event): df = self.sim.population.props person_id = hsi_event.target - # Run dx_test for anaemia... - # If a woman is not truly anaemic but the FBC returns a result of anaemia, due to tests specificity, we - # assume the reported anaemia is mild - hsi_event.get_consumables(item_codes=self.item_codes_preg_consumables['blood_test_equipment']) - hsi_event.add_equipment({'Analyser, Haematology'}) + # # Run dx_test for anaemia... + # # If a woman is not truly anaemic but the FBC returns a result of anaemia, due to tests specificity, we + # # assume the reported anaemia is mild + # hsi_event.get_consumables(item_codes=self.item_codes_preg_consumables['blood_test_equipment']) + # hsi_event.add_equipment({'Analyser, Haematology'}) + # + # test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + # dx_tests_to_run='full_blood_count_hb', hsi_event=hsi_event) - test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='full_blood_count_hb', hsi_event=hsi_event) + full_blood_count_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='full_blood_count', hsi_event=hsi_event, + q_param=None,cons=None, + opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], + equipment={'Analyser, Haematology'}, dx_test='full_blood_count_hb') - if test_result and (df.at[person_id, 'ps_anaemia_in_pregnancy'] == 'none'): + if full_blood_count_delivered and (df.at[person_id, 'ps_anaemia_in_pregnancy'] == 'none'): return 'non_severe' # If the test correctly identifies a woman's anaemia we assume it correctly identifies its severity - if test_result and (df.at[person_id, 'ps_anaemia_in_pregnancy'] != 'none'): + if full_blood_count_delivered and (df.at[person_id, 'ps_anaemia_in_pregnancy'] != 'none'): return df.at[person_id, 'ps_anaemia_in_pregnancy'] # We return a none value if no anaemia was detected diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 2a59dd0709..f9662ed7d3 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -153,8 +153,6 @@ def __init__(self, name=None, resourcefilepath=None): 'list_limits_for_defining_term_status': Parameter( Types.LIST, 'List of number of days of gestation used to define term, early preterm, late preterm and ' 'post term delivery'), - 'allowed_interventions': Parameter( - Types.LIST, 'list of interventions allowed to run, used in analysis'), # BIRTH WEIGHT... 'mean_birth_weights': Parameter( @@ -868,7 +866,7 @@ def get_and_store_labour_item_codes(self): def initialise_simulation(self, sim): # Update self.current_parameters - pregnancy_helper_functions.update_current_parameter_dictionary(self, list_position=0) + # pregnancy_helper_functions.update_current_parameter_dictionary(self, list_position=0) # We call the following function to store the required consumables for the simulation run within the appropriate # dictionary @@ -901,7 +899,6 @@ def initialise_simulation(self, sim): full_blood_count_hb_pn=DxTest( property='pn_anaemia_following_pregnancy', target_categories=['mild', 'moderate', 'severe'], - item_codes=self.item_codes_lab_consumables['full_blood_count'], sensitivity=1.0), ) @@ -1641,11 +1638,7 @@ def prophylactic_labour_interventions(self, hsi_event): params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info person_id = hsi_event.target - - # params['allowed_interventions'] contains a list of interventions delivered in this module. Removal of - # interventions from this list within test/analysis will stop this intervention from running - if 'prophylactic_labour_interventions' not in params['allowed_interventions']: - return + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' # We determine if the HCW will administer antibiotics for women with premature rupture of membranes if df.at[person_id, 'ps_premature_rupture_of_membranes']: @@ -1656,20 +1649,28 @@ def prophylactic_labour_interventions(self, hsi_event): mni[person_id]['abx_for_prom_given'] = True else: - # Run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', - hsi_event=hsi_event) - - # If she has not already receive antibiotics, we check for consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # Run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', + # hsi_event=hsi_event) + # + # # If she has not already receive antibiotics, we check for consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['abx_for_prom'], + # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) + # + # # Then query if these consumables are available during this HSI And provide if available. + # # Antibiotics for from reduce risk of newborn sepsis within the first + # # week of life + # if avail and sf_check: + + abx_prom_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='abx_for_prom', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_iv_abx'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['abx_for_prom'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # Then query if these consumables are available during this HSI And provide if available. - # Antibiotics for from reduce risk of newborn sepsis within the first - # week of life - if avail and sf_check: + if abx_prom_delivered: mni[person_id]['abx_for_prom_given'] = True # ------------------------------ STEROIDS FOR PRETERM LABOUR ------------------------------- @@ -1677,14 +1678,22 @@ def prophylactic_labour_interventions(self, hsi_event): if mni[person_id]['labour_state'] == 'early_preterm_labour' or \ mni[person_id]['labour_state'] == 'late_preterm_labour': - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['antenatal_steroids'], + # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) + # + # # If available they are given. Antenatal steroids reduce a preterm newborns chance of developing + # # respiratory distress syndrome and of death associated with prematurity + # if avail: + + steroids_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='antenatal_corticosteroids', hsi_event=hsi_event, + q_param=None, cons=self.item_codes_lab_consumables['antenatal_steroids'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # If available they are given. Antenatal steroids reduce a preterm newborns chance of developing - # respiratory distress syndrome and of death associated with prematurity - if avail: + if steroids_delivered: mni[person_id]['corticosteroids_given'] = True def determine_delivery_mode_in_spe_or_ec(self, person_id, hsi_event, complication): @@ -1723,12 +1732,10 @@ def assessment_and_treatment_of_severe_pre_eclampsia_mgso4(self, hsi_event, labo df = self.sim.population.props params = self.current_parameters person_id = hsi_event.target + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' # Women who have been admitted for delivery due to severe pre-eclampsia AND have already received magnesium # before moving to the labour ward do not receive the intervention again - if 'assessment_and_treatment_of_severe_pre_eclampsia' not in params['allowed_interventions']: - return - if (df.at[person_id, 'ps_htn_disorders'] == 'severe_pre_eclamp') or \ (df.at[person_id, 'pn_htn_disorders'] == 'severe_pre_eclamp'): @@ -1736,19 +1743,27 @@ def assessment_and_treatment_of_severe_pre_eclampsia_mgso4(self, hsi_event, labo if (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'none') and (labour_stage == 'ip'): self.determine_delivery_mode_in_spe_or_ec(person_id, hsi_event, 'spe') - # Run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', - hsi_event=hsi_event) - - # Define and check for the required consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # Run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', + # hsi_event=hsi_event) + # + # # Define and check for the required consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['magnesium_sulfate'], + # opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) + # + # # If the consumables are available - the intervention is delivered. IV magnesium reduces the + # # probability that a woman with severe pre-eclampsia will experience eclampsia in labour + # if avail and sf_check: + + mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='mgso4', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_anticonvulsant'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['magnesium_sulfate'], opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) - # If the consumables are available - the intervention is delivered. IV magnesium reduces the - # probability that a woman with severe pre-eclampsia will experience eclampsia in labour - if avail and sf_check: + if mag_sulph_delivered: df.at[person_id, 'la_severe_pre_eclampsia_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) @@ -1764,22 +1779,26 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): person_id = hsi_event.target params = self.current_parameters - # If the treatment is not allowed to be delivered or it has already been delivered the function won't run - if 'assessment_and_treatment_of_hypertension' not in params['allowed_interventions']: - return - + # If the treatment has already been delivered the function won't run if (df.at[person_id, 'ps_htn_disorders'] != 'none') or (df.at[person_id, 'pn_htn_disorders'] != 'none'): - # Then query if these consumables are available during this HSI - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, - cons=self.item_codes_lab_consumables['iv_antihypertensives'], + # # Then query if these consumables are available during this HSI + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['iv_antihypertensives'], + # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) + # + # # If they are available then the woman is started on treatment. Intravenous antihypertensive reduce a + # # womans risk of progression from mild to severe gestational hypertension ANd reduce risk of death for + # # women with severe pre-eclampsia and eclampsia + # if avail: + + iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='oral_antihypertensives', hsi_event=hsi_event, + q_param=None, cons=self.item_codes_lab_consumables['iv_antihypertensives'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # If they are available then the woman is started on treatment. Intravenous antihypertensive reduce a - # womans risk of progression from mild to severe gestational hypertension ANd reduce risk of death for - # women with severe pre-eclampsia and eclampsia - if avail: + if iv_anti_htns_delivered: df.at[person_id, 'la_maternal_hypertension_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event) @@ -1790,9 +1809,16 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): dose = (7 * 4) * 6 # approximating 4 tablets a day, for 6 weeks cons = {_i: dose for _i in self.item_codes_lab_consumables['oral_antihypertensives']} - avail = hsi_event.get_consumables(item_codes=cons) - if avail: + oral_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='oral_antihypertensives', hsi_event=hsi_event, + q_param=None, cons=cons) + + # avail = hsi_event.get_consumables(item_codes=cons) + + # if avail: + + if oral_anti_htns_delivered: df.at[person_id, 'la_gest_htn_on_treatment'] = True def assessment_and_treatment_of_eclampsia(self, hsi_event, labour_stage): @@ -1808,27 +1834,32 @@ def assessment_and_treatment_of_eclampsia(self, hsi_event, labour_stage): df = self.sim.population.props person_id = hsi_event.target params = self.current_parameters - - if 'assessment_and_treatment_of_eclampsia' not in params['allowed_interventions']: - return + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' if (df.at[person_id, 'ps_htn_disorders'] == 'eclampsia') or \ (df.at[person_id, 'pn_htn_disorders'] == 'eclampsia'): - # Run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', - hsi_event=hsi_event) - - # define and check required consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # Run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', + # hsi_event=hsi_event) + # + # # define and check required consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['magnesium_sulfate'], + # opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) + + mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='mgso4', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_anticonvulsant'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['magnesium_sulfate'], opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) if (labour_stage == 'ip') and (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'none'): self.determine_delivery_mode_in_spe_or_ec(person_id, hsi_event, 'ec') - if avail and sf_check: + # if avail and sf_check: + if mag_sulph_delivered: # Treatment with magnesium reduces a womans risk of death from eclampsia df.at[person_id, 'la_eclampsia_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) @@ -1846,6 +1877,7 @@ def assessment_for_assisted_vaginal_delivery(self, hsi_event, indication): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.current_parameters person_id = hsi_event.target + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' def refer_for_cs(): if not indication == 'other': @@ -1856,8 +1888,7 @@ def refer_for_cs(): elif indication == 'ol': mni[person_id]['cs_indication'] = indication - if ('assessment_and_treatment_of_obstructed_labour' not in params['allowed_interventions']) or \ - (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'caesarean_now') or \ + if (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'caesarean_now') or \ (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'caesarean_future'): return @@ -1866,20 +1897,29 @@ def refer_for_cs(): # We assume women with CPD cannot be delivered via AVD and will require a caesarean if not mni[person_id]['cpd']: - # If the general package is available AND the facility has the correct tools to carry out the - # delivery then it can occur - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # If the general package is available AND the facility has the correct tools to carry out the + # # delivery then it can occur + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['vacuum'], + # opt_cons=self.item_codes_lab_consumables['obstructed_labour']) + # + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='avd', + # hsi_event=hsi_event) + + # if avail and sf_check: + + avd_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='avd', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_avd'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['vacuum'], - opt_cons=self.item_codes_lab_consumables['obstructed_labour']) - - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='avd', - hsi_event=hsi_event) + opt_cons=self.item_codes_lab_consumables['obstructed_labour'], + equipment={'Delivery Forceps', 'Vacuum extractor'}) - if avail and sf_check: + if avd_delivered: # Add used equipment - hsi_event.add_equipment({'Delivery Forceps', 'Vacuum extractor'}) + # hsi_event.add_equipment({'Delivery Forceps', 'Vacuum extractor'}) pregnancy_helper_functions.log_met_need(self, f'avd_{indication}', hsi_event) @@ -1910,9 +1950,7 @@ def assessment_and_treatment_of_maternal_sepsis(self, hsi_event, labour_stage): df = self.sim.population.props params = self.current_parameters person_id = hsi_event.target - - if 'assessment_and_treatment_of_maternal_sepsis' not in params['allowed_interventions']: - return + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' if ( df.at[person_id, 'la_sepsis'] or @@ -1920,18 +1958,26 @@ def assessment_and_treatment_of_maternal_sepsis(self, hsi_event, labour_stage): ((labour_stage == 'ip') and df.at[person_id, 'ps_chorioamnionitis']) or (labour_stage == 'pp' and df.at[person_id, 'pn_sepsis_late_postpartum'])): - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', - hsi_event=hsi_event) - - # Define and check available consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', + # hsi_event=hsi_event) + # + # # Define and check available consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['maternal_sepsis_core'], + # opt_cons=self.item_codes_lab_consumables['maternal_sepsis_optional']) + # + # # If delivered this intervention reduces a womans risk of dying from sepsis + # if avail and sf_check: + + sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='sepsis_treatment', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_iv_abx'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['maternal_sepsis_core'], opt_cons=self.item_codes_lab_consumables['maternal_sepsis_optional']) - # If delivered this intervention reduces a womans risk of dying from sepsis - if avail and sf_check: + if sepsis_treatment_delivered: df.at[person_id, 'la_sepsis_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'sepsis_abx', hsi_event) @@ -1944,13 +1990,9 @@ def assessment_and_plan_for_antepartum_haemorrhage(self, hsi_event): (STR) 'hc' == health centre, 'hp' == hospital """ df = self.sim.population.props - params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info person_id = hsi_event.target - if 'assessment_and_plan_for_referral_antepartum_haemorrhage' not in params['allowed_interventions']: - return - # We assume that any woman who has been referred from antenatal inpatient care due to haemorrhage are # automatically scheduled for blood transfusion if (df.at[person_id, 'ps_antepartum_haemorrhage'] != 'none') and (df.at[person_id, @@ -1976,13 +2018,9 @@ def assessment_for_referral_uterine_rupture(self, hsi_event): (STR) 'hc' == health centre, 'hp' == hospital """ df = self.sim.population.props - params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info person_id = hsi_event.target - if 'assessment_and_plan_for_referral_uterine_rupture' not in params['allowed_interventions']: - return - # When uterine rupture is present, the mni dictionary is updated to allow treatment to be scheduled if df.at[person_id, 'la_uterine_rupture']: mni[person_id]['referred_for_surgery'] = True @@ -2000,23 +2038,29 @@ def active_management_of_the_third_stage_of_labour(self, hsi_event): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.current_parameters person_id = hsi_event.target - - if 'active_management_of_the_third_stage_of_labour' not in params['allowed_interventions']: - return - - # Define and check available consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' + + # # Define and check available consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['amtsl'], + # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) + # + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', + # hsi_event=hsi_event) + # + # # This treatment reduces a womans risk of developing uterine atony AND retained placenta, both of which are + # # preceding causes of postpartum haemorrhage + # if avail and sf_check: + + amtsl_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='amtsl', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_uterotonic'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['amtsl'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', - hsi_event=hsi_event) - - # This treatment reduces a womans risk of developing uterine atony AND retained placenta, both of which are - # preceding causes of postpartum haemorrhage - if avail and sf_check: + if amtsl_delivered: mni[person_id]['amtsl_given'] = True def assessment_and_treatment_of_pph_uterine_atony(self, hsi_event): @@ -2031,23 +2075,29 @@ def assessment_and_treatment_of_pph_uterine_atony(self, hsi_event): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.current_parameters person_id = hsi_event.target - - if 'assessment_and_treatment_of_pph_uterine_atony' not in params['allowed_interventions']: - return + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' if df.at[person_id, 'la_postpartum_haem'] and not mni[person_id]['retained_placenta']: - # Define and check available consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # Define and check available consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['pph_core'], + # opt_cons=self.item_codes_lab_consumables['pph_optional']) + # + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', + # hsi_event=hsi_event) + # + # if avail and sf_check: + + pph_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='pph_treatment_uterotonics', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_uterotonic'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['pph_core'], opt_cons=self.item_codes_lab_consumables['pph_optional']) - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', - hsi_event=hsi_event) - - if avail and sf_check: + if pph_treatment_delivered: pregnancy_helper_functions.log_met_need(self, 'uterotonics', hsi_event) # We apply a probability that this treatment will stop a womans bleeding in the first instance @@ -2075,26 +2125,32 @@ def assessment_and_treatment_of_pph_retained_placenta(self, hsi_event): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.current_parameters person_id = hsi_event.target - - if 'assessment_and_treatment_of_pph_retained_placenta' not in params['allowed_interventions']: - return + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' if ( (df.at[person_id, 'la_postpartum_haem'] and mni[person_id]['retained_placenta']) or df.at[person_id, 'pn_postpartum_haem_secondary'] ): - # Log the consumables but dont condition the treatment on their availability - the primary mechanism of this - # intervention doesnt require consumables - hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['pph_optional']) - - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='man_r_placenta', - hsi_event=hsi_event) + # # Log the consumables but dont condition the treatment on their availability - the primary mechanism of this + # # intervention doesnt require consumables + # hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['pph_optional']) + # + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='man_r_placenta', + # hsi_event=hsi_event) + # + # # Similar to uterotonics we apply a probability that this intervention will successfully stop + # # bleeding to ensure some women go on to require further care + # if sf_check: + + pph_mrrp_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='pph_treatment_mrrp', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_man_r_placenta'], params[f'mean_hcw_competence_{deliv_location}']], + cons=None, + opt_cons=self.item_codes_lab_consumables['pph_optional']) - # Similar to uterotonics we apply a probability that this intervention will successfully stop - # bleeding to ensure some women go on to require further care - if sf_check: + if pph_mrrp_delivered: pregnancy_helper_functions.log_met_need(self, 'man_r_placenta', hsi_event) if params['prob_successful_manual_removal_placenta'] > self.rng.random_sample(): @@ -2120,22 +2176,31 @@ def surgical_management_of_pph(self, hsi_event): person_id = hsi_event.target df = self.sim.population.props params = self.current_parameters - - # We log the required consumables and condition the surgery happening on the availability of the - # first consumable in this package, the anaesthetic required for the surgery - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' + + # # We log the required consumables and condition the surgery happening on the availability of the + # # first consumable in this package, the anaesthetic required for the surgery + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['obstetric_surgery_core'], + # opt_cons=self.item_codes_lab_consumables['obstetric_surgery_optional']) + # + # # run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='surg', + # hsi_event=hsi_event) + # + # if avail and sf_check: + # # Add used equipment + # hsi_event.add_equipment(hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery')) + + pph_surg_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='pph_treatment_surg', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_surg'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['obstetric_surgery_core'], - opt_cons=self.item_codes_lab_consumables['obstetric_surgery_optional']) - - # run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='surg', - hsi_event=hsi_event) - - if avail and sf_check: - # Add used equipment - hsi_event.add_equipment(hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery')) + opt_cons=self.item_codes_lab_consumables['obstetric_surgery_optional'], + equipment=hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery')) + if pph_surg_delivered: # determine if uterine preserving surgery will be successful treatment_success_pph = params['success_rate_pph_surgery'] > self.rng.random_sample() @@ -2162,20 +2227,29 @@ def blood_transfusion(self, hsi_event): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.current_parameters df = self.sim.population.props - - # Check consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' + + # # Check consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_lab_consumables['blood_transfusion'], + # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) + # + # # check HCW + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='blood_tran', + # hsi_event=hsi_event) + # + # if avail and sf_check: + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + + blood_transfusion_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='blood_transfusion', hsi_event=hsi_event, + q_param=[params['prob_hcw_avail_blood_tran'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['blood_transfusion'], - opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - - # check HCW - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='blood_tran', - hsi_event=hsi_event) - - if avail and sf_check: - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + opt_cons=self.item_codes_lab_consumables['blood_test_equipment'], + equipment={'Drip stand', 'Infusion pump'}) + if blood_transfusion_delivered: mni[person_id]['received_blood_transfusion'] = True pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) @@ -2199,27 +2273,41 @@ def assessment_and_treatment_of_anaemia(self, hsi_event): mother = df.loc[person_id] mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - # Add used equipment - hsi_event.add_equipment({'Analyser, Haematology'}) - - # Use dx_test function to assess anaemia status - test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='full_blood_count_hb_pn', hsi_event=hsi_event) - - hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['blood_test_equipment']) - - # Check consumables - if test_result: + # # Add used equipment + # hsi_event.add_equipment({'Analyser, Haematology'}) + # + # # Use dx_test function to assess anaemia status + # test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + # dx_tests_to_run='full_blood_count_hb_pn', hsi_event=hsi_event) + # + # hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['blood_test_equipment']) + # + # # Check consumables + # if test_result: + + full_blood_count_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='full_blood_count', hsi_event=hsi_event, + q_param=None, cons=None, + opt_cons=self.item_codes_lab_consumables['blood_test_equipment'], + equipment={'Analyser, Haematology'}, dx_test='full_blood_count_hb_pn') + + if full_blood_count_delivered: # Start iron and folic acid supplementation for women not already receiving this if not mother.la_iron_folic_acid_postnatal: days = int((6 - df.at[person_id, 'pn_postnatal_period_in_weeks']) * 7) dose = days * 3 cons = {_i: dose for _i in self.item_codes_lab_consumables['iron_folic_acid']} - avail = hsi_event.get_consumables(item_codes=cons) + # avail = hsi_event.get_consumables(item_codes=cons) + # + # # Start iron and folic acid treatment + # if avail: + + iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='iron_folic_acid', + hsi_event=hsi_event, q_param=None, cons=cons) - # Start iron and folic acid treatment - if avail: + if iron_folic_acid_delivered: df.at[person_id, 'la_iron_folic_acid_postnatal'] = True if self.rng.random_sample() < params['effect_of_ifa_for_resolving_anaemia']: @@ -2265,11 +2353,16 @@ def interventions_delivered_pre_discharge(self, hsi_event): # ------------------------------- Postnatal iron and folic acid --------------------------------------------- cons = {_i: params['number_ifa_tablets_required_postnatally'] for _i in self.item_codes_lab_consumables['iron_folic_acid']} - avail = hsi_event.get_consumables(item_codes=cons) + + iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='iron_folic_acid', + hsi_event=hsi_event, q_param=None, cons=cons) + + # avail = hsi_event.get_consumables(item_codes=cons) # Women are started on iron and folic acid for the next three months which reduces risk of anaemia in the # postnatal period - if avail and (self.rng.random_sample() < params['prob_adherent_ifa']): + if iron_folic_acid_delivered and (self.rng.random_sample() < params['prob_adherent_ifa']): df.at[person_id, 'la_iron_folic_acid_postnatal'] = True def run_if_receives_skilled_birth_attendance_cant_run(self, hsi_event): @@ -2882,6 +2975,7 @@ def apply(self, person_id, squeeze_factor): mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info df = self.sim.population.props params = self.module.current_parameters + deliv_location = 'hc' if self.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' if not df.at[person_id, 'is_alive']: return @@ -2908,21 +3002,28 @@ def apply(self, person_id, squeeze_factor): self.module.assessment_for_assisted_vaginal_delivery(self, indication='spe_ec') # LOG CONSUMABLES FOR DELIVERY... - # We assume all deliveries require this basic package of consumables - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, - cons=self.module.item_codes_lab_consumables['delivery_core'], + # # We assume all deliveries require this basic package of consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_lab_consumables['delivery_core'], + # opt_cons=self.module.item_codes_lab_consumables['delivery_optional']) + # + # # If the clean delivery kit consumable is available, we assume women benefit from clean delivery + # if avail: + # mni[person_id]['clean_birth_practices'] = True + + birth_kit_used = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='birth_kit', + hsi_event=self, cons=self.module.item_codes_lab_consumables['delivery_core'], opt_cons=self.module.item_codes_lab_consumables['delivery_optional']) - # Add used equipment - self.add_equipment({'Delivery set', 'Weighing scale', 'Stethoscope, foetal, monaural, Pinard, plastic', - 'Resuscitaire', 'Sphygmomanometer', 'Tray, emergency', 'Suction machine', - 'Thermometer', 'Drip stand', 'Infusion pump'}) - - # If the clean delivery kit consumable is available, we assume women benefit from clean delivery - if avail: + if birth_kit_used: mni[person_id]['clean_birth_practices'] = True + # Add used equipment + self.add_equipment({'Delivery set', 'Weighing scale', 'Stethoscope, foetal, monaural, Pinard, plastic', + 'Resuscitaire', 'Sphygmomanometer', 'Tray, emergency', 'Suction machine', + 'Thermometer', 'Drip stand', 'Infusion pump'}) # ===================================== PROPHYLACTIC CARE =================================================== # The following function manages the consumables and administration of prophylactic interventions in labour # (clean delivery practice, antibiotics for PROM, steroids for preterm labour) @@ -2990,14 +3091,22 @@ def apply(self, person_id, squeeze_factor): if not mni[person_id]['sought_care_for_complication']: # TODO: potential issue is that this consumable is being logged now for every birth as opposed to # for each birth where resuscitation of the newborn is required - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, cons=self.module.item_codes_lab_consumables['resuscitation'], opt_cons=None) - - # Run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, - sf='neo_resus', - hsi_event=self) - if sf_check and avail: + + neo_resus_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='neo_resus', hsi_event=self, + q_param=[params['prob_hcw_avail_neo_resus'], params[f'mean_hcw_competence_{deliv_location}']], + cons=self.module.item_codes_lab_consumables['resuscitation']) + + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, cons=self.module.item_codes_lab_consumables['resuscitation'], opt_cons=None) + # + # # Run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, + # sf='neo_resus', + # hsi_event=self) + # if sf_check and avail: + + if neo_resus_delivered: mni[person_id]['neo_will_receive_resus_if_needed'] = True # ========================================== SCHEDULING CEMONC CARE ========================================= @@ -3198,6 +3307,7 @@ def apply(self, person_id, squeeze_factor): df = self.sim.population.props mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info params = self.module.current_parameters + deliv_location = 'hc' if self.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' # We use the variable self.timing to differentiate between women sent to this event during labour and women # sent after labour @@ -3207,22 +3317,30 @@ def apply(self, person_id, squeeze_factor): # delivered if mni[person_id]['referred_for_cs'] and self.timing == 'intrapartum': - # We log the required consumables and condition the caesarean happening on the availability of the - # first consumable in this package, the anaesthetic required for the surgery - avail = pregnancy_helper_functions.return_cons_avail( - self.module, self, + # # We log the required consumables and condition the caesarean happening on the availability of the + # # first consumable in this package, the anaesthetic required for the surgery + # avail = pregnancy_helper_functions.return_cons_avail( + # self.module, self, + # cons=self.module.item_codes_lab_consumables['caesarean_delivery_core'], + # opt_cons=self.module.item_codes_lab_consumables['caesarean_delivery_optional']) + # + # # We check that the HCW will deliver the intervention + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, sf='surg', + # hsi_event=self) + + cs_delivered = pregnancy_helper_functions.check_int_deliverable( + self.module, int_name='abx_for_prom', hsi_event=self, + q_param=[params['prob_hcw_avail_surg'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.module.item_codes_lab_consumables['caesarean_delivery_core'], opt_cons=self.module.item_codes_lab_consumables['caesarean_delivery_optional']) - # We check that the HCW will deliver the intervention - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, sf='surg', - hsi_event=self) - # Block CS delivery for this analysis if params['la_analysis_in_progress'] and (params['cemonc_availability'] == 0.0): logger.debug(key='message', data="cs delivery blocked for this analysis") - elif (avail and sf_check) or (mni[person_id]['cs_indication'] == 'other'): + # elif (avail and sf_check) or (mni[person_id]['cs_indication'] == 'other'): + elif cs_delivered or (mni[person_id]['cs_indication'] == 'other'): + # If intervention is delivered - add used equipment self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index e8f32ed7af..c1b34abae6 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -900,8 +900,15 @@ def kangaroo_mother_care(self, hsi_event): if (df.at[person_id, 'nb_low_birth_weight_status'] != 'normal_birth_weight') or \ (df.at[person_id, 'nb_low_birth_weight_status'] != 'macrosomia'): + kmc_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='kmc', + hsi_event=hsi_event, + q_param=[params['prob_kmc_available']], + cons=None) + # Check KMC can be delivered - if self.rng.random_sample() < params['prob_kmc_available']: + # if self.rng.random_sample() < params['prob_kmc_available']: + if kmc_delivered: # Store treatment as a property of the newborn used to apply treatment effect df.at[person_id, 'nb_kangaroo_mother_care'] = True @@ -952,6 +959,8 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): """ df = self.sim.population.props person_id = int(hsi_event.target) + l_params = self.sim.modules['Labour'].current_parameters + pnc_location = 'hc' if facility_type == '1a' else 'hp' # We assume that only hospitals are able to deliver full supportive care for neonatal sepsis, full supportive # care evokes a stronger treatment effect than injectable antibiotics alone @@ -959,35 +968,53 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if df.at[person_id, 'nb_early_onset_neonatal_sepsis'] or df.at[person_id, 'pn_sepsis_late_neonatal'] or\ df.at[person_id, 'pn_sepsis_early_neonatal']: - # Run HCW check - sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - sf='iv_abx', - hsi_event=hsi_event) + + # # Run HCW check + # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], + # sf='iv_abx', + # hsi_event=hsi_event) if facility_type != '1a': - # check consumables - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # # check consumables + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_nb_consumables['sepsis_supportive_care_core'], + # opt_cons=self.item_codes_nb_consumables['sepsis_supportive_care_optional']) + + # # Then, if the consumables are available, treatment for sepsis is delivered + # if avail and sf_check: + + neo_sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='neo_sepsis_treatment_supp_care', hsi_event=hsi_event, + q_param=[l_params['prob_hcw_avail_iv_abx'], l_params[f'mean_hcw_competence_{pnc_location}']], cons=self.item_codes_nb_consumables['sepsis_supportive_care_core'], - opt_cons=self.item_codes_nb_consumables['sepsis_supportive_care_optional']) + opt_cons=self.item_codes_nb_consumables['sepsis_supportive_care_optional'], + equipment={'Drip stand', 'Infusion pump'}) - # Then, if the consumables are available, treatment for sepsis is delivered - if avail and sf_check: + if neo_sepsis_treatment_delivered: df.at[person_id, 'nb_supp_care_neonatal_sepsis'] = True pregnancy_helper_functions.log_met_need(self, 'neo_sep_supportive_care', hsi_event) - hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # The same pattern is then followed for health centre care else: - avail = pregnancy_helper_functions.return_cons_avail( - self, hsi_event, + # avail = pregnancy_helper_functions.return_cons_avail( + # self, hsi_event, + # cons=self.item_codes_nb_consumables['sepsis_abx'], + # opt_cons=self.item_codes_nb_consumables['iv_drug_equipment']) + # + # if avail and sf_check: + neo_sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( + self, int_name='neo_sepsis_treatment_abx', hsi_event=hsi_event, + q_param=[l_params['prob_hcw_avail_iv_abx'], l_params[f'mean_hcw_competence_{pnc_location}']], cons=self.item_codes_nb_consumables['sepsis_abx'], - opt_cons=self.item_codes_nb_consumables['iv_drug_equipment']) + opt_cons=self.item_codes_nb_consumables['iv_drug_equipment'], + equipment={'Drip stand', 'Infusion pump'}) - if avail and sf_check: + if neo_sepsis_treatment_delivered: df.at[person_id, 'nb_inj_abx_neonatal_sepsis'] = True pregnancy_helper_functions.log_met_need(self, 'neo_sep_abx', hsi_event) - hsi_event.add_equipment({'Drip stand', 'Infusion pump', 'Oxygen cylinder, with regulator'}) + # hsi_event.add_equipment({'Drip stand', 'Infusion pump', 'Oxygen cylinder, with regulator'}) def link_twins(self, child_one, child_two, mother_id): """ diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index dd3dbce209..ef4fab0e79 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -63,7 +63,8 @@ def get_list_of_items(self, item_list): return codes -def check_int_deliverable(self, int_name, hsi_event, q_param, cons, opt_cons=None, equipment=None, dx_test=None): +def check_int_deliverable(self, int_name, hsi_event, + q_param=None, cons=None, opt_cons=None, equipment=None, dx_test=None): """ This function is called to determine if an intervention within the MNH modules can be delivered to an individual during a given HSI. This applied to all MNH interventions. If analyses are being conducted in which the probability @@ -71,7 +72,6 @@ def check_int_deliverable(self, int_name, hsi_event, q_param, cons, opt_cons=Non intervention delivery is determined by any module-level quality parameters, consumable availability, and (if applicable) the results of any dx_tests. Equipment is also declared. - TODO: add to newborn interventions TODO: what about interventions outside mnh modules : param self: module @@ -127,18 +127,23 @@ def check_int_deliverable(self, int_name, hsi_event, q_param, cons, opt_cons=Non consumables = False test = False - if all([self.rng.random_sample() < value for value in q_param]) or (q_param is None): + if ((q_param is None) or + all([self.rng.random_sample() < value for value in q_param])): quality = True if equipment is not None: - hsi_event.add_equipment({equipment}) + hsi_event.add_equipment(equipment) - if ((hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons if not None else [])) or - (cons is None)): + if ((cons is None) or + (hsi_event.get_consumables(item_codes=cons if not None else [], + optional_item_codes=opt_cons if not None else []))): consumables = True - if (self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run=dx_test, hsi_event=hsi_event) - or (dx_test is None)): + if cons is None and opt_cons is not None: + hsi_event.get_consumables(item_codes= [], optional_item_codes=opt_cons) + + if ((dx_test is None) or + (self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run=dx_test, hsi_event=hsi_event))): test = True if quality and consumables and test: diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index efb37029bb..9c7e0c172f 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -31,7 +31,7 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=2000) + sim.make_initial_population(n=5000) sim.simulate(end_date=Date(2025, 1, 2)) output= parse_log_file(sim.log_filepath) From 0c3af64a392e4f609b331c67d5cc82121ec06680 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 16 Oct 2024 08:13:20 +0100 Subject: [PATCH 048/103] work on cohort --- .../dummy_cohort_azure_calib.py | 14 ++++++++--- src/tlo/methods/postnatal_supervisor.py | 5 ++-- src/tlo/methods/pregnancy_helper_functions.py | 24 ++++++++++++------ src/tlo/methods/pregnancy_supervisor.py | 25 ++++++++++++------- tests/test_mnh_cohort.py | 2 +- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 621bb9516f..83ceb3fbf0 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -6,10 +6,12 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' scenario_filename = 'cohort_test-2024-10-09T130546Z' +scenario_filename2 = 'cohort_test-2024-10-15T122825Z' -results_folder = get_scenario_outputs(scenario_filename, outputspath)[-1] +results_folder_old = get_scenario_outputs(scenario_filename, outputspath)[-1] +results_folder_new = get_scenario_outputs(scenario_filename2, outputspath)[-1] -def get_data_frames(key): +def get_data_frames(key, results_folder): def sort_df(_df): _x = _df.drop(columns=['date'], inplace=False) return _x.iloc[0] @@ -24,9 +26,13 @@ def sort_df(_df): return results_df -results = {k:get_data_frames(k) for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', - 'service_coverage']} +results_old = {k:get_data_frames(k, results_folder_old) for k in + ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', + 'yearly_mnh_counter_dict']} +results_new = {k:get_data_frames(k, results_folder_new) for k in + ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage', + 'yearly_mnh_counter_dict']} import matplotlib.pyplot as plt import numpy as np diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 183877f4ed..e0f6708fd5 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -646,13 +646,14 @@ def log_new_progressed_cases(disease): # Those women who die the on_death function in demography is applied for person in die_from_htn.loc[die_from_htn].index: + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', originating_module=self.sim.modules['PostnatalSupervisor']) del self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person] # ----------------------------------------- CARE SEEKING ------------------------------------------------------ - # We now use the the pn_emergency_event_mother property that has just been set for women who are experiencing + # We now use the pn_emergency_event_mother property that has just been set for women who are experiencing # severe complications to select a subset of women who may choose to seek care can_seek_care = df.loc[df['is_alive'] & df['la_is_postpartum'] & (df['pn_postnatal_period_in_weeks'] == week) & df['pn_emergency_event_mother'] & ~df['hs_is_inpatient']] @@ -838,7 +839,7 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in # If this neonate will die then we make the appropriate changes if self.rng.random_sample() < risk_of_death: - + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause}_n_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=cause, originating_module=self.sim.modules['PostnatalSupervisor']) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index ef4fab0e79..d3d36dc070 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -39,13 +39,22 @@ def generate_mnh_outcome_counter(): 'macrosomia', 'small_for_gestational_age', 'early_onset_sepsis', 'late_onset_sepsis', # death outcomes - 'direct_mat_death', 'six_week_survivors', + 'direct_mat_death', 'six_week_survivors','induced_abortion_m_death', 'spontaneous_abortion_m_death', + 'ectopic_pregnancy_m_death', 'severe_gestational_hypertension_m_death', + 'severe_pre_eclampsia_m_death', 'eclampsia_m_death', 'antepartum_haemorrhage_m_death', + 'antenatal_sepsis_m_death', + 'intrapartum_sepsis_m_death', 'postpartum_sepsis_m_death', 'uterine_rupture_m_death', + 'postpartum_haemorrhage_m_death','secondary_postpartum_haemorrhage_m_death', + 'early_onset_sepsis_n_death', 'late_onset_sepsis_n_death', 'encephalopathy_n_death', + 'neonatal_respiratory_depression_n_death', 'preterm_other_n_death', + 'respiratory_distress_syndrome_n_death', 'congenital_heart_anomaly_n_death', + 'limb_or_musculoskeletal_anomaly_n_death', 'urogenital_anomaly_n_death', 'digestive_anomaly_n_death', + 'other_anomaly_n_death', # service coverage outcomes 'anc0', 'anc1', 'anc2', 'anc3', 'anc4', 'anc5', 'anc6', 'anc7', 'anc8', 'anc8+', 'home_birth_delivery', 'hospital_delivery', 'health_centre_delivery', - 'm_pnc0', 'm_pnc1', 'm_pnc2', 'm_pnc3+', 'n_pnc0', 'n_pnc1', 'n_pnc2', 'n_pnc3+', - ] + 'm_pnc0', 'm_pnc1', 'm_pnc2', 'm_pnc3+', 'n_pnc0', 'n_pnc1', 'n_pnc2', 'n_pnc3+'] mnh_outcome_counter = {k: 0 for k in outcome_list} @@ -453,7 +462,7 @@ def log_mni_for_maternal_death(self, person_id): logger.info(key='death_mni', data=mni_to_log) -def calculate_risk_of_death_from_causes(self, risks): +def calculate_risk_of_death_from_causes(self, risks, target): """ This function calculates risk of death in the context of one or more 'death causing' complications in a mother of a newborn. In addition, it determines if the complication(s) will cause death or not. If death occurs the function @@ -481,7 +490,8 @@ def calculate_risk_of_death_from_causes(self, risks): # Now use the list of probabilities to conduct a weighted random draw to determine primary cause of death cause_of_death = self.rng.choice(list(risks.keys()), p=probs) - # Return the primary cause of death so that it can be passed to the demography function + # Return and log the primary cause of death so that it can be passed to the demography function + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause_of_death}_{target}_death'] += 1 return cause_of_death else: # Return false if death will not occur @@ -592,7 +602,7 @@ def apply_effect_of_anaemia(cause): risks.update(risk) # Call return the result from calculate_risk_of_death_from_causes function - return calculate_risk_of_death_from_causes(self, risks) + return calculate_risk_of_death_from_causes(self, risks, target='m') # if she is not at risk of death as she has no complications we return false to the module return False @@ -662,7 +672,7 @@ def check_for_risk_of_death_from_cause_neonatal(self, individual_id): risks.update(risk) # Return the result from calculate_risk_of_death_from_causes function (returns primary cause of death or False) - return calculate_risk_of_death_from_causes(self, risks) + return calculate_risk_of_death_from_causes(self, risks, target='n') # if they is not at risk of death as they has no complications we return False to the module return False diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 34ac27a916..c1cf4a67f8 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -1244,6 +1244,7 @@ def apply_risk_of_death_from_hypertension(self, gestation_of_interest): if not at_risk_of_death_htn.loc[at_risk_of_death_htn].empty: # Those women who die have InstantaneousDeath scheduled for person in at_risk_of_death_htn.loc[at_risk_of_death_htn].index: + self.mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', originating_module=self.sim.modules['PregnancySupervisor']) @@ -1997,6 +1998,7 @@ def apply(self, individual_id): pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) mni[individual_id]['delete_mni'] = True + self.module.mnh_outcome_counter[f'{self.cause}_m_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=f'{self.cause}', originating_module=self.sim.modules['PregnancySupervisor']) @@ -2278,15 +2280,20 @@ def rate (count, denom, multiplier): neonatal_deaths = len(df[(df['date_of_death'].dt.year == self.sim.date.year - 1) & (df['age_days'] <= 28)]) stillbirths = c['antenatal_stillbirth'] + c['intrapartum_stillbirth'] - logger.info(key='deaths_and_stillbirths', - data={'antenatal_sbr': rate(c['antenatal_stillbirth'], total_births, 1000), - 'intrapartum_sbr': rate(c['intrapartum_stillbirth'], total_births, 1000), - 'total_stillbirths': stillbirths, - 'sbr': rate(stillbirths, total_births, 1000), - 'neonatal_deaths': neonatal_deaths, - 'nmr' : rate(neonatal_deaths, live_births, 1000), - 'direct_maternal_deaths': c['direct_mat_death'], - 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000)}) + general_death_data = {'antenatal_sbr': rate(c['antenatal_stillbirth'], total_births, 1000), + 'intrapartum_sbr': rate(c['intrapartum_stillbirth'], total_births, 1000), + 'total_stillbirths': stillbirths, + 'sbr': rate(stillbirths, total_births, 1000), + 'neonatal_deaths': neonatal_deaths, + 'nmr' : rate(neonatal_deaths, live_births, 1000), + 'direct_maternal_deaths': c['direct_mat_death'], + 'direct_mmr': rate(c['direct_mat_death'], live_births, 100_000)} + + cause_specific_mmrs = {k: rate(c[k], total_births, 100_000) for k in c if 'm_death' in k} + cause_specific_nmrs = {k: rate(c[k], total_births, 1000) for k in c if 'n_death' in k} + general_death_data.update({**cause_specific_mmrs, **cause_specific_nmrs}) + + logger.info(key='deaths_and_stillbirths', data=general_death_data) # Finally log coverage of key health services anc1 = sum(c[f'anc{i}'] for i in range(1, 9)) + c['anc8+'] diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 9c7e0c172f..3c6a46d43d 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -31,7 +31,7 @@ def test_run_sim_with_mnh_cohort(tmpdir, seed): "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=5000) + sim.make_initial_population(n=2500) sim.simulate(end_date=Date(2025, 1, 2)) output= parse_log_file(sim.log_filepath) From 22a5e44312ad4d2f1d955b70399ae9569efb13c0 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:00:22 +0100 Subject: [PATCH 049/103] Log everything to simulation, as events logger doesn't seem to be visible to all modules. For now add person_ID to the dict of info printed as the outer dictionary key logging seems to have a problem. --- src/tlo/events.py | 13 +++++++++---- src/tlo/methods/hsi_event.py | 3 ++- src/tlo/simulation.py | 25 +++++++++++++++++-------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 03bf7c72fa..98832faecb 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -11,9 +11,13 @@ import pandas as pd + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +logger_chain = logging.getLogger('tlo.simulation') +logger_chain.setLevel(logging.INFO) + logger_summary = logging.getLogger(f"{__name__}.summary") logger_summary.setLevel(logging.INFO) @@ -89,7 +93,7 @@ def compare_population_dataframe(self,df_before, df_after): # Create a dictionary for this person # First add event info link_info = { - #'person_ID': idx, + 'person_ID': idx, 'event': str(self), 'event_date': self.sim.date, } @@ -152,13 +156,14 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> if self.target != self.sim.population: row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) - # Create and store event for this individual + # Create and store event for this individual, regardless of whether any property change occurred link_info = { #'person_ID' : self.target, + 'person_ID' : self.target, 'event' : str(self), 'event_date' : self.sim.date, } - # Store property changes as a result of the event for this individual + # Store (if any) property changes as a result of the event for this individual for key in row_before.index: if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] @@ -225,7 +230,7 @@ def run(self): # Log chain_links here if len(chain_links)>0: - logger.info(key='event_chains', + logger_chain.info(key='event_chains', data= chain_links, description='Links forming chains of events for simulated individuals') diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 0c3bc16072..6651a8704a 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -17,7 +17,7 @@ from tlo.methods.healthsystem import HealthSystem # Pointing to the logger in events -logger_chains = logging.getLogger("tlo.methods.event") +logger_chains = logging.getLogger("tlo.simulation") logger_chains.setLevel(logging.INFO) logger = logging.getLogger(__name__) @@ -246,6 +246,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> # Add event details link_info = { + 'person_ID': self.target, 'event' : str(self), 'event_date' : self.sim.date, 'appt_footprint' : str(footprint), diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 582fb4ba1c..fd9fade215 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -107,7 +107,7 @@ def __init__( self.date = self.start_date = start_date self.modules = OrderedDict() self.event_queue = EventQueue() - self.generate_event_chains = None + self.generate_event_chains = True self.generate_event_chains_overwrite_epi = None self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] @@ -292,15 +292,23 @@ def make_initial_population(self, *, n: int) -> None: # When logging events for each individual to reconstruct chains, only the changes in individual properties will be logged. # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. if self.generate_event_chains: + pop_dict = self.population.props.to_dict(orient='index') - logger_chains.info(key='event_chains', + + print(pop_dict) + print(pop_dict.keys()) + for key in pop_dict.keys(): + pop_dict[key]['person_ID'] = key + print("Length of properties", len(pop_dict[0].keys())) + #exit(-1) + logger.info(key='event_chains', data = pop_dict, description='Links forming chains of events for simulated individuals') end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") - def initialise(self, *, end_date: Date, generate_event_chains) -> None: + def initialise(self, *, end_date: Date) -> None: """Initialise all modules in simulation. :param end_date: Date to end simulation on - accessible to modules to allow initialising data structures which may depend (in size for example) on the @@ -312,7 +320,7 @@ def initialise(self, *, end_date: Date, generate_event_chains) -> None: self.date = self.start_date self.end_date = end_date # store the end_date so that others can reference it - self.generate_event_chains = generate_event_chains + #self.generate_event_chains = generate_event_chains if self.generate_event_chains: # Eventually this can be made an option self.generate_event_chains_overwrite_epi = True @@ -413,7 +421,7 @@ def run_simulation_to(self, *, to_date: Date) -> None: if self.show_progress_bar: progress_bar.stop() - def simulate(self, *, end_date: Date, generate_event_chains=False) -> None: + def simulate(self, *, end_date: Date) -> None: """Simulate until the given end date :param end_date: When to stop simulating. Only events strictly before this @@ -421,7 +429,7 @@ def simulate(self, *, end_date: Date, generate_event_chains=False) -> None: clarity. """ start = time.time() - self.initialise(end_date=end_date, generate_event_chains=generate_event_chains) + self.initialise(end_date=end_date) self.run_simulation_to(to_date=end_date) self.finalise(time.time() - start) @@ -470,9 +478,10 @@ def do_birth(self, mother_id: int) -> int: # When individual is born, store their initial properties to provide a starting point to the chain of property # changes that this individual will undergo as a result of events taking place. prop_dict = self.population.props.loc[child_id].to_dict() - + prop_dict['event'] = 'Birth' + prop_dict['event_date'] = self.date child_dict = {child_id : prop_dict} - logger_chains.info(key='event_chains', + logger.info(key='event_chains', data = child_dict, description='Links forming chains of events for simulated individuals') From 53fa8fa1228691d9671e8f6eb765468d2bbdf9f6 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 16 Oct 2024 15:35:06 +0100 Subject: [PATCH 050/103] work on cohort --- .../ResourceFile_PregnancySupervisor.xlsx | 4 ++-- .../cohort_interventions_scenario.py | 18 ++++++++++++++---- .../dummy_cohort_azure_calib.py | 3 ++- src/tlo/methods/pregnancy_helper_functions.py | 11 +++++++---- src/tlo/methods/pregnancy_supervisor.py | 10 ++++++++-- 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index 9dcd06e6f1..cf2d7636cd 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cbc67135d0dce5eb308649937ed4395ac435b8da222a1341cc6f3f5cf2a15e99 -size 23856 +oid sha256:d05fc317eafb2761d03ec2107cf5ce5322fac006289ccb1c0366f10e3ab1f685 +size 23984 diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 5a64abf546..f17877c7a6 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -1,3 +1,8 @@ +import numpy as np +import pandas as pd + +from pathlib import Path + from tlo import Date, logging from tlo.methods import mnh_cohort_module from tlo.methods.fullmodel import fullmodel @@ -11,12 +16,12 @@ def __init__(self): self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) self.pop_size = 5000 - self.number_of_draws = 1 + self.number_of_draws = 3 self.runs_per_draw = 10 def log_configuration(self): return { - 'filename': 'cohort_test', 'directory': './outputs', + 'filename': 'block_intervention_test', 'directory': './outputs', "custom_levels": { "*": logging.WARNING, "tlo.methods.demography": logging.INFO, @@ -39,8 +44,13 @@ def modules(self): mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=self.resources)] def draw_parameters(self, draw_number, rng): - return { - } + interventions_for_analysis = ['blood_transfusion', 'pph_treatment_uterotonics', 'sepsis_treatment'] + + return {'PregnancySupervisor': { + 'analysis_year': 2024, + 'interventions_analysis': True, + 'interventions_under_analysis':[interventions_for_analysis[draw_number-1]], + 'intervention_analysis_availability': 0.0}} if __name__ == '__main__': diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 83ceb3fbf0..711b48c7a0 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -6,7 +6,8 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' scenario_filename = 'cohort_test-2024-10-09T130546Z' -scenario_filename2 = 'cohort_test-2024-10-15T122825Z' +# scenario_filename2 = 'cohort_test-2024-10-15T122825Z' +scenario_filename2 = 'cohort_test-2024-10-16T071357Z' results_folder_old = get_scenario_outputs(scenario_filename, outputspath)[-1] results_folder_new = get_scenario_outputs(scenario_filename2, outputspath)[-1] diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index d3d36dc070..d0adf46621 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -95,15 +95,15 @@ def check_int_deliverable(self, int_name, hsi_event, df = self.sim.population.props individual_id = hsi_event.target - int_avail_df = self.sim.modules['PregnancySupervisor'].current_parameters['intervention_availability'] + p_params = self.sim.modules['PregnancySupervisor'].current_parameters # Firstly, we determine if an analysis is currently being conducted during which the probability of intervention # delivery is being overridden # To do: replace this parameter - if self.sim.modules['PregnancySupervisor'].current_parameters['ps_analysis_in_progress']: + if p_params['ps_analysis_in_progress'] and (int_name in p_params['interventions_under_analysis']): # If so, we determine if this intervention will be delivered given the set probability of delivery. - can_int_run_analysis = self.rng.random_sample() < int_avail_df.at[int_name, 'availability'] + can_int_run_analysis = self.rng.random_sample() < p_params['intervention_analysis_availability'] # The intervention has no effect if not can_int_run_analysis: @@ -424,7 +424,10 @@ def update_current_parameter_dictionary(self, list_position): for key, value in self.parameters.items(): if isinstance(value, list): - self.current_parameters[key] = self.parameters[key][list_position] + if not value: + self.current_parameters[key] = self.parameters[key] + else: + self.current_parameters[key] = self.parameters[key][list_position] else: if list_position == 0: self.current_parameters[key] = self.parameters[key] diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index c1cf4a67f8..568e39c915 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -392,7 +392,12 @@ def __init__(self, name=None, resourcefilepath=None): 'intervention_availability': Parameter( Types.DATA_FRAME, ''), - + 'interventions_analysis': Parameter( + Types.BOOL, ''), + 'interventions_under_analysis': Parameter( + Types.LIST, ''), + 'intervention_analysis_availability': Parameter( + Types.REAL, ''), } PROPERTIES = { @@ -2132,7 +2137,8 @@ def apply(self, population): params['alternative_anc_quality'] or \ params['alternative_ip_anc_quality'] or \ params['sens_analysis_max'] or \ - params['sens_analysis_min']: + params['sens_analysis_min'] or \ + params['interventions_analysis']: # Update this parameter which is a signal used in the pregnancy_helper_function_file to ensure that # alternative functionality for determining availability of interventions only occurs when analysis is From 699b608ae1bfcb51a6951d5caca7daa5572661d9 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 16 Oct 2024 17:05:28 +0100 Subject: [PATCH 051/103] work on cohort --- src/tlo/methods/pregnancy_helper_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index d0adf46621..91094acc48 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -424,7 +424,7 @@ def update_current_parameter_dictionary(self, list_position): for key, value in self.parameters.items(): if isinstance(value, list): - if not value: + if not value or (len(value)) == 1: self.current_parameters[key] = self.parameters[key] else: self.current_parameters[key] = self.parameters[key][list_position] From dae7e938536a836ae5f4a7cdb5eb55c6d1e40eb2 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 17 Oct 2024 09:58:19 +0100 Subject: [PATCH 052/103] work on cohort --- .../ResourceFile_PregnancySupervisor.xlsx | 4 +- .../cohort_interventions_scenario.py | 24 +++-- .../dummy_cohort_azure_calib.py | 95 +++++++++++++------ src/tlo/methods/pregnancy_supervisor.py | 2 + ...al_health_helper_and_analysis_functions.py | 30 ++++++ 5 files changed, 116 insertions(+), 39 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index cf2d7636cd..d23f90503b 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d05fc317eafb2761d03ec2107cf5ce5322fac006289ccb1c0366f10e3ab1f685 -size 23984 +oid sha256:bd3eb8760ebab77a117a6274753ae393dca9e94e667357369787c6d7bc1e67da +size 24236 diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index f17877c7a6..e55648be8e 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -15,9 +15,9 @@ def __init__(self): self.seed = 537184 self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) - self.pop_size = 5000 - self.number_of_draws = 3 - self.runs_per_draw = 10 + self.pop_size = 10_000 + self.number_of_draws = 4 + self.runs_per_draw = 20 def log_configuration(self): return { @@ -44,13 +44,17 @@ def modules(self): mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=self.resources)] def draw_parameters(self, draw_number, rng): - interventions_for_analysis = ['blood_transfusion', 'pph_treatment_uterotonics', 'sepsis_treatment'] - - return {'PregnancySupervisor': { - 'analysis_year': 2024, - 'interventions_analysis': True, - 'interventions_under_analysis':[interventions_for_analysis[draw_number-1]], - 'intervention_analysis_availability': 0.0}} + if draw_number == 0: + return {'PregnancySupervisor': { + 'analysis_year': 2024}} + else: + interventions_for_analysis = ['blood_transfusion', 'pph_treatment_uterotonics', 'sepsis_treatment'] + + return {'PregnancySupervisor': { + 'analysis_year': 2024, + 'interventions_analysis': True, + 'interventions_under_analysis':[interventions_for_analysis[draw_number-1]], + 'intervention_analysis_availability': 0.0}} if __name__ == '__main__': diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 711b48c7a0..0a4394a8c0 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -5,12 +5,12 @@ from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario_filename = 'cohort_test-2024-10-09T130546Z' -# scenario_filename2 = 'cohort_test-2024-10-15T122825Z' -scenario_filename2 = 'cohort_test-2024-10-16T071357Z' -results_folder_old = get_scenario_outputs(scenario_filename, outputspath)[-1] -results_folder_new = get_scenario_outputs(scenario_filename2, outputspath)[-1] +baseline_scenario = 'cohort_test-2024-10-16T071357Z' +intervention_scenarios = 'block_intervention_test-2024-10-16T160603Z' + +results_folder_baseline = get_scenario_outputs(baseline_scenario, outputspath)[-1] +results_folder_int = get_scenario_outputs(intervention_scenarios, outputspath)[-1] def get_data_frames(key, results_folder): def sort_df(_df): @@ -27,31 +27,72 @@ def sort_df(_df): return results_df -results_old = {k:get_data_frames(k, results_folder_old) for k in +results_baseline = {k:get_data_frames(k, results_folder_baseline) for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} -results_new = {k:get_data_frames(k, results_folder_new) for k in +results_new = {k:get_data_frames(k, results_folder_int) for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage', 'yearly_mnh_counter_dict']} -import matplotlib.pyplot as plt -import numpy as np -# Sample data -mmr_data = { - 'int_1': [(235, 250, 265), (335, 350, 365)], - 'int_2': [(170, 195, 200), (290, 305, 320)], - 'int_3': [(280, 295, 310), (295 ,310, 325)], - 'int_4': [(165, 180, 195), (385, 400, 415)] -} -# Plotting -fig, ax = plt.subplots() -for key, intervals in mmr_data.items(): - for idx, (lower, mean, upper) in enumerate(intervals): - x = np.arange(len(mmr_data)) * len(intervals) + idx - ax.plot(x, mean, 'o', label=f'{key}' if idx == 0 else "") - ax.fill_between([x, x], [lower, lower], [upper, upper], alpha=0.2) -ax.set_xticks(np.arange(len(mmr_data)) * len(intervals) + 0.5) -ax.set_xticklabels(mmr_data.keys()) -plt.legend() -plt.show() +def get_data(df, draw): + return (df.loc['direct_mmr', (draw, 'lower')], + df.loc['direct_mmr', (draw, 'mean')], + df.loc['direct_mmr', (draw, 'upper')]) + +results = {'baseline':get_data(results_baseline['deaths_and_stillbirths'], 0), + 'sepsis_treatment':get_data(results_new['deaths_and_stillbirths'], 0), + 'blood_transfusion':get_data(results_new['deaths_and_stillbirths'], 1), + 'pph_treatment_uterotonics':get_data(results_new['deaths_and_stillbirths'], 2),} + + +# scenario_filename = 'cohort_test-2024-10-09T130546Z' +# # scenario_filename2 = 'cohort_test-2024-10-15T122825Z' +# scenario_filename2 = 'cohort_test-2024-10-16T071357Z' +# +# results_folder_old = get_scenario_outputs(scenario_filename, outputspath)[-1] +# results_folder_new = get_scenario_outputs(scenario_filename2, outputspath)[-1] +# +# def get_data_frames(key, results_folder): +# def sort_df(_df): +# _x = _df.drop(columns=['date'], inplace=False) +# return _x.iloc[0] +# +# results_df = summarize (extract_results( +# results_folder, +# module="tlo.methods.pregnancy_supervisor", +# key=key, +# custom_generate_series=sort_df, +# do_scaling=False +# )) +# +# return results_df +# +# results_old = {k:get_data_frames(k, results_folder_old) for k in +# ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', +# 'yearly_mnh_counter_dict']} +# +# results_new = {k:get_data_frames(k, results_folder_new) for k in +# ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage', +# 'yearly_mnh_counter_dict']} +# +# import matplotlib.pyplot as plt +# import numpy as np +# # Sample data +# mmr_data = { +# 'int_1': [(235, 250, 265), (335, 350, 365)], +# 'int_2': [(170, 195, 200), (290, 305, 320)], +# 'int_3': [(280, 295, 310), (295 ,310, 325)], +# 'int_4': [(165, 180, 195), (385, 400, 415)] +# } +# # Plotting +# fig, ax = plt.subplots() +# for key, intervals in mmr_data.items(): +# for idx, (lower, mean, upper) in enumerate(intervals): +# x = np.arange(len(mmr_data)) * len(intervals) + idx +# ax.plot(x, mean, 'o', label=f'{key}' if idx == 0 else "") +# ax.fill_between([x, x], [lower, lower], [upper, upper], alpha=0.2) +# ax.set_xticks(np.arange(len(mmr_data)) * len(intervals) + 0.5) +# ax.set_xticklabels(mmr_data.keys()) +# plt.legend() +# plt.show() diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 568e39c915..30700391b3 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -396,6 +396,8 @@ def __init__(self, name=None, resourcefilepath=None): Types.BOOL, ''), 'interventions_under_analysis': Parameter( Types.LIST, ''), + 'all_interventions': Parameter( + Types.LIST, ''), 'intervention_analysis_availability': Parameter( Types.REAL, ''), } diff --git a/tests/test_maternal_health_helper_and_analysis_functions.py b/tests/test_maternal_health_helper_and_analysis_functions.py index daea95a5e6..7c21dfba01 100644 --- a/tests/test_maternal_health_helper_and_analysis_functions.py +++ b/tests/test_maternal_health_helper_and_analysis_functions.py @@ -49,6 +49,36 @@ def apply(self, person_id, squeeze_factor): return hsi_event +def test_interventions_are_blocked_or_maximised_during_analysis(seed): + sim = Simulation(start_date=start_date, seed=seed) + sim.register(*fullmodel(resourcefilepath=resourcefilepath)) + sim.make_initial_population(n=100) + + pparams = sim.modules['PregnancySupervisor'].parameters + pparams['analysis_year'] = 2010 + pparams['interventions_analysis'] = True + + # 'interventions_analysis': True, + # 'interventions_under_analysis': [interventions_for_analysis[draw_number - 1]], + # 'intervention_analysis_availability': 0.0 + + sim.simulate(end_date=Date(2010, 1, 2)) + + assert sim.modules['PregnancySupervisor'].current_parameters['ps_analysis_in_progress'] + + df = sim.population.props + women_repro = df.loc[df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50)] + mother_id = women_repro.index[0] + + df.at[mother_id, 'is_pregnant'] = True + df.at[mother_id, 'date_of_last_pregnancy'] = start_date + + + + + + + def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): """Test that the analysis events run when scheduled and that they update the correct parameters as expected From 12178c2e9fdad37ebc35e5eb9c640434ec181b1e Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 17 Oct 2024 17:10:04 +0100 Subject: [PATCH 053/103] updated tests --- .../ResourceFile_PregnancySupervisor.xlsx | 4 +- .../methods/care_of_women_during_pregnancy.py | 5 +- src/tlo/methods/pregnancy_helper_functions.py | 4 +- tests/test_antenatal_interventions.py | 27 +++-- tests/test_labour.py | 12 +-- ...al_health_helper_and_analysis_functions.py | 101 ++++++++++++++++-- tests/test_newborn_outcomes.py | 12 +-- tests/test_pregnancy_supervisor.py | 6 +- 8 files changed, 134 insertions(+), 37 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index d23f90503b..d352715855 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bd3eb8760ebab77a117a6274753ae393dca9e94e667357369787c6d7bc1e67da -size 24236 +oid sha256:e09d4af6900872f5379b421eb9adc9121f3171271a1aab093dc7448bec012428 +size 24223 diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 1b458156e4..045644d679 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -2496,14 +2496,15 @@ def apply(self, person_id, squeeze_factor): # Women for whom immediate delivery is indicated are schedule to move straight to the labour model where # they will have the appropriate properties set and facility delivery at a hospital scheduled (mode of # delivery will match the recommended mode here) - if df.at[person_id, 'ac_admitted_for_immediate_delivery'] in ('induction_now', 'caesarean_now'): + if df.at[person_id, 'ac_admitted_for_immediate_delivery'] in ('avd_now', 'induction_now', 'caesarean_now'): self.sim.schedule_event(LabourOnsetEvent(self.sim.modules['Labour'], person_id), self.sim.date) # Women who require delivery BUT are not in immediate risk of morbidity/mortality will remain as # inpatients until they can move to the labour model. Currently it is possible for women to go into # labour whilst as an inpatient - it is assumed they are delivered via the mode recommended here # (i.e induction/caesarean) - elif df.at[person_id, 'ac_admitted_for_immediate_delivery'] in ('caesarean_future', 'induction_future'): + elif df.at[person_id, 'ac_admitted_for_immediate_delivery'] in ('avd_future', 'caesarean_future', + 'induction_future'): # Here we calculate how many days this woman needs to remain on the antenatal ward before she can go # for delivery (assuming delivery is indicated to occur at 37 weeks) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 91094acc48..a6c33811b6 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -152,7 +152,7 @@ def check_int_deliverable(self, int_name, hsi_event, hsi_event.get_consumables(item_codes= [], optional_item_codes=opt_cons) if ((dx_test is None) or - (self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run=dx_test, hsi_event=hsi_event))): + (self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run=dx_test, hsi_event=hsi_event))): test = True if quality and consumables and test: @@ -424,7 +424,7 @@ def update_current_parameter_dictionary(self, list_position): for key, value in self.parameters.items(): if isinstance(value, list): - if not value or (len(value)) == 1: + if not value or (len(value)) == 1 or key == 'allowed_interventions': self.current_parameters[key] = self.parameters[key] else: self.current_parameters[key] = self.parameters[key][list_position] diff --git a/tests/test_antenatal_interventions.py b/tests/test_antenatal_interventions.py index ce10385ef4..a8c8bbbe68 100644 --- a/tests/test_antenatal_interventions.py +++ b/tests/test_antenatal_interventions.py @@ -593,7 +593,7 @@ def test_initiation_of_treatment_for_maternal_anaemia_during_antenatal_inpatient # and override quality parameters lparams = sim.modules['Labour'].current_parameters lparams['prob_hcw_avail_blood_tran'] = 1.0 - lparams['mean_hcw_competence_hp'] = [1.0, 1.0] + lparams['mean_hcw_competence_hp'] = 1.0 # Set anaemia status df.at[mother_id, 'ps_anaemia_in_pregnancy'] = 'severe' @@ -655,8 +655,8 @@ def test_initiation_of_treatment_for_hypertensive_disorder_during_antenatal_inpa # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1, 1] - params['mean_hcw_competence_hp'] = [1, 1] + params['mean_hcw_competence_hc'] = 1 + params['mean_hcw_competence_hp'] = 1 params['prob_hcw_avail_anticonvulsant'] = 1 # set key pregnancy characteristics @@ -791,8 +791,8 @@ def test_initiation_of_treatment_for_prom_with_or_without_chorioamnionitis_durin # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1, 1] - params['mean_hcw_competence_hp'] = [1, 1] + params['mean_hcw_competence_hc'] = 1 + params['mean_hcw_competence_hp'] = 1 params['prob_hcw_avail_iv_abx'] = 1 # set key pregnancy characteristics @@ -933,8 +933,8 @@ def test_scheduling_and_treatment_effect_of_post_abortion_care(seed): # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1, 1] - params['mean_hcw_competence_hp'] = [1, 1] + params['mean_hcw_competence_hc'] = 1 + params['mean_hcw_competence_hp'] = 1 params['prob_hcw_avail_retained_prod'] = 1 # set complications @@ -1000,7 +1000,11 @@ def test_scheduling_and_treatment_effect_of_ectopic_pregnancy_case_management(se df.at[mother_id, 'ps_gestational_age_in_weeks'] = 8 # set prob care seeking to 1 - sim.modules['PregnancySupervisor'].current_parameters['prob_care_seeking_ectopic_pre_rupture'] = 1 + params = sim.modules['PregnancySupervisor'].current_parameters + l_params = sim.modules['Labour'].current_parameters + params['prob_care_seeking_ectopic_pre_rupture'] = 1 + l_params['prob_hcw_avail_surg'] = 1 + l_params['mean_hcw_competence_hp'] = 1 # define and run ectopic event ectopic_event = pregnancy_supervisor.EctopicPregnancyEvent(individual_id=mother_id, @@ -1022,6 +1026,13 @@ def test_scheduling_and_treatment_effect_of_ectopic_pregnancy_case_management(se assert care_of_women_during_pregnancy.HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy in hsi_list # Define and run treatment events + updated_cons = \ + {k: 1.0 for (k, v) in + sim.modules['CareOfWomenDuringPregnancy'].item_codes_preg_consumables['ectopic_pregnancy_core'].items()} + + sim.modules['HealthSystem'].override_availability_of_consumables(updated_cons) + sim.modules['HealthSystem'].consumables._refresh_availability_of_consumables(date=sim.date) + ectopic_treatment = care_of_women_during_pregnancy.HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy( module=sim.modules['CareOfWomenDuringPregnancy'], person_id=updated_mother_id) ectopic_treatment.apply(person_id=updated_mother_id, squeeze_factor=0.0) diff --git a/tests/test_labour.py b/tests/test_labour.py index 21f94c200d..64706494c8 100644 --- a/tests/test_labour.py +++ b/tests/test_labour.py @@ -297,8 +297,8 @@ def test_event_scheduling_for_admissions_from_antenatal_inpatient_ward_for_caesa # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1.0, 1.0] - params['mean_hcw_competence_hp'] = [1.0, 1.0] + params['mean_hcw_competence_hc'] = 1.0 + params['mean_hcw_competence_hp'] = 1.0 params['prob_hcw_avail_surg'] = 1.0 # Run the labour onset, check she will correctly deliver at a hospital level facility @@ -514,8 +514,8 @@ def test_bemonc_treatments_are_delivered_correctly_with_no_cons_or_quality_const # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1.0, 1.0] - params['mean_hcw_competence_hp'] = [1.0, 1.0] + params['mean_hcw_competence_hc'] = 1.0 + params['mean_hcw_competence_hp'] = 1.0 params['prob_hcw_avail_iv_abx'] = 1.0 params['prob_hcw_avail_uterotonic'] = 1.0 params['prob_hcw_avail_anticonvulsant'] = 1.0 @@ -658,8 +658,8 @@ def test_cemonc_event_and_treatments_are_delivered_correct_with_no_cons_or_quali # set key parameters params = sim.modules['Labour'].current_parameters - params['mean_hcw_competence_hc'] = [1.0, 1.0] - params['mean_hcw_competence_hp'] = [1.0, 1.0] + params['mean_hcw_competence_hc'] = 1.0 + params['mean_hcw_competence_hp'] = 1.0 params['prob_hcw_avail_surg'] = 1.0 params['prob_hcw_avail_blood_tran'] = 1.0 diff --git a/tests/test_maternal_health_helper_and_analysis_functions.py b/tests/test_maternal_health_helper_and_analysis_functions.py index 7c21dfba01..53f1ffdf5f 100644 --- a/tests/test_maternal_health_helper_and_analysis_functions.py +++ b/tests/test_maternal_health_helper_and_analysis_functions.py @@ -49,7 +49,77 @@ def apply(self, person_id, squeeze_factor): return hsi_event -def test_interventions_are_blocked_or_maximised_during_analysis(seed): +def test_interventions_are_delivered_as_expected_not_during_analysis(seed): + sim = Simulation(start_date=start_date, seed=seed) + sim.register(*fullmodel(resourcefilepath=resourcefilepath)) + sim.make_initial_population(n=100) + + cw_params = sim.modules['CareOfWomenDuringPregnancy'].parameters + cw_params['sensitivity_bp_monitoring'] = 1.0 + cw_params['specificity_bp_monitoring'] = 1.0 + cw_params['sensitivity_urine_protein_1_plus'] = 1.0 + cw_params['specificity_urine_protein_1_plus'] = 1.0 + cw_params['sensitivity_poc_hb_test'] = 1.0 + cw_params['specificity_poc_hb_test'] = 1.0 + cw_params['sensitivity_fbc_hb_test'] = 1.0 + cw_params['specificity_fbc_hb_test'] = 1.0 + cw_params['sensitivity_blood_test_glucose'] = 1.0 + cw_params['specificity_blood_test_glucose'] = 1.0 + cw_params['sensitivity_blood_test_syphilis'] = 1.0 + cw_params['specificity_blood_test_syphilis'] = 1.0 + + sim.simulate(end_date=Date(2010, 1, 2)) + + df = sim.population.props + women_repro = df.loc[df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50)] + mother_id = women_repro.index[0] + + int_function = pregnancy_helper_functions.check_int_deliverable + + hsi_event = get_dummy_hsi(sim, mother_id, id=0, fl=0) + + def override_dummy_cons(value): + updated_cons = {k: value for (k, v) in sim.modules['Labour'].item_codes_lab_consumables['delivery_core'].items()} + sim.modules['HealthSystem'].override_availability_of_consumables(updated_cons) + sim.modules['HealthSystem'].consumables._refresh_availability_of_consumables(date=sim.date) + return sim.modules['Labour'].item_codes_lab_consumables['delivery_core'] + + pparams = sim.modules['PregnancySupervisor'].current_parameters + + for intervention in pparams['all_interventions']: + assert not int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[0.0], + cons=override_dummy_cons(0.0)) + assert not int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[1.0], + cons=override_dummy_cons(0.0)) + assert not int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[0.0], + cons=override_dummy_cons(1.0)) + assert int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[1.0], + cons=override_dummy_cons(1.0)) + + dx_tests = [('ps_htn_disorders', 'severe_pre_eclamp', 'bp_measurement', 'blood_pressure_measurement'), + ('ps_htn_disorders', 'severe_pre_eclamp', 'urine_dipstick', 'urine_dipstick_protein'), + ('ps_anaemia_in_pregnancy', 'moderate', 'hb_test', 'point_of_care_hb_test'), + ('ps_anaemia_in_pregnancy', 'moderate', 'full_blood_count', 'full_blood_count_hb'), + ('ps_gest_diab', 'uncontrolled', 'gdm_test', 'blood_test_glucose'), + ('ps_syphilis', True, 'syphilis_test', 'blood_test_syphilis'), + ('pn_anaemia_following_pregnancy', 'moderate', 'full_blood_count', 'full_blood_count_hb_pn')] + + for test in dx_tests: + df.at[mother_id, test[0]] = test[1] + assert int_function(sim.modules['CareOfWomenDuringPregnancy'], + test[2], hsi_event, q_param=[1.0], + cons=override_dummy_cons(1.0), dx_test=test[3]) + + assert not int_function(sim.modules['CareOfWomenDuringPregnancy'], + test[2], hsi_event, q_param=[0.0], + cons=override_dummy_cons(1.0), dx_test=test[3]) + + assert not int_function(sim.modules['CareOfWomenDuringPregnancy'], + test[2], hsi_event, q_param=[1.0], + cons=override_dummy_cons(0.0), dx_test=test[3]) + + +def test_interventions_are_delivered_as_expected_during_analysis(seed): sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) sim.make_initial_population(n=100) @@ -58,26 +128,41 @@ def test_interventions_are_blocked_or_maximised_during_analysis(seed): pparams['analysis_year'] = 2010 pparams['interventions_analysis'] = True - # 'interventions_analysis': True, - # 'interventions_under_analysis': [interventions_for_analysis[draw_number - 1]], - # 'intervention_analysis_availability': 0.0 - sim.simulate(end_date=Date(2010, 1, 2)) - assert sim.modules['PregnancySupervisor'].current_parameters['ps_analysis_in_progress'] + pparams = sim.modules['PregnancySupervisor'].current_parameters + assert pparams['ps_analysis_in_progress'] df = sim.population.props women_repro = df.loc[df.is_alive & (df.sex == 'F') & (df.age_years > 14) & (df.age_years < 50)] mother_id = women_repro.index[0] - df.at[mother_id, 'is_pregnant'] = True - df.at[mother_id, 'date_of_last_pregnancy'] = start_date + int_function = pregnancy_helper_functions.check_int_deliverable + hsi_event = get_dummy_hsi(sim, mother_id, id=0, fl=0) + + def override_dummy_cons(value): + updated_cons = {k: value for (k, v) in sim.modules['Labour'].item_codes_lab_consumables['delivery_core'].items()} + sim.modules['HealthSystem'].override_availability_of_consumables(updated_cons) + sim.modules['HealthSystem'].consumables._refresh_availability_of_consumables(date=sim.date) + return sim.modules['Labour'].item_codes_lab_consumables['delivery_core'] + for intervention in pparams['all_interventions']: + pparams['interventions_under_analysis'] = [intervention] + pparams['intervention_analysis_availability'] = 1.0 + assert int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[0.0], + cons=override_dummy_cons(0.0)) + assert int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[1.0], + cons=override_dummy_cons(0.0)) + assert int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[0.0], + cons=override_dummy_cons(1.0)) + pparams['intervention_analysis_availability'] = 0.0 + assert not int_function(sim.modules['PregnancySupervisor'], intervention, hsi_event, q_param=[1.0], + cons=override_dummy_cons(1.0)) def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): diff --git a/tests/test_newborn_outcomes.py b/tests/test_newborn_outcomes.py index 2511e14270..921b9b155c 100644 --- a/tests/test_newborn_outcomes.py +++ b/tests/test_newborn_outcomes.py @@ -336,21 +336,21 @@ def test_sba_hsi_deliveries_resuscitation_treatment_as_expected(seed): sim = Simulation(start_date=start_date, seed=seed) register_modules(sim) sim.make_initial_population(n=100) + sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) # set risk of comps very high and force care seeking - params = sim.modules['NewbornOutcomes'].parameters + params = sim.modules['NewbornOutcomes'].current_parameters params['prob_encephalopathy'] = 1.0 - params['prob_enceph_severity'] = [[0, 0, 1], [0, 0, 1]] + params['prob_enceph_severity'] = [0, 0, 1] params['treatment_effect_resuscitation'] = 0.0 params['cfr_enceph'] = 1.0 params['prob_pnc_check_newborn'] = 0.0 - la_params = sim.modules['Labour'].parameters + la_params = sim.modules['Labour'].current_parameters la_params['prob_hcw_avail_neo_resus'] = 1.0 - la_params['mean_hcw_competence_hc'] = [[1.0, 1.0], [1.0, 1.0]] - la_params['mean_hcw_competence_hp'] = [[1.0, 1.0], [1.0, 1.0]] + la_params['mean_hcw_competence_hc'] = 1.0 + la_params['mean_hcw_competence_hp'] = 1.0 - sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) df = sim.population.props mni = sim.modules['PregnancySupervisor'].mother_and_newborn_info diff --git a/tests/test_pregnancy_supervisor.py b/tests/test_pregnancy_supervisor.py index 4a1aafe0b7..b89656e797 100644 --- a/tests/test_pregnancy_supervisor.py +++ b/tests/test_pregnancy_supervisor.py @@ -529,9 +529,9 @@ def check_abortion_logic(abortion_type): params['treatment_effect_post_abortion_care'] = 0.0 lab_params = sim.modules['Labour'].current_parameters - lab_params['mean_hcw_competence_hc'] = [1, 1] - lab_params['mean_hcw_competence_hp'] = [1, 1] - lab_params['prob_hcw_avail_retained_prod'] = 1 + lab_params['mean_hcw_competence_hc'] = 1.0 + lab_params['mean_hcw_competence_hp'] = 1.0 + lab_params['prob_hcw_avail_retained_prod'] = 1.0 df = sim.population.props pregnant_women = df.loc[df.is_alive & df.is_pregnant] From 7faa81783dc43e434e26ef8c95717480cebd3816 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 18 Oct 2024 10:07:46 +0200 Subject: [PATCH 054/103] Consider all modules included as of interest --- src/tlo/simulation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index fd9fade215..15be1622e8 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -325,7 +325,7 @@ def initialise(self, *, end_date: Date) -> None: # Eventually this can be made an option self.generate_event_chains_overwrite_epi = True # For now keep these fixed, eventually they will be input from user - self.generate_event_chains_modules_of_interest = [self.modules['RTI']] + self.generate_event_chains_modules_of_interest = [self.modules] self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] else: # If not using to print chains, cannot ignore epi From 7232f976831054ed541d59d8da20c91289fa79e6 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:29:43 +0200 Subject: [PATCH 055/103] Remove pop-wide HSI warning and make epi default even when printing chains --- src/tlo/methods/hsi_event.py | 38 ++++++++++++++++++++++++++---------- src/tlo/simulation.py | 2 +- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 6651a8704a..d0cdb5bbdd 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -223,13 +223,23 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'Before' - row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) - row['level'] = self.facility_info.level + try: + row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) + row['level'] = self.facility_info.level + except: + row['appt_footprint'] = 'N/A' + row['level'] = 'N/A' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: - # Many of our HealthSystem implementations rely on the assumption that - raise RuntimeError("Cannot have population-wide HSI events") + # Once this has been removed from Chronic Syndrome mock module, make this a Runtime Error + # raise RuntimeError("Cannot have population-wide HSI events") + logger.debug( + key="message", + data=( + f"Cannot have population-wide HSI events" + ), + ) return print_chains, row_before @@ -245,12 +255,20 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> # will be stored regardless of whether individual experienced property changes. # Add event details + + try: + record_footprint = str(footprint) + record_level = self.facility_info.level + except: + record_footprint = 'N/A' + record_level = 'N/A' + link_info = { 'person_ID': self.target, 'event' : str(self), 'event_date' : self.sim.date, - 'appt_footprint' : str(footprint), - 'level' : self.facility_info.level, + 'appt_footprint' : record_footprint, + 'level' : record_level, } # Add changes to properties @@ -266,8 +284,8 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> row['event'] = str(self) row['event_date'] = self.sim.date row['when'] = 'After' - row['appt_footprint'] = footprint - row['level'] = self.facility_info.level + row['appt_footprint'] = record_footprint + row['level'] = record_level self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) return chain_links @@ -277,7 +295,7 @@ def run(self, squeeze_factor): """Make the event happen.""" - if self.sim.generate_event_chains: + if self.sim.generate_event_chains and self.target != self.sim.population: print_chains, row_before = self.store_chains_to_do_before_event() footprint = self.EXPECTED_APPT_FOOTPRINT @@ -287,7 +305,7 @@ def run(self, squeeze_factor): self._run_after_hsi_event() - if self.sim.generate_event_chains: + if self.sim.generate_event_chains and self.target != self.sim.population: # If the footprint has been updated when the event ran, change it here if updated_appt_footprint is not None: diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 15be1622e8..0c70b164d9 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -323,7 +323,7 @@ def initialise(self, *, end_date: Date) -> None: #self.generate_event_chains = generate_event_chains if self.generate_event_chains: # Eventually this can be made an option - self.generate_event_chains_overwrite_epi = True + self.generate_event_chains_overwrite_epi = False # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] From a6def2d22c0d291ce775afef561b580847ad36cf Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:39:24 +0200 Subject: [PATCH 056/103] Style fix --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index d0cdb5bbdd..041ab9cf08 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -237,7 +237,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: logger.debug( key="message", data=( - f"Cannot have population-wide HSI events" + "Cannot have population-wide HSI events" ), ) From 24a14d073a6dc1ae4735f057181cfafa3080ba1f Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 18 Oct 2024 10:46:38 +0100 Subject: [PATCH 057/103] changes to scripts --- .../cohort_interventions_scenario.py | 11 ++- .../dummy_cohort_azure_calib.py | 81 +++++++++++++++---- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index e55648be8e..b998132780 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -16,7 +16,7 @@ def __init__(self): self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) self.pop_size = 10_000 - self.number_of_draws = 4 + self.number_of_draws = 7 self.runs_per_draw = 20 def log_configuration(self): @@ -48,13 +48,18 @@ def draw_parameters(self, draw_number, rng): return {'PregnancySupervisor': { 'analysis_year': 2024}} else: - interventions_for_analysis = ['blood_transfusion', 'pph_treatment_uterotonics', 'sepsis_treatment'] + interventions_for_analysis = ['blood_transfusion', 'blood_transfusion', + 'pph_treatment_uterotonics', 'pph_treatment_uterotonics', + 'sepsis_treatment', 'sepsis_treatment'] + avail_for_draw = [0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0] return {'PregnancySupervisor': { 'analysis_year': 2024, 'interventions_analysis': True, 'interventions_under_analysis':[interventions_for_analysis[draw_number-1]], - 'intervention_analysis_availability': 0.0}} + 'intervention_analysis_availability': [avail_for_draw[draw_number-1]]}} if __name__ == '__main__': diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 0a4394a8c0..a79c1a1a79 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -2,49 +2,102 @@ import pandas as pd +import matplotlib.pyplot as plt +import numpy as np + from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize outputspath = './outputs/sejjj49@ucl.ac.uk/' -baseline_scenario = 'cohort_test-2024-10-16T071357Z' -intervention_scenarios = 'block_intervention_test-2024-10-16T160603Z' +scenario = 'block_intervention_test-2024-10-17T161423Z' -results_folder_baseline = get_scenario_outputs(baseline_scenario, outputspath)[-1] -results_folder_int = get_scenario_outputs(intervention_scenarios, outputspath)[-1] +results_folder= get_scenario_outputs(scenario, outputspath)[-1] def get_data_frames(key, results_folder): def sort_df(_df): _x = _df.drop(columns=['date'], inplace=False) return _x.iloc[0] - results_df = summarize (extract_results( + results_df = extract_results( results_folder, module="tlo.methods.pregnancy_supervisor", key=key, custom_generate_series=sort_df, do_scaling=False - )) + ) + + results_df_summ = summarize(extract_results( + results_folder, + module="tlo.methods.pregnancy_supervisor", + key=key, + custom_generate_series=sort_df, + do_scaling=False + )) - return results_df + return [results_df, results_df_summ] -results_baseline = {k:get_data_frames(k, results_folder_baseline) for k in +results_sum = {k:get_data_frames(k, results_folder)[1] for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} -results_new = {k:get_data_frames(k, results_folder_int) for k in - ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage', +results = {k:get_data_frames(k, results_folder)[0] for k in + ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} +baseline = results['deaths_and_stillbirths'].loc['direct_mmr', (0, slice(0, 19))].droplevel(0) + +def get_mmr_diffs(df, draws): + diff_results = {} + baseline = results['deaths_and_stillbirths'][0] + + for draw in draws: + # diff_df = ((results['deaths_and_stillbirths'][draw] - baseline)/baseline) * 100 + diff_df = results['deaths_and_stillbirths'][draw] - baseline + diff_df.columns = pd.MultiIndex.from_tuples([(0, v) for v in range(len(diff_df.columns))], + names=['draw', 'run']) + results_diff = summarize(diff_df) + results_diff.fillna(0) + diff_results.update({draw: results_diff}) + + return diff_results + +diff_results = get_mmr_diffs(results, range(1,4)) + def get_data(df, draw): return (df.loc['direct_mmr', (draw, 'lower')], df.loc['direct_mmr', (draw, 'mean')], df.loc['direct_mmr', (draw, 'upper')]) -results = {'baseline':get_data(results_baseline['deaths_and_stillbirths'], 0), - 'sepsis_treatment':get_data(results_new['deaths_and_stillbirths'], 0), - 'blood_transfusion':get_data(results_new['deaths_and_stillbirths'], 1), - 'pph_treatment_uterotonics':get_data(results_new['deaths_and_stillbirths'], 2),} +results = {'baseline':get_data(results['deaths_and_stillbirths'], 0), + 'blood_transfusion':get_data(results['deaths_and_stillbirths'], 1), + 'pph_treatment_uterotonics':get_data(results['deaths_and_stillbirths'], 2), + 'sepsis_treatment':get_data(results['deaths_and_stillbirths'], 3)} + +results_diff = {'blood_transfusion':get_data(diff_results[1], 1), + 'pph_treatment_uterotonics':get_data(diff_results[2], 2), + 'sepsis_treatment':get_data(diff_results[3], 3)} + +# todo: compare deaths with demography logging... + +results = data + +# Extract means and errors +labels = data.keys() +means = [vals[1] for vals in data.values()] +lower_errors = [vals[1] - vals[0] for vals in data.values()] +upper_errors = [vals[2] - vals[1] for vals in data.values()] +errors = [lower_errors, upper_errors] + +# Create bar chart with error bars +fig, ax = plt.subplots() +ax.bar(labels, means, yerr=errors, capsize=5, alpha=0.7, ecolor='black') +ax.set_ylabel('Values') +ax.set_title('Bar Chart with Error Bars') +# Adjust label size +plt.xticks(rotation=45, fontsize=8) +plt.tight_layout() +plt.show() # scenario_filename = 'cohort_test-2024-10-09T130546Z' # # scenario_filename2 = 'cohort_test-2024-10-15T122825Z' From ecea532a2843d312580accf97383cd62c457fd04 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:51:39 +0200 Subject: [PATCH 058/103] Remove data generation test, which wasn't really a test --- tests/test_data_generation.py | 82 ----------------------------------- 1 file changed, 82 deletions(-) delete mode 100644 tests/test_data_generation.py diff --git a/tests/test_data_generation.py b/tests/test_data_generation.py deleted file mode 100644 index d9885c1fab..0000000000 --- a/tests/test_data_generation.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -from pathlib import Path - -import pytest - -from tlo import Date, Simulation -from tlo.methods import ( - care_of_women_during_pregnancy, - demography, - depression, - enhanced_lifestyle, - epi, - healthburden, - healthseekingbehaviour, - healthsystem, - hiv, - cardio_metabolic_disorders, - labour, - newborn_outcomes, - postnatal_supervisor, - pregnancy_supervisor, - depression, - tb, - contraception, - rti, - symptommanager, -) - -# create simulation parameters -start_date = Date(2010, 1, 1) -end_date = Date(2012, 1, 1) -popsize = 100 - -@pytest.mark.slow -def test_data_harvesting(seed): - """ - This test runs a simulation to print all individual events of specific individuals - """ - - module_of_interest = 'RTI' - # create sim object - sim = create_basic_sim(popsize, seed) - - dependencies_list = sim.modules[module_of_interest].ADDITIONAL_DEPENDENCIES.union(sim.modules[module_of_interest].INIT_DEPENDENCIES) - - # Check that all dependencies are included - for dep in dependencies_list: - if dep not in sim.modules: - print("WARNING: dependency ", dep, "not included") - exit(-1) - - # run simulation - sim.simulate(end_date=end_date, generate_event_chains = True) - -def create_basic_sim(population_size, seed): - # create the basic outline of an rti simulation object - sim = Simulation(start_date=start_date, seed=seed) - resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' - sim.register(demography.Demography(resourcefilepath=resourcefilepath), - contraception.Contraception(resourcefilepath=resourcefilepath), - enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), - healthburden.HealthBurden(resourcefilepath=resourcefilepath), - symptommanager.SymptomManager(resourcefilepath=resourcefilepath), - healthsystem.HealthSystem(resourcefilepath=resourcefilepath, service_availability=['*']), - rti.RTI(resourcefilepath=resourcefilepath), - healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), - # simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), - epi.Epi(resourcefilepath=resourcefilepath), - hiv.Hiv(resourcefilepath=resourcefilepath), - tb.Tb(resourcefilepath=resourcefilepath), - cardio_metabolic_disorders.CardioMetabolicDisorders(resourcefilepath=resourcefilepath), - depression.Depression(resourcefilepath=resourcefilepath), - newborn_outcomes.NewbornOutcomes(resourcefilepath=resourcefilepath), - pregnancy_supervisor.PregnancySupervisor(resourcefilepath=resourcefilepath), - care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=resourcefilepath), - labour.Labour(resourcefilepath=resourcefilepath), - postnatal_supervisor.PostnatalSupervisor(resourcefilepath=resourcefilepath), - ) - - sim.make_initial_population(n=population_size) - return sim - From 9a395102e6342f5161ceb8a7b6f6ae9c1f1c9cfd Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 18 Oct 2024 12:33:50 +0100 Subject: [PATCH 059/103] fix error in analysis script --- .../cohort_analysis/cohort_interventions_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index b998132780..e10b64842f 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -59,7 +59,7 @@ def draw_parameters(self, draw_number, rng): 'analysis_year': 2024, 'interventions_analysis': True, 'interventions_under_analysis':[interventions_for_analysis[draw_number-1]], - 'intervention_analysis_availability': [avail_for_draw[draw_number-1]]}} + 'intervention_analysis_availability': avail_for_draw[draw_number-1]}} if __name__ == '__main__': From ae7a44cb5f72063c48555e3b21d5d6dd4400ee97 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:29:03 +0200 Subject: [PATCH 060/103] Change dict of properties to string in logging, and add analysis files --- .../analysis_extract_data.py | 370 ++++++++++++++++++ .../postprocess_events_chain.py | 156 ++++++++ .../scenario_generate_chains.py | 115 ++++++ src/tlo/events.py | 23 +- src/tlo/methods/hsi_event.py | 13 +- src/tlo/simulation.py | 29 +- 6 files changed, 684 insertions(+), 22 deletions(-) create mode 100644 src/scripts/analysis_data_generation/analysis_extract_data.py create mode 100644 src/scripts/analysis_data_generation/postprocess_events_chain.py create mode 100644 src/scripts/analysis_data_generation/scenario_generate_chains.py diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py new file mode 100644 index 0000000000..2cfba5315b --- /dev/null +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -0,0 +1,370 @@ +"""Produce plots to show the health impact (deaths, dalys) each the healthcare system (overall health impact) when +running under different MODES and POLICIES (scenario_impact_of_actual_vs_funded.py)""" + +# short tclose -> ideal case +# long tclose -> status quo +import argparse +from pathlib import Path +from typing import Tuple + +import pandas as pd + +from tlo import Date +from tlo.analysis.utils import extract_results +from datetime import datetime + +# Range of years considered +min_year = 2010 +max_year = 2040 + + +def all_columns(_df): + return pd.Series(_df.all()) + +def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None, ): + """Produce standard set of plots describing the effect of each TREATMENT_ID. + - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. + - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur. + """ + pd.set_option('display.max_rows', None) + pd.set_option('display.max_colwidth', None) + event_chains = extract_results( + results_folder, + module='tlo.simulation', + key='event_chains', + column='0', + #column = str(i), + #custom_generate_series=get_num_dalys_by_year, + do_scaling=False + ) + # print(event_chains.loc[0,(0, 0)]) + + eval_env = { + 'datetime': datetime, # Add the datetime class to the eval environment + 'pd': pd, # Add pandas to handle Timestamp + 'Timestamp': pd.Timestamp, # Specifically add Timestamp for eval + 'NaT': pd.NaT, + 'nan': float('nan'), # Include NaN for eval (can also use pd.NA if preferred) + } + + for item,row in event_chains.iterrows(): + value = event_chains.loc[item,(0, 0)] + if value !='': + print('') + print(value) + exit(-1) + #dict = {} + #for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]: + # dict[i] = [] + + #for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]: + # event_chains = extract_results( + # results_folder, + # module='tlo.simulation'#, + # key='event_chains', + # column = str(i), + # #custom_generate_series=get_num_dalys_by_year, + # do_scaling=False + # ) + # print(event_chains) + # print(event_chains.index) + # print(event_chains.columns.levels) + + # for index, row in event_chains.iterrows(): + # if event_chains.iloc[index,0] is not None: + # if(event_chains.iloc[index,0]['person_ID']==i): #and 'event' in event_chains.iloc[index,0].keys()): + # dict[i].append(event_chains.iloc[index,0]) + #elif (event_chains.iloc[index,0]['person_ID']==i and 'event' not in event_chains.iloc[index,0].keys()): + #print(event_chains.iloc[index,0]['de_depr']) + # exit(-1) + #for item in dict[0]: + # print(item) + + #exit(-1) + + TARGET_PERIOD = (Date(min_year, 1, 1), Date(max_year, 1, 1)) + + # Definitions of general helper functions + lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731 + + def target_period() -> str: + """Returns the target period as a string of the form YYYY-YYYY""" + return "-".join(str(t.year) for t in TARGET_PERIOD) + + def get_parameter_names_from_scenario_file() -> Tuple[str]: + """Get the tuple of names of the scenarios from `Scenario` class used to create the results.""" + from scripts.healthsystem.impact_of_actual_vs_funded.scenario_impact_of_actual_vs_funded import ( + ImpactOfHealthSystemMode, + ) + e = ImpactOfHealthSystemMode() + return tuple(e._scenarios.keys()) + + def get_num_deaths(_df): + """Return total number of Deaths (total within the TARGET_PERIOD) + """ + return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)])) + + def get_num_dalys(_df): + """Return total number of DALYs (Stacked) by label (total within the TARGET_PERIOD)""" + return pd.Series( + data=_df + .loc[_df.year.between(*[i.year for i in TARGET_PERIOD])] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum().sum() + ) + + def get_num_dalys_by_cause(_df): + """Return number of DALYs by cause by label (total within the TARGET_PERIOD)""" + return pd.Series( + data=_df + .loc[_df.year.between(*[i.year for i in TARGET_PERIOD])] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum() + ) + + def set_param_names_as_column_index_level_0(_df): + """Set the columns index (level 0) as the param_names.""" + ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)} + names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]] + assert len(names_of_cols_level0) == len(_df.columns.levels[0]) + _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0) + return _df + + def find_difference_relative_to_comparison(_ser: pd.Series, + comparison: str, + scaled: bool = False, + drop_comparison: bool = True, + ): + """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0) + within the runs (level 1), relative to where draw = `comparison`. + The comparison is `X - COMPARISON`.""" + return _ser \ + .unstack(level=0) \ + .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \ + .drop(columns=([comparison] if drop_comparison else [])) \ + .stack() + + + def get_counts_of_hsi_by_treatment_id(_df): + """Get the counts of the short TREATMENT_IDs occurring""" + _counts_by_treatment_id = _df \ + .loc[pd.to_datetime(_df['date']).between(*TARGET_PERIOD), 'TREATMENT_ID'] \ + .apply(pd.Series) \ + .sum() \ + .astype(int) + return _counts_by_treatment_id.groupby(level=0).sum() + + year_target = 2023 + def get_counts_of_hsi_by_treatment_id_by_year(_df): + """Get the counts of the short TREATMENT_IDs occurring""" + _counts_by_treatment_id = _df \ + .loc[pd.to_datetime(_df['date']).dt.year ==year_target, 'TREATMENT_ID'] \ + .apply(pd.Series) \ + .sum() \ + .astype(int) + return _counts_by_treatment_id.groupby(level=0).sum() + + def get_counts_of_hsi_by_short_treatment_id(_df): + """Get the counts of the short TREATMENT_IDs occurring (shortened, up to first underscore)""" + _counts_by_treatment_id = get_counts_of_hsi_by_treatment_id(_df) + _short_treatment_id = _counts_by_treatment_id.index.map(lambda x: x.split('_')[0] + "*") + return _counts_by_treatment_id.groupby(by=_short_treatment_id).sum() + + def get_counts_of_hsi_by_short_treatment_id_by_year(_df): + """Get the counts of the short TREATMENT_IDs occurring (shortened, up to first underscore)""" + _counts_by_treatment_id = get_counts_of_hsi_by_treatment_id_by_year(_df) + _short_treatment_id = _counts_by_treatment_id.index.map(lambda x: x.split('_')[0] + "*") + return _counts_by_treatment_id.groupby(by=_short_treatment_id).sum() + + + # Obtain parameter names for this scenario file + param_names = get_parameter_names_from_scenario_file() + print(param_names) + + # ================================================================================================ + # TIME EVOLUTION OF TOTAL DALYs + # Plot DALYs averted compared to the ``No Policy'' policy + + year_target = 2023 # This global variable will be passed to custom function + def get_num_dalys_by_year(_df): + """Return total number of DALYs (Stacked) by label (total within the TARGET_PERIOD)""" + return pd.Series( + data=_df + .loc[_df.year == year_target] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum().sum() + ) + + ALL = {} + # Plot time trend show year prior transition as well to emphasise that until that point DALYs incurred + # are consistent across different policies + this_min_year = 2010 + for year in range(this_min_year, max_year+1): + year_target = year + num_dalys_by_year = extract_results( + results_folder, + module='tlo.methods.healthburden', + key='dalys_stacked', + custom_generate_series=get_num_dalys_by_year, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + ALL[year_target] = num_dalys_by_year + # Concatenate the DataFrames into a single DataFrame + concatenated_df = pd.concat(ALL.values(), keys=ALL.keys()) + concatenated_df.index = concatenated_df.index.set_names(['date', 'index_original']) + concatenated_df = concatenated_df.reset_index(level='index_original',drop=True) + dalys_by_year = concatenated_df + print(dalys_by_year) + dalys_by_year.to_csv('ConvertedOutputs/Total_DALYs_with_time.csv', index=True) + + # ================================================================================================ + # Print population under each scenario + pop_model = extract_results(results_folder, + module="tlo.methods.demography", + key="population", + column="total", + index="date", + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + pop_model.index = pop_model.index.year + pop_model = pop_model[(pop_model.index >= this_min_year) & (pop_model.index <= max_year)] + print(pop_model) + assert dalys_by_year.index.equals(pop_model.index) + assert all(dalys_by_year.columns == pop_model.columns) + pop_model.to_csv('ConvertedOutputs/Population_with_time.csv', index=True) + + # ================================================================================================ + # DALYs BROKEN DOWN BY CAUSES AND YEAR + # DALYs by cause per year + # %% Quantify the health losses associated with all interventions combined. + + year_target = 2023 # This global variable will be passed to custom function + def get_num_dalys_by_year_and_cause(_df): + """Return total number of DALYs (Stacked) by label (total within the TARGET_PERIOD)""" + return pd.Series( + data=_df + .loc[_df.year == year_target] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum() + ) + + ALL = {} + # Plot time trend show year prior transition as well to emphasise that until that point DALYs incurred + # are consistent across different policies + this_min_year = 2010 + for year in range(this_min_year, max_year+1): + year_target = year + num_dalys_by_year = extract_results( + results_folder, + module='tlo.methods.healthburden', + key='dalys_stacked', + custom_generate_series=get_num_dalys_by_year_and_cause, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + ALL[year_target] = num_dalys_by_year #summarize(num_dalys_by_year) + + # Concatenate the DataFrames into a single DataFrame + concatenated_df = pd.concat(ALL.values(), keys=ALL.keys()) + + concatenated_df.index = concatenated_df.index.set_names(['date', 'cause']) + + df_total = concatenated_df + df_total.to_csv('ConvertedOutputs/DALYS_by_cause_with_time.csv', index=True) + + ALL = {} + # Plot time trend show year prior transition as well to emphasise that until that point DALYs incurred + # are consistent across different policies + for year in range(min_year, max_year+1): + year_target = year + + hsi_delivered_by_year = extract_results( + results_folder, + module='tlo.methods.healthsystem.summary', + key='HSI_Event', + custom_generate_series=get_counts_of_hsi_by_short_treatment_id_by_year, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + ALL[year_target] = hsi_delivered_by_year + + # Concatenate the DataFrames into a single DataFrame + concatenated_df = pd.concat(ALL.values(), keys=ALL.keys()) + concatenated_df.index = concatenated_df.index.set_names(['date', 'cause']) + HSI_ran_by_year = concatenated_df + + del ALL + + ALL = {} + # Plot time trend show year prior transition as well to emphasise that until that point DALYs incurred + # are consistent across different policies + for year in range(min_year, max_year+1): + year_target = year + + hsi_not_delivered_by_year = extract_results( + results_folder, + module='tlo.methods.healthsystem.summary', + key='Never_ran_HSI_Event', + custom_generate_series=get_counts_of_hsi_by_short_treatment_id_by_year, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + ALL[year_target] = hsi_not_delivered_by_year + + # Concatenate the DataFrames into a single DataFrame + concatenated_df = pd.concat(ALL.values(), keys=ALL.keys()) + concatenated_df.index = concatenated_df.index.set_names(['date', 'cause']) + HSI_never_ran_by_year = concatenated_df + + HSI_never_ran_by_year = HSI_never_ran_by_year.fillna(0) #clean_df( + HSI_ran_by_year = HSI_ran_by_year.fillna(0) + HSI_total_by_year = HSI_ran_by_year.add(HSI_never_ran_by_year, fill_value=0) + HSI_ran_by_year.to_csv('ConvertedOutputs/HSIs_ran_by_area_with_time.csv', index=True) + HSI_never_ran_by_year.to_csv('ConvertedOutputs/HSIs_never_ran_by_area_with_time.csv', index=True) + print(HSI_ran_by_year) + print(HSI_never_ran_by_year) + print(HSI_total_by_year) + +if __name__ == "__main__": + rfp = Path('resources') + + parser = argparse.ArgumentParser( + description="Produce plots to show the impact each set of treatments", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--output-path", + help=( + "Directory to write outputs to. If not specified (set to None) outputs " + "will be written to value of --results-path argument." + ), + type=Path, + default=None, + required=False, + ) + parser.add_argument( + "--resources-path", + help="Directory containing resource files", + type=Path, + default=Path('resources'), + required=False, + ) + parser.add_argument( + "--results-path", + type=Path, + help=( + "Directory containing results from running " + "src/scripts/analysis_data_generation/scenario_generate_chains.py " + ), + default=None, + required=False + ) + args = parser.parse_args() + assert args.results_path is not None + results_path = args.results_path + + output_path = results_path if args.output_path is None else args.output_path + + apply( + results_folder=results_path, + output_folder=output_path, + resourcefilepath=args.resources_path + ) diff --git a/src/scripts/analysis_data_generation/postprocess_events_chain.py b/src/scripts/analysis_data_generation/postprocess_events_chain.py new file mode 100644 index 0000000000..96c27a04b1 --- /dev/null +++ b/src/scripts/analysis_data_generation/postprocess_events_chain.py @@ -0,0 +1,156 @@ +import pandas as pd +from dateutil.relativedelta import relativedelta + +# Remove from every individual's event chain all events that were fired after death +def cut_off_events_after_death(df): + + events_chain = df.groupby('person_ID') + + filtered_data = pd.DataFrame() + + for name, group in events_chain: + + # Find the first non-NaN 'date_of_death' and its index + first_non_nan_index = group['date_of_death'].first_valid_index() + + if first_non_nan_index is not None: + # Filter out all rows after the first non-NaN index + filtered_group = group.loc[:first_non_nan_index] # Keep rows up to and including the first valid index + filtered_data = pd.concat([filtered_data, filtered_group]) + else: + # If there are no non-NaN values, keep the original group + filtered_data = pd.concat([filtered_data, group]) + + return filtered_data + +# Load into DataFrame +def load_csv_to_dataframe(file_path): + try: + # Load raw chains into df + df = pd.read_csv(file_path) + print("Raw event chains loaded successfully!") + return df + except FileNotFoundError: + print(f"Error: The file '{file_path}' was not found.") + except Exception as e: + print(f"An error occurred: {e}") + +file_path = 'output.csv' # Replace with the path to your CSV file + +output = load_csv_to_dataframe(file_path) + +# Some of the dates appeared not to be in datetime format. Correct here. +output['date_of_death'] = pd.to_datetime(output['date_of_death'], errors='coerce') +output['date_of_birth'] = pd.to_datetime(output['date_of_birth'], errors='coerce') +if 'hv_date_inf' in output.columns: + output['hv_date_inf'] = pd.to_datetime(output['hv_date_inf'], errors='coerce') + + +date_start = pd.to_datetime('2010-01-01') +if 'Other' in output['cause_of_death'].values: + print("ERROR: 'Other' was included in sim as possible cause of death") + exit(-1) + +# Choose which columns in individual properties to visualise +columns_to_print =['event','is_alive','hv_inf', 'hv_art','tb_inf', 'tb_date_active', 'event_date', 'when'] +#columns_to_print =['person_ID', 'date_of_birth', 'date_of_death', 'cause_of_death','hv_date_inf', 'hv_art','tb_inf', 'tb_date_active', 'event date', 'event'] + +# When checking which individuals led to *any* changes in individual properties, exclude these columns from comparison +columns_to_exclude_in_comparison = ['when', 'event', 'event_date', 'age_exact_years', 'age_years', 'age_days', 'age_range', 'level', 'appt_footprint'] + +# If considering epidemiology consistent with sim, add check here. +check_ages_of_those_HIV_inf = False +if check_ages_of_those_HIV_inf: + for index, row in output.iterrows(): + if pd.isna(row['hv_date_inf']): + continue # Skip this iteration + diff = relativedelta(output.loc[index, 'hv_date_inf'],output.loc[index, 'date_of_birth']) + if diff.years > 1 and diff.years<15: + print("Person contracted HIV infection at age younger than 15", diff) + +# Remove events after death +filtered_data = cut_off_events_after_death(output) + +print_raw_events = True # Print raw chain of events for each individual +print_selected_changes = False +print_all_changes = True +person_ID_of_interest = 494 + +pd.set_option('display.max_rows', None) + +for name, group in filtered_data.groupby('person_ID'): + list_of_dob = group['date_of_birth'] + + # Select individuals based on when they were born + if list_of_dob.iloc[0].year<2010: + + # Check that immutable properties are fixed for this individual, i.e. that events were collated properly: + all_identical_dob = group['date_of_birth'].nunique() == 1 + all_identical_sex = group['sex'].nunique() == 1 + if all_identical_dob is False or all_identical_sex is False: + print("Immutable properties are changing! This is not chain for single individual") + print(group) + exit(-1) + + print("----------------------------------------------------------------------") + print("person_ID ", group['person_ID'].iloc[0], "d.o.b ", group['date_of_birth'].iloc[0]) + print("Number of events for this individual ", group['person_ID'].iloc[0], "is :", len(group)/2) # Divide by 2 before printing Before/After for each event + number_of_events =len(group)/2 + number_of_changes=0 + if print_raw_events: + print(group) + + if print_all_changes: + # Check each row + comparison = group.drop(columns=columns_to_exclude_in_comparison).fillna(-99999).ne(group.drop(columns=columns_to_exclude_in_comparison).shift().fillna(-99999)) + + # Iterate over rows where any column has changed + for idx, row_changed in comparison.iloc[1:].iterrows(): + if row_changed.any(): # Check if any column changed in this row + number_of_changes+=1 + changed_columns = row_changed[row_changed].index.tolist() # Get the columns where changes occurred + print(f"Row {idx} - Changes detected in columns: {changed_columns}") + columns_output = ['event', 'event_date', 'appt_footprint', 'level'] + changed_columns + print(group.loc[idx, columns_output]) # Print only the changed columns + if group.loc[idx, 'when'] == 'Before': + print('-----> THIS CHANGE OCCURRED BEFORE EVENT!') + #print(group.loc[idx,columns_to_print]) + print() # For better readability + print("Number of changes is ", number_of_changes, "out of ", number_of_events, " events") + + if print_selected_changes: + tb_inf_condition = ( + ((group['tb_inf'].shift(1) == 'uninfected') & (group['tb_inf'] == 'active')) | + ((group['tb_inf'].shift(1) == 'latent') & (group['tb_inf'] == 'active')) | + ((group['tb_inf'].shift(1) == 'active') & (group['tb_inf'] == 'latent')) | + ((group['hv_inf'].shift(1) is False) & (group['hv_inf'] is True)) | + ((group['hv_art'].shift(1) == 'not') & (group['hv_art'] == 'on_not_VL_suppressed')) | + ((group['hv_art'].shift(1) == 'not') & (group['hv_art'] == 'on_VL_suppressed')) | + ((group['hv_art'].shift(1) == 'on_VL_suppressed') & (group['hv_art'] == 'on_not_VL_suppressed')) | + ((group['hv_art'].shift(1) == 'on_VL_suppressed') & (group['hv_art'] == 'not')) | + ((group['hv_art'].shift(1) == 'on_not_VL_suppressed') & (group['hv_art'] == 'on_VL_suppressed')) | + ((group['hv_art'].shift(1) == 'on_not_VL_suppressed') & (group['hv_art'] == 'not')) + ) + + alive_condition = ( + (group['is_alive'].shift(1) is True) & (group['is_alive'] is False) + ) + # Combine conditions for rows of interest + transition_condition = tb_inf_condition | alive_condition + + if list_of_dob.iloc[0].year >= 2010: + print("DETECTED OF INTEREST") + print(group[group['event'] == 'Birth'][columns_to_print]) + + # Filter the DataFrame based on the condition + filtered_transitions = group[transition_condition] + if not filtered_transitions.empty: + if list_of_dob.iloc[0].year < 2010: + print("DETECTED OF INTEREST") + print(filtered_transitions[columns_to_print]) + + +print("Number of individuals simulated ", filtered_data.groupby('person_ID').ngroups) + + + diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py new file mode 100644 index 0000000000..6bdcd02d90 --- /dev/null +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -0,0 +1,115 @@ +"""This Scenario file run the model to generate event chans + +Run on the batch system using: +``` +tlo batch-submit + src/scripts/analysis_data_generation/scenario_generate_chains.py +``` + +or locally using: +``` + tlo scenario-run src/scripts/analysis_data_generation/scenario_generate_chains.py +``` + +""" +from pathlib import Path +from typing import Dict + +import pandas as pd + +from tlo import Date, logging +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher +from tlo.scenario import BaseScenario + + +class GenerateDataChains(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = self.start_date + pd.DateOffset(months=1) + self.pop_size = 120 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 1 + self.generate_event_chains = True + + def log_configuration(self): + return { + 'filename': 'generate_event_chains', + 'directory': Path('./outputs'), # <- (specified only for local running) + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.events': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem.summary': logging.INFO, + } + } + + def modules(self): + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) + + def draw_parameters(self, draw_number, rng): + if draw_number < self.number_of_draws: + return list(self._scenarios.values())[draw_number] + else: + return + + # case 1: gfHE = -0.030, factor = 1.01074 + # case 2: gfHE = -0.020, factor = 1.02116 + # case 3: gfHE = -0.015, factor = 1.02637 + # case 4: gfHE = 0.015, factor = 1.05763 + # case 5: gfHE = 0.020, factor = 1.06284 + # case 6: gfHE = 0.030, factor = 1.07326 + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario. + """ + + self.YEAR_OF_CHANGE = 2019 + + return { + + # =========== STATUS QUO ============ + "Baseline": + mix_scenarios( + self._baseline(), + { + "HealthSystem": { + "yearly_HR_scaling_mode": "no_scaling", + }, + } + ), + + } + + def _baseline(self) -> Dict: + """Return the Dict with values for the parameter changes that define the baseline scenario. """ + return mix_scenarios( + get_parameters_for_status_quo(), + { + "HealthSystem": { + "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration + "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH + "year_mode_switch": self.YEAR_OF_CHANGE, + "scale_to_effective_capabilities": True, + "policy_name": "Naive", + "tclose_overwrite": 1, + "tclose_days_offset_overwrite": 7, + "use_funded_or_actual_staffing": "actual", + "cons_availability": "default", + } + }, + ) + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) diff --git a/src/tlo/events.py b/src/tlo/events.py index 98832faecb..00a6fe4e7d 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -11,6 +11,8 @@ import pandas as pd +FACTOR_POP_DICT = 5000 + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -83,13 +85,14 @@ def compare_population_dataframe(self,df_before, df_after): # Create an empty list to store changes for each of the individuals chain_links = {} - + len_of_diff = len(diff_mask) + # Loop through each row of the mask + for idx, row in diff_mask.iterrows(): changed_cols = row.index[row].tolist() - + if changed_cols: # Proceed only if there are changes in the row - # Create a dictionary for this person # First add event info link_info = { @@ -103,7 +106,7 @@ def compare_population_dataframe(self,df_before, df_after): link_info[col] = df_after.at[idx, col] # Append the event and changes to the individual key - chain_links = {idx : link_info} + chain_links[idx] = str(link_info) return chain_links @@ -168,7 +171,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] - chain_links = {self.target : link_info} + chain_links[self.target] = str(link_info) # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. if debug_chains: @@ -228,14 +231,18 @@ def run(self): if self.sim.generate_event_chains: chain_links = self.store_chains_to_do_after_event(print_chains, row_before, df_before) + # Create empty logger for entire pop + pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals + + pop_dict.update(chain_links) + # Log chain_links here if len(chain_links)>0: logger_chain.info(key='event_chains', - data= chain_links, - description='Links forming chains of events for simulated individuals') + data= pop_dict, + description='Links forming chains of events for simulated individuals') #print("Chain events ", chain_links) - class RegularEvent(Event): diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 041ab9cf08..d657e9d3a0 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -11,6 +11,8 @@ import pandas as pd +FACTOR_POP_DICT = 5000 + if TYPE_CHECKING: from tlo import Module, Simulation @@ -276,7 +278,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] - chain_links = {self.target : link_info} + chain_links = {self.target : str(link_info)} # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] @@ -314,10 +316,15 @@ def run(self, squeeze_factor): chain_links = self.store_chains_to_do_after_event(print_chains, row_before, str(footprint)) if len(chain_links)>0: + + pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} + # pop_dict = {i: '' for i in range(1000)} # Always include all possible individuals + + pop_dict.update(chain_links) + logger_chains.info(key='event_chains', - data = chain_links, + data = pop_dict, description='Links forming chains of events for simulated individuals') - #print(chain_links) return updated_appt_footprint diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 0c70b164d9..d9ba62c43a 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -40,6 +40,8 @@ logger_chains = logging.getLogger("tlo.methods.event") logger_chains.setLevel(logging.INFO) +FACTOR_POP_DICT = 5000 + class SimulationPreviouslyInitialisedError(Exception): """Exception raised when trying to initialise an already initialised simulation.""" @@ -294,17 +296,18 @@ def make_initial_population(self, *, n: int) -> None: if self.generate_event_chains: pop_dict = self.population.props.to_dict(orient='index') - - print(pop_dict) - print(pop_dict.keys()) for key in pop_dict.keys(): pop_dict[key]['person_ID'] = key - print("Length of properties", len(pop_dict[0].keys())) - #exit(-1) + pop_dict[key] = str(pop_dict[key]) # Log as string to avoid issues around length of properties stored later + + pop_dict_full = {i: '' for i in range(FACTOR_POP_DICT)} + pop_dict_full.update(pop_dict) + + print("Size for full sim", len(pop_dict_full)) + logger.info(key='event_chains', - data = pop_dict, + data = pop_dict_full, description='Links forming chains of events for simulated individuals') - end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") @@ -323,7 +326,7 @@ def initialise(self, *, end_date: Date) -> None: #self.generate_event_chains = generate_event_chains if self.generate_event_chains: # Eventually this can be made an option - self.generate_event_chains_overwrite_epi = False + self.generate_event_chains_overwrite_epi = True # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] @@ -480,9 +483,13 @@ def do_birth(self, mother_id: int) -> int: prop_dict = self.population.props.loc[child_id].to_dict() prop_dict['event'] = 'Birth' prop_dict['event_date'] = self.date - child_dict = {child_id : prop_dict} + + pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals + pop_dict[child_id] = str(prop_dict) # Convert to string to avoid issue of length + + print("Length at birth", len(pop_dict)) logger.info(key='event_chains', - data = child_dict, + data = pop_dict, description='Links forming chains of events for simulated individuals') # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. @@ -492,7 +499,7 @@ def do_birth(self, mother_id: int) -> int: row['event_date'] = self.date row['when'] = 'After' self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) - + return child_id def find_events_for_person(self, person_id: int) -> list[tuple[Date, Event]]: From c8905bb163a808170883277a5a6b2268f03341e8 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 31 Oct 2024 11:12:49 +0000 Subject: [PATCH 061/103] edits --- .../dummy_cohort_azure_calib.py | 82 +++++++------------ 1 file changed, 30 insertions(+), 52 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index a79c1a1a79..8565ff85d7 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -9,7 +9,7 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario = 'block_intervention_test-2024-10-17T161423Z' +scenario = 'block_intervention_test-2024-10-18T113429Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] @@ -36,15 +36,27 @@ def sort_df(_df): return [results_df, results_df_summ] +results = {k:get_data_frames(k, results_folder)[0] for k in + ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', + 'yearly_mnh_counter_dict']} + results_sum = {k:get_data_frames(k, results_folder)[1] for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} -results = {k:get_data_frames(k, results_folder)[0] for k in - ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', - 'yearly_mnh_counter_dict']} +def get_data(df, draw): + return (df.loc['direct_mmr', (draw, 'lower')], + df.loc['direct_mmr', (draw, 'mean')], + df.loc['direct_mmr', (draw, 'upper')]) -baseline = results['deaths_and_stillbirths'].loc['direct_mmr', (0, slice(0, 19))].droplevel(0) +mmrs = {'baseline':get_data(results_sum['deaths_and_stillbirths'], 0), + 'blood_transfusion_min':get_data(results_sum['deaths_and_stillbirths'], 1), + 'blood_transfusion_max': get_data(results_sum['deaths_and_stillbirths'], 2), + 'pph_treatment_uterotonics_min':get_data(results_sum['deaths_and_stillbirths'], 3), + 'pph_treatment_uterotonics_max': get_data(results_sum['deaths_and_stillbirths'], 4), + 'sepsis_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 5), + 'sepsis_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 6), + } def get_mmr_diffs(df, draws): diff_results = {} @@ -53,7 +65,7 @@ def get_mmr_diffs(df, draws): for draw in draws: # diff_df = ((results['deaths_and_stillbirths'][draw] - baseline)/baseline) * 100 diff_df = results['deaths_and_stillbirths'][draw] - baseline - diff_df.columns = pd.MultiIndex.from_tuples([(0, v) for v in range(len(diff_df.columns))], + diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], names=['draw', 'run']) results_diff = summarize(diff_df) results_diff.fillna(0) @@ -61,25 +73,19 @@ def get_mmr_diffs(df, draws): return diff_results -diff_results = get_mmr_diffs(results, range(1,4)) +diff_results = get_mmr_diffs(results, range(1,7)) -def get_data(df, draw): - return (df.loc['direct_mmr', (draw, 'lower')], - df.loc['direct_mmr', (draw, 'mean')], - df.loc['direct_mmr', (draw, 'upper')]) - -results = {'baseline':get_data(results['deaths_and_stillbirths'], 0), - 'blood_transfusion':get_data(results['deaths_and_stillbirths'], 1), - 'pph_treatment_uterotonics':get_data(results['deaths_and_stillbirths'], 2), - 'sepsis_treatment':get_data(results['deaths_and_stillbirths'], 3)} -results_diff = {'blood_transfusion':get_data(diff_results[1], 1), - 'pph_treatment_uterotonics':get_data(diff_results[2], 2), - 'sepsis_treatment':get_data(diff_results[3], 3)} +results_diff = {'blood_transfusion_min':get_data(diff_results[1], 1), + 'blood_transfusion_max':get_data(diff_results[2], 2), + 'pph_treatment_uterotonics_min':get_data(diff_results[3], 3), + 'pph_treatment_uterotonics_max': get_data(diff_results[4], 4), + 'sepsis_treatment_mins':get_data(diff_results[5], 5), + 'sepsis_treatment_max': get_data(diff_results[6], 6)} # todo: compare deaths with demography logging... -results = data +data = mmrs # Extract means and errors labels = data.keys() @@ -91,43 +97,15 @@ def get_data(df, draw): # Create bar chart with error bars fig, ax = plt.subplots() ax.bar(labels, means, yerr=errors, capsize=5, alpha=0.7, ecolor='black') -ax.set_ylabel('Values') -ax.set_title('Bar Chart with Error Bars') +ax.set_ylabel('MMR') +ax.set_title('Average MMR under each scenario') # Adjust label size -plt.xticks(rotation=45, fontsize=8) +plt.xticks(fontsize=8) plt.tight_layout() plt.show() -# scenario_filename = 'cohort_test-2024-10-09T130546Z' -# # scenario_filename2 = 'cohort_test-2024-10-15T122825Z' -# scenario_filename2 = 'cohort_test-2024-10-16T071357Z' -# -# results_folder_old = get_scenario_outputs(scenario_filename, outputspath)[-1] -# results_folder_new = get_scenario_outputs(scenario_filename2, outputspath)[-1] -# -# def get_data_frames(key, results_folder): -# def sort_df(_df): -# _x = _df.drop(columns=['date'], inplace=False) -# return _x.iloc[0] -# -# results_df = summarize (extract_results( -# results_folder, -# module="tlo.methods.pregnancy_supervisor", -# key=key, -# custom_generate_series=sort_df, -# do_scaling=False -# )) -# -# return results_df -# -# results_old = {k:get_data_frames(k, results_folder_old) for k in -# ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', -# 'yearly_mnh_counter_dict']} -# -# results_new = {k:get_data_frames(k, results_folder_new) for k in -# ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths', 'service_coverage', -# 'yearly_mnh_counter_dict']} + # # import matplotlib.pyplot as plt # import numpy as np From 6c9ae17df9bdbbd93fb4f56f5bfd89fe1e1b68c7 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 6 Nov 2024 14:47:17 +0000 Subject: [PATCH 062/103] edits --- .../ResourceFile_PregnancySupervisor.xlsx | 4 ++-- .../cohort_interventions_scenario.py | 21 +++++++++++++------ .../dummy_cohort_azure_calib.py | 2 +- src/tlo/methods/labour.py | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index d352715855..9ce97f66c5 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e09d4af6900872f5379b421eb9adc9121f3171271a1aab093dc7448bec012428 -size 24223 +oid sha256:c8b23b40eb24cb5308f320b5b920d2a31e771b675384bb273ea964d3b311cb8f +size 24202 diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index e10b64842f..bdcb387ae0 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -12,11 +12,11 @@ class BaselineScenario(BaseScenario): def __init__(self): super().__init__() - self.seed = 537184 + self.seed = 562661 self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) - self.pop_size = 10_000 - self.number_of_draws = 7 + self.pop_size = 12_000 + self.number_of_draws = 15 self.runs_per_draw = 20 def log_configuration(self): @@ -48,10 +48,19 @@ def draw_parameters(self, draw_number, rng): return {'PregnancySupervisor': { 'analysis_year': 2024}} else: - interventions_for_analysis = ['blood_transfusion', 'blood_transfusion', - 'pph_treatment_uterotonics', 'pph_treatment_uterotonics', - 'sepsis_treatment', 'sepsis_treatment'] + interventions_for_analysis = ['oral_antihypertensives', 'oral_antihypertensives', + 'iv_antihypertensives', 'iv_antihypertensives', + 'amtsl', 'amtsl' + 'mgso4', 'mgso4', + 'post_abortion_care_core', 'post_abortion_care_core', + 'caesarean_section', 'caesarean_section', + 'ectopic_pregnancy_treatment', 'ectopic_pregnancy_treatment'] + avail_for_draw = [0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, 0.0, 1.0, 0.0, 1.0] diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 8565ff85d7..f871c7b3ac 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -101,7 +101,7 @@ def get_mmr_diffs(df, draws): ax.set_title('Average MMR under each scenario') # Adjust label size -plt.xticks(fontsize=8) +plt.xticks(fontsize=8, rotation=90) plt.tight_layout() plt.show() diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index f9662ed7d3..12eae91d7d 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -3329,7 +3329,7 @@ def apply(self, person_id, squeeze_factor): # hsi_event=self) cs_delivered = pregnancy_helper_functions.check_int_deliverable( - self.module, int_name='abx_for_prom', hsi_event=self, + self.module, int_name='caesarean_section', hsi_event=self, q_param=[params['prob_hcw_avail_surg'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.module.item_codes_lab_consumables['caesarean_delivery_core'], opt_cons=self.module.item_codes_lab_consumables['caesarean_delivery_optional']) From 1d3d3b44cb142aeacbc0d3458fcfdd3543cdb24f Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 6 Nov 2024 14:49:28 +0000 Subject: [PATCH 063/103] edits --- .../cohort_analysis/cohort_interventions_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index bdcb387ae0..5eedf6456e 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -50,7 +50,7 @@ def draw_parameters(self, draw_number, rng): else: interventions_for_analysis = ['oral_antihypertensives', 'oral_antihypertensives', 'iv_antihypertensives', 'iv_antihypertensives', - 'amtsl', 'amtsl' + 'amtsl', 'amtsl', 'mgso4', 'mgso4', 'post_abortion_care_core', 'post_abortion_care_core', 'caesarean_section', 'caesarean_section', From 85a77dfaab380aa16e2cc47865d3b3adf1ca0249 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 6 Nov 2024 16:03:07 +0000 Subject: [PATCH 064/103] edits --- .../cohort_interventions_group_scenario.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_group_scenario.py diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_group_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_group_scenario.py new file mode 100644 index 0000000000..587f4e1e85 --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_group_scenario.py @@ -0,0 +1,81 @@ +import numpy as np +import pandas as pd + +from pathlib import Path + +from tlo import Date, logging +from tlo.methods import mnh_cohort_module +from tlo.methods.fullmodel import fullmodel +from tlo.scenario import BaseScenario + + +class BaselineScenario(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 562661 + self.start_date = Date(2024, 1, 1) + self.end_date = Date(2025, 1, 2) + self.pop_size = 12_000 + self.number_of_draws = 11 + self.runs_per_draw = 20 + + def log_configuration(self): + return { + 'filename': 'block_intervention_group_test', 'directory': './outputs', + "custom_levels": { + "*": logging.WARNING, + "tlo.methods.demography": logging.INFO, + "tlo.methods.demography.detail": logging.INFO, + "tlo.methods.contraception": logging.INFO, + "tlo.methods.healthsystem.summary": logging.INFO, + "tlo.methods.healthburden": logging.INFO, + "tlo.methods.labour": logging.INFO, + "tlo.methods.labour.detail": logging.INFO, + "tlo.methods.newborn_outcomes": logging.INFO, + "tlo.methods.care_of_women_during_pregnancy": logging.INFO, + "tlo.methods.pregnancy_supervisor": logging.INFO, + "tlo.methods.postnatal_supervisor": logging.INFO, + } + } + + def modules(self): + return [*fullmodel(resourcefilepath=self.resources, + module_kwargs={'Schisto': {'mda_execute': False}}), + mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=self.resources)] + + def draw_parameters(self, draw_number, rng): + if draw_number == 0: + return {'PregnancySupervisor': { + 'analysis_year': 2024}} + else: + intervention_groups = [['oral_antihypertensives', 'iv_antihypertensives', 'mgso4'], + ['oral_antihypertensives', 'iv_antihypertensives', 'mgso4'], + + ['amtsl', 'pph_treatment_uterotonics', 'pph_treatment_mrrp'], + ['amtsl', 'pph_treatment_uterotonics', 'pph_treatment_mrrp'], + + ['post_abortion_care_core', 'ectopic_pregnancy_treatment'], + ['post_abortion_care_core', 'ectopic_pregnancy_treatment'], + + ['caesarean_section', 'blood_transfusion', 'pph_treatment_surgery'], + ['caesarean_section', 'blood_transfusion', 'pph_treatment_surgery'], + + ['abx_for_prom', 'sepsis_treatment', 'birth_kit'], + ['abx_for_prom', 'sepsis_treatment', 'birth_kit']] + + avail_for_draw = [0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0] + + return {'PregnancySupervisor': { + 'analysis_year': 2024, + 'interventions_analysis': True, + 'interventions_under_analysis':intervention_groups[draw_number-1], + 'intervention_analysis_availability': avail_for_draw[draw_number-1]}} + + +if __name__ == '__main__': + from tlo.cli import scenario_run + scenario_run([__file__]) From 0f0c0f21ca37ca43ca4a00a0edb934eecdf1f53a Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 7 Nov 2024 14:47:40 +0000 Subject: [PATCH 065/103] edits --- .../dummy_cohort_azure_calib.py | 100 +++++++++++++++--- .../methods/care_of_women_during_pregnancy.py | 2 +- src/tlo/methods/labour.py | 2 +- 3 files changed, 85 insertions(+), 19 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index f871c7b3ac..be55ab39ea 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -9,7 +9,7 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario = 'block_intervention_test-2024-10-18T113429Z' +scenario = 'block_intervention_test-2024-11-06T145016Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] @@ -49,13 +49,22 @@ def get_data(df, draw): df.loc['direct_mmr', (draw, 'mean')], df.loc['direct_mmr', (draw, 'upper')]) + mmrs = {'baseline':get_data(results_sum['deaths_and_stillbirths'], 0), - 'blood_transfusion_min':get_data(results_sum['deaths_and_stillbirths'], 1), - 'blood_transfusion_max': get_data(results_sum['deaths_and_stillbirths'], 2), - 'pph_treatment_uterotonics_min':get_data(results_sum['deaths_and_stillbirths'], 3), - 'pph_treatment_uterotonics_max': get_data(results_sum['deaths_and_stillbirths'], 4), - 'sepsis_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 5), - 'sepsis_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 6), + 'oral_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 1), + 'oral_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 2), + 'iv_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 3), + 'iv_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 4), + 'amtsl_min':get_data(results_sum['deaths_and_stillbirths'], 5), + 'amtsl_max': get_data(results_sum['deaths_and_stillbirths'], 6), + 'mgso4_min':get_data(results_sum['deaths_and_stillbirths'], 7), + 'mgso4_max': get_data(results_sum['deaths_and_stillbirths'], 8), + 'post_abortion_care_core_min':get_data(results_sum['deaths_and_stillbirths'], 9), + 'post_abortion_care_core_max': get_data(results_sum['deaths_and_stillbirths'], 10), + 'caesarean_section_min':get_data(results_sum['deaths_and_stillbirths'], 11), + 'caesarean_section_max': get_data(results_sum['deaths_and_stillbirths'], 12), + 'ectopic_pregnancy_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 13), + 'ectopic_pregnancy_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 14), } def get_mmr_diffs(df, draws): @@ -73,15 +82,24 @@ def get_mmr_diffs(df, draws): return diff_results -diff_results = get_mmr_diffs(results, range(1,7)) - - -results_diff = {'blood_transfusion_min':get_data(diff_results[1], 1), - 'blood_transfusion_max':get_data(diff_results[2], 2), - 'pph_treatment_uterotonics_min':get_data(diff_results[3], 3), - 'pph_treatment_uterotonics_max': get_data(diff_results[4], 4), - 'sepsis_treatment_mins':get_data(diff_results[5], 5), - 'sepsis_treatment_max': get_data(diff_results[6], 6)} +diff_results = get_mmr_diffs(results, range(1,15)) + + +results_diff = {'oral_antihypertensives_min':get_data(diff_results[1], 1), + 'oral_antihypertensives_max':get_data(diff_results[2], 2), + 'iv_antihypertensives_min':get_data(diff_results[3], 3), + 'iv_antihypertensives_max': get_data(diff_results[4], 4), + 'amtsl_min':get_data(diff_results[5], 5), + 'amtsl_max': get_data(diff_results[6], 6), + 'mgso4_min':get_data(diff_results[7], 7), + 'mgso4_max':get_data(diff_results[8], 8), + 'post_abortion_care_core_min':get_data(diff_results[9], 9), + 'post_abortion_care_core_max': get_data(diff_results[10], 10), + 'caesarean_section_min':get_data(diff_results[11], 11), + 'caesarean_section_max': get_data(diff_results[12], 12), + 'ectopic_pregnancy_treatment_min':get_data(diff_results[13], 13), + 'ectopic_pregnancy_treatment_max': get_data(diff_results[14], 14) + } # todo: compare deaths with demography logging... @@ -105,8 +123,56 @@ def get_mmr_diffs(df, draws): plt.tight_layout() plt.show() - # +# # Example data with uncertainties +# parameters = ['Blood Transfusion', 'Uterotonics', 'Sepsis treatment'] +# base_value = results_sum['deaths_and_stillbirths'].at['direct_mmr', (0, 'mean')] # base case value for the output variable +# high_values = [results_sum['deaths_and_stillbirths'].at['direct_mmr', (1, 'mean')], +# results_sum['deaths_and_stillbirths'].at['direct_mmr', (3, 'mean')], +# results_sum['deaths_and_stillbirths'].at['direct_mmr', (5, 'mean')]] # lower-bound values for each parameter +# low_values = [results_sum['deaths_and_stillbirths'].at['direct_mmr', (2, 'mean')], +# results_sum['deaths_and_stillbirths'].at['direct_mmr', (4, 'mean')], +# results_sum['deaths_and_stillbirths'].at['direct_mmr', (6, 'mean')]] # upper-bound values for each parameter +# +# # Calculate deltas from base value +# low_deltas = [base_value - lv for lv in low_values] +# high_deltas = [hv - base_value for hv in high_values] +# +# # Sort parameters by absolute impact +# abs_impacts = [abs(low) + abs(high) for low, high in zip(low_deltas, high_deltas)] +# sorted_indices = np.argsort(abs_impacts)[::-1] +# parameters = [parameters[i] for i in sorted_indices] +# low_deltas = [low_deltas[i] for i in sorted_indices] +# high_deltas = [high_deltas[i] for i in sorted_indices] +# +# # Calculate changes from the base case +# low_deltas = [base_value - lv for lv in low_values] +# high_deltas = [hv - base_value for hv in high_values] +# +# # Sort parameters by absolute impact (for a tornado effect) +# abs_impacts = [abs(low) + abs(high) for low, high in zip(low_deltas, high_deltas)] +# sorted_indices = np.argsort(abs_impacts)[::-1] +# parameters = [parameters[i] for i in sorted_indices] +# low_deltas = [low_deltas[i] for i in sorted_indices] +# high_deltas = [high_deltas[i] for i in sorted_indices] +# +# # Plotting +# fig, ax = plt.subplots(figsize=(8, 6)) +# +# # Plot each bar for the low and high values +# for i, (param, low, high) in enumerate(zip(parameters, low_deltas, high_deltas)): +# ax.barh(param, high, left=base_value, color='skyblue') +# ax.barh(param, low, left=base_value + low, color='salmon') +# +# # Reference line for base value +# ax.axvline(base_value, color='black', linestyle='--', label="Base Value") +# +# # Labels and title +# ax.set_xlabel('Output Variable') +# ax.set_title('Tornado Plot') +# plt.legend(['Base Value']) +# plt.show() + # import matplotlib.pyplot as plt # import numpy as np # # Sample data diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 045644d679..94f01ddedf 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -1407,7 +1407,7 @@ def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event): # if avail: iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='oral_antihypertensives', hsi_event=hsi_event, + self, int_name='iv_antihypertensives', hsi_event=hsi_event, q_param=None, cons=self.item_codes_preg_consumables['iv_antihypertensives'], opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'], equipment={'Drip stand', 'Infusion pump'}) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 12eae91d7d..92416cda57 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1794,7 +1794,7 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): # if avail: iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='oral_antihypertensives', hsi_event=hsi_event, + self, int_name='iv_antihypertensives', hsi_event=hsi_event, q_param=None, cons=self.item_codes_lab_consumables['iv_antihypertensives'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) From 775776ea02c8d8b1b14ac07df4b917acf4cde278 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 7 Nov 2024 17:12:13 +0000 Subject: [PATCH 066/103] edits --- .../dummy_cohort_azure_calib.py | 50 +++++++++---------- src/tlo/methods/pregnancy_helper_functions.py | 2 +- src/tlo/methods/pregnancy_supervisor.py | 4 +- ...al_health_helper_and_analysis_functions.py | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index be55ab39ea..9bd58b2175 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -51,20 +51,20 @@ def get_data(df, draw): mmrs = {'baseline':get_data(results_sum['deaths_and_stillbirths'], 0), - 'oral_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 1), - 'oral_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 2), - 'iv_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 3), - 'iv_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 4), - 'amtsl_min':get_data(results_sum['deaths_and_stillbirths'], 5), - 'amtsl_max': get_data(results_sum['deaths_and_stillbirths'], 6), + # 'oral_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 1), + # 'oral_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 2), + # 'iv_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 3), + # 'iv_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 4), + # 'amtsl_min':get_data(results_sum['deaths_and_stillbirths'], 5), + # 'amtsl_max': get_data(results_sum['deaths_and_stillbirths'], 6), 'mgso4_min':get_data(results_sum['deaths_and_stillbirths'], 7), 'mgso4_max': get_data(results_sum['deaths_and_stillbirths'], 8), - 'post_abortion_care_core_min':get_data(results_sum['deaths_and_stillbirths'], 9), - 'post_abortion_care_core_max': get_data(results_sum['deaths_and_stillbirths'], 10), - 'caesarean_section_min':get_data(results_sum['deaths_and_stillbirths'], 11), - 'caesarean_section_max': get_data(results_sum['deaths_and_stillbirths'], 12), - 'ectopic_pregnancy_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 13), - 'ectopic_pregnancy_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 14), + # 'post_abortion_care_core_min':get_data(results_sum['deaths_and_stillbirths'], 9), + # 'post_abortion_care_core_max': get_data(results_sum['deaths_and_stillbirths'], 10), + # 'caesarean_section_min':get_data(results_sum['deaths_and_stillbirths'], 11), + # 'caesarean_section_max': get_data(results_sum['deaths_and_stillbirths'], 12), + # 'ectopic_pregnancy_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 13), + # 'ectopic_pregnancy_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 14), } def get_mmr_diffs(df, draws): @@ -82,23 +82,23 @@ def get_mmr_diffs(df, draws): return diff_results -diff_results = get_mmr_diffs(results, range(1,15)) +diff_results = get_mmr_diffs(results, [7,8]) -results_diff = {'oral_antihypertensives_min':get_data(diff_results[1], 1), - 'oral_antihypertensives_max':get_data(diff_results[2], 2), - 'iv_antihypertensives_min':get_data(diff_results[3], 3), - 'iv_antihypertensives_max': get_data(diff_results[4], 4), - 'amtsl_min':get_data(diff_results[5], 5), - 'amtsl_max': get_data(diff_results[6], 6), +results_diff = {#'oral_antihypertensives_min':get_data(diff_results[1], 1), +# 'oral_antihypertensives_max':get_data(diff_results[2], 2), +# 'iv_antihypertensives_min':get_data(diff_results[3], 3), +# 'iv_antihypertensives_max': get_data(diff_results[4], 4), +# 'amtsl_min':get_data(diff_results[5], 5), +# 'amtsl_max': get_data(diff_results[6], 6), 'mgso4_min':get_data(diff_results[7], 7), 'mgso4_max':get_data(diff_results[8], 8), - 'post_abortion_care_core_min':get_data(diff_results[9], 9), - 'post_abortion_care_core_max': get_data(diff_results[10], 10), - 'caesarean_section_min':get_data(diff_results[11], 11), - 'caesarean_section_max': get_data(diff_results[12], 12), - 'ectopic_pregnancy_treatment_min':get_data(diff_results[13], 13), - 'ectopic_pregnancy_treatment_max': get_data(diff_results[14], 14) + # 'post_abortion_care_core_min':get_data(diff_results[9], 9), + # 'post_abortion_care_core_max': get_data(diff_results[10], 10), + # 'caesarean_section_min':get_data(diff_results[11], 11), + # 'caesarean_section_max': get_data(diff_results[12], 12), + # 'ectopic_pregnancy_treatment_min':get_data(diff_results[13], 13), + # 'ectopic_pregnancy_treatment_max': get_data(diff_results[14], 14) } # todo: compare deaths with demography logging... diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index a6c33811b6..39110c52f7 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -424,7 +424,7 @@ def update_current_parameter_dictionary(self, list_position): for key, value in self.parameters.items(): if isinstance(value, list): - if not value or (len(value)) == 1 or key == 'allowed_interventions': + if not value or (len(value)) == 1 or 'intervention' in key: self.current_parameters[key] = self.parameters[key] else: self.current_parameters[key] = self.parameters[key][list_position] diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 30700391b3..7c7522ef39 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -1687,7 +1687,7 @@ def do_at_generic_first_appt_emergency( "PregnancySupervisor" ].abortion_complications if abortion_complications.has_any( - [person_id], "sepsis", "injury", "haemorrhage", first=True + [person_id], "sepsis", "injury", "haemorrhage", "other", first=True ): event = HSI_CareOfWomenDuringPregnancy_PostAbortionCaseManagement( module=self.sim.modules["CareOfWomenDuringPregnancy"], @@ -2015,7 +2015,7 @@ def apply(self, individual_id): df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' else: - self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury') + self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') df.at[individual_id, 'ac_received_post_abortion_care'] = False if individual_id in mni: diff --git a/tests/test_maternal_health_helper_and_analysis_functions.py b/tests/test_maternal_health_helper_and_analysis_functions.py index 53f1ffdf5f..82295984ed 100644 --- a/tests/test_maternal_health_helper_and_analysis_functions.py +++ b/tests/test_maternal_health_helper_and_analysis_functions.py @@ -146,7 +146,7 @@ def override_dummy_cons(value): sim.modules['HealthSystem'].consumables._refresh_availability_of_consumables(date=sim.date) return sim.modules['Labour'].item_codes_lab_consumables['delivery_core'] - for intervention in pparams['all_interventions']: + for intervention in sim.modules['PregnancySupervisor'].parameters['all_interventions']: pparams['interventions_under_analysis'] = [intervention] pparams['intervention_analysis_availability'] = 1.0 From 75291eaa3408d475b5905af5061992dd73abb164 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 7 Nov 2024 17:14:31 +0000 Subject: [PATCH 067/103] edits --- .../cohort_analysis/cohort_interventions_scenario.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 5eedf6456e..97a03474dd 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -16,8 +16,8 @@ def __init__(self): self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) self.pop_size = 12_000 - self.number_of_draws = 15 - self.runs_per_draw = 20 + self.number_of_draws = 9 + self.runs_per_draw = 30 def log_configuration(self): return { @@ -50,16 +50,10 @@ def draw_parameters(self, draw_number, rng): else: interventions_for_analysis = ['oral_antihypertensives', 'oral_antihypertensives', 'iv_antihypertensives', 'iv_antihypertensives', - 'amtsl', 'amtsl', 'mgso4', 'mgso4', - 'post_abortion_care_core', 'post_abortion_care_core', - 'caesarean_section', 'caesarean_section', - 'ectopic_pregnancy_treatment', 'ectopic_pregnancy_treatment'] + 'post_abortion_care_core', 'post_abortion_care_core'] avail_for_draw = [0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0] From f9f4e5ce00edd7d89ae157dcffcd5d3246011f3b Mon Sep 17 00:00:00 2001 From: joehcollins Date: Fri, 8 Nov 2024 09:46:39 +0000 Subject: [PATCH 068/103] edits --- src/tlo/methods/pregnancy_helper_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 39110c52f7..6ebcbe1b2d 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -424,7 +424,7 @@ def update_current_parameter_dictionary(self, list_position): for key, value in self.parameters.items(): if isinstance(value, list): - if not value or (len(value)) == 1 or 'intervention' in key: + if not value or (len(value)) == 1 or key in ('interventions_under_analysis', 'all_interventions'): self.current_parameters[key] = self.parameters[key] else: self.current_parameters[key] = self.parameters[key][list_position] From cd7c80bbe3e05705dba60517a414c1174269e0e7 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 11 Nov 2024 15:31:25 +0000 Subject: [PATCH 069/103] edits --- .../dummy_cohort_azure_calib.py | 70 ++++++++--- .../cohort_analysis/int_analysis_script.py | 118 ++++++++++++++++++ 2 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py index 9bd58b2175..abc6844cb8 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/dummy_cohort_azure_calib.py @@ -10,10 +10,14 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' scenario = 'block_intervention_test-2024-11-06T145016Z' +ordered_interventions = ['oral_antihypertensives', 'iv_antihypertensives', 'mgso4', 'post_abortion_care_core'] + +intervention_groups = [] +draws = [] results_folder= get_scenario_outputs(scenario, outputspath)[-1] -def get_data_frames(key, results_folder): +def get_ps_data_frames(key, results_folder): def sort_df(_df): _x = _df.drop(columns=['date'], inplace=False) return _x.iloc[0] @@ -25,30 +29,40 @@ def sort_df(_df): custom_generate_series=sort_df, do_scaling=False ) + results_df_summ = summarize(results_df) + + return [results_df, results_df_summ] - results_df_summ = summarize(extract_results( +all_dalys_dfs = extract_results( results_folder, - module="tlo.methods.pregnancy_supervisor", - key=key, - custom_generate_series=sort_df, - do_scaling=False - )) + module="tlo.methods.healthburden", + key="dalys_stacked", + custom_generate_series=( + lambda df: df.drop( + columns=['date', 'sex', 'age_range']).groupby(['year']).sum().stack()), + do_scaling=True) - return [results_df, results_df_summ] +mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Maternal Disorders'), :] + +mat_dalys_df = mat_disorders_all.loc[2024] +mat_dalys_df_sum = summarize(mat_dalys_df) -results = {k:get_data_frames(k, results_folder)[0] for k in +results = {k:get_ps_data_frames(k, results_folder)[0] for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} -results_sum = {k:get_data_frames(k, results_folder)[1] for k in +results_sum = {k:get_ps_data_frames(k, results_folder)[1] for k in ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} -def get_data(df, draw): - return (df.loc['direct_mmr', (draw, 'lower')], - df.loc['direct_mmr', (draw, 'mean')], - df.loc['direct_mmr', (draw, 'upper')]) +def get_data(df, key, draw): + return (df.loc[key, (draw, 'lower')], + df.loc[key, (draw, 'mean')], + df.loc[key, (draw, 'upper')]) + +mmrs_min = {f'{k}_min':get_data(results_sum['deaths_and_stillbirths'], d) for k, d in zip (ordered_interventions, draws) } +mmrs_max = { } mmrs = {'baseline':get_data(results_sum['deaths_and_stillbirths'], 0), # 'oral_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 1), @@ -67,6 +81,7 @@ def get_data(df, draw): # 'ectopic_pregnancy_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 14), } + def get_mmr_diffs(df, draws): diff_results = {} baseline = results['deaths_and_stillbirths'][0] @@ -82,6 +97,33 @@ def get_mmr_diffs(df, draws): return diff_results +# MMR + + + +# Maternal deaths +# DALYs + + + +mmrs = {'baseline':get_data(results_sum['deaths_and_stillbirths'], 0), + 'oral_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 1), + 'oral_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 2), + 'iv_antihypertensives_min':get_data(results_sum['deaths_and_stillbirths'], 3), + 'iv_antihypertensives_max': get_data(results_sum['deaths_and_stillbirths'], 4), + 'amtsl_min':get_data(results_sum['deaths_and_stillbirths'], 5), + 'amtsl_max': get_data(results_sum['deaths_and_stillbirths'], 6), + 'mgso4_min':get_data(results_sum['deaths_and_stillbirths'], 7), + 'mgso4_max': get_data(results_sum['deaths_and_stillbirths'], 8), + 'post_abortion_care_core_min':get_data(results_sum['deaths_and_stillbirths'], 9), + 'post_abortion_care_core_max': get_data(results_sum['deaths_and_stillbirths'], 10), + 'caesarean_section_min':get_data(results_sum['deaths_and_stillbirths'], 11), + 'caesarean_section_max': get_data(results_sum['deaths_and_stillbirths'], 12), + 'ectopic_pregnancy_treatment_min':get_data(results_sum['deaths_and_stillbirths'], 13), + 'ectopic_pregnancy_treatment_max': get_data(results_sum['deaths_and_stillbirths'], 14), + } + + diff_results = get_mmr_diffs(results, [7,8]) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py new file mode 100644 index 0000000000..81673ae9d9 --- /dev/null +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -0,0 +1,118 @@ +import os + +import pandas as pd + +import matplotlib.pyplot as plt +import numpy as np + +from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize + +outputspath = './outputs/sejjj49@ucl.ac.uk/' + +scenario = 'block_intervention_test-2024-11-08T094716Z' +results_folder= get_scenario_outputs(scenario, outputspath)[-1] + +interventions = ['oral_antihypertensives', 'iv_antihypertensives', 'mgso4', 'post_abortion_care_core'] + +int_analysis = ['baseline'] + +for i in interventions: + int_analysis.append(f'{i}_min') + int_analysis.append(f'{i}_max') + +draws = [x for x in range(len(int_analysis))] + +# Access dataframes generated from pregnancy supervisor +def get_ps_data_frames(key, results_folder): + def sort_df(_df): + _x = _df.drop(columns=['date'], inplace=False) + return _x.iloc[0] + + results_df = extract_results( + results_folder, + module="tlo.methods.pregnancy_supervisor", + key=key, + custom_generate_series=sort_df, + do_scaling=False + ) + results_df_summ = summarize(results_df) + + return {'crude':results_df, 'summarised':results_df_summ} + +results = {k:get_ps_data_frames(k, results_folder) for k in + ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', + 'yearly_mnh_counter_dict']} + +all_dalys_dfs = extract_results( + results_folder, + module="tlo.methods.healthburden", + key="dalys_stacked", + custom_generate_series=( + lambda df: df.drop( + columns=['date', 'sex', 'age_range']).groupby(['year']).sum().stack()), + do_scaling=False) + +mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Maternal Disorders'), :] + +mat_dalys_df = mat_disorders_all.loc[2024] +mat_dalys_df_sum = summarize(mat_dalys_df) + +results.update({'dalys':{'crude': mat_dalys_df, 'summarised': mat_dalys_df_sum}}) + +# Summarised results +def get_data(df, key, draw): + return (df.loc[key, (draw, 'lower')], + df.loc[key, (draw, 'mean')], + df.loc[key, (draw, 'upper')]) + +dalys_by_scenario = {k: get_data(results['dalys']['summarised'], 'Maternal Disorders', d) for k, d in zip ( + int_analysis, draws)} + +mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( + int_analysis, draws)} + +def barcharts(data, y_label, title): + + # Extract means and errors + labels = data.keys() + means = [vals[1] for vals in data.values()] + lower_errors = [vals[1] - vals[0] for vals in data.values()] + upper_errors = [vals[2] - vals[1] for vals in data.values()] + errors = [lower_errors, upper_errors] + + # Create bar chart with error bars + fig, ax = plt.subplots() + ax.bar(labels, means, yerr=errors, capsize=5, alpha=0.7, ecolor='black') + ax.set_ylabel(y_label) + ax.set_title(title) + + # Adjust label size + plt.xticks(fontsize=8, rotation=90) + plt.tight_layout() + plt.show() + +barcharts(dalys_by_scenario, 'DALYs', 'Total Maternal Disorders DALYs by scenario') +barcharts(mmr_by_scnario, 'MMR', 'Total Direct MMR by scenario') + +# Difference results +def get_diffs(df_key, result_key, ints, draws): + diff_results = {} + baseline = results[df_key]['crude'][0] + + for draw, int in zip(draws, ints): + diff_df = results[df_key]['crude'][draw] - baseline + diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], + names=['draw', 'run']) + results_diff = summarize(diff_df) + results_diff.fillna(0) + diff_results.update({int: results_diff.loc[result_key].values}) + + return diff_results + +mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws) +dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws) + +dalys_diffs_by_scenario = {k: get_data(dalys_diffs[d], d, d) for k, d in zip(int_analysis, draws)} + +mmr_diffs_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( + int_analysis, draws)} From 2c554a0fe697feea042040ca834f725dc77b2377 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 11 Nov 2024 16:14:33 +0000 Subject: [PATCH 070/103] removed unused variable --- .../cohort_analysis/int_analysis_script.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index 81673ae9d9..f1633829f3 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -112,7 +112,31 @@ def get_diffs(df_key, result_key, ints, draws): mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws) dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws) -dalys_diffs_by_scenario = {k: get_data(dalys_diffs[d], d, d) for k, d in zip(int_analysis, draws)} -mmr_diffs_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( - int_analysis, draws)} +def get_diff_plots(data, outcome): + categories = list(data.keys()) + mins = [arr[0] for arr in data.values()] + means = [arr[1] for arr in data.values()] + maxs = [arr[2] for arr in data.values()] + + # Error bars (top and bottom of the uncertainty interval) + errors = [(mean - min_val, max_val - mean) for mean, min_val, max_val in zip(means, mins, maxs)] + errors = np.array(errors).T + + # Plotting + plt.figure(figsize=(12, 6)) + plt.errorbar(categories, means, yerr=errors, fmt='o', capsize=5) + plt.axhline(0, color='gray', linestyle='--') # Adding a horizontal line at y=0 for reference + plt.xticks(rotation=90) + plt.xlabel('Scenarios') + plt.ylabel('Crude Difference from Baseline Scenario') + plt.title(f'Difference of {outcome} from Baseline Scenario') + plt.grid(True) + plt.tight_layout() + plt.show() + +get_diff_plots(mmr_diffs, 'MMR') +get_diff_plots(dalys_diffs, 'Maternal DALYs') + + + From cb57e5ab773e0e9ebaec8efa89391c898f1d813d Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 11 Nov 2024 17:18:07 +0000 Subject: [PATCH 071/103] editz --- src/tlo/methods/mnh_cohort_module.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 582855df5d..9b85cb2dda 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -47,15 +47,6 @@ def initialise_population(self, population): :param population: the population of individuals """ - # log_file = parse_log_file( - # '/Users/j_collins/PycharmProjects/TLOmodel/resources/maternal cohort/' - # 'fullmodel_200k_cohort__2024-04-24T072516.log', - # level=logging.DEBUG)['tlo.methods.contraception'] - # - # all_pregnancies = log_file['properties_of_pregnant_person'].loc[ - # log_file['properties_of_pregnant_person'].date.dt.year == 2024].drop(columns=['date']) - # all_pregnancies.index = [x for x in range(len(all_pregnancies))] - # Read in excel sheet with cohort all_preg_df = pd.read_excel(Path(f'{self.resourcefilepath}/maternal cohort') / 'ResourceFile_All2024PregnanciesCohortModel.xlsx') From 8934669bb775945e72d13c257fab20316c679246 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 12 Nov 2024 14:30:27 +0000 Subject: [PATCH 072/103] edit --- src/tlo/methods/mnh_cohort_module.py | 39 +++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 9b85cb2dda..40aa10c9db 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -52,7 +52,27 @@ def initialise_population(self, population): 'ResourceFile_All2024PregnanciesCohortModel.xlsx') # Only select rows equal to the desired population size - preg_pop = all_preg_df.loc[0:(len(self.sim.population.props))-1] + if len(self.sim.population.props) <= len(all_preg_df): + preg_pop = all_preg_df.loc[0:(len(self.sim.population.props))-1] + else: + # Calculate the number of rows needed to reach the desired length + additional_rows = len(self.sim.population.props) - len(all_preg_df) + + # Initialize an empty DataFrame for additional rows + rows_to_add = pd.DataFrame(columns=all_preg_df.columns) + + # Loop to fill the required additional rows + while additional_rows > 0: + if additional_rows >= len(all_preg_df): + rows_to_add = pd.concat([rows_to_add, all_preg_df], ignore_index=True) + additional_rows -= len(all_preg_df) + else: + rows_to_add = pd.concat([rows_to_add, all_preg_df.iloc[:additional_rows]], ignore_index=True) + additional_rows = 0 + + # Concatenate the original DataFrame with the additional rows + preg_pop = pd.concat([all_preg_df, rows_to_add], ignore_index=True) + # Set the dtypes and index of the cohort dataframe props_dtypes = self.sim.population.props.dtypes @@ -74,6 +94,23 @@ def initialise_population(self, population): df.loc[population.index, 'date_of_last_pregnancy'] = self.sim.start_date df.loc[population.index, 'co_contraception'] = "not_using" + # import tableone + # columns = ['age_years', 'district_of_residence', 'li_wealth', 'li_bmi', 'li_mar_stat', 'li_ed_lev', + # 'li_urban'] + # categorical = ['district_of_residence', 'li_wealth', 'li_bmi' ,'li_mar_stat', 'li_ed_lev', 'li_urban'] + # continuous = ['age_years'] + # + # rename = {'age_years': 'Age (years)', + # 'district_of_residence': 'District', + # 'li_wealth': 'Wealth Qunitle', + # 'li_bmi': 'BMI level', + # 'li_mar_stat': 'Marital Status', + # 'li_ed_lev': 'Education Level', + # 'li_urban': 'Urban/Rural'} + # from tableone import TableOne + # + # mytable = TableOne(self.sim.population.props[columns], categorical=categorical, + # continuous=continuous, rename=rename, pval=False) def initialise_simulation(self, sim): """Get ready for simulation start. From b9f915d07b32ad14cb47b3b906cdce22e886e5ed Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 12 Nov 2024 16:40:44 +0000 Subject: [PATCH 073/103] edit --- .../cohort_interventions_scenario.py | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 97a03474dd..b0e232089c 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -12,16 +12,16 @@ class BaselineScenario(BaseScenario): def __init__(self): super().__init__() - self.seed = 562661 + self.seed = 796967 self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) - self.pop_size = 12_000 - self.number_of_draws = 9 - self.runs_per_draw = 30 + self.pop_size = 30_000 + self.number_of_draws = 41 + self.runs_per_draw = 15 def log_configuration(self): return { - 'filename': 'block_intervention_test', 'directory': './outputs', + 'filename': 'block_intervention_big_pop_test', 'directory': './outputs', "custom_levels": { "*": logging.WARNING, "tlo.methods.demography": logging.INFO, @@ -48,12 +48,44 @@ def draw_parameters(self, draw_number, rng): return {'PregnancySupervisor': { 'analysis_year': 2024}} else: - interventions_for_analysis = ['oral_antihypertensives', 'oral_antihypertensives', + interventions_for_analysis = ['urine_dipstick','urine_dipstick', + 'bp_measurement','bp_measurement', + 'iron_folic_acid', 'iron_folic_acid', + 'calcium_supplement', 'calcium_supplement', + 'hb_test', 'hb_test', + 'full_blood_count', 'full_blood_count', + 'blood_transfusion', 'blood_transfusion', + 'oral_antihypertensives', 'oral_antihypertensives', 'iv_antihypertensives', 'iv_antihypertensives', 'mgso4', 'mgso4', - 'post_abortion_care_core', 'post_abortion_care_core'] + 'abx_for_prom', 'abx_for_prom', + 'post_abortion_care_core', 'post_abortion_care_core', + 'ectopic_pregnancy_treatment', 'ectopic_pregnancy_treatment', + 'birth_kit', 'birth_kit', + 'sepsis_treatment', 'sepsis_treatment', + 'amtsl', 'amtsl', + 'pph_treatment_uterotonics', 'pph_treatment_uterotonics', + 'pph_treatment_mrrp', 'pph_treatment_mrrp', + 'pph_treatment_surgery', 'pph_treatment_surgery', + 'caesarean_section', 'caesarean_section'] avail_for_draw = [0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, + 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0] From a13f14c78b8505a520c79ab90f26f975bd4ca393 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 12 Nov 2024 16:49:19 +0000 Subject: [PATCH 074/103] edit --- .../cohort_analysis/cohort_interventions_scenario.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index b0e232089c..27f364727d 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -17,7 +17,7 @@ def __init__(self): self.end_date = Date(2025, 1, 2) self.pop_size = 30_000 self.number_of_draws = 41 - self.runs_per_draw = 15 + self.runs_per_draw = 14 def log_configuration(self): return { From 91061e7ee7d33d95dd559ea4f5b39f72c117e973 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 14 Nov 2024 15:27:55 +0000 Subject: [PATCH 075/103] edit --- .../cohort_interventions_scenario.py | 42 ++----------------- .../cohort_analysis/int_analysis_script.py | 28 +++++++------ src/tlo/methods/labour.py | 5 ++- src/tlo/methods/mnh_cohort_module.py | 17 +++++--- src/tlo/methods/postnatal_supervisor.py | 2 + src/tlo/methods/pregnancy_helper_functions.py | 3 ++ src/tlo/methods/pregnancy_supervisor.py | 3 ++ tests/test_mnh_cohort.py | 4 +- 8 files changed, 46 insertions(+), 58 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py index 27f364727d..926fca6b6e 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/cohort_interventions_scenario.py @@ -16,8 +16,8 @@ def __init__(self): self.start_date = Date(2024, 1, 1) self.end_date = Date(2025, 1, 2) self.pop_size = 30_000 - self.number_of_draws = 41 - self.runs_per_draw = 14 + self.number_of_draws = 7 + self.runs_per_draw = 60 def log_configuration(self): return { @@ -48,45 +48,11 @@ def draw_parameters(self, draw_number, rng): return {'PregnancySupervisor': { 'analysis_year': 2024}} else: - interventions_for_analysis = ['urine_dipstick','urine_dipstick', - 'bp_measurement','bp_measurement', - 'iron_folic_acid', 'iron_folic_acid', - 'calcium_supplement', 'calcium_supplement', - 'hb_test', 'hb_test', - 'full_blood_count', 'full_blood_count', - 'blood_transfusion', 'blood_transfusion', - 'oral_antihypertensives', 'oral_antihypertensives', - 'iv_antihypertensives', 'iv_antihypertensives', - 'mgso4', 'mgso4', - 'abx_for_prom', 'abx_for_prom', + interventions_for_analysis = ['bp_measurement','bp_measurement', 'post_abortion_care_core', 'post_abortion_care_core', - 'ectopic_pregnancy_treatment', 'ectopic_pregnancy_treatment', - 'birth_kit', 'birth_kit', - 'sepsis_treatment', 'sepsis_treatment', - 'amtsl', 'amtsl', - 'pph_treatment_uterotonics', 'pph_treatment_uterotonics', - 'pph_treatment_mrrp', 'pph_treatment_mrrp', - 'pph_treatment_surgery', 'pph_treatment_surgery', - 'caesarean_section', 'caesarean_section'] + 'ectopic_pregnancy_treatment', 'ectopic_pregnancy_treatment'] avail_for_draw = [0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, - 0.0, 1.0, 0.0, 1.0, 0.0, 1.0] diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index f1633829f3..75b99fa1cf 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -5,14 +5,18 @@ import matplotlib.pyplot as plt import numpy as np -from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize +from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize, create_pickles_locally outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario = 'block_intervention_test-2024-11-08T094716Z' +scenario = 'block_intervention_big_pop_test-2024-11-12T165005Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] -interventions = ['oral_antihypertensives', 'iv_antihypertensives', 'mgso4', 'post_abortion_care_core'] +interventions =['urine_dipstick','bp_measurement', 'iron_folic_acid', 'calcium_supplement', 'hb_test', + 'full_blood_count','blood_transfusion', 'oral_antihypertensives','iv_antihypertensives', + 'mgso4', 'abx_for_prom', 'post_abortion_care_core', 'ectopic_pregnancy_treatment', + 'birth_kit', 'sepsis_treatment', 'amtsl', 'pph_treatment_uterotonics', 'pph_treatment_mrrp', + 'pph_treatment_surgery','caesarean_section'] int_analysis = ['baseline'] @@ -52,7 +56,7 @@ def sort_df(_df): columns=['date', 'sex', 'age_range']).groupby(['year']).sum().stack()), do_scaling=False) -mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Maternal Disorders'), :] +mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Neonatal Disorders'), :] mat_dalys_df = mat_disorders_all.loc[2024] mat_dalys_df_sum = summarize(mat_dalys_df) @@ -65,10 +69,10 @@ def get_data(df, key, draw): df.loc[key, (draw, 'mean')], df.loc[key, (draw, 'upper')]) -dalys_by_scenario = {k: get_data(results['dalys']['summarised'], 'Maternal Disorders', d) for k, d in zip ( +dalys_by_scenario = {k: get_data(results['dalys']['summarised'], 'Neonatal Disorders', d) for k, d in zip ( int_analysis, draws)} -mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( +mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'nmr', d) for k, d in zip ( int_analysis, draws)} def barcharts(data, y_label, title): @@ -91,8 +95,8 @@ def barcharts(data, y_label, title): plt.tight_layout() plt.show() -barcharts(dalys_by_scenario, 'DALYs', 'Total Maternal Disorders DALYs by scenario') -barcharts(mmr_by_scnario, 'MMR', 'Total Direct MMR by scenario') +barcharts(dalys_by_scenario, 'DALYs', 'Total Neonatal Disorders DALYs by scenario') +barcharts(mmr_by_scnario, 'NMR', 'Total NMR by scenario') # Difference results def get_diffs(df_key, result_key, ints, draws): @@ -109,8 +113,8 @@ def get_diffs(df_key, result_key, ints, draws): return diff_results -mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws) -dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws) +mmr_diffs = get_diffs('deaths_and_stillbirths', 'nmr', int_analysis, draws) +dalys_diffs = get_diffs('dalys', 'Neonatal Disorders', int_analysis, draws) def get_diff_plots(data, outcome): @@ -135,8 +139,8 @@ def get_diff_plots(data, outcome): plt.tight_layout() plt.show() -get_diff_plots(mmr_diffs, 'MMR') -get_diff_plots(dalys_diffs, 'Maternal DALYs') +get_diff_plots(mmr_diffs, 'NMR') +get_diff_plots(dalys_diffs, 'Neonatal DALYs') diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 92416cda57..7c31218ec3 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1448,7 +1448,10 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): # Or from mild to severe gestational hypertension, risk reduced by treatment if df.at[individual_id, f'{property_prefix}_htn_disorders'] == 'gest_htn': - if df.at[individual_id, 'la_maternal_hypertension_treatment']: + if (df.at[individual_id, 'la_maternal_hypertension_treatment'] or + df.at[individual_id, 'la_gest_htn_on_treatment'] or + df.at[individual_id, 'ac_gest_htn_on_treatment']): + risk_prog_gh_sgh = params['prob_progression_gest_htn'] * params[ 'anti_htns_treatment_effect_progression'] else: diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 40aa10c9db..27ee7e888e 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -95,18 +95,25 @@ def initialise_population(self, population): df.loc[population.index, 'co_contraception'] = "not_using" # import tableone - # columns = ['age_years', 'district_of_residence', 'li_wealth', 'li_bmi', 'li_mar_stat', 'li_ed_lev', - # 'li_urban'] - # categorical = ['district_of_residence', 'li_wealth', 'li_bmi' ,'li_mar_stat', 'li_ed_lev', 'li_urban'] - # continuous = ['age_years'] + # columns = ['age_years', 'la_parity', 'region_of_residence', 'li_wealth', 'li_bmi', 'li_mar_stat', 'li_ed_lev', + # 'li_urban', 'ps_prev_spont_abortion', 'ps_prev_stillbirth', 'ps_prev_pre_eclamp', 'ps_prev_gest_diab'] + # categorical = ['region_of_residence', 'li_wealth', 'li_bmi' ,'li_mar_stat', 'li_ed_lev', 'li_urban', + # 'ps_prev_spont_abortion', 'ps_prev_stillbirth', 'ps_prev_pre_eclamp', 'ps_prev_gest_diab'] + # continuous = ['age_years', 'la_parity'] # # rename = {'age_years': 'Age (years)', + # 'la_parity': 'Parity', # 'district_of_residence': 'District', # 'li_wealth': 'Wealth Qunitle', # 'li_bmi': 'BMI level', # 'li_mar_stat': 'Marital Status', # 'li_ed_lev': 'Education Level', - # 'li_urban': 'Urban/Rural'} + # 'li_urban': 'Urban/Rural', + # 'ps_prev_spont_abortion': 'Previous Miscarriage', + # 'ps_prev_stillbirth': 'Previous Stillbirth', + # 'ps_prev_pre_eclamp': 'Previous Pre-eclampsia', + # 'ps_prev_gest_diab': 'Previous Gestational Diabetes', + # } # from tableone import TableOne # # mytable = TableOne(self.sim.population.props[columns], categorical=categorical, diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index e0f6708fd5..7ee477e45a 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -647,6 +647,8 @@ def log_new_progressed_cases(disease): # Those women who die the on_death function in demography is applied for person in die_from_htn.loc[die_from_htn].index: self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', originating_module=self.sim.modules['PostnatalSupervisor']) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 6ebcbe1b2d..dc871ff63a 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -10,6 +10,7 @@ def generate_mnh_outcome_counter(): + outcome_list = [ # early/abortive outcomes 'ectopic_unruptured', 'ectopic_ruptured','multiple_pregnancy', 'twin_birth', 'placenta_praevia', 'spontaneous_abortion', 'induced_abortion', 'complicated_spontaneous_abortion', @@ -97,6 +98,8 @@ def check_int_deliverable(self, int_name, hsi_event, individual_id = hsi_event.target p_params = self.sim.modules['PregnancySupervisor'].current_parameters + assert int_name in p_params['all_interventions'] + # Firstly, we determine if an analysis is currently being conducted during which the probability of intervention # delivery is being overridden # To do: replace this parameter diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7c7522ef39..5e0cf42f5a 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -1252,6 +1252,8 @@ def apply_risk_of_death_from_hypertension(self, gestation_of_interest): # Those women who die have InstantaneousDeath scheduled for person in at_risk_of_death_htn.loc[at_risk_of_death_htn].index: self.mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 + self.mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', originating_module=self.sim.modules['PregnancySupervisor']) @@ -2006,6 +2008,7 @@ def apply(self, individual_id): mni[individual_id]['delete_mni'] = True self.module.mnh_outcome_counter[f'{self.cause}_m_death'] += 1 + self.module.mnh_outcome_counter['direct_mat_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=f'{self.cause}', originating_module=self.sim.modules['PregnancySupervisor']) diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 3c6a46d43d..303fe933cd 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -27,11 +27,11 @@ def register_modules(sim): mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath)) def test_run_sim_with_mnh_cohort(tmpdir, seed): - sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "custom_levels":{ + sim = Simulation(start_date=start_date, seed=796967, log_config={"filename": "log", "custom_levels":{ "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=2500) + sim.make_initial_population(n=200) sim.simulate(end_date=Date(2025, 1, 2)) output= parse_log_file(sim.log_filepath) From 20ca8b0b0738de49ff1c73aae03299e977809f63 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Thu, 14 Nov 2024 16:18:25 +0000 Subject: [PATCH 076/103] edit --- .../cohort_analysis/int_analysis_script.py | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index 75b99fa1cf..69ca332f42 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -47,6 +47,17 @@ def sort_df(_df): ['mat_comp_incidence', 'nb_comp_incidence', 'deaths_and_stillbirths','service_coverage', 'yearly_mnh_counter_dict']} +direct_deaths = extract_results( + results_folder, + module="tlo.methods.demography", + key="death", + custom_generate_series=( + lambda df: df.loc[(df['label'] == 'Maternal Disorders')].assign( + year=df['date'].dt.year).groupby(['year'])['year'].count()), + do_scaling=False) +dd_sum = summarize(direct_deaths) + + all_dalys_dfs = extract_results( results_folder, module="tlo.methods.healthburden", @@ -56,7 +67,7 @@ def sort_df(_df): columns=['date', 'sex', 'age_range']).groupby(['year']).sum().stack()), do_scaling=False) -mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Neonatal Disorders'), :] +mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Maternal Disorders'), :] mat_dalys_df = mat_disorders_all.loc[2024] mat_dalys_df_sum = summarize(mat_dalys_df) @@ -69,10 +80,10 @@ def get_data(df, key, draw): df.loc[key, (draw, 'mean')], df.loc[key, (draw, 'upper')]) -dalys_by_scenario = {k: get_data(results['dalys']['summarised'], 'Neonatal Disorders', d) for k, d in zip ( +dalys_by_scenario = {k: get_data(results['dalys']['summarised'], 'Maternal Disorders', d) for k, d in zip ( int_analysis, draws)} -mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'nmr', d) for k, d in zip ( +mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( int_analysis, draws)} def barcharts(data, y_label, title): @@ -96,7 +107,7 @@ def barcharts(data, y_label, title): plt.show() barcharts(dalys_by_scenario, 'DALYs', 'Total Neonatal Disorders DALYs by scenario') -barcharts(mmr_by_scnario, 'NMR', 'Total NMR by scenario') +barcharts(mmr_by_scnario, 'MMR', 'Total MMR by scenario') # Difference results def get_diffs(df_key, result_key, ints, draws): @@ -113,9 +124,22 @@ def get_diffs(df_key, result_key, ints, draws): return diff_results -mmr_diffs = get_diffs('deaths_and_stillbirths', 'nmr', int_analysis, draws) -dalys_diffs = get_diffs('dalys', 'Neonatal Disorders', int_analysis, draws) +diff_results = {} +baseline = direct_deaths[0] + +for draw, int in zip(draws, int_analysis): + diff_df = direct_deaths[draw] - baseline + diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], + names=['draw', 'run']) + results_diff = summarize(diff_df) + results_diff.fillna(0) + diff_results.update({int: results_diff.loc[2024].values}) + +mat_deaths = get_diffs('deaths_and_stillbirths', 'direct_maternal_deaths', int_analysis, draws) +mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws) +dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws) +mat_deaths_2 = diff_results def get_diff_plots(data, outcome): categories = list(data.keys()) @@ -139,8 +163,10 @@ def get_diff_plots(data, outcome): plt.tight_layout() plt.show() -get_diff_plots(mmr_diffs, 'NMR') -get_diff_plots(dalys_diffs, 'Neonatal DALYs') +get_diff_plots(mmr_diffs, 'MMR') +get_diff_plots(mat_deaths, 'Maternal Deaths (crude)') +get_diff_plots(mat_deaths_2, 'Maternal Deaths (demog log)') +get_diff_plots(dalys_diffs, 'Maternal DALYs') From 18d40752e9d8b790872000dcb29041675a2bac09 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 18 Nov 2024 12:56:57 +0000 Subject: [PATCH 077/103] edit --- .../cohort_analysis/int_analysis_script.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index 69ca332f42..69a4150990 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -9,14 +9,10 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario = 'block_intervention_big_pop_test-2024-11-12T165005Z' +scenario = 'block_intervention_big_pop_test-2024-11-14T163110Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] -interventions =['urine_dipstick','bp_measurement', 'iron_folic_acid', 'calcium_supplement', 'hb_test', - 'full_blood_count','blood_transfusion', 'oral_antihypertensives','iv_antihypertensives', - 'mgso4', 'abx_for_prom', 'post_abortion_care_core', 'ectopic_pregnancy_treatment', - 'birth_kit', 'sepsis_treatment', 'amtsl', 'pph_treatment_uterotonics', 'pph_treatment_mrrp', - 'pph_treatment_surgery','caesarean_section'] +interventions =['bp_measurement', 'post_abortion_care_core', 'ectopic_pregnancy_treatment'] int_analysis = ['baseline'] From 16299a21f43862a188f41ea6117b81c2c11d72ab Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:37:29 +0000 Subject: [PATCH 078/103] Include debugging option, final set-up of scenario to print data, analysis file now collects all relevant info and prints them --- .../analysis_extract_data.py | 157 ++++++++++++++++-- .../scenario_generate_chains.py | 53 +++++- src/tlo/events.py | 10 +- src/tlo/methods/hsi_event.py | 50 +++--- src/tlo/methods/rti.py | 17 +- src/tlo/simulation.py | 39 +++-- src/tlo/util.py | 1 + 7 files changed, 252 insertions(+), 75 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 2cfba5315b..6eb6408830 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -8,10 +8,14 @@ from typing import Tuple import pandas as pd +import matplotlib.pyplot as plt from tlo import Date from tlo.analysis.utils import extract_results from datetime import datetime +from collections import Counter +import ast + # Range of years considered min_year = 2010 @@ -28,17 +32,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No """ pd.set_option('display.max_rows', None) pd.set_option('display.max_colwidth', None) - event_chains = extract_results( - results_folder, - module='tlo.simulation', - key='event_chains', - column='0', - #column = str(i), - #custom_generate_series=get_num_dalys_by_year, - do_scaling=False - ) - # print(event_chains.loc[0,(0, 0)]) - + eval_env = { 'datetime': datetime, # Add the datetime class to the eval environment 'pd': pd, # Add pandas to handle Timestamp @@ -46,13 +40,144 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No 'NaT': pd.NaT, 'nan': float('nan'), # Include NaN for eval (can also use pd.NA if preferred) } + + initial_properties_of_interest = ['rt_inj_severity','rt_MAIS_military_score','rt_ISS_score','rt_disability','rt_polytrauma','rt_injury_1','rt_injury_2','rt_injury_3','rt_injury_4','rt_injury_5','rt_injury_6', 'rt_imm_death','sy_injury','sex','li_urban', 'li_wealth', 'li_ex_alc', 'li_exposed_to_campaign_alcohol_reduction', 'li_mar_stat', 'li_in_ed', 'li_ed_lev'] + + # Will be added through computation: age at time of RTI + + # Will be added through computation: total duration of event + + initial_rt_event_properties = set() + + num_individuals = 1000 + num_runs = 50 + record = [] + + + for i in range(0,num_individuals): - for item,row in event_chains.iterrows(): - value = event_chains.loc[item,(0, 0)] - if value !='': - print('') - print(value) + individual_event_chains = extract_results( + results_folder, + module='tlo.simulation', + key='event_chains', + column=str(i), + do_scaling=False + ) + + #print(individual_event_chains) + + + for r in range(0,num_runs): + + print("AT RUN = ", r) + + initial_properties = {} + progression_properties = {} + key_first_event = {} + key_last_event = {} + first_event = {} + last_event = {} + properties = {} + + + #ind_Counter = Counter() + ind_Counter = {'0': Counter(), '1a': Counter(), '1b' : Counter(), '2' : Counter()} + # Count total appts + + list_for_individual = [] + for item,row in individual_event_chains.iterrows(): + value = individual_event_chains.loc[item,(0, r)] + # print("The value is", value, "at run ", r) + if value !='' and isinstance(value, str): + evaluated = eval(value, eval_env) + list_for_individual.append(evaluated) + # elif not isinstance(value,str): + # print(value) + + initial_properties = list_for_individual[0] + print(initial_properties) + + # Initialise first event by gathering parameters of interest from initial_properties + first_event = {key: initial_properties[key] for key in initial_properties_of_interest if key in initial_properties} + + progression_properties = {} + for i in list_for_individual: + if 'event' in i: + print("") + print(i) + if 'RTIPolling' in i['event']: + #print("I'm in polling event") + #print(i) + + # Keep track of which properties are changed during polling events + for key,value in i.items(): + if 'rt_' in key: + initial_rt_event_properties.add(key) + + # Retain a copy of Polling event + polling_event = i.copy() + + # Update parameters of interest following RTI + key_first_event = {key: i[key] if key in i else value for key, value in first_event.items()} + + # Calculate age of individual at time of event + key_first_event['age_in_days_at_event'] = (i['rt_date_inj'] - initial_properties['date_of_birth']).days + + # Keep track of evolution in individual's properties + progression_properties = initial_properties.copy() + progression_properties.update(i) + + else: + # Progress properties of individual, even if this event is a death + progression_properties.update(i) + + #print(progression_properties) + # Update footprint + if 'appt_footprint' in i and i['appt_footprint'] != 'Counter()': + footprint = i['appt_footprint'] + if 'Counter' in footprint: + footprint = footprint[len("Counter("):-1] + apply = eval(footprint, eval_env) + ind_Counter[i['level']].update(Counter(apply)) + + if 'is_alive' in i and i['is_alive'] is False: + print("Death", i) + print("-------Total footprint", ind_Counter) + break + + + # Compute final properties of individual + key_last_event['is_alive_after_RTI'] = progression_properties['is_alive'] + key_last_event['duration_days'] = (progression_properties['event_date'] - polling_event['rt_date_inj']).days + key_last_event['rt_disability_final'] = progression_properties['rt_disability'] + key_last_event.update({'total_footprint': ind_Counter}) + + #print("-------Total footprint", ind_Counter) + #for key, value in key_first_event.items(): + # if 'rt_' in key or 'alive' in key: + # print(f"{key}: {value}") + #print(#) + #for key, value in key_last_event.items(): + #if 'rt_' in key or 'alive' in key or 'event_date' in key or 'footprint' in key: + # print(f"{key}: {value}") + + #print(key_first_event) + #print(key_last_event) + print(initial_rt_event_properties) + properties = key_first_event | key_last_event + record.append(properties) + for key, value in properties.items(): + #if 'rt_' in key or 'alive' in key or 'event_date' in key or 'footprint' in key: + print(f"{key}: {value}") + + df = pd.DataFrame(record) + df.to_csv("raw_data.csv", index=False) + + print(df) + print(initial_rt_event_properties) exit(-1) + #print(i) + #dict = {} #for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]: # dict[i] = [] diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 6bdcd02d90..79df3f55b6 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -22,18 +22,42 @@ from tlo.methods.fullmodel import fullmodel from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher from tlo.scenario import BaseScenario - +from tlo.methods import ( + alri, + cardio_metabolic_disorders, + care_of_women_during_pregnancy, + contraception, + demography, + depression, + diarrhoea, + enhanced_lifestyle, + epi, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + rti, + labour, + malaria, + newborn_outcomes, + postnatal_supervisor, + pregnancy_supervisor, + stunting, + symptommanager, + tb, + wasting, +) class GenerateDataChains(BaseScenario): def __init__(self): super().__init__() self.seed = 0 self.start_date = Date(2010, 1, 1) - self.end_date = self.start_date + pd.DateOffset(months=1) - self.pop_size = 120 + self.end_date = self.start_date + pd.DateOffset(months=13) + self.pop_size = 1000 self._scenarios = self._get_scenarios() self.number_of_draws = len(self._scenarios) - self.runs_per_draw = 1 + self.runs_per_draw = 50 self.generate_event_chains = True def log_configuration(self): @@ -51,10 +75,23 @@ def log_configuration(self): } def modules(self): - return ( - fullmodel(resourcefilepath=self.resources) - + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] - ) + # MODIFY + # Here instead of running full module + return [demography.Demography(resourcefilepath=self.resources), + enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), + healthburden.HealthBurden(resourcefilepath=self.resources), + symptommanager.SymptomManager(resourcefilepath=self.resources, spurious_symptoms=False), + rti.RTI(resourcefilepath=self.resources), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources), + #simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=self.resources, + mode_appt_constraints=1, + cons_availability='all')] + + # return ( + # fullmodel(resourcefilepath=self.resources) + # + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + # ) def draw_parameters(self, draw_number, rng): if draw_number < self.number_of_draws: diff --git a/src/tlo/events.py b/src/tlo/events.py index 00a6fe4e7d..ba8024f621 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -11,7 +11,7 @@ import pandas as pd -FACTOR_POP_DICT = 5000 +from tlo.util import FACTOR_POP_DICT logger = logging.getLogger(__name__) @@ -132,7 +132,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) - if debug_chains: + if self.sim.debug_generate_event_chains: # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target @@ -142,6 +142,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: + # This will be a population-wide event. In order to find individuals for which this led to # a meaningful change, make a copy of the pop dataframe before the event has occurred. df_before = self.sim.population.props.copy() @@ -174,7 +175,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> chain_links[self.target] = str(link_info) # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - if debug_chains: + if self.sim.debug_generate_event_chains: # Print entire row row = self.sim.population.props.loc[[abs(self.target)]] # Use abs to avoid potentil issue with direct births row['person_ID'] = self.target @@ -194,7 +195,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> chain_links = self.compare_population_dataframe(df_before, df_after) # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - if debug_chains: + if self.sim.debug_generate_event_chains: # Or print entire rows change = df_before.compare(df_after) if not change.empty: @@ -233,7 +234,6 @@ def run(self): # Create empty logger for entire pop pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals - pop_dict.update(chain_links) # Log chain_links here diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index d657e9d3a0..bdf597fba4 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -8,10 +8,9 @@ from tlo import Date, logging from tlo.events import Event from tlo.population import Population - +from tlo.util import FACTOR_POP_DICT import pandas as pd -FACTOR_POP_DICT = 5000 if TYPE_CHECKING: @@ -219,19 +218,21 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.sim.population.props.loc[[abs(self.target)]] - row['person_ID'] = self.target - row['event'] = str(self) - row['event_date'] = self.sim.date - row['when'] = 'Before' - try: - row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) - row['level'] = self.facility_info.level - except: - row['appt_footprint'] = 'N/A' - row['level'] = 'N/A' - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + if self.sim.debug_generate_event_chains: + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. + row = self.sim.population.props.loc[[abs(self.target)]] + row['person_ID'] = self.target + row['event'] = str(self) + row['event_date'] = self.sim.date + row['when'] = 'Before' + + try: + row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) + row['level'] = self.facility_info.level + except: + row['appt_footprint'] = 'N/A' + row['level'] = 'N/A' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: # Once this has been removed from Chronic Syndrome mock module, make this a Runtime Error @@ -280,15 +281,16 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> chain_links = {self.target : str(link_info)} - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.sim.population.props.loc[[abs(self.target)]] - row['person_ID'] = self.target - row['event'] = str(self) - row['event_date'] = self.sim.date - row['when'] = 'After' - row['appt_footprint'] = record_footprint - row['level'] = record_level - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + if self.sim.debug_generate_event_chains: + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. + row = self.sim.population.props.loc[[abs(self.target)]] + row['person_ID'] = self.target + row['event'] = str(self) + row['event_date'] = self.sim.date + row['when'] = 'After' + row['appt_footprint'] = record_footprint + row['level'] = record_level + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) return chain_links diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 3642365976..1ca2749af7 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -2776,7 +2776,7 @@ class RTIPollingEvent(RegularEvent, PopulationScopeEventMixin): def __init__(self, module): """Schedule to take place every month """ - super().__init__(module, frequency=DateOffset(months=1000)) + super().__init__(module, frequency=DateOffset(months=1000)) # Single polling event p = module.parameters # Parameters which transition the model between states self.base_1m_prob_rti = (p['base_rate_injrti'] / 12) @@ -2864,10 +2864,12 @@ def apply(self, population): .when('.between(70,79)', self.rr_injrti_age7079), Predictor('li_ex_alc').when(True, self.rr_injrti_excessalcohol) ) - if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi is True: - pred = 1.0 - else: - pred = eq.predict(df.loc[rt_current_non_ind]) + #if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi is True: + pred = 1.0 + #else: + # pred = eq.predict(df.loc[rt_current_non_ind]) + + random_draw_in_rti = self.module.rng.random_sample(size=len(rt_current_non_ind)) selected_for_rti = rt_current_non_ind[pred > random_draw_in_rti] @@ -4852,6 +4854,7 @@ def __init__(self, module, person_id): self.treated_code = 'none' def apply(self, person_id, squeeze_factor): + self._number_of_times_this_event_has_run += 1 df = self.sim.population.props rng = self.module.rng @@ -4900,10 +4903,12 @@ def apply(self, person_id, squeeze_factor): # injury is being treated in this surgery # find untreated injury codes that are treated with major surgery relevant_codes = np.intersect1d(injuries_to_be_treated, surgically_treated_codes) + # check that the person sent here has an appropriate code(s) assert len(relevant_codes) > 0 # choose a code at random self.treated_code = rng.choice(relevant_codes) + if request_outcome: # check the people sent here hasn't died due to rti, have had their injuries diagnosed and been through # RTI_Med @@ -4990,7 +4995,9 @@ def apply(self, person_id, squeeze_factor): # ------------------------------------- Perm disability from amputation ------------------------------------ codes = ['782', '782a', '782b', '782c', '783', '882', '883', '884'] + if self.treated_code in codes: + # Track whether they are permanently disabled df.at[person_id, 'rt_perm_disability'] = True # Find the column and code where the permanent injury is stored diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index d9ba62c43a..bb766562a0 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -11,8 +11,9 @@ from typing import Optional from typing import TYPE_CHECKING, Optional import pandas as pd - +import tlo.population import numpy as np +from tlo.util import FACTOR_POP_DICT try: import dill @@ -40,8 +41,6 @@ logger_chains = logging.getLogger("tlo.methods.event") logger_chains.setLevel(logging.INFO) -FACTOR_POP_DICT = 5000 - class SimulationPreviouslyInitialisedError(Exception): """Exception raised when trying to initialise an already initialised simulation.""" @@ -113,12 +112,15 @@ def __init__( self.generate_event_chains_overwrite_epi = None self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] + self.debug_generate_event_chains = False self.end_date = None self.output_file = None self.population: Optional[Population] = None - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - self.event_chains: Optinoal[Population] = None + + if self.debug_generate_event_chains: + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. + self.event_chains: Optional[Population] = None self.show_progress_bar = show_progress_bar self.resourcefilepath = resourcefilepath @@ -288,8 +290,9 @@ def make_initial_population(self, *, n: int) -> None: data=f"{module.name}.initialise_population() {time.time() - start1} s", ) - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint'] + ['level']) + if self.debug_generate_event_chains: + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. + self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint'] + ['level']) # When logging events for each individual to reconstruct chains, only the changes in individual properties will be logged. # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. @@ -329,7 +332,7 @@ def initialise(self, *, end_date: Date) -> None: self.generate_event_chains_overwrite_epi = True # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] - self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'HealthSeekingBehaviourPoll', 'LifestyleEvent'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] else: # If not using to print chains, cannot ignore epi self.generate_event_chains_overwrite_epi = False @@ -418,8 +421,9 @@ def run_simulation_to(self, *, to_date: Date) -> None: self.fire_single_event(event, date) self.date = to_date - # TO BE REMOVED: this is currently only used for debugging, will be removed from final PR. - self.event_chains.to_csv('output.csv', index=False) + if self.debug_generate_event_chains: + # TO BE REMOVED: this is currently only used for debugging, will be removed from final PR. + self.event_chains.to_csv('output.csv', index=False) if self.show_progress_bar: progress_bar.stop() @@ -492,13 +496,14 @@ def do_birth(self, mother_id: int) -> int: data = pop_dict, description='Links forming chains of events for simulated individuals') - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.population.props.iloc[[child_id]] - row['person_ID'] = child_id - row['event'] = 'Birth' - row['event_date'] = self.date - row['when'] = 'After' - self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) + if self.debug_generate_event_chains: + # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. + row = self.population.props.iloc[[child_id]] + row['person_ID'] = child_id + row['event'] = 'Birth' + row['event_date'] = self.date + row['when'] = 'After' + self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) return child_id diff --git a/src/tlo/util.py b/src/tlo/util.py index 168b1d41a1..f8dc67d471 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -12,6 +12,7 @@ # Default mother_id value, assigned to individuals initialised as adults at the start of the simulation. DEFAULT_MOTHER_ID = -1e7 +FACTOR_POP_DICT = 1000 def create_age_range_lookup(min_age: int, max_age: int, range_size: int = 5) -> (list, Dict[int, str]): From 0dd862f2a9b485a33933e185e3c59ad64ed33ed9 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:28:30 +0000 Subject: [PATCH 079/103] Change label of person when iterating --- .../analysis_extract_data.py | 68 ++++++++++++------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 6eb6408830..4c8e7d8197 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -41,7 +41,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No 'nan': float('nan'), # Include NaN for eval (can also use pd.NA if preferred) } - initial_properties_of_interest = ['rt_inj_severity','rt_MAIS_military_score','rt_ISS_score','rt_disability','rt_polytrauma','rt_injury_1','rt_injury_2','rt_injury_3','rt_injury_4','rt_injury_5','rt_injury_6', 'rt_imm_death','sy_injury','sex','li_urban', 'li_wealth', 'li_ex_alc', 'li_exposed_to_campaign_alcohol_reduction', 'li_mar_stat', 'li_in_ed', 'li_ed_lev'] + initial_properties_of_interest = ['rt_MAIS_military_score','rt_ISS_score','rt_disability','rt_polytrauma','rt_injury_1','rt_injury_2','rt_injury_3','rt_injury_4','rt_injury_5','rt_injury_6', 'rt_imm_death','sy_injury','sy_severe_trauma','sex','li_urban', 'li_wealth', 'li_mar_stat', 'li_in_ed', 'li_ed_lev'] # Will be added through computation: age at time of RTI @@ -54,13 +54,15 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No record = [] - for i in range(0,num_individuals): + for p in range(0,num_individuals): + + print("At person = ", p) individual_event_chains = extract_results( results_folder, module='tlo.simulation', key='event_chains', - column=str(i), + column=str(p), do_scaling=False ) @@ -69,7 +71,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No for r in range(0,num_runs): - print("AT RUN = ", r) + initial_properties = {} progression_properties = {} @@ -78,7 +80,8 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No first_event = {} last_event = {} properties = {} - + average_disability = 0 + prev_disability_incurred = 0 #ind_Counter = Counter() ind_Counter = {'0': Counter(), '1a': Counter(), '1b' : Counter(), '2' : Counter()} @@ -95,7 +98,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No # print(value) initial_properties = list_for_individual[0] - print(initial_properties) + # print(initial_properties) # Initialise first event by gathering parameters of interest from initial_properties first_event = {key: initial_properties[key] for key in initial_properties_of_interest if key in initial_properties} @@ -103,8 +106,8 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No progression_properties = {} for i in list_for_individual: if 'event' in i: - print("") - print(i) + #print("") + #print(i) if 'RTIPolling' in i['event']: #print("I'm in polling event") #print(i) @@ -126,10 +129,26 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No # Keep track of evolution in individual's properties progression_properties = initial_properties.copy() progression_properties.update(i) + + # dalys incurred + if 'rt_disability' in i: + prev_disability_incurred = i['rt_disability'] + prev_date = i['event_date'] + #print('At polling event, ', prev_disability_incurred, prev_date) else: # Progress properties of individual, even if this event is a death progression_properties.update(i) + + # If disability has changed as a result of this, recalculate + if 'rt_disability' in i and i['rt_disability'] != prev_disability_incurred: + dt_in_prev_disability = (i['event_date'] - prev_date).days + average_disability += prev_disability_incurred*dt_in_prev_disability + # Update variables + prev_disability_incurred = i['rt_disability'] + prev_date = i['event_date'] + + #print(progression_properties) # Update footprint @@ -141,34 +160,33 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No ind_Counter[i['level']].update(Counter(apply)) if 'is_alive' in i and i['is_alive'] is False: - print("Death", i) - print("-------Total footprint", ind_Counter) + #print("Death", i) + #print("-------Total footprint", ind_Counter) break # Compute final properties of individual key_last_event['is_alive_after_RTI'] = progression_properties['is_alive'] key_last_event['duration_days'] = (progression_properties['event_date'] - polling_event['rt_date_inj']).days - key_last_event['rt_disability_final'] = progression_properties['rt_disability'] + if not key_first_event['rt_imm_death'] and key_last_event['duration_days']> 0.0: + key_last_event['rt_disability_average'] = average_disability/key_last_event['duration_days'] + else: + key_last_event['rt_disability_average'] = 0.0 + key_last_event['rt_disability_permanent'] = progression_properties['rt_disability'] key_last_event.update({'total_footprint': ind_Counter}) - - #print("-------Total footprint", ind_Counter) - #for key, value in key_first_event.items(): - # if 'rt_' in key or 'alive' in key: - # print(f"{key}: {value}") - #print(#) - #for key, value in key_last_event.items(): - #if 'rt_' in key or 'alive' in key or 'event_date' in key or 'footprint' in key: - # print(f"{key}: {value}") - #print(key_first_event) - #print(key_last_event) - print(initial_rt_event_properties) + #print("Average disability", key_last_event['rt_disability_average']) + properties = key_first_event | key_last_event + + if not key_first_event['rt_imm_death'] and ((properties['rt_disability_average']-properties['rt_disability'])/properties['rt_disability'] > 1e-4): + print("Error in computed average for individual ", p, r ) + record.append(properties) - for key, value in properties.items(): + #for key, value in properties.items(): #if 'rt_' in key or 'alive' in key or 'event_date' in key or 'footprint' in key: - print(f"{key}: {value}") + #print(f"{key}: {value}") + # print("Initial event properties", initial_rt_event_properties) df = pd.DataFrame(record) df.to_csv("raw_data.csv", index=False) From 8fa97bedb927376d92c22461c025095cad43e448 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 26 Nov 2024 15:43:07 +0000 Subject: [PATCH 080/103] final updates --- .../cohort_analysis/int_analysis_script.py | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index 69a4150990..422aa01487 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -5,12 +5,13 @@ import matplotlib.pyplot as plt import numpy as np -from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize, create_pickles_locally +from tlo.analysis.utils import extract_results, get_scenario_outputs, summarize, create_pickles_locally, get_scenario_info outputspath = './outputs/sejjj49@ucl.ac.uk/' scenario = 'block_intervention_big_pop_test-2024-11-14T163110Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] +create_pickles_locally(results_folder, compressed_file_name_prefix='block_intervention_big_pop_test') interventions =['bp_measurement', 'post_abortion_care_core', 'ectopic_pregnancy_treatment'] @@ -51,8 +52,20 @@ def sort_df(_df): lambda df: df.loc[(df['label'] == 'Maternal Disorders')].assign( year=df['date'].dt.year).groupby(['year'])['year'].count()), do_scaling=False) -dd_sum = summarize(direct_deaths) +br = extract_results( + results_folder, + module="tlo.methods.demography", + key="on_birth", + custom_generate_series=( + lambda df: df.assign( + year=df['date'].dt.year).groupby(['year'])['year'].count()), + do_scaling=False + ) + +dd_sum = summarize(direct_deaths) +dd_mmr = (direct_deaths/br) * 100_000 +dd_mr_sum = summarize(dd_mmr) all_dalys_dfs = extract_results( results_folder, @@ -82,11 +95,17 @@ def get_data(df, key, draw): mmr_by_scnario = {k: get_data(results['deaths_and_stillbirths']['summarised'], 'direct_mmr', d) for k, d in zip ( int_analysis, draws)} +mmr_by_scenario_oth_log = {k: get_data(dd_mr_sum, 2024, d) for k, d in zip ( + int_analysis, draws)} + def barcharts(data, y_label, title): # Extract means and errors labels = data.keys() means = [vals[1] for vals in data.values()] + # lower_errors = [vals[0] for vals in data.values()] + # upper_errors = [vals[2] for vals in data.values()] + lower_errors = [vals[1] - vals[0] for vals in data.values()] upper_errors = [vals[2] - vals[1] for vals in data.values()] errors = [lower_errors, upper_errors] @@ -102,7 +121,8 @@ def barcharts(data, y_label, title): plt.tight_layout() plt.show() -barcharts(dalys_by_scenario, 'DALYs', 'Total Neonatal Disorders DALYs by scenario') +barcharts(dalys_by_scenario, 'DALYs', 'Total Maternal Disorders DALYs by scenario') +barcharts(mmr_by_scnario, 'MMR', 'Total MMR by scenario') barcharts(mmr_by_scnario, 'MMR', 'Total MMR by scenario') # Difference results @@ -121,10 +141,10 @@ def get_diffs(df_key, result_key, ints, draws): return diff_results diff_results = {} -baseline = direct_deaths[0] +baseline = dd_mmr[0] for draw, int in zip(draws, int_analysis): - diff_df = direct_deaths[draw] - baseline + diff_df = dd_mmr[draw] - baseline diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], names=['draw', 'run']) results_diff = summarize(diff_df) @@ -147,6 +167,8 @@ def get_diff_plots(data, outcome): errors = [(mean - min_val, max_val - mean) for mean, min_val, max_val in zip(means, mins, maxs)] errors = np.array(errors).T + # todo: the error bars are slightly off... + # Plotting plt.figure(figsize=(12, 6)) plt.errorbar(categories, means, yerr=errors, fmt='o', capsize=5) @@ -161,7 +183,7 @@ def get_diff_plots(data, outcome): get_diff_plots(mmr_diffs, 'MMR') get_diff_plots(mat_deaths, 'Maternal Deaths (crude)') -get_diff_plots(mat_deaths_2, 'Maternal Deaths (demog log)') +get_diff_plots(mat_deaths_2, 'MMR (demog log)') get_diff_plots(dalys_diffs, 'Maternal DALYs') From e7747317a2f674e0085c77a2427cf4dcef450d80 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Tue, 26 Nov 2024 16:07:50 +0000 Subject: [PATCH 081/103] final updates --- resources/ResourceFile_PregnancySupervisor.xlsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index 9ce97f66c5..04d8cbaf91 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c8b23b40eb24cb5308f320b5b920d2a31e771b675384bb273ea964d3b311cb8f -size 24202 +oid sha256:a7fa451bf9736eacfcb301044bb6beb2537799a68f86fcc2de3f918e9dc03ff3 +size 24198 From d4d3a3373d8fdeda307bc230a131a088bdb59a02 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 27 Nov 2024 08:55:05 +0000 Subject: [PATCH 082/103] fixes --- resources/ResourceFile_PregnancySupervisor.xlsx | 4 ++-- tests/test_mnh_cohort.py | 13 +++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index 04d8cbaf91..721d556702 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a7fa451bf9736eacfcb301044bb6beb2537799a68f86fcc2de3f918e9dc03ff3 -size 24198 +oid sha256:5c6354f6d82204b253ab9d2ef51af48c6d811915dfc486a71450c34b89427a51 +size 24200 diff --git a/tests/test_mnh_cohort.py b/tests/test_mnh_cohort.py index 303fe933cd..7bfcbed194 100644 --- a/tests/test_mnh_cohort.py +++ b/tests/test_mnh_cohort.py @@ -27,11 +27,11 @@ def register_modules(sim): mnh_cohort_module.MaternalNewbornHealthCohort(resourcefilepath=resourcefilepath)) def test_run_sim_with_mnh_cohort(tmpdir, seed): - sim = Simulation(start_date=start_date, seed=796967, log_config={"filename": "log", "custom_levels":{ + sim = Simulation(start_date=start_date, seed=12345, log_config={"filename": "log", "custom_levels":{ "*": logging.DEBUG},"directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=200) + sim.make_initial_population(n=3000) sim.simulate(end_date=Date(2025, 1, 2)) output= parse_log_file(sim.log_filepath) @@ -69,7 +69,7 @@ def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) register_modules(sim) - sim.make_initial_population(n=1000) + sim.make_initial_population(n=3000) sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) df = sim.population.props @@ -83,10 +83,3 @@ def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): # orig = sim.population.new_row # assert (df.dtypes == orig.dtypes).all() -# def test_mnh_cohort_module_updates_properties_as_expected(tmpdir, seed): -# sim = Simulation(start_date=start_date, seed=seed, log_config={"filename": "log", "directory": tmpdir}) -# -# register_modules(sim) -# sim.make_initial_population(n=1000) -# sim.simulate(end_date=sim.date + pd.DateOffset(days=0)) -# # to do: check properties!! From d833aee1438f14e92675650a6b142cc353006d28 Mon Sep 17 00:00:00 2001 From: joehcollins Date: Wed, 27 Nov 2024 10:59:39 +0000 Subject: [PATCH 083/103] fixes --- src/tlo/methods/pregnancy_helper_functions.py | 274 +++++++++--------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index dc871ff63a..4783d28199 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -98,7 +98,7 @@ def check_int_deliverable(self, int_name, hsi_event, individual_id = hsi_event.target p_params = self.sim.modules['PregnancySupervisor'].current_parameters - assert int_name in p_params['all_interventions'] + # assert int_name in p_params['all_interventions'] # Firstly, we determine if an analysis is currently being conducted during which the probability of intervention # delivery is being overridden @@ -152,7 +152,7 @@ def check_int_deliverable(self, int_name, hsi_event, consumables = True if cons is None and opt_cons is not None: - hsi_event.get_consumables(item_codes= [], optional_item_codes=opt_cons) + hsi_event.get_consumables(item_codes=[], optional_item_codes=opt_cons) if ((dx_test is None) or (self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run=dx_test, hsi_event=hsi_event))): @@ -165,141 +165,141 @@ def check_int_deliverable(self, int_name, hsi_event, return False -def return_cons_avail(self, hsi_event, cons, opt_cons): - """ - This function is called by majority of interventions across maternal and neonatal modules to return whether a - consumable or package of consumables are available. If analysis is not being conducted (as indicated by a series of - analysis boolean parameters) then the availability is determined via the health system module. Otherwise a - fixed probability of availability is used to force a set level of intervention coverage - availability - :param self: module - :param hsi_event: hsi_event calling the intervention - :param cons_dict: dictionary containing the consumables for that module - :param info: information on core, optional and number of consumables requested - :return: BOOL - """ - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - ps_params = self.sim.modules['PregnancySupervisor'].current_parameters - la_params = self.sim.modules['Labour'].current_parameters - - if opt_cons is None: - opt_cons = [] - - # Check if analysis is currently running, if not then availability is determined normally - if not ps_params['ps_analysis_in_progress'] and not la_params['la_analysis_in_progress']: - available = hsi_event.get_consumables(item_codes=cons, - optional_item_codes=opt_cons) - - if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): - mni[hsi_event.target]['cons_not_avail'] = True - - return available - - else: - available = hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons) - - # Depending on HSI calling this function a different parameter set is used to determine if analysis is being - # conducted - if 'AntenatalCare' in hsi_event.TREATMENT_ID: - params = self.sim.modules['PregnancySupervisor'].current_parameters - else: - params = self.sim.modules['Labour'].current_parameters - - # Store the names of the parameters which indicate that analysis is being conducted against specific HSIs - analysis_dict = {'AntenatalCare_Outpatient': ['alternative_anc_quality', 'anc_availability_probability'], - 'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], - 'AntenatalCare_FollowUp': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], - 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_cons_availability'], - 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_cons_availability'], - 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_cons_availability'], - 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], - 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], - 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} - - # Cycle through each HSI of interest. If this HSI is requesting a consumable, analysis is being conducted and - # the simulation date is greater than the predetermined analysis date, then a random draw against a probability - # of consumable availability is used - for k in analysis_dict: - if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: - if self.rng.random_sample() < params[analysis_dict[k][1]]: - available = True - else: - available = False - - # If the consumable is not available this is stored in the MNI dictionary - if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): - mni[hsi_event.target]['cons_not_avail'] = True - - return available - - -def check_emonc_signal_function_will_run(self, sf, hsi_event): - """ - Called during/after labour to determine if a B/CEmONC function will run depending on the availability of HCWs - trained in the intervention and their competence - :param self: module - :param sf: signal function of interest - :param hsi_event: hsi_event calling the intervention - :return: BOOL - """ - la_params = self.sim.modules['Labour'].current_parameters - ps_params = self.sim.modules['PregnancySupervisor'].current_parameters - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - - def see_if_sf_will_run(): - # Determine competence parameter to use (higher presumed competence in hospitals) - if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a': - competence = la_params['mean_hcw_competence_hc'][0] - else: - competence = la_params['mean_hcw_competence_hp'][1] - - comp_result = self.rng.random_sample() < competence - hcw_result = self.rng.random_sample() < la_params[f'prob_hcw_avail_{sf}'] - - # If HCW is available and staff are competent at delivering the intervention then it will be delivered - if comp_result and hcw_result: - return True - - # Otherwise log within the mni - if not comp_result and (hsi_event.target in mni): - mni[hsi_event.target]['comp_not_avail'] = True - if not hcw_result and (hsi_event.target in mni): - mni[hsi_event.target]['hcw_not_avail'] = True - - return False - - if not la_params['la_analysis_in_progress'] and not ps_params['ps_analysis_in_progress']: - return see_if_sf_will_run() - - else: - if 'AntenatalCare' in hsi_event.TREATMENT_ID: - params = self.sim.modules['PregnancySupervisor'].current_parameters - else: - params = self.sim.modules['Labour'].current_parameters - - # Define HSIs and analysis parameters of interest - analysis_dict = {'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], - 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_availability'], - 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_availability'], - 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_availability'], - 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], - 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], - 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} - - for k in analysis_dict: - # If analysis is running, the analysis date has passed and an appropriate HSI has called this function then - # probability of intervention delivery is determined by an analysis parameter - if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: - if self.rng.random_sample() < params[analysis_dict[k][1]]: - return True - - # If the event wont run it is stored in the HSI for the relevant woman - elif hsi_event.target in mni: - barrier = self.rng.choice(['comp_not_avail', 'hcw_not_avail']) - mni[hsi_event.target][barrier] = True - return False - - return see_if_sf_will_run() +# def return_cons_avail(self, hsi_event, cons, opt_cons): +# """ +# This function is called by majority of interventions across maternal and neonatal modules to return whether a +# consumable or package of consumables are available. If analysis is not being conducted (as indicated by a series of +# analysis boolean parameters) then the availability is determined via the health system module. Otherwise a +# fixed probability of availability is used to force a set level of intervention coverage +# availability +# :param self: module +# :param hsi_event: hsi_event calling the intervention +# :param cons_dict: dictionary containing the consumables for that module +# :param info: information on core, optional and number of consumables requested +# :return: BOOL +# """ +# mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info +# ps_params = self.sim.modules['PregnancySupervisor'].current_parameters +# la_params = self.sim.modules['Labour'].current_parameters +# +# if opt_cons is None: +# opt_cons = [] +# +# # Check if analysis is currently running, if not then availability is determined normally +# if not ps_params['ps_analysis_in_progress'] and not la_params['la_analysis_in_progress']: +# available = hsi_event.get_consumables(item_codes=cons, +# optional_item_codes=opt_cons) +# +# if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): +# mni[hsi_event.target]['cons_not_avail'] = True +# +# return available +# +# else: +# available = hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons) +# +# # Depending on HSI calling this function a different parameter set is used to determine if analysis is being +# # conducted +# if 'AntenatalCare' in hsi_event.TREATMENT_ID: +# params = self.sim.modules['PregnancySupervisor'].current_parameters +# else: +# params = self.sim.modules['Labour'].current_parameters +# +# # Store the names of the parameters which indicate that analysis is being conducted against specific HSIs +# analysis_dict = {'AntenatalCare_Outpatient': ['alternative_anc_quality', 'anc_availability_probability'], +# 'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], +# 'AntenatalCare_FollowUp': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], +# 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_cons_availability'], +# 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_cons_availability'], +# 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_cons_availability'], +# 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], +# 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], +# 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} +# +# # Cycle through each HSI of interest. If this HSI is requesting a consumable, analysis is being conducted and +# # the simulation date is greater than the predetermined analysis date, then a random draw against a probability +# # of consumable availability is used +# for k in analysis_dict: +# if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: +# if self.rng.random_sample() < params[analysis_dict[k][1]]: +# available = True +# else: +# available = False +# +# # If the consumable is not available this is stored in the MNI dictionary +# if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): +# mni[hsi_event.target]['cons_not_avail'] = True +# +# return available + + +# def check_emonc_signal_function_will_run(self, sf, hsi_event): +# """ +# Called during/after labour to determine if a B/CEmONC function will run depending on the availability of HCWs +# trained in the intervention and their competence +# :param self: module +# :param sf: signal function of interest +# :param hsi_event: hsi_event calling the intervention +# :return: BOOL +# """ +# la_params = self.sim.modules['Labour'].current_parameters +# ps_params = self.sim.modules['PregnancySupervisor'].current_parameters +# mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info +# +# def see_if_sf_will_run(): +# # Determine competence parameter to use (higher presumed competence in hospitals) +# if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a': +# competence = la_params['mean_hcw_competence_hc'][0] +# else: +# competence = la_params['mean_hcw_competence_hp'][1] +# +# comp_result = self.rng.random_sample() < competence +# hcw_result = self.rng.random_sample() < la_params[f'prob_hcw_avail_{sf}'] +# +# # If HCW is available and staff are competent at delivering the intervention then it will be delivered +# if comp_result and hcw_result: +# return True +# +# # Otherwise log within the mni +# if not comp_result and (hsi_event.target in mni): +# mni[hsi_event.target]['comp_not_avail'] = True +# if not hcw_result and (hsi_event.target in mni): +# mni[hsi_event.target]['hcw_not_avail'] = True +# +# return False +# +# if not la_params['la_analysis_in_progress'] and not ps_params['ps_analysis_in_progress']: +# return see_if_sf_will_run() +# +# else: +# if 'AntenatalCare' in hsi_event.TREATMENT_ID: +# params = self.sim.modules['PregnancySupervisor'].current_parameters +# else: +# params = self.sim.modules['Labour'].current_parameters +# +# # Define HSIs and analysis parameters of interest +# analysis_dict = {'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], +# 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_availability'], +# 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_availability'], +# 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_availability'], +# 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], +# 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], +# 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} +# +# for k in analysis_dict: +# # If analysis is running, the analysis date has passed and an appropriate HSI has called this function then +# # probability of intervention delivery is determined by an analysis parameter +# if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: +# if self.rng.random_sample() < params[analysis_dict[k][1]]: +# return True +# +# # If the event wont run it is stored in the HSI for the relevant woman +# elif hsi_event.target in mni: +# barrier = self.rng.choice(['comp_not_avail', 'hcw_not_avail']) +# mni[hsi_event.target][barrier] = True +# return False +# +# return see_if_sf_will_run() def log_met_need(self, intervention, hsi): From 17388df9777d2a8a33c1711445f5b76091b460ae Mon Sep 17 00:00:00 2001 From: joehcollins Date: Mon, 2 Dec 2024 13:04:43 +0000 Subject: [PATCH 084/103] fixes --- .../ResourceFile_PregnancySupervisor.xlsx | 4 +- .../cohort_analysis/int_analysis_script.py | 89 +++++- .../methods/care_of_women_during_pregnancy.py | 280 +----------------- src/tlo/methods/labour.py | 247 +-------------- src/tlo/methods/newborn_outcomes.py | 30 +- src/tlo/methods/pregnancy_helper_functions.py | 231 +++------------ src/tlo/methods/pregnancy_supervisor.py | 22 -- ...al_health_helper_and_analysis_functions.py | 23 +- 8 files changed, 143 insertions(+), 783 deletions(-) diff --git a/resources/ResourceFile_PregnancySupervisor.xlsx b/resources/ResourceFile_PregnancySupervisor.xlsx index 721d556702..c2d293b239 100644 --- a/resources/ResourceFile_PregnancySupervisor.xlsx +++ b/resources/ResourceFile_PregnancySupervisor.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5c6354f6d82204b253ab9d2ef51af48c6d811915dfc486a71450c34b89427a51 -size 24200 +oid sha256:b3e5c31d51a35b370a0317866af3b955172dfc4780ef44d053054c772ac83d0a +size 22625 diff --git a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py index 422aa01487..a0d2f1570d 100644 --- a/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py +++ b/src/scripts/maternal_perinatal_analyses/cohort_analysis/int_analysis_script.py @@ -1,4 +1,6 @@ import os +import scipy.stats as st +from scipy.stats import t, norm, shapiro import pandas as pd @@ -9,9 +11,9 @@ outputspath = './outputs/sejjj49@ucl.ac.uk/' -scenario = 'block_intervention_big_pop_test-2024-11-14T163110Z' +scenario = 'block_intervention_big_pop_test-2024-11-27T110117Z' results_folder= get_scenario_outputs(scenario, outputspath)[-1] -create_pickles_locally(results_folder, compressed_file_name_prefix='block_intervention_big_pop_test') +# create_pickles_locally(results_folder, compressed_file_name_prefix='block_intervention_big_pop_test') interventions =['bp_measurement', 'post_abortion_care_core', 'ectopic_pregnancy_treatment'] @@ -23,6 +25,38 @@ draws = [x for x in range(len(int_analysis))] +def summarize_confidence_intervals(results: pd.DataFrame) -> pd.DataFrame: + """Utility function to compute summary statistics + + Finds mean value and 95% interval across the runs for each draw. + """ + + # Calculate summary statistics + grouped = results.groupby(axis=1, by='draw', sort=False) + mean = grouped.mean() + sem = grouped.sem() # Standard error of the mean + + # Calculate the critical value for a 95% confidence level + n = grouped.size().max() # Assuming the largest group size determines the degrees of freedom + critical_value = t.ppf(0.975, df=n - 1) # Two-tailed critical value + + # Compute the margin of error + margin_of_error = critical_value * sem + + # Compute confidence intervals + lower = mean - margin_of_error + upper = mean + margin_of_error + + # Combine into a single DataFrame + summary = pd.concat({'mean': mean, 'lower': lower, 'upper': upper}, axis=1) + + # Format the DataFrame as in the original code + summary.columns = summary.columns.swaplevel(1, 0) + summary.columns.names = ['draw', 'stat'] + summary = summary.sort_index(axis=1) + + return summary + # Access dataframes generated from pregnancy supervisor def get_ps_data_frames(key, results_folder): def sort_df(_df): @@ -36,7 +70,7 @@ def sort_df(_df): custom_generate_series=sort_df, do_scaling=False ) - results_df_summ = summarize(results_df) + results_df_summ = summarize_confidence_intervals(results_df) return {'crude':results_df, 'summarised':results_df_summ} @@ -63,9 +97,9 @@ def sort_df(_df): do_scaling=False ) -dd_sum = summarize(direct_deaths) +dd_sum = summarize_confidence_intervals(direct_deaths) dd_mmr = (direct_deaths/br) * 100_000 -dd_mr_sum = summarize(dd_mmr) +dd_mr_sum = summarize_confidence_intervals(dd_mmr) all_dalys_dfs = extract_results( results_folder, @@ -79,7 +113,7 @@ def sort_df(_df): mat_disorders_all = all_dalys_dfs.loc[(slice(None), 'Maternal Disorders'), :] mat_dalys_df = mat_disorders_all.loc[2024] -mat_dalys_df_sum = summarize(mat_dalys_df) +mat_dalys_df_sum = summarize_confidence_intervals(mat_dalys_df) results.update({'dalys':{'crude': mat_dalys_df, 'summarised': mat_dalys_df_sum}}) @@ -134,11 +168,11 @@ def get_diffs(df_key, result_key, ints, draws): diff_df = results[df_key]['crude'][draw] - baseline diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], names=['draw', 'run']) - results_diff = summarize(diff_df) + results_diff = summarize_confidence_intervals(diff_df) results_diff.fillna(0) diff_results.update({int: results_diff.loc[result_key].values}) - return diff_results + return [diff_results, diff_df] diff_results = {} baseline = dd_mmr[0] @@ -147,14 +181,14 @@ def get_diffs(df_key, result_key, ints, draws): diff_df = dd_mmr[draw] - baseline diff_df.columns = pd.MultiIndex.from_tuples([(draw, v) for v in range(len(diff_df.columns))], names=['draw', 'run']) - results_diff = summarize(diff_df) + results_diff = summarize_confidence_intervals(diff_df) results_diff.fillna(0) diff_results.update({int: results_diff.loc[2024].values}) -mat_deaths = get_diffs('deaths_and_stillbirths', 'direct_maternal_deaths', int_analysis, draws) -mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws) -dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws) +mat_deaths = get_diffs('deaths_and_stillbirths', 'direct_maternal_deaths', int_analysis, draws)[0] +mmr_diffs = get_diffs('deaths_and_stillbirths', 'direct_mmr', int_analysis, draws)[0] +dalys_diffs = get_diffs('dalys', 'Maternal Disorders', int_analysis, draws)[0] mat_deaths_2 = diff_results def get_diff_plots(data, outcome): @@ -187,4 +221,35 @@ def get_diff_plots(data, outcome): get_diff_plots(dalys_diffs, 'Maternal DALYs') +# NORMALITY OF MMR ESTIMATES ACROSS RUNS (NOT DIFFERENCES) +for draw in draws: + data = results['deaths_and_stillbirths']['crude'].loc['direct_mmr', draw].values + + # Importing Shapiro-Wilk test for normality + # Conducting Shapiro-Wilk test + stat, p_value = shapiro(data) + # Plotting histogram + plt.hist(data, bins=15, density=True, alpha=0.6, color='skyblue', edgecolor='black') + + # Overlay normal distribution (optional) + mean, std = np.mean(data), np.std(data) + xmin, xmax = plt.xlim() + x = np.linspace(xmin, xmax, 100) + p = norm.pdf(x, mean, std) + plt.axvline(mean, color='green', linestyle='-', linewidth=2, label='Data Mean') + plt.plot(x, p, 'r--', linewidth=2, label='Normal Curve') + + # Adding labels and legend + plt.title(f'MMR data Histogram with Normality Test (p-value = {p_value:.4f}) (Draw {draw})') + plt.xlabel('Value') + plt.ylabel('Density') + plt.legend() + # Show plot + plt.show() + # Printing Shapiro-Wilk test results + print(f"Shapiro-Wilk Test Statistic: {stat:.4f}, p-value: {p_value:.4f}") + if p_value > 0.05: + print("Result: Data likely follows a normal distribution (p > 0.05).") + else: + print("Result: Data likely does not follow a normal distribution (p ≤ 0.05).") diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 94f01ddedf..61e2446594 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -739,52 +739,18 @@ def screening_interventions_delivered_at_every_contact(self, hsi_event): self, int_name='bp_measurement', hsi_event=hsi_event, q_param=[params['prob_intervention_delivered_bp']], - cons=None, equipment={'Sphygmomanometer'}, dx_test='blood_pressure_measurement') if hypertension_diagnosed and not df.at[person_id, 'ac_gest_htn_on_treatment'] and \ (df.at[person_id, 'ps_htn_disorders'] != 'none') and pd.isnull(mni[person_id]['hypertension_onset']): + # We store date of onset to calculate dalys- only women who are aware of diagnosis experience DALYs # (see daly weight for hypertension) pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'hypertension_onset', self.sim.date) - # # Delivery of the intervention is conditioned on a random draw against a probability that the intervention - # # would be delivered (used to calibrate to SPA data - acts as proxy for clinical quality) - # if self.rng.random_sample() < params['prob_intervention_delivered_urine_ds']: - # - # # check consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, cons=self.item_codes_preg_consumables['urine_dipstick'], opt_cons=None) - # - # # If the intervention will be delivered the dx_manager runs, returning True if the consumables are - # # available and the test detects protein in the urine - # if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - # dx_tests_to_run='urine_dipstick_protein', hsi_event=hsi_event): - # - # # We use a temporary variable to store if proteinuria is detected - # proteinuria_diagnosed = True - # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'dipstick'}) - # - # # The process is repeated for blood pressure monitoring - # if self.rng.random_sample() < params['prob_intervention_delivered_bp']: - # hsi_event.add_equipment({'Sphygmomanometer'}) - # - # if self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='blood_pressure_measurement', - # hsi_event=hsi_event): - # hypertension_diagnosed = True - # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'bp_measurement'}) - # - # if not df.at[person_id, 'ac_gest_htn_on_treatment'] and\ - # (df.at[person_id, 'ps_htn_disorders'] != 'none') and pd.isnull(mni[person_id]['hypertension' - # '_onset']): - # - # # We store date of onset to calculate dalys- only women who are aware of diagnosis experience DALYs - # # (see daly weight for hypertension) - # pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'hypertension_onset', self.sim.date) - # - # # If either high blood pressure or proteinuria are detected (or both) we assume this woman needs to be admitted - # # for further treatment following this ANC contact + # If either high blood pressure or proteinuria are detected (or both) we assume this woman needs to be admitted + # for further treatment following this ANC contact # Only women who are not on treatment OR are determined to have severe disease whilst on treatment are admitted if hypertension_diagnosed or proteinuria_diagnosed: @@ -819,14 +785,8 @@ def iron_and_folic_acid_supplementation(self, hsi_event): iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='iron_folic_acid', hsi_event=hsi_event, - q_param=None, cons=updated_cons) - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, cons=updated_cons, opt_cons=None) - # - # if avail: - if iron_folic_acid_delivered: logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'iron_folic_acid'}) @@ -862,13 +822,7 @@ def balance_energy_and_protein_supplementation(self, hsi_event): self.item_codes_preg_consumables['balanced_energy_protein'].items()} bep_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='protein_supplement', hsi_event=hsi_event, q_param=None, cons=updated_cons) - - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, cons=updated_cons, opt_cons=None) - # - # # And she is deemed to be at risk (i.e. BMI < 18) she is started on supplements - # if avail and (df.at[person_id, 'li_bmi'] == 1): + self, int_name='protein_supplement', hsi_event=hsi_event, cons=updated_cons) if bep_delivered and (df.at[person_id, 'li_bmi'] == 1): df.at[person_id, 'ac_receiving_bep_supplements'] = True @@ -933,12 +887,7 @@ def calcium_supplementation(self, hsi_event): self.item_codes_preg_consumables['calcium'].items()} calcium_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='calcium_supplement', hsi_event=hsi_event, q_param=None, cons=updated_cons) - - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, cons=updated_cons, opt_cons=None) - # - # if avail: + self, int_name='calcium_supplement', hsi_event=hsi_event, cons=updated_cons) if calcium_delivered: df.at[person_id, 'ac_receiving_calcium_supplements'] = True @@ -960,21 +909,11 @@ def point_of_care_hb_testing(self, hsi_event): logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'hb_screen'}) hb_test_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='hb_test', hsi_event=hsi_event, q_param=None, + self, int_name='hb_test', hsi_event=hsi_event, cons=self.item_codes_preg_consumables['hb_test'], opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], dx_test='point_of_care_hb_test') - # # Run check against probability of testing being delivered - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['hb_test'], - # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - - # # We run the test through the dx_manager and if a woman has anaemia and its detected she will be admitted - # # for further care - # if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='point_of_care_hb_test', - # hsi_event=hsi_event): if hb_test_delivered: df.at[person_id, 'ac_to_be_admitted'] = True @@ -1015,7 +954,7 @@ def hep_b_testing(self, hsi_event): if not self.check_intervention_should_run_and_update_mni(person_id, 'hep_b_1', 'hep_b_2'): return - # This intervention is a place holder prior to the Hepatitis B module being coded + # This intervention is a placeholder prior to the Hepatitis B module being coded # Define the consumables avail = hsi_event.get_consumables(item_codes=cons['hep_b_test'], optional_item_codes=cons['blood_test_equipment']) @@ -1046,37 +985,15 @@ def syphilis_screening_and_treatment(self, hsi_event): opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], dx_test='blood_test_syphilis') - - # # See if she will receive testing - # if self.rng.random_sample() < params['prob_intervention_delivered_syph_test']: - # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_test'}) - # - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['syphilis_test'], - # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - # - # test = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - # dx_tests_to_run='blood_test_syphilis', hsi_event=hsi_event) - - # # If the testing occurs and detects syphilis she will get treatment (if consumables are available) - # if avail and test: - if syph_test_delivered: syph_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='syphilis_treatment', hsi_event=hsi_event, - q_param=None, cons=self.item_codes_preg_consumables['syphilis_treatment'], opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['syphilis_treatment'], - # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - # - # # if avail: if syph_treatment_delivered: + # We assume that treatment is 100% effective at curing infection df.at[person_id, 'ps_syphilis'] = False logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_treat'}) @@ -1143,30 +1060,6 @@ def gdm_screening(self, hsi_event): dx_test='blood_test_glucose', equipment={'Glucometer'}) - - # # If they are available, the test is conducted - # if self.rng.random_sample() < params['prob_intervention_delivered_gdm_test']: - # - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['gdm_test'], - # opt_cons=self.item_codes_preg_consumables['blood_test_equipment']) - # - # # If the test accurately detects a woman has gestational diabetes the consumables are recorded and - # # she is referred for treatment - # if avail: - # hsi_event.add_equipment({'Glucometer'}) - # - # if ( - # self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - # dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event) - # ): - # logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'}) - # mni[person_id]['anc_ints'].append('gdm_screen') - - # We assume women with a positive GDM screen will be admitted (if they are not already receiving - # outpatient care) - if gdm_screening_delivered: if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none': @@ -1288,23 +1181,13 @@ def full_blood_count_testing(self, hsi_event): df = self.sim.population.props person_id = hsi_event.target - # # Run dx_test for anaemia... - # # If a woman is not truly anaemic but the FBC returns a result of anaemia, due to tests specificity, we - # # assume the reported anaemia is mild - # hsi_event.get_consumables(item_codes=self.item_codes_preg_consumables['blood_test_equipment']) - # hsi_event.add_equipment({'Analyser, Haematology'}) - # - # test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - # dx_tests_to_run='full_blood_count_hb', hsi_event=hsi_event) - full_blood_count_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='full_blood_count', hsi_event=hsi_event, - q_param=None,cons=None, opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], equipment={'Analyser, Haematology'}, dx_test='full_blood_count_hb') if full_blood_count_delivered and (df.at[person_id, 'ps_anaemia_in_pregnancy'] == 'none'): - return 'non_severe' + return 'none' # If the test correctly identifies a woman's anaemia we assume it correctly identifies its severity if full_blood_count_delivered and (df.at[person_id, 'ps_anaemia_in_pregnancy'] != 'none'): @@ -1328,19 +1211,6 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event): store_dalys_in_mni = pregnancy_helper_functions.store_dalys_in_mni mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - # # Check for consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['blood_transfusion'], - # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) - # - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - # sf='blood_tran', - # hsi_event=hsi_event) - # - # # If the blood is available we assume the intervention can be delivered - # if avail and sf_check: - blood_transfusion_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='blood_transfusion', hsi_event=hsi_event, q_param=[l_params['prob_hcw_avail_blood_tran'], l_params['mean_hcw_competence_hp']], @@ -1349,9 +1219,6 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event): equipment={'Drip stand', 'Infusion pump'}) if blood_transfusion_delivered: - pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) - - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # If the woman is receiving blood due to anaemia we apply a probability that a transfusion of 2 units # RBCs will correct this woman's severe anaemia @@ -1375,13 +1242,7 @@ def initiate_maintenance_anti_hypertensive_treatment(self, individual_id, hsi_ev self.item_codes_preg_consumables['oral_antihypertensives'].items()} oral_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='oral_antihypertensives', hsi_event=hsi_event, - q_param=None, cons=updated_cons) - # - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, cons=updated_cons, opt_cons=None) - # # If the consumables are available then the woman is started on treatment - # if avail: + self, int_name='oral_antihypertensives', hsi_event=hsi_event, cons=updated_cons) if oral_anti_htns_delivered: df.at[individual_id, 'ac_gest_htn_on_treatment'] = True @@ -1397,23 +1258,13 @@ def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event): """ df = self.sim.population.props - # # Define the consumables and check their availability - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['iv_antihypertensives'], - # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) - # - # # If they are available then the woman is started on treatment - # if avail: - iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='iv_antihypertensives', hsi_event=hsi_event, - q_param=None, cons=self.item_codes_preg_consumables['iv_antihypertensives'], - opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'], equipment={'Drip stand', 'Infusion pump'}) + cons=self.item_codes_preg_consumables['iv_antihypertensives'], + opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'], + equipment={'Drip stand', 'Infusion pump'}) if iv_anti_htns_delivered: - pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event) - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # We assume women treated with antihypertensives would no longer be severely hypertensive- meaning they # are not at risk of death from severe gestational hypertension in the PregnancySupervisor event @@ -1439,19 +1290,6 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve df = self.sim.population.props l_params = self.sim.modules['Labour'].current_parameters - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['magnesium_sulfate'], - # opt_cons=self.item_codes_preg_consumables['eclampsia_management_optional']) - # - # # check HCW will deliver intervention - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - # sf='anticonvulsant', - # hsi_event=hsi_event) - # - # # If available deliver the treatment - # if avail and sf_check: - mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='mgso4', hsi_event=hsi_event, q_param=[l_params['prob_hcw_avail_anticonvulsant'], l_params['mean_hcw_competence_hp']], @@ -1461,8 +1299,6 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve if mag_sulph_delivered: df.at[individual_id, 'ac_mag_sulph_treatment'] = True - pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def antibiotics_for_prom(self, individual_id, hsi_event): """ @@ -1474,17 +1310,6 @@ def antibiotics_for_prom(self, individual_id, hsi_event): df = self.sim.population.props l_params = self.sim.modules['Labour'].current_parameters - # # check consumables and whether HCW are available to deliver the intervention - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_preg_consumables['abx_for_prom'], - # opt_cons=self.item_codes_preg_consumables['iv_drug_equipment']) - # - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - # sf='iv_abx', - # hsi_event=hsi_event) - # - # if avail and sf_check: abx_prom_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='abx_for_prom', hsi_event=hsi_event, q_param=[l_params['prob_hcw_avail_iv_abx'], l_params['mean_hcw_competence_hp']], @@ -1494,7 +1319,6 @@ def antibiotics_for_prom(self, individual_id, hsi_event): if abx_prom_delivered: df.at[individual_id, 'ac_received_abx_for_prom'] = True - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def ectopic_pregnancy_treatment_doesnt_run(self, hsi_event): """ @@ -2641,12 +2465,6 @@ def schedule_gdm_event_and_checkup(): updated_cons = {k: v * days for (k, v) in self.module.item_codes_preg_consumables['oral_diabetic_treatment'].items()} - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, cons=updated_cons, opt_cons=None) - # - # # If the meds are available women are started on that treatment - # if avail: - gdm_orals_delivered = pregnancy_helper_functions.check_int_deliverable( self.module, int_name='gdm_treatment_orals', hsi_event=self, q_param=None, cons=updated_cons) @@ -2671,11 +2489,6 @@ def schedule_gdm_event_and_checkup(): updated_cons = {k: v * required_vials for (k, v) in self.module.item_codes_preg_consumables['insulin_treatment'].items()} - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, cons=updated_cons, opt_cons=None) - # - # if avail: - gdm_insulin_delivered = pregnancy_helper_functions.check_int_deliverable( self.module, int_name='gdm_treatment_insulin', hsi_event=self, q_param=None, cons=updated_cons) @@ -2746,62 +2559,6 @@ def apply(self, person_id, squeeze_factor): if pac_delivered: df.at[person_id, 'ac_received_post_abortion_care'] = True - # # Request baseline PAC consumables - # baseline_cons = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['post_abortion_care_core'], - # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_optional']) - # - # # Check HCW availability to deliver surgical removal of retained products - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - # sf='retained_prod', - # hsi_event=self) - # - # # Add used equipment if intervention can happen - # if baseline_cons and sf_check: - # self.add_equipment({'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) - # - # # Then we determine if a woman gets treatment for her complication depending on availability of the baseline - # # consumables (misoprostol) or a HCW who can conduct MVA/DC (we dont model equipment) and additional - # # consumables for management of her specific complication - # if abortion_complications.has_any([person_id], 'sepsis', first=True): - # - # cons_for_sepsis_pac = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_core'], - # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_optional']) - # - # if cons_for_sepsis_pac and (baseline_cons or sf_check): - # df.at[person_id, 'ac_received_post_abortion_care'] = True - # - # elif abortion_complications.has_any([person_id], 'haemorrhage', first=True): - # cons_for_haemorrhage = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['blood_transfusion'], - # opt_cons=self.module.item_codes_preg_consumables['iv_drug_equipment']) - # - # cons_for_shock = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], - # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) - # - # if cons_for_haemorrhage and cons_for_shock and (baseline_cons or sf_check): - # df.at[person_id, 'ac_received_post_abortion_care'] = True - # - # elif abortion_complications.has_any([person_id], 'injury', first=True): - # cons_for_shock = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'], - # opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional']) - # - # if cons_for_shock and (baseline_cons or sf_check): - # df.at[person_id, 'ac_received_post_abortion_care'] = True - # - # elif abortion_complications.has_any([person_id], 'other', first=True) and (baseline_cons or sf_check): - # df.at[person_id, 'ac_received_post_abortion_care'] = True - - if df.at[person_id, 'ac_received_post_abortion_care']: - pregnancy_helper_functions.log_met_need(self.module, 'pac', self) def did_not_run(self): logger.debug(key='message', data='HSI_CareOfWomenDuringPregnancy_PostAbortionCaseManagement: did not run') @@ -2843,19 +2600,8 @@ def apply(self, person_id, squeeze_factor): opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional'], equipment={'Laparotomy Set'}) - # # We define the required consumables and check their availability - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_core'], - # opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional']) - # - # # If they are available then treatment can go ahead - # if avail: - if ep_surg_delivered: self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person_id]['delete_mni'] = True - pregnancy_helper_functions.log_met_need(self.module, 'ep_case_mang', self) - # self.add_equipment({'Laparotomy Set'}) # For women who have sought care after they have experienced rupture we use this treatment variable to # reduce risk of death (women who present prior to rupture do not pass through the death event as we assume diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 7c31218ec3..85acf3fccd 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1412,12 +1412,6 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if property_prefix == 'ps': - timing = 'intrapartum' - current_log = logger - else: - timing = 'postnatal' - current_log = logger_pn # n.b. on birth women whose hypertension will continue into the postnatal period have their disease state stored # in a new property therefore antenatal/intrapartum hypertension is 'ps_htn_disorders' and postnatal is @@ -1652,20 +1646,6 @@ def prophylactic_labour_interventions(self, hsi_event): mni[person_id]['abx_for_prom_given'] = True else: - # # Run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', - # hsi_event=hsi_event) - # - # # If she has not already receive antibiotics, we check for consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['abx_for_prom'], - # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # - # # Then query if these consumables are available during this HSI And provide if available. - # # Antibiotics for from reduce risk of newborn sepsis within the first - # # week of life - # if avail and sf_check: abx_prom_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='abx_for_prom', hsi_event=hsi_event, @@ -1681,15 +1661,6 @@ def prophylactic_labour_interventions(self, hsi_event): if mni[person_id]['labour_state'] == 'early_preterm_labour' or \ mni[person_id]['labour_state'] == 'late_preterm_labour': - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['antenatal_steroids'], - # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # - # # If available they are given. Antenatal steroids reduce a preterm newborns chance of developing - # # respiratory distress syndrome and of death associated with prematurity - # if avail: - steroids_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='antenatal_corticosteroids', hsi_event=hsi_event, q_param=None, @@ -1746,20 +1717,6 @@ def assessment_and_treatment_of_severe_pre_eclampsia_mgso4(self, hsi_event, labo if (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'none') and (labour_stage == 'ip'): self.determine_delivery_mode_in_spe_or_ec(person_id, hsi_event, 'spe') - # # Run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', - # hsi_event=hsi_event) - # - # # Define and check for the required consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['magnesium_sulfate'], - # opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) - # - # # If the consumables are available - the intervention is delivered. IV magnesium reduces the - # # probability that a woman with severe pre-eclampsia will experience eclampsia in labour - # if avail and sf_check: - mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='mgso4', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_anticonvulsant'], params[f'mean_hcw_competence_{deliv_location}']], @@ -1768,7 +1725,6 @@ def assessment_and_treatment_of_severe_pre_eclampsia_mgso4(self, hsi_event, labo if mag_sulph_delivered: df.at[person_id, 'la_severe_pre_eclampsia_treatment'] = True - pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): """ @@ -1780,30 +1736,17 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): """ df = self.sim.population.props person_id = hsi_event.target - params = self.current_parameters # If the treatment has already been delivered the function won't run if (df.at[person_id, 'ps_htn_disorders'] != 'none') or (df.at[person_id, 'pn_htn_disorders'] != 'none'): - # # Then query if these consumables are available during this HSI - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['iv_antihypertensives'], - # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # - # # If they are available then the woman is started on treatment. Intravenous antihypertensive reduce a - # # womans risk of progression from mild to severe gestational hypertension ANd reduce risk of death for - # # women with severe pre-eclampsia and eclampsia - # if avail: - iv_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='iv_antihypertensives', hsi_event=hsi_event, - q_param=None, cons=self.item_codes_lab_consumables['iv_antihypertensives'], + cons=self.item_codes_lab_consumables['iv_antihypertensives'], opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) if iv_anti_htns_delivered: df.at[person_id, 'la_maternal_hypertension_treatment'] = True - pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event) if (labour_stage == 'ip') and (df.at[person_id, 'ps_htn_disorders'] == 'severe_gest_htn'): df.at[person_id, 'ps_htn_disorders'] = 'gest_htn' @@ -1814,12 +1757,7 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage): cons = {_i: dose for _i in self.item_codes_lab_consumables['oral_antihypertensives']} oral_anti_htns_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='oral_antihypertensives', hsi_event=hsi_event, - q_param=None, cons=cons) - - # avail = hsi_event.get_consumables(item_codes=cons) - - # if avail: + self, int_name='oral_antihypertensives', hsi_event=hsi_event, cons=cons) if oral_anti_htns_delivered: df.at[person_id, 'la_gest_htn_on_treatment'] = True @@ -1842,16 +1780,6 @@ def assessment_and_treatment_of_eclampsia(self, hsi_event, labour_stage): if (df.at[person_id, 'ps_htn_disorders'] == 'eclampsia') or \ (df.at[person_id, 'pn_htn_disorders'] == 'eclampsia'): - # # Run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='anticonvulsant', - # hsi_event=hsi_event) - # - # # define and check required consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['magnesium_sulfate'], - # opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional']) - mag_sulph_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='mgso4', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_anticonvulsant'], params[f'mean_hcw_competence_{deliv_location}']], @@ -1861,11 +1789,10 @@ def assessment_and_treatment_of_eclampsia(self, hsi_event, labour_stage): if (labour_stage == 'ip') and (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'none'): self.determine_delivery_mode_in_spe_or_ec(person_id, hsi_event, 'ec') - # if avail and sf_check: if mag_sulph_delivered: + # Treatment with magnesium reduces a womans risk of death from eclampsia df.at[person_id, 'la_eclampsia_treatment'] = True - pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) def assessment_for_assisted_vaginal_delivery(self, hsi_event, indication): """ @@ -1897,22 +1824,10 @@ def refer_for_cs(): # Define the consumables... if df.at[person_id, 'la_obstructed_labour'] or (indication in ('spe_ec', 'other')): + # We assume women with CPD cannot be delivered via AVD and will require a caesarean if not mni[person_id]['cpd']: - # # If the general package is available AND the facility has the correct tools to carry out the - # # delivery then it can occur - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['vacuum'], - # opt_cons=self.item_codes_lab_consumables['obstructed_labour']) - # - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='avd', - # hsi_event=hsi_event) - - # if avail and sf_check: - avd_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='avd', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_avd'], params[f'mean_hcw_competence_{deliv_location}']], @@ -1921,10 +1836,6 @@ def refer_for_cs(): equipment={'Delivery Forceps', 'Vacuum extractor'}) if avd_delivered: - # Add used equipment - # hsi_event.add_equipment({'Delivery Forceps', 'Vacuum extractor'}) - - pregnancy_helper_functions.log_met_need(self, f'avd_{indication}', hsi_event) # If AVD was successful then we record the mode of delivery. We use this variable to reduce # risk of intrapartum still birth when applying risk in the death event @@ -1961,19 +1872,6 @@ def assessment_and_treatment_of_maternal_sepsis(self, hsi_event, labour_stage): ((labour_stage == 'ip') and df.at[person_id, 'ps_chorioamnionitis']) or (labour_stage == 'pp' and df.at[person_id, 'pn_sepsis_late_postpartum'])): - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='iv_abx', - # hsi_event=hsi_event) - # - # # Define and check available consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['maternal_sepsis_core'], - # opt_cons=self.item_codes_lab_consumables['maternal_sepsis_optional']) - # - # # If delivered this intervention reduces a womans risk of dying from sepsis - # if avail and sf_check: - sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='sepsis_treatment', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_iv_abx'], params[f'mean_hcw_competence_{deliv_location}']], @@ -1982,7 +1880,6 @@ def assessment_and_treatment_of_maternal_sepsis(self, hsi_event, labour_stage): if sepsis_treatment_delivered: df.at[person_id, 'la_sepsis_treatment'] = True - pregnancy_helper_functions.log_met_need(self, 'sepsis_abx', hsi_event) def assessment_and_plan_for_antepartum_haemorrhage(self, hsi_event): """ @@ -2043,20 +1940,6 @@ def active_management_of_the_third_stage_of_labour(self, hsi_event): person_id = hsi_event.target deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' - # # Define and check available consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['amtsl'], - # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', - # hsi_event=hsi_event) - # - # # This treatment reduces a womans risk of developing uterine atony AND retained placenta, both of which are - # # preceding causes of postpartum haemorrhage - # if avail and sf_check: - amtsl_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='amtsl', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_uterotonic'], params[f'mean_hcw_competence_{deliv_location}']], @@ -2082,18 +1965,6 @@ def assessment_and_treatment_of_pph_uterine_atony(self, hsi_event): if df.at[person_id, 'la_postpartum_haem'] and not mni[person_id]['retained_placenta']: - # # Define and check available consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['pph_core'], - # opt_cons=self.item_codes_lab_consumables['pph_optional']) - # - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic', - # hsi_event=hsi_event) - # - # if avail and sf_check: - pph_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='pph_treatment_uterotonics', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_uterotonic'], params[f'mean_hcw_competence_{deliv_location}']], @@ -2101,7 +1972,6 @@ def assessment_and_treatment_of_pph_uterine_atony(self, hsi_event): opt_cons=self.item_codes_lab_consumables['pph_optional']) if pph_treatment_delivered: - pregnancy_helper_functions.log_met_need(self, 'uterotonics', hsi_event) # We apply a probability that this treatment will stop a womans bleeding in the first instance # meaning she will not require further treatment @@ -2135,32 +2005,17 @@ def assessment_and_treatment_of_pph_retained_placenta(self, hsi_event): df.at[person_id, 'pn_postpartum_haem_secondary'] ): - # # Log the consumables but dont condition the treatment on their availability - the primary mechanism of this - # # intervention doesnt require consumables - # hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['pph_optional']) - # - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='man_r_placenta', - # hsi_event=hsi_event) - # - # # Similar to uterotonics we apply a probability that this intervention will successfully stop - # # bleeding to ensure some women go on to require further care - # if sf_check: - pph_mrrp_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='pph_treatment_mrrp', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_man_r_placenta'], params[f'mean_hcw_competence_{deliv_location}']], - cons=None, opt_cons=self.item_codes_lab_consumables['pph_optional']) if pph_mrrp_delivered: - pregnancy_helper_functions.log_met_need(self, 'man_r_placenta', hsi_event) if params['prob_successful_manual_removal_placenta'] > self.rng.random_sample(): df.at[person_id, 'la_postpartum_haem'] = False mni[person_id]['retained_placenta'] = False - pregnancy_helper_functions.log_met_need(self, 'man_r_placenta', hsi_event) if df.at[person_id, 'pn_postpartum_haem_secondary']: df.at[person_id, 'pn_postpartum_haem_secondary'] = False @@ -2181,21 +2036,6 @@ def surgical_management_of_pph(self, hsi_event): params = self.current_parameters deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' - # # We log the required consumables and condition the surgery happening on the availability of the - # # first consumable in this package, the anaesthetic required for the surgery - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['obstetric_surgery_core'], - # opt_cons=self.item_codes_lab_consumables['obstetric_surgery_optional']) - # - # # run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='surg', - # hsi_event=hsi_event) - # - # if avail and sf_check: - # # Add used equipment - # hsi_event.add_equipment(hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery')) - pph_surg_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='pph_treatment_surg', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_surg'], params[f'mean_hcw_competence_{deliv_location}']], @@ -2215,10 +2055,6 @@ def surgical_management_of_pph(self, hsi_event): self.pph_treatment.set(person_id, 'hysterectomy') df.at[person_id, 'la_has_had_hysterectomy'] = True - # log intervention delivery - if self.pph_treatment.has_all(person_id, 'surgery') or df.at[person_id, 'la_has_had_hysterectomy']: - pregnancy_helper_functions.log_met_need(self, 'pph_surg', hsi_event) - def blood_transfusion(self, hsi_event): """ This function represents the blood transfusion during or after labour. This function is called during @@ -2232,19 +2068,6 @@ def blood_transfusion(self, hsi_event): df = self.sim.population.props deliv_location = 'hc' if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a' else 'hp' - # # Check consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_lab_consumables['blood_transfusion'], - # opt_cons=self.item_codes_lab_consumables['iv_drug_equipment']) - # - # # check HCW - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='blood_tran', - # hsi_event=hsi_event) - # - # if avail and sf_check: - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) - blood_transfusion_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='blood_transfusion', hsi_event=hsi_event, q_param=[params['prob_hcw_avail_blood_tran'], params[f'mean_hcw_competence_{deliv_location}']], @@ -2254,7 +2077,6 @@ def blood_transfusion(self, hsi_event): if blood_transfusion_delivered: mni[person_id]['received_blood_transfusion'] = True - pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) # We assume that anaemia is corrected by blood transfusion if df.at[person_id, 'pn_anaemia_following_pregnancy'] != 'none': @@ -2276,21 +2098,8 @@ def assessment_and_treatment_of_anaemia(self, hsi_event): mother = df.loc[person_id] mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - # # Add used equipment - # hsi_event.add_equipment({'Analyser, Haematology'}) - # - # # Use dx_test function to assess anaemia status - # test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - # dx_tests_to_run='full_blood_count_hb_pn', hsi_event=hsi_event) - # - # hsi_event.get_consumables(item_codes=self.item_codes_lab_consumables['blood_test_equipment']) - # - # # Check consumables - # if test_result: - full_blood_count_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='full_blood_count', hsi_event=hsi_event, - q_param=None, cons=None, opt_cons=self.item_codes_lab_consumables['blood_test_equipment'], equipment={'Analyser, Haematology'}, dx_test='full_blood_count_hb_pn') @@ -2301,14 +2110,9 @@ def assessment_and_treatment_of_anaemia(self, hsi_event): days = int((6 - df.at[person_id, 'pn_postnatal_period_in_weeks']) * 7) dose = days * 3 cons = {_i: dose for _i in self.item_codes_lab_consumables['iron_folic_acid']} - # avail = hsi_event.get_consumables(item_codes=cons) - # - # # Start iron and folic acid treatment - # if avail: iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='iron_folic_acid', - hsi_event=hsi_event, q_param=None, cons=cons) + self, int_name='iron_folic_acid', hsi_event=hsi_event, cons=cons) if iron_folic_acid_delivered: df.at[person_id, 'la_iron_folic_acid_postnatal'] = True @@ -2358,10 +2162,7 @@ def interventions_delivered_pre_discharge(self, hsi_event): self.item_codes_lab_consumables['iron_folic_acid']} iron_folic_acid_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='iron_folic_acid', - hsi_event=hsi_event, q_param=None, cons=cons) - - # avail = hsi_event.get_consumables(item_codes=cons) + self, int_name='iron_folic_acid', hsi_event=hsi_event, cons=cons) # Women are started on iron and folic acid for the next three months which reduces risk of anaemia in the # postnatal period @@ -3004,17 +2805,6 @@ def apply(self, person_id, squeeze_factor): elif df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'avd_now': self.module.assessment_for_assisted_vaginal_delivery(self, indication='spe_ec') - # LOG CONSUMABLES FOR DELIVERY... - # # We assume all deliveries require this basic package of consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_lab_consumables['delivery_core'], - # opt_cons=self.module.item_codes_lab_consumables['delivery_optional']) - # - # # If the clean delivery kit consumable is available, we assume women benefit from clean delivery - # if avail: - # mni[person_id]['clean_birth_practices'] = True - birth_kit_used = pregnancy_helper_functions.check_int_deliverable( self.module, int_name='birth_kit', hsi_event=self, cons=self.module.item_codes_lab_consumables['delivery_core'], @@ -3100,15 +2890,6 @@ def apply(self, person_id, squeeze_factor): q_param=[params['prob_hcw_avail_neo_resus'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.module.item_codes_lab_consumables['resuscitation']) - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, cons=self.module.item_codes_lab_consumables['resuscitation'], opt_cons=None) - # - # # Run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, - # sf='neo_resus', - # hsi_event=self) - # if sf_check and avail: - if neo_resus_delivered: mni[person_id]['neo_will_receive_resus_if_needed'] = True @@ -3320,17 +3101,6 @@ def apply(self, person_id, squeeze_factor): # delivered if mni[person_id]['referred_for_cs'] and self.timing == 'intrapartum': - # # We log the required consumables and condition the caesarean happening on the availability of the - # # first consumable in this package, the anaesthetic required for the surgery - # avail = pregnancy_helper_functions.return_cons_avail( - # self.module, self, - # cons=self.module.item_codes_lab_consumables['caesarean_delivery_core'], - # opt_cons=self.module.item_codes_lab_consumables['caesarean_delivery_optional']) - # - # # We check that the HCW will deliver the intervention - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, sf='surg', - # hsi_event=self) - cs_delivered = pregnancy_helper_functions.check_int_deliverable( self.module, int_name='caesarean_section', hsi_event=self, q_param=[params['prob_hcw_avail_surg'], params[f'mean_hcw_competence_{deliv_location}']], @@ -3341,7 +3111,6 @@ def apply(self, person_id, squeeze_factor): if params['la_analysis_in_progress'] and (params['cemonc_availability'] == 0.0): logger.debug(key='message', data="cs delivery blocked for this analysis") - # elif (avail and sf_check) or (mni[person_id]['cs_indication'] == 'other'): elif cs_delivered or (mni[person_id]['cs_indication'] == 'other'): # If intervention is delivered - add used equipment @@ -3370,9 +3139,6 @@ def apply(self, person_id, squeeze_factor): # further checks and surgical repair is assume to have occurred during their caesarean surgery if mni[person_id]['mode_of_delivery'] == 'caesarean_section': - # log treatment is delivered - pregnancy_helper_functions.log_met_need(self.module, 'ur_surg', self) - # Determine if the uterus can be repaired treatment_success_ur = params['success_rate_uterine_repair'] > self.module.rng.random_sample() @@ -3523,7 +3289,6 @@ def apply(self, population): nb_params['prob_timings_pnc_newborns'] = [1.0, 0] if params['alternative_pnc_quality']: - nb_params['prob_kmc_available'] = params['pnc_availability_probability'] params['prob_intervention_delivered_anaemia_assessment_pnc'] = params['pnc_availability_probability'] if params['pnc_sens_analysis_max'] or params['pnc_sens_analysis_min']: diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index c1b34abae6..3b8cad4da0 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -901,13 +901,9 @@ def kangaroo_mother_care(self, hsi_event): (df.at[person_id, 'nb_low_birth_weight_status'] != 'macrosomia'): kmc_delivered = pregnancy_helper_functions.check_int_deliverable( - self, int_name='kmc', - hsi_event=hsi_event, - q_param=[params['prob_kmc_available']], - cons=None) + self, int_name='kmc', hsi_event=hsi_event, q_param=[params['prob_kmc_available']]) # Check KMC can be delivered - # if self.rng.random_sample() < params['prob_kmc_available']: if kmc_delivered: # Store treatment as a property of the newborn used to apply treatment effect df.at[person_id, 'nb_kangaroo_mother_care'] = True @@ -968,22 +964,8 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if df.at[person_id, 'nb_early_onset_neonatal_sepsis'] or df.at[person_id, 'pn_sepsis_late_neonatal'] or\ df.at[person_id, 'pn_sepsis_early_neonatal']: - - # # Run HCW check - # sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'], - # sf='iv_abx', - # hsi_event=hsi_event) if facility_type != '1a': - # # check consumables - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_nb_consumables['sepsis_supportive_care_core'], - # opt_cons=self.item_codes_nb_consumables['sepsis_supportive_care_optional']) - - # # Then, if the consumables are available, treatment for sepsis is delivered - # if avail and sf_check: - neo_sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='neo_sepsis_treatment_supp_care', hsi_event=hsi_event, q_param=[l_params['prob_hcw_avail_iv_abx'], l_params[f'mean_hcw_competence_{pnc_location}']], @@ -993,17 +975,9 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if neo_sepsis_treatment_delivered: df.at[person_id, 'nb_supp_care_neonatal_sepsis'] = True - pregnancy_helper_functions.log_met_need(self, 'neo_sep_supportive_care', hsi_event) - # hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # The same pattern is then followed for health centre care else: - # avail = pregnancy_helper_functions.return_cons_avail( - # self, hsi_event, - # cons=self.item_codes_nb_consumables['sepsis_abx'], - # opt_cons=self.item_codes_nb_consumables['iv_drug_equipment']) - # - # if avail and sf_check: neo_sepsis_treatment_delivered = pregnancy_helper_functions.check_int_deliverable( self, int_name='neo_sepsis_treatment_abx', hsi_event=hsi_event, q_param=[l_params['prob_hcw_avail_iv_abx'], l_params[f'mean_hcw_competence_{pnc_location}']], @@ -1013,8 +987,6 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if neo_sepsis_treatment_delivered: df.at[person_id, 'nb_inj_abx_neonatal_sepsis'] = True - pregnancy_helper_functions.log_met_need(self, 'neo_sep_abx', hsi_event) - # hsi_event.add_equipment({'Drip stand', 'Infusion pump', 'Oxygen cylinder, with regulator'}) def link_twins(self, child_one, child_two, mother_id): """ diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 4783d28199..ad0a69de82 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -82,9 +82,7 @@ def check_int_deliverable(self, int_name, hsi_event, intervention delivery is determined by any module-level quality parameters, consumable availability, and (if applicable) the results of any dx_tests. Equipment is also declared. - TODO: what about interventions outside mnh modules - -: param self: module + :param self: module param int_name: items for code look up param hsi_event: module param q_param: items for code look up @@ -97,13 +95,15 @@ def check_int_deliverable(self, int_name, hsi_event, df = self.sim.population.props individual_id = hsi_event.target p_params = self.sim.modules['PregnancySupervisor'].current_parameters + l_params = self.sim.modules['Labour'].current_parameters # assert int_name in p_params['all_interventions'] # Firstly, we determine if an analysis is currently being conducted during which the probability of intervention # delivery is being overridden # To do: replace this parameter - if p_params['ps_analysis_in_progress'] and (int_name in p_params['interventions_under_analysis']): + if (p_params['interventions_analysis'] and p_params['ps_analysis_in_progress'] and + (int_name in p_params['interventions_under_analysis'])): # If so, we determine if this intervention will be delivered given the set probability of delivery. can_int_run_analysis = self.rng.random_sample() < p_params['intervention_analysis_availability'] @@ -131,6 +131,35 @@ def check_int_deliverable(self, int_name, hsi_event, else: return True + elif (l_params['la_analysis_in_progress'] or + (p_params['ps_analysis_in_progress'] and not p_params['interventions_under_analysis'])): + + if 'AntenatalCare' in hsi_event.TREATMENT_ID: + params = self.sim.modules['PregnancySupervisor'].current_parameters + else: + params = self.sim.modules['Labour'].current_parameters + + # Define HSIs and analysis parameters of interest + analysis_dict = {'AntenatalCare_Outpatient': ['alternative_anc_quality', 'anc_availability_probability'], + 'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], + 'AntenatalCare_FollowUp': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], + 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_cons_availability'], + 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_cons_availability'], + 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_cons_availability'], + 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], + 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], + 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} + + for k in analysis_dict: + # If analysis is running, the analysis date has passed and an appropriate HSI has called this function then + # probability of intervention delivery is determined by an analysis parameter + if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: + if self.rng.random_sample() < params[analysis_dict[k][1]]: + return True + + else: + return False + else: # If analysis is not being conducted, intervention delivery is dependent on quality parameters, consumable @@ -143,6 +172,7 @@ def check_int_deliverable(self, int_name, hsi_event, all([self.rng.random_sample() < value for value in q_param])): quality = True + # todo: should this only be if qual and cons are also true? if equipment is not None: hsi_event.add_equipment(equipment) @@ -165,199 +195,6 @@ def check_int_deliverable(self, int_name, hsi_event, return False -# def return_cons_avail(self, hsi_event, cons, opt_cons): -# """ -# This function is called by majority of interventions across maternal and neonatal modules to return whether a -# consumable or package of consumables are available. If analysis is not being conducted (as indicated by a series of -# analysis boolean parameters) then the availability is determined via the health system module. Otherwise a -# fixed probability of availability is used to force a set level of intervention coverage -# availability -# :param self: module -# :param hsi_event: hsi_event calling the intervention -# :param cons_dict: dictionary containing the consumables for that module -# :param info: information on core, optional and number of consumables requested -# :return: BOOL -# """ -# mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info -# ps_params = self.sim.modules['PregnancySupervisor'].current_parameters -# la_params = self.sim.modules['Labour'].current_parameters -# -# if opt_cons is None: -# opt_cons = [] -# -# # Check if analysis is currently running, if not then availability is determined normally -# if not ps_params['ps_analysis_in_progress'] and not la_params['la_analysis_in_progress']: -# available = hsi_event.get_consumables(item_codes=cons, -# optional_item_codes=opt_cons) -# -# if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): -# mni[hsi_event.target]['cons_not_avail'] = True -# -# return available -# -# else: -# available = hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons) -# -# # Depending on HSI calling this function a different parameter set is used to determine if analysis is being -# # conducted -# if 'AntenatalCare' in hsi_event.TREATMENT_ID: -# params = self.sim.modules['PregnancySupervisor'].current_parameters -# else: -# params = self.sim.modules['Labour'].current_parameters -# -# # Store the names of the parameters which indicate that analysis is being conducted against specific HSIs -# analysis_dict = {'AntenatalCare_Outpatient': ['alternative_anc_quality', 'anc_availability_probability'], -# 'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], -# 'AntenatalCare_FollowUp': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], -# 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_cons_availability'], -# 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_cons_availability'], -# 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_cons_availability'], -# 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], -# 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], -# 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} -# -# # Cycle through each HSI of interest. If this HSI is requesting a consumable, analysis is being conducted and -# # the simulation date is greater than the predetermined analysis date, then a random draw against a probability -# # of consumable availability is used -# for k in analysis_dict: -# if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: -# if self.rng.random_sample() < params[analysis_dict[k][1]]: -# available = True -# else: -# available = False -# -# # If the consumable is not available this is stored in the MNI dictionary -# if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'): -# mni[hsi_event.target]['cons_not_avail'] = True -# -# return available - - -# def check_emonc_signal_function_will_run(self, sf, hsi_event): -# """ -# Called during/after labour to determine if a B/CEmONC function will run depending on the availability of HCWs -# trained in the intervention and their competence -# :param self: module -# :param sf: signal function of interest -# :param hsi_event: hsi_event calling the intervention -# :return: BOOL -# """ -# la_params = self.sim.modules['Labour'].current_parameters -# ps_params = self.sim.modules['PregnancySupervisor'].current_parameters -# mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info -# -# def see_if_sf_will_run(): -# # Determine competence parameter to use (higher presumed competence in hospitals) -# if hsi_event.ACCEPTED_FACILITY_LEVEL == '1a': -# competence = la_params['mean_hcw_competence_hc'][0] -# else: -# competence = la_params['mean_hcw_competence_hp'][1] -# -# comp_result = self.rng.random_sample() < competence -# hcw_result = self.rng.random_sample() < la_params[f'prob_hcw_avail_{sf}'] -# -# # If HCW is available and staff are competent at delivering the intervention then it will be delivered -# if comp_result and hcw_result: -# return True -# -# # Otherwise log within the mni -# if not comp_result and (hsi_event.target in mni): -# mni[hsi_event.target]['comp_not_avail'] = True -# if not hcw_result and (hsi_event.target in mni): -# mni[hsi_event.target]['hcw_not_avail'] = True -# -# return False -# -# if not la_params['la_analysis_in_progress'] and not ps_params['ps_analysis_in_progress']: -# return see_if_sf_will_run() -# -# else: -# if 'AntenatalCare' in hsi_event.TREATMENT_ID: -# params = self.sim.modules['PregnancySupervisor'].current_parameters -# else: -# params = self.sim.modules['Labour'].current_parameters -# -# # Define HSIs and analysis parameters of interest -# analysis_dict = {'AntenatalCare_Inpatient': ['alternative_ip_anc_quality', 'ip_anc_availability_probability'], -# 'DeliveryCare_Basic': ['alternative_bemonc_availability', 'bemonc_availability'], -# 'DeliveryCare_Neonatal': ['alternative_bemonc_availability', 'bemonc_availability'], -# 'DeliveryCare_Comprehensive': ['alternative_cemonc_availability', 'cemonc_availability'], -# 'PostnatalCare_Maternal': ['alternative_pnc_quality', 'pnc_availability_probability'], -# 'PostnatalCare_Comprehensive': ['alternative_pnc_quality', 'pnc_availability_probability'], -# 'PostnatalCare_Neonatal': ['alternative_pnc_quality', 'pnc_availability_probability']} -# -# for k in analysis_dict: -# # If analysis is running, the analysis date has passed and an appropriate HSI has called this function then -# # probability of intervention delivery is determined by an analysis parameter -# if (hsi_event.TREATMENT_ID == k) and params[analysis_dict[k][0]]: -# if self.rng.random_sample() < params[analysis_dict[k][1]]: -# return True -# -# # If the event wont run it is stored in the HSI for the relevant woman -# elif hsi_event.target in mni: -# barrier = self.rng.choice(['comp_not_avail', 'hcw_not_avail']) -# mni[hsi_event.target][barrier] = True -# return False -# -# return see_if_sf_will_run() - - -def log_met_need(self, intervention, hsi): - """ - This function is called whenever a woman receives an intervention used as treatment for a potentially fatal - complication within the maternal or neonatal health modules. The intervention is logged within the labour.detail - logging and is used to calculate treatment coverage/met need - :param self: module - :param intervention: intervention to be logged - :param hsi: hsi_event calling the intervention - """ - df = self.sim.population.props - logger = logging.getLogger("tlo.methods.labour.detail") - person = df.loc[hsi.target] - person_id = hsi.target - - # Interventions with single indications are simply logged - if intervention in ('ep_case_mang', 'pac', 'avd_other', 'avd_ol', 'avd_spe_ec', 'uterotonics', 'man_r_placenta', - 'pph_surg', 'ur_surg', 'neo_resus', 'neo_sep_supportive_care', 'neo_sep_abx'): - logger.info(key='intervention', data={'person_id': person_id, 'int': intervention}) - - # For interventions with multiple indications the HSI and properties of the individual are used to determine the - # correct name for the logged intervention. - elif (intervention == 'mag_sulph') or (intervention == 'iv_htns'): - if hsi.TREATMENT_ID == 'AntenatalCare_Inpatient': - tp = ['an', 'ps'] - elif hsi.TREATMENT_ID == 'DeliveryCare_Basic': - tp = ['la', 'ps'] - else: - tp = ['pn', 'pn'] - - logger.info(key='intervention', - data={'person_id': person_id, - 'int': f'{intervention}_{tp[0]}_{df.at[person_id, f"{tp[1]}_htn_disorders"]}'}) - - elif intervention == 'sepsis_abx': - if (hsi.TREATMENT_ID == 'DeliveryCare_Basic') or (hsi.TREATMENT_ID == 'AntenatalCare_Inpatient'): - logger.info(key='intervention', data={'person_id': person_id, 'int': 'abx_an_sepsis'}) - - elif hsi.TREATMENT_ID == 'PostnatalCare_Maternal': - logger.info(key='intervention', data={'person_id': person_id, 'int': 'abx_pn_sepsis'}) - - elif intervention == 'blood_tran': - if hsi.TREATMENT_ID == 'AntenatalCare_Inpatient': - logger.info(key='intervention', data={'person_id': person_id, 'int': 'blood_tran_anaemia'}) - - elif hsi.TREATMENT_ID == 'DeliveryCare_Comprehensive': - if ((person.la_antepartum_haem != 'none') or (person.ps_antepartum_haemorrhage != 'none')) and \ - not person.la_is_postpartum: - logger.info(key='intervention', data={'person_id': person_id, 'int': 'blood_tran_aph'}) - - elif person.la_uterine_rupture and not person.la_is_postpartum: - logger.info(key='intervention', data={'person_id': person_id, 'int': 'blood_tran_ur'}) - - elif (person.la_postpartum_haem or person.pn_postpartum_haem_secondary) and person.la_is_postpartum: - logger.info(key='intervention', data={'person_id': person_id, 'int': 'blood_tran_pph'}) - - def scale_linear_model_at_initialisation(self, model, parameter_key): """ This function scales the intercept value of linear models according to the distribution of predictor values diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 5e0cf42f5a..07d8b559f1 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -390,8 +390,6 @@ def __init__(self, name=None, resourcefilepath=None): Types.BOOL, 'Signals within the analysis event and code that sensitivity analysis is being undertaken in ' 'which the maximum coverage of ANC is enforced'), - 'intervention_availability': Parameter( - Types.DATA_FRAME, ''), 'interventions_analysis': Parameter( Types.BOOL, ''), 'interventions_under_analysis': Parameter( @@ -454,8 +452,6 @@ def read_parameters(self, data_folder): workbook = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx', sheet_name=None) self.load_parameters_from_dataframe(workbook["parameter_values"]) - workbook['intervention_availability'].set_index('intervention', inplace=True) - p['intervention_availability'] = workbook['intervention_availability'] # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module. # Currently this module calculates and reports all DALY weights from all maternal modules @@ -2169,11 +2165,6 @@ def apply(self, population): params['prob_anc1_months_2_to_4'] = [1.0, 0, 0] params['prob_late_initiation_anc4'] = 0 - # Finally, remove squeeze factor threshold for ANC attendance to ensure that higher levels of ANC - # coverage can be reached with current logic - self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters['squeeze_factor_threshold_anc'] = \ - 10_000 - if params['alternative_anc_quality'] or params['sens_analysis_max']: # Override the availability of IPTp consumables with the set level of coverage @@ -2182,23 +2173,10 @@ def apply(self, population): self.sim.modules['HealthSystem'].override_availability_of_consumables( {iptp: params['anc_availability_probability']}) - # And then override the quality parameters in the model - for parameter in ['prob_intervention_delivered_urine_ds', 'prob_intervention_delivered_bp', - 'prob_intervention_delivered_syph_test', 'prob_intervention_delivered_gdm_test']: - self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters[parameter] = \ - params['anc_availability_probability'] - - if params['alternative_ip_anc_quality']: - self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters['squeeze_factor_threshold_an'] = \ - 10_000 - if params['sens_analysis_max']: for parameter in ['prob_seek_anc5', 'prob_seek_anc6', 'prob_seek_anc7', 'prob_seek_anc8']: self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters[parameter] = 1.0 - self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters['squeeze_factor_threshold_anc'] = \ - 10_000 - params['prob_seek_care_pregnancy_complication'] = 1.0 self.sim.modules['CareOfWomenDuringPregnancy'].current_parameters['prob_adherent_ifa'] = 1.0 diff --git a/tests/test_maternal_health_helper_and_analysis_functions.py b/tests/test_maternal_health_helper_and_analysis_functions.py index 82295984ed..9230670cd7 100644 --- a/tests/test_maternal_health_helper_and_analysis_functions.py +++ b/tests/test_maternal_health_helper_and_analysis_functions.py @@ -170,7 +170,7 @@ def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): when they run""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) + lparams = sim.modules['Labour'].parameters pparams = sim.modules['PregnancySupervisor'].parameters @@ -200,11 +200,11 @@ def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): unchanged_odds_anc = pparams['odds_early_init_anc4'][0] unchanged_odds_pnc = lparams['odds_will_attend_pnc'][0] + sim.make_initial_population(n=100) # run the model for 1 day sim.simulate(end_date=Date(2010, 1, 2)) p_current_params = sim.modules['PregnancySupervisor'].current_parameters - c_current_params = sim.modules['CareOfWomenDuringPregnancy'].current_parameters l_current_params = sim.modules['Labour'].current_parameters nbparams = sim.modules['NewbornOutcomes'].current_parameters @@ -216,10 +216,6 @@ def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): assert p_current_params['prob_late_initiation_anc4'] == 0 assert p_current_params['odds_early_init_anc4'] != unchanged_odds_anc - for parameter in ['prob_intervention_delivered_urine_ds', 'prob_intervention_delivered_bp', - 'prob_intervention_delivered_syph_test', 'prob_intervention_delivered_gdm_test']: - assert c_current_params[parameter] == new_avail_prob - # Now check corrent labour/newborn/postnatal parameters have been updated assert l_current_params['prob_intervention_delivered_anaemia_assessment_pnc'] == new_avail_prob @@ -234,7 +230,7 @@ def test_analysis_analysis_events_run_as_expected_and_update_parameters(seed): def test_analysis_analysis_events_run_as_expected_when_using_sensitivity_max_parameters(seed): sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) + lparams = sim.modules['Labour'].parameters pparams = sim.modules['PregnancySupervisor'].parameters @@ -250,6 +246,7 @@ def test_analysis_analysis_events_run_as_expected_when_using_sensitivity_max_par pnc_avail_prob = 1.0 lparams['pnc_availability_probability'] = pnc_avail_prob + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) p_current_params = sim.modules['PregnancySupervisor'].current_parameters @@ -285,7 +282,6 @@ def test_analysis_analysis_events_run_as_expected_when_using_sensitivity_max_par def test_analysis_analysis_events_run_as_expected_when_using_sensitivity_min_parameters(seed): sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) lparams = sim.modules['Labour'].parameters pparams = sim.modules['PregnancySupervisor'].parameters @@ -300,6 +296,7 @@ def test_analysis_analysis_events_run_as_expected_when_using_sensitivity_min_par pnc_avail_prob = 0.0 lparams['pnc_availability_probability'] = pnc_avail_prob + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) p_current_params = sim.modules['PregnancySupervisor'].current_parameters @@ -328,7 +325,6 @@ def test_analysis_events_force_availability_of_consumables_when_scheduled_in_anc via some pre-defined analysis parameter and not via the health system within the ANC HSIs""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) # Set the analysis event to run on the first day of the simulation pparams = sim.modules['PregnancySupervisor'].parameters @@ -340,6 +336,7 @@ def test_analysis_events_force_availability_of_consumables_when_scheduled_in_anc cparams['sensitivity_blood_test_syphilis'] = [1.0, 1.0] cparams['specificity_blood_test_syphilis'] = [1.0, 1.0] + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) # check the event ran @@ -411,7 +408,6 @@ def test_analysis_events_force_availability_of_consumables_for_sba_analysis(seed via some pre-defined analysis parameter and not via the health system within the SBA HSIs""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) # Set the analysis event to run at simulation start lparams = sim.modules['Labour'].parameters @@ -423,6 +419,7 @@ def test_analysis_events_force_availability_of_consumables_for_sba_analysis(seed lparams['bemonc_availability'] = 1.0 lparams['cemonc_availability'] = 1.0 + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) params = sim.modules['Labour'].current_parameters @@ -547,7 +544,6 @@ def test_analysis_events_force_availability_of_consumables_for_pnc_analysis(seed via some pre-defined analysis parameter and not via the health system within the PNC HSIs""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) # Set the analysis event to run at simulation start lparams = sim.modules['Labour'].parameters @@ -558,6 +554,7 @@ def test_analysis_events_force_availability_of_consumables_for_pnc_analysis(seed # Set availability lparams['pnc_availability_probability'] = 1.0 + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) params = sim.modules['Labour'].current_parameters @@ -615,7 +612,6 @@ def test_analysis_events_force_availability_of_consumables_for_newborn_hsi(seed) via some pre-defined analysis parameter and not via the health system within the newborn HSIs""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) # Set the analysis event to run at simulation start lparams = sim.modules['Labour'].parameters @@ -628,6 +624,7 @@ def test_analysis_events_force_availability_of_consumables_for_newborn_hsi(seed) lparams['pnc_availability_probability'] = 1.0 lparams['bemonc_availability'] = 1.0 + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) df = sim.population.props @@ -685,7 +682,6 @@ def test_analysis_events_circumnavigates_sf_and_competency_parameters(seed): functions can run""" sim = Simulation(start_date=start_date, seed=seed) sim.register(*fullmodel(resourcefilepath=resourcefilepath)) - sim.make_initial_population(n=100) # Set the analysis event to run at simulation start lparams = sim.modules['Labour'].parameters @@ -697,6 +693,7 @@ def test_analysis_events_circumnavigates_sf_and_competency_parameters(seed): lparams['bemonc_availability'] = 1.0 lparams['cemonc_availability'] = 1.0 + sim.make_initial_population(n=100) sim.simulate(end_date=Date(2010, 1, 2)) params = sim.modules['Labour'].current_parameters From 84f826322ba13f6fa1631d639944c2bac50667f6 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:55:03 +0000 Subject: [PATCH 085/103] Correctly retrieve event name --- src/tlo/events.py | 12 ++++++------ src/tlo/methods/hsi_event.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index ba8024f621..f67b54458a 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -97,7 +97,7 @@ def compare_population_dataframe(self,df_before, df_after): # First add event info link_info = { 'person_ID': idx, - 'event': str(self), + 'event': type(self).__name__, 'event_date': self.sim.date, } @@ -136,7 +136,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target - row['event'] = str(self) + row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'Before' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) @@ -164,7 +164,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> link_info = { #'person_ID' : self.target, 'person_ID' : self.target, - 'event' : str(self), + 'event' : type(self).__name__, 'event_date' : self.sim.date, } # Store (if any) property changes as a result of the event for this individual @@ -179,7 +179,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> # Print entire row row = self.sim.population.props.loc[[abs(self.target)]] # Use abs to avoid potentil issue with direct births row['person_ID'] = self.target - row['event'] = str(self) + row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'After' self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) @@ -202,13 +202,13 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> indices = change.index new_rows_before = df_before.loc[indices] new_rows_before['person_ID'] = new_rows_before.index - new_rows_before['event'] = self + new_rows_before['event'] = type(self).__name__ new_rows_before['event_date'] = self.sim.date new_rows_before['when'] = 'Before' new_rows_after = df_after.loc[indices] new_rows_after['person_ID'] = new_rows_after.index - new_rows_after['event'] = self + new_rows_after['event'] = type(self).__name__ new_rows_after['event_date'] = self.sim.date new_rows_after['when'] = 'After' diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index f267181b56..978b26d7c5 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -222,7 +222,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target - row['event'] = str(self) + row['event'] = type(self).__name__ #str(self.event_name) row['event_date'] = self.sim.date row['when'] = 'Before' @@ -268,7 +268,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> link_info = { 'person_ID': self.target, - 'event' : str(self), + 'event' : type(self).__name__, 'event_date' : self.sim.date, 'appt_footprint' : record_footprint, 'level' : record_level, @@ -285,7 +285,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target - row['event'] = str(self) + row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'After' row['appt_footprint'] = record_footprint From a490d1995c12ac20beda2fbd16271d22f0e4f8fe Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 20 Jan 2025 11:34:02 +0000 Subject: [PATCH 086/103] Modify scenario file such that can exclude specific services, and corrected analysis file such as for small number of cases where the DALYs are not explicitly resolved the average DALYs are still computed correctly [skip ci] --- .../analysis_extract_data.py | 105 ++++++++++-------- .../scenario_generate_chains.py | 58 +++++++--- 2 files changed, 103 insertions(+), 60 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 4c8e7d8197..3afad7adcc 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -16,6 +16,9 @@ from collections import Counter import ast +# Time simulated to collect data +start_date = Date(2010, 1, 1) +end_date = start_date + pd.DateOffset(months=13) # Range of years considered min_year = 2010 @@ -25,6 +28,13 @@ def all_columns(_df): return pd.Series(_df.all()) +def check_if_beyond_time_range_considered(progression_properties): + matching_keys = [key for key in progression_properties.keys() if "rt_date_to_remove_daly" in key] + if matching_keys: + for key in matching_keys: + if progression_properties[key] > end_date: + print("Beyond time range considered, need at least ",progression_properties[key]) + def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None, ): """Produce standard set of plots describing the effect of each TREATMENT_ID. - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. @@ -44,19 +54,21 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No initial_properties_of_interest = ['rt_MAIS_military_score','rt_ISS_score','rt_disability','rt_polytrauma','rt_injury_1','rt_injury_2','rt_injury_3','rt_injury_4','rt_injury_5','rt_injury_6', 'rt_imm_death','sy_injury','sy_severe_trauma','sex','li_urban', 'li_wealth', 'li_mar_stat', 'li_in_ed', 'li_ed_lev'] # Will be added through computation: age at time of RTI - # Will be added through computation: total duration of event initial_rt_event_properties = set() - + num_individuals = 1000 num_runs = 50 record = [] - + # Include results folder in output file name + name_tag = str(results_folder).replace("outputs/", "") + + for p in range(0,num_individuals): - print("At person = ", p) + print("At person = ", p, " out of ", num_individuals) individual_event_chains = extract_results( results_folder, @@ -66,51 +78,41 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No do_scaling=False ) - #print(individual_event_chains) - - for r in range(0,num_runs): - - - initial_properties = {} - progression_properties = {} key_first_event = {} key_last_event = {} first_event = {} last_event = {} properties = {} average_disability = 0 + total_dt_included = 0 + dt_in_prev_disability = 0 prev_disability_incurred = 0 - - #ind_Counter = Counter() ind_Counter = {'0': Counter(), '1a': Counter(), '1b' : Counter(), '2' : Counter()} # Count total appts list_for_individual = [] for item,row in individual_event_chains.iterrows(): value = individual_event_chains.loc[item,(0, r)] - # print("The value is", value, "at run ", r) if value !='' and isinstance(value, str): evaluated = eval(value, eval_env) list_for_individual.append(evaluated) - # elif not isinstance(value,str): - # print(value) + # These are the properties of the individual before the start of the chain of events initial_properties = list_for_individual[0] - # print(initial_properties) # Initialise first event by gathering parameters of interest from initial_properties first_event = {key: initial_properties[key] for key in initial_properties_of_interest if key in initial_properties} + # The changing or adding of properties from the first_event will be stored in progression_properties progression_properties = {} + for i in list_for_individual: + # Skip the initial_properties, or in other words only consider these if they are 'proper' events if 'event' in i: - #print("") #print(i) if 'RTIPolling' in i['event']: - #print("I'm in polling event") - #print(i) # Keep track of which properties are changed during polling events for key,value in i.items(): @@ -130,67 +132,80 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No progression_properties = initial_properties.copy() progression_properties.update(i) - # dalys incurred + # Initialise chain of Dalys incurred if 'rt_disability' in i: prev_disability_incurred = i['rt_disability'] prev_date = i['event_date'] - #print('At polling event, ', prev_disability_incurred, prev_date) else: # Progress properties of individual, even if this event is a death progression_properties.update(i) - # If disability has changed as a result of this, recalculate - if 'rt_disability' in i and i['rt_disability'] != prev_disability_incurred: + # If disability has changed as a result of this, recalculate and add previous to rolling average + if 'rt_disability' in i: + dt_in_prev_disability = (i['event_date'] - prev_date).days + #print("Detected change in disability", i['rt_disability'], "after dt=", dt_in_prev_disability) + #print("Adding the following to the average", prev_disability_incurred, " x ", dt_in_prev_disability ) average_disability += prev_disability_incurred*dt_in_prev_disability + total_dt_included += dt_in_prev_disability # Update variables prev_disability_incurred = i['rt_disability'] prev_date = i['event_date'] - - - #print(progression_properties) - # Update footprint + # Update running footprint if 'appt_footprint' in i and i['appt_footprint'] != 'Counter()': footprint = i['appt_footprint'] if 'Counter' in footprint: footprint = footprint[len("Counter("):-1] apply = eval(footprint, eval_env) ind_Counter[i['level']].update(Counter(apply)) - + + # If the individual has died, ensure chain of event is interrupted here and update rolling average of DALYs if 'is_alive' in i and i['is_alive'] is False: - #print("Death", i) - #print("-------Total footprint", ind_Counter) + if ((i['event_date'] - polling_event['rt_date_inj']).days) > total_dt_included: + dt_in_prev_disability = (i['event_date'] - prev_date).days + average_disability += prev_disability_incurred*dt_in_prev_disability + total_dt_included += dt_in_prev_disability break - - + + # check_if_beyond_time_range_considered(progression_properties) + # Compute final properties of individual key_last_event['is_alive_after_RTI'] = progression_properties['is_alive'] key_last_event['duration_days'] = (progression_properties['event_date'] - polling_event['rt_date_inj']).days - if not key_first_event['rt_imm_death'] and key_last_event['duration_days']> 0.0: + + # If individual didn't die and the key_last_event didn't result in a final change in DALYs, ensure that the last change is recorded here + if not key_first_event['rt_imm_death'] and (total_dt_included < key_last_event['duration_days']): + #print("Number of events", len(list_for_individual)) + #for i in list_for_individual: + # if 'event' in i: + # print(i) + dt_in_prev_disability = (progression_properties['event_date'] - prev_date).days + average_disability += prev_disability_incurred*dt_in_prev_disability + total_dt_included += dt_in_prev_disability + + # Now calculate the average disability incurred, and store any permanent disability and total footprint + if not key_first_event['rt_imm_death'] and key_last_event['duration_days']> 0: key_last_event['rt_disability_average'] = average_disability/key_last_event['duration_days'] else: key_last_event['rt_disability_average'] = 0.0 + key_last_event['rt_disability_permanent'] = progression_properties['rt_disability'] key_last_event.update({'total_footprint': ind_Counter}) - #print("Average disability", key_last_event['rt_disability_average']) + if key_last_event['duration_days']!=total_dt_included: + print("The duration of event and total_dt_included don't match", key_last_event['duration_days'], total_dt_included) + exit(-1) properties = key_first_event | key_last_event - - if not key_first_event['rt_imm_death'] and ((properties['rt_disability_average']-properties['rt_disability'])/properties['rt_disability'] > 1e-4): - print("Error in computed average for individual ", p, r ) record.append(properties) - #for key, value in properties.items(): - #if 'rt_' in key or 'alive' in key or 'event_date' in key or 'footprint' in key: - #print(f"{key}: {value}") - # print("Initial event properties", initial_rt_event_properties) - - df = pd.DataFrame(record) - df.to_csv("raw_data.csv", index=False) + + df = pd.DataFrame(record) + df.to_csv("new_raw_data_" + name_tag + ".csv", index=False) + print(df) print(initial_rt_event_properties) exit(-1) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 79df3f55b6..822bf13ad8 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -18,7 +18,7 @@ import pandas as pd from tlo import Date, logging -from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios, get_filtered_treatment_ids from tlo.methods.fullmodel import fullmodel from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher from tlo.scenario import BaseScenario @@ -92,7 +92,35 @@ def modules(self): # fullmodel(resourcefilepath=self.resources) # + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] # ) + """ + def draw_parameters(self, draw_number, rng): + return mix_scenarios( + get_parameters_for_status_quo(), + { + 'HealthSystem': { + 'Service_Availability': list(self._scenarios.values())[draw_number], + }, + } + ) + def _get_scenarios(self) -> Dict[str, list[str]]: + Return the Dict with values for the parameter `Service_Availability` keyed by a name for the scenario. + The sequences of scenarios systematically omits one of the TREATMENT_ID's that is defined in the model. + + # Generate list of TREATMENT_IDs and filter to the resolution needed + treatments = get_filtered_treatment_ids(depth=2) + treatments_RTI = [item for item in treatments if 'Rti' in item] + + # Return 'Service_Availability' values, with scenarios for everything, nothing, and ones for which each + # treatment is omitted + service_availability = dict({"Everything": ["*", "Nothing": []}) + #service_availability.update( + # {f"No {t.replace('_*', '*')}": [x for x in treatments if x != t] for t in treatments_RTI} + #) + + return service_availability + + """ def draw_parameters(self, draw_number, rng): if draw_number < self.number_of_draws: return list(self._scenarios.values())[draw_number] @@ -107,20 +135,27 @@ def draw_parameters(self, draw_number, rng): # case 6: gfHE = 0.030, factor = 1.07326 def _get_scenarios(self) -> Dict[str, Dict]: - """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario. - """ + #Return the Dict with values for the parameters that are changed, keyed by a name for the scenario. + + treatments = get_filtered_treatment_ids(depth=2) + treatments_RTI = [item for item in treatments if 'Rti' in item] - self.YEAR_OF_CHANGE = 2019 + # Return 'Service_Availability' values, with scenarios for everything, nothing, and ones for which each + # treatment is omitted + service_availability = dict({"Everything": ["*"], "Nothing": []}) + service_availability.update( + {f"No {t.replace('_*', '*')}": [x for x in treatments if x != t] for t in treatments_RTI} + ) + print(service_availability.keys()) return { - # =========== STATUS QUO ============ "Baseline": mix_scenarios( self._baseline(), { "HealthSystem": { - "yearly_HR_scaling_mode": "no_scaling", + "Service_Availability": service_availability["No Rti_BurnManagement*"], }, } ), @@ -128,20 +163,13 @@ def _get_scenarios(self) -> Dict[str, Dict]: } def _baseline(self) -> Dict: - """Return the Dict with values for the parameter changes that define the baseline scenario. """ + #Return the Dict with values for the parameter changes that define the baseline scenario. return mix_scenarios( get_parameters_for_status_quo(), { "HealthSystem": { "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration - "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH - "year_mode_switch": self.YEAR_OF_CHANGE, - "scale_to_effective_capabilities": True, - "policy_name": "Naive", - "tclose_overwrite": 1, - "tclose_days_offset_overwrite": 7, - "use_funded_or_actual_staffing": "actual", - "cons_availability": "default", + "cons_availability": "all", } }, ) From 08a5d9a29c9e2e8af7832ca49bfca1cb75f6d8d6 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Sat, 12 Apr 2025 11:34:07 +0100 Subject: [PATCH 087/103] Change seed in scenario file --- .../analysis_data_generation/scenario_generate_chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 822bf13ad8..3bc75978d2 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -51,7 +51,7 @@ class GenerateDataChains(BaseScenario): def __init__(self): super().__init__() - self.seed = 0 + self.seed = 42 self.start_date = Date(2010, 1, 1) self.end_date = self.start_date + pd.DateOffset(months=13) self.pop_size = 1000 From 3dda343f65c49e429c677b89d1536531fa83833a Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 14 Apr 2025 18:06:08 +0200 Subject: [PATCH 088/103] latest scenario --- .../analysis_data_generation/scenario_generate_chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 3bc75978d2..1297c6b18b 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -155,7 +155,7 @@ def _get_scenarios(self) -> Dict[str, Dict]: self._baseline(), { "HealthSystem": { - "Service_Availability": service_availability["No Rti_BurnManagement*"], + "Service_Availability": service_availability["No Rti_FractureCast*"], }, } ), From d9e3f66138c0e372b2b0fa0ac10e7393457bcaf8 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:35:47 +0100 Subject: [PATCH 089/103] Latest scenario version --- .../analysis_data_generation/scenario_generate_chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 1297c6b18b..b4ad946154 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -155,7 +155,7 @@ def _get_scenarios(self) -> Dict[str, Dict]: self._baseline(), { "HealthSystem": { - "Service_Availability": service_availability["No Rti_FractureCast*"], + "Service_Availability": service_availability["No Rti_MinorSurgeries*"], }, } ), From ddf6f689b6b9184e3f09ac1906417e6fa0495a7f Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 29 Apr 2025 15:44:41 +0100 Subject: [PATCH 090/103] Latest version of scenario file --- .../analysis_data_generation/scenario_generate_chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index b4ad946154..35b7d75e1c 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -155,7 +155,7 @@ def _get_scenarios(self) -> Dict[str, Dict]: self._baseline(), { "HealthSystem": { - "Service_Availability": service_availability["No Rti_MinorSurgeries*"], + "Service_Availability": service_availability["No Rti_ShockTreatment*"], }, } ), From 0e38408d5e37ccb4f894bb89c4d3c93673ae09a3 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 9 Oct 2025 09:20:35 +0100 Subject: [PATCH 091/103] Ensure changes to mni dataframe are captured as well --- .../scenario_generate_chains.py | 30 ++-- src/tlo/events.py | 164 ++++++++++++++++-- src/tlo/methods/hsi_event.py | 112 ++++++++---- src/tlo/methods/pregnancy_helper_functions.py | 50 +----- src/tlo/methods/pregnancy_supervisor.py | 50 ++++++ src/tlo/methods/rti.py | 4 +- src/tlo/simulation.py | 13 +- src/tlo/util.py | 2 +- 8 files changed, 314 insertions(+), 111 deletions(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 35b7d75e1c..64fa70d055 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -53,11 +53,11 @@ def __init__(self): super().__init__() self.seed = 42 self.start_date = Date(2010, 1, 1) - self.end_date = self.start_date + pd.DateOffset(months=13) + self.end_date = self.start_date + pd.DateOffset(months=36) self.pop_size = 1000 self._scenarios = self._get_scenarios() self.number_of_draws = len(self._scenarios) - self.runs_per_draw = 50 + self.runs_per_draw = 1 self.generate_event_chains = True def log_configuration(self): @@ -77,21 +77,31 @@ def log_configuration(self): def modules(self): # MODIFY # Here instead of running full module + """ return [demography.Demography(resourcefilepath=self.resources), enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), healthburden.HealthBurden(resourcefilepath=self.resources), - symptommanager.SymptomManager(resourcefilepath=self.resources, spurious_symptoms=False), - rti.RTI(resourcefilepath=self.resources), + symptommanager.SymptomManager(resourcefilepath=self.resources, spurious_symptoms=False),#, + #rti.RTI(resourcefilepath=self.resources), + pregnancy_supervisor.PregnancySupervisor(resourcefilepath=self.resources), + labour.Labour(resourcefilepath=self.resources), + care_of_women_during_pregnancy.CareOfWomenDuringPregnancy(resourcefilepath=self.resources), + contraception.Contraception(resourcefilepath=self.resources), + newborn_outcomes.NewbornOutcomes(resourcefilepath=self.resources), + postnatal_supervisor.PostnatalSupervisor(resourcefilepath=self.resources), + hiv.Hiv(resourcefilepath=self.resources), + tb.Tb(resourcefilepath=self.resources), + epi.Epi(resourcefilepath=self.resources), healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources), - #simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + #simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), healthsystem.HealthSystem(resourcefilepath=self.resources, mode_appt_constraints=1, cons_availability='all')] - - # return ( - # fullmodel(resourcefilepath=self.resources) - # + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] - # ) + """ + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) """ def draw_parameters(self, draw_number, rng): return mix_scenarios( diff --git a/src/tlo/events.py b/src/tlo/events.py index f67b54458a..3a8f4f58c7 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -13,6 +13,7 @@ from tlo.util import FACTOR_POP_DICT +import copy logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -76,23 +77,85 @@ def apply(self, target): """ raise NotImplementedError - def compare_population_dataframe(self,df_before, df_after): + def values_differ(self, v1, v2): + + if isinstance(v1, list) and isinstance(v2, list): + return v1 != v2 # simple element-wise comparison + + if pd.isna(v1) and pd.isna(v2): + return False # treat both NaT/NaN as equal + return v1 != v2 + + def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): + diffs = {} + """ + will_pause = False + + target_attribute = 'hcw_not_avail' + if len(entire_mni_after)>0: + print("Default target value before", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) + person = next(iter(entire_mni_after)) + entire_mni_after[person][target_attribute] = True + will_pause = True + print("Default target value after", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) + + + if will_pause: + print("Reprint") + print(entire_mni_before) + print(entire_mni_after) + print("Default target value", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) + """ + all_individuals = set(entire_mni_before.keys()) | set(entire_mni_after.keys()) + + for person in all_individuals: + if person not in entire_mni_before: # but is afterward + for key in entire_mni_after[person]: + if self.values_differ(entire_mni_after[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if person not in diffs: + diffs[person] = {} + diffs[person][key] = entire_mni_after[person][key] + + elif person not in entire_mni_after: # but is beforehand + for key in entire_mni_before[person]: + if self.values_differ(entire_mni_before[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if person not in diffs: + diffs[person] = {} + diffs[person][key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] + + else: # person is in both + # Compare properties + for key in entire_mni_before[person]: + if self.values_differ(entire_mni_before[person][key],entire_mni_after[person][key]): + if person not in diffs: + diffs[person] = {} + diffs[person][key] = entire_mni_after[person][key] + + if len(diffs)>0: + print("DIfferences for ", diffs) + return diffs + + def compare_population_dataframe(self,df_before, df_after, entire_mni_before, entire_mni_after): """ This function compares the population dataframe before/after a population-wide event has occurred. It allows us to identify the individuals for which this event led to a significant (i.e. property) change, and to store the properties which have changed as a result of it. """ # Create a mask of where values are different diff_mask = (df_before != df_after) & ~(df_before.isna() & df_after.isna()) + + diff_mni = self.compare_entire_mni_dicts(entire_mni_before, entire_mni_after) # Create an empty list to store changes for each of the individuals chain_links = {} len_of_diff = len(diff_mask) # Loop through each row of the mask + persons_changed = [] for idx, row in diff_mask.iterrows(): changed_cols = row.index[row].tolist() if changed_cols: # Proceed only if there are changes in the row + persons_changed.append(idx) # Create a dictionary for this person # First add event info link_info = { @@ -104,19 +167,47 @@ def compare_population_dataframe(self,df_before, df_after): # Store the new values from df_after for the changed columns for col in changed_cols: link_info[col] = df_after.at[idx, col] - + + if idx in diff_mni: + # This person has also undergone changes in the mni dictionary, so add these here + for key in diff_mni[idx]: + link_info[col] = diff_mni[idx][key] + # Append the event and changes to the individual key chain_links[idx] = str(link_info) - + + # Check individuals + if len(diff_mni)>0: + print("Non-zero changes in mni") + for key in diff_mni: + if key not in persons_changed: + print("Individual ", key, "is changing in mni alone") + # If individual hadn't been previously added due to changes in pop df, add it here + link_info = { + 'person_ID': key, + 'event': type(self).__name__, + 'event_date': self.sim.date, + } + + for key_prop in diff_mni[key]: + link_info[key_prop] = diff_mni[key][key_prop] + + chain_links[key] = str(link_info) + print("Change for ", key, " is ", str(link_info)) + return chain_links - def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame]: - """ This function checks whether this event should be logged as part of the event chains, and if so stored required information before the event has occurred. """ + def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame, dict, dict, bool]: + """ This function checks whether this event should be logged as part of the event chains, and if so stored required information before the event has occurred. """ + # Initialise these variables print_chains = False df_before = [] row_before = pd.Series() + mni_instances_before = False + mni_row_before = {} + entire_mni_before = {} # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore #if (self.module in self.sim.generate_event_chains_modules_of_interest) and .. @@ -129,9 +220,16 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame # Target is single individual if self.target != self.sim.population: + # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + + if self.target in mni: + mni_instances_before = True + mni_row_before = mni[self.target].copy() + if self.sim.debug_generate_event_chains: # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] @@ -139,6 +237,13 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'Before' + if not mni_instances_before: + for key in self.sim.modules['PregnancySupervisor'].default_mni_values: + row[key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] + else: + for key in mni_row_before: + row[key] = mni_row_before[key] + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: @@ -146,20 +251,30 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame # This will be a population-wide event. In order to find individuals for which this led to # a meaningful change, make a copy of the pop dataframe before the event has occurred. df_before = self.sim.population.props.copy() + entire_mni_before = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) - return print_chains, row_before, df_before + return print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before - def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> dict: + def store_chains_to_do_after_event(self, print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) -> dict: """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ chain_links = {} - + + if print_chains: # Target is single individual if self.target != self.sim.population: + + mni_instances_after = False + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + + if self.target in mni: + mni_instances_after = True + # Create and store event for this individual, regardless of whether any property change occurred link_info = { #'person_ID' : self.target, @@ -167,11 +282,35 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> 'event' : type(self).__name__, 'event_date' : self.sim.date, } + # Store (if any) property changes as a result of the event for this individual for key in row_before.index: if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] + # Now store changes in the mni dictionary, accounting for following cases: + + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in mni_row_before: + if self.values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + print("INDIVIDUAL WAS ADDED") + exit(-1) + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in default: + if self.values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Else, no need to do anything + chain_links[self.target] = str(link_info) # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. @@ -182,6 +321,7 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'After' + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: @@ -190,9 +330,10 @@ def store_chains_to_do_after_event(self, print_chains, row_before, df_before) -> # Population frame after event df_after = self.sim.population.props + entire_mni_after = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) # Create and store the event and dictionary of changes for affected individuals - chain_links = self.compare_population_dataframe(df_before, df_after) + chain_links = self.compare_population_dataframe(df_before, df_after, entire_mni_before, entire_mni_after) # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. if self.sim.debug_generate_event_chains: @@ -222,7 +363,7 @@ def run(self): # Collect relevant information before event takes place if self.sim.generate_event_chains: - print_chains, row_before, df_before = self.store_chains_to_do_before_event() + print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before = self.store_chains_to_do_before_event() self.apply(self.target) self.post_apply_hook() @@ -230,7 +371,7 @@ def run(self): # Collect event info + meaningful property changes of individuals. Combined, these will constitute a 'link' # in the individual's event chain. if self.sim.generate_event_chains: - chain_links = self.store_chains_to_do_after_event(print_chains, row_before, df_before) + chain_links = self.store_chains_to_do_after_event(print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) # Create empty logger for entire pop pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals @@ -238,6 +379,7 @@ def run(self): # Log chain_links here if len(chain_links)>0: + print(chain_links) logger_chain.info(key='event_chains', data= pop_dict, description='Links forming chains of events for simulated individuals') diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 978b26d7c5..41342f117e 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -195,65 +195,83 @@ def _run_after_hsi_event(self) -> None: item_codes=self._EQUIPMENT, facility_id=self.facility_info.id ) - - def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series]: + + def values_differ(self, v1, v2): + + if isinstance(v1, list) and isinstance(v2, list): + return v1 != v2 # simple element-wise comparison + + if pd.isna(v1) and pd.isna(v2): + return False # treat both NaT/NaN as equal + return v1 != v2 + + + def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, dict, bool]: """ This function checks whether this event should be logged as part of the event chains, and if so stored required information before the event has occurred. """ # Initialise these variables print_chains = False row_before = pd.Series() + mni_instances_before = False + mni_row_before = {} # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - # if (self.module in self.sim.generate_event_chains_modules_of_interest) and + #if (self.module in self.sim.generate_event_chains_modules_of_interest) and .. if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): # Will eventually use this once I can actually GET THE NAME OF THE SELF - # if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): - + #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): + + print_chains = True + + # Target is single individual if self.target != self.sim.population: - # In the case of HSI events, only individual events should exist and therefore be logged - print_chains = True - # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) - + + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + + if self.target in mni: + mni_instances_before = True + mni_row_before = mni[self.target].copy() + if self.sim.debug_generate_event_chains: # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. row = self.sim.population.props.loc[[abs(self.target)]] row['person_ID'] = self.target - row['event'] = type(self).__name__ #str(self.event_name) + row['event'] = type(self).__name__ row['event_date'] = self.sim.date row['when'] = 'Before' - - try: - row['appt_footprint'] = str(self.EXPECTED_APPT_FOOTPRINT) - row['level'] = self.facility_info.level - except: - row['appt_footprint'] = 'N/A' - row['level'] = 'N/A' + if not mni_instances_before: + for key in self.sim.modules['PregnancySupervisor'].default_mni_values: + row[key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] + else: + for key in mni_row_before: + row[key] = mni_row_before[key] + self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: - # Once this has been removed from Chronic Syndrome mock module, make this a Runtime Error - # raise RuntimeError("Cannot have population-wide HSI events") - logger.debug( - key="message", - data=( - "Cannot have population-wide HSI events" - ), - ) - + print("ERROR: there shouldn't be pop-wide HSI event") - return print_chains, row_before + return print_chains, row_before, mni_row_before, mni_instances_before - def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> dict: + def store_chains_to_do_after_event(self, print_chains, row_before, footprint, mni_row_before, mni_instances_before) -> dict: """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ if print_chains: # For HSI event, this will only ever occur for individual events - + chain_links = {} + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) + mni_instances_after = False + + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + + if self.target in mni: + mni_instances_after = True + # Create and store dictionary of changes. Note that person_ID, event, event_date, appt_foot, and level # will be stored regardless of whether individual experienced property changes. @@ -278,8 +296,35 @@ def store_chains_to_do_after_event(self, print_chains, row_before, footprint) -> for key in row_before.index: if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] - - chain_links = {self.target : str(link_info)} + + # Now store changes in the mni dictionary, accounting for following cases: + + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + print("--------------------------------------------->",link_info[key]) + exit(-1) + + + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in mni_row_before: + if self.values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + print("--------------------------------------------->",link_info[key]) + exit(-1) + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in default: + if self.values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + print("--------------------------------------------->",link_info[key]) + exit(-1) + chain_links[self.target] = str(link_info) if self.sim.debug_generate_event_chains: # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. @@ -300,7 +345,7 @@ def run(self, squeeze_factor): if self.sim.generate_event_chains and self.target != self.sim.population: - print_chains, row_before = self.store_chains_to_do_before_event() + print_chains, row_before, mni_row_before, mni_instances_before = self.store_chains_to_do_before_event() footprint = self.EXPECTED_APPT_FOOTPRINT @@ -315,10 +360,9 @@ def run(self, squeeze_factor): if updated_appt_footprint is not None: footprint = updated_appt_footprint - chain_links = self.store_chains_to_do_after_event(print_chains, row_before, str(footprint)) + chain_links = self.store_chains_to_do_after_event(print_chains, row_before, str(footprint), mni_row_before, mni_instances_before) if len(chain_links)>0: - pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # pop_dict = {i: '' for i in range(1000)} # Always include all possible individuals diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 8f7faa0503..79483cddaa 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -542,55 +542,7 @@ def update_mni_dictionary(self, individual_id): if self == self.sim.modules['PregnancySupervisor']: - mni[individual_id] = {'delete_mni': False, # if True, mni deleted in report_daly_values function - 'didnt_seek_care': False, - 'cons_not_avail': False, - 'comp_not_avail': False, - 'hcw_not_avail': False, - 'ga_anc_one': 0, - 'anc_ints': [], - 'abortion_onset': pd.NaT, - 'abortion_haem_onset': pd.NaT, - 'abortion_sep_onset': pd.NaT, - 'eclampsia_onset': pd.NaT, - 'mild_mod_aph_onset': pd.NaT, - 'severe_aph_onset': pd.NaT, - 'chorio_onset': pd.NaT, - 'chorio_in_preg': False, # use in predictor in newborn linear models - 'ectopic_onset': pd.NaT, - 'ectopic_rupture_onset': pd.NaT, - 'gest_diab_onset': pd.NaT, - 'gest_diab_diagnosed_onset': pd.NaT, - 'gest_diab_resolution': pd.NaT, - 'mild_anaemia_onset': pd.NaT, - 'mild_anaemia_resolution': pd.NaT, - 'moderate_anaemia_onset': pd.NaT, - 'moderate_anaemia_resolution': pd.NaT, - 'severe_anaemia_onset': pd.NaT, - 'severe_anaemia_resolution': pd.NaT, - 'mild_anaemia_pp_onset': pd.NaT, - 'mild_anaemia_pp_resolution': pd.NaT, - 'moderate_anaemia_pp_onset': pd.NaT, - 'moderate_anaemia_pp_resolution': pd.NaT, - 'severe_anaemia_pp_onset': pd.NaT, - 'severe_anaemia_pp_resolution': pd.NaT, - 'hypertension_onset': pd.NaT, - 'hypertension_resolution': pd.NaT, - 'obstructed_labour_onset': pd.NaT, - 'sepsis_onset': pd.NaT, - 'uterine_rupture_onset': pd.NaT, - 'mild_mod_pph_onset': pd.NaT, - 'severe_pph_onset': pd.NaT, - 'secondary_pph_onset': pd.NaT, - 'vesicovaginal_fistula_onset': pd.NaT, - 'vesicovaginal_fistula_resolution': pd.NaT, - 'rectovaginal_fistula_onset': pd.NaT, - 'rectovaginal_fistula_resolution': pd.NaT, - 'test_run': False, # used by labour module when running some model tests - 'pred_syph_infect': pd.NaT, # date syphilis is predicted to onset - 'new_onset_spe': False, - 'cs_indication': 'none' - } + mni[individual_id] = self.sim.modules['PregnancySupervisor'].default_mni_values.copy() elif self == self.sim.modules['Labour']: labour_variables = {'labour_state': None, diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7dd8819ab6..f634d9b971 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -61,6 +61,56 @@ def __init__(self, name=None, resourcefilepath=None): # This variable will store a Bitset handler for the property ps_abortion_complications self.abortion_complications = None + + self.default_mni_values = {'delete_mni': False, # if True, mni deleted in report_daly_values function + 'didnt_seek_care': False, + 'cons_not_avail': False, + 'comp_not_avail': False, + 'hcw_not_avail': False, + 'ga_anc_one': 0, + 'anc_ints': [], + 'abortion_onset': pd.NaT, + 'abortion_haem_onset': pd.NaT, + 'abortion_sep_onset': pd.NaT, + 'eclampsia_onset': pd.NaT, + 'mild_mod_aph_onset': pd.NaT, + 'severe_aph_onset': pd.NaT, + 'chorio_onset': pd.NaT, + 'chorio_in_preg': False, # use in predictor in newborn linear models + 'ectopic_onset': pd.NaT, + 'ectopic_rupture_onset': pd.NaT, + 'gest_diab_onset': pd.NaT, + 'gest_diab_diagnosed_onset': pd.NaT, + 'gest_diab_resolution': pd.NaT, + 'mild_anaemia_onset': pd.NaT, + 'mild_anaemia_resolution': pd.NaT, + 'moderate_anaemia_onset': pd.NaT, + 'moderate_anaemia_resolution': pd.NaT, + 'severe_anaemia_onset': pd.NaT, + 'severe_anaemia_resolution': pd.NaT, + 'mild_anaemia_pp_onset': pd.NaT, + 'mild_anaemia_pp_resolution': pd.NaT, + 'moderate_anaemia_pp_onset': pd.NaT, + 'moderate_anaemia_pp_resolution': pd.NaT, + 'severe_anaemia_pp_onset': pd.NaT, + 'severe_anaemia_pp_resolution': pd.NaT, + 'hypertension_onset': pd.NaT, + 'hypertension_resolution': pd.NaT, + 'obstructed_labour_onset': pd.NaT, + 'sepsis_onset': pd.NaT, + 'uterine_rupture_onset': pd.NaT, + 'mild_mod_pph_onset': pd.NaT, + 'severe_pph_onset': pd.NaT, + 'secondary_pph_onset': pd.NaT, + 'vesicovaginal_fistula_onset': pd.NaT, + 'vesicovaginal_fistula_resolution': pd.NaT, + 'rectovaginal_fistula_onset': pd.NaT, + 'rectovaginal_fistula_resolution': pd.NaT, + 'test_run': False, # used by labour module when running some model tests + 'pred_syph_infect': pd.NaT, # date syphilis is predicted to onset + 'new_onset_spe': False, + 'cs_indication': 'none' + } INIT_DEPENDENCIES = {'Demography'} diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index c79b26314d..e772366d57 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -2865,9 +2865,9 @@ def apply(self, population): Predictor('li_ex_alc').when(True, self.rr_injrti_excessalcohol) ) #if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi is True: - pred = 1.0 + #pred = 1.0 #else: - # pred = eq.predict(df.loc[rt_current_non_ind]) + pred = eq.predict(df.loc[rt_current_non_ind]) random_draw_in_rti = self.module.rng.random_sample(size=len(rt_current_non_ind)) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index bb766562a0..045e86bdd8 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -109,7 +109,7 @@ def __init__( self.modules = OrderedDict() self.event_queue = EventQueue() self.generate_event_chains = True - self.generate_event_chains_overwrite_epi = None + self.generate_event_chains_overwrite_epi = False self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] self.debug_generate_event_chains = False @@ -299,6 +299,12 @@ def make_initial_population(self, *, n: int) -> None: if self.generate_event_chains: pop_dict = self.population.props.to_dict(orient='index') + + #if "PregnancySupervisor" in self.modules: + # print("I found it!") + # print(self.modules['PregnancySupervisor'].mother_and_newborn_info) + # exit(-1) + for key in pop_dict.keys(): pop_dict[key]['person_ID'] = key pop_dict[key] = str(pop_dict[key]) # Log as string to avoid issues around length of properties stored later @@ -329,10 +335,10 @@ def initialise(self, *, end_date: Date) -> None: #self.generate_event_chains = generate_event_chains if self.generate_event_chains: # Eventually this can be made an option - self.generate_event_chains_overwrite_epi = True + self.generate_event_chains_overwrite_epi = False # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] - self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'HealthSeekingBehaviourPoll', 'LifestyleEvent'] #['TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler'] + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'LifestyleEvent', 'TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler', 'RTIPollingEvent'] else: # If not using to print chains, cannot ignore epi self.generate_event_chains_overwrite_epi = False @@ -491,7 +497,6 @@ def do_birth(self, mother_id: int) -> int: pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals pop_dict[child_id] = str(prop_dict) # Convert to string to avoid issue of length - print("Length at birth", len(pop_dict)) logger.info(key='event_chains', data = pop_dict, description='Links forming chains of events for simulated individuals') diff --git a/src/tlo/util.py b/src/tlo/util.py index e246fcf05b..c9130e3f07 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -13,7 +13,7 @@ # Default mother_id value, assigned to individuals initialised as adults at the start of the simulation. DEFAULT_MOTHER_ID = -1e7 -FACTOR_POP_DICT = 1000 +FACTOR_POP_DICT = 50000 def create_age_range_lookup(min_age: int, max_age: int, range_size: int = 5) -> (list, Dict[int, str]): From 9b8f01ff383bdb0954146b93849c6c7a18008b2d Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 9 Oct 2025 11:24:39 +0100 Subject: [PATCH 092/103] Tidy up --- .../analysis_extract_data.py | 2 +- src/tlo/events.py | 199 +++++++----------- src/tlo/methods/hiv.py | 32 ++- src/tlo/methods/hsi_event.py | 165 ++++++--------- src/tlo/methods/tb.py | 5 +- src/tlo/simulation.py | 41 +--- 6 files changed, 151 insertions(+), 293 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 3afad7adcc..8068db203a 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -59,7 +59,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No initial_rt_event_properties = set() num_individuals = 1000 - num_runs = 50 + num_runs = 1 record = [] # Include results folder in output file name name_tag = str(results_folder).replace("outputs/", "") diff --git a/src/tlo/events.py b/src/tlo/events.py index 3a8f4f58c7..9f762fd3c6 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -77,7 +77,7 @@ def apply(self, target): """ raise NotImplementedError - def values_differ(self, v1, v2): + def mni_values_differ(self, v1, v2): if isinstance(v1, list) and isinstance(v2, list): return v1 != v2 # simple element-wise comparison @@ -111,14 +111,14 @@ def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): for person in all_individuals: if person not in entire_mni_before: # but is afterward for key in entire_mni_after[person]: - if self.values_differ(entire_mni_after[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if self.mni_values_differ(entire_mni_after[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): if person not in diffs: diffs[person] = {} diffs[person][key] = entire_mni_after[person][key] elif person not in entire_mni_after: # but is beforehand for key in entire_mni_before[person]: - if self.values_differ(entire_mni_before[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if self.mni_values_differ(entire_mni_before[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): if person not in diffs: diffs[person] = {} diffs[person][key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] @@ -126,7 +126,7 @@ def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): else: # person is in both # Compare properties for key in entire_mni_before[person]: - if self.values_differ(entire_mni_before[person][key],entire_mni_after[person][key]): + if self.mni_values_differ(entire_mni_before[person][key],entire_mni_after[person][key]): if person not in diffs: diffs[person] = {} diffs[person][key] = entire_mni_after[person][key] @@ -135,13 +135,12 @@ def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): print("DIfferences for ", diffs) return diffs - def compare_population_dataframe(self,df_before, df_after, entire_mni_before, entire_mni_after): - """ This function compares the population dataframe before/after a population-wide event has occurred. + def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_before, entire_mni_after): + """ This function compares the population dataframe and mni dictionary before/after a population-wide event has occurred. It allows us to identify the individuals for which this event led to a significant (i.e. property) change, and to store the properties which have changed as a result of it. """ # Create a mask of where values are different diff_mask = (df_before != df_after) & ~(df_before.isna() & df_after.isna()) - diff_mni = self.compare_entire_mni_dicts(entire_mni_before, entire_mni_after) # Create an empty list to store changes for each of the individuals @@ -176,12 +175,10 @@ def compare_population_dataframe(self,df_before, df_after, entire_mni_before, en # Append the event and changes to the individual key chain_links[idx] = str(link_info) - # Check individuals + # For individuals which only underwent changes in mni dictionary, save changes here if len(diff_mni)>0: - print("Non-zero changes in mni") for key in diff_mni: if key not in persons_changed: - print("Individual ", key, "is changing in mni alone") # If individual hadn't been previously added due to changes in pop df, add it here link_info = { 'person_ID': key, @@ -193,7 +190,6 @@ def compare_population_dataframe(self,df_before, df_after, entire_mni_before, en link_info[key_prop] = diff_mni[key][key_prop] chain_links[key] = str(link_info) - print("Change for ", key, " is ", str(link_info)) return chain_links @@ -210,7 +206,6 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame entire_mni_before = {} # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and .. if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): # Will eventually use this once I can actually GET THE NAME OF THE SELF @@ -224,140 +219,88 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + # Check if individual is already in mni dictionary, if so copy her original status mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: mni_instances_before = True mni_row_before = mni[self.target].copy() - - if self.sim.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.sim.population.props.loc[[abs(self.target)]] - row['person_ID'] = self.target - row['event'] = type(self).__name__ - row['event_date'] = self.sim.date - row['when'] = 'Before' - if not mni_instances_before: - for key in self.sim.modules['PregnancySupervisor'].default_mni_values: - row[key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] - else: - for key in mni_row_before: - row[key] = mni_row_before[key] - - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: # This will be a population-wide event. In order to find individuals for which this led to - # a meaningful change, make a copy of the pop dataframe before the event has occurred. + # a meaningful change, make a copy of the while pop dataframe/mni before the event has occurred. df_before = self.sim.population.props.copy() entire_mni_before = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) return print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before - def store_chains_to_do_after_event(self, print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) -> dict: + def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) -> dict: """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ chain_links = {} - - - if print_chains: - - # Target is single individual - if self.target != self.sim.population: + + # Target is single individual + if self.target != self.sim.population: + + # Copy full new status for individual + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) - mni_instances_after = False + # Check if individual is in mni after the event + mni_instances_after = False + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_after = True - row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) - - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - - if self.target in mni: - mni_instances_after = True - - # Create and store event for this individual, regardless of whether any property change occurred - link_info = { - #'person_ID' : self.target, - 'person_ID' : self.target, - 'event' : type(self).__name__, - 'event_date' : self.sim.date, - } - - # Store (if any) property changes as a result of the event for this individual - for key in row_before.index: - if row_before[key] != row_after[key]: # Note: used fillna previously - link_info[key] = row_after[key] - - # Now store changes in the mni dictionary, accounting for following cases: - - # Individual is in mni dictionary before and after - if mni_instances_before and mni_instances_after: - for key in mni_row_before: - if self.values_differ(mni_row_before[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - # Individual is only in mni dictionary before event - elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_mni_values - for key in mni_row_before: - if self.values_differ(mni_row_before[key], default[key]): - link_info[key] = default[key] - # Individual is only in mni dictionary after event - elif mni_instances_after and not mni_instances_before: - print("INDIVIDUAL WAS ADDED") - exit(-1) - default = self.sim.modules['PregnancySupervisor'].default_mni_values - for key in default: - if self.values_differ(default[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - # Else, no need to do anything - - chain_links[self.target] = str(link_info) - - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - if self.sim.debug_generate_event_chains: - # Print entire row - row = self.sim.population.props.loc[[abs(self.target)]] # Use abs to avoid potentil issue with direct births - row['person_ID'] = self.target - row['event'] = type(self).__name__ - row['event_date'] = self.sim.date - row['when'] = 'After' - - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) - - else: - # Target is entire population. Identify individuals for which properties have changed - # and store their changes. - - # Population frame after event - df_after = self.sim.population.props - entire_mni_after = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) - - # Create and store the event and dictionary of changes for affected individuals - chain_links = self.compare_population_dataframe(df_before, df_after, entire_mni_before, entire_mni_after) - - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - if self.sim.debug_generate_event_chains: - # Or print entire rows - change = df_before.compare(df_after) - if not change.empty: - indices = change.index - new_rows_before = df_before.loc[indices] - new_rows_before['person_ID'] = new_rows_before.index - new_rows_before['event'] = type(self).__name__ - new_rows_before['event_date'] = self.sim.date - new_rows_before['when'] = 'Before' - - new_rows_after = df_after.loc[indices] - new_rows_after['person_ID'] = new_rows_after.index - new_rows_after['event'] = type(self).__name__ - new_rows_after['event_date'] = self.sim.date - new_rows_after['when'] = 'After' - - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_before], ignore_index=True) - self.sim.event_chains = pd.concat([self.sim.event_chains,new_rows_after], ignore_index=True) + # Create and store event for this individual, regardless of whether any property change occurred + link_info = { + #'person_ID' : self.target, + 'person_ID' : self.target, + 'event' : type(self).__name__, + 'event_date' : self.sim.date, + } + + # Store (if any) property changes as a result of the event for this individual + for key in row_before.index: + if row_before[key] != row_after[key]: # Note: used fillna previously, so this is safe + link_info[key] = row_after[key] + + # Now check and store changes in the mni dictionary, accounting for following cases: + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.mni_values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in mni_row_before: + if self.mni_values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in default: + if self.mni_values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Else, no need to do anything + + # Add individual to the chain links + chain_links[self.target] = str(link_info) + + else: + # Target is entire population. Identify individuals for which properties have changed + # and store their changes. + + # Population frame after event + df_after = self.sim.population.props + entire_mni_after = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) + + # Create and store the event and dictionary of changes for affected individuals + chain_links = self.compare_population_dataframe_and_mni(df_before, df_after, entire_mni_before, entire_mni_after) return chain_links + def run(self): """Make the event happen.""" @@ -370,8 +313,8 @@ def run(self): # Collect event info + meaningful property changes of individuals. Combined, these will constitute a 'link' # in the individual's event chain. - if self.sim.generate_event_chains: - chain_links = self.store_chains_to_do_after_event(print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) + if self.sim.generate_event_chains and print_chains: + chain_links = self.store_chains_to_do_after_event(row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) # Create empty logger for entire pop pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals @@ -384,7 +327,7 @@ def run(self): data= pop_dict, description='Links forming chains of events for simulated individuals') - #print("Chain events ", chain_links) + print("Chain events ", chain_links) class RegularEvent(Event): diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 8487eaa467..0a80f8b41b 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -631,12 +631,11 @@ def initialise_population(self, population): df.loc[df.is_alive, "hv_date_treated"] = pd.NaT df.loc[df.is_alive, "hv_date_last_ART"] = pd.NaT - if self.sim.generate_event_chains is False or self.sim.generate_event_chains is None or self.sim.generate_event_chains_overwrite_epi is False: - # Launch sub-routines for allocating the right number of people into each category - self.initialise_baseline_prevalence(population) # allocate baseline prevalence + # Launch sub-routines for allocating the right number of people into each category + self.initialise_baseline_prevalence(population) # allocate baseline prevalence - self.initialise_baseline_art(population) # allocate baseline art coverage - self.initialise_baseline_tested(population) # allocate baseline testing coverage + self.initialise_baseline_art(population) # allocate baseline art coverage + self.initialise_baseline_tested(population) # allocate baseline testing coverage def initialise_baseline_prevalence(self, population): """ @@ -906,16 +905,10 @@ def initialise_simulation(self, sim): df = sim.population.props p = self.parameters - if self.sim.generate_event_chains is True and self.sim.generate_event_chains_overwrite_epi: - print("Should be generating data") - sim.schedule_event( - HivPollingEventForDataGeneration(self), sim.date + DateOffset(days=0) - ) - else: - # 1) Schedule the Main HIV Regular Polling Event - sim.schedule_event( - HivRegularPollingEvent(self), sim.date + DateOffset(days=0) - ) + # 1) Schedule the Main HIV Regular Polling Event + sim.schedule_event( + HivRegularPollingEvent(self), sim.date + DateOffset(days=0) + ) # 2) Schedule the Logging Event sim.schedule_event(HivLoggingEvent(self), sim.date + DateOffset(years=1)) @@ -1901,12 +1894,11 @@ def vmmc_for_child(): priority=0, ) - if self.sim.generate_event_chains is False or self.sim.generate_event_chains is None or self.sim.generate_event_chains_overwrite_epi is False: - # Horizontal transmission: Male --> Female - horizontal_transmission(from_sex="M", to_sex="F") + # Horizontal transmission: Male --> Female + horizontal_transmission(from_sex="M", to_sex="F") - # Horizontal transmission: Female --> Male - horizontal_transmission(from_sex="F", to_sex="M") + # Horizontal transmission: Female --> Male + horizontal_transmission(from_sex="F", to_sex="M") # testing # if year later than 2020, set testing rates to those reported in 2020 diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 41342f117e..dbca98da5c 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -216,8 +216,7 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, dict, bool]: mni_row_before = {} # Only print event if it belongs to modules of interest and if it is not in the list of events to ignore - #if (self.module in self.sim.generate_event_chains_modules_of_interest) and .. - if all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): + if (self.module in self.sim.generate_event_chains_modules_of_interest) and all(sub not in str(self) for sub in self.sim.generate_event_chains_ignore_events): # Will eventually use this once I can actually GET THE NAME OF THE SELF #if not set(self.sim.generate_event_chains_ignore_events).intersection(str(self)): @@ -230,112 +229,75 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, dict, bool]: # Save row for comparison after event has occurred row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) + # Check if individual is in mni dictionary before the event, if so store its original status mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: mni_instances_before = True mni_row_before = mni[self.target].copy() - - if self.sim.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.sim.population.props.loc[[abs(self.target)]] - row['person_ID'] = self.target - row['event'] = type(self).__name__ - row['event_date'] = self.sim.date - row['when'] = 'Before' - if not mni_instances_before: - for key in self.sim.modules['PregnancySupervisor'].default_mni_values: - row[key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] - else: - for key in mni_row_before: - row[key] = mni_row_before[key] - - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) else: print("ERROR: there shouldn't be pop-wide HSI event") + exit(-1) return print_chains, row_before, mni_row_before, mni_instances_before - def store_chains_to_do_after_event(self, print_chains, row_before, footprint, mni_row_before, mni_instances_before) -> dict: + def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, mni_instances_before) -> dict: """ If print_chains=True, this function logs the event and identifies and logs the any property changes that have occured to one or multiple individuals as a result of the event taking place. """ - if print_chains: - # For HSI event, this will only ever occur for individual events - chain_links = {} - row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) - - mni_instances_after = False - - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - - if self.target in mni: - mni_instances_after = True - - # Create and store dictionary of changes. Note that person_ID, event, event_date, appt_foot, and level - # will be stored regardless of whether individual experienced property changes. + # For HSI event, this will only ever occur for individual events + chain_links = {} - # Add event details + row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) + + mni_instances_after = False + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_after = True - try: - record_footprint = str(footprint) - record_level = self.facility_info.level - except: - record_footprint = 'N/A' - record_level = 'N/A' - - link_info = { - 'person_ID': self.target, - 'event' : type(self).__name__, - 'event_date' : self.sim.date, - 'appt_footprint' : record_footprint, - 'level' : record_level, - } + # Create and store dictionary of changes. Note that person_ID, event, event_date, appt_foot, and level + # will be stored regardless of whether individual experienced property changes or not. + + # Add event details + try: + record_footprint = str(footprint) + record_level = self.facility_info.level + except: + record_footprint = 'N/A' + record_level = 'N/A' - # Add changes to properties - for key in row_before.index: - if row_before[key] != row_after[key]: # Note: used fillna previously - link_info[key] = row_after[key] - - # Now store changes in the mni dictionary, accounting for following cases: + link_info = { + 'person_ID': self.target, + 'event' : type(self).__name__, + 'event_date' : self.sim.date, + 'appt_footprint' : record_footprint, + 'level' : record_level, + } + + # Add changes to properties + for key in row_before.index: + if row_before[key] != row_after[key]: # Note: used fillna previously + link_info[key] = row_after[key] - # Individual is in mni dictionary before and after - if mni_instances_before and mni_instances_after: - for key in mni_row_before: - if self.values_differ(mni_row_before[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - print("--------------------------------------------->",link_info[key]) - exit(-1) - - - # Individual is only in mni dictionary before event - elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_mni_values - for key in mni_row_before: - if self.values_differ(mni_row_before[key], default[key]): - link_info[key] = default[key] - print("--------------------------------------------->",link_info[key]) - exit(-1) - # Individual is only in mni dictionary after event - elif mni_instances_after and not mni_instances_before: - default = self.sim.modules['PregnancySupervisor'].default_mni_values - for key in default: - if self.values_differ(default[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - print("--------------------------------------------->",link_info[key]) - exit(-1) - chain_links[self.target] = str(link_info) - - if self.sim.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.sim.population.props.loc[[abs(self.target)]] - row['person_ID'] = self.target - row['event'] = type(self).__name__ - row['event_date'] = self.sim.date - row['when'] = 'After' - row['appt_footprint'] = record_footprint - row['level'] = record_level - self.sim.event_chains = pd.concat([self.sim.event_chains, row], ignore_index=True) + # Now store changes in the mni dictionary, accounting for following cases: + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in mni_row_before: + if self.values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + default = self.sim.modules['PregnancySupervisor'].default_mni_values + for key in default: + if self.values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + + chain_links[self.target] = str(link_info) return chain_links @@ -360,17 +322,16 @@ def run(self, squeeze_factor): if updated_appt_footprint is not None: footprint = updated_appt_footprint - chain_links = self.store_chains_to_do_after_event(print_chains, row_before, str(footprint), mni_row_before, mni_instances_before) + if print_chains: + chain_links = self.store_chains_to_do_after_event(row_before, str(footprint), mni_row_before, mni_instances_before) - if len(chain_links)>0: - pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} - # pop_dict = {i: '' for i in range(1000)} # Always include all possible individuals - - pop_dict.update(chain_links) - - logger_chains.info(key='event_chains', - data = pop_dict, - description='Links forming chains of events for simulated individuals') + if len(chain_links)>0: + pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} + pop_dict.update(chain_links) + + logger_chains.info(key='event_chains', + data = pop_dict, + description='Links forming chains of events for simulated individuals') return updated_appt_footprint diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 33edeb63c8..fe5d19c964 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -890,10 +890,7 @@ def initialise_simulation(self, sim): sim.schedule_event(TbRegularEvents(self), sim.date) sim.schedule_event(TbSelfCureEvent(self), sim.date) - if sim.generate_event_chains is True and sim.generate_event_chains_overwrite_epi is True: - sim.schedule_event(TbActiveCasePollGenerateData(self), sim.date + DateOffset(days=0)) - else: - sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) + sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) # 2) log at the end of the year diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 045e86bdd8..8356424901 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -109,19 +109,13 @@ def __init__( self.modules = OrderedDict() self.event_queue = EventQueue() self.generate_event_chains = True - self.generate_event_chains_overwrite_epi = False self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] - self.debug_generate_event_chains = False self.end_date = None self.output_file = None self.population: Optional[Population] = None - - if self.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - self.event_chains: Optional[Population] = None - + self.show_progress_bar = show_progress_bar self.resourcefilepath = resourcefilepath @@ -289,21 +283,12 @@ def make_initial_population(self, *, n: int) -> None: key="debug", data=f"{module.name}.initialise_population() {time.time() - start1} s", ) - - if self.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - self.event_chains = pd.DataFrame(columns= list(self.population.props.columns)+['person_ID'] + ['event'] + ['event_date'] + ['when'] + ['appt_footprint'] + ['level']) # When logging events for each individual to reconstruct chains, only the changes in individual properties will be logged. # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. if self.generate_event_chains: pop_dict = self.population.props.to_dict(orient='index') - - #if "PregnancySupervisor" in self.modules: - # print("I found it!") - # print(self.modules['PregnancySupervisor'].mother_and_newborn_info) - # exit(-1) for key in pop_dict.keys(): pop_dict[key]['person_ID'] = key @@ -311,12 +296,11 @@ def make_initial_population(self, *, n: int) -> None: pop_dict_full = {i: '' for i in range(FACTOR_POP_DICT)} pop_dict_full.update(pop_dict) - - print("Size for full sim", len(pop_dict_full)) logger.info(key='event_chains', data = pop_dict_full, description='Links forming chains of events for simulated individuals') + end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") @@ -334,15 +318,9 @@ def initialise(self, *, end_date: Date) -> None: #self.generate_event_chains = generate_event_chains if self.generate_event_chains: - # Eventually this can be made an option - self.generate_event_chains_overwrite_epi = False # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] - self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'LifestyleEvent', 'TbActiveCasePollGenerateData','HivPollingEventForDataGeneration','SimplifiedBirthsPoll', 'AgeUpdateEvent', 'HealthSystemScheduler', 'RTIPollingEvent'] - else: - # If not using to print chains, cannot ignore epi - self.generate_event_chains_overwrite_epi = False - + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'LifestyleEvent', 'TbActiveCasePollGenerateData','HivPollingEventForDataGeneration', 'RTIPollingEvent'] # Reorder columns to place the new columns at the front pd.set_option('display.max_columns', None) @@ -426,10 +404,6 @@ def run_simulation_to(self, *, to_date: Date) -> None: self._update_progress_bar(progress_bar, date) self.fire_single_event(event, date) self.date = to_date - - if self.debug_generate_event_chains: - # TO BE REMOVED: this is currently only used for debugging, will be removed from final PR. - self.event_chains.to_csv('output.csv', index=False) if self.show_progress_bar: progress_bar.stop() @@ -500,15 +474,6 @@ def do_birth(self, mother_id: int) -> int: logger.info(key='event_chains', data = pop_dict, description='Links forming chains of events for simulated individuals') - - if self.debug_generate_event_chains: - # TO BE REMOVED This is currently just used for debugging. Will be removed from final version of PR. - row = self.population.props.iloc[[child_id]] - row['person_ID'] = child_id - row['event'] = 'Birth' - row['event_date'] = self.date - row['when'] = 'After' - self.event_chains = pd.concat([self.event_chains, row], ignore_index=True) return child_id From 3b81de6546cb498938ff9918c852e39369b29ca3 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:32:50 +0100 Subject: [PATCH 093/103] All fixes made --- .../analysis_extract_data.py | 8 +++- .../scenario_generate_chains.py | 2 +- src/tlo/events.py | 33 +++---------- src/tlo/methods/hsi_event.py | 4 +- src/tlo/methods/pregnancy_helper_functions.py | 46 ++++--------------- src/tlo/methods/pregnancy_supervisor.py | 40 ++++++++++++++++ 6 files changed, 64 insertions(+), 69 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 8068db203a..7fe15f0eb4 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -98,7 +98,11 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No if value !='' and isinstance(value, str): evaluated = eval(value, eval_env) list_for_individual.append(evaluated) - + + for i in list_for_individual: + print(i) + + """ # These are the properties of the individual before the start of the chain of events initial_properties = list_for_individual[0] @@ -201,7 +205,7 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No properties = key_first_event | key_last_event record.append(properties) - + """ df = pd.DataFrame(record) df.to_csv("new_raw_data_" + name_tag + ".csv", index=False) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index 64fa70d055..e9291a50ce 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -53,7 +53,7 @@ def __init__(self): super().__init__() self.seed = 42 self.start_date = Date(2010, 1, 1) - self.end_date = self.start_date + pd.DateOffset(months=36) + self.end_date = self.start_date + pd.DateOffset(months=18) self.pop_size = 1000 self._scenarios = self._get_scenarios() self.number_of_draws = len(self._scenarios) diff --git a/src/tlo/events.py b/src/tlo/events.py index 9f762fd3c6..993c27090c 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -88,40 +88,23 @@ def mni_values_differ(self, v1, v2): def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): diffs = {} - """ - will_pause = False - - target_attribute = 'hcw_not_avail' - if len(entire_mni_after)>0: - print("Default target value before", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) - person = next(iter(entire_mni_after)) - entire_mni_after[person][target_attribute] = True - will_pause = True - print("Default target value after", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) - - if will_pause: - print("Reprint") - print(entire_mni_before) - print(entire_mni_after) - print("Default target value", self.sim.modules['PregnancySupervisor'].default_mni_values[target_attribute]) - """ all_individuals = set(entire_mni_before.keys()) | set(entire_mni_after.keys()) for person in all_individuals: if person not in entire_mni_before: # but is afterward for key in entire_mni_after[person]: - if self.mni_values_differ(entire_mni_after[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if self.mni_values_differ(entire_mni_after[person][key],self.sim.modules['PregnancySupervisor'].default_all_mni_values[key]): if person not in diffs: diffs[person] = {} diffs[person][key] = entire_mni_after[person][key] elif person not in entire_mni_after: # but is beforehand for key in entire_mni_before[person]: - if self.mni_values_differ(entire_mni_before[person][key],self.sim.modules['PregnancySupervisor'].default_mni_values[key]): + if self.mni_values_differ(entire_mni_before[person][key],self.sim.modules['PregnancySupervisor'].default_all_mni_values[key]): if person not in diffs: diffs[person] = {} - diffs[person][key] = self.sim.modules['PregnancySupervisor'].default_mni_values[key] + diffs[person][key] = self.sim.modules['PregnancySupervisor'].default_all_mni_values[key] else: # person is in both # Compare properties @@ -131,8 +114,6 @@ def compare_entire_mni_dicts(self,entire_mni_before, entire_mni_after): diffs[person] = {} diffs[person][key] = entire_mni_after[person][key] - if len(diffs)>0: - print("DIfferences for ", diffs) return diffs def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_before, entire_mni_after): @@ -272,13 +253,13 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, link_info[key] = mni[self.target][key] # Individual is only in mni dictionary before event elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_mni_values + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values for key in mni_row_before: if self.mni_values_differ(mni_row_before[key], default[key]): link_info[key] = default[key] # Individual is only in mni dictionary after event elif mni_instances_after and not mni_instances_before: - default = self.sim.modules['PregnancySupervisor'].default_mni_values + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values for key in default: if self.mni_values_differ(default[key], mni[self.target][key]): link_info[key] = mni[self.target][key] @@ -322,12 +303,10 @@ def run(self): # Log chain_links here if len(chain_links)>0: - print(chain_links) + logger_chain.info(key='event_chains', data= pop_dict, description='Links forming chains of events for simulated individuals') - - print("Chain events ", chain_links) class RegularEvent(Event): diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index dbca98da5c..85ac6da3e2 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -286,13 +286,13 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, link_info[key] = mni[self.target][key] # Individual is only in mni dictionary before event elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_mni_values + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values for key in mni_row_before: if self.values_differ(mni_row_before[key], default[key]): link_info[key] = default[key] # Individual is only in mni dictionary after event elif mni_instances_after and not mni_instances_before: - default = self.sim.modules['PregnancySupervisor'].default_mni_values + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values for key in default: if self.values_differ(default[key], mni[self.target][key]): link_info[key] = mni[self.target][key] diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 79483cddaa..2456f57e8b 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -545,40 +545,12 @@ def update_mni_dictionary(self, individual_id): mni[individual_id] = self.sim.modules['PregnancySupervisor'].default_mni_values.copy() elif self == self.sim.modules['Labour']: - labour_variables = {'labour_state': None, - # Term Labour (TL), Early Preterm (EPTL), Late Preterm (LPTL) or Post Term (POTL) - 'birth_weight': 'normal_birth_weight', - 'birth_size': 'average_for_gestational_age', - 'delivery_setting': None, # home_birth, health_centre, hospital - 'twins': df.at[individual_id, 'ps_multiple_pregnancy'], - 'twin_count': 0, - 'twin_one_comps': False, - 'pnc_twin_one': 'none', - 'bf_status_twin_one': 'none', - 'eibf_status_twin_one': False, - 'an_placental_abruption': df.at[individual_id, 'ps_placental_abruption'], - 'corticosteroids_given': False, - 'clean_birth_practices': False, - 'abx_for_prom_given': False, - 'abx_for_pprom_given': False, - 'endo_pp': False, - 'retained_placenta': False, - 'uterine_atony': False, - 'amtsl_given': False, - 'cpd': False, - 'mode_of_delivery': 'vaginal_delivery', - 'neo_will_receive_resus_if_needed': False, - # vaginal_delivery, instrumental, caesarean_section - 'hsi_cant_run': False, # True (T) or False (F) - 'sought_care_for_complication': False, # True (T) or False (F) - 'sought_care_labour_phase': 'none', - 'referred_for_cs': False, # True (T) or False (F) - 'referred_for_blood': False, # True (T) or False (F) - 'received_blood_transfusion': False, # True (T) or False (F) - 'referred_for_surgery': False, # True (T) or False (F)' - 'death_in_labour': False, # True (T) or False (F) - 'single_twin_still_birth': False, # True (T) or False (F) - 'will_receive_pnc': 'none', - 'passed_through_week_one': False} - - mni[individual_id].update(labour_variables) + + labour_default = self.sim.modules['PregnancySupervisor'].default_labour_values.copy() + mni[individual_id].update(labour_default) + + # Update from default based on individual case + mni[individual_id]['twins'] = df.at[individual_id, 'ps_multiple_pregnancy'] + mni[individual_id]['an_placental_abruption'] = df.at[individual_id, 'ps_placental_abruption'] + + diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index f634d9b971..5d747d44c2 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -82,6 +82,8 @@ def __init__(self, name=None, resourcefilepath=None): 'gest_diab_onset': pd.NaT, 'gest_diab_diagnosed_onset': pd.NaT, 'gest_diab_resolution': pd.NaT, + 'none_anaemia_onset': pd.NaT, + 'none_anaemia_resolution': pd.NaT, 'mild_anaemia_onset': pd.NaT, 'mild_anaemia_resolution': pd.NaT, 'moderate_anaemia_onset': pd.NaT, @@ -111,6 +113,44 @@ def __init__(self, name=None, resourcefilepath=None): 'new_onset_spe': False, 'cs_indication': 'none' } + self.default_labour_values = {'labour_state': None, + # Term Labour (TL), Early Preterm (EPTL), Late Preterm (LPTL) or Post Term (POTL) + 'birth_weight': 'normal_birth_weight', + 'birth_size': 'average_for_gestational_age', + 'delivery_setting': None, # home_birth, health_centre, hospital + 'twins': None, + 'twin_count': 0, + 'twin_one_comps': False, + 'pnc_twin_one': 'none', + 'bf_status_twin_one': 'none', + 'eibf_status_twin_one': False, + 'an_placental_abruption': None, + 'corticosteroids_given': False, + 'clean_birth_practices': False, + 'abx_for_prom_given': False, + 'abx_for_pprom_given': False, + 'endo_pp': False, + 'retained_placenta': False, + 'uterine_atony': False, + 'amtsl_given': False, + 'cpd': False, + 'mode_of_delivery': 'vaginal_delivery', + 'neo_will_receive_resus_if_needed': False, + # vaginal_delivery, instrumental, caesarean_section + 'hsi_cant_run': False, # True (T) or False (F) + 'sought_care_for_complication': False, # True (T) or False (F) + 'sought_care_labour_phase': 'none', + 'referred_for_cs': False, # True (T) or False (F) + 'referred_for_blood': False, # True (T) or False (F) + 'received_blood_transfusion': False, # True (T) or False (F) + 'referred_for_surgery': False, # True (T) or False (F)' + 'death_in_labour': False, # True (T) or False (F) + 'single_twin_still_birth': False, # True (T) or False (F) + 'will_receive_pnc': 'none', + 'passed_through_week_one': False} + + self.default_all_mni_values = self.default_mni_values + self.default_all_mni_values.update(self.default_labour_values) INIT_DEPENDENCIES = {'Demography'} From 070f83beb3e62ac288fe777fc99051dda21f6fbc Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:14:15 +0100 Subject: [PATCH 094/103] Allowing individuals to always partially survive death event, plus refactor risk of death function so that this can be obtained outside of functions get_risk_of_death_from_cause --- .../scenario_generate_chains.py | 12 +-- src/tlo/events.py | 4 +- src/tlo/methods/demography.py | 35 +++++++ src/tlo/methods/postnatal_supervisor.py | 66 +++++++++---- src/tlo/methods/pregnancy_helper_functions.py | 50 +++++++--- src/tlo/methods/pregnancy_supervisor.py | 95 +++++++++++++------ src/tlo/simulation.py | 2 +- 7 files changed, 198 insertions(+), 66 deletions(-) diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index e9291a50ce..b5fb2e9df1 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -54,7 +54,7 @@ def __init__(self): self.seed = 42 self.start_date = Date(2010, 1, 1) self.end_date = self.start_date + pd.DateOffset(months=18) - self.pop_size = 1000 + self.pop_size = 10000 self._scenarios = self._get_scenarios() self.number_of_draws = len(self._scenarios) self.runs_per_draw = 1 @@ -77,7 +77,6 @@ def log_configuration(self): def modules(self): # MODIFY # Here instead of running full module - """ return [demography.Demography(resourcefilepath=self.resources), enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), healthburden.HealthBurden(resourcefilepath=self.resources), @@ -97,11 +96,10 @@ def modules(self): healthsystem.HealthSystem(resourcefilepath=self.resources, mode_appt_constraints=1, cons_availability='all')] - """ - return ( - fullmodel(resourcefilepath=self.resources) - + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] - ) + #return ( + # fullmodel(resourcefilepath=self.resources) + # +# [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + #) """ def draw_parameters(self, draw_number, rng): return mix_scenarios( diff --git a/src/tlo/events.py b/src/tlo/events.py index 993c27090c..cdda334a11 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -278,7 +278,7 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Create and store the event and dictionary of changes for affected individuals chain_links = self.compare_population_dataframe_and_mni(df_before, df_after, entire_mni_before, entire_mni_after) - + return chain_links @@ -303,7 +303,7 @@ def run(self): # Log chain_links here if len(chain_links)>0: - + print(chain_links) logger_chain.info(key='event_chains', data= pop_dict, description='Links forming chains of events for simulated individuals') diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index 64a99fb625..856e9979ab 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -121,8 +121,13 @@ def __init__(self, name=None, resourcefilepath=None, equal_allocation_by_distric # as optional if they can be undefined for a given individual. PROPERTIES = { 'is_alive': Property(Types.BOOL, 'Whether this individual is alive'), + 'aliveness_weight': Property(Types.REAL, 'If allowing individual to survive, track decreasing weight'), + 'death_weight': Property(Types.LIST, 'If allowing individual to survive, track weight of death'), 'date_of_birth': Property(Types.DATE, 'Date of birth of this individual'), 'date_of_death': Property(Types.DATE, 'Date of death of this individual'), + 'date_of_partial_death': Property(Types.LIST, 'Date of latest partial death of this individual'), + 'cause_of_partial_death': Property(Types.LIST, 'Date of latest partial death of this individual'), + 'sex': Property(Types.CATEGORICAL, 'Male or female', categories=['M', 'F']), 'mother_id': Property(Types.INT, 'Unique identifier of mother of this individual'), @@ -278,9 +283,14 @@ def initialise_population(self, population): # Assign the characteristics df.is_alive.values[:] = True + alive_count = sum(df.is_alive) + df.aliveness_weight.values[:] = 1 + df.loc[df.is_alive, 'death_weight'] = pd.Series([[] for _ in range(alive_count)]) df.loc[df.is_alive, 'date_of_birth'] = demog_char_to_assign['date_of_birth'] df.loc[df.is_alive, 'date_of_death'] = pd.NaT + df.loc[df.is_alive, 'date_of_partial_death'] = pd.Series([[] for _ in range(alive_count)]) df.loc[df.is_alive, 'cause_of_death'] = np.nan + df.loc[df.is_alive, 'cause_of_partial_death'] = pd.Series([[] for _ in range(alive_count)]) df.loc[df.is_alive, 'sex'] = demog_char_to_assign['Sex'] df.loc[df.is_alive, 'mother_id'] = DEFAULT_MOTHER_ID # Motherless, and their characterists are not inherited df.loc[df.is_alive, 'district_num_of_residence'] = demog_char_to_assign['District_Num'].values[:] @@ -353,9 +363,13 @@ def on_birth(self, mother_id, child_id): child = { 'is_alive': True, + 'aliveness_weight': 1.0, + 'death_weight': [], 'date_of_birth': self.sim.date, 'date_of_death': pd.NaT, 'cause_of_death': np.nan, + 'date_of_partial_death': [], + 'cause_of_partial_death': [], 'sex': 'M' if rng.random_sample() < fraction_of_births_male else 'F', 'mother_id': mother_id, 'district_num_of_residence': _district_num_of_residence, @@ -479,6 +493,7 @@ def process_causes_of_death(self): data=mapper_from_gbd_causes ) + def do_death(self, individual_id: int, cause: str, originating_module: Module): """Register and log the death of an individual from a specific cause. * 'individual_id' is the index in the population.props dataframe to the (one) person. @@ -751,6 +766,26 @@ def apply(self, population): self.sim.date + DateOffset(days=self.module.rng.randint(0, 30))) +class InstantaneousPartialDeath(Event, IndividualScopeEventMixin): + """ + Call the do_death function to cause the person to die. + + Note that no checking is done here. (Checking is done within `do_death` which can also be called directly.) + + The 'individual_id' is the index in the population.props dataframe. It is for _one_ person only. + The 'cause' is the cause that is defined by the disease module (aka, "tlo cause"). + The 'module' passed to this event is the disease module that is causing the death. + """ + + def __init__(self, module, individual_id, cause, weight): + super().__init__(module, person_id=individual_id) + self.cause = cause + self.weight = weight + + def apply(self, individual_id): + self.sim.modules['Demography'].do_partial_death(individual_id, cause=self.cause, originating_module=self.module, weight=self.weight) + + class InstantaneousDeath(Event, IndividualScopeEventMixin): """ Call the do_death function to cause the person to die. diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 7ee477e45a..f2c6e47764 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -641,18 +641,34 @@ def log_new_progressed_cases(disease): (df['pn_postnatal_period_in_weeks'] == week) & (df['pn_htn_disorders'] == 'severe_gest_htn')] - die_from_htn = pd.Series(self.rng.random_sample(len(at_risk_of_death_htn)) < - params['weekly_prob_death_severe_gest_htn'], index=at_risk_of_death_htn.index) - # Those women who die the on_death function in demography is applied - for person in die_from_htn.loc[die_from_htn].index: - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + if self.sim.generate_event_chains: + + # If generating chains of events, all women at risk will partially die, partially survive + for person in at_risk_of_death_htn.index: + df.loc[person,'aliveness_weight'] *= (1. - params['weekly_prob_death_severe_gest_htn']) + # Individual is partially dead + death_weight = df.loc[individual_id]['aliveness_weight'] * params['weekly_prob_death_severe_gest_htn'] + + df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[individual_id, 'death_weight'].append(death_weight) + df.loc[individual_id, 'cause_of_partial_death'].append('severe_gestational_hypertension') - self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', - originating_module=self.sim.modules['PostnatalSupervisor']) + else: + + die_from_htn = pd.Series(self.rng.random_sample(len(at_risk_of_death_htn)) < + params['weekly_prob_death_severe_gest_htn'], index=at_risk_of_death_htn.index) + + for person in die_from_htn.loc[die_from_htn].index: + + # Those women who die the on_death function in demography is applied + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + + self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', + originating_module=self.sim.modules['PostnatalSupervisor']) - del self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person] + del self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person] # ----------------------------------------- CARE SEEKING ------------------------------------------------------ # We now use the pn_emergency_event_mother property that has just been set for women who are experiencing @@ -839,17 +855,33 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in else: cause = 'early_onset_sepsis' - # If this neonate will die then we make the appropriate changes - if self.rng.random_sample() < risk_of_death: - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause}_n_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=cause, - originating_module=self.sim.modules['PostnatalSupervisor']) - - # Otherwise we reset the variables in the data frame - else: + if self.sim.generate_event_chains: + + # This neonate will always partially die, partially survive + # Individual is partially less alive + df.loc[individual_id,'aliveness_weight'] *= (1. - risk_of_death) + # Individual is partially dead + death_weight = df.loc[individual_id]['aliveness_weight'] * risk_of_death + df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[individual_id, 'death_weight'].append(death_weight) + df.loc[individual_id, 'cause_of_partial_death'].append(cause) + + # Because it always survives, always reset variables in the dataframe df.at[individual_id, 'pn_sepsis_late_neonatal'] = False df.at[individual_id, 'pn_sepsis_early_neonatal'] = False + else: + # If this neonate will die then we make the appropriate changes + if self.rng.random_sample() < risk_of_death: + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause}_n_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=cause, + originating_module=self.sim.modules['PostnatalSupervisor']) + + # Otherwise we reset the variables in the data frame + else: + df.at[individual_id, 'pn_sepsis_late_neonatal'] = False + df.at[individual_id, 'pn_sepsis_early_neonatal'] = False + class PostnatalSupervisorEvent(RegularEvent, PopulationScopeEventMixin): """ diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index eb32c2e759..9c7ef03959 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -341,12 +341,10 @@ def calculate_risk_of_death_from_causes(self, risks, target): return False -def check_for_risk_of_death_from_cause_maternal(self, individual_id, timing): +def get_risk_of_death_from_cause_maternal(self, individual_id, timing): """ - This function calculates the risk of death associated with one or more causes being experience by an individual and - determines if they will die and which of a number of competing cause is the primary cause of death - :param individual_id: individual_id of woman at risk of death - return: cause of death or False + This function calculates the risk of death associated with one or more causes being experience by an individual + return: causes and associated risks """ params = self.current_parameters df = self.sim.population.props @@ -389,10 +387,11 @@ def check_for_risk_of_death_from_cause_maternal(self, individual_id, timing): if mother.pn_postpartum_haem_secondary and (timing == 'postnatal'): causes.append('secondary_postpartum_haemorrhage') + risks = dict() + # If this list is not empty, use either CFR parameters or linear models to calculate risk of death from each # complication she is experiencing and store in a dictionary, using each cause as the key if causes: - risks = dict() def apply_effect_of_anaemia(cause): lab_params = self.sim.modules['Labour'].current_parameters @@ -443,7 +442,21 @@ def apply_effect_of_anaemia(cause): apply_effect_of_anaemia(cause) risks.update(risk) + + return risks + +def check_for_risk_of_death_from_cause_maternal(self, individual_id, timing): + """ + This function calculates the risk of death associated with one or more causes being experience by an individual and + determines if they will die and which of a number of competing cause is the primary cause of death + :param individual_id: individual_id of woman at risk of death + return: cause of death or False + """ + + risks = get_risk_of_death_from_cause_maternal(self, individual_id, timing) + + if len(risks)>0: # Call return the result from calculate_risk_of_death_from_causes function return calculate_risk_of_death_from_causes(self, risks, target='m') @@ -451,12 +464,10 @@ def apply_effect_of_anaemia(cause): return False -def check_for_risk_of_death_from_cause_neonatal(self, individual_id): +def get_risk_of_death_from_cause_neonatal(self, individual_id): """ - This function calculates the risk of death associated with one or more causes being experience by an individual and - determines if they will die and which of a number of competing cause is the primary cause of death - :param individual_id: individual_id of woman at risk of death - return: cause of death or False + This function calculates the risk of death associated with one or more causes being experience by an individual + return: causes and associated risks """ params = self.current_parameters df = self.sim.population.props @@ -501,10 +512,11 @@ def check_for_risk_of_death_from_cause_neonatal(self, individual_id): if self.congeintal_anomalies.has_all(individual_id, 'other'): causes.append('other_anomaly') + risks = dict() + # If this list is not empty, use either CFR parameters or linear models to calculate risk of death from each # complication they experiencing and store in a dictionary, using each cause as the key if causes: - risks = dict() for cause in causes: if f'{cause}_death' in self.nb_linear_models.keys(): risk = {cause: self.nb_linear_models[f'{cause}_death'].predict( @@ -513,7 +525,21 @@ def check_for_risk_of_death_from_cause_neonatal(self, individual_id): risk = {cause: params[f'cfr_{cause}']} risks.update(risk) + + return risks + + +def check_for_risk_of_death_from_cause_neonatal(self, individual_id): + """ + This function calculates the risk of death associated with one or more causes being experience by an individual and + determines if they will die and which of a number of competing cause is the primary cause of death + :param individual_id: individual_id of woman at risk of death + return: cause of death or False + """ + + risks = get_risk_of_death_from_cause_neonatal(self, individual_id) + if len(risks)>0: # Return the result from calculate_risk_of_death_from_causes function (returns primary cause of death or False) return calculate_risk_of_death_from_causes(self, risks, target='n') diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 881ec8a6e7..1ae7fa5c09 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -28,7 +28,7 @@ from tlo.methods.causes import Cause from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.util import BitsetHandler - +from tlo.methods.demography import InstantaneousPartialDeath if TYPE_CHECKING: from tlo.methods.hsi_generic_first_appts import HSIEventScheduler from tlo.population import IndividualProperties @@ -1331,19 +1331,36 @@ def apply_risk_of_death_from_hypertension(self, gestation_of_interest): (df.ps_ectopic_pregnancy == 'none') & ~df.la_currently_in_labour & \ (df.ps_htn_disorders == 'severe_gest_htn') - at_risk_of_death_htn = pd.Series(self.rng.random_sample(len(at_risk.loc[at_risk])) < - params['prob_monthly_death_severe_htn'], index=at_risk.loc[at_risk].index) + if self.sim.generate_event_chains: + + # Every woman at risk will partially die + for person in df.index[at_risk]: + # Individual is partially less alive + df.loc[person,'aliveness_weight'] *= (1. - params['prob_monthly_death_severe_htn']) + # Individual is partially dead + death_weight = df.loc[person]['aliveness_weight'] * params['prob_monthly_death_severe_htn'] + df.loc[person, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[person, 'death_weight'].append(death_weight) + df.loc[person, 'cause_of_partial_death'].append('severe_gestational_hypertension') + + # Not deleting woman from mni because she has survived + + else: + + at_risk_of_death_htn = pd.Series(self.rng.random_sample(len(at_risk.loc[at_risk])) < + params['prob_monthly_death_severe_htn'], index=at_risk.loc[at_risk].index) - if not at_risk_of_death_htn.loc[at_risk_of_death_htn].empty: - # Those women who die have InstantaneousDeath scheduled - for person in at_risk_of_death_htn.loc[at_risk_of_death_htn].index: - self.mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 - self.mnh_outcome_counter['direct_mat_death'] += 1 + if not at_risk_of_death_htn.loc[at_risk_of_death_htn].empty: + # Those women who die have InstantaneousDeath scheduled + for person in at_risk_of_death_htn.loc[at_risk_of_death_htn].index: + self.mnh_outcome_counter['severe_gestational_hypertension_m_death'] += 1 + self.mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', - originating_module=self.sim.modules['PregnancySupervisor']) + self.sim.modules['Demography'].do_death(individual_id=person, cause='severe_gestational_hypertension', + originating_module=self.sim.modules['PregnancySupervisor']) - del mni[person] + del mni[person] + def apply_risk_of_placental_abruption(self, gestation_of_interest): """ @@ -2086,29 +2103,53 @@ def apply(self, individual_id): risk_of_death = self.module.ps_linear_models[f'{self.cause}_death'].predict( df.loc[[individual_id]])[individual_id] - # If the death occurs we record it here - if self.module.rng.random_sample() < risk_of_death: + if self.sim.generate_event_chains: + + # Individual partially survives + df.loc[individual_id,'aliveness_weight'] *= (1. - risk_of_death) + + # Individual partially dies + death_weight = df.loc[individual_id]['aliveness_weight'] * risk_of_death + df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[individual_id, 'death_weight'].append(death_weight) + df.loc[individual_id, 'cause_of_partial_death'].append('severe_gestational_hypertension') + + # Individual always survives, so always update variables + if self.cause == 'ectopic_pregnancy': + df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' - if individual_id in mni: - pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) - mni[individual_id]['delete_mni'] = True + else: + self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') + df.at[individual_id, 'ac_received_post_abortion_care'] = False - self.module.mnh_outcome_counter[f'{self.cause}_m_death'] += 1 - self.module.mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=f'{self.cause}', - originating_module=self.sim.modules['PregnancySupervisor']) + if individual_id in mni: + mni[individual_id]['delete_mni'] = True else: - # Otherwise we reset any variables - if self.cause == 'ectopic_pregnancy': - df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' + + # If the death occurs we record it here + if self.module.rng.random_sample() < risk_of_death: + + if individual_id in mni: + pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) + mni[individual_id]['delete_mni'] = True + + self.module.mnh_outcome_counter[f'{self.cause}_m_death'] += 1 + self.module.mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=f'{self.cause}', + originating_module=self.sim.modules['PregnancySupervisor']) else: - self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') - df.at[individual_id, 'ac_received_post_abortion_care'] = False + # Otherwise we reset any variables + if self.cause == 'ectopic_pregnancy': + df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' + + else: + self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') + df.at[individual_id, 'ac_received_post_abortion_care'] = False - if individual_id in mni: - mni[individual_id]['delete_mni'] = True + if individual_id in mni: + mni[individual_id]['delete_mni'] = True class GestationalDiabetesGlycaemicControlEvent(Event, IndividualScopeEventMixin): diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 8356424901..bfa7c2e1c5 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -320,7 +320,7 @@ def initialise(self, *, end_date: Date) -> None: if self.generate_event_chains: # For now keep these fixed, eventually they will be input from user self.generate_event_chains_modules_of_interest = [self.modules] - self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'LifestyleEvent', 'TbActiveCasePollGenerateData','HivPollingEventForDataGeneration', 'RTIPollingEvent'] + self.generate_event_chains_ignore_events = ['AgeUpdateEvent','HealthSystemScheduler', 'SimplifiedBirthsPoll','DirectBirth', 'LifestyleEvent', 'TbActiveCasePollGenerateData','HivPollingEventForDataGeneration', 'RTIPollingEvent', 'DepressionPollingEvent','Get_Current_DALYs', 'PostnatalSupervisorEvent','PregnancySupervisorEvent', 'MalariaPollingEventDistrict','CopdPollEvent', 'CardioMetabolicDisorders_MainPollingEvent', 'CardioMetabolicDisorders_LoggingEvent'] # Reorder columns to place the new columns at the front pd.set_option('display.max_columns', None) From 1c146fefb49a2d06dbe7d13f3edfb7ac08a9c3ba Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:21:38 +0100 Subject: [PATCH 095/103] All partial deaths now captured, when multiple partial deaths now rely on unique function --- src/tlo/methods/labour.py | 72 ++++++++++------ src/tlo/methods/mnh_cohort_module.py | 1 + src/tlo/methods/postnatal_supervisor.py | 67 +++++++++------ src/tlo/methods/pregnancy_helper_functions.py | 21 +++++ src/tlo/methods/pregnancy_supervisor.py | 85 +++++++++++-------- 5 files changed, 161 insertions(+), 85 deletions(-) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 85acf3fccd..ba4d2214ae 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1488,25 +1488,37 @@ def apply_risk_of_early_postpartum_death(self, individual_id): # Check the right women are at risk of death self.postpartum_characteristics_checker(individual_id) - # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death - # (either from one or multiple causes) and if death occurs returns the cause - potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( - self, individual_id=individual_id, timing='postnatal') + survived = True + + if self.sim.generate_event_chains: + + risks = pregnancy_helper_functions.get_risk_of_death_from_cause_maternal( + self, individual_id=individual_id, timing='postnatal') + + pregnancy_helper_functions.apply_multiple_partial_deaths(self, risks, individual_id=individual_id) + + else: + + # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death + # (either from one or multiple causes) and if death occurs returns the cause + potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( + self, individual_id=individual_id, timing='postnatal') - # Log df row containing complications and treatments to calculate met need + # Log df row containing complications and treatments to calculate met need - # If a cause is returned death is scheduled - if potential_cause_of_death: - pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + # If a cause is returned death is scheduled + if potential_cause_of_death: + survived = False + pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['Labour']) + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, + originating_module=self.sim.modules['Labour']) # If she hasn't died from any complications, we reset some key properties that resolve after risk of death # has been applied - else: + if survived: if df.at[individual_id, 'pn_htn_disorders'] == 'eclampsia': df.at[individual_id, 'pn_htn_disorders'] = 'severe_pre_eclamp' @@ -2552,19 +2564,31 @@ def apply(self, individual_id): self.module.labour_characteristics_checker(individual_id) - # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death - # (either from one or multiple causes) and if death occurs returns the cause - potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( - self.module, individual_id=individual_id, timing='intrapartum') + survived = True + + if self.sim.generate_event_chains: + + risks = pregnancy_helper_functions.get_risk_of_death_from_cause_maternal( + self.module, individual_id=individual_id, timing='intrapartum') + + pregnancy_helper_functions.apply_multiple_partial_deaths(self, risks, individual_id=individual_id) + + else: + + # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death + # (either from one or multiple causes) and if death occurs returns the cause + potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( + self.module, individual_id=individual_id, timing='intrapartum') - # If a cause is returned death is scheduled - if potential_cause_of_death: - pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['Labour']) + # If a cause is returned death is scheduled + if potential_cause_of_death: + survived = False + pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, + originating_module=self.sim.modules['Labour']) - mni[individual_id]['death_in_labour'] = True + mni[individual_id]['death_in_labour'] = True # Next we determine if she will experience a stillbirth during her delivery outcome_of_still_birth_equation = self.module.predict(self.module.la_linear_models['intrapartum_still_birth'], @@ -2611,7 +2635,7 @@ def apply(self, individual_id): mni[individual_id]['didnt_seek_care'] = False # Finally, reset some treatment variables - if not potential_cause_of_death: + if survived: df.at[individual_id, 'la_maternal_hypertension_treatment'] = False df.at[individual_id, 'ac_iv_anti_htn_treatment'] = False df.at[individual_id, 'ac_mag_sulph_treatment'] = False diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 27ee7e888e..1a15242e52 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -77,6 +77,7 @@ def initialise_population(self, population): # Set the dtypes and index of the cohort dataframe props_dtypes = self.sim.population.props.dtypes preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) + preg_pop_final.index.name = 'person' # For the below columns we manually overwrite the dtypes diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index f2c6e47764..420e4f1d42 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -646,13 +646,15 @@ def log_new_progressed_cases(disease): # If generating chains of events, all women at risk will partially die, partially survive for person in at_risk_of_death_htn.index: + + original_aliveness_weight = df.loc[person,'aliveness_weight'] df.loc[person,'aliveness_weight'] *= (1. - params['weekly_prob_death_severe_gest_htn']) # Individual is partially dead - death_weight = df.loc[individual_id]['aliveness_weight'] * params['weekly_prob_death_severe_gest_htn'] + death_weight = original_aliveness_weight * params['weekly_prob_death_severe_gest_htn'] - df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) - df.loc[individual_id, 'death_weight'].append(death_weight) - df.loc[individual_id, 'cause_of_partial_death'].append('severe_gestational_hypertension') + df.loc[person, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[person, 'death_weight'].append(death_weight) + df.loc[person, 'cause_of_partial_death'].append('severe_gestational_hypertension') else: @@ -815,21 +817,33 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in # Create a list of all the causes that may cause death in the individual (matched to GBD labels) if mother_or_child == 'mother': - # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death - # (either from one or multiple causes) and if death occurs returns the cause - potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( - self, individual_id=individual_id, timing='postnatal') - - # If a cause is returned death is scheduled - if potential_cause_of_death: - mni[individual_id]['didnt_seek_care'] = True - pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['PostnatalSupervisor']) - del mni[individual_id] + survived = True + + if self.sim.generate_event_chains: + + # If collecting events, woman will always partially survive and partially die from all possible causes + risks = pregnancy_helper_functions.get_risk_of_death_from_cause_maternal( + self, individual_id=individual_id, timing='postnatal') + + pregnancy_helper_functions.apply_multiple_partial_deaths(self, risks, individual_id=individual_id) else: + # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death + # (either from one or multiple causes) and if death occurs returns the cause + potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( + self, individual_id=individual_id, timing='postnatal') + + # If a cause is returned death is scheduled + if potential_cause_of_death: + survived = False + mni[individual_id]['didnt_seek_care'] = True + pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) + self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, + originating_module=self.sim.modules['PostnatalSupervisor']) + del mni[individual_id] + + if survived: # Reset variables for women who survive if mother.pn_postpartum_haem_secondary: df.at[individual_id, 'pn_postpartum_haem_secondary'] = False @@ -844,6 +858,8 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in if mother_or_child == 'child': # Neonates can have either early or late onset sepsis, not both at once- so we use either equation # depending on this neonates current condition + + survived = True if child.pn_sepsis_early_neonatal: risk_of_death = params['cfr_early_onset_neonatal_sepsis'] elif child.pn_sepsis_late_neonatal: @@ -858,29 +874,28 @@ def apply_risk_of_maternal_or_neonatal_death_postnatal(self, mother_or_child, in if self.sim.generate_event_chains: # This neonate will always partially die, partially survive + original_aliveness_weight = df.loc[individual_id,'aliveness_weight'] # Individual is partially less alive df.loc[individual_id,'aliveness_weight'] *= (1. - risk_of_death) # Individual is partially dead - death_weight = df.loc[individual_id]['aliveness_weight'] * risk_of_death + death_weight = original_aliveness_weight * risk_of_death df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) df.loc[individual_id, 'death_weight'].append(death_weight) df.loc[individual_id, 'cause_of_partial_death'].append(cause) - - # Because it always survives, always reset variables in the dataframe - df.at[individual_id, 'pn_sepsis_late_neonatal'] = False - df.at[individual_id, 'pn_sepsis_early_neonatal'] = False else: # If this neonate will die then we make the appropriate changes if self.rng.random_sample() < risk_of_death: + + survived = False self.sim.modules['PregnancySupervisor'].mnh_outcome_counter[f'{cause}_n_death'] += 1 self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=cause, originating_module=self.sim.modules['PostnatalSupervisor']) - # Otherwise we reset the variables in the data frame - else: - df.at[individual_id, 'pn_sepsis_late_neonatal'] = False - df.at[individual_id, 'pn_sepsis_early_neonatal'] = False + # If individual survived we reset the variables in the data frame + if survived: + df.at[individual_id, 'pn_sepsis_late_neonatal'] = False + df.at[individual_id, 'pn_sepsis_early_neonatal'] = False class PostnatalSupervisorEvent(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py index 9c7ef03959..16d4c84087 100644 --- a/src/tlo/methods/pregnancy_helper_functions.py +++ b/src/tlo/methods/pregnancy_helper_functions.py @@ -305,6 +305,27 @@ def log_mni_for_maternal_death(self, person_id): logger.info(key='death_mni', data=mni_to_log) +def apply_multiple_partial_deaths(self, risks, individual_id): + """ + This function applies multiple causes of partial death on individual + """ + df = self.sim.population.props + + total_risk_of_death = 0 + for key, value in risks.items(): + total_risk_of_death += value + + # Individual is partially less alive + original_aliveness_weight = df.loc[individual_id,'aliveness_weight'] + df.loc[individual_id,'aliveness_weight'] *= (1. - total_risk_of_death) + for key, value in risks.items(): + # Individual is partially dead + death_weight = original_aliveness_weight * value + df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) + df.loc[individual_id, 'death_weight'].append(death_weight) + df.loc[individual_id, 'cause_of_partial_death'].append(key) + + def calculate_risk_of_death_from_causes(self, risks, target): """ This function calculates risk of death in the context of one or more 'death causing' complications in a mother of a diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 1ae7fa5c09..0e66ffb70d 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -1335,15 +1335,17 @@ def apply_risk_of_death_from_hypertension(self, gestation_of_interest): # Every woman at risk will partially die for person in df.index[at_risk]: + + original_aliveness_weight = df.loc[person,'aliveness_weight'] # Individual is partially less alive df.loc[person,'aliveness_weight'] *= (1. - params['prob_monthly_death_severe_htn']) # Individual is partially dead - death_weight = df.loc[person]['aliveness_weight'] * params['prob_monthly_death_severe_htn'] + death_weight = original_aliveness_weight * params['prob_monthly_death_severe_htn'] df.loc[person, 'date_of_partial_death'].append(str(self.sim.date)) df.loc[person, 'death_weight'].append(death_weight) df.loc[person, 'cause_of_partial_death'].append('severe_gestational_hypertension') - # Not deleting woman from mni because she has survived + # Not deleting woman from mni because she always survives else: @@ -1667,21 +1669,36 @@ def apply_risk_of_death_from_monthly_complications(self, individual_id): mother = df.loc[individual_id] - # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death - # (either from one or multiple causes) and if death occurs returns the cause - potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( + survived = True + + if self.sim.generate_event_chains: + + # Get all potential causes of death and associated risks + risks = pregnancy_helper_functions.get_risk_of_death_from_cause_maternal( self, individual_id=individual_id, timing='antenatal') - - # If a cause is returned death is scheduled - if potential_cause_of_death: - pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['PregnancySupervisor']) - self.mnh_outcome_counter['direct_mat_death'] += 1 - del mni[individual_id] + + pregnancy_helper_functions.apply_multiple_partial_deaths(self, risks, individual_id=individual_id) + + else: + + # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death + # (either from one or multiple causes) and if death occurs returns the cause + potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( + self, individual_id=individual_id, timing='antenatal') + + # If a cause is returned death is scheduled + if potential_cause_of_death: + + # If woman has been selected to die, set survived to false + survived = False + pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) + self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, + originating_module=self.sim.modules['PregnancySupervisor']) + self.mnh_outcome_counter['direct_mat_death'] += 1 + del mni[individual_id] # If not we reset variables and the woman survives - else: + if survived: mni[individual_id]['didnt_seek_care'] = False if (mother.ps_htn_disorders == 'severe_pre_eclamp') and mni[individual_id]['new_onset_spe']: @@ -2099,36 +2116,32 @@ def apply(self, individual_id): if not df.at[individual_id, 'is_alive']: return + # Woman assumed to survive, will overwrite if selected for death + survived = True + # Individual risk of death is calculated through the linear model risk_of_death = self.module.ps_linear_models[f'{self.cause}_death'].predict( df.loc[[individual_id]])[individual_id] if self.sim.generate_event_chains: + original_aliveness_weight = df.loc[individual_id,'aliveness_weight'] + # Individual partially survives df.loc[individual_id,'aliveness_weight'] *= (1. - risk_of_death) # Individual partially dies - death_weight = df.loc[individual_id]['aliveness_weight'] * risk_of_death + death_weight = original_aliveness_weight * risk_of_death df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) df.loc[individual_id, 'death_weight'].append(death_weight) df.loc[individual_id, 'cause_of_partial_death'].append('severe_gestational_hypertension') - - # Individual always survives, so always update variables - if self.cause == 'ectopic_pregnancy': - df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' - - else: - self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') - df.at[individual_id, 'ac_received_post_abortion_care'] = False - - if individual_id in mni: - mni[individual_id]['delete_mni'] = True else: # If the death occurs we record it here if self.module.rng.random_sample() < risk_of_death: + + survived = False if individual_id in mni: pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) @@ -2139,17 +2152,19 @@ def apply(self, individual_id): self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=f'{self.cause}', originating_module=self.sim.modules['PregnancySupervisor']) - else: - # Otherwise we reset any variables - if self.cause == 'ectopic_pregnancy': - df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' + # If individual survived, updated variables + if survived: + + # If woman survived we reset any variables + if self.cause == 'ectopic_pregnancy': + df.at[individual_id, 'ps_ectopic_pregnancy'] = 'none' - else: - self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') - df.at[individual_id, 'ac_received_post_abortion_care'] = False + else: + self.module.abortion_complications.unset(individual_id, 'sepsis', 'haemorrhage', 'injury', 'other') + df.at[individual_id, 'ac_received_post_abortion_care'] = False - if individual_id in mni: - mni[individual_id]['delete_mni'] = True + if individual_id in mni: + mni[individual_id]['delete_mni'] = True class GestationalDiabetesGlycaemicControlEvent(Event, IndividualScopeEventMixin): From b62c54f6219505207a535433477c54f001b232e2 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:46:41 +0100 Subject: [PATCH 096/103] Ensure mnh cohort has missing columns initialised to correct values --- src/tlo/methods/mnh_cohort_module.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index 1a15242e52..ab079828d8 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -54,6 +54,8 @@ def initialise_population(self, population): # Only select rows equal to the desired population size if len(self.sim.population.props) <= len(all_preg_df): preg_pop = all_preg_df.loc[0:(len(self.sim.population.props))-1] + #preg_pop = all_preg_df.iloc[0:(len(self.sim.population.props))-1].copy() + else: # Calculate the number of rows needed to reach the desired length additional_rows = len(self.sim.population.props) - len(all_preg_df) @@ -76,8 +78,26 @@ def initialise_population(self, population): # Set the dtypes and index of the cohort dataframe props_dtypes = self.sim.population.props.dtypes + + preg_pop.loc[:,'aliveness_weight'] = 1 + preg_pop.loc[:,'death_weight'] = [[] for _ in range(len(preg_pop))] + preg_pop.loc[:,'cause_of_partial_death'] = [[] for _ in range(len(preg_pop))] + preg_pop.loc[:,'date_of_partial_death'] = [[] for _ in range(len(preg_pop))] + + print(preg_pop.columns) + missing_cols = [col for col in self.sim.population.props.columns if col not in preg_pop.columns] + if missing_cols: + print("Missing columns") + for col in missing_cols: + print(col) + exit(-1) + + # Reorder columns + preg_pop = preg_pop[self.sim.population.props.columns] + preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) + preg_pop_final.index.name = 'person' # For the below columns we manually overwrite the dtypes From b8819c484c8c6f04d12e2bff7a94cb90b6938239 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Tue, 14 Oct 2025 16:21:52 +0100 Subject: [PATCH 097/103] Ensure series within pop dataframe are copied deeply when saving row [skip ci] --- src/tlo/events.py | 28 +++++++++++++++++++------ src/tlo/methods/mnh_cohort_module.py | 3 +-- src/tlo/methods/pregnancy_supervisor.py | 2 -- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index cdda334a11..92d276ae86 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -172,6 +172,12 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be chain_links[key] = str(link_info) + # Ensure the partial death events are cleared after event + #df = self.sim.population.props + #df.loc[:,'death_weight'] = pd.Series([[] for _ in range(len(df))]) + #df.loc[:,'cause_of_partial_death'] = pd.Series([[] for _ in range(len(df))]) + #df.loc[:,'date_of_partial_death'] = pd.Series([[] for _ in range(len(df))]) + return chain_links @@ -198,8 +204,14 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame if self.target != self.sim.population: # Save row for comparison after event has occurred - row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) - + row_before = self.sim.population.props.loc[[abs(self.target)], :].copy(deep=True).iloc[0] + row_before = row_before.fillna(-99999) + # Recursively deep copy all object columns (like lists, Series, dicts) + for col in row_before.index: + val = row_before[col] + if isinstance(val, (list, dict, pd.Series)): + row_before[col] = copy.deepcopy(val) + # Check if individual is already in mni dictionary, if so copy her original status mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info if self.target in mni: @@ -222,10 +234,10 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Target is single individual if self.target != self.sim.population: - + # Copy full new status for individual row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) - + # Check if individual is in mni after the event mni_instances_after = False mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info @@ -268,6 +280,11 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Add individual to the chain links chain_links[self.target] = str(link_info) + # Ensure the partial death events are cleared after event + #df = self.sim.population.props + #df.at[self.target,'death_weight'] = [] + #df.at[self.target,'cause_of_partial_death'] = [] + #df.at[self.target,'date_of_partial_death'] = [] else: # Target is entire population. Identify individuals for which properties have changed # and store their changes. @@ -278,7 +295,7 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Create and store the event and dictionary of changes for affected individuals chain_links = self.compare_population_dataframe_and_mni(df_before, df_after, entire_mni_before, entire_mni_after) - + return chain_links @@ -303,7 +320,6 @@ def run(self): # Log chain_links here if len(chain_links)>0: - print(chain_links) logger_chain.info(key='event_chains', data= pop_dict, description='Links forming chains of events for simulated individuals') diff --git a/src/tlo/methods/mnh_cohort_module.py b/src/tlo/methods/mnh_cohort_module.py index ab079828d8..77ba618748 100644 --- a/src/tlo/methods/mnh_cohort_module.py +++ b/src/tlo/methods/mnh_cohort_module.py @@ -96,8 +96,7 @@ def initialise_population(self, population): preg_pop = preg_pop[self.sim.population.props.columns] preg_pop_final = preg_pop.astype(props_dtypes.to_dict()) - - + preg_pop_final.index.name = 'person' # For the below columns we manually overwrite the dtypes diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 0e66ffb70d..8da8fa575c 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -2124,7 +2124,6 @@ def apply(self, individual_id): df.loc[[individual_id]])[individual_id] if self.sim.generate_event_chains: - original_aliveness_weight = df.loc[individual_id,'aliveness_weight'] # Individual partially survives @@ -2135,7 +2134,6 @@ def apply(self, individual_id): df.loc[individual_id, 'date_of_partial_death'].append(str(self.sim.date)) df.loc[individual_id, 'death_weight'].append(death_weight) df.loc[individual_id, 'cause_of_partial_death'].append('severe_gestational_hypertension') - else: # If the death occurs we record it here From bc61e1efbf7c79c4b85273b5b3c893c0030b362d Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Mon, 17 Nov 2025 10:21:22 +0000 Subject: [PATCH 098/103] Cleaned and [skip ci] --- src/tlo/simulation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 8356424901..ef2fe4518e 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -108,9 +108,11 @@ def __init__( self.date = self.start_date = start_date self.modules = OrderedDict() self.event_queue = EventQueue() + self.generate_event_chains = True self.generate_event_chains_modules_of_interest = [] self.generate_event_chains_ignore_events = [] + self.end_date = None self.output_file = None self.population: Optional[Population] = None From e084e3949c03a8e19bc49f42aea56a154d09dabf Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:38:07 +0000 Subject: [PATCH 099/103] Start logging data in EAV format --- src/tlo/events.py | 17 ++++++++++------- src/tlo/simulation.py | 11 ++++++++++- src/tlo/util.py | 23 +++++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 993c27090c..9e9865cdad 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -11,7 +11,7 @@ import pandas as pd -from tlo.util import FACTOR_POP_DICT +from tlo.util import FACTOR_POP_DICT, convert_dict_into_eav import copy @@ -233,12 +233,12 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, mni_instances_after = True # Create and store event for this individual, regardless of whether any property change occurred - link_info = { - #'person_ID' : self.target, - 'person_ID' : self.target, - 'event' : type(self).__name__, - 'event_date' : self.sim.date, - } + link_info = {} + # #'person_ID' : self.target, + # 'person_ID' : self.target, + # 'event' : type(self).__name__, + # 'event_date' : self.sim.date, + #} # Store (if any) property changes as a result of the event for this individual for key in row_before.index: @@ -265,6 +265,9 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, link_info[key] = mni[self.target][key] # Else, no need to do anything + eav = convert_dict_into_eav(link_info, self.target, self.sim.date, type(self).__name__) + print(eav) + exit(-1) # Add individual to the chain links chain_links[self.target] = str(link_info) diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index ef2fe4518e..ef27fa6381 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -13,7 +13,7 @@ import pandas as pd import tlo.population import numpy as np -from tlo.util import FACTOR_POP_DICT +from tlo.util import FACTOR_POP_DICT, df_to_eav try: import dill @@ -290,6 +290,11 @@ def make_initial_population(self, *, n: int) -> None: # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. if self.generate_event_chains: + print(len(self.population.props), n) + # EAV structure to capture status of individuals at the start of the simulation + eav = df_to_eav(self.population.props, self.date, 'StartOfSimulation') + + """ pop_dict = self.population.props.to_dict(orient='index') for key in pop_dict.keys(): @@ -302,6 +307,10 @@ def make_initial_population(self, *, n: int) -> None: logger.info(key='event_chains', data = pop_dict_full, description='Links forming chains of events for simulated individuals') + """ + logger.info(key='event_chains', + data = eav.to_dict(), + description='Links forming chains of events for simulated individuals') end = time.time() logger.info(key="info", data=f"make_initial_population() {end - start} s") diff --git a/src/tlo/util.py b/src/tlo/util.py index c9130e3f07..e83e19baab 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -94,6 +94,29 @@ def transition_states(initial_series: pd.Series, prob_matrix: pd.DataFrame, rng: return final_states +def df_to_eav(df, date, event_name): + """Function to convert dataframe into EAV""" + eav = df.stack().reset_index() + eav.columns = ['E', 'A', 'V'] + eav['Date'] = date + eav['NameEvent'] = event_name + eav = eav[["E", "Date", "NameEvent", "A", "V"]] + + return eav + + +def convert_dict_into_eav(link_info, target, date, event_name): + "Function to convert link info in the form of dictionary into an EAV" + eav = pd.DataFrame(list(link_info.items()), columns=['A', 'V']) + eav.columns = ['A', 'V'] + eav['E'] = target + eav['Date'] = date + eav['NameEvent'] = event_name + eav = eav[['E', 'Date', 'NameEvent', 'A', 'V']] + + return eav + + def sample_outcome(probs: pd.DataFrame, rng: np.random.RandomState): """ Helper function to randomly sample an outcome for each individual in a population from a set of probabilities that are specific to each individual. From ac617e80ff416976229b3f3bdd915198a26da96c Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 21 Nov 2025 13:15:07 +0000 Subject: [PATCH 100/103] Log event chains via EAV approach --- .../analysis_extract_data.py | 27 ++++- .../scenario_generate_chains.py | 4 +- src/tlo/analysis/utils.py | 111 ++++++++++++++++++ src/tlo/events.py | 41 +++---- src/tlo/methods/hsi_event.py | 21 ++-- src/tlo/simulation.py | 40 ++----- src/tlo/util.py | 30 +++-- 7 files changed, 200 insertions(+), 74 deletions(-) diff --git a/src/scripts/analysis_data_generation/analysis_extract_data.py b/src/scripts/analysis_data_generation/analysis_extract_data.py index 7fe15f0eb4..9ee37cabef 100644 --- a/src/scripts/analysis_data_generation/analysis_extract_data.py +++ b/src/scripts/analysis_data_generation/analysis_extract_data.py @@ -11,7 +11,7 @@ import matplotlib.pyplot as plt from tlo import Date -from tlo.analysis.utils import extract_results +from tlo.analysis.utils import extract_results, extract_event_chains from datetime import datetime from collections import Counter import ast @@ -35,6 +35,27 @@ def check_if_beyond_time_range_considered(progression_properties): if progression_properties[key] > end_date: print("Beyond time range considered, need at least ",progression_properties[key]) +def print_filtered_df(df): + """ + Prints rows of the DataFrame excluding EventName 'Initialise' and 'Birth'. + """ + pd.set_option('display.max_colwidth', None) + filtered = df#[~df['EventName'].isin(['StartOfSimulation', 'Birth'])] + + dict_cols = ["Info"] + max_items = 2 + # Step 2: Truncate dictionary columns for display + if dict_cols is not None: + for col in dict_cols: + def truncate_dict(d): + if isinstance(d, dict): + items = list(d.items())[:max_items] # keep only first `max_items` + return dict(items) + return d + filtered[col] = filtered[col].apply(truncate_dict) + print(filtered) + + def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None, ): """Produce standard set of plots describing the effect of each TREATMENT_ID. - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. @@ -43,6 +64,10 @@ def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = No pd.set_option('display.max_rows', None) pd.set_option('display.max_colwidth', None) + individual_event_chains = extract_event_chains(results_folder) + print_filtered_df(individual_event_chains[0]) + exit(-1) + eval_env = { 'datetime': datetime, # Add the datetime class to the eval environment 'pd': pd, # Add pandas to handle Timestamp diff --git a/src/scripts/analysis_data_generation/scenario_generate_chains.py b/src/scripts/analysis_data_generation/scenario_generate_chains.py index e9291a50ce..6cfbd040fa 100644 --- a/src/scripts/analysis_data_generation/scenario_generate_chains.py +++ b/src/scripts/analysis_data_generation/scenario_generate_chains.py @@ -53,11 +53,11 @@ def __init__(self): super().__init__() self.seed = 42 self.start_date = Date(2010, 1, 1) - self.end_date = self.start_date + pd.DateOffset(months=18) + self.end_date = self.start_date + pd.DateOffset(months=1) self.pop_size = 1000 self._scenarios = self._get_scenarios() self.number_of_draws = len(self._scenarios) - self.runs_per_draw = 1 + self.runs_per_draw = 3 self.generate_event_chains = True def log_configuration(self): diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index e605400332..f762f1eb92 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -341,6 +341,117 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: _concat = pd.concat(res, axis=1) _concat.columns.names = ['draw', 'run'] # name the levels of the columns multi-index return _concat + + +import pandas as pd + +def unpack_dict_rows(df): + """ + Reconstruct a full dataframe from rows whose columns contain dictionaries + mapping local-row-index → value. Preserves original column order. + """ + original_cols = ['E', 'EventDate', 'EventName', 'A', 'V'] + reconstructed_rows = [] + + for _, row in df.iterrows(): + # Determine how many rows this block has (using the first dict column) + first_dict_col = next(col for col in original_cols if isinstance(row[col], dict)) + block_length = len(row[first_dict_col]) + + # Build each reconstructed row + for i in range(block_length): + new_row = {} + for col in original_cols: + cell = row[col] + if not isinstance(cell, dict): + raise ValueError(f"Column {col} does not contain a dictionary") + new_row[col] = cell.get(str(i)) + reconstructed_rows.append(new_row) + + # Build DataFrame and enforce the original column order + out = pd.DataFrame(reconstructed_rows)[original_cols] + return out.reset_index(drop=True) + + +def print_filtered_df(df): + """ + Prints rows of the DataFrame excluding EventName 'Initialise' and 'Birth'. + """ + pd.set_option('display.max_colwidth', None) + filtered = df#[~df['EventName'].isin(['StartOfSimulation', 'Birth'])] + + dict_cols = ["Info"] + max_items = 2 + # Step 2: Truncate dictionary columns for display + if dict_cols is not None: + for col in dict_cols: + def truncate_dict(d): + if isinstance(d, dict): + items = list(d.items())[:max_items] # keep only first `max_items` + return dict(items) + return d + filtered[col] = filtered[col].apply(truncate_dict) + print(filtered) + + +def extract_event_chains(results_folder: Path, + ) -> dict: + """Utility function to collect chains of events. Individuals across runs of the same draw will be combined into unique df. + Returns dictionary where keys are draws, and each draw is associated with a dataframe of format 'E', 'EventDate', 'EventName', 'Info' where 'Info' is a dictionary that combines A&Vs for a particular individual + date + event name combination. + """ + module = 'tlo.simulation' + key = 'event_chains' + + # get number of draws and numbers of runs + info = get_scenario_info(results_folder) + + # Collect results from each draw/run. Individuals across runs of the same draw will be combined into unique df. + res = dict() + + for draw in range(info['number_of_draws']): + + # All individuals in same draw will be combined across runs, so their ID will be offset. + dfs_from_runs = [] + ID_offset = 0 + + for run in range(info['runs_per_draw']): + + try: + df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key] + del df['date'] + recon = unpack_dict_rows(df) + # For now convert value to string in all cases to facilitate manipulation. This can be reversed later. + recon['V'] = recon['V'].apply(str) + # Collapse into 'E', 'EventDate', 'EventName', 'Info' format where 'Info' is dict listing attributes (e.g. {a1:v1, a2:v2, a3:v3, ...} ) + df_collapsed = ( + recon.groupby(['E', 'EventDate', 'EventName']) + .apply(lambda g: dict(zip(g['A'], g['V']))) + .reset_index(name='Info') + ) + df_final = df_collapsed.sort_values(by=['E','EventDate'], ascending=True).reset_index(drop=True) + birth_count = (df_final['EventName'] == 'Birth').sum() + + print("Birth count for run ", run, "is ", birth_count) + df_final['E'] = df_final['E'] + ID_offset + + # Calculate ID offset for next run + ID_offset = (max(df_final['E']) + 1) + + # Append these chains to list + dfs_from_runs.append(df_final) + + except KeyError: + # Some logs could not be found - probably because this run failed. + # Simply to not append anything to the df collecting chains. + print("Run failed") + + # Combine all dfs into a single DataFrame + res[draw] = pd.concat(dfs_from_runs, ignore_index=True) + + # Optionally, sort by 'E' and 'EventDate' after combining + res[draw] = res[draw].sort_values(by=['E', 'EventDate']).reset_index(drop=True) + + return res def summarize(results: pd.DataFrame, only_mean: bool = False, collapse_columns: bool = False) -> pd.DataFrame: diff --git a/src/tlo/events.py b/src/tlo/events.py index 9e9865cdad..ba91218dbc 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -11,7 +11,7 @@ import pandas as pd -from tlo.util import FACTOR_POP_DICT, convert_dict_into_eav +from tlo.util import convert_chain_links_into_EAV import copy @@ -139,9 +139,8 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be # Create a dictionary for this person # First add event info link_info = { - 'person_ID': idx, - 'event': type(self).__name__, - 'event_date': self.sim.date, + 'EventDate': self.sim.date, + 'EventName': type(self).__name__, } # Store the new values from df_after for the changed columns @@ -154,7 +153,7 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be link_info[col] = diff_mni[idx][key] # Append the event and changes to the individual key - chain_links[idx] = str(link_info) + chain_links[idx] = link_info # For individuals which only underwent changes in mni dictionary, save changes here if len(diff_mni)>0: @@ -162,15 +161,14 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be if key not in persons_changed: # If individual hadn't been previously added due to changes in pop df, add it here link_info = { - 'person_ID': key, - 'event': type(self).__name__, - 'event_date': self.sim.date, + 'EventDate': self.sim.date, + 'EventName': type(self).__name__, } for key_prop in diff_mni[key]: link_info[key_prop] = diff_mni[key][key_prop] - chain_links[key] = str(link_info) + chain_links[key] = link_info return chain_links @@ -233,12 +231,10 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, mni_instances_after = True # Create and store event for this individual, regardless of whether any property change occurred - link_info = {} - # #'person_ID' : self.target, - # 'person_ID' : self.target, - # 'event' : type(self).__name__, - # 'event_date' : self.sim.date, - #} + link_info = { + 'EventDate' : self.sim.date, + 'EventName' : type(self).__name__, + } # Store (if any) property changes as a result of the event for this individual for key in row_before.index: @@ -265,11 +261,8 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, link_info[key] = mni[self.target][key] # Else, no need to do anything - eav = convert_dict_into_eav(link_info, self.target, self.sim.date, type(self).__name__) - print(eav) - exit(-1) # Add individual to the chain links - chain_links[self.target] = str(link_info) + chain_links[self.target] = link_info else: # Target is entire population. Identify individuals for which properties have changed @@ -300,6 +293,14 @@ def run(self): if self.sim.generate_event_chains and print_chains: chain_links = self.store_chains_to_do_after_event(row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before) + if chain_links: + # Convert chain_links into EAV + ednav = convert_chain_links_into_EAV(chain_links) + + logger_chain.info(key='event_chains', + data= ednav.to_dict(), + description='Links forming chains of events for simulated individuals') + """ # Create empty logger for entire pop pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals pop_dict.update(chain_links) @@ -310,7 +311,7 @@ def run(self): logger_chain.info(key='event_chains', data= pop_dict, description='Links forming chains of events for simulated individuals') - + """ class RegularEvent(Event): """An event that automatically reschedules itself at a fixed frequency.""" diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 85ac6da3e2..59b7b1f60a 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -8,7 +8,7 @@ from tlo import Date, logging from tlo.events import Event from tlo.population import Population -from tlo.util import FACTOR_POP_DICT +from tlo.util import convert_chain_links_into_EAV import pandas as pd @@ -266,9 +266,8 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, record_level = 'N/A' link_info = { - 'person_ID': self.target, - 'event' : type(self).__name__, - 'event_date' : self.sim.date, + 'EventName' : type(self).__name__, + 'EventDate' : self.sim.date, 'appt_footprint' : record_footprint, 'level' : record_level, } @@ -297,7 +296,7 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, if self.values_differ(default[key], mni[self.target][key]): link_info[key] = mni[self.target][key] - chain_links[self.target] = str(link_info) + chain_links[self.target] = link_info return chain_links @@ -325,13 +324,13 @@ def run(self, squeeze_factor): if print_chains: chain_links = self.store_chains_to_do_after_event(row_before, str(footprint), mni_row_before, mni_instances_before) - if len(chain_links)>0: - pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} - pop_dict.update(chain_links) + if chain_links: - logger_chains.info(key='event_chains', - data = pop_dict, - description='Links forming chains of events for simulated individuals') + # Convert chain_links into EAV + ednav = convert_chain_links_into_EAV(chain_links) + logger_chain.info(key='event_chains', + data = ednav, + description='Links forming chains of events for simulated individuals') return updated_appt_footprint diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index ef27fa6381..da55d42efc 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -13,7 +13,7 @@ import pandas as pd import tlo.population import numpy as np -from tlo.util import FACTOR_POP_DICT, df_to_eav +from tlo.util import df_to_EAV, convert_chain_links_into_EAV try: import dill @@ -290,26 +290,11 @@ def make_initial_population(self, *, n: int) -> None: # At the start of the simulation + when a new individual is born, we therefore want to store all of their properties at the start. if self.generate_event_chains: - print(len(self.population.props), n) - # EAV structure to capture status of individuals at the start of the simulation - eav = df_to_eav(self.population.props, self.date, 'StartOfSimulation') - - """ - pop_dict = self.population.props.to_dict(orient='index') - - for key in pop_dict.keys(): - pop_dict[key]['person_ID'] = key - pop_dict[key] = str(pop_dict[key]) # Log as string to avoid issues around length of properties stored later - - pop_dict_full = {i: '' for i in range(FACTOR_POP_DICT)} - pop_dict_full.update(pop_dict) - - logger.info(key='event_chains', - data = pop_dict_full, - description='Links forming chains of events for simulated individuals') - """ + # EDNAV structure to capture status of individuals at the start of the simulation + ednav = df_to_EAV(self.population.props, self.date, 'StartOfSimulation') + logger.info(key='event_chains', - data = eav.to_dict(), + data = ednav.to_dict(), description='Links forming chains of events for simulated individuals') end = time.time() @@ -475,15 +460,16 @@ def do_birth(self, mother_id: int) -> int: if self.generate_event_chains: # When individual is born, store their initial properties to provide a starting point to the chain of property # changes that this individual will undergo as a result of events taking place. - prop_dict = self.population.props.loc[child_id].to_dict() - prop_dict['event'] = 'Birth' - prop_dict['event_date'] = self.date - - pop_dict = {i: '' for i in range(FACTOR_POP_DICT)} # Always include all possible individuals - pop_dict[child_id] = str(prop_dict) # Convert to string to avoid issue of length + link_info = self.population.props.loc[child_id].to_dict() + link_info['EventName'] = 'Birth' + link_info['EventDate'] = self.date + chain_links = {} + chain_links[child_id] = link_info # Convert to string to avoid issue of length + ednav = convert_chain_links_into_EAV(chain_links) + logger.info(key='event_chains', - data = pop_dict, + data = ednav.to_dict(), description='Links forming chains of events for simulated individuals') return child_id diff --git a/src/tlo/util.py b/src/tlo/util.py index e83e19baab..ee29445e9a 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -13,7 +13,6 @@ # Default mother_id value, assigned to individuals initialised as adults at the start of the simulation. DEFAULT_MOTHER_ID = -1e7 -FACTOR_POP_DICT = 50000 def create_age_range_lookup(min_age: int, max_age: int, range_size: int = 5) -> (list, Dict[int, str]): @@ -94,25 +93,30 @@ def transition_states(initial_series: pd.Series, prob_matrix: pd.DataFrame, rng: return final_states -def df_to_eav(df, date, event_name): +def df_to_EAV(df, date, event_name): """Function to convert dataframe into EAV""" eav = df.stack().reset_index() eav.columns = ['E', 'A', 'V'] - eav['Date'] = date - eav['NameEvent'] = event_name - eav = eav[["E", "Date", "NameEvent", "A", "V"]] + eav['EventDate'] = date + eav['EventName'] = event_name + eav = eav[["E", "EventDate", "EventName", "A", "V"]] return eav -def convert_dict_into_eav(link_info, target, date, event_name): - "Function to convert link info in the form of dictionary into an EAV" - eav = pd.DataFrame(list(link_info.items()), columns=['A', 'V']) - eav.columns = ['A', 'V'] - eav['E'] = target - eav['Date'] = date - eav['NameEvent'] = event_name - eav = eav[['E', 'Date', 'NameEvent', 'A', 'V']] +def convert_chain_links_into_EAV(chain_links): + df = pd.DataFrame.from_dict(chain_links, orient="index") + id_cols = ["EventDate", "EventName"] + + eav = df.reset_index().melt( + id_vars=["index"] + id_cols, # index = person ID + var_name="A", + value_name="V" + ) + + eav.rename(columns={"index": "E"}, inplace=True) + + eav = eav[["E", "EventDate", "EventName", "A", "V"]] return eav From 5234550934fd0bf156e43603d593945c66d888c0 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 21 Nov 2025 13:44:52 +0000 Subject: [PATCH 101/103] No need to store EventDate since this is already stored in logger by default --- src/tlo/analysis/utils.py | 62 ++++++++++++++++++++++++++++++++---- src/tlo/events.py | 3 -- src/tlo/methods/hsi_event.py | 1 - src/tlo/simulation.py | 1 - src/tlo/util.py | 7 ++-- 5 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index f762f1eb92..00a297030b 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -345,7 +345,7 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series: import pandas as pd -def unpack_dict_rows(df): +def old_unpack_dict_rows(df): """ Reconstruct a full dataframe from rows whose columns contain dictionaries mapping local-row-index → value. Preserves original column order. @@ -372,6 +372,54 @@ def unpack_dict_rows(df): out = pd.DataFrame(reconstructed_rows)[original_cols] return out.reset_index(drop=True) + +def unpack_dict_rows(df, non_dict_cols=None): + """ + Reconstruct a full DataFrame from rows where most columns are dictionaries. + Non-dict columns (e.g., 'date') are propagated to all reconstructed rows. + + Parameters: + df: pd.DataFrame + non_dict_cols: list of columns that are NOT dictionaries + """ + if non_dict_cols is None: + non_dict_cols = [] + + original_cols = ['E', 'date', 'EventName', 'A', 'V'] + + reconstructed_rows = [] + + for _, row in df.iterrows(): + # Determine dict columns for this row + dict_cols = [col for col in original_cols if col not in non_dict_cols] + + if not dict_cols: + # No dict columns, just append row + reconstructed_rows.append(row.to_dict()) + continue + + # Use the first dict column to get the block length + first_dict_col = dict_cols[0] + block_length = len(row[first_dict_col]) + + # Build each expanded row + for i in range(block_length): + new_row = {} + for col in original_cols: + cell = row[col] + if col in dict_cols: + # Access the dict using string or integer keys + new_row[col] = cell.get(str(i), cell.get(i)) + else: + # Propagate non-dict value + new_row[col] = cell + reconstructed_rows.append(new_row) + + # Build DataFrame in original column order + out = pd.DataFrame(reconstructed_rows)[original_cols] + + return out.reset_index(drop=True) + def print_filtered_df(df): """ @@ -418,17 +466,19 @@ def extract_event_chains(results_folder: Path, try: df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key] - del df['date'] - recon = unpack_dict_rows(df) + + recon = unpack_dict_rows(df, ['date']) + print(recon) + #del recon['EventDate'] # For now convert value to string in all cases to facilitate manipulation. This can be reversed later. recon['V'] = recon['V'].apply(str) # Collapse into 'E', 'EventDate', 'EventName', 'Info' format where 'Info' is dict listing attributes (e.g. {a1:v1, a2:v2, a3:v3, ...} ) df_collapsed = ( - recon.groupby(['E', 'EventDate', 'EventName']) + recon.groupby(['E', 'date', 'EventName']) .apply(lambda g: dict(zip(g['A'], g['V']))) .reset_index(name='Info') ) - df_final = df_collapsed.sort_values(by=['E','EventDate'], ascending=True).reset_index(drop=True) + df_final = df_collapsed.sort_values(by=['E','date'], ascending=True).reset_index(drop=True) birth_count = (df_final['EventName'] == 'Birth').sum() print("Birth count for run ", run, "is ", birth_count) @@ -449,7 +499,7 @@ def extract_event_chains(results_folder: Path, res[draw] = pd.concat(dfs_from_runs, ignore_index=True) # Optionally, sort by 'E' and 'EventDate' after combining - res[draw] = res[draw].sort_values(by=['E', 'EventDate']).reset_index(drop=True) + res[draw] = res[draw].sort_values(by=['E', 'date']).reset_index(drop=True) return res diff --git a/src/tlo/events.py b/src/tlo/events.py index ba91218dbc..4b62c16932 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -139,7 +139,6 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be # Create a dictionary for this person # First add event info link_info = { - 'EventDate': self.sim.date, 'EventName': type(self).__name__, } @@ -161,7 +160,6 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be if key not in persons_changed: # If individual hadn't been previously added due to changes in pop df, add it here link_info = { - 'EventDate': self.sim.date, 'EventName': type(self).__name__, } @@ -232,7 +230,6 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Create and store event for this individual, regardless of whether any property change occurred link_info = { - 'EventDate' : self.sim.date, 'EventName' : type(self).__name__, } diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 59b7b1f60a..d59f8e2404 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -267,7 +267,6 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, link_info = { 'EventName' : type(self).__name__, - 'EventDate' : self.sim.date, 'appt_footprint' : record_footprint, 'level' : record_level, } diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index da55d42efc..35f6818f66 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -462,7 +462,6 @@ def do_birth(self, mother_id: int) -> int: # changes that this individual will undergo as a result of events taking place. link_info = self.population.props.loc[child_id].to_dict() link_info['EventName'] = 'Birth' - link_info['EventDate'] = self.date chain_links = {} chain_links[child_id] = link_info # Convert to string to avoid issue of length diff --git a/src/tlo/util.py b/src/tlo/util.py index ee29445e9a..d678aa09ef 100644 --- a/src/tlo/util.py +++ b/src/tlo/util.py @@ -97,16 +97,15 @@ def df_to_EAV(df, date, event_name): """Function to convert dataframe into EAV""" eav = df.stack().reset_index() eav.columns = ['E', 'A', 'V'] - eav['EventDate'] = date eav['EventName'] = event_name - eav = eav[["E", "EventDate", "EventName", "A", "V"]] + eav = eav[["E", "EventName", "A", "V"]] return eav def convert_chain_links_into_EAV(chain_links): df = pd.DataFrame.from_dict(chain_links, orient="index") - id_cols = ["EventDate", "EventName"] + id_cols = ["EventName"] eav = df.reset_index().melt( id_vars=["index"] + id_cols, # index = person ID @@ -116,7 +115,7 @@ def convert_chain_links_into_EAV(chain_links): eav.rename(columns={"index": "E"}, inplace=True) - eav = eav[["E", "EventDate", "EventName", "A", "V"]] + eav = eav[["E", "EventName", "A", "V"]] return eav From 2f20cb392a9aaee1c8d004a82e4f31957d2130b8 Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 21 Nov 2025 14:45:16 +0000 Subject: [PATCH 102/103] Check if PregnancySupervisor is included before considering in chain of events production --- src/tlo/events.py | 101 ++++++++++++++++++++--------------- src/tlo/methods/hsi_event.py | 53 +++++++++--------- 2 files changed, 87 insertions(+), 67 deletions(-) diff --git a/src/tlo/events.py b/src/tlo/events.py index 4b62c16932..f03f150f92 100644 --- a/src/tlo/events.py +++ b/src/tlo/events.py @@ -122,7 +122,10 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be # Create a mask of where values are different diff_mask = (df_before != df_after) & ~(df_before.isna() & df_after.isna()) - diff_mni = self.compare_entire_mni_dicts(entire_mni_before, entire_mni_after) + if 'PregnancySupervisor' in self.sim.modules: + diff_mni = self.compare_entire_mni_dicts(entire_mni_before, entire_mni_after) + else: + diff_mni = [] # Create an empty list to store changes for each of the individuals chain_links = {} @@ -154,19 +157,20 @@ def compare_population_dataframe_and_mni(self,df_before, df_after, entire_mni_be # Append the event and changes to the individual key chain_links[idx] = link_info - # For individuals which only underwent changes in mni dictionary, save changes here - if len(diff_mni)>0: - for key in diff_mni: - if key not in persons_changed: - # If individual hadn't been previously added due to changes in pop df, add it here - link_info = { - 'EventName': type(self).__name__, - } - - for key_prop in diff_mni[key]: - link_info[key_prop] = diff_mni[key][key_prop] + if 'PregnancySupervisor' in self.sim.modules: + # For individuals which only underwent changes in mni dictionary, save changes here + if len(diff_mni)>0: + for key in diff_mni: + if key not in persons_changed: + # If individual hadn't been previously added due to changes in pop df, add it here + link_info = { + 'EventName': type(self).__name__, + } - chain_links[key] = link_info + for key_prop in diff_mni[key]: + link_info[key_prop] = diff_mni[key][key_prop] + + chain_links[key] = link_info return chain_links @@ -197,17 +201,23 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, pd.DataFrame row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) # Check if individual is already in mni dictionary, if so copy her original status - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: - mni_instances_before = True - mni_row_before = mni[self.target].copy() + if 'PregnancySupervisor' in self.sim.modules: + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_before = True + mni_row_before = mni[self.target].copy() + else: + mni_row_before = None else: # This will be a population-wide event. In order to find individuals for which this led to # a meaningful change, make a copy of the while pop dataframe/mni before the event has occurred. df_before = self.sim.population.props.copy() - entire_mni_before = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) + if 'PregnancySupervisor' in self.sim.modules: + entire_mni_before = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) + else: + entire_mni_before = None return print_chains, row_before, df_before, mni_row_before, entire_mni_before, mni_instances_before @@ -224,9 +234,12 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Check if individual is in mni after the event mni_instances_after = False - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: - mni_instances_after = True + if 'PregnancySupervisor' in self.sim.modules: + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_after = True + else: + mni_instances_after = None # Create and store event for this individual, regardless of whether any property change occurred link_info = { @@ -237,26 +250,27 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, for key in row_before.index: if row_before[key] != row_after[key]: # Note: used fillna previously, so this is safe link_info[key] = row_after[key] - - # Now check and store changes in the mni dictionary, accounting for following cases: - # Individual is in mni dictionary before and after - if mni_instances_before and mni_instances_after: - for key in mni_row_before: - if self.mni_values_differ(mni_row_before[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - # Individual is only in mni dictionary before event - elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_all_mni_values - for key in mni_row_before: - if self.mni_values_differ(mni_row_before[key], default[key]): - link_info[key] = default[key] - # Individual is only in mni dictionary after event - elif mni_instances_after and not mni_instances_before: - default = self.sim.modules['PregnancySupervisor'].default_all_mni_values - for key in default: - if self.mni_values_differ(default[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - # Else, no need to do anything + + if 'PregnancySupervisor' in self.sim.modules: + # Now check and store changes in the mni dictionary, accounting for following cases: + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.mni_values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values + for key in mni_row_before: + if self.mni_values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values + for key in default: + if self.mni_values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Else, no need to do anything # Add individual to the chain links chain_links[self.target] = link_info @@ -267,7 +281,10 @@ def store_chains_to_do_after_event(self, row_before, df_before, mni_row_before, # Population frame after event df_after = self.sim.population.props - entire_mni_after = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) + if 'PregnancySupervisor' in self.sim.modules: + entire_mni_after = copy.deepcopy(self.sim.modules['PregnancySupervisor'].mother_and_newborn_info) + else: + entire_mni_after = None # Create and store the event and dictionary of changes for affected individuals chain_links = self.compare_population_dataframe_and_mni(df_before, df_after, entire_mni_before, entire_mni_after) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 7d960077fc..edb5d3df3b 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -239,10 +239,11 @@ def store_chains_to_do_before_event(self) -> tuple[bool, pd.Series, dict, bool]: row_before = self.sim.population.props.loc[abs(self.target)].copy().fillna(-99999) # Check if individual is in mni dictionary before the event, if so store its original status - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: - mni_instances_before = True - mni_row_before = mni[self.target].copy() + if 'PregnancySupervisor' in self.sim.modules: + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_before = True + mni_row_before = mni[self.target].copy() else: print("ERROR: there shouldn't be pop-wide HSI event") @@ -259,9 +260,10 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, row_after = self.sim.population.props.loc[abs(self.target)].fillna(-99999) mni_instances_after = False - mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info - if self.target in mni: - mni_instances_after = True + if 'PregnancySupervisor' in self.sim.modules: + mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + if self.target in mni: + mni_instances_after = True # Create and store dictionary of changes. Note that person_ID, event, event_date, appt_foot, and level # will be stored regardless of whether individual experienced property changes or not. @@ -285,24 +287,25 @@ def store_chains_to_do_after_event(self, row_before, footprint, mni_row_before, if row_before[key] != row_after[key]: # Note: used fillna previously link_info[key] = row_after[key] - # Now store changes in the mni dictionary, accounting for following cases: - # Individual is in mni dictionary before and after - if mni_instances_before and mni_instances_after: - for key in mni_row_before: - if self.values_differ(mni_row_before[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] - # Individual is only in mni dictionary before event - elif mni_instances_before and not mni_instances_after: - default = self.sim.modules['PregnancySupervisor'].default_all_mni_values - for key in mni_row_before: - if self.values_differ(mni_row_before[key], default[key]): - link_info[key] = default[key] - # Individual is only in mni dictionary after event - elif mni_instances_after and not mni_instances_before: - default = self.sim.modules['PregnancySupervisor'].default_all_mni_values - for key in default: - if self.values_differ(default[key], mni[self.target][key]): - link_info[key] = mni[self.target][key] + if 'PregnancySupervisor' in self.sim.modules: + # Now store changes in the mni dictionary, accounting for following cases: + # Individual is in mni dictionary before and after + if mni_instances_before and mni_instances_after: + for key in mni_row_before: + if self.values_differ(mni_row_before[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] + # Individual is only in mni dictionary before event + elif mni_instances_before and not mni_instances_after: + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values + for key in mni_row_before: + if self.values_differ(mni_row_before[key], default[key]): + link_info[key] = default[key] + # Individual is only in mni dictionary after event + elif mni_instances_after and not mni_instances_before: + default = self.sim.modules['PregnancySupervisor'].default_all_mni_values + for key in default: + if self.values_differ(default[key], mni[self.target][key]): + link_info[key] = mni[self.target][key] chain_links[self.target] = link_info From 71b73bf1abce96188cabd31a4c9862986da49baa Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Fri, 21 Nov 2025 15:51:19 +0000 Subject: [PATCH 103/103] Fix last conflicts --- .../methods/care_of_women_during_pregnancy.py | 9 +---- src/tlo/methods/labour.py | 36 ------------------- src/tlo/methods/pregnancy_supervisor.py | 3 +- 3 files changed, 3 insertions(+), 45 deletions(-) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index bb8655f133..029e84c6e5 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -1220,11 +1220,7 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event): q_param=[l_params['prob_hcw_avail_blood_tran'], l_params['mean_hcw_competence_hp']], cons=self.item_codes_preg_consumables['blood_transfusion'], opt_cons=self.item_codes_preg_consumables['blood_test_equipment'], -<<<<<<< HEAD - equipment={'Drip stand', 'Infusion pump'}) -======= equipment=hsi_event.healthcare_system.equipment.from_pkg_names('Blood Transfusion')) ->>>>>>> master if blood_transfusion_delivered: @@ -2565,11 +2561,8 @@ def apply(self, person_id, squeeze_factor): q_param=[l_params['prob_hcw_avail_retained_prod'], l_params['mean_hcw_competence_hp']], cons=pac_cons, opt_cons=pac_opt_cons, -<<<<<<< HEAD - equipment={'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) -======= + equipment={'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump', 'Evacuation set'}) ->>>>>>> master if pac_delivered: df.at[person_id, 'ac_received_post_abortion_care'] = True diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 3c57b6b27d..4eb933cf30 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1409,10 +1409,6 @@ def progression_of_hypertensive_disorders(self, individual_id, property_prefix): params = self.current_parameters mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info -<<<<<<< HEAD - -======= ->>>>>>> master # n.b. on birth women whose hypertension will continue into the postnatal period have their disease state stored # in a new property therefore antenatal/intrapartum hypertension is 'ps_htn_disorders' and postnatal is # 'pn_htn_disorders' hence the use of property prefix variable (as this function is called before and after @@ -1515,17 +1511,6 @@ def apply_risk_of_early_postpartum_death(self, individual_id): self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, originating_module=self.sim.modules['Labour']) -<<<<<<< HEAD -======= - # If a cause is returned death is scheduled - if potential_cause_of_death: - pregnancy_helper_functions.log_mni_for_maternal_death(self, individual_id) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 - - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['Labour']) ->>>>>>> master - # If she hasn't died from any complications, we reset some key properties that resolve after risk of death # has been applied @@ -2096,11 +2081,7 @@ def blood_transfusion(self, hsi_event): q_param=[params['prob_hcw_avail_blood_tran'], params[f'mean_hcw_competence_{deliv_location}']], cons=self.item_codes_lab_consumables['blood_transfusion'], opt_cons=self.item_codes_lab_consumables['blood_test_equipment'], -<<<<<<< HEAD - equipment={'Drip stand', 'Infusion pump'}) -======= equipment=hsi_event.healthcare_system.equipment.from_pkg_names('Blood Transfusion')) ->>>>>>> master if blood_transfusion_delivered: mni[person_id]['received_blood_transfusion'] = True @@ -2590,19 +2571,10 @@ def apply(self, individual_id): else: -<<<<<<< HEAD # Function checks df for any potential cause of death, uses CFR parameters to determine risk of death # (either from one or multiple causes) and if death occurs returns the cause potential_cause_of_death = pregnancy_helper_functions.check_for_risk_of_death_from_cause_maternal( self.module, individual_id=individual_id, timing='intrapartum') -======= - # If a cause is returned death is scheduled - if potential_cause_of_death: - pregnancy_helper_functions.log_mni_for_maternal_death(self.module, individual_id) - self.sim.modules['PregnancySupervisor'].mnh_outcome_counter['direct_mat_death'] += 1 - self.sim.modules['Demography'].do_death(individual_id=individual_id, cause=potential_cause_of_death, - originating_module=self.sim.modules['Labour']) ->>>>>>> master # If a cause is returned death is scheduled if potential_cause_of_death: @@ -2864,9 +2836,6 @@ def apply(self, person_id, squeeze_factor): # Add used equipment self.add_equipment({'Delivery set', 'Weighing scale', 'Stethoscope, foetal, monaural, Pinard, plastic', 'Resuscitaire', 'Sphygmomanometer', 'Tray, emergency', 'Suction machine', -<<<<<<< HEAD - 'Thermometer', 'Drip stand', 'Infusion pump'}) -======= 'Thermometer', 'Drip stand', 'Infusion pump', 'Board for Cord Knotting', 'Cot, baby (bassinet), hospital-type', 'Delivery Beds with Stirrups', 'Fetoscope', 'Mucous Extractor for neonates', 'Amnio hook', 'Incubator, infant'}) @@ -2874,7 +2843,6 @@ def apply(self, person_id, squeeze_factor): if self.ACCEPTED_FACILITY_LEVEL == '2': self.add_equipment({'Cardiotocography'}) ->>>>>>> master # ===================================== PROPHYLACTIC CARE =================================================== # The following function manages the consumables and administration of prophylactic interventions in labour # (clean delivery practice, antibiotics for PROM, steroids for preterm labour) @@ -2946,12 +2914,8 @@ def apply(self, person_id, squeeze_factor): neo_resus_delivered = pregnancy_helper_functions.check_int_deliverable( self.module, int_name='neo_resus', hsi_event=self, q_param=[params['prob_hcw_avail_neo_resus'], params[f'mean_hcw_competence_{deliv_location}']], -<<<<<<< HEAD - cons=self.module.item_codes_lab_consumables['resuscitation']) -======= cons=self.module.item_codes_lab_consumables['resuscitation'], equipment={'Ambu bag, infant with mask', 'Resuscitator, manual, infant'}) ->>>>>>> master if neo_resus_delivered: mni[person_id]['neo_will_receive_resus_if_needed'] = True diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index d13075dbec..ef522d62bd 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -28,7 +28,7 @@ from tlo.methods.causes import Cause from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin -from tlo.util import BitsetHandler +from tlo.util import BitsetHandler,read_csv_files from tlo.methods.demography import InstantaneousPartialDeath if TYPE_CHECKING: @@ -549,6 +549,7 @@ def read_parameters(self, resourcefilepath: Optional[Path] = None): files='parameter_values') self.load_parameters_from_dataframe(parameter_dataframe) + # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module. # Currently this module calculates and reports all DALY weights from all maternal modules if 'HealthBurden' in self.sim.modules.keys():