From 5200ee04961858adcf8c6df2b9b4c92b827bfc2a Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Fri, 18 Mar 2022 14:52:13 -0400 Subject: [PATCH 01/15] Initial testing of get_dataloaders --- sbi/inference/base.py | 75 ++++++++++++++++++++++++++++++++- sbi/inference/snpe/snpe_base.py | 16 +++---- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 5a2fb7c57..23450a29e 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -161,6 +161,7 @@ def get_simulations( starting_round: int = 0, exclude_invalid_x: bool = True, warn_on_invalid: bool = True, + warn_if_zscoring: Optional[bool] = True, ) -> Tuple[Tensor, Tensor, Tensor]: r"""Returns all $\theta$, $x$, and prior_masks from rounds >= `starting_round`. @@ -190,7 +191,8 @@ def get_simulations( # Check for NaNs in simulations. is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) # Check for problematic z-scoring - warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) if warn_on_invalid: warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) warn_on_invalid_x_for_snpec_leakage( @@ -216,6 +218,74 @@ def train( ) -> NeuralPosterior: raise NotImplementedError + def get_dataloaders_all( + self, + starting_round: int = 0, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + training_batch_size: int = 50, + validation_fraction: float = 0.1, + resume_training: bool = False, + dataloader_kwargs: Optional[dict] = None, + warn_if_zscoring: Optional[bool] = True, + ) -> Tuple[data.DataLoader, data.DataLoader]: + """Return dataloaders for training and validation. + + Args: + dataset: holding all theta and x, optionally masks. + training_batch_size: training arg of inference methods. + resume_training: Whether the current call is resuming training so that no + new training and validation indices into the dataset have to be created. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn). + + Returns: + Tuple of dataloaders for training and validation. + + """ + + datset = data.TensorDataset( + *self.get_simulations( + starting_round,exclude_invalid_x, warn_on_invalid = warn_on_invalid, + warn_if_zscoring = warn_if_zscoring + ) + + ) + # Get total number of training examples. + num_examples = len(dataset) + + # Select random train and validation splits from (theta, x) pairs. + num_training_examples = int((1 - validation_fraction) * num_examples) + num_validation_examples = num_examples - num_training_examples + + if not resume_training: + permuted_indices = torch.randperm(num_examples) + self.train_indices, self.val_indices = ( + permuted_indices[:num_training_examples], + permuted_indices[num_training_examples:], + ) + + # Create training and validation loaders using a subset sampler. + # Intentionally use dicts to define the default dataloader args + # Then, use dataloader_kwargs to override (or add to) any of these defaults + # https://stackoverflow.com/questions/44784577/in-method-call-args-how-to-override-keyword-argument-of-unpacked-dict + train_loader_kwargs = { + "batch_size": min(training_batch_size, num_training_examples), + "drop_last": True, + "sampler": SubsetRandomSampler(torch.arange(len(self.train_indices).tolist()) ), + } + val_loader_kwargs = { + "batch_size": min(training_batch_size, num_validation_examples), + "shuffle": False, + "drop_last": True, + "sampler": SubsetRandomSampler(torch.arange(len(self.val_indices).tolist()) ), + } + if dataloader_kwargs is not None: + train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) + val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) + + return data.DataLoader(dataset[train_indices], **train_loader_kwargs), data.DataLoader(dataset[val_indices], **val_loader_kwargs) + def get_dataloaders( self, dataset: data.TensorDataset, @@ -358,7 +428,8 @@ def _report_convergence_at_end( ) def _summarize( - self, round_: int, x_o: Union[Tensor, None], theta_bank: Tensor, x_bank: Tensor + self, round_: int, x_o: Union[Tensor, None], theta_bank: Union[Tensor,None] + , x_bank: Union[Tensor,None] ) -> None: """Update the summary_writer with statistics for a given round. diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index da950d59a..459cd75fd 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -166,6 +166,7 @@ def train( discard_prior_samples: bool = False, retrain_from_scratch: bool = False, show_train_summary: bool = False, + warn_if_zscoring: Optional[bool] = True, dataloader_kwargs: Optional[dict] = None, ) -> nn.Module: r"""Return density estimator that approximates the distribution $p(\theta|x)$. @@ -217,25 +218,20 @@ def train( if self.use_non_atomic_loss or hasattr(self, "_ran_final_round"): start_idx = self._round - theta, x, prior_masks = self.get_simulations( - start_idx, exclude_invalid_x, warn_on_invalid=True - ) - - # Dataset is shared for training and validation loaders. - dataset = data.TensorDataset(theta, x, prior_masks) - # Set the proposal to the last proposal that was passed by the user. For # atomic SNPE, it does not matter what the proposal is. For non-atomic # SNPE, we only use the latest data that was passed, i.e. the one from the # last proposal. proposal = self._proposal_roundwise[-1] - train_loader, val_loader = self.get_dataloaders( - dataset, + train_loader, val_loader = self.get_dataloaders_all( + start_idx, + exclude_invalid_x, training_batch_size, validation_fraction, resume_training, dataloader_kwargs=dataloader_kwargs, + warn_if_zscoring=warn_if_zscoring, ) # First round or if retraining from scratch: @@ -341,7 +337,7 @@ def train( self._summary["best_validation_log_probs"].append(self._best_val_log_prob) # Update tensorboard and summary dict. - self._summarize(round_=self._round, x_o=None, theta_bank=theta, x_bank=x) + self._summarize(round_=self._round, x_o=None, theta_bank=None, x_bank=None) # Update description for progress bar. if show_train_summary: From 750fb6518326722d4baad561bcf29f5f5d9100e6 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Fri, 18 Mar 2022 15:02:24 -0400 Subject: [PATCH 02/15] Minor typo --- sbi/inference/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 23450a29e..4dfc4d61d 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -244,7 +244,7 @@ def get_dataloaders_all( """ - datset = data.TensorDataset( + dataset = data.TensorDataset( *self.get_simulations( starting_round,exclude_invalid_x, warn_on_invalid = warn_on_invalid, warn_if_zscoring = warn_if_zscoring From 178e3d1ade694cb3722ce9268e8536c0916ae760 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Wed, 23 Mar 2022 14:14:20 -0400 Subject: [PATCH 03/15] Changes to SNPE append_simulations and get_dataloaders. --- sbi/inference/base.py | 27 +++++++------- sbi/inference/snpe/snpe_base.py | 64 ++++++++++++++++++++++++++------- sbi/inference/snpe/snpe_c.py | 1 + 3 files changed, 65 insertions(+), 27 deletions(-) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 4dfc4d61d..86897fecc 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -129,11 +129,14 @@ def __init__( # Initialize roundwise (theta, x, prior_masks) for storage of parameters, # simulations and masks indicating if simulations came from prior. self._theta_roundwise, self._x_roundwise, self._prior_masks = [], [], [] + self._dataset = None + self._num_sims_per_round = [] self._model_bank = [] # Initialize list that indicates the round from which simulations were drawn. self._data_round_index = [] + self._round = 0 self._val_log_prob = float("-Inf") @@ -221,13 +224,10 @@ def train( def get_dataloaders_all( self, starting_round: int = 0, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, training_batch_size: int = 50, validation_fraction: float = 0.1, resume_training: bool = False, dataloader_kwargs: Optional[dict] = None, - warn_if_zscoring: Optional[bool] = True, ) -> Tuple[data.DataLoader, data.DataLoader]: """Return dataloaders for training and validation. @@ -244,22 +244,19 @@ def get_dataloaders_all( """ - dataset = data.TensorDataset( - *self.get_simulations( - starting_round,exclude_invalid_x, warn_on_invalid = warn_on_invalid, - warn_if_zscoring = warn_if_zscoring - ) + if starting_round == 0: + indices = torch.arange(sum(self._num_sims_per_round)) + else: + indices = torch.arange(sum(self._num_sims_per_round[:starting_round]), sum(self._num_sims_per_round)) - ) # Get total number of training examples. - num_examples = len(dataset) - + num_examples = len(indices) # Select random train and validation splits from (theta, x) pairs. num_training_examples = int((1 - validation_fraction) * num_examples) num_validation_examples = num_examples - num_training_examples if not resume_training: - permuted_indices = torch.randperm(num_examples) + permuted_indices = indices[torch.randperm(num_examples)] self.train_indices, self.val_indices = ( permuted_indices[:num_training_examples], permuted_indices[num_training_examples:], @@ -272,19 +269,19 @@ def get_dataloaders_all( train_loader_kwargs = { "batch_size": min(training_batch_size, num_training_examples), "drop_last": True, - "sampler": SubsetRandomSampler(torch.arange(len(self.train_indices).tolist()) ), + "sampler": SubsetRandomSampler(torch.arange(len(self.train_indices)).tolist() ), } val_loader_kwargs = { "batch_size": min(training_batch_size, num_validation_examples), "shuffle": False, "drop_last": True, - "sampler": SubsetRandomSampler(torch.arange(len(self.val_indices).tolist()) ), + "sampler": SubsetRandomSampler(torch.arange(len(self.val_indices)).tolist() ), } if dataloader_kwargs is not None: train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) - return data.DataLoader(dataset[train_indices], **train_loader_kwargs), data.DataLoader(dataset[val_indices], **val_loader_kwargs) + return data.DataLoader(self._dataset, **train_loader_kwargs), data.DataLoader(self._dataset, **val_loader_kwargs) def get_dataloaders( self, diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 459cd75fd..55b919016 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -29,6 +29,10 @@ test_posterior_net_for_multi_d_x, validate_theta_and_x, x_shape_from_simulation, + handle_invalid_x, + warn_if_zscoring_changes_data, + warn_on_invalid_x, + warn_on_invalid_x_for_snpec_leakage, ) from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior @@ -88,6 +92,11 @@ def append_simulations( theta: Tensor, x: Tensor, proposal: Optional[DirectPosterior] = None, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + warn_if_zscoring: bool = True, + return_self: bool = True, + data_device: str = None ) -> "PosteriorEstimator": r"""Store parameters and simulation outputs to use them for later training. @@ -108,7 +117,26 @@ def append_simulations( NeuralInference object (returned so that this function is chainable). """ - theta, x = validate_theta_and_x(theta, x, training_device=self._device) + # Add ability to specify device data is saved on + if data_device is None: data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) + + + is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) + + # Check for problematic z-scoring + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_on_invalid: + warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + warn_on_invalid_x_for_snpec_leakage( + num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round + ) + + x = x[is_valid_x] + theta = theta[is_valid_x] + + self._check_proposal(proposal) if ( @@ -122,7 +150,7 @@ def append_simulations( # with MLE loss or with atomic loss (see, in `train()`: # self._round = max(self._data_round_index)) self._data_round_index.append(0) - self._prior_masks.append(mask_sims_from_prior(0, theta.size(0))) + prior_masks = mask_sims_from_prior(0, theta.size(0)) else: if not self._data_round_index: # This catches a pretty specific case: if, in the first round, one @@ -130,10 +158,16 @@ def append_simulations( self._data_round_index.append(1) else: self._data_round_index.append(max(self._data_round_index) + 1) - self._prior_masks.append(mask_sims_from_prior(1, theta.size(0))) + prior_masks = mask_sims_from_prior(1, theta.size(0)) - self._theta_roundwise.append(theta) - self._x_roundwise.append(x) + + if self._dataset is None: + #If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + else: + self._dataset.datasets.append( data.TensorDataset(theta,x,prior_masks) ) + + self._num_sims_per_round.append(theta.size(0)) self._proposal_roundwise.append(proposal) if self._prior is None or isinstance(self._prior, ImproperEmpirical): @@ -150,7 +184,11 @@ def append_simulations( theta_prior = self.get_simulations()[0] self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) - return self + #Add ability to not return self + if return_self: + return self + else: + return 1 def train( self, @@ -226,22 +264,24 @@ def train( train_loader, val_loader = self.get_dataloaders_all( start_idx, - exclude_invalid_x, training_batch_size, validation_fraction, resume_training, dataloader_kwargs=dataloader_kwargs, - warn_if_zscoring=warn_if_zscoring, ) - # First round or if retraining from scratch: # Call the `self._build_neural_net` with the rounds' thetas and xs as # arguments, which will build the neural network. # This is passed into NeuralPosterior, to create a neural posterior which # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: + + #Get test theta,x + test_theta = self._dataset.datasets[0].tensors[0][:100] + test_x = self._dataset.datasets[0].tensors[1][:100] + self._neural_net = self._build_neural_net( - theta[self.train_indices], x[self.train_indices] + test_theta, test_x ) # If data on training device already move net as well. if ( @@ -250,8 +290,8 @@ def train( ): self._neural_net.to(self._device) - test_posterior_net_for_multi_d_x(self._neural_net, theta, x) - self._x_shape = x_shape_from_simulation(x) + test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) + self._x_shape = x_shape_from_simulation(test_x) # Move entire net to device for training. self._neural_net.to(self._device) diff --git a/sbi/inference/snpe/snpe_c.py b/sbi/inference/snpe/snpe_c.py index 065a45f1f..6d3b0e8e5 100644 --- a/sbi/inference/snpe/snpe_c.py +++ b/sbi/inference/snpe/snpe_c.py @@ -100,6 +100,7 @@ def train( retrain_from_scratch: bool = False, show_train_summary: bool = False, dataloader_kwargs: Optional[Dict] = None, + warn_if_zscoring: Optional[bool] = True ) -> nn.Module: r"""Return density estimator that approximates the distribution $p(\theta|x)$. From e1f7fd35ec331ad348967977ad6616fe5eb2b1f7 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 24 Mar 2022 11:13:49 -0400 Subject: [PATCH 04/15] Minor bug fix --- sbi/inference/snpe/snpe_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 55b919016..787844693 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -276,7 +276,7 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - #Get test theta,x + #Get theta,x from dataset to initialize NN test_theta = self._dataset.datasets[0].tensors[0][:100] test_x = self._dataset.datasets[0].tensors[1][:100] @@ -286,7 +286,7 @@ def train( # If data on training device already move net as well. if ( not self._device == "cpu" - and f"{x.device.type}:{x.device.index}" == self._device + and f"{test_x.device.type}:{test_x.device.index}" == self._device ): self._neural_net.to(self._device) From d77bed7b64734d6c34cc60a5dcdcc821e813d007 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 24 Mar 2022 15:24:43 -0400 Subject: [PATCH 05/15] Minor bugfix when using multiple rounds --- sbi/inference/snpe/snpe_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 787844693..186ae5a18 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -165,7 +165,8 @@ def append_simulations( #If first round, set up ConcatDataset self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) else: - self._dataset.datasets.append( data.TensorDataset(theta,x,prior_masks) ) + #Otherwise append to Dataset + self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) self._num_sims_per_round.append(theta.size(0)) self._proposal_roundwise.append(proposal) From 48ace32258a1ba19101bb2fca321b154abfe142b Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 21 Apr 2022 14:39:03 -0400 Subject: [PATCH 06/15] Added to SNRE and SNLE --- README.md | 238 +- sbi/inference/base.py | 115 +- sbi/inference/snle/snle_base.py | 73 +- sbi/inference/snpe/snpe_a.py | 1721 +- sbi/inference/snpe/snpe_base.py | 1191 +- sbi/inference/snpe/snpe_c.py | 1254 +- sbi/inference/snre/snre_a.py | 3 - sbi/inference/snre/snre_b.py | 3 - sbi/inference/snre/snre_base.py | 71 +- sbi/utils/sbiutils.py | 23 + tests/linearGaussian_snpe_test.py | 1166 +- tutorials/07_conditional_distributions.ipynb | 33176 ++++++++--------- 12 files changed, 19522 insertions(+), 19512 deletions(-) diff --git a/README.md b/README.md index 96012b76c..9d9fc26e5 100644 --- a/README.md +++ b/README.md @@ -1,119 +1,119 @@ -[![PyPI version](https://badge.fury.io/py/sbi.svg)](https://badge.fury.io/py/sbi) -[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/mackelab/sbi/blob/master/CONTRIBUTING.md) -[![Tests](https://github.com/mackelab/sbi/workflows/Tests/badge.svg?branch=main)](https://github.com/mackelab/sbi/actions) -[![codecov](https://codecov.io/gh/mackelab/sbi/branch/main/graph/badge.svg)](https://codecov.io/gh/mackelab/sbi) -[![GitHub license](https://img.shields.io/github/license/mackelab/sbi)](https://github.com/mackelab/sbi/blob/master/LICENSE.txt) -[![DOI](https://joss.theoj.org/papers/10.21105/joss.02505/status.svg)](https://doi.org/10.21105/joss.02505) - -## sbi: simulation-based inference -[Getting Started](https://www.mackelab.org/sbi/tutorial/00_getting_started/) | [Documentation](https://www.mackelab.org/sbi/) - -`sbi` is a PyTorch package for simulation-based inference. Simulation-based inference is -the process of finding parameters of a simulator from observations. - -`sbi` takes a Bayesian approach and returns a full posterior distribution -over the parameters, conditional on the observations. This posterior can be amortized (i.e. -useful for any observation) or focused (i.e. tailored to a particular observation), with different -computational trade-offs. - -`sbi` offers a simple interface for one-line posterior inference. - -```python -from sbi.inference import infer -# import your simulator, define your prior over the parameters -parameter_posterior = infer(simulator, prior, method='SNPE', num_simulations=100) -``` -See below for the available methods of inference, `SNPE`, `SNRE` and `SNLE`. - - -## Installation - -`sbi` requires Python 3.6 or higher. We recommend to use a [`conda`](https://docs.conda.io/en/latest/miniconda.html) virtual -environment ([Miniconda installation instructions](https://docs.conda.io/en/latest/miniconda.html])). If `conda` is installed on the system, an environment for -installing `sbi` can be created as follows: -```commandline -# Create an environment for sbi (indicate Python 3.6 or higher); activate it -$ conda create -n sbi_env python=3.7 && conda activate sbi_env -``` - -Independent of whether you are using `conda` or not, `sbi` can be installed using `pip`: -```commandline -$ pip install sbi -``` - -To test the installation, drop into a python prompt and run -```python -from sbi.examples.minimal import simple -posterior = simple() -print(posterior) -``` - -## Inference Algorithms - -The following algorithms are currently available: - -#### Sequential Neural Posterior Estimation (SNPE) - -* [`SNPE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_a.SNPE_A) from Papamakarios G and Murray I [_Fast ε-free Inference of Simulation Models with Bayesian Conditional Density Estimation_](https://proceedings.neurips.cc/paper/2016/hash/6aca97005c68f1206823815f66102863-Abstract.html) (NeurIPS 2016). - -* [`SNPE_C`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_c.SNPE_C) or `APT` from Greenberg D, Nonnenmacher M, and Macke J [_Automatic - Posterior Transformation for likelihood-free - inference_](https://arxiv.org/abs/1905.07488) (ICML 2019). - - -#### Sequential Neural Likelihood Estimation (SNLE) -* [`SNLE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snle.snle_a.SNLE_A) or just `SNL` from Papamakarios G, Sterrat DC and Murray I [_Sequential - Neural Likelihood_](https://arxiv.org/abs/1805.07226) (AISTATS 2019). - - -#### Sequential Neural Ratio Estimation (SNRE) - -* [`SNRE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_a.SNRE_A) or `AALR` from Hermans J, Begy V, and Louppe G. [_Likelihood-free Inference with Amortized Approximate Likelihood Ratios_](https://arxiv.org/abs/1903.04057) (ICML 2020). - -* [`SNRE_B`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_b.SNRE_B) or `SRE` from Durkan C, Murray I, and Papamakarios G. [_On Contrastive Learning for Likelihood-free Inference_](https://arxiv.org/abs/2002.03712) (ICML 2020). - -#### Sequential Neural Variational Inference (SNVI) - -* [`SNVI`](https://www.mackelab.org/sbi/reference/#sbi.inference.posteriors.vi_posterior) from Glöckler M, Deistler M, Macke J, [_Variational methods for simulation-based inference_](https://openreview.net/forum?id=kZ0UYdhqkNY) (ICLR 2022). - -## Feedback and Contributions - -We would like to hear how `sbi` is working for your inference problems as well as receive bug reports, pull requests and other feedback (see -[contribute](http://www.mackelab.org/sbi/contribute/)). - - -## Acknowledgements - -`sbi` is the successor (using PyTorch) of the -[`delfi`](https://github.com/mackelab/delfi) package. It was started as a fork of Conor -M. Durkan's `lfi`. `sbi` runs as a community project; development is coordinated at the -[mackelab](https://uni-tuebingen.de/en/research/core-research/cluster-of-excellence-machine-learning/research/research/cluster-research-groups/professorships/machine-learning-in-science/). See also [credits](https://github.com/mackelab/sbi/blob/master/docs/docs/credits.md). - - -## Support - -`sbi` has been supported by the German Federal Ministry of Education and Research (BMBF) through the project ADIMEM, FKZ 01IS18052 A-D). [ADIMEM](https://fit.uni-tuebingen.de/Project/Details?id=9199) is a collaborative project between the groups of Jakob Macke (Uni Tübingen), Philipp Berens (Uni Tübingen), Philipp Hennig (Uni Tübingen) and Marcel Oberlaender (caesar Bonn) which aims to develop inference methods for mechanistic models. - - -## License - -[Affero General Public License v3 (AGPLv3)](https://www.gnu.org/licenses/) - - -## Citation -If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.21105/joss.02505), in addition to the original research articles describing the specifc sbi-algorithm(s) you are using: - -``` -@article{tejero-cantero2020sbi, - doi = {10.21105/joss.02505}, - url = {https://doi.org/10.21105/joss.02505}, - year = {2020}, - publisher = {The Open Journal}, - volume = {5}, - number = {52}, - pages = {2505}, - author = {Alvaro Tejero-Cantero and Jan Boelts and Michael Deistler and Jan-Matthis Lueckmann and Conor Durkan and Pedro J. Gonçalves and David S. Greenberg and Jakob H. Macke}, - title = {sbi: A toolkit for simulation-based inference}, - journal = {Journal of Open Source Software} -} -``` +[![PyPI version](https://badge.fury.io/py/sbi.svg)](https://badge.fury.io/py/sbi) +[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/mackelab/sbi/blob/master/CONTRIBUTING.md) +[![Tests](https://github.com/mackelab/sbi/workflows/Tests/badge.svg?branch=main)](https://github.com/mackelab/sbi/actions) +[![codecov](https://codecov.io/gh/mackelab/sbi/branch/main/graph/badge.svg)](https://codecov.io/gh/mackelab/sbi) +[![GitHub license](https://img.shields.io/github/license/mackelab/sbi)](https://github.com/mackelab/sbi/blob/master/LICENSE.txt) +[![DOI](https://joss.theoj.org/papers/10.21105/joss.02505/status.svg)](https://doi.org/10.21105/joss.02505) + +## sbi: simulation-based inference +[Getting Started](https://www.mackelab.org/sbi/tutorial/00_getting_started/) | [Documentation](https://www.mackelab.org/sbi/) + +`sbi` is a PyTorch package for simulation-based inference. Simulation-based inference is +the process of finding parameters of a simulator from observations. + +`sbi` takes a Bayesian approach and returns a full posterior distribution +over the parameters, conditional on the observations. This posterior can be amortized (i.e. +useful for any observation) or focused (i.e. tailored to a particular observation), with different +computational trade-offs. + +`sbi` offers a simple interface for one-line posterior inference. + +```python +from sbi.inference import infer +# import your simulator, define your prior over the parameters +parameter_posterior = infer(simulator, prior, method='SNPE', num_simulations=100) +``` +See below for the available methods of inference, `SNPE`, `SNRE` and `SNLE`. + + +## Installation + +`sbi` requires Python 3.6 or higher. We recommend to use a [`conda`](https://docs.conda.io/en/latest/miniconda.html) virtual +environment ([Miniconda installation instructions](https://docs.conda.io/en/latest/miniconda.html])). If `conda` is installed on the system, an environment for +installing `sbi` can be created as follows: +```commandline +# Create an environment for sbi (indicate Python 3.6 or higher); activate it +$ conda create -n sbi_env python=3.7 && conda activate sbi_env +``` + +Independent of whether you are using `conda` or not, `sbi` can be installed using `pip`: +```commandline +$ pip install sbi +``` + +To test the installation, drop into a python prompt and run +```python +from sbi.examples.minimal import simple +posterior = simple() +print(posterior) +``` + +## Inference Algorithms + +The following algorithms are currently available: + +#### Sequential Neural Posterior Estimation (SNPE) + +* [`SNPE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_a.SNPE_A) from Papamakarios G and Murray I [_Fast ε-free Inference of Simulation Models with Bayesian Conditional Density Estimation_](https://proceedings.neurips.cc/paper/2016/hash/6aca97005c68f1206823815f66102863-Abstract.html) (NeurIPS 2016). + +* [`SNPE_C`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_c.SNPE_C) or `APT` from Greenberg D, Nonnenmacher M, and Macke J [_Automatic + Posterior Transformation for likelihood-free + inference_](https://arxiv.org/abs/1905.07488) (ICML 2019). + + +#### Sequential Neural Likelihood Estimation (SNLE) +* [`SNLE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snle.snle_a.SNLE_A) or just `SNL` from Papamakarios G, Sterrat DC and Murray I [_Sequential + Neural Likelihood_](https://arxiv.org/abs/1805.07226) (AISTATS 2019). + + +#### Sequential Neural Ratio Estimation (SNRE) + +* [`SNRE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_a.SNRE_A) or `AALR` from Hermans J, Begy V, and Louppe G. [_Likelihood-free Inference with Amortized Approximate Likelihood Ratios_](https://arxiv.org/abs/1903.04057) (ICML 2020). + +* [`SNRE_B`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_b.SNRE_B) or `SRE` from Durkan C, Murray I, and Papamakarios G. [_On Contrastive Learning for Likelihood-free Inference_](https://arxiv.org/abs/2002.03712) (ICML 2020). + +#### Sequential Neural Variational Inference (SNVI) + +* [`SNVI`](https://www.mackelab.org/sbi/reference/#sbi.inference.posteriors.vi_posterior) from Glöckler M, Deistler M, Macke J, [_Variational methods for simulation-based inference_](https://openreview.net/forum?id=kZ0UYdhqkNY) (ICLR 2022). + +## Feedback and Contributions + +We would like to hear how `sbi` is working for your inference problems as well as receive bug reports, pull requests and other feedback (see +[contribute](http://www.mackelab.org/sbi/contribute/)). + + +## Acknowledgements + +`sbi` is the successor (using PyTorch) of the +[`delfi`](https://github.com/mackelab/delfi) package. It was started as a fork of Conor +M. Durkan's `lfi`. `sbi` runs as a community project; development is coordinated at the +[mackelab](https://uni-tuebingen.de/en/research/core-research/cluster-of-excellence-machine-learning/research/research/cluster-research-groups/professorships/machine-learning-in-science/). See also [credits](https://github.com/mackelab/sbi/blob/master/docs/docs/credits.md). + + +## Support + +`sbi` has been supported by the German Federal Ministry of Education and Research (BMBF) through the project ADIMEM, FKZ 01IS18052 A-D). [ADIMEM](https://fit.uni-tuebingen.de/Project/Details?id=9199) is a collaborative project between the groups of Jakob Macke (Uni Tübingen), Philipp Berens (Uni Tübingen), Philipp Hennig (Uni Tübingen) and Marcel Oberlaender (caesar Bonn) which aims to develop inference methods for mechanistic models. + + +## License + +[Affero General Public License v3 (AGPLv3)](https://www.gnu.org/licenses/) + + +## Citation +If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.21105/joss.02505), in addition to the original research articles describing the specifc sbi-algorithm(s) you are using: + +``` +@article{tejero-cantero2020sbi, + doi = {10.21105/joss.02505}, + url = {https://doi.org/10.21105/joss.02505}, + year = {2020}, + publisher = {The Open Journal}, + volume = {5}, + number = {52}, + pages = {2505}, + author = {Alvaro Tejero-Cantero and Jan Boelts and Michael Deistler and Jan-Matthis Lueckmann and Conor Durkan and Pedro J. Gonçalves and David S. Greenberg and Jakob H. Macke}, + title = {sbi: A toolkit for simulation-based inference}, + journal = {Journal of Open Source Software} +} +``` diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 86897fecc..daae107f1 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -26,7 +26,7 @@ warn_on_invalid_x, warn_on_invalid_x_for_snpec_leakage, ) -from sbi.utils.sbiutils import get_simulations_since_round +from sbi.utils.sbiutils import get_simulations_indcies from sbi.utils.torchutils import check_if_prior_on_device, process_device from sbi.utils.user_input_checks import prepare_for_sbi @@ -128,7 +128,6 @@ def __init__( # Initialize roundwise (theta, x, prior_masks) for storage of parameters, # simulations and masks indicating if simulations came from prior. - self._theta_roundwise, self._x_roundwise, self._prior_masks = [], [], [] self._dataset = None self._num_sims_per_round = [] self._model_bank = [] @@ -162,9 +161,6 @@ def __init__( def get_simulations( self, starting_round: int = 0, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: Optional[bool] = True, ) -> Tuple[Tensor, Tensor, Tensor]: r"""Returns all $\theta$, $x$, and prior_masks from rounds >= `starting_round`. @@ -181,28 +177,22 @@ def get_simulations( Returns: Parameters, simulation outputs, prior masks. """ - theta = get_simulations_since_round( - self._theta_roundwise, self._data_round_index, starting_round - ) - x = get_simulations_since_round( - self._x_roundwise, self._data_round_index, starting_round - ) - prior_masks = get_simulations_since_round( - self._prior_masks, self._data_round_index, starting_round - ) + #This is a pretty clunky implementation but not sure this will be used much with + #new implementation of `get_dataloaders` + indicies = get_simulations_indcies(self._num_sims_per_round, self._data_round_index, starting_round) + theta,x,prior_masks = [],[],[] - # Check for NaNs in simulations. - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) - warn_on_invalid_x_for_snpec_leakage( - num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round - ) + for ind in indicies: + theta_cur,x_cur,prior_mask_cur = self._dataset[ind] + theta.append(theta_cur) + x.append(x_cur) + prior_masks.append(prior_mask_cur) + + theta = torch.stack(theta).squeeze() + x = torch.stack(x).squeeze() + prior_masks = torch.stack(prior_masks).squeeze() - return theta[is_valid_x], x[is_valid_x], prior_masks[is_valid_x] + return theta, x, prior_masks @abstractmethod def train( @@ -221,7 +211,7 @@ def train( ) -> NeuralPosterior: raise NotImplementedError - def get_dataloaders_all( + def get_dataloaders( self, starting_round: int = 0, training_batch_size: int = 50, @@ -243,11 +233,9 @@ def get_dataloaders_all( Tuple of dataloaders for training and validation. """ - - if starting_round == 0: - indices = torch.arange(sum(self._num_sims_per_round)) - else: - indices = torch.arange(sum(self._num_sims_per_round[:starting_round]), sum(self._num_sims_per_round)) + + #Generate indicies to use based on starting_round + indices = get_simulations_indcies(self._num_sims_per_round, self._data_round_index, starting_round) # Get total number of training examples. num_examples = len(indices) @@ -256,6 +244,7 @@ def get_dataloaders_all( num_validation_examples = num_examples - num_training_examples if not resume_training: + # Seperate indicies for training and validation permuted_indices = indices[torch.randperm(num_examples)] self.train_indices, self.val_indices = ( permuted_indices[:num_training_examples], @@ -281,68 +270,10 @@ def get_dataloaders_all( train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) - return data.DataLoader(self._dataset, **train_loader_kwargs), data.DataLoader(self._dataset, **val_loader_kwargs) - - def get_dataloaders( - self, - dataset: data.TensorDataset, - training_batch_size: int = 50, - validation_fraction: float = 0.1, - resume_training: bool = False, - dataloader_kwargs: Optional[dict] = None, - ) -> Tuple[data.DataLoader, data.DataLoader]: - """Return dataloaders for training and validation. - - Args: - dataset: holding all theta and x, optionally masks. - training_batch_size: training arg of inference methods. - resume_training: Whether the current call is resuming training so that no - new training and validation indices into the dataset have to be created. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn). - - Returns: - Tuple of dataloaders for training and validation. - - """ - - # Get total number of training examples. - num_examples = len(dataset) - - # Select random train and validation splits from (theta, x) pairs. - num_training_examples = int((1 - validation_fraction) * num_examples) - num_validation_examples = num_examples - num_training_examples - - if not resume_training: - permuted_indices = torch.randperm(num_examples) - self.train_indices, self.val_indices = ( - permuted_indices[:num_training_examples], - permuted_indices[num_training_examples:], - ) - - # Create training and validation loaders using a subset sampler. - # Intentionally use dicts to define the default dataloader args - # Then, use dataloader_kwargs to override (or add to) any of these defaults - # https://stackoverflow.com/questions/44784577/in-method-call-args-how-to-override-keyword-argument-of-unpacked-dict - train_loader_kwargs = { - "batch_size": min(training_batch_size, num_training_examples), - "drop_last": True, - "sampler": SubsetRandomSampler(self.train_indices.tolist()), - } - val_loader_kwargs = { - "batch_size": min(training_batch_size, num_validation_examples), - "shuffle": False, - "drop_last": True, - "sampler": SubsetRandomSampler(self.val_indices.tolist()), - } - if dataloader_kwargs is not None: - train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) - val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) - - train_loader = data.DataLoader(dataset, **train_loader_kwargs) - val_loader = data.DataLoader(dataset, **val_loader_kwargs) + train_loader = data.DataLoader(self._dataset, **train_loader_kwargs) + val_loader = data.DataLoader(self._dataset, **val_loader_kwargs) - return train_loader, val_loader + return train_loader,val_loader def _converged(self, epoch: int, stop_after_epochs: int) -> bool: """Return whether the training converged yet and save best model state so far. diff --git a/sbi/inference/snle/snle_base.py b/sbi/inference/snle/snle_base.py index e9e7baace..33aefa8c4 100644 --- a/sbi/inference/snle/snle_base.py +++ b/sbi/inference/snle/snle_base.py @@ -22,7 +22,10 @@ check_estimator_arg, check_prior, mask_sims_from_prior, + handle_invalid_x, validate_theta_and_x, + warn_if_zscoring_changes_data, + warn_on_invalid_x, x_shape_from_simulation, ) @@ -83,6 +86,11 @@ def append_simulations( theta: Tensor, x: Tensor, from_round: int = 0, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + warn_if_zscoring: bool = True, + return_self: bool = True, + data_device: str = None, ) -> "LikelihoodEstimator": r"""Store parameters and simulation outputs to use them for later training. @@ -99,19 +107,46 @@ def append_simulations( With default settings, this is not used at all for `SNLE`. Only when the user later on requests `.train(discard_prior_samples=True)`, we use these indices to find which training data stemmed from the prior. - + exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` + during training. Expect errors, silent or explicit, when `False`. + warn_on_invalid: Whether to warn if data is invalid + warn_if_zscoring: Whether to test if z-scoring causes duplicates + return_self: Whether to return a instance of the class, allows chaining + with `.train()`. Setting `False` decreases memory overhead. + data_device: Where to store the data, default is on the same device where + the training is happening. If training a large dataset on a GPU with not + much VRAM can set to 'cpu' to store data on system memory instead. Returns: NeuralInference object (returned so that this function is chainable). """ - theta, x = validate_theta_and_x(theta, x, training_device=self._device) + is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) + + # Check for problematic z-scoring + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_on_invalid: + warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + + x = x[is_valid_x] + theta = theta[is_valid_x] + + if data_device is None: data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) + prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - self._theta_roundwise.append(theta) - self._x_roundwise.append(x) - self._prior_masks.append(mask_sims_from_prior(int(from_round), theta.size(0))) - self._data_round_index.append(int(from_round)) + if self._dataset is None: + #If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + else: + #Otherwise append to Dataset + self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) - return self + self._num_sims_per_round.append(theta.size(0)) + self._data_round_index.append(int(from_round) ) + + if return_self: + return self def train( self, @@ -121,7 +156,6 @@ def train( stop_after_epochs: int = 20, max_num_epochs: int = 2**31 - 1, clip_max_norm: Optional[float] = 5.0, - exclude_invalid_x: bool = True, resume_training: bool = False, discard_prior_samples: bool = False, retrain_from_scratch: bool = False, @@ -131,8 +165,6 @@ def train( r"""Train the density estimator to learn the distribution $p(x|\theta)$. Args: - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. resume_training: Can be used in case training time is limited, e.g. on a cluster. If `True`, the split between train and validation set, the optimizer, the number of epochs, and the best validation log-prob will @@ -155,15 +187,9 @@ def train( start_idx = int(discard_prior_samples and self._round > 0) # Load data from most recent round. self._round = max(self._data_round_index) - theta, x, _ = self.get_simulations( - start_idx, exclude_invalid_x, warn_on_invalid=True - ) - - # Dataset is shared for training and validation loaders. - dataset = data.TensorDataset(theta, x) train_loader, val_loader = self.get_dataloaders( - dataset, + start_idx, training_batch_size, validation_fraction, resume_training, @@ -176,10 +202,15 @@ def train( # This is passed into NeuralPosterior, to create a neural posterior which # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: + + #Get theta,x from dataset to initialize NN + test_theta = self._dataset.datasets[0].tensors[0][:100] + test_x = self._dataset.datasets[0].tensors[1][:100] + self._neural_net = self._build_neural_net( - theta[self.train_indices], x[self.train_indices] + test_theta, test_x ) - self._x_shape = x_shape_from_simulation(x) + self._x_shape = x_shape_from_simulation(test_x) assert ( len(self._x_shape) < 3 ), "SNLE cannot handle multi-dimensional simulator output." @@ -257,8 +288,8 @@ def train( self._summarize( round_=self._round, x_o=None, - theta_bank=theta, - x_bank=x, + theta_bank=None, + x_bank=None, ) # Update description for progress bar. diff --git a/sbi/inference/snpe/snpe_a.py b/sbi/inference/snpe/snpe_a.py index e0edefd12..664e62fff 100644 --- a/sbi/inference/snpe/snpe_a.py +++ b/sbi/inference/snpe/snpe_a.py @@ -1,862 +1,859 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - -import warnings -from copy import deepcopy -from functools import partial -from typing import Any, Callable, Dict, Optional, Union - -import torch -import torch.nn as nn -from pyknos.mdn.mdn import MultivariateGaussianMDN -from pyknos.nflows import flows -from pyknos.nflows.transforms import CompositeTransform -from torch import Tensor -from torch.distributions import Distribution, MultivariateNormal - -import sbi.utils as utils -from sbi.inference.posteriors.direct_posterior import DirectPosterior -from sbi.inference.snpe.snpe_base import PosteriorEstimator -from sbi.types import TensorboardSummaryWriter, TorchModule -from sbi.utils import torchutils - - -class SNPE_A(PosteriorEstimator): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "mdn_snpe_a", - num_components: int = 10, - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[TensorboardSummaryWriter] = None, - show_progress_bars: bool = True, - ): - r"""SNPE-A [1]. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - - This class implements SNPE-A. SNPE-A trains across multiple rounds with a - maximum-likelihood-loss. This will make training converge to the proposal - posterior instead of the true posterior. To correct for this, SNPE-A applies a - post-hoc correction after training. This correction has to be performed - analytically. Thus, SNPE-A is limited to Gaussian distributions for all but the - last round. In the last round, SNPE-A can use a Mixture of Gaussians. - - Args: - prior: A probability distribution that expresses prior knowledge about the - parameters, e.g. which ranges are meaningful for them. Any - object with `.log_prob()`and `.sample()` (for example, a PyTorch - distribution) can be used. - density_estimator: If it is a string (only "mdn_snpe_a" is valid), use a - pre-configured mixture of densities network. Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. Note that until the last round only a - single (multivariate) Gaussian component is used for training (see - Algorithm 1 in [1]). In the last round, this component is replicated - `num_components` times, its parameters are perturbed with a very small - noise, and then the last training round is done with the expanded - Gaussian mixture as estimator for the proposal posterior. - num_components: Number of components of the mixture of Gaussians in the - last round. This overrides the `num_components` value passed to - `posterior_nn()`. - device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". - logging_level: Minimum severity of messages to log. One of the strings - INFO, WARNING, DEBUG, ERROR and CRITICAL. - summary_writer: A tensorboard `SummaryWriter` to control, among others, log - file location (default is `/logs`.) - show_progress_bars: Whether to show a progressbar during training. - """ - - # Catch invalid inputs. - if not ((density_estimator == "mdn_snpe_a") or callable(density_estimator)): - raise TypeError( - "The `density_estimator` passed to SNPE_A needs to be a " - "callable or the string 'mdn_snpe_a'!" - ) - - # `num_components` will be used to replicate the Gaussian in the last round. - self._num_components = num_components - self._ran_final_round = False - - # WARNING: sneaky trick ahead. We proxy the parent's `train` here, - # requiring the signature to have `num_atoms`, save it for use below, and - # continue. It's sneaky because we are using the object (self) as a namespace - # to pass arguments between functions, and that's implicit state management. - kwargs = utils.del_entries( - locals(), - entries=("self", "__class__", "num_components"), - ) - super().__init__(**kwargs) - - def train( - self, - final_round: bool = False, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - exclude_invalid_x: bool = True, - resume_training: bool = False, - force_first_round_loss: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - dataloader_kwargs: Optional[Dict] = None, - component_perturbation: float = 5e-3, - ) -> nn.Module: - r"""Return density estimator that approximates the proposal posterior. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - - Training is performed with maximum likelihood on samples from the latest round, - which leads the algorithm to converge to the proposal posterior. - - Args: - final_round: Whether we are in the last round of training or not. For all - but the last round, Algorithm 1 from [1] is executed. In last the - round, Algorithm 2 from [1] is executed once. - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - force_first_round_loss: If `True`, train with maximum likelihood, - regardless of the proposal distribution. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. Not supported for - SNPE-A. - show_train_summary: Whether to print the number of epochs and validation - loss and leakage after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - component_perturbation: The standard deviation applied to all weights and - biases when, in the last round, the Mixture of Gaussians is build from - a single Gaussian. This value can be problem-specific and also depends - on the number of mixture components. - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - - assert not retrain_from_scratch, """Retraining from scratch is not supported in SNPE-A yet. The reason for - this is that, if we reininitialized the density estimator, the z-scoring would - change, which would break the posthoc correction. This is a pure implementation - issue.""" - - kwargs = utils.del_entries( - locals(), - entries=("self", "__class__", "final_round", "component_perturbation"), - ) - - # SNPE-A always discards the prior samples. - kwargs["discard_prior_samples"] = True - - self._round = max(self._data_round_index) - - if final_round: - # If there is (will be) only one round, train with Algorithm 2 from [1]. - if self._round == 0: - self._build_neural_net = partial( - self._build_neural_net, num_components=self._num_components - ) - # Run Algorithm 2 from [1]. - elif not self._ran_final_round: - # Now switch to the specified number of components. This method will - # only be used if `retrain_from_scratch=True`. Otherwise, - # the MDN will be built from replicating the single-component net for - # `num_component` times (via `_expand_mog()`). - self._build_neural_net = partial( - self._build_neural_net, num_components=self._num_components - ) - - # Extend the MDN to the originally desired number of components. - self._expand_mog(eps=component_perturbation) - else: - warnings.warn( - "You have already run SNPE-A with `final_round=True`. Running it" - "again with this setting will not allow computing the posthoc" - "correction applied in SNPE-A. Thus, you will get an error when " - "calling `.build_posterior()` after training.", - UserWarning, - ) - else: - # Run Algorithm 1 from [1]. - # Wrap the function that builds the MDN such that we can make - # sure that there is only one component when running. - self._build_neural_net = partial(self._build_neural_net, num_components=1) - - if final_round: - self._ran_final_round = True - - return super().train(**kwargs) - - def correct_for_proposal( - self, - density_estimator: Optional[TorchModule] = None, - ) -> "SNPE_A_MDN": - r"""Build mixture of Gaussians that approximates the posterior. - - Returns a `SNPE_A_MDN` object, which applies the posthoc-correction required in - SNPE-A. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. - """ - if density_estimator is None: - density_estimator = deepcopy( - self._neural_net - ) # PosteriorEstimator.train() also returns a deepcopy, mimic this here - # If internal net is used device is defined. - device = self._device - else: - # Otherwise, infer it from the device of the net parameters. - device = str(next(density_estimator.parameters()).device) - - # Set proposal of the density estimator. - # This also evokes the z-scoring correction if necessary. - if ( - self._proposal_roundwise[-1] is self._prior - or self._proposal_roundwise[-1] is None - ): - proposal = self._prior - assert isinstance( - proposal, (MultivariateNormal, utils.BoxUniform) - ), """Prior must be `torch.distributions.MultivariateNormal` or `sbi.utils. - BoxUniform`""" - else: - assert isinstance( - self._proposal_roundwise[-1], DirectPosterior - ), """The proposal you passed to `append_simulations` is neither the prior - nor a `DirectPosterior`. SNPE-A currently only supports these scenarios. - """ - proposal = self._proposal_roundwise[-1] - - # Create the SNPE_A_MDN - wrapped_density_estimator = SNPE_A_MDN( - flow=density_estimator, proposal=proposal, prior=self._prior, device=device - ) - return wrapped_density_estimator - - def build_posterior( - self, - density_estimator: Optional[TorchModule] = None, - prior: Optional[Distribution] = None, - ) -> "DirectPosterior": - r"""Build posterior from the neural density estimator. - - This method first corrects the estimated density with `correct_for_proposal` - and then returns a `DirectPosterior`. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - prior: Prior distribution. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. - """ - if prior is None: - assert ( - self._prior is not None - ), """You did not pass a prior. You have to pass the prior either at - initialization `inference = SNPE_A(prior)` or to `.build_posterior - (prior=prior)`.""" - prior = self._prior - - wrapped_density_estimator = self.correct_for_proposal( - density_estimator=density_estimator - ) - self._posterior = DirectPosterior( - posterior_estimator=wrapped_density_estimator, - prior=prior, - ) - return deepcopy(self._posterior) - - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - ) -> Tensor: - """Return the log-probability of the proposal posterior. - - For SNPE-A this is the same as `self._neural_net.log_prob(theta, x)` in - `_loss()` to be found in `snpe_base.py`. - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - proposal: Proposal distribution. - - Returns: Log-probability of the proposal posterior. - """ - return self._neural_net.log_prob(theta, x) - - def _expand_mog(self, eps: float = 1e-5): - """ - Replicate a singe Gaussian trained with Algorithm 1 before continuing - with Algorithm 2. The weights and biases of the associated MDN layers - are repeated `num_components` times, slightly perturbed to break the - symmetry such that the gradients in the subsequent training are not - all identical. - - Args: - eps: Standard deviation for the random perturbation. - """ - assert isinstance(self._neural_net._distribution, MultivariateGaussianMDN) - - # Increase the number of components - self._neural_net._distribution._num_components = self._num_components - - # Expand the 1-dim Gaussian. - for name, param in self._neural_net.named_parameters(): - if any( - key in name for key in ["logits", "means", "unconstrained", "upper"] - ): - if "bias" in name: - param.data = param.data.repeat(self._num_components) - param.data.add_(torch.randn_like(param.data) * eps) - param.grad = None # let autograd construct a new gradient - elif "weight" in name: - param.data = param.data.repeat(self._num_components, 1) - param.data.add_(torch.randn_like(param.data) * eps) - param.grad = None # let autograd construct a new gradient - - -class SNPE_A_MDN(nn.Module): - """Generates a posthoc-corrected MDN which approximates the posterior. - - This class takes as input the density estimator (abbreviated with `_d` suffix, aka - the proposal posterior) and the proposal prior (abbreviated with `_pp` suffix) from - which the simulations were drawn. It uses the algorithm presented in SNPE-A [1] to - compute the approximate posterior (abbreviated with `_p` suffix) from the two. The - approximate posterior is a MoG. This class also implements log-prob calculation - sampling from the approximate posterior. It inherits from `nn.Module` since the - constructor of `DirectPosterior` expects the argument `neural_net` to be a - `nn.Module`. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - """ - - def __init__( - self, - flow: flows.Flow, - proposal: Union["utils.BoxUniform", "MultivariateNormal", "DirectPosterior"], - prior: Distribution, - device: str, - ): - """Constructor. - - Args: - flow: The trained normalizing flow, passed when building the posterior. - proposal: The proposal distribution. - prior: The prior distribution. - """ - # Call nn.Module's constructor. - super().__init__() - - self._neural_net = flow - self._prior = prior - self._device = device - - # Set the proposal using the `default_x`. - if isinstance(proposal, (utils.BoxUniform, MultivariateNormal)): - self._apply_correction = False - else: - self._apply_correction = True - logits_pp, m_pp, prec_pp = proposal.posterior_estimator._posthoc_correction( - proposal.default_x - ) - self._logits_pp, self._m_pp, self._prec_pp = ( - logits_pp.detach(), - m_pp.detach(), - prec_pp.detach(), - ) - - # Take care of z-scoring, pre-compute and store prior terms. - self._set_state_for_mog_proposal() - - def log_prob(self, inputs: Tensor, context: Tensor) -> Tensor: - inputs, context = inputs.to(self._device), context.to(self._device) - - if not self._apply_correction: - return self._neural_net.log_prob(inputs, context) - else: - # When we want to compute the approx. posterior, a proposal prior \tilde{p} - # has already been observed. To analytically calculate the log-prob of the - # Gaussian, we first need to compute the mixture components. - - # Compute the mixture components of the proposal posterior. - logits_pp, m_pp, prec_pp = self._posthoc_correction(context) - - # z-score theta if it z-scoring had been requested. - theta = self._maybe_z_score_theta(inputs) - - # Compute the log_prob of theta under the product. - log_prob_proposal_posterior = utils.mog_log_prob( - theta, logits_pp, m_pp, prec_pp - ) - utils.assert_all_finite( - log_prob_proposal_posterior, "proposal posterior eval" - ) - return log_prob_proposal_posterior # \hat{p} from eq (3) in [1] - - def sample(self, num_samples: int, context: Tensor, batch_size: int = 1) -> Tensor: - context = context.to(self._device) - - if not self._apply_correction: - return self._neural_net.sample(num_samples, context, batch_size) - else: - # When we want to sample from the approx. posterior, a proposal prior - # \tilde{p} has already been observed. To analytically calculate the - # log-prob of the Gaussian, we first need to compute the mixture components. - return self._sample_approx_posterior_mog(num_samples, context, batch_size) - - def _sample_approx_posterior_mog( - self, num_samples, x: Tensor, batch_size: int - ) -> Tensor: - r"""Sample from the approximate posterior. - - Args: - num_samples: Desired number of samples. - x: Conditioning context for posterior $p(\theta|x)$. - batch_size: Batch size for sampling. - - Returns: - Samples from the approximate mixture of Gaussians posterior. - """ - - # Compute the mixture components of the posterior. - logits_p, m_p, prec_p = self._posthoc_correction(x) - - # Compute the precision factors which represent the upper triangular matrix - # of the cholesky decomposition of the prec_p. - prec_factors_p = torch.linalg.cholesky(prec_p, upper=True) - - assert logits_p.ndim == 2 - assert m_p.ndim == 3 - assert prec_p.ndim == 4 - assert prec_factors_p.ndim == 4 - - # Replicate to use batched sampling from pyknos. - if batch_size is not None and batch_size > 1: - logits_p = logits_p.repeat(batch_size, 1) - m_p = m_p.repeat(batch_size, 1, 1) - prec_factors_p = prec_factors_p.repeat(batch_size, 1, 1, 1) - - # Get (optionally z-scored) MoG samples. - theta = MultivariateGaussianMDN.sample_mog( - num_samples, logits_p, m_p, prec_factors_p - ) - - embedded_context = self._neural_net._embedding_net(x) - if embedded_context is not None: - # Merge the context dimension with sample dimension in order to - # apply the transform. - theta = torchutils.merge_leading_dims(theta, num_dims=2) - embedded_context = torchutils.repeat_rows( - embedded_context, num_reps=num_samples - ) - - theta, _ = self._neural_net._transform.inverse(theta, context=embedded_context) - - if embedded_context is not None: - # Split the context dimension from sample dimension. - theta = torchutils.split_leading_dim(theta, shape=[-1, num_samples]) - - return theta - - def _posthoc_correction(self, x: Tensor): - """ - Compute the mixture components of the posterior given the current density - estimator and the proposal. - - Args: - x: Conditioning context for posterior. - - Returns: - Mixture components of the posterior. - """ - - # Evaluate the density estimator. - encoded_x = self._neural_net._embedding_net(x) - dist = self._neural_net._distribution # defined to avoid black formatting. - logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) - - # The following if case is needed because, in the constructor, we call - # `_posthoc_correction` regardless of whether the `proposal` itself had a - # `proposal` or not. - if not self._apply_correction: - return norm_logits_d, m_d, prec_d - else: - logits_pp, m_pp, prec_pp = self._logits_pp, self._m_pp, self._prec_pp - - # Compute the MoG parameters of the posterior. - logits_p, m_p, prec_p, cov_p = self._proposal_posterior_transformation( - logits_pp, m_pp, prec_pp, norm_logits_d, m_d, prec_d - ) - return logits_p, m_p, prec_p - - def _proposal_posterior_transformation( - self, - logits_pp: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Transforms the proposal posterior (the MDN) into the posterior. - - The approximate posterior is: - $p(\theta|x) = 1/Z * q(\theta|x) * p(\theta) / prop(\theta)$ - In words: posterior = proposal posterior estimate * prior / proposal. - - Since the proposal posterior estimate and the proposal are MoG, and the - prior is either Gaussian or uniform, we can solve this in closed-form. - - This function implements Appendix C from [1], and is highly similar to - `SNPE_C._automatic_posterior_transformation()`. - - Args: - logits_pp: Component weight of each Gaussian of the proposal prior. - means_pp: Mean of each Gaussian of the proposal prior. - precisions_pp: Precision matrix of each Gaussian of the proposal prior. - logits_d: Component weight for each Gaussian of the density estimator. - means_d: Mean of each Gaussian of the density estimator. - precisions_d: Precision matrix of each Gaussian of the density estimator. - - Returns: (Component weight, mean, precision matrix, covariance matrix) of each - Gaussian of the approximate posterior. - """ - - precisions_post, covariances_post = self._precisions_posterior( - precisions_pp, precisions_d - ) - - means_post = self._means_posterior( - covariances_post, means_pp, precisions_pp, means_d, precisions_d - ) - - logits_post = SNPE_A_MDN._logits_posterior( - means_post, - precisions_post, - covariances_post, - logits_pp, - means_pp, - precisions_pp, - logits_d, - means_d, - precisions_d, - ) - - return logits_post, means_post, precisions_post, covariances_post - - def _set_state_for_mog_proposal(self) -> None: - """ - Set state variables of the SNPE_A_MDN instance every time `set_proposal()` - is called, i.e. every time a posterior is build using - `SNPE_A.build_posterior()`. - - This function is almost identical to `SNPE_C._set_state_for_mog_proposal()`. - - Three things are computed: - 1) Check if z-scoring was requested. To do so, we check if the `_transform` - argument of the net had been a `CompositeTransform`. See pyknos mdn.py. - 2) Define a (potentially standardized) prior. It's standardized if z-scoring - had been requested. - 3) Compute (Precision * mean) for the prior. This quantity is used at every - training step if the prior is Gaussian. - """ - - self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) - - self._set_maybe_z_scored_prior() - - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - self.prec_m_prod_prior = torch.mv( - self._maybe_z_scored_prior.precision_matrix, # type: ignore - self._maybe_z_scored_prior.loc, # type: ignore - ) - - def _set_maybe_z_scored_prior(self) -> None: - r""" - Compute and store potentially standardized prior (if z-scoring was requested). - - This function is highly similar to `SNPE_C._set_maybe_z_scored_prior()`. - - The proposal posterior is: - $p(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - - Let's denote z-scored theta by `a`: a = (theta - mean) / std - Then $p'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ - - The ' indicates that the evaluation occurs in standardized space. The constant - scaling factor has been absorbed into $Z_2$. - From the above equation, we see that we need to evaluate the prior **in - standardized space**. We build the standardized prior in this function. - - The standardize transform that is applied to the samples theta does not use - the exact prior mean and std (due to implementation issues). Hence, the z-scored - prior will not be exactly have mean=0 and std=1. - """ - if self.z_score_theta: - scale = self._neural_net._transform._transforms[0]._scale - shift = self._neural_net._transform._transforms[0]._shift - - # Following the definition of the linear transform in - # `standardizing_transform` in `sbiutils.py`: - # shift=-mean / std - # scale=1 / std - # Solving these equations for mean and std: - estim_prior_std = 1 / scale - estim_prior_mean = -shift * estim_prior_std - - # Compute the discrepancy of the true prior mean and std and the mean and - # std that was empirically estimated from samples. - # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) - # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean - # and std (estimated from samples and used to build standardize transform). - almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std - almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std - - if isinstance(self._prior, MultivariateNormal): - self._maybe_z_scored_prior = MultivariateNormal( - almost_zero_mean, torch.diag(almost_one_std) - ) - else: - range_ = torch.sqrt(almost_one_std * 3.0) - self._maybe_z_scored_prior = utils.BoxUniform( - almost_zero_mean - range_, almost_zero_mean + range_ - ) - else: - self._maybe_z_scored_prior = self._prior - - def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: - """Return potentially standardized theta if z-scoring was requested.""" - - if self.z_score_theta: - theta, _ = self._neural_net._transform(theta) - - return theta - - def _precisions_posterior(self, precisions_pp: Tensor, precisions_d: Tensor): - r"""Return the precisions and covariances of the MoG posterior. - - As described at the end of Appendix C in [1], it can happen that the - proposal's precision matrix is not positive definite. - - $S_k^\prime = ( S_k^{-1} - S_0^{-1} )^{-1}$ - (see eq (23) in Appendix C of [1]) - - Args: - precisions_pp: Precision matrices of the proposal prior. - precisions_d: Precision matrices of the density estimator. - - Returns: (Precisions, Covariances) of the MoG posterior. - """ - - num_comps_p = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Check if precision matrices are positive definite. - for batches in precisions_pp: - for pprior in batches: - eig_pprior = torch.linalg.eigvalsh(pprior, UPLO="U") - if not (eig_pprior > 0).all(): - raise AssertionError( - "The precision matrix of the proposal is not positive definite!" - ) - for batches in precisions_d: - for d in batches: - eig_d = torch.linalg.eigvalsh(d, UPLO="U") - if not (eig_d > 0).all(): - raise AssertionError( - "The precision matrix of the density estimator is not " - "positive definite!" - ) - - precisions_pp_rep = precisions_pp.repeat_interleave(num_comps_d, dim=1) - precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) - - precisions_p = precisions_d_rep - precisions_pp_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - precisions_p += self._maybe_z_scored_prior.precision_matrix - - # Check if precision matrix is positive definite. - for idx_batch, batches in enumerate(precisions_p): - for idx_comp, pp in enumerate(batches): - eig_pp = torch.symeig(pp, eigenvectors=False).eigenvalues - if not (eig_pp > 0).all(): - raise AssertionError( - "The precision matrix of a posterior is not positive " - "definite! This is a known issue for SNPE-A. Either try a " - "different parameter setting, e.g. a different number of " - "mixture components (when contracting SNPE-A), or a different " - "value for the parameter perturbation (when building the " - "posterior)." - ) - - covariances_p = torch.inverse(precisions_p) - return precisions_p, covariances_p - - def _means_posterior( - self, - covariances_p: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Return the means of the MoG posterior. - - $m_k^\prime = S_k^\prime ( S_k^{-1} m_k - S_0^{-1} m_0 )$ - (see eq (24) in Appendix C of [1]) - - Args: - covariances_post: Covariance matrices of the MoG posterior. - means_pp: Means of the proposal prior. - precisions_pp: Precision matrices of the proposal prior. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Means of the MoG posterior. - """ - - num_comps_pp = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute the products P_k * m_k and P_0 * m_0. - prec_m_prod_pp = utils.batched_mixture_mv(precisions_pp, means_pp) - prec_m_prod_d = utils.batched_mixture_mv(precisions_d, means_d) - - # Repeat them to allow for matrix operations: same trick as for the precisions. - prec_m_prod_pp_rep = prec_m_prod_pp.repeat_interleave(num_comps_d, dim=1) - prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_pp, 1) - - # Compute the means P_k^prime * (P_k * m_k - P_0 * m_0). - summed_cov_m_prod_rep = prec_m_prod_d_rep - prec_m_prod_pp_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - summed_cov_m_prod_rep += self.prec_m_prod_prior - - means_p = utils.batched_mixture_mv(covariances_p, summed_cov_m_prod_rep) - return means_p - - @staticmethod - def _logits_posterior( - means_post: Tensor, - precisions_post: Tensor, - covariances_post: Tensor, - logits_pp: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Return the component weights (i.e. logits) of the MoG posterior. - - $\alpha_k^\prime = \frac{ \alpha_k exp(-0.5 c_k) }{ \sum{j} \alpha_j exp(-0.5 - c_j) } $ - with - $c_k = logdet(S_k) - logdet(S_0) - logdet(S_k^\prime) + - + m_k^T P_k m_k - m_0^T P_0 m_0 - m_k^\prime^T P_k^\prime m_k^\prime$ - (see eqs. (25, 26) in Appendix C of [1]) - - Args: - means_post: Means of the posterior. - precisions_post: Precision matrices of the posterior. - covariances_post: Covariance matrices of the posterior. - logits_pp: Component weights (i.e. logits) of the proposal prior. - means_pp: Means of the proposal prior. - precisions_pp: Precision matrices of the proposal prior. - logits_d: Component weights (i.e. logits) of the density estimator. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Component weights of the proposal posterior. - """ - - num_comps_pp = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute the ratio of the logits similar to eq (10) in Appendix A.1 of [2] - logits_pp_rep = logits_pp.repeat_interleave(num_comps_d, dim=1) - logits_d_rep = logits_d.repeat(1, num_comps_pp) - logit_factors = logits_d_rep - logits_pp_rep - - # Compute the log-determinants - logdet_covariances_post = torch.logdet(covariances_post) - logdet_covariances_pp = -torch.logdet(precisions_pp) - logdet_covariances_d = -torch.logdet(precisions_d) - - # Repeat the proposal and density estimator terms such that there are LK terms. - # Same trick as has been used above. - logdet_covariances_pp_rep = logdet_covariances_pp.repeat_interleave( - num_comps_d, dim=1 - ) - logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_pp) - - log_sqrt_det_ratio = 0.5 * ( # similar to eq (14) in Appendix A.1 of [2] - logdet_covariances_post - + logdet_covariances_pp_rep - - logdet_covariances_d_rep - ) - - # Compute for proposal, density estimator, and proposal posterior: - exponent_pp = utils.batched_mixture_vmv( - precisions_pp, means_pp # m_0 in eq (26) in Appendix C of [1] - ) - exponent_d = utils.batched_mixture_vmv( - precisions_d, means_d # m_k in eq (26) in Appendix C of [1] - ) - exponent_post = utils.batched_mixture_vmv( - precisions_post, means_post # m_k^\prime in eq (26) in Appendix C of [1] - ) - - # Extend proposal and density estimator exponents to get LK terms. - exponent_pp_rep = exponent_pp.repeat_interleave(num_comps_d, dim=1) - exponent_d_rep = exponent_d.repeat(1, num_comps_pp) - exponent = -0.5 * ( - exponent_d_rep - exponent_pp_rep - exponent_post # eq (26) in [1] - ) - - logits_post = logit_factors + log_sqrt_det_ratio + exponent - return logits_post +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + +import warnings +from copy import deepcopy +from functools import partial +from typing import Any, Callable, Dict, Optional, Union + +import torch +import torch.nn as nn +from pyknos.mdn.mdn import MultivariateGaussianMDN +from pyknos.nflows import flows +from pyknos.nflows.transforms import CompositeTransform +from torch import Tensor +from torch.distributions import Distribution, MultivariateNormal + +import sbi.utils as utils +from sbi.inference.posteriors.direct_posterior import DirectPosterior +from sbi.inference.snpe.snpe_base import PosteriorEstimator +from sbi.types import TensorboardSummaryWriter, TorchModule +from sbi.utils import torchutils + + +class SNPE_A(PosteriorEstimator): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "mdn_snpe_a", + num_components: int = 10, + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[TensorboardSummaryWriter] = None, + show_progress_bars: bool = True, + ): + r"""SNPE-A [1]. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + + This class implements SNPE-A. SNPE-A trains across multiple rounds with a + maximum-likelihood-loss. This will make training converge to the proposal + posterior instead of the true posterior. To correct for this, SNPE-A applies a + post-hoc correction after training. This correction has to be performed + analytically. Thus, SNPE-A is limited to Gaussian distributions for all but the + last round. In the last round, SNPE-A can use a Mixture of Gaussians. + + Args: + prior: A probability distribution that expresses prior knowledge about the + parameters, e.g. which ranges are meaningful for them. Any + object with `.log_prob()`and `.sample()` (for example, a PyTorch + distribution) can be used. + density_estimator: If it is a string (only "mdn_snpe_a" is valid), use a + pre-configured mixture of densities network. Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. Note that until the last round only a + single (multivariate) Gaussian component is used for training (see + Algorithm 1 in [1]). In the last round, this component is replicated + `num_components` times, its parameters are perturbed with a very small + noise, and then the last training round is done with the expanded + Gaussian mixture as estimator for the proposal posterior. + num_components: Number of components of the mixture of Gaussians in the + last round. This overrides the `num_components` value passed to + `posterior_nn()`. + device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". + logging_level: Minimum severity of messages to log. One of the strings + INFO, WARNING, DEBUG, ERROR and CRITICAL. + summary_writer: A tensorboard `SummaryWriter` to control, among others, log + file location (default is `/logs`.) + show_progress_bars: Whether to show a progressbar during training. + """ + + # Catch invalid inputs. + if not ((density_estimator == "mdn_snpe_a") or callable(density_estimator)): + raise TypeError( + "The `density_estimator` passed to SNPE_A needs to be a " + "callable or the string 'mdn_snpe_a'!" + ) + + # `num_components` will be used to replicate the Gaussian in the last round. + self._num_components = num_components + self._ran_final_round = False + + # WARNING: sneaky trick ahead. We proxy the parent's `train` here, + # requiring the signature to have `num_atoms`, save it for use below, and + # continue. It's sneaky because we are using the object (self) as a namespace + # to pass arguments between functions, and that's implicit state management. + kwargs = utils.del_entries( + locals(), + entries=("self", "__class__", "num_components"), + ) + super().__init__(**kwargs) + + def train( + self, + final_round: bool = False, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[Dict] = None, + component_perturbation: float = 5e-3, + ) -> nn.Module: + r"""Return density estimator that approximates the proposal posterior. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + + Training is performed with maximum likelihood on samples from the latest round, + which leads the algorithm to converge to the proposal posterior. + + Args: + final_round: Whether we are in the last round of training or not. For all + but the last round, Algorithm 1 from [1] is executed. In last the + round, Algorithm 2 from [1] is executed once. + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + force_first_round_loss: If `True`, train with maximum likelihood, + regardless of the proposal distribution. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. Not supported for + SNPE-A. + show_train_summary: Whether to print the number of epochs and validation + loss and leakage after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + component_perturbation: The standard deviation applied to all weights and + biases when, in the last round, the Mixture of Gaussians is build from + a single Gaussian. This value can be problem-specific and also depends + on the number of mixture components. + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + + assert not retrain_from_scratch, """Retraining from scratch is not supported in SNPE-A yet. The reason for + this is that, if we reininitialized the density estimator, the z-scoring would + change, which would break the posthoc correction. This is a pure implementation + issue.""" + + kwargs = utils.del_entries( + locals(), + entries=("self", "__class__", "final_round", "component_perturbation"), + ) + + # SNPE-A always discards the prior samples. + kwargs["discard_prior_samples"] = True + + self._round = max(self._data_round_index) + + if final_round: + # If there is (will be) only one round, train with Algorithm 2 from [1]. + if self._round == 0: + self._build_neural_net = partial( + self._build_neural_net, num_components=self._num_components + ) + # Run Algorithm 2 from [1]. + elif not self._ran_final_round: + # Now switch to the specified number of components. This method will + # only be used if `retrain_from_scratch=True`. Otherwise, + # the MDN will be built from replicating the single-component net for + # `num_component` times (via `_expand_mog()`). + self._build_neural_net = partial( + self._build_neural_net, num_components=self._num_components + ) + + # Extend the MDN to the originally desired number of components. + self._expand_mog(eps=component_perturbation) + else: + warnings.warn( + "You have already run SNPE-A with `final_round=True`. Running it" + "again with this setting will not allow computing the posthoc" + "correction applied in SNPE-A. Thus, you will get an error when " + "calling `.build_posterior()` after training.", + UserWarning, + ) + else: + # Run Algorithm 1 from [1]. + # Wrap the function that builds the MDN such that we can make + # sure that there is only one component when running. + self._build_neural_net = partial(self._build_neural_net, num_components=1) + + if final_round: + self._ran_final_round = True + + return super().train(**kwargs) + + def correct_for_proposal( + self, + density_estimator: Optional[TorchModule] = None, + ) -> "SNPE_A_MDN": + r"""Build mixture of Gaussians that approximates the posterior. + + Returns a `SNPE_A_MDN` object, which applies the posthoc-correction required in + SNPE-A. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. + """ + if density_estimator is None: + density_estimator = deepcopy( + self._neural_net + ) # PosteriorEstimator.train() also returns a deepcopy, mimic this here + # If internal net is used device is defined. + device = self._device + else: + # Otherwise, infer it from the device of the net parameters. + device = str(next(density_estimator.parameters()).device) + + # Set proposal of the density estimator. + # This also evokes the z-scoring correction if necessary. + if ( + self._proposal_roundwise[-1] is self._prior + or self._proposal_roundwise[-1] is None + ): + proposal = self._prior + assert isinstance( + proposal, (MultivariateNormal, utils.BoxUniform) + ), """Prior must be `torch.distributions.MultivariateNormal` or `sbi.utils. + BoxUniform`""" + else: + assert isinstance( + self._proposal_roundwise[-1], DirectPosterior + ), """The proposal you passed to `append_simulations` is neither the prior + nor a `DirectPosterior`. SNPE-A currently only supports these scenarios. + """ + proposal = self._proposal_roundwise[-1] + + # Create the SNPE_A_MDN + wrapped_density_estimator = SNPE_A_MDN( + flow=density_estimator, proposal=proposal, prior=self._prior, device=device + ) + return wrapped_density_estimator + + def build_posterior( + self, + density_estimator: Optional[TorchModule] = None, + prior: Optional[Distribution] = None, + ) -> "DirectPosterior": + r"""Build posterior from the neural density estimator. + + This method first corrects the estimated density with `correct_for_proposal` + and then returns a `DirectPosterior`. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + prior: Prior distribution. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. + """ + if prior is None: + assert ( + self._prior is not None + ), """You did not pass a prior. You have to pass the prior either at + initialization `inference = SNPE_A(prior)` or to `.build_posterior + (prior=prior)`.""" + prior = self._prior + + wrapped_density_estimator = self.correct_for_proposal( + density_estimator=density_estimator + ) + self._posterior = DirectPosterior( + posterior_estimator=wrapped_density_estimator, + prior=prior, + ) + return deepcopy(self._posterior) + + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + ) -> Tensor: + """Return the log-probability of the proposal posterior. + + For SNPE-A this is the same as `self._neural_net.log_prob(theta, x)` in + `_loss()` to be found in `snpe_base.py`. + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + proposal: Proposal distribution. + + Returns: Log-probability of the proposal posterior. + """ + return self._neural_net.log_prob(theta, x) + + def _expand_mog(self, eps: float = 1e-5): + """ + Replicate a singe Gaussian trained with Algorithm 1 before continuing + with Algorithm 2. The weights and biases of the associated MDN layers + are repeated `num_components` times, slightly perturbed to break the + symmetry such that the gradients in the subsequent training are not + all identical. + + Args: + eps: Standard deviation for the random perturbation. + """ + assert isinstance(self._neural_net._distribution, MultivariateGaussianMDN) + + # Increase the number of components + self._neural_net._distribution._num_components = self._num_components + + # Expand the 1-dim Gaussian. + for name, param in self._neural_net.named_parameters(): + if any( + key in name for key in ["logits", "means", "unconstrained", "upper"] + ): + if "bias" in name: + param.data = param.data.repeat(self._num_components) + param.data.add_(torch.randn_like(param.data) * eps) + param.grad = None # let autograd construct a new gradient + elif "weight" in name: + param.data = param.data.repeat(self._num_components, 1) + param.data.add_(torch.randn_like(param.data) * eps) + param.grad = None # let autograd construct a new gradient + + +class SNPE_A_MDN(nn.Module): + """Generates a posthoc-corrected MDN which approximates the posterior. + + This class takes as input the density estimator (abbreviated with `_d` suffix, aka + the proposal posterior) and the proposal prior (abbreviated with `_pp` suffix) from + which the simulations were drawn. It uses the algorithm presented in SNPE-A [1] to + compute the approximate posterior (abbreviated with `_p` suffix) from the two. The + approximate posterior is a MoG. This class also implements log-prob calculation + sampling from the approximate posterior. It inherits from `nn.Module` since the + constructor of `DirectPosterior` expects the argument `neural_net` to be a + `nn.Module`. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + """ + + def __init__( + self, + flow: flows.Flow, + proposal: Union["utils.BoxUniform", "MultivariateNormal", "DirectPosterior"], + prior: Distribution, + device: str, + ): + """Constructor. + + Args: + flow: The trained normalizing flow, passed when building the posterior. + proposal: The proposal distribution. + prior: The prior distribution. + """ + # Call nn.Module's constructor. + super().__init__() + + self._neural_net = flow + self._prior = prior + self._device = device + + # Set the proposal using the `default_x`. + if isinstance(proposal, (utils.BoxUniform, MultivariateNormal)): + self._apply_correction = False + else: + self._apply_correction = True + logits_pp, m_pp, prec_pp = proposal.posterior_estimator._posthoc_correction( + proposal.default_x + ) + self._logits_pp, self._m_pp, self._prec_pp = ( + logits_pp.detach(), + m_pp.detach(), + prec_pp.detach(), + ) + + # Take care of z-scoring, pre-compute and store prior terms. + self._set_state_for_mog_proposal() + + def log_prob(self, inputs: Tensor, context: Tensor) -> Tensor: + inputs, context = inputs.to(self._device), context.to(self._device) + + if not self._apply_correction: + return self._neural_net.log_prob(inputs, context) + else: + # When we want to compute the approx. posterior, a proposal prior \tilde{p} + # has already been observed. To analytically calculate the log-prob of the + # Gaussian, we first need to compute the mixture components. + + # Compute the mixture components of the proposal posterior. + logits_pp, m_pp, prec_pp = self._posthoc_correction(context) + + # z-score theta if it z-scoring had been requested. + theta = self._maybe_z_score_theta(inputs) + + # Compute the log_prob of theta under the product. + log_prob_proposal_posterior = utils.mog_log_prob( + theta, logits_pp, m_pp, prec_pp + ) + utils.assert_all_finite( + log_prob_proposal_posterior, "proposal posterior eval" + ) + return log_prob_proposal_posterior # \hat{p} from eq (3) in [1] + + def sample(self, num_samples: int, context: Tensor, batch_size: int = 1) -> Tensor: + context = context.to(self._device) + + if not self._apply_correction: + return self._neural_net.sample(num_samples, context, batch_size) + else: + # When we want to sample from the approx. posterior, a proposal prior + # \tilde{p} has already been observed. To analytically calculate the + # log-prob of the Gaussian, we first need to compute the mixture components. + return self._sample_approx_posterior_mog(num_samples, context, batch_size) + + def _sample_approx_posterior_mog( + self, num_samples, x: Tensor, batch_size: int + ) -> Tensor: + r"""Sample from the approximate posterior. + + Args: + num_samples: Desired number of samples. + x: Conditioning context for posterior $p(\theta|x)$. + batch_size: Batch size for sampling. + + Returns: + Samples from the approximate mixture of Gaussians posterior. + """ + + # Compute the mixture components of the posterior. + logits_p, m_p, prec_p = self._posthoc_correction(x) + + # Compute the precision factors which represent the upper triangular matrix + # of the cholesky decomposition of the prec_p. + prec_factors_p = torch.linalg.cholesky(prec_p, upper=True) + + assert logits_p.ndim == 2 + assert m_p.ndim == 3 + assert prec_p.ndim == 4 + assert prec_factors_p.ndim == 4 + + # Replicate to use batched sampling from pyknos. + if batch_size is not None and batch_size > 1: + logits_p = logits_p.repeat(batch_size, 1) + m_p = m_p.repeat(batch_size, 1, 1) + prec_factors_p = prec_factors_p.repeat(batch_size, 1, 1, 1) + + # Get (optionally z-scored) MoG samples. + theta = MultivariateGaussianMDN.sample_mog( + num_samples, logits_p, m_p, prec_factors_p + ) + + embedded_context = self._neural_net._embedding_net(x) + if embedded_context is not None: + # Merge the context dimension with sample dimension in order to + # apply the transform. + theta = torchutils.merge_leading_dims(theta, num_dims=2) + embedded_context = torchutils.repeat_rows( + embedded_context, num_reps=num_samples + ) + + theta, _ = self._neural_net._transform.inverse(theta, context=embedded_context) + + if embedded_context is not None: + # Split the context dimension from sample dimension. + theta = torchutils.split_leading_dim(theta, shape=[-1, num_samples]) + + return theta + + def _posthoc_correction(self, x: Tensor): + """ + Compute the mixture components of the posterior given the current density + estimator and the proposal. + + Args: + x: Conditioning context for posterior. + + Returns: + Mixture components of the posterior. + """ + + # Evaluate the density estimator. + encoded_x = self._neural_net._embedding_net(x) + dist = self._neural_net._distribution # defined to avoid black formatting. + logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) + + # The following if case is needed because, in the constructor, we call + # `_posthoc_correction` regardless of whether the `proposal` itself had a + # `proposal` or not. + if not self._apply_correction: + return norm_logits_d, m_d, prec_d + else: + logits_pp, m_pp, prec_pp = self._logits_pp, self._m_pp, self._prec_pp + + # Compute the MoG parameters of the posterior. + logits_p, m_p, prec_p, cov_p = self._proposal_posterior_transformation( + logits_pp, m_pp, prec_pp, norm_logits_d, m_d, prec_d + ) + return logits_p, m_p, prec_p + + def _proposal_posterior_transformation( + self, + logits_pp: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Transforms the proposal posterior (the MDN) into the posterior. + + The approximate posterior is: + $p(\theta|x) = 1/Z * q(\theta|x) * p(\theta) / prop(\theta)$ + In words: posterior = proposal posterior estimate * prior / proposal. + + Since the proposal posterior estimate and the proposal are MoG, and the + prior is either Gaussian or uniform, we can solve this in closed-form. + + This function implements Appendix C from [1], and is highly similar to + `SNPE_C._automatic_posterior_transformation()`. + + Args: + logits_pp: Component weight of each Gaussian of the proposal prior. + means_pp: Mean of each Gaussian of the proposal prior. + precisions_pp: Precision matrix of each Gaussian of the proposal prior. + logits_d: Component weight for each Gaussian of the density estimator. + means_d: Mean of each Gaussian of the density estimator. + precisions_d: Precision matrix of each Gaussian of the density estimator. + + Returns: (Component weight, mean, precision matrix, covariance matrix) of each + Gaussian of the approximate posterior. + """ + + precisions_post, covariances_post = self._precisions_posterior( + precisions_pp, precisions_d + ) + + means_post = self._means_posterior( + covariances_post, means_pp, precisions_pp, means_d, precisions_d + ) + + logits_post = SNPE_A_MDN._logits_posterior( + means_post, + precisions_post, + covariances_post, + logits_pp, + means_pp, + precisions_pp, + logits_d, + means_d, + precisions_d, + ) + + return logits_post, means_post, precisions_post, covariances_post + + def _set_state_for_mog_proposal(self) -> None: + """ + Set state variables of the SNPE_A_MDN instance every time `set_proposal()` + is called, i.e. every time a posterior is build using + `SNPE_A.build_posterior()`. + + This function is almost identical to `SNPE_C._set_state_for_mog_proposal()`. + + Three things are computed: + 1) Check if z-scoring was requested. To do so, we check if the `_transform` + argument of the net had been a `CompositeTransform`. See pyknos mdn.py. + 2) Define a (potentially standardized) prior. It's standardized if z-scoring + had been requested. + 3) Compute (Precision * mean) for the prior. This quantity is used at every + training step if the prior is Gaussian. + """ + + self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) + + self._set_maybe_z_scored_prior() + + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + self.prec_m_prod_prior = torch.mv( + self._maybe_z_scored_prior.precision_matrix, # type: ignore + self._maybe_z_scored_prior.loc, # type: ignore + ) + + def _set_maybe_z_scored_prior(self) -> None: + r""" + Compute and store potentially standardized prior (if z-scoring was requested). + + This function is highly similar to `SNPE_C._set_maybe_z_scored_prior()`. + + The proposal posterior is: + $p(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + + Let's denote z-scored theta by `a`: a = (theta - mean) / std + Then $p'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ + + The ' indicates that the evaluation occurs in standardized space. The constant + scaling factor has been absorbed into $Z_2$. + From the above equation, we see that we need to evaluate the prior **in + standardized space**. We build the standardized prior in this function. + + The standardize transform that is applied to the samples theta does not use + the exact prior mean and std (due to implementation issues). Hence, the z-scored + prior will not be exactly have mean=0 and std=1. + """ + if self.z_score_theta: + scale = self._neural_net._transform._transforms[0]._scale + shift = self._neural_net._transform._transforms[0]._shift + + # Following the definition of the linear transform in + # `standardizing_transform` in `sbiutils.py`: + # shift=-mean / std + # scale=1 / std + # Solving these equations for mean and std: + estim_prior_std = 1 / scale + estim_prior_mean = -shift * estim_prior_std + + # Compute the discrepancy of the true prior mean and std and the mean and + # std that was empirically estimated from samples. + # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) + # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean + # and std (estimated from samples and used to build standardize transform). + almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std + almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std + + if isinstance(self._prior, MultivariateNormal): + self._maybe_z_scored_prior = MultivariateNormal( + almost_zero_mean, torch.diag(almost_one_std) + ) + else: + range_ = torch.sqrt(almost_one_std * 3.0) + self._maybe_z_scored_prior = utils.BoxUniform( + almost_zero_mean - range_, almost_zero_mean + range_ + ) + else: + self._maybe_z_scored_prior = self._prior + + def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: + """Return potentially standardized theta if z-scoring was requested.""" + + if self.z_score_theta: + theta, _ = self._neural_net._transform(theta) + + return theta + + def _precisions_posterior(self, precisions_pp: Tensor, precisions_d: Tensor): + r"""Return the precisions and covariances of the MoG posterior. + + As described at the end of Appendix C in [1], it can happen that the + proposal's precision matrix is not positive definite. + + $S_k^\prime = ( S_k^{-1} - S_0^{-1} )^{-1}$ + (see eq (23) in Appendix C of [1]) + + Args: + precisions_pp: Precision matrices of the proposal prior. + precisions_d: Precision matrices of the density estimator. + + Returns: (Precisions, Covariances) of the MoG posterior. + """ + + num_comps_p = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Check if precision matrices are positive definite. + for batches in precisions_pp: + for pprior in batches: + eig_pprior = torch.linalg.eigvalsh(pprior, UPLO="U") + if not (eig_pprior > 0).all(): + raise AssertionError( + "The precision matrix of the proposal is not positive definite!" + ) + for batches in precisions_d: + for d in batches: + eig_d = torch.linalg.eigvalsh(d, UPLO="U") + if not (eig_d > 0).all(): + raise AssertionError( + "The precision matrix of the density estimator is not " + "positive definite!" + ) + + precisions_pp_rep = precisions_pp.repeat_interleave(num_comps_d, dim=1) + precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) + + precisions_p = precisions_d_rep - precisions_pp_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + precisions_p += self._maybe_z_scored_prior.precision_matrix + + # Check if precision matrix is positive definite. + for idx_batch, batches in enumerate(precisions_p): + for idx_comp, pp in enumerate(batches): + eig_pp = torch.symeig(pp, eigenvectors=False).eigenvalues + if not (eig_pp > 0).all(): + raise AssertionError( + "The precision matrix of a posterior is not positive " + "definite! This is a known issue for SNPE-A. Either try a " + "different parameter setting, e.g. a different number of " + "mixture components (when contracting SNPE-A), or a different " + "value for the parameter perturbation (when building the " + "posterior)." + ) + + covariances_p = torch.inverse(precisions_p) + return precisions_p, covariances_p + + def _means_posterior( + self, + covariances_p: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Return the means of the MoG posterior. + + $m_k^\prime = S_k^\prime ( S_k^{-1} m_k - S_0^{-1} m_0 )$ + (see eq (24) in Appendix C of [1]) + + Args: + covariances_post: Covariance matrices of the MoG posterior. + means_pp: Means of the proposal prior. + precisions_pp: Precision matrices of the proposal prior. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Means of the MoG posterior. + """ + + num_comps_pp = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute the products P_k * m_k and P_0 * m_0. + prec_m_prod_pp = utils.batched_mixture_mv(precisions_pp, means_pp) + prec_m_prod_d = utils.batched_mixture_mv(precisions_d, means_d) + + # Repeat them to allow for matrix operations: same trick as for the precisions. + prec_m_prod_pp_rep = prec_m_prod_pp.repeat_interleave(num_comps_d, dim=1) + prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_pp, 1) + + # Compute the means P_k^prime * (P_k * m_k - P_0 * m_0). + summed_cov_m_prod_rep = prec_m_prod_d_rep - prec_m_prod_pp_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + summed_cov_m_prod_rep += self.prec_m_prod_prior + + means_p = utils.batched_mixture_mv(covariances_p, summed_cov_m_prod_rep) + return means_p + + @staticmethod + def _logits_posterior( + means_post: Tensor, + precisions_post: Tensor, + covariances_post: Tensor, + logits_pp: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Return the component weights (i.e. logits) of the MoG posterior. + + $\alpha_k^\prime = \frac{ \alpha_k exp(-0.5 c_k) }{ \sum{j} \alpha_j exp(-0.5 + c_j) } $ + with + $c_k = logdet(S_k) - logdet(S_0) - logdet(S_k^\prime) + + + m_k^T P_k m_k - m_0^T P_0 m_0 - m_k^\prime^T P_k^\prime m_k^\prime$ + (see eqs. (25, 26) in Appendix C of [1]) + + Args: + means_post: Means of the posterior. + precisions_post: Precision matrices of the posterior. + covariances_post: Covariance matrices of the posterior. + logits_pp: Component weights (i.e. logits) of the proposal prior. + means_pp: Means of the proposal prior. + precisions_pp: Precision matrices of the proposal prior. + logits_d: Component weights (i.e. logits) of the density estimator. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Component weights of the proposal posterior. + """ + + num_comps_pp = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute the ratio of the logits similar to eq (10) in Appendix A.1 of [2] + logits_pp_rep = logits_pp.repeat_interleave(num_comps_d, dim=1) + logits_d_rep = logits_d.repeat(1, num_comps_pp) + logit_factors = logits_d_rep - logits_pp_rep + + # Compute the log-determinants + logdet_covariances_post = torch.logdet(covariances_post) + logdet_covariances_pp = -torch.logdet(precisions_pp) + logdet_covariances_d = -torch.logdet(precisions_d) + + # Repeat the proposal and density estimator terms such that there are LK terms. + # Same trick as has been used above. + logdet_covariances_pp_rep = logdet_covariances_pp.repeat_interleave( + num_comps_d, dim=1 + ) + logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_pp) + + log_sqrt_det_ratio = 0.5 * ( # similar to eq (14) in Appendix A.1 of [2] + logdet_covariances_post + + logdet_covariances_pp_rep + - logdet_covariances_d_rep + ) + + # Compute for proposal, density estimator, and proposal posterior: + exponent_pp = utils.batched_mixture_vmv( + precisions_pp, means_pp # m_0 in eq (26) in Appendix C of [1] + ) + exponent_d = utils.batched_mixture_vmv( + precisions_d, means_d # m_k in eq (26) in Appendix C of [1] + ) + exponent_post = utils.batched_mixture_vmv( + precisions_post, means_post # m_k^\prime in eq (26) in Appendix C of [1] + ) + + # Extend proposal and density estimator exponents to get LK terms. + exponent_pp_rep = exponent_pp.repeat_interleave(num_comps_d, dim=1) + exponent_d_rep = exponent_d.repeat(1, num_comps_pp) + exponent = -0.5 * ( + exponent_d_rep - exponent_pp_rep - exponent_post # eq (26) in [1] + ) + + logits_post = logit_factors + log_sqrt_det_ratio + exponent + return logits_post diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 26c862282..1a56f9c75 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -1,594 +1,597 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . -import time -from abc import ABC, abstractmethod -from copy import deepcopy -from typing import Any, Callable, Dict, Optional, Union -from warnings import warn - -import torch -from torch import Tensor, nn, ones, optim -from torch.distributions import Distribution -from torch.nn.utils.clip_grad import clip_grad_norm_ -from torch.utils import data -from torch.utils.tensorboard.writer import SummaryWriter - -from sbi import utils as utils -from sbi.inference import NeuralInference, check_if_proposal_has_default_x -from sbi.inference.posteriors import ( - DirectPosterior, - MCMCPosterior, - RejectionPosterior, - VIPosterior, -) -from sbi.inference.posteriors.base_posterior import NeuralPosterior -from sbi.inference.potentials import posterior_estimator_based_potential -from sbi.utils import ( - RestrictedPrior, - check_estimator_arg, - test_posterior_net_for_multi_d_x, - validate_theta_and_x, - x_shape_from_simulation, - handle_invalid_x, - warn_if_zscoring_changes_data, - warn_on_invalid_x, - warn_on_invalid_x_for_snpec_leakage, -) -from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior - - -class PosteriorEstimator(NeuralInference, ABC): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "maf", - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[SummaryWriter] = None, - show_progress_bars: bool = True, - ): - """Base class for Sequential Neural Posterior Estimation methods. - - Args: - density_estimator: If it is a string, use a pre-configured network of the - provided type (one of nsf, maf, mdn, made). Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. - - See docstring of `NeuralInference` class for all other arguments. - """ - - super().__init__( - prior=prior, - device=device, - logging_level=logging_level, - summary_writer=summary_writer, - show_progress_bars=show_progress_bars, - ) - - # As detailed in the docstring, `density_estimator` is either a string or - # a callable. The function creating the neural network is attached to - # `_build_neural_net`. It will be called in the first round and receive - # thetas and xs as inputs, so that they can be used for shape inference and - # potentially for z-scoring. - check_estimator_arg(density_estimator) - if isinstance(density_estimator, str): - self._build_neural_net = utils.posterior_nn(model=density_estimator) - else: - self._build_neural_net = density_estimator - - self._proposal_roundwise = [] - self.use_non_atomic_loss = False - - # Extra SNPE-specific fields summary_writer. - self._summary.update({"rejection_sampling_acceptance_rates": []}) # type:ignore - - def append_simulations( - self, - theta: Tensor, - x: Tensor, - proposal: Optional[DirectPosterior] = None, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: bool = True, - return_self: bool = True, - data_device: str = None - ) -> "PosteriorEstimator": - r"""Store parameters and simulation outputs to use them for later training. - - Data are stored as entries in lists for each type of variable (parameter/data). - - Stores $\theta$, $x$, prior_masks (indicating if simulations are coming from the - prior or not) and an index indicating which round the batch of simulations came - from. - - Args: - theta: Parameter sets. - x: Simulation outputs. - proposal: The distribution that the parameters $\theta$ were sampled from. - Pass `None` if the parameters were sampled from the prior. If not - `None`, it will trigger a different loss-function. - - Returns: - NeuralInference object (returned so that this function is chainable). - """ - - # Add ability to specify device data is saved on - if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) - - - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) - warn_on_invalid_x_for_snpec_leakage( - num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round - ) - - x = x[is_valid_x] - theta = theta[is_valid_x] - - - self._check_proposal(proposal) - - if ( - proposal is None - or proposal is self._prior - or ( - isinstance(proposal, RestrictedPrior) and proposal._prior is self._prior - ) - ): - # The `_data_round_index` will later be used to infer if one should train - # with MLE loss or with atomic loss (see, in `train()`: - # self._round = max(self._data_round_index)) - self._data_round_index.append(0) - prior_masks = mask_sims_from_prior(0, theta.size(0)) - else: - if not self._data_round_index: - # This catches a pretty specific case: if, in the first round, one - # passes data that does not come from the prior. - self._data_round_index.append(1) - else: - self._data_round_index.append(max(self._data_round_index) + 1) - prior_masks = mask_sims_from_prior(1, theta.size(0)) - - - if self._dataset is None: - #If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) - else: - #Otherwise append to Dataset - self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) - - self._num_sims_per_round.append(theta.size(0)) - self._proposal_roundwise.append(proposal) - - if self._prior is None or isinstance(self._prior, ImproperEmpirical): - if proposal is not None: - raise ValueError( - "You had not passed a prior at initialization, but now you " - "passed a proposal. If you want to run multi-round SNPE, you have " - "to specify a prior (set the `.prior` argument or re-initialize " - "the object with a prior distribution). If the samples you passed " - "to `append_simulations()` were sampled from the prior, you can " - "run single-round inference with " - "`append_simulations(..., proposal=None)`." - ) - theta_prior = self.get_simulations()[0] - self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) - - #Add ability to not return self - if return_self: - return self - else: - return 1 - - def train( - self, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - exclude_invalid_x: bool = True, - resume_training: bool = False, - force_first_round_loss: bool = False, - discard_prior_samples: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - warn_if_zscoring: Optional[bool] = True, - dataloader_kwargs: Optional[dict] = None, - ) -> nn.Module: - r"""Return density estimator that approximates the distribution $p(\theta|x)$. - - Args: - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - discard_prior_samples: Whether to discard samples simulated in round 1, i.e. - from the prior. Training may be sped up by ignoring such less targeted - samples. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. - show_train_summary: Whether to print the number of epochs and validation - loss after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - if self._round == 0 and self._neural_net is not None: - assert force_first_round_loss, ( - "You have already trained this neural network. After you had trained " - "the network, you again appended simulations with `append_simulations" - "(theta, x)`, but you did not provide a proposal. If the new " - "simulations are sampled from the prior, you can set " - "`.train(..., force_first_round_loss=True`). However, if the new " - "simulations were not sampled from the prior, you should pass the " - "proposal, i.e. `append_simulations(theta, x, proposal)`. If " - "your samples are not sampled from the prior and you do not pass a " - "proposal and you set `force_first_round_loss=True`, the result of " - "SNPE will not be the true posterior. Instead, it will be the proposal " - "posterior, which (usually) is more narrow than the true posterior." - ) - - # Calibration kernels proposed in Lueckmann, Gonçalves et al., 2017. - if calibration_kernel is None: - calibration_kernel = lambda x: ones([len(x)], device=self._device) - - # Starting index for the training set (1 = discard round-0 samples). - start_idx = int(discard_prior_samples and self._round > 0) - - # For non-atomic loss, we can not reuse samples from previous rounds as of now. - # SNPE-A can, by construction of the algorithm, only use samples from the last - # round. SNPE-A is the only algorithm that has an attribute `_ran_final_round`, - # so this is how we check for whether or not we are using SNPE-A. - if self.use_non_atomic_loss or hasattr(self, "_ran_final_round"): - start_idx = self._round - - # Set the proposal to the last proposal that was passed by the user. For - # atomic SNPE, it does not matter what the proposal is. For non-atomic - # SNPE, we only use the latest data that was passed, i.e. the one from the - # last proposal. - proposal = self._proposal_roundwise[-1] - - train_loader, val_loader = self.get_dataloaders_all( - start_idx, - training_batch_size, - validation_fraction, - resume_training, - dataloader_kwargs=dataloader_kwargs, - ) - # First round or if retraining from scratch: - # Call the `self._build_neural_net` with the rounds' thetas and xs as - # arguments, which will build the neural network. - # This is passed into NeuralPosterior, to create a neural posterior which - # can `sample()` and `log_prob()`. The network is accessible via `.net`. - if self._neural_net is None or retrain_from_scratch: - - #Get theta,x from dataset to initialize NN - test_theta = self._dataset.datasets[0].tensors[0][:100] - test_x = self._dataset.datasets[0].tensors[1][:100] - - self._neural_net = self._build_neural_net( - test_theta, test_x - ) - # If data on training device already move net as well. - if ( - not self._device == "cpu" - and f"{test_x.device.type}:{test_x.device.index}" == self._device - ): - self._neural_net.to(self._device) - - test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) - self._x_shape = x_shape_from_simulation(test_x) - - # Move entire net to device for training. - self._neural_net.to(self._device) - - if not resume_training: - self.optimizer = optim.Adam( - list(self._neural_net.parameters()), lr=learning_rate - ) - self.epoch, self._val_log_prob = 0, float("-Inf") - - while self.epoch <= max_num_epochs and not self._converged( - self.epoch, stop_after_epochs - ): - - # Train for a single epoch. - self._neural_net.train() - train_log_probs_sum = 0 - epoch_start_time = time.time() - for batch in train_loader: - self.optimizer.zero_grad() - # Get batches on current device. - theta_batch, x_batch, masks_batch = ( - batch[0].to(self._device), - batch[1].to(self._device), - batch[2].to(self._device), - ) - - train_losses = self._loss( - theta_batch, x_batch, masks_batch, proposal, calibration_kernel - ) - train_loss = torch.mean(train_losses) - train_log_probs_sum -= train_losses.sum().item() - - train_loss.backward() - if clip_max_norm is not None: - clip_grad_norm_( - self._neural_net.parameters(), max_norm=clip_max_norm - ) - self.optimizer.step() - - self.epoch += 1 - - train_log_prob_average = train_log_probs_sum / ( - len(train_loader) * train_loader.batch_size # type: ignore - ) - self._summary["train_log_probs"].append(train_log_prob_average) - - # Calculate validation performance. - self._neural_net.eval() - val_log_prob_sum = 0 - - with torch.no_grad(): - for batch in val_loader: - theta_batch, x_batch, masks_batch = ( - batch[0].to(self._device), - batch[1].to(self._device), - batch[2].to(self._device), - ) - # Take negative loss here to get validation log_prob. - val_losses = self._loss( - theta_batch, - x_batch, - masks_batch, - proposal, - calibration_kernel, - ) - val_log_prob_sum -= val_losses.sum().item() - - # Take mean over all validation samples. - self._val_log_prob = val_log_prob_sum / ( - len(val_loader) * val_loader.batch_size # type: ignore - ) - # Log validation log prob for every epoch. - self._summary["validation_log_probs"].append(self._val_log_prob) - self._summary["epoch_durations_sec"].append(time.time() - epoch_start_time) - - self._maybe_show_progress(self._show_progress_bars, self.epoch) - - self._report_convergence_at_end(self.epoch, stop_after_epochs, max_num_epochs) - - # Update summary. - self._summary["epochs"].append(self.epoch) - self._summary["best_validation_log_probs"].append(self._best_val_log_prob) - - # Update tensorboard and summary dict. - self._summarize(round_=self._round, x_o=None, theta_bank=None, x_bank=None) - - # Update description for progress bar. - if show_train_summary: - print(self._describe_round(self._round, self._summary)) - - # Avoid keeping the gradients in the resulting network, which can - # cause memory leakage when benchmarking. - self._neural_net.zero_grad(set_to_none=True) - - return deepcopy(self._neural_net) - - def build_posterior( - self, - density_estimator: Optional[nn.Module] = None, - prior: Optional[Distribution] = None, - sample_with: str = "rejection", - mcmc_method: str = "slice_np", - vi_method: str = "rKL", - mcmc_parameters: Dict[str, Any] = {}, - vi_parameters: Dict[str, Any] = {}, - rejection_sampling_parameters: Dict[str, Any] = {}, - ) -> Union[MCMCPosterior, RejectionPosterior, VIPosterior, DirectPosterior]: - r"""Build posterior from the neural density estimator. - - For SNPE, the posterior distribution that is returned here implements the - following functionality over the raw neural density estimator: - - correct the calculation of the log probability such that it compensates for - the leakage. - - reject samples that lie outside of the prior bounds. - - alternatively, if leakage is very high (which can happen for multi-round - SNPE), sample from the posterior with MCMC. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - prior: Prior distribution. - sample_with: Method to use for sampling from the posterior. Must be one of - [`mcmc` | `rejection` | `vi`]. - mcmc_method: Method used for MCMC sampling, one of `slice_np`, `slice`, - `hmc`, `nuts`. Currently defaults to `slice_np` for a custom numpy - implementation of slice sampling; select `hmc`, `nuts` or `slice` for - Pyro-based sampling. - vi_method: Method used for VI, one of [`rKL`, `fKL`, `IW`, `alpha`]. Note - some of the methods admit a `mode seeking` property (e.g. rKL) whereas - some admit a `mass covering` one (e.g fKL). - mcmc_parameters: Additional kwargs passed to `MCMCPosterior`. - vi_parameters: Additional kwargs passed to `VIPosterior`. - rejection_sampling_parameters: Additional kwargs passed to - `RejectionPosterior` or `DirectPosterior`. By default, - `DirectPosterior` is used. Only if `rejection_sampling_parameters` - contains `proposal`, a `RejectionPosterior` is instantiated. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods - (the returned log-probability is unnormalized). - """ - if prior is None: - assert self._prior is not None, ( - "You did not pass a prior. You have to pass the prior either at " - "initialization `inference = SNPE(prior)` or to " - "`.build_posterior(prior=prior)`." - ) - prior = self._prior - else: - utils.check_prior(prior) - - if density_estimator is None: - posterior_estimator = self._neural_net - # If internal net is used device is defined. - device = self._device - else: - posterior_estimator = density_estimator - # Otherwise, infer it from the device of the net parameters. - device = next(density_estimator.parameters()).device.type - - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator=posterior_estimator, prior=prior, x_o=None - ) - - if sample_with == "rejection": - if "proposal" in rejection_sampling_parameters.keys(): - self._posterior = RejectionPosterior( - potential_fn=potential_fn, - device=device, - x_shape=self._x_shape, - **rejection_sampling_parameters, - ) - else: - self._posterior = DirectPosterior( - posterior_estimator=posterior_estimator, - prior=prior, - x_shape=self._x_shape, - device=device, - ) - elif sample_with == "mcmc": - self._posterior = MCMCPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - proposal=prior, - method=mcmc_method, - device=device, - x_shape=self._x_shape, - **mcmc_parameters, - ) - elif sample_with == "vi": - self._posterior = VIPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - prior=prior, # type: ignore - vi_method=vi_method, - device=device, - x_shape=self._x_shape, - **vi_parameters, - ) - else: - raise NotImplementedError - - # Store models at end of each round. - self._model_bank.append(deepcopy(self._posterior)) - - return deepcopy(self._posterior) - - @abstractmethod - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - ) -> Tensor: - raise NotImplementedError - - def _loss( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - calibration_kernel: Callable, - ) -> Tensor: - """Return loss with proposal correction (`round_>0`) or without it (`round_=0`). - - The loss is the negative log prob. Irrespective of the round or SNPE method - (A, B, or C), it can be weighted with a calibration kernel. - - Returns: - Calibration kernel-weighted negative log prob. - """ - if self._round == 0: - # Use posterior log prob (without proposal correction) for first round. - log_prob = self._neural_net.log_prob(theta, x) - else: - log_prob = self._log_prob_proposal_posterior(theta, x, masks, proposal) - - return -(calibration_kernel(x) * log_prob) - - def _check_proposal(self, proposal): - """ - Check for validity of the provided proposal distribution. - - If the proposal is a `NeuralPosterior`, we check if the default_x is set. - If the proposal is **not** a `NeuralPosterior`, we warn since it is likely that - the user simply passed the prior, but this would still trigger atomic loss. - """ - if proposal is not None: - check_if_proposal_has_default_x(proposal) - - if isinstance(proposal, RestrictedPrior): - if proposal._prior is not self._prior: - warn( - "The proposal you passed is a `RestrictedPrior`, but the " - "proposal distribution it uses is not the prior (it can be " - "accessed via `RestrictedPrior._prior`). We do not " - "recommend to mix the `RestrictedPrior` with multi-round " - "SNPE." - ) - elif ( - not isinstance(proposal, NeuralPosterior) - and proposal is not self._prior - ): - warn( - "The proposal you passed is neither the prior nor a " - "`NeuralPosterior` object. If you are an expert user and did so " - "for research purposes, this is fine. If not, you might be doing " - "something wrong: feel free to create an issue on Github." - ) - elif self._round > 0: - raise ValueError( - "A proposal was passed but no prior was passed at initialisation. When " - "running multi-round inference, a prior needs to be specified upon " - "initialisation. Potential fix: setting the `._prior` attribute or " - "re-initialisation. If the samples passed to `append_simulations()` " - "were sampled from the prior, single-round inference can be performed " - "with `append_simulations(..., proprosal=None)`." - ) +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . +import time +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Any, Callable, Dict, Optional, Union +from warnings import warn + +import torch +from torch import Tensor, nn, ones, optim +from torch.distributions import Distribution +from torch.nn.utils.clip_grad import clip_grad_norm_ +from torch.utils import data +from torch.utils.tensorboard.writer import SummaryWriter + +from sbi import utils as utils +from sbi.inference import NeuralInference, check_if_proposal_has_default_x +from sbi.inference.posteriors import ( + DirectPosterior, + MCMCPosterior, + RejectionPosterior, + VIPosterior, +) +from sbi.inference.posteriors.base_posterior import NeuralPosterior +from sbi.inference.potentials import posterior_estimator_based_potential +from sbi.utils import ( + RestrictedPrior, + check_estimator_arg, + test_posterior_net_for_multi_d_x, + validate_theta_and_x, + x_shape_from_simulation, + handle_invalid_x, + warn_if_zscoring_changes_data, + warn_on_invalid_x, + warn_on_invalid_x_for_snpec_leakage, +) +from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior + + +class PosteriorEstimator(NeuralInference, ABC): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "maf", + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[SummaryWriter] = None, + show_progress_bars: bool = True, + ): + """Base class for Sequential Neural Posterior Estimation methods. + + Args: + density_estimator: If it is a string, use a pre-configured network of the + provided type (one of nsf, maf, mdn, made). Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. + + See docstring of `NeuralInference` class for all other arguments. + """ + + super().__init__( + prior=prior, + device=device, + logging_level=logging_level, + summary_writer=summary_writer, + show_progress_bars=show_progress_bars, + ) + + # As detailed in the docstring, `density_estimator` is either a string or + # a callable. The function creating the neural network is attached to + # `_build_neural_net`. It will be called in the first round and receive + # thetas and xs as inputs, so that they can be used for shape inference and + # potentially for z-scoring. + check_estimator_arg(density_estimator) + if isinstance(density_estimator, str): + self._build_neural_net = utils.posterior_nn(model=density_estimator) + else: + self._build_neural_net = density_estimator + + self._proposal_roundwise = [] + self.use_non_atomic_loss = False + + # Extra SNPE-specific fields summary_writer. + self._summary.update({"rejection_sampling_acceptance_rates": []}) # type:ignore + + def append_simulations( + self, + theta: Tensor, + x: Tensor, + proposal: Optional[DirectPosterior] = None, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + warn_if_zscoring: bool = True, + return_self: bool = True, + data_device: str = None, + ) -> "PosteriorEstimator": + r"""Store parameters and simulation outputs to use them for later training. + + Data are stored as entries in lists for each type of variable (parameter/data). + + Stores $\theta$, $x$, prior_masks (indicating if simulations are coming from the + prior or not) and an index indicating which round the batch of simulations came + from. + + Args: + theta: Parameter sets. + x: Simulation outputs. + proposal: The distribution that the parameters $\theta$ were sampled from. + Pass `None` if the parameters were sampled from the prior. If not + `None`, it will trigger a different loss-function. + exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` + during training. Expect errors, silent or explicit, when `False`. + warn_on_invalid: Whether to warn if data is invalid + warn_if_zscoring: Whether to test if z-scoring causes duplicates + return_self: Whether to return a instance of the class, allows chaining + with `.train()`. Setting `False` decreases memory overhead. + data_device: Where to store the data, default is on the same device where + the training is happening. If training a large dataset on a GPU with not + much VRAM can set to 'cpu' to store data on system memory instead. + + Returns: + NeuralInference object (returned so that this function is chainable). + """ + + # Add ability to specify device data is saved on + if data_device is None: data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) + + + is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) + + # Check for problematic z-scoring + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_on_invalid: + warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + warn_on_invalid_x_for_snpec_leakage( + num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round + ) + + x = x[is_valid_x] + theta = theta[is_valid_x] + + + self._check_proposal(proposal) + + if ( + proposal is None + or proposal is self._prior + or ( + isinstance(proposal, RestrictedPrior) and proposal._prior is self._prior + ) + ): + # The `_data_round_index` will later be used to infer if one should train + # with MLE loss or with atomic loss (see, in `train()`: + # self._round = max(self._data_round_index)) + self._data_round_index.append(0) + prior_masks = mask_sims_from_prior(0, theta.size(0)) + else: + if not self._data_round_index: + # This catches a pretty specific case: if, in the first round, one + # passes data that does not come from the prior. + self._data_round_index.append(1) + else: + self._data_round_index.append(max(self._data_round_index) + 1) + prior_masks = mask_sims_from_prior(1, theta.size(0)) + + + if self._dataset is None: + #If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + else: + #Otherwise append to Dataset + self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) + + self._num_sims_per_round.append(theta.size(0)) + self._proposal_roundwise.append(proposal) + + if self._prior is None or isinstance(self._prior, ImproperEmpirical): + if proposal is not None: + raise ValueError( + "You had not passed a prior at initialization, but now you " + "passed a proposal. If you want to run multi-round SNPE, you have " + "to specify a prior (set the `.prior` argument or re-initialize " + "the object with a prior distribution). If the samples you passed " + "to `append_simulations()` were sampled from the prior, you can " + "run single-round inference with " + "`append_simulations(..., proposal=None)`." + ) + theta_prior = self.get_simulations()[0] + self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) + + #Add ability to not return self + if return_self: + return self + + def train( + self, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + discard_prior_samples: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[dict] = None, + ) -> nn.Module: + r"""Return density estimator that approximates the distribution $p(\theta|x)$. + + Args: + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + discard_prior_samples: Whether to discard samples simulated in round 1, i.e. + from the prior. Training may be sped up by ignoring such less targeted + samples. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. + show_train_summary: Whether to print the number of epochs and validation + loss after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + if self._round == 0 and self._neural_net is not None: + assert force_first_round_loss, ( + "You have already trained this neural network. After you had trained " + "the network, you again appended simulations with `append_simulations" + "(theta, x)`, but you did not provide a proposal. If the new " + "simulations are sampled from the prior, you can set " + "`.train(..., force_first_round_loss=True`). However, if the new " + "simulations were not sampled from the prior, you should pass the " + "proposal, i.e. `append_simulations(theta, x, proposal)`. If " + "your samples are not sampled from the prior and you do not pass a " + "proposal and you set `force_first_round_loss=True`, the result of " + "SNPE will not be the true posterior. Instead, it will be the proposal " + "posterior, which (usually) is more narrow than the true posterior." + ) + + # Calibration kernels proposed in Lueckmann, Gonçalves et al., 2017. + if calibration_kernel is None: + calibration_kernel = lambda x: ones([len(x)], device=self._device) + + # Starting index for the training set (1 = discard round-0 samples). + start_idx = int(discard_prior_samples and self._round > 0) + + # For non-atomic loss, we can not reuse samples from previous rounds as of now. + # SNPE-A can, by construction of the algorithm, only use samples from the last + # round. SNPE-A is the only algorithm that has an attribute `_ran_final_round`, + # so this is how we check for whether or not we are using SNPE-A. + if self.use_non_atomic_loss or hasattr(self, "_ran_final_round"): + start_idx = self._round + + # Set the proposal to the last proposal that was passed by the user. For + # atomic SNPE, it does not matter what the proposal is. For non-atomic + # SNPE, we only use the latest data that was passed, i.e. the one from the + # last proposal. + proposal = self._proposal_roundwise[-1] + + train_loader, val_loader = self.get_dataloaders( + start_idx, + training_batch_size, + validation_fraction, + resume_training, + dataloader_kwargs=dataloader_kwargs, + ) + # First round or if retraining from scratch: + # Call the `self._build_neural_net` with the rounds' thetas and xs as + # arguments, which will build the neural network. + # This is passed into NeuralPosterior, to create a neural posterior which + # can `sample()` and `log_prob()`. The network is accessible via `.net`. + if self._neural_net is None or retrain_from_scratch: + + #Get theta,x from dataset to initialize NN + test_theta = self._dataset.datasets[0].tensors[0][:100] + test_x = self._dataset.datasets[0].tensors[1][:100] + + self._neural_net = self._build_neural_net( + test_theta, test_x + ) + # If data on training device already move net as well. + if ( + not self._device == "cpu" + and f"{test_x.device.type}:{test_x.device.index}" == self._device + ): + self._neural_net.to(self._device) + + test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) + self._x_shape = x_shape_from_simulation(test_x) + + # Move entire net to device for training. + self._neural_net.to(self._device) + + if not resume_training: + self.optimizer = optim.Adam( + list(self._neural_net.parameters()), lr=learning_rate + ) + self.epoch, self._val_log_prob = 0, float("-Inf") + + while self.epoch <= max_num_epochs and not self._converged( + self.epoch, stop_after_epochs + ): + + # Train for a single epoch. + self._neural_net.train() + train_log_probs_sum = 0 + epoch_start_time = time.time() + for batch in train_loader: + self.optimizer.zero_grad() + # Get batches on current device. + theta_batch, x_batch, masks_batch = ( + batch[0].to(self._device), + batch[1].to(self._device), + batch[2].to(self._device), + ) + + train_losses = self._loss( + theta_batch, x_batch, masks_batch, proposal, calibration_kernel + ) + train_loss = torch.mean(train_losses) + train_log_probs_sum -= train_losses.sum().item() + + train_loss.backward() + if clip_max_norm is not None: + clip_grad_norm_( + self._neural_net.parameters(), max_norm=clip_max_norm + ) + self.optimizer.step() + + self.epoch += 1 + + train_log_prob_average = train_log_probs_sum / ( + len(train_loader) * train_loader.batch_size # type: ignore + ) + self._summary["train_log_probs"].append(train_log_prob_average) + + # Calculate validation performance. + self._neural_net.eval() + val_log_prob_sum = 0 + + with torch.no_grad(): + for batch in val_loader: + theta_batch, x_batch, masks_batch = ( + batch[0].to(self._device), + batch[1].to(self._device), + batch[2].to(self._device), + ) + # Take negative loss here to get validation log_prob. + val_losses = self._loss( + theta_batch, + x_batch, + masks_batch, + proposal, + calibration_kernel, + ) + val_log_prob_sum -= val_losses.sum().item() + + # Take mean over all validation samples. + self._val_log_prob = val_log_prob_sum / ( + len(val_loader) * val_loader.batch_size # type: ignore + ) + # Log validation log prob for every epoch. + self._summary["validation_log_probs"].append(self._val_log_prob) + self._summary["epoch_durations_sec"].append(time.time() - epoch_start_time) + + self._maybe_show_progress(self._show_progress_bars, self.epoch) + + self._report_convergence_at_end(self.epoch, stop_after_epochs, max_num_epochs) + + # Update summary. + self._summary["epochs"].append(self.epoch) + self._summary["best_validation_log_probs"].append(self._best_val_log_prob) + + # Update tensorboard and summary dict. + self._summarize(round_=self._round, x_o=None, theta_bank=None, x_bank=None) + + # Update description for progress bar. + if show_train_summary: + print(self._describe_round(self._round, self._summary)) + + # Avoid keeping the gradients in the resulting network, which can + # cause memory leakage when benchmarking. + self._neural_net.zero_grad(set_to_none=True) + + return deepcopy(self._neural_net) + + def build_posterior( + self, + density_estimator: Optional[nn.Module] = None, + prior: Optional[Distribution] = None, + sample_with: str = "rejection", + mcmc_method: str = "slice_np", + vi_method: str = "rKL", + mcmc_parameters: Dict[str, Any] = {}, + vi_parameters: Dict[str, Any] = {}, + rejection_sampling_parameters: Dict[str, Any] = {}, + ) -> Union[MCMCPosterior, RejectionPosterior, VIPosterior, DirectPosterior]: + r"""Build posterior from the neural density estimator. + + For SNPE, the posterior distribution that is returned here implements the + following functionality over the raw neural density estimator: + - correct the calculation of the log probability such that it compensates for + the leakage. + - reject samples that lie outside of the prior bounds. + - alternatively, if leakage is very high (which can happen for multi-round + SNPE), sample from the posterior with MCMC. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + prior: Prior distribution. + sample_with: Method to use for sampling from the posterior. Must be one of + [`mcmc` | `rejection` | `vi`]. + mcmc_method: Method used for MCMC sampling, one of `slice_np`, `slice`, + `hmc`, `nuts`. Currently defaults to `slice_np` for a custom numpy + implementation of slice sampling; select `hmc`, `nuts` or `slice` for + Pyro-based sampling. + vi_method: Method used for VI, one of [`rKL`, `fKL`, `IW`, `alpha`]. Note + some of the methods admit a `mode seeking` property (e.g. rKL) whereas + some admit a `mass covering` one (e.g fKL). + mcmc_parameters: Additional kwargs passed to `MCMCPosterior`. + vi_parameters: Additional kwargs passed to `VIPosterior`. + rejection_sampling_parameters: Additional kwargs passed to + `RejectionPosterior` or `DirectPosterior`. By default, + `DirectPosterior` is used. Only if `rejection_sampling_parameters` + contains `proposal`, a `RejectionPosterior` is instantiated. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods + (the returned log-probability is unnormalized). + """ + if prior is None: + assert self._prior is not None, ( + "You did not pass a prior. You have to pass the prior either at " + "initialization `inference = SNPE(prior)` or to " + "`.build_posterior(prior=prior)`." + ) + prior = self._prior + else: + utils.check_prior(prior) + + if density_estimator is None: + posterior_estimator = self._neural_net + # If internal net is used device is defined. + device = self._device + else: + posterior_estimator = density_estimator + # Otherwise, infer it from the device of the net parameters. + device = next(density_estimator.parameters()).device.type + + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator=posterior_estimator, prior=prior, x_o=None + ) + + if sample_with == "rejection": + if "proposal" in rejection_sampling_parameters.keys(): + self._posterior = RejectionPosterior( + potential_fn=potential_fn, + device=device, + x_shape=self._x_shape, + **rejection_sampling_parameters, + ) + else: + self._posterior = DirectPosterior( + posterior_estimator=posterior_estimator, + prior=prior, + x_shape=self._x_shape, + device=device, + ) + elif sample_with == "mcmc": + self._posterior = MCMCPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + proposal=prior, + method=mcmc_method, + device=device, + x_shape=self._x_shape, + **mcmc_parameters, + ) + elif sample_with == "vi": + self._posterior = VIPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + prior=prior, # type: ignore + vi_method=vi_method, + device=device, + x_shape=self._x_shape, + **vi_parameters, + ) + else: + raise NotImplementedError + + # Store models at end of each round. + self._model_bank.append(deepcopy(self._posterior)) + + return deepcopy(self._posterior) + + @abstractmethod + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + ) -> Tensor: + raise NotImplementedError + + def _loss( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + calibration_kernel: Callable, + ) -> Tensor: + """Return loss with proposal correction (`round_>0`) or without it (`round_=0`). + + The loss is the negative log prob. Irrespective of the round or SNPE method + (A, B, or C), it can be weighted with a calibration kernel. + + Returns: + Calibration kernel-weighted negative log prob. + """ + if self._round == 0: + # Use posterior log prob (without proposal correction) for first round. + log_prob = self._neural_net.log_prob(theta, x) + else: + log_prob = self._log_prob_proposal_posterior(theta, x, masks, proposal) + + return -(calibration_kernel(x) * log_prob) + + def _check_proposal(self, proposal): + """ + Check for validity of the provided proposal distribution. + + If the proposal is a `NeuralPosterior`, we check if the default_x is set. + If the proposal is **not** a `NeuralPosterior`, we warn since it is likely that + the user simply passed the prior, but this would still trigger atomic loss. + """ + if proposal is not None: + check_if_proposal_has_default_x(proposal) + + if isinstance(proposal, RestrictedPrior): + if proposal._prior is not self._prior: + warn( + "The proposal you passed is a `RestrictedPrior`, but the " + "proposal distribution it uses is not the prior (it can be " + "accessed via `RestrictedPrior._prior`). We do not " + "recommend to mix the `RestrictedPrior` with multi-round " + "SNPE." + ) + elif ( + not isinstance(proposal, NeuralPosterior) + and proposal is not self._prior + ): + warn( + "The proposal you passed is neither the prior nor a " + "`NeuralPosterior` object. If you are an expert user and did so " + "for research purposes, this is fine. If not, you might be doing " + "something wrong: feel free to create an issue on Github." + ) + elif self._round > 0: + raise ValueError( + "A proposal was passed but no prior was passed at initialisation. When " + "running multi-round inference, a prior needs to be specified upon " + "initialisation. Potential fix: setting the `._prior` attribute or " + "re-initialisation. If the samples passed to `append_simulations()` " + "were sampled from the prior, single-round inference can be performed " + "with `append_simulations(..., proprosal=None)`." + ) diff --git a/sbi/inference/snpe/snpe_c.py b/sbi/inference/snpe/snpe_c.py index f09428ccb..b9cd11984 100644 --- a/sbi/inference/snpe/snpe_c.py +++ b/sbi/inference/snpe/snpe_c.py @@ -1,629 +1,625 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - - -from typing import Callable, Dict, Optional, Union - -import torch -from pyknos.mdn.mdn import MultivariateGaussianMDN as mdn -from pyknos.nflows.transforms import CompositeTransform -from torch import Tensor, eye, nn, ones -from torch.distributions import Distribution, MultivariateNormal, Uniform - -from sbi import utils as utils -from sbi.inference.posteriors.direct_posterior import DirectPosterior -from sbi.inference.snpe.snpe_base import PosteriorEstimator -from sbi.types import TensorboardSummaryWriter -from sbi.utils import ( - batched_mixture_mv, - batched_mixture_vmv, - check_dist_class, - clamp_and_warn, - del_entries, - repeat_rows, -) - - -class SNPE_C(PosteriorEstimator): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "maf", - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[TensorboardSummaryWriter] = None, - show_progress_bars: bool = True, - ): - r"""SNPE-C / APT [1]. - - [1] _Automatic Posterior Transformation for Likelihood-free Inference_, - Greenberg et al., ICML 2019, https://arxiv.org/abs/1905.07488. - - This class implements two loss variants of SNPE-C: the non-atomic and the atomic - version. The atomic loss of SNPE-C can be used for any density estimator, - i.e. also for normalizing flows. However, it suffers from leakage issues. On - the other hand, the non-atomic loss can only be used only if the proposal - distribution is a mixture of Gaussians, the density estimator is a mixture of - Gaussians, and the prior is either Gaussian or Uniform. It does not suffer from - leakage issues. At the beginning of each round, we print whether the non-atomic - or the atomic version is used. - - In this codebase, we will automatically switch to the non-atomic loss if the - following criteria are fulfilled:
- - proposal is a `DirectPosterior` with density_estimator `mdn`, as built - with `utils.sbi.posterior_nn()`.
- - the density estimator is a `mdn`, as built with - `utils.sbi.posterior_nn()`.
- - `isinstance(prior, MultivariateNormal)` (from `torch.distributions`) or - `isinstance(prior, sbi.utils.BoxUniform)` - - Note that custom implementations of any of these densities (or estimators) will - not trigger the non-atomic loss, and the algorithm will fall back onto using - the atomic loss. - - Args: - prior: A probability distribution that expresses prior knowledge about the - parameters, e.g. which ranges are meaningful for them. - density_estimator: If it is a string, use a pre-configured network of the - provided type (one of nsf, maf, mdn, made). Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. - device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". - logging_level: Minimum severity of messages to log. One of the strings - INFO, WARNING, DEBUG, ERROR and CRITICAL. - summary_writer: A tensorboard `SummaryWriter` to control, among others, log - file location (default is `/logs`.) - show_progress_bars: Whether to show a progressbar during training. - """ - - kwargs = del_entries(locals(), entries=("self", "__class__")) - super().__init__(**kwargs) - - def train( - self, - num_atoms: int = 10, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - exclude_invalid_x: bool = True, - resume_training: bool = False, - force_first_round_loss: bool = False, - discard_prior_samples: bool = False, - use_combined_loss: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - dataloader_kwargs: Optional[Dict] = None, - warn_if_zscoring: Optional[bool] = True - ) -> nn.Module: - r"""Return density estimator that approximates the distribution $p(\theta|x)$. - - Args: - num_atoms: Number of atoms to use for classification. - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - discard_prior_samples: Whether to discard samples simulated in round 1, i.e. - from the prior. Training may be sped up by ignoring such less targeted - samples. - use_combined_loss: Whether to train the neural net also on prior samples - using maximum likelihood in addition to training it on all samples using - atomic loss. The extra MLE loss helps prevent density leaking with - bounded priors. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. - show_train_summary: Whether to print the number of epochs and validation - loss and leakage after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - - # WARNING: sneaky trick ahead. We proxy the parent's `train` here, - # requiring the signature to have `num_atoms`, save it for use below, and - # continue. It's sneaky because we are using the object (self) as a namespace - # to pass arguments between functions, and that's implicit state management. - self._num_atoms = num_atoms - self._use_combined_loss = use_combined_loss - kwargs = del_entries( - locals(), entries=("self", "__class__", "num_atoms", "use_combined_loss") - ) - - self._round = max(self._data_round_index) - - if self._round > 0: - # Set the proposal to the last proposal that was passed by the user. For - # atomic SNPE, it does not matter what the proposal is. For non-atomic - # SNPE, we only use the latest data that was passed, i.e. the one from the - # last proposal. - proposal = self._proposal_roundwise[-1] - self.use_non_atomic_loss = ( - isinstance(proposal.posterior_estimator._distribution, mdn) - and isinstance(self._neural_net._distribution, mdn) - and check_dist_class( - self._prior, class_to_check=(Uniform, MultivariateNormal) - )[0] - ) - - algorithm = "non-atomic" if self.use_non_atomic_loss else "atomic" - print(f"Using SNPE-C with {algorithm} loss") - - if self.use_non_atomic_loss: - # Take care of z-scoring, pre-compute and store prior terms. - self._set_state_for_mog_proposal() - - return super().train(**kwargs) - - def _set_state_for_mog_proposal(self) -> None: - """Set state variables that are used at each training step of non-atomic SNPE-C. - - Three things are computed: - 1) Check if z-scoring was requested. To do so, we check if the `_transform` - argument of the net had been a `CompositeTransform`. See pyknos mdn.py. - 2) Define a (potentially standardized) prior. It's standardized if z-scoring - had been requested. - 3) Compute (Precision * mean) for the prior. This quantity is used at every - training step if the prior is Gaussian. - """ - - self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) - - self._set_maybe_z_scored_prior() - - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - self.prec_m_prod_prior = torch.mv( - self._maybe_z_scored_prior.precision_matrix, # type: ignore - self._maybe_z_scored_prior.loc, # type: ignore - ) - - def _set_maybe_z_scored_prior(self) -> None: - r"""Compute and store potentially standardized prior (if z-scoring was done). - - The proposal posterior is: - $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - - Let's denote z-scored theta by `a`: a = (theta - mean) / std - Then pp'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ - - The ' indicates that the evaluation occurs in standardized space. The constant - scaling factor has been absorbed into Z_2. - From the above equation, we see that we need to evaluate the prior **in - standardized space**. We build the standardized prior in this function. - - The standardize transform that is applied to the samples theta does not use - the exact prior mean and std (due to implementation issues). Hence, the z-scored - prior will not be exactly have mean=0 and std=1. - """ - - if self.z_score_theta: - scale = self._neural_net._transform._transforms[0]._scale - shift = self._neural_net._transform._transforms[0]._shift - - # Following the definintion of the linear transform in - # `standardizing_transform` in `sbiutils.py`: - # shift=-mean / std - # scale=1 / std - # Solving these equations for mean and std: - estim_prior_std = 1 / scale - estim_prior_mean = -shift * estim_prior_std - - # Compute the discrepancy of the true prior mean and std and the mean and - # std that was empirically estimated from samples. - # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) - # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean - # and std (estimated from samples and used to build standardize transform). - almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std - almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std - - if isinstance(self._prior, MultivariateNormal): - self._maybe_z_scored_prior = MultivariateNormal( - almost_zero_mean, torch.diag(almost_one_std) - ) - else: - range_ = torch.sqrt(almost_one_std * 3.0) - self._maybe_z_scored_prior = utils.BoxUniform( - almost_zero_mean - range_, almost_zero_mean + range_ - ) - else: - self._maybe_z_scored_prior = self._prior - - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: DirectPosterior, - ) -> Tensor: - """Return the log-probability of the proposal posterior. - - If the proposal is a MoG, the density estimator is a MoG, and the prior is - either Gaussian or uniform, we use non-atomic loss. Else, use atomic loss (which - suffers from leakage). - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - proposal: Proposal distribution. - - Returns: Log-probability of the proposal posterior. - """ - - if self.use_non_atomic_loss: - return self._log_prob_proposal_posterior_mog(theta, x, proposal) - else: - return self._log_prob_proposal_posterior_atomic(theta, x, masks) - - def _log_prob_proposal_posterior_atomic( - self, theta: Tensor, x: Tensor, masks: Tensor - ): - """Return log probability of the proposal posterior for atomic proposals. - - We have two main options when evaluating the proposal posterior. - (1) Generate atoms from the proposal prior. - (2) Generate atoms from a more targeted distribution, such as the most - recent posterior. - If we choose the latter, it is likely beneficial not to do this in the first - round, since we would be sampling from a randomly-initialized neural density - estimator. - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - - Returns: - Log-probability of the proposal posterior. - """ - - batch_size = theta.shape[0] - - num_atoms = int( - clamp_and_warn("num_atoms", self._num_atoms, min_val=2, max_val=batch_size) - ) - - # Each set of parameter atoms is evaluated using the same x, - # so we repeat rows of the data x, e.g. [1, 2] -> [1, 1, 2, 2] - repeated_x = repeat_rows(x, num_atoms) - - # To generate the full set of atoms for a given item in the batch, - # we sample without replacement num_atoms - 1 times from the rest - # of the theta in the batch. - probs = ones(batch_size, batch_size) * (1 - eye(batch_size)) / (batch_size - 1) - - choices = torch.multinomial(probs, num_samples=num_atoms - 1, replacement=False) - contrasting_theta = theta[choices] - - # We can now create our sets of atoms from the contrasting parameter sets - # we have generated. - atomic_theta = torch.cat((theta[:, None, :], contrasting_theta), dim=1).reshape( - batch_size * num_atoms, -1 - ) - - # Evaluate large batch giving (batch_size * num_atoms) log prob posterior evals. - log_prob_posterior = self._neural_net.log_prob(atomic_theta, repeated_x) - utils.assert_all_finite(log_prob_posterior, "posterior eval") - log_prob_posterior = log_prob_posterior.reshape(batch_size, num_atoms) - - # Get (batch_size * num_atoms) log prob prior evals. - log_prob_prior = self._prior.log_prob(atomic_theta) - log_prob_prior = log_prob_prior.reshape(batch_size, num_atoms) - utils.assert_all_finite(log_prob_prior, "prior eval") - - # Compute unnormalized proposal posterior. - unnormalized_log_prob = log_prob_posterior - log_prob_prior - - # Normalize proposal posterior across discrete set of atoms. - log_prob_proposal_posterior = unnormalized_log_prob[:, 0] - torch.logsumexp( - unnormalized_log_prob, dim=-1 - ) - utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") - - # XXX This evaluates the posterior on _all_ prior samples - if self._use_combined_loss: - log_prob_posterior_non_atomic = self._neural_net.log_prob(theta, x) - masks = masks.reshape(-1) - log_prob_proposal_posterior = ( - masks * log_prob_posterior_non_atomic + log_prob_proposal_posterior - ) - - return log_prob_proposal_posterior - - def _log_prob_proposal_posterior_mog( - self, theta: Tensor, x: Tensor, proposal: DirectPosterior - ) -> Tensor: - """Return log-probability of the proposal posterior for MoG proposal. - - For MoG proposals and MoG density estimators, this can be done in closed form - and does not require atomic loss (i.e. there will be no leakage issues). - - Notation: - - m are mean vectors. - prec are precision matrices. - cov are covariance matrices. - - _p at the end indicates that it is the proposal. - _d indicates that it is the density estimator. - _pp indicates the proposal posterior. - - All tensors will have shapes (batch_dim, num_components, ...) - - Args: - theta: Batch of parameters θ. - x: Batch of data. - proposal: Proposal distribution. - - Returns: - Log-probability of the proposal posterior. - """ - - # Evaluate the proposal. MDNs do not have functionality to run the embedding_net - # and then get the mixture_components (**without** calling log_prob()). Hence, - # we call them separately here. - encoded_x = proposal.posterior_estimator._embedding_net(proposal.default_x) - dist = ( - proposal.posterior_estimator._distribution - ) # defined to avoid ugly black formatting. - logits_p, m_p, prec_p, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_p = logits_p - torch.logsumexp(logits_p, dim=-1, keepdim=True) - - # Evaluate the density estimator. - encoded_x = self._neural_net._embedding_net(x) - dist = self._neural_net._distribution # defined to avoid black formatting. - logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) - - # z-score theta if it z-scoring had been requested. - theta = self._maybe_z_score_theta(theta) - - # Compute the MoG parameters of the proposal posterior. - logits_pp, m_pp, prec_pp, cov_pp = self._automatic_posterior_transformation( - norm_logits_p, m_p, prec_p, norm_logits_d, m_d, prec_d - ) - - # Compute the log_prob of theta under the product. - log_prob_proposal_posterior = utils.mog_log_prob( - theta, logits_pp, m_pp, prec_pp - ) - utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") - - return log_prob_proposal_posterior - - def _automatic_posterior_transformation( - self, - logits_p: Tensor, - means_p: Tensor, - precisions_p: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Returns the MoG parameters of the proposal posterior. - - The proposal posterior is: - $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - In words: proposal posterior = posterior estimate * proposal / prior. - - If the posterior estimate and the proposal are MoG and the prior is either - Gaussian or uniform, we can solve this in closed-form. The is implemented in - this function. - - This function implements Appendix A1 from Greenberg et al. 2019. - - We have to build L*K components. How do we do this? - Example: proposal has two components, density estimator has three components. - Let's call the two components of the proposal i,j and the three components - of the density estimator x,y,z. We have to multiply every component of the - proposal with every component of the density estimator. So, what we do is: - 1) for the proposal, build: i,i,i,j,j,j. Done with torch.repeat_interleave() - 2) for the density estimator, build: x,y,z,x,y,z. Done with torch.repeat() - 3) Multiply them with simple matrix operations. - - Args: - logits_p: Component weight of each Gaussian of the proposal. - means_p: Mean of each Gaussian of the proposal. - precisions_p: Precision matrix of each Gaussian of the proposal. - logits_d: Component weight for each Gaussian of the density estimator. - means_d: Mean of each Gaussian of the density estimator. - precisions_d: Precision matrix of each Gaussian of the density estimator. - - Returns: (Component weight, mean, precision matrix, covariance matrix) of each - Gaussian of the proposal posterior. Has L*K terms (proposal has L terms, - density estimator has K terms). - """ - - precisions_pp, covariances_pp = self._precisions_proposal_posterior( - precisions_p, precisions_d - ) - - means_pp = self._means_proposal_posterior( - covariances_pp, means_p, precisions_p, means_d, precisions_d - ) - - logits_pp = self._logits_proposal_posterior( - means_pp, - precisions_pp, - covariances_pp, - logits_p, - means_p, - precisions_p, - logits_d, - means_d, - precisions_d, - ) - - return logits_pp, means_pp, precisions_pp, covariances_pp - - def _precisions_proposal_posterior( - self, precisions_p: Tensor, precisions_d: Tensor - ): - """Return the precisions and covariances of the proposal posterior. - - Args: - precisions_p: Precision matrices of the proposal distribution. - precisions_d: Precision matrices of the density estimator. - - Returns: (Precisions, Covariances) of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - precisions_p_rep = precisions_p.repeat_interleave(num_comps_d, dim=1) - precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) - - precisions_pp = precisions_p_rep + precisions_d_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - precisions_pp -= self._maybe_z_scored_prior.precision_matrix - - covariances_pp = torch.inverse(precisions_pp) - - return precisions_pp, covariances_pp - - def _means_proposal_posterior( - self, - covariances_pp: Tensor, - means_p: Tensor, - precisions_p: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - """Return the means of the proposal posterior. - - means_pp = C_ix * (P_i * m_i + P_x * m_x - P_o * m_o). - - Args: - covariances_pp: Covariance matrices of the proposal posterior. - means_p: Means of the proposal distribution. - precisions_p: Precision matrices of the proposal distribution. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Means of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - # First, compute the product P_i * m_i and P_j * m_j - prec_m_prod_p = batched_mixture_mv(precisions_p, means_p) - prec_m_prod_d = batched_mixture_mv(precisions_d, means_d) - - # Repeat them to allow for matrix operations: same trick as for the precisions. - prec_m_prod_p_rep = prec_m_prod_p.repeat_interleave(num_comps_d, dim=1) - prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_p, 1) - - # Means = C_ij * (P_i * m_i + P_x * m_x - P_o * m_o). - summed_cov_m_prod_rep = prec_m_prod_p_rep + prec_m_prod_d_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - summed_cov_m_prod_rep -= self.prec_m_prod_prior - - means_pp = batched_mixture_mv(covariances_pp, summed_cov_m_prod_rep) - - return means_pp - - @staticmethod - def _logits_proposal_posterior( - means_pp: Tensor, - precisions_pp: Tensor, - covariances_pp: Tensor, - logits_p: Tensor, - means_p: Tensor, - precisions_p: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - """Return the component weights (i.e. logits) of the proposal posterior. - - Args: - means_pp: Means of the proposal posterior. - precisions_pp: Precision matrices of the proposal posterior. - covariances_pp: Covariance matrices of the proposal posterior. - logits_p: Component weights (i.e. logits) of the proposal distribution. - means_p: Means of the proposal distribution. - precisions_p: Precision matrices of the proposal distribution. - logits_d: Component weights (i.e. logits) of the density estimator. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Component weights of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute log(alpha_i * beta_j) - logits_p_rep = logits_p.repeat_interleave(num_comps_d, dim=1) - logits_d_rep = logits_d.repeat(1, num_comps_p) - logit_factors = logits_p_rep + logits_d_rep - - # Compute sqrt(det()/(det()*det())) - logdet_covariances_pp = torch.logdet(covariances_pp) - logdet_covariances_p = -torch.logdet(precisions_p) - logdet_covariances_d = -torch.logdet(precisions_d) - - # Repeat the proposal and density estimator terms such that there are LK terms. - # Same trick as has been used above. - logdet_covariances_p_rep = logdet_covariances_p.repeat_interleave( - num_comps_d, dim=1 - ) - logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_p) - - log_sqrt_det_ratio = 0.5 * ( - logdet_covariances_pp - - (logdet_covariances_p_rep + logdet_covariances_d_rep) - ) - - # Compute for proposal, density estimator, and proposal posterior: - # mu_i.T * P_i * mu_i - exponent_p = batched_mixture_vmv(precisions_p, means_p) - exponent_d = batched_mixture_vmv(precisions_d, means_d) - exponent_pp = batched_mixture_vmv(precisions_pp, means_pp) - - # Extend proposal and density estimator exponents to get LK terms. - exponent_p_rep = exponent_p.repeat_interleave(num_comps_d, dim=1) - exponent_d_rep = exponent_d.repeat(1, num_comps_p) - exponent = -0.5 * (exponent_p_rep + exponent_d_rep - exponent_pp) - - logits_pp = logit_factors + log_sqrt_det_ratio + exponent - - return logits_pp - - def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: - """Return potentially standardized theta if z-scoring was requested.""" - - if self.z_score_theta: - theta, _ = self._neural_net._transform(theta) - - return theta +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + + +from typing import Callable, Dict, Optional, Union + +import torch +from pyknos.mdn.mdn import MultivariateGaussianMDN as mdn +from pyknos.nflows.transforms import CompositeTransform +from torch import Tensor, eye, nn, ones +from torch.distributions import Distribution, MultivariateNormal, Uniform + +from sbi import utils as utils +from sbi.inference.posteriors.direct_posterior import DirectPosterior +from sbi.inference.snpe.snpe_base import PosteriorEstimator +from sbi.types import TensorboardSummaryWriter +from sbi.utils import ( + batched_mixture_mv, + batched_mixture_vmv, + check_dist_class, + clamp_and_warn, + del_entries, + repeat_rows, +) + + +class SNPE_C(PosteriorEstimator): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "maf", + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[TensorboardSummaryWriter] = None, + show_progress_bars: bool = True, + ): + r"""SNPE-C / APT [1]. + + [1] _Automatic Posterior Transformation for Likelihood-free Inference_, + Greenberg et al., ICML 2019, https://arxiv.org/abs/1905.07488. + + This class implements two loss variants of SNPE-C: the non-atomic and the atomic + version. The atomic loss of SNPE-C can be used for any density estimator, + i.e. also for normalizing flows. However, it suffers from leakage issues. On + the other hand, the non-atomic loss can only be used only if the proposal + distribution is a mixture of Gaussians, the density estimator is a mixture of + Gaussians, and the prior is either Gaussian or Uniform. It does not suffer from + leakage issues. At the beginning of each round, we print whether the non-atomic + or the atomic version is used. + + In this codebase, we will automatically switch to the non-atomic loss if the + following criteria are fulfilled:
+ - proposal is a `DirectPosterior` with density_estimator `mdn`, as built + with `utils.sbi.posterior_nn()`.
+ - the density estimator is a `mdn`, as built with + `utils.sbi.posterior_nn()`.
+ - `isinstance(prior, MultivariateNormal)` (from `torch.distributions`) or + `isinstance(prior, sbi.utils.BoxUniform)` + + Note that custom implementations of any of these densities (or estimators) will + not trigger the non-atomic loss, and the algorithm will fall back onto using + the atomic loss. + + Args: + prior: A probability distribution that expresses prior knowledge about the + parameters, e.g. which ranges are meaningful for them. + density_estimator: If it is a string, use a pre-configured network of the + provided type (one of nsf, maf, mdn, made). Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. + device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". + logging_level: Minimum severity of messages to log. One of the strings + INFO, WARNING, DEBUG, ERROR and CRITICAL. + summary_writer: A tensorboard `SummaryWriter` to control, among others, log + file location (default is `/logs`.) + show_progress_bars: Whether to show a progressbar during training. + """ + + kwargs = del_entries(locals(), entries=("self", "__class__")) + super().__init__(**kwargs) + + def train( + self, + num_atoms: int = 10, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + discard_prior_samples: bool = False, + use_combined_loss: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[Dict] = None, + ) -> nn.Module: + r"""Return density estimator that approximates the distribution $p(\theta|x)$. + + Args: + num_atoms: Number of atoms to use for classification. + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + discard_prior_samples: Whether to discard samples simulated in round 1, i.e. + from the prior. Training may be sped up by ignoring such less targeted + samples. + use_combined_loss: Whether to train the neural net also on prior samples + using maximum likelihood in addition to training it on all samples using + atomic loss. The extra MLE loss helps prevent density leaking with + bounded priors. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. + show_train_summary: Whether to print the number of epochs and validation + loss and leakage after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + + # WARNING: sneaky trick ahead. We proxy the parent's `train` here, + # requiring the signature to have `num_atoms`, save it for use below, and + # continue. It's sneaky because we are using the object (self) as a namespace + # to pass arguments between functions, and that's implicit state management. + self._num_atoms = num_atoms + self._use_combined_loss = use_combined_loss + kwargs = del_entries( + locals(), entries=("self", "__class__", "num_atoms", "use_combined_loss") + ) + + self._round = max(self._data_round_index) + + if self._round > 0: + # Set the proposal to the last proposal that was passed by the user. For + # atomic SNPE, it does not matter what the proposal is. For non-atomic + # SNPE, we only use the latest data that was passed, i.e. the one from the + # last proposal. + proposal = self._proposal_roundwise[-1] + self.use_non_atomic_loss = ( + isinstance(proposal.posterior_estimator._distribution, mdn) + and isinstance(self._neural_net._distribution, mdn) + and check_dist_class( + self._prior, class_to_check=(Uniform, MultivariateNormal) + )[0] + ) + + algorithm = "non-atomic" if self.use_non_atomic_loss else "atomic" + print(f"Using SNPE-C with {algorithm} loss") + + if self.use_non_atomic_loss: + # Take care of z-scoring, pre-compute and store prior terms. + self._set_state_for_mog_proposal() + + return super().train(**kwargs) + + def _set_state_for_mog_proposal(self) -> None: + """Set state variables that are used at each training step of non-atomic SNPE-C. + + Three things are computed: + 1) Check if z-scoring was requested. To do so, we check if the `_transform` + argument of the net had been a `CompositeTransform`. See pyknos mdn.py. + 2) Define a (potentially standardized) prior. It's standardized if z-scoring + had been requested. + 3) Compute (Precision * mean) for the prior. This quantity is used at every + training step if the prior is Gaussian. + """ + + self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) + + self._set_maybe_z_scored_prior() + + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + self.prec_m_prod_prior = torch.mv( + self._maybe_z_scored_prior.precision_matrix, # type: ignore + self._maybe_z_scored_prior.loc, # type: ignore + ) + + def _set_maybe_z_scored_prior(self) -> None: + r"""Compute and store potentially standardized prior (if z-scoring was done). + + The proposal posterior is: + $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + + Let's denote z-scored theta by `a`: a = (theta - mean) / std + Then pp'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ + + The ' indicates that the evaluation occurs in standardized space. The constant + scaling factor has been absorbed into Z_2. + From the above equation, we see that we need to evaluate the prior **in + standardized space**. We build the standardized prior in this function. + + The standardize transform that is applied to the samples theta does not use + the exact prior mean and std (due to implementation issues). Hence, the z-scored + prior will not be exactly have mean=0 and std=1. + """ + + if self.z_score_theta: + scale = self._neural_net._transform._transforms[0]._scale + shift = self._neural_net._transform._transforms[0]._shift + + # Following the definintion of the linear transform in + # `standardizing_transform` in `sbiutils.py`: + # shift=-mean / std + # scale=1 / std + # Solving these equations for mean and std: + estim_prior_std = 1 / scale + estim_prior_mean = -shift * estim_prior_std + + # Compute the discrepancy of the true prior mean and std and the mean and + # std that was empirically estimated from samples. + # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) + # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean + # and std (estimated from samples and used to build standardize transform). + almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std + almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std + + if isinstance(self._prior, MultivariateNormal): + self._maybe_z_scored_prior = MultivariateNormal( + almost_zero_mean, torch.diag(almost_one_std) + ) + else: + range_ = torch.sqrt(almost_one_std * 3.0) + self._maybe_z_scored_prior = utils.BoxUniform( + almost_zero_mean - range_, almost_zero_mean + range_ + ) + else: + self._maybe_z_scored_prior = self._prior + + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: DirectPosterior, + ) -> Tensor: + """Return the log-probability of the proposal posterior. + + If the proposal is a MoG, the density estimator is a MoG, and the prior is + either Gaussian or uniform, we use non-atomic loss. Else, use atomic loss (which + suffers from leakage). + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + proposal: Proposal distribution. + + Returns: Log-probability of the proposal posterior. + """ + + if self.use_non_atomic_loss: + return self._log_prob_proposal_posterior_mog(theta, x, proposal) + else: + return self._log_prob_proposal_posterior_atomic(theta, x, masks) + + def _log_prob_proposal_posterior_atomic( + self, theta: Tensor, x: Tensor, masks: Tensor + ): + """Return log probability of the proposal posterior for atomic proposals. + + We have two main options when evaluating the proposal posterior. + (1) Generate atoms from the proposal prior. + (2) Generate atoms from a more targeted distribution, such as the most + recent posterior. + If we choose the latter, it is likely beneficial not to do this in the first + round, since we would be sampling from a randomly-initialized neural density + estimator. + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + + Returns: + Log-probability of the proposal posterior. + """ + + batch_size = theta.shape[0] + + num_atoms = int( + clamp_and_warn("num_atoms", self._num_atoms, min_val=2, max_val=batch_size) + ) + + # Each set of parameter atoms is evaluated using the same x, + # so we repeat rows of the data x, e.g. [1, 2] -> [1, 1, 2, 2] + repeated_x = repeat_rows(x, num_atoms) + + # To generate the full set of atoms for a given item in the batch, + # we sample without replacement num_atoms - 1 times from the rest + # of the theta in the batch. + probs = ones(batch_size, batch_size) * (1 - eye(batch_size)) / (batch_size - 1) + + choices = torch.multinomial(probs, num_samples=num_atoms - 1, replacement=False) + contrasting_theta = theta[choices] + + # We can now create our sets of atoms from the contrasting parameter sets + # we have generated. + atomic_theta = torch.cat((theta[:, None, :], contrasting_theta), dim=1).reshape( + batch_size * num_atoms, -1 + ) + + # Evaluate large batch giving (batch_size * num_atoms) log prob posterior evals. + log_prob_posterior = self._neural_net.log_prob(atomic_theta, repeated_x) + utils.assert_all_finite(log_prob_posterior, "posterior eval") + log_prob_posterior = log_prob_posterior.reshape(batch_size, num_atoms) + + # Get (batch_size * num_atoms) log prob prior evals. + log_prob_prior = self._prior.log_prob(atomic_theta) + log_prob_prior = log_prob_prior.reshape(batch_size, num_atoms) + utils.assert_all_finite(log_prob_prior, "prior eval") + + # Compute unnormalized proposal posterior. + unnormalized_log_prob = log_prob_posterior - log_prob_prior + + # Normalize proposal posterior across discrete set of atoms. + log_prob_proposal_posterior = unnormalized_log_prob[:, 0] - torch.logsumexp( + unnormalized_log_prob, dim=-1 + ) + utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") + + # XXX This evaluates the posterior on _all_ prior samples + if self._use_combined_loss: + log_prob_posterior_non_atomic = self._neural_net.log_prob(theta, x) + masks = masks.reshape(-1) + log_prob_proposal_posterior = ( + masks * log_prob_posterior_non_atomic + log_prob_proposal_posterior + ) + + return log_prob_proposal_posterior + + def _log_prob_proposal_posterior_mog( + self, theta: Tensor, x: Tensor, proposal: DirectPosterior + ) -> Tensor: + """Return log-probability of the proposal posterior for MoG proposal. + + For MoG proposals and MoG density estimators, this can be done in closed form + and does not require atomic loss (i.e. there will be no leakage issues). + + Notation: + + m are mean vectors. + prec are precision matrices. + cov are covariance matrices. + + _p at the end indicates that it is the proposal. + _d indicates that it is the density estimator. + _pp indicates the proposal posterior. + + All tensors will have shapes (batch_dim, num_components, ...) + + Args: + theta: Batch of parameters θ. + x: Batch of data. + proposal: Proposal distribution. + + Returns: + Log-probability of the proposal posterior. + """ + + # Evaluate the proposal. MDNs do not have functionality to run the embedding_net + # and then get the mixture_components (**without** calling log_prob()). Hence, + # we call them separately here. + encoded_x = proposal.posterior_estimator._embedding_net(proposal.default_x) + dist = ( + proposal.posterior_estimator._distribution + ) # defined to avoid ugly black formatting. + logits_p, m_p, prec_p, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_p = logits_p - torch.logsumexp(logits_p, dim=-1, keepdim=True) + + # Evaluate the density estimator. + encoded_x = self._neural_net._embedding_net(x) + dist = self._neural_net._distribution # defined to avoid black formatting. + logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) + + # z-score theta if it z-scoring had been requested. + theta = self._maybe_z_score_theta(theta) + + # Compute the MoG parameters of the proposal posterior. + logits_pp, m_pp, prec_pp, cov_pp = self._automatic_posterior_transformation( + norm_logits_p, m_p, prec_p, norm_logits_d, m_d, prec_d + ) + + # Compute the log_prob of theta under the product. + log_prob_proposal_posterior = utils.mog_log_prob( + theta, logits_pp, m_pp, prec_pp + ) + utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") + + return log_prob_proposal_posterior + + def _automatic_posterior_transformation( + self, + logits_p: Tensor, + means_p: Tensor, + precisions_p: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Returns the MoG parameters of the proposal posterior. + + The proposal posterior is: + $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + In words: proposal posterior = posterior estimate * proposal / prior. + + If the posterior estimate and the proposal are MoG and the prior is either + Gaussian or uniform, we can solve this in closed-form. The is implemented in + this function. + + This function implements Appendix A1 from Greenberg et al. 2019. + + We have to build L*K components. How do we do this? + Example: proposal has two components, density estimator has three components. + Let's call the two components of the proposal i,j and the three components + of the density estimator x,y,z. We have to multiply every component of the + proposal with every component of the density estimator. So, what we do is: + 1) for the proposal, build: i,i,i,j,j,j. Done with torch.repeat_interleave() + 2) for the density estimator, build: x,y,z,x,y,z. Done with torch.repeat() + 3) Multiply them with simple matrix operations. + + Args: + logits_p: Component weight of each Gaussian of the proposal. + means_p: Mean of each Gaussian of the proposal. + precisions_p: Precision matrix of each Gaussian of the proposal. + logits_d: Component weight for each Gaussian of the density estimator. + means_d: Mean of each Gaussian of the density estimator. + precisions_d: Precision matrix of each Gaussian of the density estimator. + + Returns: (Component weight, mean, precision matrix, covariance matrix) of each + Gaussian of the proposal posterior. Has L*K terms (proposal has L terms, + density estimator has K terms). + """ + + precisions_pp, covariances_pp = self._precisions_proposal_posterior( + precisions_p, precisions_d + ) + + means_pp = self._means_proposal_posterior( + covariances_pp, means_p, precisions_p, means_d, precisions_d + ) + + logits_pp = self._logits_proposal_posterior( + means_pp, + precisions_pp, + covariances_pp, + logits_p, + means_p, + precisions_p, + logits_d, + means_d, + precisions_d, + ) + + return logits_pp, means_pp, precisions_pp, covariances_pp + + def _precisions_proposal_posterior( + self, precisions_p: Tensor, precisions_d: Tensor + ): + """Return the precisions and covariances of the proposal posterior. + + Args: + precisions_p: Precision matrices of the proposal distribution. + precisions_d: Precision matrices of the density estimator. + + Returns: (Precisions, Covariances) of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + precisions_p_rep = precisions_p.repeat_interleave(num_comps_d, dim=1) + precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) + + precisions_pp = precisions_p_rep + precisions_d_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + precisions_pp -= self._maybe_z_scored_prior.precision_matrix + + covariances_pp = torch.inverse(precisions_pp) + + return precisions_pp, covariances_pp + + def _means_proposal_posterior( + self, + covariances_pp: Tensor, + means_p: Tensor, + precisions_p: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + """Return the means of the proposal posterior. + + means_pp = C_ix * (P_i * m_i + P_x * m_x - P_o * m_o). + + Args: + covariances_pp: Covariance matrices of the proposal posterior. + means_p: Means of the proposal distribution. + precisions_p: Precision matrices of the proposal distribution. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Means of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + # First, compute the product P_i * m_i and P_j * m_j + prec_m_prod_p = batched_mixture_mv(precisions_p, means_p) + prec_m_prod_d = batched_mixture_mv(precisions_d, means_d) + + # Repeat them to allow for matrix operations: same trick as for the precisions. + prec_m_prod_p_rep = prec_m_prod_p.repeat_interleave(num_comps_d, dim=1) + prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_p, 1) + + # Means = C_ij * (P_i * m_i + P_x * m_x - P_o * m_o). + summed_cov_m_prod_rep = prec_m_prod_p_rep + prec_m_prod_d_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + summed_cov_m_prod_rep -= self.prec_m_prod_prior + + means_pp = batched_mixture_mv(covariances_pp, summed_cov_m_prod_rep) + + return means_pp + + @staticmethod + def _logits_proposal_posterior( + means_pp: Tensor, + precisions_pp: Tensor, + covariances_pp: Tensor, + logits_p: Tensor, + means_p: Tensor, + precisions_p: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + """Return the component weights (i.e. logits) of the proposal posterior. + + Args: + means_pp: Means of the proposal posterior. + precisions_pp: Precision matrices of the proposal posterior. + covariances_pp: Covariance matrices of the proposal posterior. + logits_p: Component weights (i.e. logits) of the proposal distribution. + means_p: Means of the proposal distribution. + precisions_p: Precision matrices of the proposal distribution. + logits_d: Component weights (i.e. logits) of the density estimator. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Component weights of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute log(alpha_i * beta_j) + logits_p_rep = logits_p.repeat_interleave(num_comps_d, dim=1) + logits_d_rep = logits_d.repeat(1, num_comps_p) + logit_factors = logits_p_rep + logits_d_rep + + # Compute sqrt(det()/(det()*det())) + logdet_covariances_pp = torch.logdet(covariances_pp) + logdet_covariances_p = -torch.logdet(precisions_p) + logdet_covariances_d = -torch.logdet(precisions_d) + + # Repeat the proposal and density estimator terms such that there are LK terms. + # Same trick as has been used above. + logdet_covariances_p_rep = logdet_covariances_p.repeat_interleave( + num_comps_d, dim=1 + ) + logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_p) + + log_sqrt_det_ratio = 0.5 * ( + logdet_covariances_pp + - (logdet_covariances_p_rep + logdet_covariances_d_rep) + ) + + # Compute for proposal, density estimator, and proposal posterior: + # mu_i.T * P_i * mu_i + exponent_p = batched_mixture_vmv(precisions_p, means_p) + exponent_d = batched_mixture_vmv(precisions_d, means_d) + exponent_pp = batched_mixture_vmv(precisions_pp, means_pp) + + # Extend proposal and density estimator exponents to get LK terms. + exponent_p_rep = exponent_p.repeat_interleave(num_comps_d, dim=1) + exponent_d_rep = exponent_d.repeat(1, num_comps_p) + exponent = -0.5 * (exponent_p_rep + exponent_d_rep - exponent_pp) + + logits_pp = logit_factors + log_sqrt_det_ratio + exponent + + return logits_pp + + def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: + """Return potentially standardized theta if z-scoring was requested.""" + + if self.z_score_theta: + theta, _ = self._neural_net._transform(theta) + + return theta diff --git a/sbi/inference/snre/snre_a.py b/sbi/inference/snre/snre_a.py index 1416abd1c..ce06a96e9 100644 --- a/sbi/inference/snre/snre_a.py +++ b/sbi/inference/snre/snre_a.py @@ -55,7 +55,6 @@ def train( stop_after_epochs: int = 20, max_num_epochs: int = 2**31 - 1, clip_max_norm: Optional[float] = 5.0, - exclude_invalid_x: bool = True, resume_training: bool = False, discard_prior_samples: bool = False, retrain_from_scratch: bool = False, @@ -75,8 +74,6 @@ def train( we train until validation loss increases (see also `stop_after_epochs`). clip_max_norm: Value at which to clip the total gradient norm in order to prevent exploding gradients. Use None for no clipping. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. resume_training: Can be used in case training time is limited, e.g. on a cluster. If `True`, the split between train and validation set, the optimizer, the number of epochs, and the best validation log-prob will diff --git a/sbi/inference/snre/snre_b.py b/sbi/inference/snre/snre_b.py index d2db9212a..bb53cd09d 100644 --- a/sbi/inference/snre/snre_b.py +++ b/sbi/inference/snre/snre_b.py @@ -56,7 +56,6 @@ def train( stop_after_epochs: int = 20, max_num_epochs: int = 2**31 - 1, clip_max_norm: Optional[float] = 5.0, - exclude_invalid_x: bool = True, resume_training: bool = False, discard_prior_samples: bool = False, retrain_from_scratch: bool = False, @@ -77,8 +76,6 @@ def train( we train until validation loss increases (see also `stop_after_epochs`). clip_max_norm: Value at which to clip the total gradient norm in order to prevent exploding gradients. Use None for no clipping. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. resume_training: Can be used in case training time is limited, e.g. on a cluster. If `True`, the split between train and validation set, the optimizer, the number of epochs, and the best validation log-prob will diff --git a/sbi/inference/snre/snre_base.py b/sbi/inference/snre/snre_base.py index 09898527e..698a57612 100644 --- a/sbi/inference/snre/snre_base.py +++ b/sbi/inference/snre/snre_base.py @@ -17,8 +17,11 @@ check_estimator_arg, check_prior, clamp_and_warn, + handle_invalid_x, validate_theta_and_x, x_shape_from_simulation, + warn_if_zscoring_changes_data, + warn_on_invalid_x, ) from sbi.utils.sbiutils import mask_sims_from_prior @@ -82,6 +85,11 @@ def append_simulations( theta: Tensor, x: Tensor, from_round: int = 0, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + warn_if_zscoring: bool = True, + return_self: bool = True, + data_device: str = None, ) -> "RatioEstimator": r"""Store parameters and simulation outputs to use them for later training. @@ -98,19 +106,48 @@ def append_simulations( With default settings, this is not used at all for `SNRE`. Only when the user later on requests `.train(discard_prior_samples=True)`, we use these indices to find which training data stemmed from the prior. - + exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` + during training. Expect errors, silent or explicit, when `False`. + warn_on_invalid: Whether to warn if data is invalid + warn_if_zscoring: Whether to test if z-scoring causes duplicates + return_self: Whether to return a instance of the class, allows chaining + with `.train()`. Setting `False` decreases memory overhead. + data_device: Where to store the data, default is on the same device where + the training is happening. If training a large dataset on a GPU with not + much VRAM can set to 'cpu' to store data on system memory instead. Returns: NeuralInference object (returned so that this function is chainable). """ theta, x = validate_theta_and_x(theta, x, training_device=self._device) - self._theta_roundwise.append(theta) - self._x_roundwise.append(x) - self._prior_masks.append(mask_sims_from_prior(int(from_round), theta.size(0))) - self._data_round_index.append(int(from_round)) + is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) + + # Check for problematic z-scoring + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_on_invalid: + warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + + x = x[is_valid_x] + theta = theta[is_valid_x] + + if data_device is None: data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) + prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - return self + if self._dataset is None: + #If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + else: + #Otherwise append to Dataset + self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) + + self._num_sims_per_round.append(theta.size(0)) + self._data_round_index.append(int(from_round) ) + + if return_self: + return self def train( self, @@ -154,15 +191,9 @@ def train( start_idx = int(discard_prior_samples and self._round > 0) # Load data from most recent round. self._round = max(self._data_round_index) - theta, x, _ = self.get_simulations( - start_idx, exclude_invalid_x, warn_on_invalid=True - ) - - # Dataset is shared for training and validation loaders. - dataset = data.TensorDataset(theta, x) train_loader, val_loader = self.get_dataloaders( - dataset, + start_idx, training_batch_size, validation_fraction, resume_training, @@ -183,11 +214,15 @@ def train( # This is passed into NeuralPosterior, to create a neural posterior which # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: + + #Get theta,x from dataset to initialize NN + test_theta = self._dataset.datasets[0].tensors[0][:100] + test_x = self._dataset.datasets[0].tensors[1][:100] + self._neural_net = self._build_neural_net( - theta[self.train_indices], x[self.train_indices] + test_theta, test_x ) - self._x_shape = x_shape_from_simulation(x) - + self._x_shape = x_shape_from_simulation(test_x) self._neural_net.to(self._device) if not resume_training: @@ -260,8 +295,8 @@ def train( self._summarize( round_=self._round, x_o=None, - theta_bank=theta, - x_bank=x, + theta_bank=None, + x_bank=None, ) # Update description for progress bar. diff --git a/sbi/utils/sbiutils.py b/sbi/utils/sbiutils.py index bde9d7375..4f26cf848 100644 --- a/sbi/utils/sbiutils.py +++ b/sbi/utils/sbiutils.py @@ -348,6 +348,29 @@ def get_simulations_since_round( [t for t, r in zip(data, data_round_indices) if r >= starting_round_index] ) +def get_simulations_indcies( + num_sims_per_round: List, data_round_indices: List, starting_round_index: int +) -> Tensor: + """ + Returns indicies for for all simulations round >= `starting_round`. Used in + `get_dataloaders` and `get_simulations` + + Args: + num_sims_per_round: Number of simulations per round + data_round_indices: List with same length as data, each entry is an integer that + indicates which round the data is from. + starting_round_index: From which round onwards to return the data. We start + counting from 0. + """ + inds = [] + for j, (n,r) in enumerate(zip(num_sims_per_round,data_round_indices)): + + #Where to start counting + s_ind = sum(num_sims_per_round[:j]) + + if r >= starting_round_index: + inds.append(torch.arange(s_ind,s_ind + n) ) + return torch.cat(inds) def mask_sims_from_prior(round_: int, num_simulations: int) -> Tensor: """Returns Tensor True where simulated from prior parameters. diff --git a/tests/linearGaussian_snpe_test.py b/tests/linearGaussian_snpe_test.py index 2d84aa759..b43bd0144 100644 --- a/tests/linearGaussian_snpe_test.py +++ b/tests/linearGaussian_snpe_test.py @@ -1,583 +1,583 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - -from __future__ import annotations - -import numpy as np -import pytest -import torch -from scipy.stats import gaussian_kde -from torch import eye, ones, zeros -from torch.distributions import MultivariateNormal - -from sbi import analysis as analysis -from sbi import utils as utils -from sbi.analysis import ConditionedMDN, conditonal_potential -from sbi.inference import ( - SNPE_A, - SNPE_B, - SNPE_C, - DirectPosterior, - MCMCPosterior, - RejectionPosterior, - posterior_estimator_based_potential, - prepare_for_sbi, - simulate_for_sbi, -) -from sbi.simulators.linear_gaussian import ( - linear_gaussian, - samples_true_posterior_linear_gaussian_mvn_prior_different_dims, - samples_true_posterior_linear_gaussian_uniform_prior, - true_posterior_linear_gaussian_mvn_prior, -) -from tests.sbiutils_test import conditional_of_mvn -from tests.test_utils import ( - check_c2st, - get_dkl_gaussian_prior, - get_normalization_uniform_prior, - get_prob_outside_uniform_prior, -) - - -@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) -@pytest.mark.parametrize( - "num_dim, prior_str, num_trials", - ( - (2, "gaussian", 1), - (2, "uniform", 1), - (1, "gaussian", 1), - # no iid x in snpe. - pytest.param(1, "gaussian", 2, marks=pytest.mark.xfail), - pytest.param(2, "gaussian", 2, marks=pytest.mark.xfail), - ), -) -def test_c2st_snpe_on_linearGaussian( - snpe_method, num_dim: int, prior_str: str, num_trials: int -): - """Test whether SNPE infers well a simple example with available ground truth.""" - - x_o = zeros(num_trials, num_dim) - num_samples = 1000 - num_simulations = 2600 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - if prior_str == "gaussian": - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - gt_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - target_samples = gt_posterior.sample((num_samples,)) - else: - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - target_samples = samples_true_posterior_linear_gaussian_uniform_prior( - x_o, likelihood_shift, likelihood_cov, prior=prior, num_samples=num_samples - ) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - - inference = snpe_method(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, num_simulations, simulation_batch_size=1000 - ) - posterior_estimator = inference.append_simulations(theta, x).train( - training_batch_size=100 - ) - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg="snpe_c") - - map_ = posterior.map(num_init_samples=1_000, show_progress_bars=False) - - # Checks for log_prob() - if prior_str == "gaussian": - # For the Gaussian prior, we compute the KLd between ground truth and posterior. - dkl = get_dkl_gaussian_prior( - posterior, x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - - max_dkl = 0.15 - - assert ( - dkl < max_dkl - ), f"D-KL={dkl} is more than 2 stds above the average performance." - - assert ((map_ - gt_posterior.mean) ** 2).sum() < 0.5 - - elif prior_str == "uniform": - # Check whether the returned probability outside of the support is zero. - posterior_prob = get_prob_outside_uniform_prior(posterior, prior, num_dim) - assert ( - posterior_prob == 0.0 - ), "The posterior probability outside of the prior support is not zero" - - # Check whether normalization (i.e. scaling up the density due - # to leakage into regions without prior support) scales up the density by the - # correct factor. - ( - posterior_likelihood_unnorm, - posterior_likelihood_norm, - acceptance_prob, - ) = get_normalization_uniform_prior(posterior, prior, x=x_o) - # The acceptance probability should be *exactly* the ratio of the unnormalized - # and the normalized likelihood. However, we allow for an error margin of 1%, - # since the estimation of the acceptance probability is random (based on - # rejection sampling). - assert ( - acceptance_prob * 0.99 - < posterior_likelihood_unnorm / posterior_likelihood_norm - < acceptance_prob * 1.01 - ), "Normalizing the posterior density using the acceptance probability failed." - - assert ((map_ - ones(num_dim)) ** 2).sum() < 0.5 - - -def test_c2st_snpe_on_linearGaussian_different_dims(): - """Test whether SNPE B/C infer well a simple example with available ground truth. - - This example has different number of parameters theta than number of x. Also - this implicitly tests simulation_batch_size=1. It also impleictly tests whether the - prior can be `None` and whether we can stop and resume training. - - """ - - theta_dim = 3 - x_dim = 2 - discard_dims = theta_dim - x_dim - - x_o = zeros(1, x_dim) - num_samples = 1000 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(x_dim) - likelihood_cov = 0.3 * eye(x_dim) - - prior_mean = zeros(theta_dim) - prior_cov = eye(theta_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - target_samples = samples_true_posterior_linear_gaussian_mvn_prior_different_dims( - x_o, - likelihood_shift, - likelihood_cov, - prior_mean, - prior_cov, - num_discarded_dims=discard_dims, - num_samples=num_samples, - ) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian( - theta, likelihood_shift, likelihood_cov, num_discarded_dims=discard_dims - ), - prior, - ) - # Test whether prior can be `None`. - inference = SNPE_C(prior=None, density_estimator="maf", show_progress_bars=False) - - # type: ignore - theta, x = simulate_for_sbi(simulator, prior, 2000, simulation_batch_size=1) - - inference = inference.append_simulations(theta, x) - posterior_estimator = inference.train( - max_num_epochs=10 - ) # Test whether we can stop and resume. - posterior_estimator = inference.train( - resume_training=True, force_first_round_loss=True - ) - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg="snpe_c") - - -# Test multi-round SNPE. -@pytest.mark.slow -@pytest.mark.parametrize( - "method_str", - ( - "snpe_a", - pytest.param( - "snpe_b", - marks=pytest.mark.xfail( - raises=NotImplementedError, reason="""SNPE-B not implemented""" - ), - ), - "snpe_c", - "snpe_c_non_atomic", - ), -) -def test_c2st_multi_round_snpe_on_linearGaussian(method_str: str): - """Test whether SNPE B/C infer well a simple example with available ground truth. - . - """ - - num_dim = 2 - x_o = zeros((1, num_dim)) - num_samples = 1000 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - gt_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - target_samples = gt_posterior.sample((num_samples,)) - - if method_str == "snpe_c_non_atomic": - # Test whether SNPE works properly with structured z-scoring. - density_estimator = utils.posterior_nn( - "mdn", z_score_x="structured", num_components=5 - ) - method_str = "snpe_c" - elif method_str == "snpe_a": - density_estimator = "mdn_snpe_a" - else: - density_estimator = "maf" - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - creation_args = dict( - prior=prior, - density_estimator=density_estimator, - show_progress_bars=False, - ) - - if method_str == "snpe_b": - inference = SNPE_B(**creation_args) - theta, x = simulate_for_sbi(simulator, prior, 500, simulation_batch_size=10) - posterior_estimator = inference.append_simulations(theta, x).train() - posterior1 = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - theta, x = simulate_for_sbi( - simulator, posterior1, 1000, simulation_batch_size=10 - ) - posterior_estimator = inference.append_simulations( - theta, x, proposal=posterior1 - ).train() - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - elif method_str == "snpe_c": - inference = SNPE_C(**creation_args) - theta, x = simulate_for_sbi(simulator, prior, 900, simulation_batch_size=50) - posterior_estimator = inference.append_simulations(theta, x).train() - posterior1 = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - theta = posterior1.sample((1000,)) - x = simulator(theta) - _ = inference.append_simulations(theta, x, proposal=posterior1).train() - posterior = inference.build_posterior().set_default_x(x_o) - elif method_str == "snpe_a": - inference = SNPE_A(**creation_args) - proposal = prior - final_round = False - num_rounds = 3 - for r in range(num_rounds): - if r == 2: - final_round = True - theta, x = simulate_for_sbi( - simulator, proposal, 500, simulation_batch_size=50 - ) - inference = inference.append_simulations(theta, x, proposal=proposal) - _ = inference.train(max_num_epochs=200, final_round=final_round) - posterior = inference.build_posterior().set_default_x(x_o) - proposal = posterior - - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg=method_str) - - -# Testing rejection and mcmc sampling methods. -@pytest.mark.slow -@pytest.mark.parametrize( - "sample_with, mcmc_method, prior_str", - ( - ("mcmc", "slice_np", "gaussian"), - ("mcmc", "slice", "gaussian"), - # XXX (True, "slice", "uniform"), - # XXX takes very long. fix when refactoring pyro sampling - ("rejection", "rejection", "uniform"), - ), -) -def test_api_snpe_c_posterior_correction(sample_with, mcmc_method, prior_str): - """Test that leakage correction applied to sampling works, with both MCMC and - rejection. - - """ - - num_dim = 2 - x_o = zeros(1, num_dim) - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - if prior_str == "gaussian": - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - else: - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - inference = SNPE_C(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi(simulator, prior, 1000) - posterior_estimator = inference.append_simulations(theta, x).train() - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator, prior, x_o - ) - if sample_with == "mcmc": - posterior = MCMCPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - proposal=prior, - method=mcmc_method, - ) - elif sample_with == "rejection": - posterior = RejectionPosterior( - potential_fn=potential_fn, - proposal=prior, - theta_transform=theta_transform, - ) - - # Posterior should be corrected for leakage even if num_rounds just 1. - samples = posterior.sample((10,)) - - # Evaluate the samples to check correction factor. - _ = posterior.log_prob(samples) - - -@pytest.mark.slow -def test_sample_conditional(): - """ - Test whether sampling from the conditional gives the same results as evaluating. - - This compares samples that get smoothed with a Gaussian kde to evaluating the - conditional log-probability with `eval_conditional_density`. - - `eval_conditional_density` is itself tested in `sbiutils_test.py`. Here, we use - a bimodal posterior to test the conditional. - """ - - num_dim = 3 - dim_to_sample_1 = 0 - dim_to_sample_2 = 2 - - x_o = zeros(1, num_dim) - - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.1 * eye(num_dim) - - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - - def simulator(theta): - if torch.rand(1) > 0.5: - return linear_gaussian(theta, likelihood_shift, likelihood_cov) - else: - return linear_gaussian(theta, -likelihood_shift, likelihood_cov) - - # Test whether SNPE works properly with structured z-scoring. - net = utils.posterior_nn("maf", z_score_x="structured", hidden_features=20) - - simulator, prior = prepare_for_sbi(simulator, prior) - - inference = SNPE_C(prior, density_estimator=net, show_progress_bars=False) - - # We need a pretty big dataset to properly model the bimodality. - theta, x = simulate_for_sbi(simulator, prior, 10000) - posterior_estimator = inference.append_simulations(theta, x).train( - max_num_epochs=60 - ) - - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((50,)) - - # Evaluate the conditional density be drawing samples and smoothing with a Gaussian - # kde. - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator, prior=prior, x_o=x_o - ) - (conditioned_potential_fn, restricted_tf, restricted_prior,) = conditonal_potential( - potential_fn=potential_fn, - theta_transform=theta_transform, - prior=prior, - condition=samples[0], - dims_to_sample=[dim_to_sample_1, dim_to_sample_2], - ) - mcmc_posterior = MCMCPosterior( - potential_fn=conditioned_potential_fn, - theta_transform=restricted_tf, - proposal=restricted_prior, - ) - cond_samples = mcmc_posterior.sample((500,)) - - _ = analysis.pairplot( - cond_samples, - limits=[[-2, 2], [-2, 2], [-2, 2]], - figsize=(2, 2), - diag="kde", - upper="kde", - ) - - limits = [[-2, 2], [-2, 2], [-2, 2]] - - density = gaussian_kde(cond_samples.numpy().T, bw_method="scott") - - X, Y = np.meshgrid( - np.linspace(limits[0][0], limits[0][1], 50), - np.linspace(limits[1][0], limits[1][1], 50), - ) - positions = np.vstack([X.ravel(), Y.ravel()]) - sample_kde_grid = np.reshape(density(positions).T, X.shape) - - # Evaluate the conditional with eval_conditional_density. - eval_grid = analysis.eval_conditional_density( - posterior, - condition=samples[0], - dim1=dim_to_sample_1, - dim2=dim_to_sample_2, - limits=torch.tensor([[-2, 2], [-2, 2], [-2, 2]]), - ) - - # Compare the two densities. - sample_kde_grid = sample_kde_grid / np.sum(sample_kde_grid) - eval_grid = eval_grid / torch.sum(eval_grid) - - error = np.abs(sample_kde_grid - eval_grid.numpy()) - - max_err = np.max(error) - assert max_err < 0.0027 - - -def test_mdn_conditional_density(num_dim: int = 3, cond_dim: int = 1): - """Test whether the conditional density infered from MDN parameters of a - `DirectPosterior` matches analytical results for MVN. This uses a n-D joint and - conditions on the last m values to generate a conditional. - - Gaussian prior used for easier ground truthing of conditional posterior. - - Args: - num_dim: Dimensionality of the MVM. - cond_dim: Dimensionality of the condition. - """ - - assert ( - num_dim > cond_dim - ), "The number of dimensions needs to be greater than that of the condition!" - - x_o = zeros(1, num_dim) - num_samples = 1000 - num_simulations = 2700 - condition = 0.1 * ones(1, num_dim) - - dims = list(range(num_dim)) - dims2sample = dims[-cond_dim:] - dims2condition = dims[:-cond_dim] - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - joint_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - joint_cov = joint_posterior.covariance_matrix - joint_mean = joint_posterior.loc - - conditional_mean, conditional_cov = conditional_of_mvn( - joint_mean, joint_cov, condition[0, dims2condition] - ) - conditional_dist_gt = MultivariateNormal(conditional_mean, conditional_cov) - - conditional_samples_gt = conditional_dist_gt.sample((num_samples,)) - - def simulator(theta): - return linear_gaussian(theta, likelihood_shift, likelihood_cov) - - simulator, prior = prepare_for_sbi(simulator, prior) - inference = SNPE_C(density_estimator="mdn", show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, num_simulations, simulation_batch_size=1000 - ) - posterior_mdn = inference.append_simulations(theta, x).train( - training_batch_size=100 - ) - conditioned_mdn = ConditionedMDN( - posterior_mdn, x_o, condition=condition, dims_to_sample=[0] - ) - conditional_samples_sbi = conditioned_mdn.sample((num_samples,)) - check_c2st( - conditional_samples_sbi, - conditional_samples_gt, - alg="analytic_mdn_conditioning_of_direct_posterior", - ) - - -@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) -def test_example_posterior(snpe_method: type): - """Return an inferred `NeuralPosterior` for interactive examination.""" - num_dim = 2 - x_o = zeros(1, num_dim) - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - if snpe_method == SNPE_A: - extra_kwargs = dict(final_round=True) - else: - extra_kwargs = dict() - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - inference = snpe_method(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, 1000, simulation_batch_size=10, num_workers=6 - ) - posterior_estimator = inference.append_simulations(theta, x).train(**extra_kwargs) - if snpe_method == SNPE_A: - posterior_estimator = inference.correct_for_proposal() - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - assert posterior is not None +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + +from __future__ import annotations + +import numpy as np +import pytest +import torch +from scipy.stats import gaussian_kde +from torch import eye, ones, zeros +from torch.distributions import MultivariateNormal + +from sbi import analysis as analysis +from sbi import utils as utils +from sbi.analysis import ConditionedMDN, conditonal_potential +from sbi.inference import ( + SNPE_A, + SNPE_B, + SNPE_C, + DirectPosterior, + MCMCPosterior, + RejectionPosterior, + posterior_estimator_based_potential, + prepare_for_sbi, + simulate_for_sbi, +) +from sbi.simulators.linear_gaussian import ( + linear_gaussian, + samples_true_posterior_linear_gaussian_mvn_prior_different_dims, + samples_true_posterior_linear_gaussian_uniform_prior, + true_posterior_linear_gaussian_mvn_prior, +) +from tests.sbiutils_test import conditional_of_mvn +from tests.test_utils import ( + check_c2st, + get_dkl_gaussian_prior, + get_normalization_uniform_prior, + get_prob_outside_uniform_prior, +) + + +@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) +@pytest.mark.parametrize( + "num_dim, prior_str, num_trials", + ( + (2, "gaussian", 1), + (2, "uniform", 1), + (1, "gaussian", 1), + # no iid x in snpe. + pytest.param(1, "gaussian", 2, marks=pytest.mark.xfail), + pytest.param(2, "gaussian", 2, marks=pytest.mark.xfail), + ), +) +def test_c2st_snpe_on_linearGaussian( + snpe_method, num_dim: int, prior_str: str, num_trials: int +): + """Test whether SNPE infers well a simple example with available ground truth.""" + + x_o = zeros(num_trials, num_dim) + num_samples = 1000 + num_simulations = 2600 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + if prior_str == "gaussian": + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + gt_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + target_samples = gt_posterior.sample((num_samples,)) + else: + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + target_samples = samples_true_posterior_linear_gaussian_uniform_prior( + x_o, likelihood_shift, likelihood_cov, prior=prior, num_samples=num_samples + ) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + + inference = snpe_method(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, num_simulations, simulation_batch_size=1000 + ) + posterior_estimator = inference.append_simulations(theta, x).train( + training_batch_size=100 + ) + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg="snpe_c") + + map_ = posterior.map(num_init_samples=1_000, show_progress_bars=False) + + # Checks for log_prob() + if prior_str == "gaussian": + # For the Gaussian prior, we compute the KLd between ground truth and posterior. + dkl = get_dkl_gaussian_prior( + posterior, x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + + max_dkl = 0.15 + + assert ( + dkl < max_dkl + ), f"D-KL={dkl} is more than 2 stds above the average performance." + + assert ((map_ - gt_posterior.mean) ** 2).sum() < 0.5 + + elif prior_str == "uniform": + # Check whether the returned probability outside of the support is zero. + posterior_prob = get_prob_outside_uniform_prior(posterior, prior, num_dim) + assert ( + posterior_prob == 0.0 + ), "The posterior probability outside of the prior support is not zero" + + # Check whether normalization (i.e. scaling up the density due + # to leakage into regions without prior support) scales up the density by the + # correct factor. + ( + posterior_likelihood_unnorm, + posterior_likelihood_norm, + acceptance_prob, + ) = get_normalization_uniform_prior(posterior, prior, x=x_o) + # The acceptance probability should be *exactly* the ratio of the unnormalized + # and the normalized likelihood. However, we allow for an error margin of 1%, + # since the estimation of the acceptance probability is random (based on + # rejection sampling). + assert ( + acceptance_prob * 0.99 + < posterior_likelihood_unnorm / posterior_likelihood_norm + < acceptance_prob * 1.01 + ), "Normalizing the posterior density using the acceptance probability failed." + + assert ((map_ - ones(num_dim)) ** 2).sum() < 0.5 + + +def test_c2st_snpe_on_linearGaussian_different_dims(): + """Test whether SNPE B/C infer well a simple example with available ground truth. + + This example has different number of parameters theta than number of x. Also + this implicitly tests simulation_batch_size=1. It also impleictly tests whether the + prior can be `None` and whether we can stop and resume training. + + """ + + theta_dim = 3 + x_dim = 2 + discard_dims = theta_dim - x_dim + + x_o = zeros(1, x_dim) + num_samples = 1000 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(x_dim) + likelihood_cov = 0.3 * eye(x_dim) + + prior_mean = zeros(theta_dim) + prior_cov = eye(theta_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + target_samples = samples_true_posterior_linear_gaussian_mvn_prior_different_dims( + x_o, + likelihood_shift, + likelihood_cov, + prior_mean, + prior_cov, + num_discarded_dims=discard_dims, + num_samples=num_samples, + ) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian( + theta, likelihood_shift, likelihood_cov, num_discarded_dims=discard_dims + ), + prior, + ) + # Test whether prior can be `None`. + inference = SNPE_C(prior=None, density_estimator="maf", show_progress_bars=False) + + # type: ignore + theta, x = simulate_for_sbi(simulator, prior, 2000, simulation_batch_size=1) + + inference = inference.append_simulations(theta, x) + posterior_estimator = inference.train( + max_num_epochs=10 + ) # Test whether we can stop and resume. + posterior_estimator = inference.train( + resume_training=True, force_first_round_loss=True + ) + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg="snpe_c") + + +# Test multi-round SNPE. +@pytest.mark.slow +@pytest.mark.parametrize( + "method_str", + ( + "snpe_a", + pytest.param( + "snpe_b", + marks=pytest.mark.xfail( + raises=NotImplementedError, reason="""SNPE-B not implemented""" + ), + ), + "snpe_c", + "snpe_c_non_atomic", + ), +) +def test_c2st_multi_round_snpe_on_linearGaussian(method_str: str): + """Test whether SNPE B/C infer well a simple example with available ground truth. + . + """ + + num_dim = 2 + x_o = zeros((1, num_dim)) + num_samples = 1000 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + gt_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + target_samples = gt_posterior.sample((num_samples,)) + + if method_str == "snpe_c_non_atomic": + # Test whether SNPE works properly with structured z-scoring. + density_estimator = utils.posterior_nn( + "mdn", z_score_x="structured", num_components=5 + ) + method_str = "snpe_c" + elif method_str == "snpe_a": + density_estimator = "mdn_snpe_a" + else: + density_estimator = "maf" + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + creation_args = dict( + prior=prior, + density_estimator=density_estimator, + show_progress_bars=False, + ) + + if method_str == "snpe_b": + inference = SNPE_B(**creation_args) + theta, x = simulate_for_sbi(simulator, prior, 500, simulation_batch_size=10) + posterior_estimator = inference.append_simulations(theta, x).train() + posterior1 = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + theta, x = simulate_for_sbi( + simulator, posterior1, 1000, simulation_batch_size=10 + ) + posterior_estimator = inference.append_simulations( + theta, x, proposal=posterior1 + ).train() + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + elif method_str == "snpe_c": + inference = SNPE_C(**creation_args) + theta, x = simulate_for_sbi(simulator, prior, 900, simulation_batch_size=50) + posterior_estimator = inference.append_simulations(theta, x).train() + posterior1 = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + theta = posterior1.sample((1000,)) + x = simulator(theta) + _ = inference.append_simulations(theta, x, proposal=posterior1).train() + posterior = inference.build_posterior().set_default_x(x_o) + elif method_str == "snpe_a": + inference = SNPE_A(**creation_args) + proposal = prior + final_round = False + num_rounds = 3 + for r in range(num_rounds): + if r == 2: + final_round = True + theta, x = simulate_for_sbi( + simulator, proposal, 500, simulation_batch_size=50 + ) + inference = inference.append_simulations(theta, x, proposal=proposal) + _ = inference.train(max_num_epochs=200, final_round=final_round) + posterior = inference.build_posterior().set_default_x(x_o) + proposal = posterior + + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg=method_str) + + +# Testing rejection and mcmc sampling methods. +@pytest.mark.slow +@pytest.mark.parametrize( + "sample_with, mcmc_method, prior_str", + ( + ("mcmc", "slice_np", "gaussian"), + ("mcmc", "slice", "gaussian"), + # XXX (True, "slice", "uniform"), + # XXX takes very long. fix when refactoring pyro sampling + ("rejection", "rejection", "uniform"), + ), +) +def test_api_snpe_c_posterior_correction(sample_with, mcmc_method, prior_str): + """Test that leakage correction applied to sampling works, with both MCMC and + rejection. + + """ + + num_dim = 2 + x_o = zeros(1, num_dim) + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + if prior_str == "gaussian": + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + else: + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + inference = SNPE_C(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi(simulator, prior, 1000) + posterior_estimator = inference.append_simulations(theta, x).train() + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator, prior, x_o + ) + if sample_with == "mcmc": + posterior = MCMCPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + proposal=prior, + method=mcmc_method, + ) + elif sample_with == "rejection": + posterior = RejectionPosterior( + potential_fn=potential_fn, + proposal=prior, + theta_transform=theta_transform, + ) + + # Posterior should be corrected for leakage even if num_rounds just 1. + samples = posterior.sample((10,)) + + # Evaluate the samples to check correction factor. + _ = posterior.log_prob(samples) + + +@pytest.mark.slow +def test_sample_conditional(): + """ + Test whether sampling from the conditional gives the same results as evaluating. + + This compares samples that get smoothed with a Gaussian kde to evaluating the + conditional log-probability with `eval_conditional_density`. + + `eval_conditional_density` is itself tested in `sbiutils_test.py`. Here, we use + a bimodal posterior to test the conditional. + """ + + num_dim = 3 + dim_to_sample_1 = 0 + dim_to_sample_2 = 2 + + x_o = zeros(1, num_dim) + + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.1 * eye(num_dim) + + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + + def simulator(theta): + if torch.rand(1) > 0.5: + return linear_gaussian(theta, likelihood_shift, likelihood_cov) + else: + return linear_gaussian(theta, -likelihood_shift, likelihood_cov) + + # Test whether SNPE works properly with structured z-scoring. + net = utils.posterior_nn("maf", z_score_x="structured", hidden_features=20) + + simulator, prior = prepare_for_sbi(simulator, prior) + + inference = SNPE_C(prior, density_estimator=net, show_progress_bars=False) + + # We need a pretty big dataset to properly model the bimodality. + theta, x = simulate_for_sbi(simulator, prior, 10000) + posterior_estimator = inference.append_simulations(theta, x).train( + max_num_epochs=60 + ) + + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((50,)) + + # Evaluate the conditional density be drawing samples and smoothing with a Gaussian + # kde. + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator, prior=prior, x_o=x_o + ) + (conditioned_potential_fn, restricted_tf, restricted_prior,) = conditonal_potential( + potential_fn=potential_fn, + theta_transform=theta_transform, + prior=prior, + condition=samples[0], + dims_to_sample=[dim_to_sample_1, dim_to_sample_2], + ) + mcmc_posterior = MCMCPosterior( + potential_fn=conditioned_potential_fn, + theta_transform=restricted_tf, + proposal=restricted_prior, + ) + cond_samples = mcmc_posterior.sample((500,)) + + _ = analysis.pairplot( + cond_samples, + limits=[[-2, 2], [-2, 2], [-2, 2]], + figsize=(2, 2), + diag="kde", + upper="kde", + ) + + limits = [[-2, 2], [-2, 2], [-2, 2]] + + density = gaussian_kde(cond_samples.numpy().T, bw_method="scott") + + X, Y = np.meshgrid( + np.linspace(limits[0][0], limits[0][1], 50), + np.linspace(limits[1][0], limits[1][1], 50), + ) + positions = np.vstack([X.ravel(), Y.ravel()]) + sample_kde_grid = np.reshape(density(positions).T, X.shape) + + # Evaluate the conditional with eval_conditional_density. + eval_grid = analysis.eval_conditional_density( + posterior, + condition=samples[0], + dim1=dim_to_sample_1, + dim2=dim_to_sample_2, + limits=torch.tensor([[-2, 2], [-2, 2], [-2, 2]]), + ) + + # Compare the two densities. + sample_kde_grid = sample_kde_grid / np.sum(sample_kde_grid) + eval_grid = eval_grid / torch.sum(eval_grid) + + error = np.abs(sample_kde_grid - eval_grid.numpy()) + + max_err = np.max(error) + assert max_err < 0.0027 + + +def test_mdn_conditional_density(num_dim: int = 3, cond_dim: int = 1): + """Test whether the conditional density infered from MDN parameters of a + `DirectPosterior` matches analytical results for MVN. This uses a n-D joint and + conditions on the last m values to generate a conditional. + + Gaussian prior used for easier ground truthing of conditional posterior. + + Args: + num_dim: Dimensionality of the MVM. + cond_dim: Dimensionality of the condition. + """ + + assert ( + num_dim > cond_dim + ), "The number of dimensions needs to be greater than that of the condition!" + + x_o = zeros(1, num_dim) + num_samples = 1000 + num_simulations = 2700 + condition = 0.1 * ones(1, num_dim) + + dims = list(range(num_dim)) + dims2sample = dims[-cond_dim:] + dims2condition = dims[:-cond_dim] + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + joint_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + joint_cov = joint_posterior.covariance_matrix + joint_mean = joint_posterior.loc + + conditional_mean, conditional_cov = conditional_of_mvn( + joint_mean, joint_cov, condition[0, dims2condition] + ) + conditional_dist_gt = MultivariateNormal(conditional_mean, conditional_cov) + + conditional_samples_gt = conditional_dist_gt.sample((num_samples,)) + + def simulator(theta): + return linear_gaussian(theta, likelihood_shift, likelihood_cov) + + simulator, prior = prepare_for_sbi(simulator, prior) + inference = SNPE_C(density_estimator="mdn", show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, num_simulations, simulation_batch_size=1000 + ) + posterior_mdn = inference.append_simulations(theta, x).train( + training_batch_size=100 + ) + conditioned_mdn = ConditionedMDN( + posterior_mdn, x_o, condition=condition, dims_to_sample=[0] + ) + conditional_samples_sbi = conditioned_mdn.sample((num_samples,)) + check_c2st( + conditional_samples_sbi, + conditional_samples_gt, + alg="analytic_mdn_conditioning_of_direct_posterior", + ) + + +@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) +def test_example_posterior(snpe_method: type): + """Return an inferred `NeuralPosterior` for interactive examination.""" + num_dim = 2 + x_o = zeros(1, num_dim) + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + if snpe_method == SNPE_A: + extra_kwargs = dict(final_round=True) + else: + extra_kwargs = dict() + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + inference = snpe_method(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, 1000, simulation_batch_size=10, num_workers=6 + ) + posterior_estimator = inference.append_simulations(theta, x).train(**extra_kwargs) + if snpe_method == SNPE_A: + posterior_estimator = inference.correct_for_proposal() + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + assert posterior is not None diff --git a/tutorials/07_conditional_distributions.ipynb b/tutorials/07_conditional_distributions.ipynb index f84014a88..2e46f8cd0 100644 --- a/tutorials/07_conditional_distributions.ipynb +++ b/tutorials/07_conditional_distributions.ipynb @@ -1,16588 +1,16588 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Analysing variability and compensation mechansims with conditional distributions\n", - "\n", - "A central advantage of `sbi` over parameter search methods such as genetic algorithms is that the posterior captures **all** models that can reproduce experimental data. This allows us to analyse whether parameters can be variable or have to be narrowly tuned, and to analyse compensation mechanisms between different parameters. See also [Marder and Taylor, 2011](https://www.nature.com/articles/nn.2735?page=2) for further motivation to identify all models that capture experimental data. \n", - "\n", - "In this tutorial, we will show how one can use the posterior distribution to identify whether parameters can be variable or have to be finely tuned, and how we can use the posterior to find potential compensation mechanisms between model parameters. To investigate this, we will extract **conditional distributions** from the posterior inferred with `sbi`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note, you can find the original version of this notebook at [https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb](https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb) in the `sbi` repository." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main syntax" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sbi.analysis import conditional_pairplot, conditional_corrcoeff\n", - "\n", - "# Plot slices through posterior, i.e. conditionals.\n", - "_ = conditional_pairplot(\n", - " density=posterior,\n", - " condition=posterior.sample((1,)),\n", - " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", - ")\n", - "\n", - "# Compute the matrix of correlation coefficients of the slices.\n", - "cond_coeff_mat = conditional_corrcoeff(\n", - " density=posterior,\n", - " condition=posterior.sample((1,)),\n", - " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", - ")\n", - "plt.imshow(cond_coeff_mat, clim=[-1, 1])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analysing variability and compensation mechanisms in a toy example\n", - "Below, we use a simple toy example to demonstrate the above described features. For an application of these features to a neuroscience problem, see figure 6 in [Gonçalves, Lueckmann, Deistler et al., 2019](https://arxiv.org/abs/1907.00770)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from sbi import utils as utils\n", - "from sbi.analysis import pairplot, conditional_pairplot, conditional_corrcoeff\n", - "import torch\n", - "import numpy as np\n", - "\n", - "import matplotlib.pyplot as plt\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "from matplotlib import animation, rc\n", - "from IPython.display import HTML, Image\n", - "\n", - "_ = torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's say we have used SNPE to obtain a posterior distribution over three parameters. In this tutorial, we just load the posterior from a file:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from toy_posterior_for_07_cc import ExamplePosterior\n", - "posterior = ExamplePosterior()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we specify the experimental observation $x_o$ at which we want to evaluate and sample the posterior $p(\\theta|x_o)$:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "x_o = torch.ones(1, 20) # simulator output was 20-dimensional\n", - "posterior.set_default_x(x_o)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, we can inspect the posterior marginals with the `pairplot()` function:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAACaY0lEQVR4nOz9eZxk6VXfCX/P8zz3xpJZVa3eW2q11m6phQDtrELDgAxYzHhszD4sBmPw6xljjD3GxvYwHuN5PV7GC/BigxnMYoTB7GB4sREWHiEktO9Sa+vWvnR3VWVGxL33WeaP8zz33ojK6s6uyqqs7I7Tn/hEdlZmxM2IG797zu/8zu9ISoltbGMb2zgpYY77ALaxjW1s45HEFrS2sY1tnKjYgtY2trGNExVb0NrGNrZxomILWtvYxjZOVGxBaxvb2MaJCncJv7PVSBxNyOU+wPP//D9NsQI/FcIUooM4SSSrXyebSAaSBUzSZxRIMnoLL3YU+UckyfCOJ71JlHwP0gmSwHQgXjA+fx3AtiA+Ybx+bXzCNgnbJsQn3DJguohZdUgTkM4jTQs+kLoOQoAQSd5DjBACKUQ9lBDyMcV8f3mn5e/GX7jk9+Ol5qsf+sklP7QYxIjeWwPGIM6BtWBHX1cOjCFVjlQ5cIZYW1JliZUhTC3Jgp8ZohNCJYSJvs+hFpKDaCE5SALJMHrvyzGN/l+S/pwFTP6dcu64pN8zeq9vdnmM/GeXc6ScF14g6rkgQfJ5oueN8SAeTADToOfGKuFWCdvBH778ex/2fbgU0NrGNRJ+KsQa/AzCLBFrCJNEcolUJbB6ExsRk5B80gkgkvqTTjZOk/7zn4SUz/KU/x8UJ1IUYhTwBoIoeHnBdDIAWAsmA5lt9GS1jWBbPUGTFYxPOCeYKiCNxRgBH/RD3XYkGxVXUyShn7WUEoLV70WjByRy2cB1ReJigGUt0gOURZwF58DZDFQKUql2RGeI0wxYtSFMheiEbibECmKtoBUdxJp80UoboKUXrdQDzgBYCAMwCXreOD1njEuIJMRErNWvjSnnTSIlISWI0RCjEIMhBiEFQ/D5/IggnUFCwnhR0PJgKunPAwpoHiIeVaDlQ+Te+xc8+YYdPfkf5REnEGoFLD+DWCfSLIBLmDpgXcDahHMBIwlrItboiWdNxGQAAzCy/oGPSfTiWYAqiX4vCSHm+yR0nSNGwXeW2BliZ5BWgcw2+QTtREErCHaVs64WojOYLpGsw1QGW1mwgnRBsxBrkc6DCClEREzOvvSWkiAEBS49avLBXpXX/2FjE7CsRSTfVzmrck4zrAJW1pImTgGrssSJJTpDmGbAqgWfwcrPhFBDrPJFy+VzwGmGlFwaMqMenEYXqvw9MQkRMCYDlUlUlcdKonIBayKVif054yT2f6JPhpiEEA1tsIRoWHWOEAy+s4RgSEFInd7HIEhn9EJmIXVCEs3Ckjnc+/aoAa1//9r7+JeveA/33b/k1tNTvunznsRffMnTHtXgFV2+VflknURkEjBVpKoCVeVxJjIpJ6ANOBMxDKBlJGFImNGJCBDzJTmiYFVuACEZfDR6otaWLhiarqLrLMFbgrPghWAN0mm5kqyCl5args1XVWu1vLQiYASJCTGCycCjWVZCJJByKdhnWzGSxCAmkuI1AlQlDsqwFBn0Zm3OsBSwSlmYrB0AyxlCZTTDmihghVrwU4iV6IWq0uw6TBWo4jTmTCkhrmTYObs2cTgsUaAqmZNIwpmIswpOtVWwmlhPbQNOArUN+VwZXmufz4U2Otpg6aLF2YrWW1rr6DpLDIZgEikYktdzIFotIZMkxCsID7XrQ8ejArR+5tUf5G//ylv57Cdex5/7/Kfw++/+JP/od97FPZ/Y4x9+1WdRu0dnvyE6SPmkTdOATAOTeUdVeeZ1x6zqqE1gp2pwJlKbwMR4jKR8H6lEgQzAEvsTsgBUl3P2mISAXlV90pPTJ8MyVLTBst9NWPqKlXcs2wrvLW3j1rIv8YJdCaEF0wqu1lIyVgY7SbhGiFawrcU6g6kd0npkZREftJzynuQ9YnL2RacZlyRS6F8ZvTvujEvMcF8yrVISlgyrrkguA1atGVaYWFIpBTNv1c2NZtUTwe9AqMDvJGKdlMecRqSKVBOPdRHnApUNWJNwNvRZdXmvpVywJGFF3/faBJzR82FmO2rjmeRbJYHKKGhZiYR8UWuio0uWZahZhoplqDjnpnTBst/VtLXFB0PrNfsKXm/RG4K1pEYfR4JgD/kxPfGg9er3fZof+LW38cXPuIkf/5YXYo3w577gyfzI77+Xf/Q772Kv8fzINz6P6rCvyEkKGRGkNiFW0/qJC0ydZ6dqqY1nt2qoJF818wk4MR6bQauSgJGIHWVcJdMKCDEZQjJEhJAMXbJ0yeKjYWlrmuA0o3Oepa8wkmitRSThrSU4QzSW5A1gSEZINmlJYHNpkIlhCQYMSFSwNCKKQSLIKPtSXgtStEgIJKNNg7WM67h4rhFJKCZnkH2WJUOGZYwCllvPrgpgxcoQKiFWWgaGzF2Fmh6s4kQzbDPzWBeYTjsqG6hdYGJLaZcz7FFWXS5OpdwzkqhMyOdHZGZbKgnM8/34PIHh/FglRxOr/t+MJHwytOKICEYSncnnkjF4Y0AsYiB4Q4qJ5HLzIHKoONGgtWg9f/Xn38gdN8z551//XGwuBUWEv/TFT+fU1PF3f/VtfM/Pv5H/62uf86gDrtIZTC4hVdSTtvLMq47duuF0tWJmO3Zcw8R45rZlIgpWE9P1J+JUWi0fxQNgc7uwAJZ+bQhJiBja5Hrg2gtTumQ562fs+wn7vmZiPU1w7FU1TedovaN1eqWNzhJbg2kNyWjGpSSsnrgkiFUGLCvYLmIBnBk+/NYoYInovTEKXDBkXOmQn4CjjouVhdYqAe9cvh+VhIVwrzPZPrEKVBPBT83AX00gTHKGVSXCToRJwE0C01lL7TynJi0T65m5jqnrcBL77LpkSkAPMCW7NpLWwGluGyoJ7Jgmf89jc5YFShEEhEWcsIoV582UKmfyAEtTAVCZQBctlYmEJLTe0diI95EmQhRL8IJp0O7jIeJEg9YP/d49fOTsil/4rs/j9LS64N+/+fOezKoL/IPfeief2mv4kW98Ptfv1MdwpFc4MplqrfJWE+uZu5Yd1zKzLafdiqnpmJu2B6uptNT9Serz134t24L1jEuzLUObLF0GrnOmpUuWiXj2TMuOU9BahorKBJrK0XjHwlV0wdLYCl8rcCVjMZWAaFkYXebMWkhicZUQS/nQGYwIxhjovHI01iqnJX7oLBIQ7LGXigWw1srCUhJW1UC655Iw1nYAq2mWMdQqZQmVEGZD0yXMEnESkbmnmnimk45T04ZZ1XGmXjK1nh3XMLNdn1VXErQMzKBVwGecYRfAqsUzlXyemJaK0GfloOdEQOiSw5LWvg9aMpZzaBUcPloqE5QDtQFnHa1Vkt6DUgiVxQQOFScWtN77yT1+7A/ex1c973Ze+OTrL/pzf+GLnsaNuxO+75fewhf/49/nq553O896/GkE+Pj5Fas2cGZe85K7buLpN+9evT/gCCPJ0P0pXZ7CX02MZ27aDFoN0x60OqzEfHLqSV0zlIl9tpUzh75MzCdrmywdFiORVayI1uiHIiRikr7DtPJV/3WbNVZiEt5YYhCiMYQwaMGMV3LWeEGSASIhWIwI4iMpWSXnvZ7hEgIpGSQZUoyICAkl8686OT/KsoZvjcpCMb0mC6PZVnJGS0KnZWByCt6xlIWVylpCIdxriJMIdcTVnsmkYz5p2a0b5q7lTM6uZ7ZlN2dLU9P1oGQZLkilRLTEDGixv5AVsCoXM/3d4ZzoksWSCEbL95VUTExHSKbPtryx+T5iQiQag42xf+6mCiqRsKmXaRwmTixo/e+/8XamzvJ9X/HMh/3ZP/O823nGraf4kd9/Lz/1hx/Aj07mQnv8oMCffu7t/G9/6jPYnZyQl2WDvtHKaQCscuKesismpmPHNOyYBoOC1fiErIlU+UpcSzxwVCJCBi1Dlwwthql0dMZRS2A/Tpgb5UAa2zCzLUtX00TLXjehjY79umbZVTSdY99O8J3FW0esDLHWP6J0GWMFtjEkEWwlJCPYJmAar8dXMi4RzbjIpeLFyPkrmW09pB4rl4V11UsbUl31GZZmWSpriE5ULFzrvZ/r6+B3lL8K84jseFztObWzYqfuOD1Zcf1kn13Xcn21z9y0zG3D3LRrWVMBps0oWdc4oyoXsGkGKzM62bT9EWklEKIQxDA1HV2yRGOY27YvOWfR0iUFZp8MTXDUJrAKji5YUoLQqXhWwqO4PPy9d36c33/XJ/nbL7ubm05NDvU7n/H4M/zwNzyPc6uOB/c7QkrcfGrCrLJ84nzDv/mv7+Mn/p8PcN/9C37y217IvD45L41stIoLX1E6Q1r++QxQngq9+pYTswBWVUoFwMqFM14RqEhYcuubRJSWFYmp6ZT3wjC3jXaYMMqVRH0tfVRuxZmIKxyHdTRB0OuIwXTKVRkPIQGIlg1ikICS8ckiPmYiPqpMgkzKMyLnNzOuq0nMmw15g4y+dlY5uky6JyskO2RYoRr0V7FSPV4h3qkirgrUdWBWeeZVy27VsOtadjJQFcA6bZZ91qTva+wBajNKiWiJ/UXMSKIiYvuMTIvvzZkYm39vzIuVk6eSwCRpZhaT6bNuI5E9O6GxEbFJFfj2UQpaqy7wv//GO3jqTTt88+c9+RH//ulpdQH/deuZKd//smfxWbdfx3e//A1818+8nv/7W1/YE/vXakgeq4Hhszh0hQaOopSD5VZJYEfazEdk3ZYkBSSBiqyb4uLAVUmkSxEMVCkQEP1Q5Ct5Ix5DojOWJjpq4/HRMrMd+65m4WqMJJauIiWhM4loHD4ajMudQNGMS6KOpZAMSfTYJCbECiZGKKCUSw+JaeguBrKO6woq5w/MsuyQZfWKd0tyttdixUozkFAPOqxYFcI981l50iHOEmkSsDMl3ec5wzpdrzhTLbnOLdi1Dde7vZ4K2DFNf4GyxAv4ys0otMAYrMpFbAgFLiMJm9LodyK1eKII88xzFq5r3HluoqMygT1fs19NtAPtKjqnwHWYOHGg9S/+83t4/6f2+alve9GR66/+u89+POdWHd//y2/lR//Le/lLX/z0I338I480uqGfxXiAQE91OMpX9ASrhAvAygAWBYXyytqN4USLJhGkhBEIaPmwI22PcAFDnTuRXbJMjWWe9ERexYqdULF0NU4Ci6pGJOWrbiKkiuQMklBphAOS6Bxj0vJPJREOYwQJSUHDZEmEyEDKh5BfFLMOXFciDlK9l7LQWdVjVU67hRM76hSOhKOTLG0oncKpjmjFSSTNAnbqmU47Tk8bdqqW6ycLTlcrTrkVj6v2mYrnlF3pxcm07EhLJZ5p32C5OCgUvqqcE8P/D1HogSI4DqNzw5KYSofJ0oqi6+t/N/OhRR7hJHDOzVj6CucinUmPzjGeN3/oQf7VK9/H17zgdr7orpuuyHN8w4vu4NXvu59/+rvv5nOfegPPf9LjrsjzHGmMgEuV6weDuWEg2cfc1RiwDhMWKWIpqjw5XUnQq7p0dKKn1cR0VMn3ncaA9GRwJQGfDM5EmuD60aBVq7la6KxmUEkwNUAeto165TZd/lB1EWJEQia4kwpQMTpoJzlbY9yZOupsaywiBR1BkhHxnjVZhXhP1qwT7m5UClbSfx1qFY+mWseyqiowqVQ0PHOdEu6m1UaL+NwZ9kxNu9ZgKWU/sMZNXfC+jspA/VmNAvPaQVawGjdmStkZELQ337I5iB9RrV8lIZ8Lhtp6Kht0lrGMFR0iTgxoPbDf8pd/7g3cuFvz/S971hV7HhHhB//0s3nDvQ/wV37+DfzH7/6ia5qY7wft81xgSnol9NHq1S4LQ0uYzF3VEqkephR8uLAieoKnxFQCgQ6MnqBV8liU1+qSzboeQzQrOutYJcfMduyFiWqJ7Jzzled+oK0dQcgfbgDB1mRphGKARINxml2pflMUvIzR8R5AJCrnFcJo3Ofoy8SxvKGIRnvVe1VlLZYbtFgTh88D0H5qetFoN88D8HMI04SfJdKOx0wCO7sr5nXHmcmKG6b77LiWm+rzzG3Lrl1xyi6ZijZbhmzar12kxhnUYaJkViGfUyFnWC2GmAzt6DJXQLI/NzKHNjyWSmUWUTloQ2LHtezZCcbEwYXkEHHtfhpH0frId/3M6/jI2RU/9x2fw5nZhZqso4zT04p/9rXP4Wv+1R/yA7/2Nv7xV3/2FX2+o4riyBDzSdYlm082PclivkeGxP1ipWDoBaYHn+QRCKMPvSFRE4kEulIaSuh5rpCzkJCELgWq5MFp6drWqp4GWNR6rV61lhhBkkHPc0EmJdMSQp3lEK1BvHahpMsCU5fl1SkpgOiLc+XKxAJY1uZBb8PaIPSm6r0aJA5DVqWA1c8S5llSMw1UtWdaeXbqlp2qYce17LiGuS1Z1iAULiC1Jm14iJJvHONX5GJgBfSZEuSsrBD4DIQ+QD1KbwPCKunndj9OWBjfzzE+Uub4mgets4uO/8+/ex1/9P77+edf9xye/6SLa7KOMl7w5Ov5n7746fyL37uHz3vqDXzV82+/Ks97uTEMNeuJVpTrbS7PyglILgm1LBTNVPrHSFj1Txi+N3qOcECGYvOISIWWiQBTkwWpMsr2BAIdMZcK0wxwpRu58hXORoI3dFQEAQmWZHTsRwl6tTZJYpBcMgJIO2i4JDoFsKKUt9r7IuZaURT0jiTbKkp9yZlW8cY6UN4w4rGKgDTzV0rAJ8JUB59lGphMO6Z1x5npilPVisfVS66rFj1gzU2j+ru+HPRrWrtNHmu4WI3fz9HXlJnToskzfSlYMqtN+qGU/AWwxjKJUnKukqXK+pPzpmMaPc6Evsv9SOKaBq3XfuB+/sYvvpn7Hljwj/7sZ/GnnvOEq/r8f/lL7uSPP/gAf/OX38LTb97ls5943VV9/ocNYZjZE53UhxFwZR6hywr2VazppKWVQKe6cSIDl9VnWuWETppplayqXDfXAWz4f328pFdYUe7JSiKgpShrv2f0BDcdoKDlTKCNlomd4oNhXxLeVISQ1fJa82GsmshFKxnITJ5brBR8Q0T67mEYysTSjUxHK4NY6xSOiXdnSZOaNHXEyhKmNvthmUGLNSbd53kAeq7D7/Ws4/R8xazquGG6zynXcF214IxbMjdaFk6lzYJhf4GcQfmmw/9tJbvqMD1YrVLVc1djmsFmYCxTFFPxvWxmKtpxrEX6C1yF/lyXwXXcxQxjE8FDxDU5jPfJ8w1//RfexFf/6B+y6gI/8+2fw1e/4IlX/TicNfzQNzyPm3YnfMdP/TH33b+46sfwsLHxPg8uDWZkKWPWMi7lmGTtCjuOAl7j7KuEAthw60naja7lWGU9lCxDJjaWYJyyS07ZFWfcklNVw07VMK076jpg60CqI6lO2YJnVE7le/WZysryyq6VZGoBMyrdMkHe81CwNuB8SSEydAyzmV9xbsDpiEpyphfQ9kr38vfUw98VJwmZRFwdmNSeWdWxU7Wccs1aSTgxXT+KVWYCx5MM/fuVS7y19+aAPzf29+sZ+nDOODp0CqKQ6vo+p54nnUpgKompCFMRJmKYG8tUjPKno4bA+PhS0qvvYROuay7T+pU3fJi/8ytvZeUD3/mSp/LdX3LnsQo9r9+p+YlvfSFf86/+kG/88T/iF77r87jl9PTYjmccJctKJmUd4+CLVK5khQjvkmOVKlapwqZIkzqsJNpCZCOYCxrc5BNes6wCWIWghQGsYg90mYDN5Z8WZYOViR2duGpx0vXgVklgNamY2Q6fLFYS+zZwPgnBWQJKztui43LDiZ6swQSnn8gYMSnl8lBJ9xSHzOuo+S0Zk+6TWjOsWZ1N/Bxh6oi1wc9Nr3QvpHs3V6DyMwinAtSR6U7LbNJyZrbiptkep1zDTfX5frrhlF322ruSYVUccnDvgBhPOqwyUK1SRZvv10BKIoHItKjoc3dyLoGpwFQME3GZKzWZ5Ux0dDSjjDYmg4/qFBKjqPvtIf+EaybTijHx93797fyVn38jd992mt/+K1/E3/yKu68JZfozbj3Fv/22F/HpvYZv+jd/xAP77XEfksaoPCSXh8V+xBZFPAW8iq2M09nBXAJ0GYRiSsQ18lZPjZJtjQn7vvWdr+IFsApfdpCI0cqFauySIWibXgWRp+yKHduw61SLNK07qtpjJ5pxxTrbStcMIsyJDMLMyhDzEDKV8kvDvJ8M2dZYOCyX+TEo3u7ODVqsyhJzt1A5LMHnW5hk19mJAlaYoOZ9dcROAtO6Y153nKobzlTKZZ1xyx6w+izVtBdcBIqq/eHioL+4cJ4B6WdL+znTwoludKL7LFqgEuVGK7E4LJXYjQthAcfiEpItj6J5ROXh8SMC6o30937j7fzkqz7At37+k/n+l919zdnIPOeJ1/Fj3/ICvvX/fi3f+pOv5d/9+c9h57ilEDLchMEbyZmAO+DkLVlXOQk7Ip1EQs62KiASB8DCZPGoZlub0eWfO0jQCloi5oHANSEi0INaD4RiwNK3xPfrCb64BkwcIolVUD/yIEZ9t0TtnMtCBTs1OFBrX+80o6qcvkTBjVTy5khdT5V0V7AqZn5xkgFranWmMFvNhIl6+/uploRhCmEaSdOIm3omU3VsOD1ZcV296DmsM3aRVe7tgWM5DyVlGPNaB5WGPZeVaYSSkcciV8m2RCqX0y5h/9ii0xHaORQqLAbBypj/EkJMGRBHVEXU+cMQMjd5kvy0fuwP3sdPvuoDfNsXPIW/85V365XwGozPf9qN/PA3PI/v/Ok/5m/+0lv451/3nGM91j7LQkdVrIm9pfJkw3VyfFLryehYkahSZKWFV54fk/wThbOQTKqXqzk5O7tQFV1i7MO19v2+g6jloxl94OpcG5yySwBWVUVIOi7SBIezkRiFFvURCzGLRgtgGsF49eiS4CBqg0F8VN+tMOoU2pFOK5eJlxVVHoSe1sRZRawdYeb6IWhdQCF0O5ueWOB3A0wibubZ3Vkxn7TcONvjTL3ixnqPm+tznDIrHc/JGVY9MuPbjHEXb7N7aDbu9T1hTdpQMvGSDbVJOSzI8pcE8WEy00jqS+4uBToCq6QdxOK/tYrqeOuj0RI9Hlpbevzl4R+979P8f//jO3nZZ97G337ZtQtYJV76rFv43j/xDH7tTR/h377qA8d9OBpjv+9RtlUGWM1G2VAM3AqPEYGO3CnMH+zIZiknByrmxyXFWCl9wc+lC8sEbRaUrC72DgMT0zE1WfFt1cxu6jx17XFVQOpAqnKnrRD0a4pyUdK7MiQ32MCIyXOAZX3XAVYylxRlpjA3AlKeJ4zZ011vI8J9Uu4T1BEzyVqsulMDx6rhlFuxWwags6RBS8LMYxGpKVKD4XapMVa4DzczAFYyFzRb1n9fI/b/JTyBiM6odvlcG3ux+dwQSkn6bPlQL/cl/5VHEPfvt/zll7+BJ92wwz/8s591YpZQ/MWXPI3Xf/AB/sFvvZP/5hk38+Qbd47nQMr73AMWuOzlPfbRGsSHWcOTPb6DqL3MKlkgsCJRC5A04woXIXcPGuU48OeymLX8e+zLyZJxDU6pViIkVVZj4IxdqmSCSDNx1DYQkjqVNtbRRCFZ25eQyeoKM7VvzhyJaKZlclYlKSHGkGJYd4W4TMlDynOFcVrlDMvid5R072bSzxL6+aDDCjvKYVU7LdNcEt4w01nCWybnOeOW3OjOc73bY8c0nDIrakI/hFwcF0x5bTe4poeKoeNLvmAVqyG7lmmFJAdcbEyvvYvZosiSNJvKpHvR98WUWKVIByxixfk4ZREnLKL6yfto8MHqgMJoFO3h4tgyrZQS/8svvokH9jv+5dc/95oeldkMY4T/4898JrUz/K+/9rbLPukvOdLGfQ6TS72SuZSrcy1q6rbpqaTEq363TYmORJeikvP5FkgXQNj4irxeVuhtrO+JbF7FJX8w1jVAhZifmI65bbJddMvctcyrjkmlGZet9UOfCjGfCe1Cyit/ZHoV+iB/MD1xviaDuJwYqd3V232UYU2Uvwq1juaEiTo2MA3YaWA27diZtJzK9tinM+l+xi57p4axO0fRQhkZZCSHjYPkKm1ZAZYnKMbvRcxWQ+U2dlnryfpkWSXLKplcAiaaFGlSZJVi/p6wyoT+QOrrNqc4OoevecnDz/7Rvfynd3yCv/OVz+LZTzhzXIdxyXHz6Snf89K7+N9/4+38zts+xpc/+7ZjOY7Nj1u/Fqz3NvL9/dhSuRC4Ot4TiSJ0qVzFEhbN2LNuvI8w4rNimSfMaX+JcZdwDbA2y0aJgFkDXdV1GabS0khFZ3RkJSZh5Sp8nUdJag+SCEGy86lmM5LAT9UBVaL6rZMS0lkITrMtr1bPmJAdIi7v2q2AZYhlGcWoS6hEu/ph+VkiTXQ3ZTVTHdqpacNu3XDdZMn19T6n3Yozdskpu+S0XbEjOqZT/K3WQGrcH5E4rH3LfGHxvooifeY5frHbbBnTjudD+4vIRbLnJHRYbHIg9KM5TX5cveAVioEsoTA9ub9KdV6KoiNmZV72kVw2jgW03vmxc/z933w7L77zRv7c5z/5OA7hSOJbPu9J/Pxr7+X//J138dJn3XrV/bckkh0edNtziMOgdEzSmwEWwJpKt8YdjeUQZfWwIeUO4oUdqS5dyGGVk73LjFcp6cZRgKt8XUI1PHEYvKYAnqcWy9w0ACxijSVm/k3JeR8MxjhW2UAwGIPxpaM4aH5sYwCnZWLZ5hOz+iy53E28PCJ+PAStCvesdJ9qSeh3EmGSiLsBmQQms46dWcNO3XHjbI/dquH6esGN1R67djUqCZdrGdbw/gwzphcr00OyRBIhl91GEl1af0+HUZ2ReDRnyGNt1vh97KdWY+74GjSDMh1TOmxK/bGWecUuWfZTzSpV+lzFzbQXlnLo0lBfg6sc51Ydf/FnXs+pacU/+ZrPPjE81kHhrOF7vvQu3vfJfX7jzR+5+geQ1oGrODwMJ/VItcy6B/hBpnDFK6lNJhOnrN36Ido0lIUFsIoGLI5KxfH31q7iDF3HoTO1fh4MWaJquLQTGpjaTlehuaCLRV0El5SY7xfXZtV8JapAr3OZWFxDi4NodhS93PIwZY/3YjdTfN1jWfWV7WVkEnql+07d9a6jp10zIt3btXKwdAFhXbrQi3vXCHSzcSszg6YHiWKV3fXfH7RYF2TCG1EAq5SN62BXpBIul4t27fHHHcl+gH8zm7sWy8MQE9/779/Evfcv+Lnv+FxuPnVtKMsvJ77sM27lmbee4p//5/fwlZ/1+KuabUlEW8VBiDFv+g2WJrh+iWYBGLWgyWaArH8YYDzVXyTmsDnIOuY+WorWazjpN7uGB231GUfoP4xmTW9kZRi4BjhjXe/DBOAk0kWLNbG/WgdjCa2O5hgv2W884RujizKKPxdguqzdSulIll/EiW7SUf2VZll+pq6jficRdnWWcH6qYVZ3XDdb8rjJglOu4cbJno4w2SVn7D5T03HKLJV/ZLA9BtYcF9Y4qFHDY3M+EOjV8uVCpa/98HtFSFrKeH1PYj8/qu9fucjY/mJlJRKioRJLJ6638t6kBzpsL3Mo52R53+J4hOdaBK0f/M138Ltv/zg/8N89ixc95eq4NVzpMEb47i+5k7/4s6/nP771o3zlZz3+qj23JJAoECEGIQRDFy1ttDT5BNGUXBdPFBM2GMqE3vc7xzgLYqOcGE/7j6+gIZUPzXpb3F4kkR+XpZslaGkSWBEdvDawSh0BYdc2/Uk/c52WH5XHe6vAVSdi0I01ppUsOch2NnXetVhlM76UEB/yiq/LKzjK+rPoZE3WEGqI04RMA27imU9aduo2uzWoPfIZu2RuG07ZJTum7cdyikyl56VG79HYz2q8Iam8pv1xpaEkt6JFfzc67rUmyUWyrOKJtvb3Jr04hGTy++Xy+6g811jGUrLwnszPAtbCiyayfO5aJOL/9Svfy0/8P+/n277gKXzrFzzlaj3tVYkv+4xbecqNO/zYH7yfl33mbVdNayah3GQAraBK4yY6VtnatogEpxt6pE3g6ku2ET+yfpKbtQ9JKSk2RYglxknMWsdSLuS9Lli8kIljQ2Qnc1udtX22tQyZAPaOtspbXSr14IqN1fIwQKj0NbK5TJSoXT5JSbVb2cLmckI1YdJ3DUs3M07V172e6m7CYi9zw2TBDdU+c9uuebqXtV2bq74uVgqO+cSDwKcfWJc8NL/xPmyW5uu/mwWio2xrDF4hKaB1OGLWiA3HOVx4+g5xoRVKt7IM80cDURXxbJaLF4mrAlo//eoP8g9+65287LNu4/tfdvfVeMqrGsYI3/aFT+Hv/Mpbed0HH+AFD7GH8SijBy0v4A0hGFbesd/V7Luac1bL77nRWcneTdKMMpoLpu6HD8UYvNbb4OM5Rtu3w9eI2hz9h68I10cfmiK/WLsfDeLWKaiHk2Ft+3UlgbZ2OBPwccjuuokjJohTQ/BoR3GqpUdoBTPRTqVptKOIt4gPKjq9jOhFpMVipiykmAXc3LM7X3Fq0nLTdI9T1Yqb6j2ud/vMTcN1djHS0hWn0XFnV2OznCtOoMPrPnRzy+veJd1LaVNaA7CLhT6/IZQLCIY2lXEsHSzffH9LBDTrCsn073VM69ZIY1FpG3WJSQhGh6Uj187A9Mtfcy9/51feypfefTP/7Gufc81vuLnU+KrnPYHr5hU/9gfvu2rPKXG4EYQUBB9U/7LKvFYT1dq48AmFNH0oUSiMwKsMtZKv6KNRj4GMP0DOkOOgk1zHhcrVOK4BVlF6q5By0JYVK+Fev2VaVcvbrJZ3AWsj4hLJJZKD5FBy3kF0uqZLCXOjpLwxw0D1ZYSuABs9V5WIWUdW1b5f9bXjdNv3mnmfGcz7DmqMBGQk+hwyWm2arIPC8F5INn1c57oesgzsu5OxH8JWIXKxvSkXufX7h4txg6C3u8nuDqE4PGSK45qYPXz5a+7l+37pLbzkrpv4oW943jU3BH2UMa8d3/g5d/Ajv/9e7rt/wROvn1/x5zR+uIkXYmdpvWPRRSZ2wtRqdjIxnjh67YMxWJOo8CAPPWw7zqxKOdH2oJfTftbLh0LaXuiaGbP7RFyTYGzuYOyPE12EUZZjlA/PVCZ0lWOS15KpLimx6vR07jpD6JQL0kxLCC14r9IOW1lMTOrIEB4Kug8Xoc66rKn0a7+YBeq5mvhdP1twXb3gplqV7o9z+5w2Sy0LpVkD7OFvz3q0XA73F4mc4Y4zq/IeXEDCJ82QDlzQOnrPy79XEvJ7qRnumJMcP4YZZYPKwQ1mgP3xjzRjIUnPsTbRsQwVq1DReEvwFvHSn8eHiSuGIv/+tff1gPWvvun5TKvLu5qdhPgfP/dJGBF+5o8+eFWeT3zqAUtLRMF7Q+e1g7gKjn0/YRkqzgcdodiPEx1azQT90IE63Ed380p90C7qsYVuObnN6ApdytSxfmwqnmnWI00kMMlrzqbZkrnIAIpK/JTV2bxT1YrdqmHmOqaVp6o9UsVBAjHedNP7sueZRJuFpZdbHpasrhokDraO1NnEb+5adl078nRv16YULgZYpbwKm9lSzx09BGAxbNcZR59FEdfeg3KrxK+V7L07aQEmiWuLfzcFy3YjUxzPGq5ipZl/cDTB4b0l5nN3rK17uLgimdavvekj/I1fejMvvvPGxwxgAdx2ZsaXfcYt/Pxr7+N7vvSuK/53mwAxlEyLDFqW1kaaYFl4XYi6F9Tu5bxM++5hsTkGLihLIMsVksmkquHCEZ6BSzqo5OjLiI0PkJHhg1I+EAdtuB5zOeUY9QH0MXsldtIrt0+Gc9WUEIWmDoTKkLyQKiEG1WzZCqJPWiJm4JLcSbycKNqsUGlpmOo0LKMorqO2Ydeu+mHwIg+42JxgKes2M93SqdXXXQ68aOjLtH6RGLvIbpaA5ef093SWsE3oHCooIme+i9GZ0Ptp5cfuj30EumNNl2ZZtTaKvMN7Q/JGz99weE7ryEHrdR98gL/2C2/ihU+6nh/75hc8ZgCrxDd/3pP5rbd8jF9740f4mhdeWYto04KxCdMKphWSNYTG0gILq6VYSoIzER+HDSorq6VWsToGPcnL9hQFiJhLRy2zaoG2jIIkk1HS9R0moNfZjPkR/f9iexPXruhFQKlZlfrIl2WxxZM+krD4fhNMOb7OLpT4RVSzJYm9iYJz6x0Ln7tUU+1K2QmETiUioTYQE1I7pAvIYbeEXiSKqV+cpN65YVp37NQqHi02yWorM6z1utiK+rFU4aAweWRHX9eQLXyGxxq/3kOmFHsNVfleeS3XfjclyNlfKU3LgP3YkmYMfGuvBYPLR8DQJsci1pwPU/bChH1fc76dsN9W+NZBZzCdYDswHYeKIwWtT55v+M6ffh23np7yo4+hDGscn/OU67nrll3+3WvuvfKgFRJmxAfE3EWMLuKDCk2dcax8RVkcMdHNp+wb3QhdRXWFqIGwcRJa8kBPHgXRcRvV5ZBnFktXKY4EojDwV+Vxen+nUdeyfK+WiBVdhKDjQ6LyKejBq5aYxZUhc12qlJ/GjrltaaJj7lq6YHUBqIuEKhKdQYplTSHkHdmyppSHl5lpWd3HmCwklxAXcTZ7m+WRqZLhjjOSsW6u/x7rZV5MJktSBsGnXjTiaO5wDOj5giGjEm8DsMwFWdZ4njF/L29sAjBpWFAy/vlNrmxzoDqMvOb7plBQ4z8fLMmbviw8tkzrH/3OO3lw0fKzf/7FXL9TH+VDn5gQEb7mBU/k7//mO3j3x89z1y2nrthz2SaRRLAN2JWQJBFXhpgcjY7hEaIgkmjzgCqotYuVSGMqQibop6ZlB3IZMBrMFd+XDHbU0i4q9jbZvk1+wfGNuoTF5K+4OAxZgJ7qFVAhVGIoBoQdgZizPvX7Klogz460WJOIVolpw1AGN0Hb6Usgrmw/TG06QWLCTxRobWOQzvYziZcaw7INoIq4KjBxnon1zGwex2HdkE//DjMabjZrgDMWaOrrV8S4atBYMZjxHcRnjTOhcWdy0yRw0yjQFmNHCTnrKiC1jiib0w5FBBtGI12rVLOINYswYT9MON9N2e8mLNqKVVNBazCNYBvBtGDbw70PR0bEv/lDD/ILr/sQf+4Lnswzbr1yH9STEH/6uU/AGeEX/vi+K/o8mmElTJswHdhWMI0grSG0hra1NF3FyjuWvmKv0xPnfDflrJ9xPkw5H6fsx5r9OOlJ036Uo2RCDG4Rteg2nbG2qurJ2HVDus0refkArvEyhbPqt1wbzehEKPsYbZ+BpdEHbViGUaQDu7bhlGvUNLDyVFX2lZ9EQq1K+bLFZ0zIJ3eZLg8ly7IJbEJMWS5SRmZ00LsbkdLjOcxxOdW/tyO+qF/8wfDalxJbM862/7qXT2xktptxEGDp846WumaQKxeZwkduvtflfV6XYOhURpPJd+0Y6nnYeqddw070Vs7jQ3YPjyzT+sHffAc37NT8z19y51E95ImNG3YnfOndt/DLb/gw/8uXP/OKST1sG3ULTadXqmQ1m0gGkrN4myAJC1P326eNqEvAJGROI5cvEaMGfJSTNfRfA9kFYugOlVIh5Cs/yWElXNCFfCg5RZmJBC0DJ8b0W1wA3ZWYQgYsjTI0bHvA1DKxS+q91aWG3aphUdX4YGjqihTyeI0bBqmNL9otQ7rM92cALRCTVC+WF4yMIyY1Xiz+Y+AVqGQorMZjMyZ/X/mr8bbmMlM4ZFmDXGHMJa6X4uXrzdgc1TKSNMsCkJBL1HUCvv97Dug6Fy6rdAyL6d/C16y8o20tsTNIl6mNbrgAHyaOBLQ+8Kl9/uj99/N9X/FMTk+v7Mr6kxJf/YLb+e23fYxXvvuTfMndt1yR5zCNKpxdpWJKSTr/JgG0ce3oKj2RfTB0WXi6dBUxCUunV8GQDI2t0JGZVk9+01Cn8RU/YEpWMCoZ27zqq5C1RdtzEIkcMZkb0+5UlYneKoNhkxQsq1x+Qrm/kOyw2XalANfcNJxxupdyWdes8pjPclbRkhXx3qhma6qEvJ3qPCKXScTHkmUZzbKGv1fwSQfZV1IxMR3E3NEzI2BJA7CMy8T+b90AnLVZURk6dXDhRaP/+fx+tJLL9tF86UFqlzLaZXsrkQvDSFQLnCzLKCvH9rM7aSHfz3dTzrVT9toJi6bGN460srhGsCvBtrmpdDWJ+F9544cRgT/1nKs3LHytx4vvvIlTE8fvvv3jVw60fCL5hO0S3gumSznTEqxDW/sYfKcfyrI5KyahMsr/GBIzq2fL3DSqx0leAUVgmnmknufKUojN7iIjwr6Q8mPgGpZa0LfTu6SDtp0YuhSpSARJyjEdUh5dSlR1GQjZW16dTpvgqKqA95Ywtq5xQnRJwcYZkrt8p4fxkhGgdzAoZLRLkS46rEm9sh2gGpPcyaxxRcBaqTb8zRolA1IOqjzxhaNXwPB6Zk7Sjl/jAxw9DjqGcWyODY0ticbke5mDbYKjDZauGwh405UxtFweXmx78EZcNmillPjVN36Ez33KDdx2Zna5D/eoidoZXvKMm/hP7/gEMaYr4htmGs1AkqVfEZ9sPhGSXl1jmwhJM64Q8nyiU//vpa9Y1RURYcfqDr0uOULeMzWVDmuGTAtYAy5L6InaQFzrMhXCfhi2LR8kJe675AhGBsdTo3mCTboUoboI3bq5LVl/NWbhqc5YrlzFstJG0N5US+NFawmdIMmoSj4KoRHiSh1OLytGgFWcOH0chteXQY9lmjMtI8WQsKVKo4+g+AFUuLDDd4G8oGjf+gvHaAh6Q4jaJdu/T2XDknYIcxZ7SNzeHKrfXARcysG9nGWd66ac66bstzXLtqJrHDQGs5JMwmtDybYJ21wl0HrTh87y/k/t8xdf8rTLfahHXbz0WbfwG2/+KG/60IM8947HHfnjm05Bw9RGOy+inazc3iMZHURN1pCiXvnbBMEZRBIhf1idBDpnmdlW7UZy2YWBOoVcDq7byPRaIYoTwMVLrE2yuSO35mNNEI9NkaxhZT96KkmMV4VEDioQh1DblchUWqII82zR7JNhXnW03rKqK0JtiV3KJPygkBd/BBeUNNxiVNDySWUn3hp85ndMzrQURBxdGrqqppcyFOAaLhRDxpXWnnPdVmhdiFouCMOY1eD4AAx2zZmLvFhGNY7Ngfri9lH8slZJyfdVrFiGmoWvWPqKZVvRto7UaUk+1mYNo2hXCbT+41s/Sm0NX/6Zt17uQz3q4r+562asEf7TOz5+RUBLuoARwbQR69TjyDZ6JiYj6IVUifkYikLZkapIYxIxGlLeLeiTZWbVo6qsXIeRVOEhuI21f89t76IxKrE5SzcM0oqCFijfYxJTsrNq7hqGAyQJQ9cx5g9SKRNDtnpp8dYycx1N5VhUQXVbffdwGKQ27jJBq3hBRSHlW8iZlk86UmVIzIyhIRsaZvuZAizFc105v9D/jeMc8KBO3yZw6cu4PlA9TBWoti4wUrCXUhFgpNM7CKzK98duE6XjXPisJlYsghLvw4yho8tzhhQCvgBWp/TGVS0P//gDD/CZt5/ZEvAHxJl5xQuf/Dh+9+0f569/2TOP/PFl1UHQleTIcJE2vkzN6weUlG2HfSJ4HWFpo+CrQNdZQhRWtZ4Ky6pSMlgiK1sp2ZpL24oLnQgeSRQ3iL61n6AWPel3TEMnegwrCQTTUaXUb0QOaXA92PxAFcuVSgI7punHRgCuq5fadOgcXet0EcZUOZUwEUKVssvppUcxY5SYtFMZDa23OBtovKM2XpfOxkrvk6NK2r2tcqZVMs3BWkYGQBmByRhUNpfl9rOK2YljFau1cSsrCZtG2rmSv+ZylHShQGI851iAa2yNs8hSmbIabBEmWfk+Ya+bsJfV723rCCuLWZleV2ga9Os2YdqIaQ/LY15GND7wlg+d5flPOvos4tES/80zbubdH9/jU3vN0T94CEgISIhIFzFdzGR8yh2ZlEV7enKYlqzjEmgNsbX4ztJ0jmWrafzCV+z7fALGuidXi9VJiU375E0niLUFFmk02EvxKre5XB1a46tU5QUIrl9LVfzpIxy4IWb8YR60ZGUFmdrXzF3LtPJYF6CKWbleZApqWXNZUZw3Iznb0m+FXCL2y0YYLQKJrnfxHGxlyhjzxT+WDzfcvu7HP7zu4xX3w0yg9Iss+tKyB6nRHgAGi5sxYJXzoifgs8V3Ub+30dLljDN2BnzWZHXa4dbsqqjhE3I1Mq23fvgcbYg87wqUPo+WKID+xnsf5EufdbRdRGk6iEmXkUq+0gva2r8g09JSUQLETj9Qmn0JqwRdZRFJdBNtYVdGea65aXW2zUYt4wTq0bm16Ze1Odx7YZkx4kOwkKATq4tjiw2L6Id4Kh11iv12l4OyCyjOBYOEoByDJXFdtSAm4Xw1ZW/SaTNikhBP3pF4eEuUi74PcXCQJQopNzy8MQNwJbXCdtlJtksWm2Jvk2zT4P5ZiVfdW5Z1DN1CuWiXbww6BaB6gMyvq5HUWy8TVS5RXB1s/j2VXZTnW+9EjheYlA7oIk7okuvlDYtYr2VZi6amaSpSY5HGDOr3kmU1CdtEbBeVoz1EXBZovf6DDwBsM62HiGc//gzOCK+/94EjBy28Vw1D55FWFU0267KsaHkRouqI9HMveWxNu4xFnhCteqyvqgojUNnAuW6KIXHGLbFkz6sU+g8SDFlOKUv66X4G/uOghRab/9b13zPUMRCyB1QUQ5SO8abrgx5zPHJEckylozMKfHPbsrQ1M6eLXtvKsaoiyVm1lLHKa11OSGSNiE+JXsy7GWs8XwYFUwBeVMcWspi0iG9VlHr4xaxDljV4s+v3h+5ry9A46flH8Rc0VMJGVlZAcJUqQjL9WrAmuTXl+6pIHLxm89KZfrC/12X51N+ki4g/XHl4WaD1ug8+wJNumHPTqcnlPMyjOma15e7bTvOGex888sdOMQ52wfkqZZzJ/IjkYWb9YIYkIErQxwjWSi8sDE61213lWJnEwlUsqorKBBaxZmI6VrFixzS0yfYOAQcKGdOF7phjO+YLfj4LHwtwrVLm1FIkogstpnQXqLbHqvHhBSnuEyrXCLmTuHINc9dS24BzAVwiukSykoedL1OnlUvDfqNMGpaQXrAm64DolfJJsFKI9KiKd9bV6OPYJOIvdFwwDEt183H0ZL+usDfkbC8D/qbFzGY3csiyxnqs3C2MajuzCpVuhOocvtPS0HSiw/2ZgLdd0u6hT4hPiL8KoJVS4nX3PsCLn37jpT7EYyaed8d1/MLrPkSI6WjtptuOFNMgEwoOI6LcQEoQLeINkD2LokBKxFqzMAlCSJDEqFVzFlnuSWJeKYycrWYYktodR71EHqTaBoaO1YbwcMxFaYmiH7CxZ1ZRVoOOpIQktMb2wtGLWaHoYw4umkWpX5oHu3ZFl6yaBdYNbbDs15FUm76LmA6pxL5YHFSxiSREEtZEnOitMgFnSmd0GIlat1hez8Q2XSBg05VByfrekYPBYUG5w8HoEegHtEt5GETlMfpcHnLnd7PE3wSr4kS6iDWrWHHeqx5rFVwm32uWTYVvHDRWle+rMtyfcmkYsauIbQKmDcghy8NLJuI/9MCST55veN62NHzYeO4dj2PRBt71sfNH+8ApqpYhBChZl48qhfBRU+8uDml4l/Ksl3I6kl1PjRcIQvK60ccHo1tusqJ5c9B3DEgPFcWv/KGIZVhvp4/J6bH2pzitljb7xQjpMsCrW218bwszMZ7aBiobEBtJNuVbP+FydNEDVsourjo8XWyKy9eXE+Pfv1jZGPqyTi8IXRyyo0G57ta+Lsr2Tf/5gwCrvD+94j3anGVZ2ixxSFniIB5kTZc1Kg1DhBjBX2FO6x0fPQfAs59w5lIf4jETpVHxhvse4FmPP31kj5s6r+Q7OdNyDowg0Q0r4LNvVMz+6MpzSS4Z9ReVCxZ8bYgGOutYdhXWRPZ8TW08czPllF1iUmSaOmqGmbjNKKB2UCcR6AWppszEjQZwlZy3RClLQANtsr0Ga3DgzFlXVpGbNVlA6G1bdkxDY6q1JRjWRS0P++UXR5P99tNMeWja2cDEeqbZomZmWyZGB7wrMzIDvIgTw2YUBwZDyiu+8j/kbCtI7AXDsXQJe75R+nuTTJZX5DI7DtuOygaf8h4WsIpJBv5qBFb7YUITHOf9hHPtlJWv2M/ku185ZGWwS4NdCm6lWZbLN7uKmCYgTUAaj3SH64hcMmi95xN7ADztpp2H+cltPPH6GTfs1Lzh3gf5xs950tE9cFAXKzFCMhbBI14/uiKCGKMLHJyWiNak/AFNWhYZGZwhJCFe2/8hqM6odXrlbKOjSaM9itje1+mCQxq12g/alweFixkbDGZCWobdeWWIedjkMtg0ly3Zxe1gGH/ZtALWTlyxOK6NGh4aG8EkdcMwl59p9Y9h0CzLqjVNZSK1UeCaGL2VzK+31+mbGuvmgIb10nCw90mDueIGp1Uy1rFeq8+0xqBF6nVh2lgx5cqVn2uQSoy5q0WYrEkamlix55XHWviaRVdcHBy+tb1fVpHd9PddtlPyEZMrA11SeYUzrfd+Yo9bT085tRWVPmyICM96/Gne/fGjLQ9TSrraPUREPAmnJSKQMmgBmE4zm2QE22l6ZTpIGcTEq4Gn+LxiKwyK7jbkjlAu06oYCMbQ5VOnuAEcRLQ/XAlZfOjHwDWMmMTe7yXKYJQX8f2adzXGKzOPG4CVP3QlkyklojMRYxRYklETxcsGrUwqJpMGP63sXFpbT20UOAtglVvxq3qoYWW4sOGx2ZQYR9HChVGm1QNWEkD9/kvHsMNiUuy7ivr94eIzLgeb5Oii8lhtLglXWfW+nwFr1Tl854itRVrJtkmDyZ9tswK+U22hdIPWkHCFifh7PrnHnbfsXuqvP+bi6Tfv8vOvve9oh6djoiy1T4CkmEfJ8gZlQKLTDCA67SRmyUN0Rrtco/LQNJCMITlL21qMcez7mtp69u2EhZ1gSZyPM92OY4Z5tfE+vpCKaFHWgGv8AVVeJ2T6OHcPM4D1/Ev+vU5sNqALdGLzmJFoCWgYhrQZ3ChiLh2L39bEdEwygDiXea1eYHp5b0MvVnUgLuFcYOo8M9ex41p2rXp9zW1DWTZ70AYei4EUCSI6VI7Ns59lpGdQw9v+nS/aN+k7ff0255GYtbwX/Wbo0R9tUinXU3/hKTzYKrketPa8Zlplw1MbLefbKUtfsd9WLJtale/7DmkMbmFw+0q+Vwsl390qYZcR20RM43Np2EHnVcJziLika0xKiXs+scfTbtqC1mHjzptPsWgDHzm7PLoHTRFdqZyJzKhlV4p61ZKg5KZ4Fe5JT86n0c7ETMoHMHmzD3kUJYQh22qiLihYpXUytxDj45b4gVf7wnNtSABMXsIwHg9as/LNH8R1Etn2z3GxWHP+LI+Plm1WkqpB5GjKQ6Q8VgKTMGa9a+hMPLAk3NwT2SviS3k3FndmYW0c/X95fcpj9ZumR699efyHy3pjKvKIchsy37JgddPrfeFrdQrxjqar6NqcYfWarEy8t9l+pivnXs60fERyE4kQSFcy0/rI2RWLNvD0m7egddgoWel7PrHH7Y87mkWuKSbERPLFWMtFgGog5yVGsAZSUjmhCBIMzqlNSzK68EGSWhAnA9GNtvpMKyobmPoJO071eGWgep6afvPLeGSkzLyVq3UJkz9QFb4HKwAkf4+Bk+kYAKtXaEfJgtOB7ym6sYr1ZRn6uKWLWDqIyms5GzEmqj6rcFGX8z4YtBPpEtZFnMsEvFPyfb0cXC8FiyJeR5kHcOk3J6VhTjBIGpoX/e8b2qKAZzDkW3vdM4f4cMA1/vcikxh3CEuGtecn6tzQKeneekuzqgkr5bHcvsE04BaCWyr5Xi1Tr343ReLQ+pxhBZLP+/AOEZcEWvdkEn4LWoePp+es9J6P7/HFz7j5aB40RVI0ClxJkBgVuAqhmX3X6byCmYiKT0EdOyWSjOqVQNdskUEsNJZgEstWu4iVCezYtneB6GxDl1z2Dy/KeFkDq1KaQHHotD3Y6HZrQ+81LkMbP2adVRcdjMwElaBOubslvYfVOHrf84wNYy91l/8OayJiMqdlVHB7WW/DiIg3okS5M7H/e0qXNebyt6jO15T9uWtnUyKKKDlu6B0wSjkYCP0W7jHRPp43BPqMzpSxHdXd98cz5tbGSy/WgCtntz6a7As2+LwvuopV51SL5S1hZZGVRRrBrnTG1a5Gmqw2YfNQdK/JKtVACEPVcIi4LNC6cwtah47H7dTcuDvhPZ84eq1WiuqPpQ200GdZB3YUnUFEsG0EMRibsK2q5UOnq7BMB7ETklWnycY6lq7ivNdMa2Y7IkI0LVXmm2D4EPY8CkMHsZQx6nuuRNok298MdsJ630H+8BaCnqHrlaOUQyU2N8wUos9kEWf5gDpRq2NjUvYcy0T65bwF42wta7TM6DYcs6yNyZSsqP8bRqVjftH0NTHQjlXxo+PtdVS5n7sZNhsOGgn9a1hejx7YNvi1YVGsDAsq8mjOKmiGtWx1QYXvHKE10BgFrLxcZdwtVBeHXBp2hbZQXSFxoDOuaPfwnk+c53Hziht2t+M7jyTuvHm3l4ocSaSk2VTKJWISBacYSdHq5zZaBavgSDAo5gUk2n6UR4LkGUXRxzGGGITG1cQw+G4tXE1EVHdkWuWINjpgw1D0uhK+EMAT8VTG912r8Ybiumx6MbEXNRIH0BsDnC0e8VlIOt5UrSMwOgLUiWXHNMxNy8yOxnmKuPRy13PmiSgFLFXDl9fBRxVhYqALwxN1fba18RpJpMml7DRrp9pkwUKH2mDXo07peCZwnNVWBKbiCdmyZ3PUp2xQWivT0Wy3A+U0k/rbl5nC/U7Lwv0mq927TLp3BrMwmlk1QrUAk8l3t8wdw1XQoeg267I6ryDlQ59pHXbT9yVnWtvS8JHHnbfs8suv/7CWcHKZl/cDIsU0ZFwpabotSWcUs2I+2aC8lo+IEYxVRbw1enVMFozTKyRGiI1u9Vm1FftVTUzC1E6UmLXqdrp54m8SzMD6zxggQie5M1bEoZl7GRPVFRCM9FnJsGg0UYxchv1+BbAGHZO6RORSKOuknFFyHJN6Mv5I34c0kOYRdTFdoRq38ppcDLSMpP71weTOHihPKAZMq6M4+bUcG/71j8No9RcGta8uz1FGnoZxorXHGr0YJdPyydBGq7esdve+bNQxWdqAZlmZeLdrpLsa/IlPmmGVxlFfGqahoXSIuCTQet8n93npUTsWPAbizpt3Od94Pn6u4dYz06N50P7qVJYUCERDKuVESio+LTOKIhAjxqqVDaAqeq9yB3UsUJGpBEjWEJKjSXBOEk2lp8zEevWpsjVGkoo2H2I0pYyzdMlSxUBn9EM7NdmHXnQTz7DUVQ3r2uTynJz0c3O6ldr3+/+molnJVHSTT7+WDGglMKVjatqcaan0oXIBcZGUB6ePKorDQxcsrbEs81YglwLLfGSDdmrdsaK8ht4YnBlkCZ3kJRjiCUi/c/Kg0J2Epl9QAlwIaFLGidYfo3dxKBY00fZZ1sLXLLuKRVvRNBVh5aBVpbtpBbfIHFZDn125VcItM481ni/svJaGpWMYAqmA1yHiEYPWXuP59H7LHTccTQfssRRPv1mX2L774+ePDrQ2onQUicN4DJ1+kpNXwakAtF47iiKotYB6p0Nu3zv1l4+VZF7J0ZiE98qfTSpPFy0rqwR3ceccl4ljECug5aNlYnXfX9nBWAZ4VXOVeZb8oTS5VCwEfylFByK5qMxjD1jTsnmZRCfqFaHApsr4qe2obMC4Uh4egctDfuFSNJm4tlT5A69/h9O/t8+0hk5pCZ8bDRGhJjubSui7pWXkpohtx2UilGaHCm0rwsGNChle13EUqUWJIi0pavexz3vMavfBH0t6o8l+SUWrX5u2eGUNEocixUkxFh/wnG1doUzrvvt1t9wd129B65FGGXn6wKf3+SJuOtoHH/NbpaNYpBBhpJvOKnl80KzLBKQ1WCA2Ss4nMyzIsNkRIllDdI4UhZXTGbeUhOAMlQ14YwbyeU1zNRKTilFyHR2mtsS+G2lNUuvhfKBlrrDsUbSYvpwBRtuUw8BloYBVZdAyWcg5zaBVRnrKiI0xEW8vP9MaLGlyaTjyiG+Nza/HOucXudC2ppj8lZ+1RFaifFiTeTGbP7I10ObXaTwDWmYI4cLZ0LXFJCPQKmaMw3GMtkRnX6zeG6sMQWdvrLIouCjetUxkKA2LV1YXwMeBw8raLGLquawrxml98NMKWk+6fjtz+EjjplMTZpXtX8Mjj4cCLujHe4iZ1g6x128lb3X0JyQkGJIY3VotynmJF7oEaWJYJaGtA523NJXH2UjtvJYdmS8yXNhFK2DWGkttdHNx4cQ2SWQrnWYSEqhSWBvxMRL7zKkm9DxWAayJDKNlURp0U49nJ5eIO7ZlYnVwusvZ1uW97sW9VIghz24GqzyeifikjqX9MeXycByGRDShz0hjkjWw67PWEf+m15WIpZR9sR9xKjzfuAQ8mHcswl/93iqVFWB5yWo35Xw7YW81oW0qQmP7IWjTsl4SrhS43FJtv+0q5LLQqybLZwLeB5L3A6dVysMrJXnYZlqXHiLCHdfPrxxowcHAlWSQQpQOVrnSdQJWO42q3cozim2+2jeAyaLTWnQuzek4SJufMpQ1WEZn/lISJGuVUkrENeCSnn6rTOgV1pUEVskxyUZ0OsqiUcqe8YKHujg+SMxtfO2M6qjxmNwWqkQuHwsRrxY1xsQse7j88rC/xVGmFZUX0gMZfrxkWOXeiNaXw/cNURJdMkzSaOC5+JRJ2eQ9GqhemwAYdFdwMeua2FvXjBdhFMuZsk1n6bO0IRj1ee9ErWaCDtqbjt7myOTFwRJSnsIYJjF0MiOsARUhaHZVpjkOGY8YtO69f8HpqePMfDsofSlxxw1zPvjp/Sv7JJvARSCJfpST96qEF6MdRdCfDRaT3UxdTsuMN3l3YlLvc9GV8snYnFEITRS8tYTaqx2LiQQXEElaguVso2RaIoloBJ9Lgc7Y3rUgIpnDUbI55A+hoXhkDWEYabI2wsqI3E42Z2IlO1N7mNpohohNY4ODSwqJjJZaDNt4RBKdyeVw1tKtv00K7n1ZaBIQldtKsS/TTNJdiSamXEYrJ1hsqSHr1ChdVM8F0wE5xhlr6ciuYp0zrAln/ZxzfsrZbsZep2Z+i0bJ97TSIWjbjAz9Mo/lVgnXZNX7SqUNptEMS7ohw6LrNMvyviffU9Ku8WHjkkBrS8Jfejzp+jmvfPcnr/wTPVzGJX4Y9REhpYRZmT4LsiYDmANNE1Im58sWa6NjNUDKBH6wkej0eYyJ4OjJ5wJcjoiPyn/5DC7LUPXlz0Q8WNiPE/WdwlPnxOmgjlmXLFFC3tqjg79dCmvZFuRuZJZVFNmDLbKHy8y01laIRfrFuCEvbS1Z58WeZrOBUaLMAMZs8xNl2JCjw806WF2WrR6k3LhgxVj+/VWs6LCsYs35OGU/Trjf7/CAn3O2m/FgM2PR1eytJn23UFrBZPLd9lue0uDg0KRe8V46hdJkPVbWZQ2ApdTEuCxMV6o8vPf+Bc+67eiM7B5r8aQbd2gO6YV92VGAi0HD1XcURbTsizLY2Vgl542hJ6fVmjl7y+t2d2KTyXmjBH1KghcwI2LZJsEIGRgiJinARRFMtnPyWTjaBIfLpV6THDZqVjRsqEl5i/SmPinm0kY/xF2KgHYLbS4Vdff0EL1zaFauk5Xxl/c60/Na/cLWlIfOk+CKV/xoVOnhooz8wOD1XmYKlROLfdNCrZKH3w1F6DXqBhawKtMK4zX258Nsjcc6301YdHXeV2j7bqHJOqzegXTk927WBqJjXmunJWAvb8jlYcpOpT1gpaiAdchs6xGD1oceWPBln7HdJn2p8aSrzQUmLTkAyl7QXg+T0uB8WjqMMaqdDfTKeeMTEo2O/XSZnM8WuiHq7GKKynXFyhArNdqLSVTIaSPRBqwkUsm6SLQB4mjuzyfNwIqGa5V0U/RU2gu0RTr3aOnE0Ylu4Im0VBJZie6asSKElFglsjuFulIMXFLG9MuUaY3Lw3ILwWCMdln9aLSnjNKUUnGtUSEbkwW9ONWqPU9S7doqOc2sUiLieylEJ7oMhLSelRYHjjLMrtugda9lEys+1e2yDDWfand4sJmx1054cDGjbS3dfo2s1OPdLbIma1kI+OxAmsn3nnhfdpplrdrecia17YW6rEsALLgE0OpC2pLwlxFPOo7SepRx6WBqvoqL5FEKIblsICjKXUmnngvJGZIVrEnYWgtKW6MWv1aUD0oQrSElBcAw0iMlN7T2kxl9kEbKbyOWNpcGrdVTshpN/Mc8iqJzdJnDEeV5OgkEI71v/FQ8XdZt2ZT6TGwRB5/5YUvNUb2++neoMJecadFv5EkouF+sDOz/zqQvSJFEhCR00eoOysxt6Shi6mUQm8PVQYYu7Ngbq2RZSra7fhnvItQ82M3Z9zUPNjPO512FbTO4j0ozMvPrhnlC242yq7IsuGzVKdIG73OGlUHqMgELLlERfywfvEdJPP662dFu5Dls5IwrxVHXST9ZOcPKGVd2Q6XMKBpD2exTWSXn+xnFCCDESiULscpr4YOAS6RaiCFinZoTWjFEF/KqLM26osmllFVwO9dNmRivnuS2Ymo02ypiUhhGUUqpV0ddOLpjWqxE5tL0tsxAv6fvwTDnfJyqgV1wGVQ4cJvOI4mSaUlS2YO+BoZgEsFqJ1FMHHRZolXpmJgvZLyWfwnPwC+Ou41eFHCr5OjyDKd2XqsLZkDHbhvlVjqExWZmGSoebGcsfcW51WRk5FdBJ9h9m8l2zbBMC3aZSfeWXvFuVwGz6pAmQNNqSdh2pK7TkrDz6xzWJmAdUqMFlwha20zr0qOyhidcNzueJy+lopg+40rkWUTQjIsROR8jYgVbxH95tCda039Qk6iGK2tBiV6UB3NJv64iwUVCEIxJ+Giy5XGksrHvMrbR0hiHT5baeJrodOTGeBZZy1WyLVjnhmyWPVQjD/ZKAnXZz5g/uPf7XfbChLPdjH1f03oHfgCHSw0JGbTydqPktDxEEq3Rj1i0w2vrTFzXsOW/pTQmXM6Q1EBQtVvLUKlUQyIT6zEkJqbrX5cybF2i/M0xCU3U17WLOlbkk2HhaxqvNjN7TU0XLKtljW8yf7WwuRRUR1vbgluoeLRfTNFudAqbgLTdoMXquoF07+UNlwdYcAmgZY1w2xUaQXmsxLFnqqOOItGsLciALEQt4z6dgpoxMmyvbqUgm25oDmTXCO2gSRL9XjLaSQuaTZk8LhOtYIwhRgWvZAWbpN8k04rFRx3UbYylSzpL5/KH86AYE+xT0/Vq+FImdcmyFybsh0k/RxeCyWXdZSri4+gWcrblDVEgBJ0lLBunrdHX3koiiWrYoBj16eMV37AOizUqUi0A5kykiQ4jkYlx/aD5GMRLaemj6rvKei8fLaug84XLrtI1cd6yaipC9sTSZRQGu8rDz0XWkMd0TEcvbTDdqFPY6jYd8WFNPHrUgAWXAFp/4YueirNHPBb/GItjzVRHA9Zj1Xz5vmqycqnoTa+ctz4iCUxtkQShLVuDDaFSoaEuPxVCrVxWqFVtHitDqhPBJUJlEZsQGzFW5yStTZjscVWysNoFqixLmFidaxwb69H/FevjMGPl/ZhDikl0GYN3PLCasWhqmmWFNLox5nLCdNpdNY1gspgsWEvwCti+sxirjqbGDBlWWeh60cfNNjflZ4vBoM3c4EO9HiHPQIYk+GAJWfDqg9poe2+J3pB89sLqBLcy2aUhG/l1qnRX1wYtBW2XsrFfUHnDIpPuTYu0nQJW0yhgFfI9ZS5rk7+6BMCCSwCtv/Hlz7ykJ9rGENeMrc/FMi6AYDJwjbb7tB4D2H5PoCGZpNyWDFkGEV3yELKVc0CzLas8lzorGKKLiDGEvMHGSKKzCl6tz7ouG6hs1ZeRMGi/QMEosW7xUn5GYJ3jiQYfLPsr5W3SyqqdyuH2KVw0jEctfbx+4BGIrZCSySR85uyCvtbGDL5bBwlOx6E/M/ysoNmagtj6ccQR+R+y5CLmTmbRjkVvVJbRGcjjWabJK+sbFYwaD3aZFe6rgXR3K7WXsU3uEnYR81DjOUU4GtOFGqxLBCy4jG0827j0+HNf8JTjPoSHnlMs5LzN232iU3I+6XiG3idMZ5BoiZVmXNFBqMDWWgL2a+etarySHbKv5BKx0q5XtCqtCHkFFwJi1RJZDJhsj2yyOrN80AuRHqOow0IGsBLlMz025UtRBt1RXiJqLzPTsi2ArmMr247AEPPfmKqoUwS26MIUpNcOshz4CLTGTV/yqI/ke/27hp8rv5+yTowkEKQfLSplsHj92pZRHK/WyCbk8q8dwMp4JdzLnkK3Ckina+xV5T4i3ZtWu4RF3lBI9+JGehnl4GZsQeuxHBcBruH7Kbufpn7kJ0V1ZxIfkc4iMRGdwXRqbROdECrNqkKlkohoVaSaLIR8n6x2HZPRrKzfjJNBK+WZwGQymAlw0Ac9kX95xE2NifUxHuWft3l+TslldSm4nHCrVFY3IkkIXea18t8YnVVwdsPfdqA8Pv8tsvFnkv9EJGWH1OHHN39PYn7o0t1NQ2eTiA6/56aBCXpvWzSDagexqC32yG0cJA2NR0LErPzgidV2mqEfRLqXkvCIYwtaj/U4ALhKqQjovRGSB7FBl8Nmkt6gingT1AFUgkF0F0WWRWg30ViIIen3csmYbO60FdAqIGWz2l5rO8WjEZBt6hMkyqBID7JmE7P+g/lnyB9YnxeIZnL5csJ4Xfqqzq+5wDYM0pCgf5uE0d8m0idV/Z9U/o6D3qb+j4ALgLg8Rhq0YmW0iFGTgKRApeClNwkpZ1fZWsarut2uIiYU0FJ1u2mzvUyXV9jHOAKqA0j3/hiPLsvSv/WIHmgb29jGNq5GbNuA29jGNk5UbEFrG9vYxomKLWhtYxvbOFGxBa1tbGMbJyoecfdQRN4KrK7AsRxl3Ah86rgP4mFimlJ69nEfxDa2cdLiUiQPq5TSC478SI4wROSPT8IxHvcxbGMbJzG25eE2trGNExVb0NrGNrZxouJSQOtfH/lRHH1sj3Eb23iUxlYRv41tbONExbY83MY2tnGi4hGBloh8o4i8WUTeIiKvEpHPvlIHdqkhIl8uIu8SkXtE5PuO+3g2Q0SeKCKvEJG3i8jbROS7j/uYtrGNkxSPqDwUkc8H3pFSekBEvgL4gZTS51yxo3uEISIWeDfwUuBDwGuBr08pvf1YD2wUInIbcFtK6fUicgp4HfA/XEvHuI1tXMvxiDKtlNKrUkoP5P99NXD70R/SZcWLgHtSSu9LKbXAy4E/dczHtBYppY+mlF6fvz4PvAN4wvEe1Ta2cXLicjitbwf+41EdyBHFE4D7Rv//Ia5hQBCRJwPPBf7omA9lG9s4MXFJJoAi8sUoaH3h0R7OYydEZBf4D8BfSSmdu8SH2bZ+jyYuy2/5pearr/z7IKKr38hbk8Qg1oC1YAziHNh87xw4S6orcJY4qUgTS6wsYWbVXXZqCJVaYvtJ3qpUC9GqKWMsxoy23NLwPZf6f1tzk42og2reFVCcUSXoZh8JjL6n5ovGp37Dj/Hwhy//3od9Lx4WtETkLwHfkf/3T6JzfT8OfEVK6dOX8PJfyfgw8MTR/9+ev3dNhYhUKGD9bErpl477eLZxDUcxgh8DlrWICFTVAFzOIcbApCZVClpxWpMqQ5w5wkS9/Lu5IVaCn+rWpFgJYZKBqspglQEKg1pi2wQWkotgE1IVz/7Bfx+yZ39El2cEIXiD5OUZMvak96LusZ1ucbKNgqE9pIPsw4JWSumHgR/W10/uAH4J+KaU0rsfyWt/leK1wJ0i8hQUrL4O+IbjPaT1EBEB/g3a0Pinx30827iGYwRYJbvCiGZTZsiqxFmoK7CWNK1JlSXVjjB3RGfwc5tBSvCznF3Ny6o3CNOUffwTyWWAqtRsXqqIcbrirao91kYqG7BGV5mVbUchGrpgiNHQeru2pix6A23eMdnpPgHxulDEdDljK2u3DxGPtDz8u8ANwI/oZw9/LQ0mp5S8iPxPwO8AFviJlNLbjvmwNuMLgG8C3iIib8zf+1sppd86vkM6fLQ+8qtv/DC/9ZaPcmZW8ewnnOHrXnQHu5PtuoErEjnD6gHLWsRaXYpYAMu5vhRMtSNOHbGy+Jnrs6sw0YyqmytohRmEOml2NU15vVsElxCbsHVQoKoCzkacDcwqT2UDE+txJuLyRuuI4KOhi1Y3VXtHFwytd3SdJQaDd5bkTc7iDOS1bWV7UaxY20T0kC/JVhF/ouOqvnmfOLfi6/71q3nfp/a54/o5ISY+/OCS63dq/upL7+IbXnQHZnMZ38mIa4vTOiDDKmWg2CGrkrqGypEqR5pphhXmDj+1xNrQ7RhCLXRzCFMFLT/XjCrMErGOUCXM1CM2Udce5wKVDUwrT20DM9cxdy21CZyqVlQSmZgOl3dQxqT7FJtYsQwVTXQsfEUbHUtfZQCzrDpHCIaudfhOAUyWFmkEtxCqPS0T3/JPvufyOa1tbAPg7LLjm3/iNXzs3Iof/+YX8CV334yI8IZ7H+Af/vY7+du/8lZ+880f5e//6WfztJuukWW0JzEuUhL2gFW5gWyvHGlSkSq7ll2FqRBqQzfTfZN+roAVJ2SwSsRZgDpiay37nFOgmji97VQttfGcqhpmtqM2nl3bUElgmtcXGYl00dElSxM9E9PRxIrKBJrgcBKpTKCLFmsirbe6qNakfgM3yRBbbQDIIdFom2md7Lhqb953/fTr+M/v/Dg/8a0v5MV33rR+ECnx8tfexz/4zXew8oH/8XOfxDd97pN46skBr2sj03qoDKtyA+E+nfT8VekM+nml2dWuwU+UaPc7ylv5ncxb1YmwE6GKuJmnqj2VC+xOGyoT2a0bprZjaj3XVUsmpuO0WzG3DVPxzE2DlUglgZBLuVWqiUlYxAmrWNEkx56f0CXLvp/QREcbLXvdhC5all3Foq1ovWO5XxNXDrNnqc4Z7Are8YPbTGsbRxCveNcn+O23fYy//mXPuACwAESEr3/RHXzp3bfwj37nnfzUH36Q//v/+QBPu2mHu287zY27E3YmFitCZQ3TynLdvOKmUxNuf9ycO66fU7vH8BjsZocwc1ZrHcKq7gn3NNGyMM4q4sQRppZuV7uD7a7BT7Uc9HO0JNyJxEkiTSJ2x2NdYD5tmdYdU+c5lcHqTLViZltmtuN6t8/EdJwyK6amYyodlXhs5rFCMkQMdQy0yWIkaRaWLJUEumSZ2Y5lqPDJMrUdbXQsXI0zU1Y+4L2hTULshFiL7oU8RGxBaxsPGasu8AO/9jaedtMO3/Hipz7kz950asL/+Wc/m7/2J57BL7/hw7z2Aw/wpg89yIP7Hfut52LLhp0Rnn7zLi991i38mefdzlNu3LkCf8k1GrL+QRUjA2BZO+KyjJaEuURMlVXAmljCxOitVpAKUyFMwc9GRPskIpNAPemoXGBn0jKrOmau43S1YmY7rqsW7NqGuW04Y5dMpeW0XVGJpyJQiy7wDQgdjkAkZKCNeDBg8mLWLoNXATAngWWoMSR8NIgk9qua4C3ejnRfh4gtaG3jIeNnXv1BPvjpBT/75z/n0NnQzaenfOdLnsZ3vmT9+yklfEwsu8CD+x0fP7/ivvsX3POJPd5w74P88Cvu4f/3++/l+77imXz7Fz4FkRNJ6h8+LiZpsHYAqpJhTWpSXZHqijhXHqvbdYSpwU8N7a7yV92pTLjPEn5HO4Kyo6XgZOI5PVtR28CZesXctey4luuqBXPb8ji3zymzYsc0XGcXVOKZSoclYSQSk77/LRldkqMS3/85pWycSkfAEJMQMJnzqljEmj0/YWI957sJK+9ISfCNJVaJjIkPGycKtETkB4C9lNI/FpG/B7wypfSfLvGxfgL4SuAT2wUTB8ei9fzof3kvX/j0G/mCp9942Y8nIlRWS8TT04o7bpjzwidf3//7x8+t+Lu/+lb+/m++g/d+co9/8Kc/89ELXBcDrHGGNZY0VIV4t8Rab2GaM6yJECaiRPsk81e5HKSOuAxYs7pjXnVMrGe3athxSrKX7OqUWXHKLtkxjfJXRKYjUIpolhWTIaSSYRmsRAICCayAHQlOQxIihn2ZUGVUamJFFy0TG1iYiNiU1fWHe+lOFGiNI6X0dy/zIX4S+CHgpy7/aB6d8bOvvpdP7bV895feeVWe75bTU370f3w+//C338WP/pf38tm3X8fXveiOq/LcVzUeLsMaj+RMapKzpEmtgDXJotFK8FOjhPusSBnAz5OC1iwic4+rAjsz5a9265ZT1Yqp9ZyuVuzYhl3bcL3bZ24abnB7zKVhxyjxrhlWyhmTHnPAKHDl+xKWRF26isSe+wLlv+amYRWrnI0pOn3K7TCpKvZdJNpEOqRc5poHLRH5fuBbgE+gw9Cvy9//SeA3Ukq/KCIfAH4O+ApUtvYXgP8DeDrwj1JKP7r5uCmlV+aB5W0cEMs28K9eqVnWOBu60iEi/PUvewZv+8hZ/u6vvY3n3HEdz7z19FV7/isam2AFw0hOkTRkHZb0koZaRaOzSvmrqcXPjUoadvI4zix3CCcJvxOVv5oGpvOWygVOTRtmrmOnajhTr1TK4FbMTcspq9nV3DScMkum0jEVTyWDDqvFEJNhlSraZOmwPRFfohJPLQFDpJagpP2oub2falamAmARa7pkmVrVghkTCSYdVlt6bTuXisjz0VGc56Bzjy98iB+/N6X0HOAP0CzqzwKfC/xvV/QgH6Xxc6/RLOsvf8nVybLGYY3wz772OexOHP/rr76NR60sR0wPWJg8Q9gPPVvNsCpHrF1fEsZaASvU6Oxgf6+ke6oTMom4OjCptCScOb1Nrac2nonxTE3H3DZMcmdQu4NKnI/BppSDLbYHrC45uuT6jAnIUggFrrkpEomuv+1Iq89jOiZGn8uZgJGkWC6AXJkxnqsdLwZ+OaW0ABCRX3uIny3/9hZgN3tVnReRRkSuSyk9eGUP9dETjdcs63Oecj0vesrVy7LGccPuhL/60rv427/yVn77rR/jKz7ztmM5jiOJi+mvjFnvEFZVn2mlyl1Ausfa4GeGdieT7rtKuod5ottVDsvsdFQTz2zS9RnWqVpLwsJfTYxnblrmpu35q6l01AyA1SUt/1apoksKVKtUEZIS6yZnYlUuBXtQEs/ceCoSU4HCrVdEqhTokuNB0zI1HbVR9b0xKYPW4V7OazrTeoTR5Ps4+rr8/7UOztdU/MIff4iPn2v4n//bq59ljePrXvhEnnnrKX7wt95B4w/ZWrrW4mIK9w3AYqxyz4BVSHd1aMiEe5E1TBS4wjTp4PMki0brQF17zbJcx8R5ahOY5AzLGVWpl6yqkoCVgYMKKH+lQKW3VaqG0nCkS7AkzbAIudPomUhgLokdI0zFMBWhFqGWqBwZEUvEXIYu+loHrVcC/4OIzLI18X933Af0aI/GB37kFffw3Duu4wuefsOxHouzhu9/2d186IElP/2HHzzWY7mkuEiGJZls7yUNVaVZVgGsaTWQ7lPtEvpZVrlPRYWjM5U1hClKuk8DduaZTjrmdcdO3fZzgzOrHcOJyeWZeCoJ1OJ77qlkWDFnUgWoVqliFesMWI7ARkmIjvXsSMfceHZMZEcMc7FMxDEVy0QMlUCVwbHczCHLwc24pkEr2xL/PPAm1CX1tUf12CLyc8AfAs8QkQ+JyLcf1WOf5Hj5a+7jI2dX/NWX3nVNyA1efOdNvPjOG/mhV9zD2eUhDZeuhSimfWKGrmDlVOFeVzr0XNX69aTWknBak2Y1cVoRZhVh5ggzJd67uaGbi5Lvc+0WhnkiziNpFnDTjum0Y3fasFO37FRtHnT2TDKX5YyO4FRGwUqznmwtg9CyAVYZsAqvBWDRxyg82I5p2JGWufHMJXFKDHNTMZGKiTgqsdiNuq+XS2TmPSV0IO2QGHbNl00ppR8EfvCA73/r6Osnj77+SZSIv+DfNn7/64/sIB8lseoCP/yKe3jRk6/nC49Al3VU8Te+/Jl85b/8r/zI79/D3/yKu4/7cB4+RmB/gcK9EO/Wqmmf069TVrmXW6ytEu4TUYfROjuLZvI91jpLmOqImQTqOlA77cYNIBWYmICTgB2VZUXKsK6nMliSloeZzyr3Y8AyOUsqHcKpeKYSmEpiKkIlBv3voS94m8AFHNqa5poHrW1cvfi3r/oAnzjf8C++/rnXRJZV4tlPOMOfed4T+In/+n6++vm38/SbTx33IV08coZ1oGlfXYGxvWCU3B3sx3KmatoX5k4Bayq0O2aNdPez0SzhrsdOQp9hjecIp7ZjZjuqPOA8MV65LDOUhnakp4oY2kSvvyqSBuWtlE8sHcKKwI5Rx4e5eKYSmYswFUuVbwAhRUJKBBJdUnJfO4/KjflkSEmIUZAoh20eXtvl4TauXjy4aPnhV9zDFz/jJj73qcfLZR0Uf+tP3s28dvytX37rtSuBGPm4b5r2rbmMbhLuObOKlSWWWcKp4LNxX+jvIU6y2j3bytS1p3aeqRuVgRIzf6TjN0ZSP9BcSdDv5azpoCgEuyGDVL6VknCaJQtTCdQSqQQsQsmxFKwikUQk0qWYy09Dm6yO+GTjQB+NZlgJbZkdIragtQ0AfvgV97DXeL7vGi2/btyd8H1f8Uxe8/77+alrkZTPmakY0duIcKfKgFVV2aUh30qHcOqUdJ9l0j0DVk+6TyFM1QsrTBNpGpFZYJKdGuZ1x7xqey2WOotqWVht3CwpdwxT380D1u4L11WLDkkX7qpII3akZSeXhROBWgSzkZmPAatLKWdZdsi0osUnq2M+SdSK+dHCaW3jysd99y/4t6/6IF/1vNt5xq3Xbun1tS94Ir/79o/z93/z7Tz7Cad5/pOOR0N2QWyUhNoVrEbzg1nOUFckpy4NsXbgDGFiSVUWjM50Q47aI6tbQ7czHs+JpGnE7nRUVWA+6ZhVCljFXXSSuaxSFjoTezGplold3zUsYtAy9FwGo8dhR2T9AHqRqWgfsRbBQE+2R6LOKKZER2CVIqsE+1nntUoVi5B9toLFB/WRlyCHHpjeZlrb4P/8nXdhDHzvn3jGcR/KQ4Yxwv/1Nc/h8dfN+K6feT333b847kO6MLIGa0y4M1K4673RLTmV6QErFuCqB9Jdt+WMSfeE1FEtkUcuo7UN1EYV5gWw9OvY67BMBrG6lzuEvgQs2VeVea6aQE1YU8mX0Z5yM0B1AO0ZUiKkUZYFdL3ua+CzmqhC1RAFoiCRQ3cPt6D1GI833vcgv/6mj/AdL34qt56ZHvfhPGycmVf82De/gNZHvvHH/4iPnV0d9yENyyaK/ipnVVJVSF2rlGGS5QyzijCvCXOVNHQ7Dj+3dLuqdO92hG5XMyy/A35HLWbiTkDmnnresjNrODVt2K0bdlzLbnZrmNkui0i7Pruamq5Xv6+P7AweWT1XJT7rrTpOGS0BT0nHPItGp/lWkTKPNURAM6uOQJM8qxRYpcR+NCyi43yccS5OOR9m7IUJ+6Fm2amDacprxow/XPNnC1qP4Ugp8Q9+6x3cuFvznS952nEfzqHjrltO8W+/7UV8eq/hm/7NH/Hgoj3uQ1rflDMi3jclDbF2pNroDOHIVqZkWEX1rl8r6R7riNQBVwWqKlC7QG1DHjj2axlWKQk3uaySZRXJQgGsqnwvl32VRB25GZH5/Y2kpeAB2BJT6m8diVVKNAmaZNkvZWGcsIg1TXQ0weGjIQSjmVaCi/QFLnypj/SN28aJit9/1yd5zfvv57u/5M4TtwLsOU+8jh/7lhfwwU8v+I6f+mNW3TGO+RT9VV8SmjVJQ5q4vkOYakMopWAGLSXdycT7yBOrTsTeEytQ1Z5p5Zlnx9GpU2nDZMRljQFr0g8ne+rc/esdGHLJaEnUrINVLVGzqY3bJliUV1x9thSsxoC1nxz7qWI/TvrbItYsQ80quLwfMZeHgW33cBsPHTEm/uFvv5Mn3TA/sZ5Vn/+0G/mnX/vZ/PEHH+B/+/XjW2/ZC0fHXcIiZ6irfhdhmOp6r6Jy9zOhmxWFe/Z0n6rSXT2xEkwiZhqoJ55p3fUWyWU8R2/t2hD03LT9MooiURj4qWFsp4hFTc6ixtnUZozxJCT9/wi0KdGmRJcUrEpJeD5WnI81D8a53sKcs37OWT/jvJ+w8DVdsMRgEZ+7h1vQ2sZDxa+96SO882Pn+d4/8Qwqe3JPg6/8rMfznV/0NH7uNffxO2/72PEchMmODaPtOcmanHEZYqXke6wkE+5CqBiU7pUuK9XV9Hnbc15HLy5ibcSaSJW3O5eRHNcT47kELIPQWUBaiWeszSpC0iJvGM8cmtHX4yjgtPl1SOvf6/L3VklokmWVHPtxwirWrLLVcpMcbe4adlG3UMdQAOvw4tKTVRNs40jCh8g//8/v4Zm3nuIrT7LlS46/+tK7+K/3fJLv+w9v5kVPvp7H7dRX9wDGiyeKLmuUYcVJ9sHKkoYw0fX0oVIDv1ipJ1aYJaLTAehURWQSqSY+yxtapnkfYbGZ2XFNr3afmxabF6kOHb8WK+kCQ74SAcEgVOiW6DYZrCQOKrRDkn7sJ6CAN15UojY2li4Zzscpq1RxLk55MOywiDVn/Zxzfsq5bsrC16y8w3tD8gaT5Q7bTGsbF41ffsOHef+n9vmel951UjdCr0XtDP/kq5/D2WXHv/y9e676869vzNEsKzm9Raeke6wySNXDrTfwK7IGB6lKJJd0Pb2L/cbnysRe8T64NnjmVsvB0iUsncBafC734gWA1Y/qZEfSLt8ioj5aSfrvbf5b+fc2mf62SpZFdKySZT/VeosTzocZi1izCBP2wqTnspqgW6djNH1ZKOM07mFim2k9xqILkX/xe+/h2U84zZ941i3HfThHFs+49RRf84In8tOv/gDf8vlP4kk3XMU1ZGLA2DUH0lTlsZxa9xGOh5/7kZwqDz9XqQcuXIIqIlXEuoCzkcoFJs4ztb5XvM8yUE3Er7mBFh1WP9ycAWu8rxCJmDIgLXEYbk6Hy2HGljLFP35sZ3M+zPQ+TtkLUxahZhkqlqFiFSq6YOm8JQWBoPsO5RFwWlvQeozFL7/hw9x3/5If+JbPuKaGoo8i/upL7+JX3/gR/vH//938y69/7tV7YltU8K7vFsaJywZ+6jYaqjyWUw8dwljlLqFTwEqTCDZhJgHrQr9BZ+o8O67NK78aTjldoHrGLvsu4WYpuLlYordGFlTEme9LxnVQRnbRyD82ztYGK5uKB8OcVao46+d9hnW+m7IKjqWvWHbaOYydRbxBAhgP5pDOQ9vy8DEUPkR++BX38JlPOMN/+8ybj/twjjxuPj3lmz//Sfzmmz9yddXydpRlOUvKRHx0QnJCdNKT7es3XZ1VykFsAhcxJuFcxJlIbbU8rDOPVUSjE/G9pGEqbd8h7LVXF6m1invD+L6o1VssLbZ3Ly3/f9BNbZgdqzh4bxVZw3hUp9zaaGmjloU+KAlPUD/mInfYZlrbuCB+9Y0f4YOfXvBj3/yCR12WVeLPff5T+Dd/8H7+zX99Pz/w33/GVXlOGWmyYq27CcNM/bC6uaGbKX/lZ0VAOsqwJpFUZQ5rEjAuUk+6finF7mhl/Y5rOOOWnLHLvLJ+qYr2rMXazJQCMZeD9N5YACFZbB6NLvbHltRvh96MsnVnvDKsgF6bbJ9p7ccJTaw4G2asYsW+z8r3ULHXTWiCY9lVdN7iOwedYFrBNoJtwDaPjsUW2zii8CHyQ6+4h2fddpovvfvRl2WVuPXMlP/+OY/n3//xfXzPl97FmXl15Z+0bNJxViUOTrOsWA/ZVT9TuEm615phKYelPFbtdJPO1HmmtmPuVIs1N22fYRX9VcmsKsIFJSEofxVHXFX/tcQReEUCEXvAhWy8KqyUmP3C1rw9OmTQWsSaLmr2VTisZahYeV3O2gZL6y2+s0QvSGcwXhAPxieMv+DpD365L+1d2sZJi19/80d4/6f2+ctfcuejNssq8R0vfiqLNvDy1957dZ7Qmr5jGCuTV32Nu4QDYIV6IN01w1LS3YzGdKaZx1IRqUobdAu0ikZ3TFmo2o26hZsjO2HNTnkcCjau94Nv07AWrNxUX1X3PFUpA/djzaKUgfn7w3hOlRXvFU10PWCtgqPxjtY7Om8J3pA6g+lAOuWyTAum22Za28gRYuJf/t49PPPWU4+qjuHF4u7bTvOCJz2On3/tffyFL3rqFQfpNNJlhanLnlimX0ThsxbLzxLJoRYzdeavpgFrIy6DVe0Cu3kpxa5ruK5aMrMtZ9wyq90HX6u1sZyRrUxMhpC/bpOKSQth3uZtOuMN0SXbGm/aKQssSialvz8Q7+V31RvL0WR+q4mOfa981sLXLH1FExyLVoej29YRG4s0FrsS3FJwS3CrhFsdDrS2mdZjIH79TR/hfZ/c57u/5M5HhS7rMPG1L3wi7/vUPq/9wANX/smMIVkhOkNykr9Gb9Vwn4rS3Q0ZlrVaFlZuGISeWN/bJRfHhqJwLyVhuRXAKl5XNfFAFfw4xkS8fi153f3oliRnYYOdzLADcbCXaWLVm/op4T4m3vXWBKsupd4QvQFvkE4wneSuoZaGtt1mWttAdVn/1396N3ffdpov+4xbj/twrlq87LNu4+/9+tt5+WvuveILZ9PEqfp9MkgcysqvMMtK9xrCLGdYdcROAsaqrKF2Ogi9W7VMnOd0Jt13bcPjqgUT07FrV31ZOJemt5QxEvuB5xIGwSahI66tri9lYRhxVJaY5Q/D7yvB7ohJeg+sgNBFl/+9eMiLjuNg+pKwjY6Fr2hjkTdUNJ2jaSp8Z0kri1manGWBW4BbQrWI2OXhht63mdajPH7xdR/ig59e8Nf+xKND/X7YmNeO//45j+c33/JRzq2u7OqxZIuQdDRXuEa85wyrVuGo8leeug5Mq3XnhqLFmo0yLVW6t73baHFoGAPW2o1hCLqELquQtb2FJcbyiD4Lyz/bZ1bRrWVZJbvSjMvhk0oalkEBqwmuJ967kMn31iKtwTTaMTS5Y+iahGkjtjuc5mELWo/iWHWBf/Gf38Pz7rjuUanLerj4quffTuMj/+ntH7+iz5OqMrIjWf0+jOiESVLF+2QMWGULdLFLVsDarRp2bMuu1SzrlF0NoznFqYFMtBPWAWtkIWM3ACtmHqpkXTH7so+38WzGGLCKlisWB9K4Dl4+WpowAFWXb20h3ztL7Ay0BmkF29IDlm3AthHbRqQ9HGhty8NHcfzkqz7AR8+u+Kdf85xHfcfwoHjuE6/jCdfN+M03f5Q/87zbr9jzlKUUfqaaLD9T8j1M1WImzPPK+pnvCff5pKW2oV/5teNadqy6i552K+a2YW7atS5hz2Vl4JqMtuGU7CNCNuLLC1hzdzAwqOJNnkmsRvOJQCbZ8+OkTLiPAGucgfmo3FfJqpahYuFr2mjZ72raYFm2Fc2qIngDS4tpDG4huIVgl1DtQ7WMuEXELj1mdTjNwzbTepTGA/u6Euy/febNfN7Trr2VYFcjRIQ/+Zm38sr3fPKKbqdOToi2KN83VO95nlDqmCUNvtdgzbKJ33TkizWzJavyF0gYhnVeYVgRBnmxhN4MZbwmd/p6/qp0CuMaYBXnUp1TjBdV0gOj7uFwr1mXwedbF5V077xV4j0IqTWqycpCUtPmLKtLSsK3EekCckgjxy1oPUrjh19xD/uN5298+TOP+1CONV72WY+nC4nfvYIlohr7ZfK9mPrNEn6eSDPdnjOZdezOGk5PG66bLnncZMH1kwU3Tva5vl5wnVuo2t0tOGMXzM1IjzVybyj7Biti3upM3uwsWJG8Cac4MmQCPWlBVZatFu/4Ta2XyUD4cFHKS58sTR7PWYWKpa9YdBWLtmLVVnSNIy4csrS4fck3qPaSZll7kWov4PY7zLKD5nC22VvQehTGx86u+KlXf5A/c42vBLsa8dm3n+H2x834zTd/5Io9R6yL7QzESSbeJzoAbaaeulbX0Z26Zbdu2K0aTlWNEu6902i2mMmZ1OA26jOPlbfoZB5rKpEKFKzWxmtydy+DVekQ2rystTzuZtlpD3CEKHGgbKLPqlTp3ngVkK461WJ1rSOuHNIY7NKoHmsFbpmyLitiVwHTBKT10HZIty0PH7PxQ694DyklvvtL7jzuQzn2EBG+9O5beNV7P33FfOQH8l0G8n06eLsXDmsNsArh7poesPoB6DxLON6YMwassg2nANZ4UWqEtSyrlIelJNRZxSFz65+LAlzZ3bSUjL3b6bgTmT22kvSD0E1WvWun0BEaq4C1Mkq6r8Au9d6tIm6ZsKuAXXmk9QpY7eFK+C0R/yiL++5f8POvvY+vfeETeeL18+M+nGsiXnLXTfzkqz7Aa95/P190101H/vhlIYWf5y3Qs4TMAm7asTNrODNbsVO1XD/ZZ2Y7dq1KGioJzG3T81Y7phnd+7WScKfsHSSxYyTvHTRYhEDq/dpXydIWPVYm3ntveInsSIuRyFQuBIiDBqZN9t7ql1gkwecMq5DvS1+x39bsr2ra1hH2HNIa3L7B7eswdH1OO4X1XqTa0yzL7neYpkOWDTQtyW8zrcdk/NQffoCU4C998dOP+1Cumficp15PbQ2vfPcnr8jjR6flYVHBpyphqkhVBWb1+iIK3Us4kOzFp73qZwXjqCQ8OMMaA1aJkBJdImdAw5gNMGRYudycSked5xI3pRHjONAzPknvYlqI9y5YGn+AFmuV3RtW9C4OdqXEu2kz8d558IEUIoSt5OExF40P/OLrPsRLn3ULt52ZHffhXDMxrx0vfMrjeOV7rgxo+alyWWGaVPU+DUxnLfNJy5nJiuvqJaeqFafdam1FfTHwK46jhW8aL0+d5k7hXBK1CBVCJQZTNFfEXuLQIbQYWnQAupR7Fcpl1QTmpssbdxJd7gCOYyyN2IyiyyoZ1sLXnG8mLNuK5WKCXziksbjzqniv9sHtJ9wK6vNRy8L9gF10mMZjFivoPKlpoWlIYds9fMzF77zt4zyw6Pj6E7oS7ErGF915E+/++B4fPbs88sdOI6lDWV1fO1W5T0ebn/sV9Xm4ucgMLGVl/XjjswLWXAJzSUxFmIhhIo5KbG8j0+8bTAOXVexnLKnnxHTNve8Bq0SxmemSUz1XBrGSrRUniKJ8L+4NC1+z6GrtFDYVfqldQruftVhLBaxqUcj3iF1F7MpjVh2y6jTL6jrwXgFrC1qPvXj5a+7l9sfN+MKn33jch3LNReGy/uA9nzryx+6Hoh3gdAB6Uvk8+Kz34xVfMJReRtIgRejLN9+voC+ShqlYKiyVWAyjTCslQkq0ZenEyL3BZMX7sPNw2BpdYtB0Sa+aL49RxKVFCX8BYHUVq5VKG2RlsQuDXa7PFLplpFpE3DL0AlJpPNJ20HUk75XLCoGUti4Pj6n4+LkVr3rvp/maFzzxMTVjeNh45q2neNy84o8/cP+RP/ZYSCp11E3QzjNxClhOAkbSmlf7YAsTs/ZKS8JTZsVcPHMJnDLC3FjmUjGRiom4fglFJNIR8kZnXT/f5nEdUMAqXcfxgouQdAaxQ7foDN7ug3dWcW9YxJq9MMkLVqec7Wbc3+xw/3LOA4sZZ/dmdOcncLbCnTXUZ4X6LNRnE/W5xOR8pD4fqfY8bq/D7rfIokFWDawaLQsLeIW4zbQea/FfMsn80seAX9alhIjw3Dsex+vvffDIHztaSDY7kJqEMQkjWoaVUqxwRxElsoF+AcWYKK9RDdZUUP4KLQULWFkZu5Dm7c6jTGlMwOtzxA2yfSDSh/lBN7KhWR+MLs4Ne75mr5uogDSXhF2jOiyzMheQ7q5J2CZm0n1QvEsh3r2Wg6lkWCmS4taa5jEVr3z3J7np1IRnPsbFpA8Vz33idfzeOz/B2WXHmdnR2TAno15ZySasSZi8/RkUJHw21zMxdwlN6k37xlnWVDxz49kxSrgX/sog62BFpEuBVYprXFbLYOJXnCBg5O0+yvL6DdDZ273fAt27kdY00XG+m7IfFLAeWM1YNDV7+1Ml3VeW6mwm3c9DtZ+wK5icD5gmUS08Zum1U7hoEB+g7UhNozzWiMtKMa3Z4zxUbDOtR0GEmPiv93yKF99542NyMPqw8bwnPQ6AN9734NE+sMl4MHrpfeaCmuBUOZ5sP7NXwhJ7RfpUOiYSqEiDpCFnWGPACikSUiKgm6C7ESel/276W/FxLxlYi2GVXL6tb9FZxAmLoLbJi1hz3k8530052015YDXnwdWMs4sZ+6VLuLDYPTOM5uwn3CJRLSN2lXBZ7W7aMIhHO09qOwiRFKICVkyPCLDyy72Nkx5v/fBZHlx0vOQKCCcfTfFZt59BBF7/waN1M00CG1VZFmGqlqmJg51LCe0eprUh6OLYUCQNZuPjGVIkknKmpVlWmTPU59z4+VwyqgTC9tlYNwatzF+tklomL0LNvp+w52vO+wnn2yl7Xc1+UyvpvsqzhIs8mtN3B9UuWbuEeTynjOjkkpCugxh64p0CVgWwDknEb8vDR0G88t2fRIRt1/Bh4tS04hm3nOINR51pjSIlIQRDGyxGEs5kQ75Rx6509KYyHoRWwLIHPGZIpdQcAKtske9yp08dSYVu9Ahdsv1qsDKOU6QNZTnFIk76lV8PdnOWoWI/1DzYzFj5inOrCYtVrcPP5ytMY6jOD/Yy9Tk18av2s6ShidhFh3QRs2qRptPyr221JGy7nsta47EOCViwzbQeFfGq936aZ912mht2J8d9KNd8PPeOx/GGex8gHpL0PXQkvaUIMRpCvhVHhJDGCnXtJo4HlQ9Un6OZlX6tgNWXhin1JWHJsMalYRnjKfOHZctO2bwzlIdubW39eT9h4Wv22gn7bc2yyYC1cthsk2yXClhulTLhnjBtwvZKdyXeyWR7fx8CxAPKwkcAWLDNtE58xJh4y4fP8qef+4TjPpQTEc994nX83Gvu5X2f2ufpN+8eyWNK0u3IEoQUDMEbumCwxtAFS7RZN5WdFooCfryAYiz4jCkRJSpRJrF3uAopqcyhmPzl0rCY/XW59INCvltsSmsuDatY0SbL+TBT7ipMedBrhvVAM2ffq/7q7GJG21q6/VpFoyuhOmewDVTn6cvBek+7g24RMI3vMyx8UC1W22mnsO20U+i9AlbOtC4ltqB1wuP9n95nr/F85u1njvtQTkQ86/GnAXjnx84dGWgR0UwrQoqaWcU4jMiU8nBs6lfU8cNDCCGBEVW5k7RcLB0/Xaia6FIkoBY0pTQMmB6wCm8WUEuayMBtAX3H8HyYshcm7IUJ57opC19xtp1qdtVWLBe1zhHuq2jU5bEc22TSfZVwjS6jsF3UsZzRPKH4MEgbxor3TdL9EWZZsC0PT3y85UNnASWZt/Hwcectu1gjvOOj547sMftMK0oGMFmb6SuarbXVXge4hMbRTbuDObMiqJNDSms/o6XnYItc9hO2hXQfLWJtR0LSsXB030/Y6ybsd1oOLpqaZlURy1jOwvT8lVuUDmHK23MG0v1gwPJQSPdNPdYllIUltpnWCY83f+gss8ry9JuOKGt4lMfEWZ520w7v+Oj5o3vQVAArQRJSykAlicoGapNHeUygyjOI48WqEUOHySWipm0G6FJYW1WvPBZ0Cdpk+uHoAkZdXphaZBXFG2uVva+65DgbZixCzae7Hc53U/b8hPuXc5adY28xpVs50spiz1tsI7i9oUNYn0/YNpPuTVDh6KpT0WiTZQ0hDCXheEQnxCMBLNiC1omPt3z4QT7j8adxdps0Hzbuvu00r33/0Y3zHORQLJIQhtLQjaxoNqPorDqMlpnZJCaKApWFwc8q/3yX9Vfra+3XZRWRoVQsy1UXQXVYKmuYsJ+HnpvO4VvdS6jmfernrvOD6tTgMuFeVO4luxIfB6I9RAWsuE6+rwHWZcYWtE5whJh464fP8XUveuJxH8qJirtvO82vvvEjPLhouW5eX/4D5s6hJEASImDNkGXVJbvKKnhL1AxLhjX1Gn5tzKcajeAYBsBaJV0msZ90XnCRFe1lJGccIQ87L4Kuqv90t8O+n3B/M+dcM2XRVqpw7yxpz+nQ80qozgt2BfX5QYNV7Wt2ZRcdpg3gI9K0SIhZi+UVsLpWyfbCZV1Gp/Cg2ILWCY73fnKPZRe2fNYjjLtvUzL+HR89f6SbiorIVCRhTcTlWyV6v+m1HpKhE0tJvgp5XqEZWZdMvznakHrv9/GgcyHWV6nqM68SZWdhkxx7fsIy1jzQzln4mrPNlL3VRGcIFxW0BrdnVc6wgmpPTfuUdM8arIXH+Ji7hMpbaUkYe8eGonbvtViPcETnMLEFrRMcb84k/Gc+YQtajyTuvk3nM9/x0XNHu15NAJMQE7EZpNx4PdfYbQGDIarrgwAJbNZU9eAlgZAkzymmPAYkmb+yrGKdSXbX+2iNO4WlXNzzk550P9dOWflqAKzGQT/0rHYyhXS37QBYbhWwTc6uNgh3ipQhxPWSsGRYJY4gy4ItaJ3oeNfHzjFxhqfcuCXhH0nctDvhhp2ad37siDqIkoemDYhJWJuoSpZlQk/Kl1B3UMFiWFFhU6KT2Hu018V6OQ9UWyKkskzVcC5O6ZLjfJzm4eZJn1WVpapj074Hu1nfIbx/OVdJw36tHcLGUJ032EaHnt1icBrVTMv33UGz7IbsqvMqFG11NIcul4IpkTp/IYd1RIAFW9A60fGuj+/1LfxtHD5EhLtvO31kHUQtCxOY1JeHIoM9jeXg0igU4l0iIVnseJhahoypeLcXAem4U1iyrrX19Ul5rP0woQmOvW6incK2ZtFUtE11wXov044I95VmWGWRai9n2FC26/1YzrAhGr0CgAVb0DrR8e6Pnefzn/7Y3B59uXHXLaf4udfcS4zpSEwTez5r008rg9d4ryBkIEqRgFVfG+jnEwPSK9nbZLES+9GcLlnOxZmWfWF6AWD5OGRZZzudH7x/NWe/rVg2Nau9Cakx2D3bj+RU55W/qs8PQ8/VvqrbdY4wl4PZoaEn3McZVgGuTQ7riAELtqB1ouNj51Y845atf9alxF237LLsAh96YMkdN1zmqjUZbpIzLQWr2Ls5jH3hgV4IWgAJII4ysg6wSX8+Rv3Z0iEs5eAi1r17RMmwlqGmiZZVHstZdjr0vFrW+MbBnsuloOl3EdbndX6w3s+C0UZJd3zsXRrW+SvVYJHiUBJeJcCCLWid+LhrC1qXFHfm1+3dHz9/2aBVsqwkqRdtlSyrxAWdw0y2tyOJQtkGPfyMEJOaFSpguTzgPOmzqVIK6teGfV+zCi4PPdesyqacxsLK4vYNps1k+3JkK9Mk7DL2PljSetVfdX7oEHqv4BSzQ0PpEo4Bq39RrgxgwRa0TnzctXUqvaQoc4fv/sR5vvRyLaplVB6yzmlZWee0VAFfrGbsmikgoNYyafjZsil6DFRNdPhks7mg7h5cBbVF3i8LJ9pKDfuK/mo50l81UO2N9Ff90HOnHlhdWNdfZf5q3CHsS8JNpTtcUcCCLWid6NidOB5/Znrch3Ei48ys4tbTU97z8b2r8nzFSRSJMM6uRu6BvcVM/l4XB5V7k78uK+i7ZGijU6PBaFn6ijbafuC5aRzdUvVXRTBqs2mfaVPuEhZJQ8B0EWkCplXDPvFFthAHcCoANS4JrzDpflBsQesEx1237G7tlS8j7rxll3d//Ag6iAe8BWnko9UlS0VgFas173agH3YG1uQKhVRvYkWXDMtQ6Tr6ZGiD0wwr6P833rHsHF2wyl3lcRyzb7Gt4HJ25ZY5w2rSsJq+idhlnh9cdcP8YAatnnCPaT3DSumK6bAeLragdYLjGdvS8LLirltO8TOv/uDlP1Ba/zIlUXDJewIrico9GbvGc4094ws35aPN5d9Q9vl8H5LR7mDQpaqtt/hg8cHQNhXBG9LCIa3gVurfblpVt7uVLp2o9yK21U3PJmdWvf6qdAdj7H2v+hnC3gMr9YB1KVbJRxFb0DrBsSXhLy/uumWXxh/NeIn0zqVCjEKICjAKXLFfeLoJWrEXhAptdIQkrEJFGxxt8ZYPlm7khuqDilO7zhG8IQRDXFnwBrNQoahd6cIJ2w7+V7ZJuEXAjuYHpVNXhrXuYNFgZYK9B6xRhvVIVn4ddWxB6wTHl33Grcd9CCc67jwi0JdscCVeiJ2hs45zqwlNUNA5Z6Y6h5i3S5eRnAJYPoNRmwGqzdmTjwbvLTEKMai9TIpC8oYUBTpBvEE6oVoJ0qHeV81g1mfb7H21StgmYPe7dXfRom4vvFUpBVNcy67UJlkBPpWlqkc0AP1IYwtaJzgef93suA/hRMedR+VcmsAEQbyQOoO3lqZzfWZUWx3lsWbI6sacV/m51qvWyntLCNn91BuIQgqiXjURCIIEQTrBdILxYJd67xYqFLUNVIuE6RJuof5X0uZh5w0pw1opWLKoXCb2gFXAaZN4P4bYgtY2HrNxalrxT776sy/7cUync4e2FVKjWdASMDaxtAFrUy+DAAWslM0CYxRSNHo/yqBICkx40QmhAERBgn4tAUyrQGXavNm5UxmDbVHeahExXcQu/dAZbFotBcfZVRaKEgtftQFWm+XgMfBY49iC1jYe0/FVz7/9sh/DNnqfrE5OxyoRu4pgE8FUdJIG1Xz5nMfcciw2zUEBiKj3peQ0QfLPKFBJUpCUqGBluoTplGg3XkHLdFnVvtKsyiy7QdVeVniNjfouBlawDljHDFYltqC1jW1cZhifSEYwHWCk94tPNvWbp0eW8T1pT9INPgWMxOvvKXgV3/nRLWSw6hISFSyNz6DV6ICzXWVX0S5gVjqK0xPtxahvM7vqOaqN7AqOlXC/WEg6ZtTcxja2sY1HEltj8W1sYxsnKragtY1tbONExRa0trGNbZyo2ILWNraxjRMV2+7hCQ4ReSuwOu7jeJi4EfjUcR/Ew8Q0pfTs4z6IbRwutqB1smOVUnrBcR/EQ4WI/PFJOMbjPoZtHD625eE2trGNExVb0NrGNrZxomILWic7/vVxH8AhYnuM2zjS2Crit7GNbZyo2GZa29jGNk5UbEFrG9vYxomKLWidwBCRbxSRN4vIW0TkVSJy+aZQRxwi8uUi8i4RuUdEvu+4j2czROSJIvIKEXm7iLxNRL77uI9pG4eLLad1AkNEPh94R0rpARH5CuAHUkqfc9zHVUJELPBu4KXAh4DXAl+fUnr7sR7YKETkNuC2lNLrReQU8Drgf7iWjnEbB8c20zqBkVJ6VUrpgfy/rwYu38nuaONFwD0ppfellFrg5cCfOuZjWouU0kdTSq/PX58H3gE84XiPahuHiS1onfz4duA/HvdBbMQTgPtG//8hrmFAEJEnA88F/uiYD2Ubh4jtGM8JDhH5YhS0vvC4j+WkhojsAv8B+CsppXOX+DBbjuVo4lCbh7eZ1gkJEflLIvLGfHu8iHwW8OPAn0opffq4j28jPgw8cfT/t+fvXVMhIhUKWD+bUvql4z6ebRwutkT8CQwRuQP4PeCbU0qvOu7j2QwRcSgR/yUoWL0W+IaU0tuO9cBGISIC/Fvg/pTSX7nMhztRH6IYE/fev+AT5xvOzCruumUXfTmOPQ51EFvQOoEhIj8OfBVQdrr7a81JQUT+JPDPAAv8RErpB4/3iNZDRL4Q+APgLeg2QYC/lVL6rUt4uGv6QxRi4mPnVrzh3gd45bs/ySve9Uk+eb7p//3W01O+6yVP5Zs/78kYc6zgtQWtbWzjKsWxfojOLjped+/9vOOj57nnE3t86IEFn9prOb/yNF1g2QV83qhzaup4yV038YVPv5HHXzfjY2dX/PIbPswfvu/TfMHTb+BHvvH5nJlVx/WnbEFrG9u4SnHVP0QpJV75nk/xo7//Xl79/k/3qwhvOzPljuvn3HRqwqlpxbQyzGvL46+b8cxbT/PZt5/BWXPBY/3ca+7jf/21t/LsJ5zhp7/9c9idHEuPbgta29jGVYqr+iFadYG//otv5tff9BFuPT3la174RD7vqTfw7Cec5tT00rOk337rx/hL/+71vOjJ1/NT3/4iKnvV+3Rb0NrGNq5SXLUP0X7j+eafeA2v++ADfO9L7+IvvOSpTJw9ssf/xdd9iL/2C2/iO178FL7/Zc86ssc9ZBwKtLY6rW1s44RESom/8R/e/P+2d+/BUVV3HMC/v91NSFgSAuQFCSGBhBBEwfCQ8qqCrYBFGcQpYNXWR0eLTp2pb1sHnVqf7ThOHR8VRKc4WkVbtTx8AYoMEEQChCRgHiQgGxOSkISQ969/7GYaIyT7uLt3L34/Mxl2l3PP/U1m8p17zp57Lr6uqMPzK3Jx5UXDDT/H0smpyK+sxz++KENu2hAsuND4cwSK67SILGLtjnJ8uP8E7r4iOyiB1e1PvxiPiSPjcN/6/ThefyZo5/EXQ4v8IiKrRORuz+tHReRyP/vhbgteqKxtxhMbizB3XCJumzMmqOeKdNjw3LJJ6FLgrje/RkdnV/8HhRBDiwKmqg+r6id+Ht4B4A+qOh7AdAArRSTkkynh7rH/FsImgj8vnhCStVSjhjnx58UTkFdeh2c+Ohz08/mCoUVeE5GHROSwiGwHkN3j87UistTzulxEHvfcbrRHRHJFZLOIlIjIbb375G4L/dt+pAabCly4Y24mRsRFh+y8iy9OwXWXpOHFbSXYdPBEyM7bH4YWeUVEJgNYBmASgIUApvbRvEJVJ8G94nwtgKVwX0U90s850sHdFr5HVfH0R8VIiYvGzbMyQn7+hxe557fuemsf8sprQ37+s2FokbdmA3hPVZs9uyG830fb7v87AGCXqjaqajWAVhGJO9sBBu22cN7Zerga+ZX1uHNuJqIijFva4K0BDjtW3zgFI+KicdOredhXWR/yGnpjaFEwdN/Y1tXjdff7Hyyz4W4LZ6eqePaTI0gdEo0luebt8xg/aADW3XIJ4pwRWP7yTnxaWGVaLQBDi7z3OYDFIhLt2Z54kRGdenZbWA339tF/M6LP88UXR2qQX1mPlZdlItJh7p/q8MHRWH/7DGQmDsKtr+/Bm7srTKuFoUVe8UyWvwUgH+6dUvMM6nomgOsBzO2xX9hCg/q2tBe2liA5NgpLcsPje4nEmCi8+dvpmJ2VgPvfPYDnt3xjSh28jYcocIb/Ee2rrMfi57/EQwtzcOuc0UZ3H5D2zi7c83Y+/r3vW6xaNB6/nmnYFwS8jYfIql7cWoLYKAeWX5Jmdik/EGG34ZlrJ6K5rROrPjiExNgoLAzh7T4cHhKFmWJXIzYVuHDjjHSztojpl8Nuw3PLL8bFaXG45+18lNWcDtm5GVpEYea5z47AGWk3ZV2WL6Ii7Pj7ilxEOGxYuW4vWto7Q3JehhZRGDlS1YgNB07gxhnpiBsYaXY5/UqJi8Zfr52IQycaQjYxz9AiCiNPbirGwAg7bpkdXpPvfZmXk4QluSl4YWsJilzBXxfM0CIKE1uKvsMnhVW4c14WhjrD/yqrpz9eOR6x0RF44N0DCPaKBIYWURhoae/EIx8UYHSCEzcZt4QgZIY6I3H/gnH4uqIeH+4P7s3VDC2iMPDExiKUn2zGo1dNMH31u7+uyU3FuOQYPLW5CK0dwZuUt+Zvh+g88lGBC2t3lOOmmRmYlRVvdjl+s9sEDy7MQWXtGazbGbzbfBhaRCY6VteMe97ZjwkpsbhvQXb/B4S5OWMTMH30ULz0eUnQrrYYWkQmae3oxMp1e9GliudX5Br6VB0z3XFZFqoaWvHOV8eC0j9Di8gkT20qRv6xU3h66USMGuY0uxzDzMwchokj4/DitpKg7C/P0CIywe6yWqz5sgy/mp6G+ROSzS7HUCKClZeOQWXtGWw46DK8f4YWUYidaevEve/kI3VINB5YkGN2OUFxeU4SMuKdWLO9zPC+GVpEIfbCthKUn2zGk0sugjNMb4gOlM0m+M3MdOyrrMfeijpj+za0NyLqU2VtM17aVoJFE0dgRqZ1lzd445rcVMREObDa4KsthhZRCP1lg/v5hQ8sGGd2KUHnHODAsqkjsfmgC65TLYb1y9AiCpFdpSex8aALt186JqTPLzTT9dPT0amKNwzcU56hRRQCXV2KxzYUYvjgKNxqoR0cApU2bCDmZifijV0VaOswZvkDQ4soBP6Tfxz7j53CvfOzER15fiwi9dYNM9JR09SKjQY9pZqhRRRkLe2deHpTMS5MGYyrJ4bHk3VCaXZmPNKHDcS6XcYMERlaREG25ssyfHuqBQ8uzIHN5tUDZ84rNpvgl1PTsLusFiXVTYH3Z0BNRHQOtafb8MKWElyek4SfjBlmdjmmuWZyChw2wVt5lQH3xdAiCqLV20vR1NaB++ZbfweHQCTGRGFeTiLWf3Us4Al5hhZRkJw6047XdxzFggnJyEqKMbsc0y2bmoaTp9uw7XB1QP0wtIiC5J87j6KxtQO/uzTT7FLCwqyseMRGObC5ILCbqBlaREHQ0dmFV78sx6XZCZiQMtjscsJChN2GeTlJ+LSwKqAtaxhaREGwo+QkappasXxa+D3W3kxXXJCEuuZ27C6v9bsPhhZREHyQ/y1iBjjw07EJZpcSVuaMTcAAhw0fFVT53QdDi8hgrR2d2FTgws8vSEZUxI9r9Xt/BkY6MGdsAj4+VOX38xEZWkQG21ZcjcaWDlw1aYTZpYSl2VnxOF5/Bsfrz/h1PEOLyGCbC6oQNzACM37Ei0n7kps2BADw1VH/NgdkaBEZLK+8FtPShyLCzj+vsxmXHANnpJ2hRRQOqhpaUFHbjGkZQ80uJWw57DZMSovDnnKGFpHp8jxf5U9NZ2j1ZfKooShyNaCptcPnYxlaRAbKK6tFdIQd40fEml1KWJsyagi6FPjaj4deMLSIDJRXXofcUXGcz+rHpLQ4iMCvISJ/s0QGaWhpR6GrAVNGcWjYn9ioCIxJGISCbxt8PpahRWSQ/Mp6qAJT0oeYXYolZCfH4HBVo8/HMbSIDFJ4wn3VcMEI3iDtjeykGFTUNqO5zbfJeIYWkUGKXI1IiBmAoc5Is0uxhLGePcaOVPm2BTNDi8ggxa5GjEvmZn/eyvb8ropdvg0RGVpEBujo7MKR75qQzR1KvZY2dCCiImwo9nFei6FFZIDyk81o6+jCuOFcn+Utu02Qlej7ZDxDi8gA3UMcDg99MzYphsNDIjMUuxpgEyAzcZDZpVhKdvIgfNfYirrTbV4fw9AiMkCRqxHp8U5u+uej7GT3cNqXeS2GFpEBiqv4zaE/xiQ4AQBlNae9PoahRRSglvZOVNQ2IyuRoeWrEYOjEemwMbSIQunoyWaoAqM9Vw3kPZtNkDHMidJqhhZRyJRWu1d0j47nJLw/MuKdKKvxflU8Q4soQKWeoU0Gr7T8kh7vREVts9ftGVpEASqtPo2k2AEYNMBhdimWNDreifZO7x8nxtAiClBpTROHhgHw9QqVoUUUoNLq05yED0BGPEOLKKROnWn3+Q+P/m+YMxIxUd4PrRlaRAYYk8Dhob9EBKN9CH2GFpEBODwMjC9XqgwtogBF2m1IHTLQ7DIs7aLUOK/bMrSIAjQ1YwjsNjG7DEu7aVaG121F1fv1EUR0VvwjMoZXyc8rLSKyFIYWEVkK7zsgChwntEKIV1pEZCkMLSKyFIYWEVkKQ4uILIUT8UQBEpGDAFrMrqMf8QBqzC6iH1GqOqG/RgwtosC1qOoUs4voi4jssUKN3rTj8JCILIWhRUSWwtAiCtzLZhfghfOmRt4wTUSWwistIrIUhhZRAETkOhHZLyIHRGSHiEw0u6beRGS+iBSLyDcicr/Z9fQmIiNFZIuIHBKRAhH5fZ/tOTwk8p+IzABQqKp1IrIAwCpVvcTsurqJiB3AYQA/A3AMQB6A5ap6yNTCehCR4QCGq+peEYkB8BWAxeeqkVdaRAFQ1R2qWud5uxNAqpn1nMU0AN+oaqmqtgF4E8DVJtf0Pap6QlX3el43AigEkHKu9gwtIuPcDGCj2UX0kgKgssf7Y+gjEMwmIukALgaw61xtuCKeyAAichncoTXL7FqsSkQGAVgP4C5VbThXO15pEflIRFaKyD7PzwgRuQjAKwCuVtWTZtfXy3EAI3u8T/V8FlZEJALuwFqnqu/22ZYT8UT+E5E0AJ8BuEFVd5hdT28i4oB7In4e3GGVB2CFqhaYWlgPIiIAXgNQq6p39dueoUXkPxF5BcA1AI56PuoItxuTRWQhgGcB2AGsUdXHzK3o+0RkFoAvABwA0OX5+EFV3XDW9gwtIrISzmkRkaUwtIjIUhhaRGQpDC0ishSGFhFZCkOLyMJEZJWI3O15/aiIXO5nP1EisltE8j07LTxibKXG4W08ROcJVX04gMNbAcxV1SbP6vTtIrJRVXcaVJ5heKVFZDEi8pCIHBaR7QCye3y+VkSWel6Xi8jjnluN9ohIrohsFpESEbmtd5/q1uR5G+H5CctFnAwtIgsRkckAlgGYBGAhgKl9NK9Q1UlwrzZfC2ApgOkAzjr0ExG7iOwD8B2Aj1X1nDstmImhRWQtswG8p6rNnp0Q3u+jbff/HQCwS1UbVbUaQKuIxPVurKqdnpBLBTBNRPp9cKoZGFpE569Wz79dPV53vz/nfLaq1gPYAmB+0CoLAEOLyFo+B7BYRKI9WxMvMqJTEUnovvoSkWi4t2cuMqJvo/HbQyIL8eyj/haAfLjnnvIM6no4gNc8e8rbAPxLVT80qG9DcZcHIrIUDg+JyFIYWkRkKQwtIrIUhhYRWQpDi4gshaFFRJbC0CIiS2FoEZGl/A/6tLyGhCJesQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "posterior_samples = posterior.sample((5000,))\n", - "\n", - "fig, ax = pairplot(\n", - " samples=posterior_samples,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - " upper=['kde'],\n", - " diag=['kde'],\n", - " figsize=(5,5)\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The 1D and 2D marginals of the posterior fill almost the entire parameter space! Also, the Pearson correlation coefficient matrix of the marginal shows rather weak interactions (low correlations):" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWYklEQVR4nO3df6xcZZ3H8fen97Y0u6wWKCmI2EJokBpWkAYwRGUBtRBDu4rabpTiQroY3F1FDSKJbHBJym4iiz+xgQooCygi1ohhkR+LRmEpbKH8WOQKKq2VSvmhhFK4vd/94zxThunMnXM7Z849c+7nZU46c84z85yJuV+eX+f5KiIwM+u3aZN9A2Y2NTjYmFkpHGzMrBQONmZWCgcbMyuFg42ZlcLBxqymJK2WtFnSgx2uS9KXJY1IekDS25quLZf0WDqWF3E/DjZm9XUFsGic6ycC89OxAvgGgKQ9gfOBo4AjgfMl7dHrzTjYmNVURNwJPDNOkcXAVZG5C5glaV/gvcAtEfFMRDwL3ML4QSuX4V6/wMyK85f7z4ztL43lKrvt6VceAl5qOrUqIlZNoLr9gCeb3m9I5zqd74mDjVmFjL00xgHvn52r7P+t2vRSRCzs8y0Vxt0osyoRTJumXEcBNgL7N71/YzrX6XxPHGzMKkbKdxRgDXBqmpU6Gng+IjYBNwPvkbRHGhh+TzrXE3ejzCpEwLSCmgCSrgGOBWZL2kA2wzQdICIuBW4CTgJGgBeBj6Vrz0j6InBP+qoLImK8geZcHGzMqkQwNFxMsyUilnW5HsBZHa6tBlYXciOJg41ZxaimgxsONmYVIsG0ggZkqsbBxqxi3LIxs1IUNUBcNQ42ZhUiuWVjZiUZGvKYjZn1meRulJmVQqiYRxEqx8HGrErcsjGzsniA2Mz6TvIAcVtp+8DrgHnAb4APpZ29WsttB9ant7+LiJN7qdeszurajer1Z30OuDUi5gO3pvftbI2Iw9LhQGPWgQBNU65j0PQabBYDV6bXVwJLevw+s6ktDRDnOQZNr2M2c9JmOwB/AOZ0KDdT0lpgFFgZETe2KyRpBdku72hYR8yYNXWGlA7+iwMn+xZKs23W1sm+hVI9+sDvno6IvfOWr+lzmN2DjaSfAvu0uXRe85uICEnR4WvmRsRGSQcCt0laHxG/bi2UNmteBTBz7xkxb0m+vVjr4OYjLp/sWyjNyOK2aYxq6x1vOPO3ectmm2fVM9p0DTYRcUKna5KekrRvRGxKKSA2d/iOjenfxyXdARwO7BRszKa8AjfPqppee35rgEa2vOXAD1sLpH1Md0uvZwPHAA/3WK9ZLQkxTfmOQdNrsFkJvFvSY8AJ6T2SFkq6LJU5BFgr6X7gdrIxGwcbs3YKzq4gaZGkR1OK3Z1miyVdLGldOn4l6bmma9ubrq3p9af1NAIbEVuA49ucXwuckV7/Aji0l3rMpooix2wkDQFfA95NlmjuHklrmv9jHxGfair/j2RDHA1bI+KwQm4Gp3Ixq5xpmpbryOFIYCQiHo+Il4FryZardLIMuKaAn9CWg41ZlShfFypn6yd3Gl1Jc4EDgNuaTs+UtFbSXZKW7OIv2mHqLGQxGwAChodytwFmp/VrDRPN9d1sKXB9RGxvOpdryUpeDjZmFZJtnpU72DzdJdf3RNLoLqUlh1TRS1bcjTKrlEK7UfcA8yUdIGkGWUDZaVZJ0puBPYBfNp0rfMmKWzZmVVJg3qiIGJX0CbI83UPA6oh4SNIFwNqIaASepcC1KUNmwyHANyWNkTVKel6y4mBjViFFP64QETeR5fRuPveFlvf/0uZzhS9ZcbAxqxKJoaGhyb6LvnCwMauQKf0gppmVy8HGzPpOIu/q4IHjYGNWKfkfshw0DjZmFTOI20fk4WBjViESDA97NsrM+qyxeVYdOdiYVYk8G2VmJXGwMbO+E576NrMyyFPfZlYCIYaHpk/2bfRFIe21HDu47ybpunT9bknziqjXrG4EDGko1zFoeg42TTu4nwgsAJZJWtBS7HTg2Yg4CLgYuKjXes1qSWLatKFcx6ApomWTZwf3xcCV6fX1wPFSTRcTmPVomoZyHYOmiDGbdju4H9WpTNo97HlgL+DpAuo3qw2hiexBPFAqNUAsaQWwAmB498GL3Ga9ksT0oRmTfRt9UUSwybODe6PMBknDwOuBLa1flNJQrAKYufeMaL1uVn+q7TqbIn5Vnh3c1wDL0+tTgNtaNlc2MxqpXIobIM4xU3yapD825fQ+o+nackmPpWN562cnqueWTc4d3C8Hvi1pBHiGLCCZ2U5U2LR2nlzfyXUR8YmWz+4JnA8sBAK4N3322V29n0LGbLrt4B4RLwEfLKIuszor+HGFHTPFAJIaM8V5UrK8F7glIp5Jn70FWEQPucDr2Tk0G1gTWmczO+XibhwrWr4sb67vD0h6QNL1khrjr7nzhOdVqdkos6lugrNR3dLv5vEj4JqI2CbpH8jWwx3X43e25ZaNWYU0ulF5jhy6zhRHxJaI2JbeXgYckfezE+VgY1YlxT6u0HWmWNK+TW9PBh5Jr28G3pNyfu8BvCed22XuRplVigp7FCHnTPE/SToZGCWbKT4tffYZSV8kC1gAFzQGi3eVg41ZhWQZMYvrcOSYKT4XOLfDZ1cDq4u6Fwcbs0opbp1N1TjYmFWIJIb9bJSZ9Zv3IDazckgMDeDGWHk42JhVSNaycbAxs76r7xYTDjZmFSLE8DQPEJtZv0nI3SgzK4PHbMys74SYhoONmZXALRsz6zsV+CBm1TjYmFWKGJJno8yszySvszGzktS1G1VICO0lN42ZNZNzfXfSS24aM3st4UV94+klN42ZtfA6m87a5Zc5qk25D0h6J/Ar4FMR8WRrgZT3ZgXAfnvO4Y4jvlfA7Q2GY++dOjn8PnLwoZN9C5UlFftslKRFwCVkexBfFhErW66fDZxBtgfxH4G/j4jfpmvbgfWp6O8i4uRe7qWsYe8fAfMi4q+BW8hy0+wkIlZFxMKIWLjX7rNKujWzKiluzKZpiONEYAGwTNKClmL/CyxMf5vXA//WdG1rRByWjp4CDRQTbHrJTWNmr5GN2eQ5ctgxxBERLwONIY4dIuL2iHgxvb2L7O+3L4oINr3kpjGzJiIbs8lzUFz63YbTgZ80vZ+ZvvcuSUt6/W09j9n0kpvGzFpNaFFfEel3s1qljwALgXc1nZ4bERslHQjcJml9RPx6V+soZFFfL7lpzOxVBQ8Q50qhK+kE4DzgXU3DHUTExvTv45LuAA4HdjnY1HNdtNkAE0O5jhzyDHEcDnwTODkiNjed30PSbun1bOAYelzO4scVzCqkyKe+cw5x/DuwO/A9SfDqFPchwDcljZE1Sla2Wag7IQ42ZpVS7BYTOYY4TujwuV8AhS6IcrAxqxjVdHTDwcascjTZN9AXDjZmFeI9iM2sRO5GmVkJ5G6UmfWfkLcFNbNyuGVjZn3nAWIzK427UWbWZ8IDxGZWCnkFsZmVxS0bMyuBWzZmVgLl3atm4DjYmFVINkDslo2ZlcCzUWbWfxL4cQUzK0NdWzaFhFBJqyVtlvRgh+uS9GVJI5IekPS2Iuo1q59snU2eI9e3SYskPZr+9j7X5vpukq5L1++WNK/p2rnp/KOS3tvrLyuqvXYFsGic6ycC89OxAvhGQfWa1U5R2RVypt89HXg2Ig4CLgYuSp9dQJaN4S1kf9tfV840nJ0UEmwi4k6y5HOdLAauisxdwKyWLJlmxquPK+T5Xw5d0++m91em19cDxytLs7AYuDYitkXEE8BI+r5dVtZIVK40oJJWNFKJbnnhuZJuzaxKNIGjkPS7O8pExCjwPLBXzs9OSKUGiCNiFbAK4K1z3xyTfDtm5Yt05FNY+t0ylNWyyZUG1MwCRb4jhzx/dzvKSBoGXg9syfnZCSkr2KwBTk2zUkcDz0fEppLqNhssYzmP7rqm303vl6fXpwC3RUSk80vTbNUBZJM7/9PDryqmGyXpGuBYsj7kBuB8YDpARFxKlpHvJLJBpheBjxVRr1kt5Wu15PiaXOl3Lwe+LWmEbJJnafrsQ5K+S5bfexQ4KyK293I/hQSbiFjW5XoAZxVRl1mtBajA0coc6XdfAj7Y4bMXAhcWdS+VGiA2MyYyQDxQHGzMqqagblTVONiYVU09Y42DjVmlBHmntQeOg41Z1dQz1jjYmFWOg42ZlcLdKDMrQ5HrbKrEwcasSib2IOZAcbAxq5SAsXpGGwcbswoR9e1G1XMbdzOrHLdszKrGs1Fm1nceIDazssgDxGZWinrGGgcbs0oJPPVtZmUIoqYDxJ76Nqua4jY870jSnpJukfRY+nePNmUOk/RLSQ+ltNkfbrp2haQnJK1Lx2Hd6nSwMauQCIixyHX06HPArRExH7g1vW/1InBqRDRS8P6HpFlN1z8bEYelY123Ct2NMquY2N5jsyWfxWQZUSBLv3sHcM5r7iPiV02vfy9pM7A38NyuVFhIy0bSakmbJT3Y4fqxkp5vanJ9oV05symvMUCc5+iefnc8c5pyt/0BmDNeYUlHAjOAXzedvjB1ry6WtFu3Cotq2VwBfBW4apwyP4uI9xVUn1lNTWiAeNz0u5J+CuzT5tJ5r6kxIqTOT2RJ2hf4NrA8IhrNrnPJgtQMspTZ5wAXjHezReWNulPSvCK+q2HbrK2MLG7bUKqljxx86GTfQmm+85/rJ/sWqq2gXlREnNDpmqSnJO0bEZtSMNncodzrgB8D50XEXU3f3WgVbZP0LeAz3e6nzAHit0u6X9JPJL2lXQFJKxpNwue2vFDirZlVR0TkOnrUnHZ3OfDD1gIpZe8PgKsi4vqWa/umfwUsAbq2DMoKNvcBcyPircBXgBvbFYqIVRGxMCIWztpr95JuzaxCJjZm04uVwLslPQackN4jaaGky1KZDwHvBE5rM8V9taT1wHpgNvCv3SosZTYqIv7U9PomSV+XNDsini6jfrNBUsZsVERsAY5vc34tcEZ6/R3gOx0+f9xE6ywl2EjaB3gqDUQdSdai2lJG3WaDJKKQNTSVVEiwkXQN2Zz9bEkbgPOB6QARcSlwCvBxSaPAVmBp1HVNtlmvSllmU76iZqOWdbn+VbKpcTProq7/HfYKYrMq8VPfZlaWkh5XKJ2DjVmVBB6zMbMyeDbKzMriAWIz67u0n00dOdiYVY2DjZn1W0R4NsrMyuFgY2b95zEbMyuHu1FmVoYAxhxszKzPIoKxV7ZP9m30hYONWcW4G2Vm/VfjbpQzYppVSr5smL3OWOVJv5vKbW/af3hN0/kDJN0taUTSdWlz9HE52JhVSWTdqDxHj/Kk3wXY2pRi9+Sm8xcBF0fEQcCzwOndKnSwMauQAGJsLNfRo8VkaXdJ/y7J+8GUvuU4oJHeJdfnPWZjViURRP7ZqNmS1ja9XxURq3J+Nm/63ZmpjlFgZUTcCOwFPBcRo6nMBmC/bhU62JhVSUxoNqqM9LtzI2KjpAOB21KuqOfz3mCznoONpP3JcnzPIWsFroqIS1rKCLgEOAl4ETgtIu7rtW6z+okiukjZNxWQfjciNqZ/H5d0B3A48H1glqTh1Lp5I7Cx2/0UMWYzCnw6IhYARwNnSVrQUuZEYH46VgDfKKBes/oJYHvkO3qTJ/3uHpJ2S69nA8cAD6c0TLeTpWjq+PlWPQebiNjUaKVExJ+BR9i5/7aYLF9wpOTksxq5gs3stUoaIM6TfvcQYK2k+8mCy8qIeDhdOwc4W9II2RjO5d0qLHTMRtI8smbW3S2X9gOebHrfGFDahJntUNZ+NjnT7/4COLTD5x8HjpxInYUFG0m7k/XlPtmc23uC37GCrJvFnP32LOrWzAZHMJHZqIFSyDobSdPJAs3VEXFDmyIbgf2b3rcdUIqIVRGxMCIWztpr9yJuzWzARFndqNL1HGzSTNPlwCMR8aUOxdYApypzNPB80xy/mTWUt4K4dEV0o44BPgqsl7Qunfs88CaAiLgUuIls2nuEbOr7YwXUa1ZDxU19V03PwSYifg6oS5kAzuq1LrPaa0x915BXEJtVSEQw9vJo94IDyMHGrEom9rjCQHGwMauSgBh1sDGzvnN2BTMrQbhlY2aliHCwMbMSBIxtq+fjCg42ZlVS0oOYk8HBxqxCPGZjZuXwmI2ZlcXdKDPrP3ejzKwMEcHYtno+G+UkdWZVksZs8hy9yJN+V9LfNKXeXSfpJUlL0rUrJD3RdO2wbnU62JhVSYXS70bE7Y3Uu2QZMF8E/qupyGebUvOu61ahg41ZlaQxm363bJh4+t1TgJ9ExIu7WqGDjVmllNONIn/63YalwDUt5y6U9ICkixv5pcbjAWKzComxCT2uMG6u74LS75JyvB0K3Nx0+lyyIDUDWEWWR+qC8W7WwcasUib0uMK4ub6LSL+bfAj4QUS80vTdjVbRNknfAj7T7WbdjTKrkvLGbLqm322yjJYuVCOjbcqusgR4sFuFbtmYVUl5jyusBL4r6XTgt2StFyQtBM6MiDPS+3lkOd/+u+XzV0vamyzZwTrgzG4V9hxsJO0PXEU2wBRk/cZLWsocSxY5n0inboiIcft3ZlNRlLQHcZ70u+n9b8hSZbeWO26idRbRshkFPh0R90n6K+BeSbc0JSBv+FlEvK+A+sxqzc9GdZAGijal13+W9AhZJGwNNmbWRUQwOubNs7pK/bvDgbvbXH67pPuB3wOfiYiH2nx+BbAivX3hHW8489Ei7y+n2cDTk1DvZJlKv3eyfuvciRTeHm7ZjEvS7sD3gU9GxJ9aLt8HzI2IFySdBNwIzG/9jrRGYFXr+TJJWjvedGLdTKXfOwi/NQjGahpsCpn6ljSdLNBcHRE3tF6PiD9FxAvp9U3AdEmzi6jbrG7GInIdg6aI2SgBlwOPRMSXOpTZB3gqrVQ8kizIbem1brM6qmvLpohu1DHAR4H1ktalc58H3gQQEZeSPcT1cUmjwFZgaURlQ/OkduMmwVT6vZX/rRH17Uapun/zZlPPQcNz4kuv+3Cusouf/cq9VR+DauYVxGaVEp6NMrP+CxjIwd88/CBmE0mLJD0qaUTSTjuX1Ymk1ZI2S+r6AN2gk7S/pNslPSzpIUn/PNn31FFkA8R5jkHjYJNIGgK+BpwILACWSVowuXfVV1cAiyb7JkrSeKRmAXA0cFZ1/7+N2gYbd6NedSQwEhGPA0i6lmzrxFo+dhERd6YV37U3SI/UBPhxhSlgP+DJpvcbgKMm6V6sT7o8UjPpwgPEZoOvyyM11RBe1DcVbCTbJKjhjemc1UC3R2qqos6zUQ42r7oHmC/pALIgsxT4u8m9JStCnkdqqqO+K4g9G5VExCjwCbId5B8BvttuG4y6kHQN8EvgYEkb0vaQddV4pOa4pgyOJ032TbWTtWw8G1V76Yn0myb7PsoQEcsm+x7KEhE/J9srt/IigldqOhvllo1ZxZTRspH0wbTAcSxtct6pXNuFrpIOkHR3On+dpBnd6nSwMauYkvazeRB4P3BnpwJdFrpeBFwcEQcBzwJdu+EONmYVEiWtII6IRyKi27a7Oxa6RsTLwLXA4jTgfhxwfSqXJ1e4x2zMqmQDz918dtyQdxfLmeOl3y1Ap4WuewHPpUmVxvmd0r20crAxq5CIKOx5tfFyfUfEeBkw+8LBxqymxsv1nVOnha5bgFmShlPrJtcCWI/ZmFknOxa6ptmmpcCatKXv7WTb/UL3XOGAg43ZlCTpbyVtAN4O/FjSzen8GyTdBF0Xup4DnC1phGwM5/KudXoPYjMrg1s2ZlYKBxszK4WDjZmVwsHGzErhYGNmpXCwMbNSONiYWSn+H4jRLO0e+4NOAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "corr_matrix_marginal = np.corrcoef(posterior_samples.T)\n", - "fig, ax = plt.subplots(1,1, figsize=(4, 4))\n", - "im = plt.imshow(corr_matrix_marginal, clim=[-1, 1], cmap='PiYG')\n", - "_ = fig.colorbar(im)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It might be tempting to conclude that the experimental data barely constrains our parameters and that almost all parameter combinations can reproduce the experimental data. As we will show below, this is not the case." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Because our toy posterior has only three parameters, we can plot posterior samples in a 3D plot:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "rc('animation', html='html5')\n", - "\n", - "# First set up the figure, the axis, and the plot element we want to animate\n", - "fig = plt.figure(figsize=(6,6))\n", - "ax = fig.add_subplot(111, projection='3d')\n", - "\n", - "ax.set_xlim((-2, 2))\n", - "ax.set_ylim((-2, 2))\n", - "\n", - "def init():\n", - " line, = ax.plot([], [], lw=2)\n", - " line.set_data([], [])\n", - " return (line,)\n", - "\n", - "def animate(angle):\n", - " num_samples_vis = 1000\n", - " line = ax.scatter(posterior_samples[:num_samples_vis, 0], posterior_samples[:num_samples_vis, 1], posterior_samples[:num_samples_vis, 2], zdir='z', s=15, c='#2171b5', depthshade=False)\n", - " ax.view_init(20, angle)\n", - " return (line,)\n", - "\n", - "anim = animation.FuncAnimation(fig, animate, init_func=init,\n", - " frames=range(0,360,5), interval=150, blit=True)\n", - "\n", - "plt.close()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "HTML(anim.to_html5_video())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Clearly, the range of admissible parameters is constrained to a narrow region in parameter space, which had not been evident from the marginals." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the posterior has more than three dimensions, inspecting all dimensions at once will not be possible anymore. One way to still reveal structures in high-dimensional posteriors is to inspect 2D-slices through the posterior. In `sbi`, this can be done with the `conditional_pairplot()` function, which computes the conditional distributions within the posterior. We can slice (i.e. condition) the posterior at any location, given by the `condition`. In the plot below, for all upper diagonal plots, we keep all but two parameters constant at values sampled from the posterior, and inspect what combinations of the remaining two parameters can reproduce experimental data. For the plots on the diagonal (the 1D conditionals), we keep all but one parameter constant." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABtpElEQVR4nO39eZBl2X3fiX3OOXe/b8utMmvr7qruQi+AQIBLN0Q0uAIjkiJNOkzNaAkqFNbMWLYUI4Ul2wprhqYUZsyE7dDIMZ6xR5YYlMLykNJIsjUayRSapEg0IXSDBAkC6G50F6qX2nJ9+da733v8x3lZVWj0Uktmvsys84nI6Fxe3ncyq/P7zvnd7+/7E1prLBaL5bgg570Ai8ViuResaFkslmOFFS2LxXKssKJlsViOFVa0LBbLscKKlsViOVY49/E91iOxP4gHvcAfa/85rasKnef7sZ6Hms83/+S+/z0+J//E/P4mhAAhUQtdRBjSLHep2j5l2yFbUFShIO8JqgiqSFP1akRQE3dTumHGcjjl8dYWp7wxTwfXOesMWFUFChhrwSvFKm8Vy9wserwxXmE3j9gat0inHk3i4AwcVCZwx+CNNU4CYb/CSWrc3Qw5mKAnU+r+AJr6Q3+cu/l3uB/RshwR5EIPnabUVrQeWoRSCM9DtFvoOKTq+pQth6KlKFuCKhRUsRGsOm6QcYkXVCxEKcvhhLVwzPmgz5oz5DG3z5qqWZAh3ygq1usObxXLXM0W2cg7bCZtxplvBGvqIhOJMxU4mREsd6JxE407qlBJYQSrP6CeTO9KsO6WYy9a46ykaaAbufNeyqHT9NpIz0WVFU2SoKtq3kuyHCLCcZCtGNFq0fRa1JFL0XEpWpIyFhRtQe1D2dLUcY2IaqJWTivIORWNORMOOesPuOhtsuYMedTRJI3mzSrjrWqV9bLLW9kyN9IuO1nMIAnJMpdm4qImCicFdwIqA3ei8UcNzrTGGabISYYeDGnSbF8FC05ATesv/r9/n5/7pZfmvYy50EQuTRQg4gjhOOaoYHk4EMLssIIAHQXUkUsVOVShpAoEdWAEqw41TdCA3+D4FbFf0PFylvwpS+6UZWfMKTVmSeZEwiPTsFWHrJddtqs2/SJimIeMc58sc6lTB5lJVA4qE6gMnFTjphqV1qi0QiQ5pBnNNEVX5b7/6Md6p7U5yvjCG1toDVf7CecXo3kv6VApFgOc0MHBvPqIaUI9Gs17WZaDRiqk5yKXFtGdmKoTUCx4lJEk7wrK2NSwyo6mDhvolARRSTvKONMasuQnPBFtct7tc97d4Uk3RwrF5TLn7WqJ6+UCb6SrbBUtrk4W6E8j0tSjHnnIVOKOJO4UnAT8YYObavzdEmeUI8cZbPdpshxdFgfz4x/IVQ+Jf/W1m+y1Tv76qxvzXcwcqCJJGTk0LQ8dh4goRLie3XGdcGTgI+II3QppIo86cigjSRVIU8MKoQqNYGm/wQ0qoiCnG2Qs+QmL3pQVZ8yaM2RFpdRoxk3NRt1is2qzUXbplzGDImKU+aSZS5nt7bDM7kqloFKNk2lU2qDSCjnJEUmGzvIDLVUc653Wv/zDmzy52qZqGl54dZM/9+kL817SoVK0JI4jkLWHKCOklMgkpcmFvaN4UhEC2WlDFFJ1Q6rYpeg4FC1JHUAZQxVrqkij4xoV1HTijMUwYSWccC7YZdUd8pi7zaPOiFXlcbMu2Gl83iqXeTtfZqPocDPpMMwDxtOAcuohUoUzEeZO4YRbRXdvWOGkNWqYIoZjmvGEJs/hAIMYjq1o3Rik/O7bu/zVz32ESVHxSy++ySgr6QQPT0G+aAtqF0SjEFWAoyRO3kVOE+qy2vcCqGW+yCBAtGJ0t03T8im6HlWsTOG9LYxotbWpY8U1Xqsg8EuWoimnoxGn/DEX/E1OOWMuuiMk0G8K3qq6bFUd3sxXuJH12MjabE1jksynnHiIROHs3SlMZ0X3cYMzbXBHBWpaIHZHRrDS7EAFC47x8fD/9/V1AP74x0/zuadXKWvNb31za86rOlzqAOpAmMJrKKkj1xwTgwDpufaYeJIQAjE7FjYtnzp0qWdF9yqY/b/gQ+NrmrBBBDWBX9IKchb9hEV3yqo74pQzZkWN6UoFwLBR7NQttqoOO0WL3SJklAekuUeRO4hcoVKJyoy1QWXmSOikDU5aI5PSHAmTFF2Uh/JCeWx3Wi+8usGlUy0urrR4dEmzGHv8+qsb/NR3nZn30g6NogPKg73XHteXiCrCcRVSCNjapsmyua7Rsg9IhVpaRLRj6m5MsRBQxYqsq0zRPYaio6kDTd2pUXFJFOWstccsBAkX423OeX3Oezs86e4QS8G40VyrQq5XC7yanmWnjHlrush2EjNOfdJhALkpujsTs8PyBuYuoT+ozQ5rkiO3d9FJeqg3gI7lTmuYlrz8Zp/PPrMKgJKCH3nqFL/x2iZl3cx5dYdHHZijQBUye8UV1JFDHXvoVohoxcggmPcyLQ+AcD1kHCFaEU0roI5d6lDNiu57u+3ZLivQyLDC90vaQc5CkLDoJSy7Y9bcIStqjCug0Jrt2mWzbrNeddkpY7bzFoMsZJp75JkHuby1w1K5sTU4mcbJGpy0MkfCJEcnqalhHSLHUrR+6/Utqkbz2adXb33us0+vMsoqfvet3Tmu7HCpW41pzYigbAmKlqBoK8qOR90OEJ02otuxx8RjjIxDZKdN04moOgFlx6VoS4qWoJrtsqpYU7dqdFwRRgW9OGUpTFgLRpzzd3nM2+a80+cxx7TnJFrwTrXA28UK7+RL3Ei7bKRthmlAMvWppw7ORJka1nRWdJ+CN25wRzXOMEeOEhiMqCfTQ7/pcyyPhy+8ssFyy+MT53u3PveZS8t4juSFVzf4o48vzW9xh0mrpFYOaIFoBFoJVCHRSiAaH9FopOug8hyd5faoeIwQrmdqWAs9dBRQLoSm6N5WFG3TnlO27mjPaZn2nMU4udWe81iwzZoz5KK7TVtUgOBG7bFVt7mcr3E973Ez67I+7TDOfJKJTzN1UdPZkfBd7TnesLzdnjMc0+xze87dcux2WmXd8Jvf3ORHnjqFkrd3ELHv8OnHl3jh1Q0eltx7x68QYUUTNOao6EMVzI6KoaQOXZrIR4QhwrP+rWODEAjXQUQhOvRpIo8qVLfc7sbxbtzudaDRQY3rV4R+QcfPWPBSFl3jxVpxRizK6taxcKeO2azabJetmRfLHAmz3KWZud1lLmaOd/O2V3hXaYlMCvQ0RafpgZlHP4xjt9P68pt9xlnFj95xNNzjs8+s8jf++dd5Y3PCR1bbc1jd4bLYSZjmHlOg1C6NIxGNoHFBC3N3yPUkouwhfQ+pNc00sVaIo4wQqHbbNEB3W5SLEVXskPcUZSQoW4KyY8yjZaeGsMaPC5Y7Uzp+xsXWNmveiEf9bT7q3aArS9pScaPS3KjbfDM/w0bZ4UqyzFbaYjcJGY1DmtRBjRROInASgTfUOCkEgxp3XOOOC9TOGD1JqLe3D9zW8EEcu53W51/dwHckn7m0/B1f+9GnjJC98JC447t+RivIcYMKHTSzV16oQ2auaEkVKeOYjwJkHBkrxOx2t+WIIRXSn7nd45Am9qlihyoyETN1KGb/tmaHRbDndjc7rEU/YcmdsuoOWVEjurIkENCva7aaiPWqx0bZYatosZtFjLKAJPNoUgeRzYruqUClGD9WOrM2JCVymqMnCTpJ5ipYcMx2WlprXnh1g+efWCbyvnPpa92AP3K2ywuvbPC/+qEn5rDCw2UlnOCqmqJSDGtJLTVV7gICUQuKErSQqNRFC4FTN4i8QGiNzu1u66ghAx/h++hOi6YdULY9yraiCm73E1aRpoq1ac+JCuKwYCFKOR2OWPSmPOpvc97d4bwzYlEpSt3wRhVztVziWrHIzazLTh7TTyPGiU+RGPOoSgXOVMx6CjXetMFJZubRYYoYJ9SD4dyOhHdyrETr9Y0JV/vpBwrSZ59e5e/8+utsjXNW2v4hru7weSzaYctp0WhB00im0qMsJVpJ0HJWnAdVOmhHIrRG1TXK96g2t+0x8aggFcJ1kN0OOg6pFyLTntN1yNvydtF9rz2nVaHCim7rdnvOI2GfVXdoYmbUlFUl2agbBo3HG/ka14pFbuRdrk17jAqf4eR2e447fo/2nEGFk1So3cS054zGB5LYcD8cq+Ph3rHvR5869b6P+ewzp9AafvO1zcNa1txYdscsuVN6XkrsFwRBCX5DE5jjQ3WHY94U5h105EMYIFzHHhOPCEKZYyFhgI586tAxhfe9gvu3ebEaVFDj+yUtP6frpyx6U5bdsSm8y4S20CgEg8bcKezXMTtlzG4RMSp8prlHlTuQS2Qmbnuxsju9WDXqDrd7kx1sP+G9cKx2Wp9/ZYPvOtflVOf9DZPPnO5wphvwb17Z4N//vvOHuLrD5yn/JotqghSaBsGOG1NVitx1qfAAiXYEopY0jgbhgo5wPAdV1ejxmHownPeP8VAjXA/ZaSG6HeqlNlXski26lJGkbEHRnfUUdhqaqEa1KjrthE6Qc761y+lgxDlvl6f8G6ypCR9xAzbrhG+Wktfy02xXHV6frrGRtdlJI/qjmDJzYOTiTk0/oTcy5lF/qE3RfVLh9KeISUqztU1zSO05d8ux2WltjjP+4Org2wyl74UQgs8+s8qLl7fIyqPziz4I1tSINWfIqjti2Zuy4CfEQYEXVOiwNrutPcd8CGUoqCJFHbvGMR/PHPPWCnH4CHE7eTQybvcqdm8V3c2/mRGsKpj1E4Y1flDQCXJ6fsqKN2HVHbHqDlhTE7qyZrdJ2Woc1usOG1WXjbLDTh4xyELGmU+ZOejUQSXSFN1nIX5Oqk0NKzFRyWIyszVUFeij1WVybHZav/GqOe7tte58EJ99epV/+O/e5ncub7+nNeKkcM6p8MWQRPuM6wApGkZRgBCauhaUpQChqDKBKc6DKhUIkGmAAkRVIfqDI1FgfagQEuE4iG7b1LHaAWXLMRlpsaCKbhfe67BBRBVBVNCJMpbDCUv+lLP+gHPeDo84fc454OJxuWq4WvW4WixxPV9gO2+xnbYYpAFJ4qMTB5nKmbUB43ifGtFyJxVqVCAnKXo4oskPNhfrfjk2ovXCq5uc7YU8tfbh/qvnLi7S8h1eeHXzRIsWQFs2XHS3KbQiUjnTysdTNVoLdhtJrTRl6aLVTLgaaByFLH20I03qaV2bukWSzPvHeWhQ3Q4iCml6LaqWR9l1ybvGPFq0jbWhjDVVpwa/ptXO6IQZK+GUR6M+p7wxT/jrPOLscs6pGDeacaO5Up7iarHEtWKBa0mP3TwyyaOJRzN1cEYKlXOr6G7SR2tU2uDupsjhFD2emJkD9dE8qRyL42Fa1Lx4eYvPPbOKuIujjO8ofvAjK/z6qxs0zdEoHh4E2awwuigrltSEFWfMojel5yW0/BzfL5FBbRzzvnHMf1thPnBoIs845kN7TDwUhLjVorOX7V5HjmmAvjPbPdA0gQa/xvFrYr+g7eUszLxYC86UU2pMW5YEQjFuJP0mYLPqsF216Bcxw8Jku+eZS5MpRC6Rxe1sd1N8b24lj4okR6eZeavrI1N4fzfHYqf1O5e3ycrmQ+tZd/LZZ07xP37tJl+7PuS77uhRPEl8s+yyoqY84UguOkN6MmXa+LTU7QbWgRMyrAWV44BQCC1oXIGsFI0r0DLArzUi8JBFaaJy7VHxwJBhiFzo0Sx1aCLPZLvHirwtTNHdn2W7Rw06rok6GZFfcqY1ZCWYcD7Y5cngJmvOgCfdnEzDRl1xpVpmvezxRrrKzazLVtpiY9wiSz2qkWfGfSUCb2h6Cv1hgzdpcCY1Xj81wyi2d80O64in3h4L0Xrh1Q3avsOzFxbv+nt++EnTm/j5VzZOrGitVz1qLenJXQIBK6pgzRnQaMHIC5j4PloLksyjaAR1KagKCVpQRoCWyFLhpB4KkHGEFoK6Ko/sq+xxRkYRIo5NDSv2b2W7l3tu9+CObPegwQ1L4qC4le2+5o845Y5YcwYsyowSzVgLtuqQ6+Ui22WbjbxNP4/YzULSxKfKFTKTOKlxuzvpLMhvVnh3kgoxzRBJRpOm6PLo1bDezZEXrabRvPDqJj/wkRU85+5Ps73I43sfXeCFVzf4a3/syQNc4fy4ViySOS6xzHnGzViWPn1nFyUaksZnUhtz7TjymABFbYQLASoXgEbWEpW7aAFup4UARJoZI6EVrv1DKkS7Be2YphOa9pxYUcaScq/oHmtqX6OjGhVVhGHBYpjQ81POBANOuwMecXc47yREQpBoTb82QX43ix5bRZvtrEU/jZikPlXqIDKFSgQqEbiJcbu7qcad1DiTEjXKEePpzIt1PFJAjrxoffXagO1Jzufu4q7hu/ncM6v8H//HV0/seLEr6TJDN0SJBldcZ0WmPOpIIrlDrSWlVoSqJKtdXNWwCxSNmBXixaw4D6JWNI5AVjHKUSig3tm1x8R9QsaxqRkudmlaAUXXo+g5lKGk6MzGfcWasmUSG7x2QRzmLMUJ5+MBi96Ux/0Nzrq7POYMkUCiNVfKDu+Ui1wrlngrXWInj9mYtJgkAUXiIsd7I+tn7TnT2UDVpMHbzVHjDDGa0gyGxot1TDjyhfgXXt1AScEPPblyz9+7d+fwpI4X2y0iBlXEVtU2zufGwxWSSGhOqTHLzphld0LPS2n7OYFfIoIa7d+OsrldmBfUoSnME4WIwLeO+f1AKpOLFd4e91UH73K7+9D4oP07st39gq6X0nNN8ugpZ8ySTGhLQQOMG8Vm3Wa76hi3ex7dynYvcwdydcvt/mHZ7kfNPPphHPmd1guvbPJ9jy3Qi7x7/t4LyzGPr8QndrzY9UmXSelTa4FCM3ZDInGdtoQn3ZyaG7RVSta4hKpEotFA6vhUtY9WEi3FLM5GImuXxpV4QqCaBhEE1FsP17CQ/WTP7c5ijyYOTLZ7y0zPybsmLrnomMjsOm5wOgVBWLDWHrMUTDkbDrgUbsyK7jsEAiSCq5VJbHg9O82NvMd62ubGpMM080hHgZmeM5V4I3OX0Btq3KnGm9R4uwVqmiN3BjSTKc14PO9f0z1zpEXraj/hmxtj/tM//vR9X+Ozz6zy979wMseLjTOfRgsip+Cm00WKhjVnSMOUQDX0ZA7OLhtuj1IrprXHpPTQWjDJHGoNopZUufFwlalANBJZuog0QgqBnEboojiSJsOjjMl2Nzn9TRxQx2bcVxnermHtWRv2zKNhlBP7BQtBwrI/4ZQ34qzbZ0WNCQSUQL+WrFc91qsu63mX7Tymn8VMs1m2eyaRe9nud7jd3aTBSWrUOEcmGc1kis6O9l3C9+NIi9Zeg/T91LP2+NzTq/y3v3WF3/rm1omb1JMmPk0j2XEqIsfUn1Yc88q5KEcsyppIJKy7Jjc/bxwmpY8A8syl1FBXgqowxfkyE4BEVA4y80GCmMQghBWte0EII1hxjG5FVG2Ti1XGctZKZdp09nZYIqzxw9K05wQpa8GI096Q826fs2rIoiqJpGKjbrhedbg6u1O4mbfYyWKGaUCWmFwsNVW33O7G6Q7epDH9hOMCOUnQ0/RY7rD2OPKidelUi0eX4vu+xicfWWAx9njhBI4Xq0YedSnZAqTQZLVLS+VMPR8pGh5zJrSl4JK3SSzNq2rVKDadFnnlMFYNuYCycUxjdWWibEAiag/Xk7h51wQHak2TpvaO4odwZ7Z7E4dUCyFFzzRA552ZYMVmqGoTNsh2SRCaYRRnWkOW/SmPB1uccXd5zN1mVZW4QvBmKbleL/BWscKVdIXtvMWNSZdhGpgXr7GLSiXu+LZgeSONmzZ4gxJnbIZRNDu7x3aHtceRLcQP05KXrvTvqtfwg9gbL/abJ3C8mCgEOlMUucM49xnmoRkHVbbZqjpMG0mpNT1ZsagmrDgjem7CwswxH/olyn9vx3wdSqrQoYl8dOgjwgDhnKzj9b6zN1A1CNBRQBPNxn2F7+V2b9B+g+dXBF5p3O5eMst2H7GkJizKAiUEpdZszgaqbpdt+kXMqAzMuK/coc6MF0sWexEzt7Pd1SzbXSQ5JOmJMA8f2Z3We40Ju18++/Qq//3vXePLb/X5/se/M6b5uOKOJLUvKYEBkJcOvlMxrYw/S4mGM84uH3M1gSjwWKfQirZaIK1dfFUhhGaoI2rHoayNDUILU9tyXYEsAxxXoqRE1jXNlGP/P/2BIBUy8JG9LroV3sp2LzqKvGvuzhYdcySsWg10ZtNz2lMWgpSz0YAnok1WnSGXvA1WVcGqCnm9LNiqY76Zn+Fm0eVa1uPapMc490y2e+IgpwpvJJH5XtEdvGmDv1uiktJku4/G1MPRiTjmH1nReq8xYffLrfFir2yeKNFyEoFoNNpVlJ4DQtNPIySa2MnpOqYBek2t4wKrquTsrL617bdotCSvHbLCJQfqTCK0RNRiZj4Fp6UQjYsoA1SrhRSSemCNp9/GTLBEu4Vu3c52v2UcDe+oYYUaHdZ4s2z3hSBlJZiYmBlnyJoz5IwyLwobdcp63b2V7b6Rd9jOWowy/3a2e6pu9xLeynY3Xixneke2e5Yf2Qboe+VIitbemLAf++jat40Ju1/uHC/2n/3k03fVdH0ccFJAz6bvuA6lFowCHyk0gVPScYzDeU0NWVUpF9wWg8YUYLe9Do0WZLVDWrqMhSYJHSptImyqYnZHMZOIxjEzFLMIIQRiPD7SDbWHjXAdYx5tx7dysYxg3XGncDbyqwmN270VZbT9gpVgwil/zDmvzyNunxU15ZSK2GlSrlYe18sFNqqu6SfMW+xmIdPU5GKJROHMst2dhNt3CacNzqRETjLExIys12V1Yv69jqRo7Y0Je5C7hu/ms8+s8pv//Otc3pxw6YSMF/OGGjkTF4SkqgRTN6CqjCnUEQ1pbdp8pu4OMKQnoS2HTJt1fFkihaZqFJ6qqSpF4biUwpmlnQpkJWZ+LhdRxSjPQZUlzXhCM53O9ec/KqiVZXToUy/EFAs+VaRmPixB2Z55sQJN06lwgop2K2W1NWHRT3g82uKUO+Jxb4Mn3BGxkLxVJazXEW+VK7yenaZfxlyb9tjNQkbTgGLkI/K9oruZAu0NjXnU361wJyVqlMH2gCbL0EVxYgQLjmghfm9M2PPvMSbsftkbL/b5E+SOV7lGZXo2WNO4n5vMochdJrnHoAjpl7FxzFcd+o0x6AZCcEqZQZ4LTsKCn9DxMwK/xPFnjnmfWWrmzDEfSKpQUe855sMA4RzJ17xDR0cm270JHergXQNVfWh8Pct2r/AD43bveSbbfcGZcsoZsaKmBLMTwE7js1V32KraDMqI/izbPZm53cWHZLvLpDBu9ywzdwpPkGDBEdxp7Y0J+/T7jAm7X9a6AR8/d7LGiwWDmiqUmNeemWXBUdS1YESEEJq0Mm74pPapkSjvBsuq5KKbEckNFJpSK2JVkNcOfVUzFiGlnrnkK5MzjwBZOzSeRFZtpKOQQtLs7p6I4u6DUC/G1KFjpud0lIm27sxmT7Ya6naNDCs6rZRumHE6GvFYtMMpb8RT/g3OqDGPOg7bTUW/dngtP83NcoGr2SJvTRcZZCG744gi8WDq4IxNVLI7NtYGd6rxhzXOtMLZTUw/4XB0YgfzHjnR2hsT9r/8wf0Xlh996mSNF3OmNTTguoLGA4S5ra6FpFYOU9f8jJt+C4C2ylhSE2rGnHdKYlFx1tllx2uhRMN2ENNoQVUrqkLR4Bi3fGMEUeUSLcBJXJwmQFU1Ms9p0uyhvqNYxS51sFfDgnqv8B6YXCwZmR1WN8xYCqYs+xNOe0NW3QFn1JhYNkx0yXrts1W32ai6bBbGPLo7y3YvUhedzuYTJqbw7iZ69mZqWGpaIsYJemoK7ydRsOAIitatMWFPv/+YsPvls8+c4r984XV+47UN/oPve2Tfr3/YOJMCUbs0nqD2FKCp/Zm73ZEUnvFV7QTGnBurnK4ydxTPqB0iAWeclJ2mj6Jh02/TaEFZK/LCoWB2R7EB0QjK3Ax/dWOF0B6iapBphATqh1i0ypYy1pPImEf3iu71bBhFEBbEQcFSMOVUMGHNG3HG3WXNGbCmoEQwaDCtOVWP9bzLRt4xTdBpQJZ66NRku985AdqZCZaJmSkQ0ww9Hp/4F5EjJ1p7Y8JWP2BM2P2yN17shVc3T4Roqf4EEQV7p0PK0hTPRW3acUrpkleSLadFUZvivBSacRMQyZwVmXDGEVx0+rRlRtL4prFaaMpGMnF9JpWkVAot5ayxGpN66gi0FHhVg/A9ZFU9tD2KeUdRB1C0TeG9Ck22uwgrwlbOUithMUh4NOqz5g+56G3xlLdBV9ZMNfQbh6tVj8v5GhtlhzenS/SzmH4SMh0H6FThDPfuFII3Mu05waDGmdY4wxy5O0ZPE+rR5MhNz9lvjlQh/m7HhN0ve+PFvvDGCRkvlqTIJEMlFSptbhVkVWYK8zITkEvyzGWS+QyLkH5x2zE/1i6lbohlw4pMWHFGLDpTFr2EjpcTeiVqL8rmlmOe24X5QNFEnnHM+/5DW5jf+53ccrv7e9nuFZFf0vEzOl7Kkjs1A1WdEZGo8YQw2e51NHO735nt7pFlLnqW7a5ykPme0/2ObPekRCY5OjHZ7jQn34pypP4v25sK/aCtOx/ESRovVm1uo1oxCvAEyNKjdl1EBWBqW7ISlMpl0pg7U2pmg4hkwbTxaPQGT3sFq8oh0RsEssQVNWWjCJ2SqpZMZEApPOOYd5jZIEBLhax9HE/hVDVi4CCq6qHbbeVdU1MsO5qq3aADk+0eBwWn4gmPxX1WvPGtbPeLM9PvtNFcqZa4Xi7yTr7ElWSZQR6yNYlNVPLURY0UKhe3ewons6J72uDuZshJCoPxQ3VD5EiJ1udfufsxYffL7fFiG8detNCN8eCMp6jA2Bm8UAIK7UDjGQ9XHSgaAanyGHghAOteByUaYlnQlessqYJVpanZJWtc+r6pgw2LgLqRNI2kKSSVMLfamZlQnVyhBcgkRNUNsqmpd4cntgj8XtQh1L4ZjKvD2nixwpyun7ESTFjzh5xyR5x1dunKHCUEO7Vg2PhcLxe5WfS4mXXZyWLGuW8EK3UQezWsbGYevaOGpdIKOUkR4ynNdHpi3O53w5ERrb0xYX/y+x45UMf67fFimzSNRu6D435uaE1TlMjxGBn4KMCJXbQwdoXaMz9bnQq0UJSOy9gzdxS3/RZSaCJZsKQmwISLrkdDTuIM2XI7AOz4EXUjqRvJNFc0KKpIz0RLzOJsFE7LQzQNsq4Rkyk6f3j+iKrQHAl1YAQrjEzEzKKfcMofc9odsOYOWFXprSC/fhOwXvW4WfTYKDrs5DHDLGCaeTPBUqhkNpAi2xMsjTudteckxUywkoduXuWREa29MWEHcdfw3eyNF/vD68N96W2cK01NkzcwGCKKAtdVyMIHPLQQyApTnK+g0g6JCChL44DPaodGC1xRMXCHBGIdV8CT7ohC36CrpqS1S6AqlGxoGkHuupS1dzv1tJbUnkbULtoROFKi6sYUhXd35/3bORSqjklscFsF3XZKJ8g4Hw9Y9Udc8Le45K+zMotKHjeaK1XIa/kZNsou30qW2c5abCcxg1FEmTnIkWN8WBOBNwKVaoJhgzOtcUclqj9FTFOT418dn2z3/eLIiNbemLDnLiwd+HPtjRf79Vc3jr9oAWiNLgrTF5jkSEfipAonFGhpirhamjt/TaCopGaSe7gqpOXmbFdtXFHTdzx6smBRSpbUlBLFkjslb1ySymPoBUa4fIe6EsjSRNmIBmrfFOZl5CJDH6H1Q9OjqP0G4dczt3tOd+Z2X3ZN0X1JprRlQ6lhrB226g7bVZudMmZQRExmbvdqL9s9Nw3rptNh7+bKLHk0vZ3tfpJtDR/EkRCtvTFhP/jkvY0Ju1/2xot9/pUN/uq/dzLGi+k8py4KlOugZrPrtApnOy1TkDe2dkVdCUZuSFmr2U6rIfE9Yplz3t0hEglnVEFPbjMNfAJZIkVD0Sh23ZDtSlFJKKVC1LPhrzU0Spm007qF8lxknpvBCSf8+OK2c4KgZClOOBsPWfYnPB5sctbtc9Hps6I0rlB8vfBZr3pcKVa4ki6zk8esT9pMUp9s6sHYwUlNtruTzoruIz2bnlOYqOThhGZ759iM+zoIjoRoPciYsPvlRI4X0xo9TUzcr+vgeA40mipwQZhjoinOSyrfJQWGqmbLa9EgWHYnSNEQiIpVVRAJOOvs0mhJ1rgMAlPEn0YeCVA3GMe8EJSRgAZEo1CZMbU6qbmhIur6xDXt3kkryom8kqVgyqo/Ys0f8oi7w4oas6I0idZMa831aoEb5QLX8wW2sha7uZlPmKcuOnVwEpPtvmccdZLZfMJ0lu0+SdDj8UNzl/D9OBI+rVtjwj5y8PWsPU7qeLEmy9CTKWKSoKY5TlKZO06zPwInBZUKRCapU4dp6tNPI3bymJtFlxvlAut1hwZwhWBNJZxxdznj7bLqj1kOJnQCs7MQYU0dNlSBNnfQQpMdVbYUVezStAJEHCF9H8SR+F/tQOgEOYthYtzu/tDkYqkRK6qgKwPGjeJG3eZGucBm0WE7b9GfZbvnqUuTOmZs/d4E6FQbe0OicRIzVFVOU/RkSj2aPPSidSR2WntjwrrR4cX5XliOeeJUi8+/unHixos1eY7e2kZJicgjfCWRpQNamRNiKdCOibIpGsGO1OSVgycr0tqj0RKXmlNqwhOuxhUTPGoK7dBSOUXj4KqaHdkwaQSVqxCNMnUzJczwV1cg6ghHKZRSSK3RaXoi/+DOtQb03JSL4RZP+OusOUM+4goSLXi9LHijXOF6ucDlZJXtImZ92mFrHJNnnsl2TyTuROKOTCaWPzTWBm9Y4fYT5CSj2e6f6H7Ce2HuL3/v7JgxYZ97Zu3Qn/uzT6/y0pU+o+yE3YHRGl1V6CRFJBkqq3DS2hR0Z65qE2siEIWkyB2S3J055iO2yxZbdYedJqLUDS6wolKW1IRld8yil9D1MmK/wPEr8Bsa32RG7Q0gNQNgFU3k0kQBIggQ/vFvUn8vFr0pS96EZWfEKTVmSeY0NCRas1XHbFYddquYQRkyyI3bPc9d6txku6tc3HK734qaSRtUVpls9zQ7UcmjD8rcd1p7DdKfPQSrw7v57NOn+H/81rdO5HgxgGY8RuQ50ndxS1OP0spFlnK2E7rdo5hWkk3VUM56FH1ZMXYD2jLjjMp53G0x1Zu0ZUrSeDiiRoqGqlZMnIasEmip0NI45mvX2C2Q4EqBW5qpPs1eq8kJ4mK4xaKa8BFvgyfcjLb0uVxWrNddvlWs8q3sFNt5i+tTMz1nPAmpR94syE+aovsYM7I+1fiD0hwJhwnsDGjSh/dO4XtxJETrI6sPNibsfjnJ48UA88pcGBOiABxX4foKNNSB6bLWamZE1ZD6HlJoXFXTdjMaBFedJWAHT0zpSY1yRmy5AxotqbRiXAYIoSkLM/y10qbtRAsocoFoFDSg0gApBSrp0CQJOj/eY6zu5Iy7yyk15oyTUmrYqHOuVwusV12uFwvczDr085hBEjJNjXl0b6CqcbrfNo86SWPmE44zxGhCk87G1ltuMVfRGiYlL73Z53/xAxfn8vx748X+zTfWKesGV839tLy/7B0TxxMzLdp1cAIHcKhCgZbQuAKVznZcgctUaFynZsczLyKn3C6uqIjFDmeUJlINZ51dSq1IGo9+YO68TgOPtBY0jaDKZu1DmaCsQTSSKvVwANVpIeua+gSJ1llnl0WZsSw9NuqCYeOyXnW5WS6wVbSNYGUhSeZRZe7tYRSzwruTatxUm8SGpEZOc8Q0NZHWRXnidqYPylxF69++vknd6ANtkP4wTup4sTtp0hSaBlE3uFIic4/GCZGVBA0IQVUKtHIoK8FAC5RsmJQmqiZpPErtIL112lLzhDsiELcbq2NVUNaKvmpIVEBVm92bqE1hvlFylvsl8WqNkhLlOiemR/FJN6XUmu2m4ErVZavq8Hp2mo28w/Wky8akRZL5Jts9lTiTWb57Ohv5lTR44wa/nyOTwmS7J7P2nBNqE3kQ5rq1eOHVTTMm7FxvbmvYGy/2+VdOlvXh29jbcWUZIsmQWTkrzDe3BnuqHGRmCvNV7jDNPSaFPyvMt9mq2vSbgEQLAiFoy4IVNWLZnbDoTen4pjDv+SXab8wA2FuF+VnOfChpIhcdBYgwNJOrpZr3b+eBUQhqoF+77MyGqvbLmN0iZJQHpLlHMct2V9nsWJjddrs7qcZJK8Se2z3L0IUd0/Z+zG2nVVQN//abm/z4x9bm2rR853ixn//JZ07MeLF3Y46JY4TjIMsK11WI2tzN01IhKmF6FLWk0g5TZab6+E6PojE9ipHMmTpDev6ARVkTubtk2iWSOWltivNKaDYrSem4lJWLFhizayloHIUszeccJRBliUxSmvF4vr+cB2SsG/q1y+vlKS5na2yWbd6ZLjDIQvqTiHTsQ6ZM0X02PccdmSOhPzQ+LGeUIXdH6On02P8+Dpq5idaX3zJjwg4q8O9eMOPFtnhjc8JHTsh4sfejSRJkXSNDHwfQSlB7clacN7UoLSSV65Jpwe4sysaTFQvOEqV2OKUmdGVJWwrWnCFKNPQr46pvtCQpXKYCikJSNwqhMcV5qXFycyNAaHDSFlLK27fzj+lR8WoVsVO3uFYscSPvsV3E7KQRk8y43UXiIGfzCVVmhlF4U2NrcMclapIjRwl6PDF3Vy0fyNxE6/Ov7P+YsPvlR59a5W/w9dmdzJMtWjrPqcsKJ46QUqI8BzcwGVxVahqrtSNoMkkjFEngoaTGVy02PdNDeN3t4opdVoRgReYoNOvukFw75LVzq92nzB3q2phNqxBAUIYgaomsHVQcgNaIiQ9Zjj6morVe9dipW2yUHbaLmN3MCFaWejSJcysTy0TMzPLdU5PaoKazbPfJ9MRnu+8XcxEtrTW//toGz+/zmLD75SSOF/tAmpp6u48sSxytQbSQpUvjOMjKDLHQUlCVDrnyqStF3Qgc2TD2AwJZknkemd7mCVexqGoyfYNAlviiIm8c+m5E1UhTmFcaGueWN0wriVYCUYU4gYNTN+jBiGbcHEvH/Dez0+xWEW8ni9yYdBlnPtNRgE4d1FiZonsG/tC0UnmTBr9fopICuTMy7TnD0bHdaR42cynE740JO0rJoZ99epXfvzpga3xybsV/ELoq0bkZ6qmSEpVUd+TLc2sQqMgVVa5Ic49x4bNbhMYxX7XZqWOSpqbRmhWVzgrzYxa9KT0vJfYLXK9C+A1NcNsxf2uYaSipA8cU5qPQOOaPYU1xp4xNtnseMs09svzbs91vdyHc4XZPTZCf3vOsWcG6a+ayzZmnC/79+OzTq/ztz7/Ob762yb//fefnvZyDR2szzLMskY7CqSI8TyFqhdASrYxlQTuSqnHIgG0npqgVkVNSakWNJJAla2rKo46Hyy5KNCSNZ6wSlTGrDmRDMhskKyqJFgKkMFN93NmOSwhT38rzY7fbupouMMxDtqYx42lAnTioiboV5OeOTfOzP65xJjXesED1J5CkVP1de5fwHpmLaO2NCTt1AGPC7penT7c52wv5/KsbD4doAeiGpigRkymybnBDF/DQUpg0Um2ibDSSWjqknk/TCLZcM/zVFTU9lVBqRVsMcQU85gwZuDEKzSAMZ8V5QZG7VBLqXCA0gJg55yUydxGN+cOVaRedZjTT6dx+LffKZtJmWnhMEp966hov1nRmHp2CN5kV3Uc1TlIiRyl6NEYnqRWs++DQRWtvTNhf/dxHDvupPxAhBJ99+hS/+rtXycqawD3+/qEPRWvQ9a0/HjmNcJQJ9XMC04qjQtPqox1B7StyPIZ5gJINoSrZcLsoGgZqTFs2LErJmjOgRrDhdSgah6JWTEKfDKhCaXZzNVShAA1VZFIoRNXgtGITY3OMjJWjzCcrXON2zyRqNpDCyZh5sBozBTopkZPCuN0n04c6yO9BOHTR+o1XD35M2P3y2WdW+QcnZLzYvdCkKaIokI6DKlv4dYyWAlWaY6KsTBZ8Lh3qUtKXMUWlqBuJK2vGtSnOn3f6nFEF552EtiwotUMgSzxZUTaKkRvQbwSVZHb8nDVuNxItnZmLvkH6Hqqqjk2P4mgcUecKMXZwx2aH5c1iZrxxg79b4SQVameCmCTU/V0Timi5Lw5dtF54dePAx4TdL89dWDo548XuBa3RtdlxScdBei5O6oIEJ5NopU2PYi5AGMd86niM3Ip+EeGIms2qQyRy2rKkLTRtUbGiRoydgMT12PFjGi2Y+D55JWlqQR1oRGOK8rKUyBrq0DUN1nFkehTL6sgXqetMQSFntoZvz3Y3O60alczc7mmKLqtjs4s8ihyqaJkxYdv8B997/kg6zz1H8oMfWeGFVzf5xeM+Xuxe0do4sZsGCWaqT+lRexIxG/TaKDPVp3QdMg27GnxVk1UuoTLF+QbJx7wd2lLwuGsK866oSRoPT9YUtWJXaDKhqUoP7TCLyDFDOGTh4boSUbaRgNSaZjI50n/kYuKYgaoTgTuZBfmNZ9nugwJnkJoj4U7f3GQ44iJ81DlU0dobEzaPwL+75USNF7sPdJ7T7A5MX2DT4AUKoRVaKmqPWXFeUuNQCOh7IVUj6XgdzFTEhp5MWFQZK1KSqQm1K+n7Ma6omVQ+jTYimBZqJlQSoU3WvJNLtHKQRWDmKAKU5ZFOO3CS2wNV3eks231c4yYVzjhHjKbGPFqUoJt5L/fYc6iitTcm7NkLi4f5tPfEiRsvdo/o2Vh7kaRIIVCxT+MIHFfgBBJmE6a1lNSOIstcBNAPYnxZ48uKdbeLFA3nlGZRFtTOmHVnCJghsXnlUNWKPHRptEMdSEQFooYyFKAlVWjmN8q6QUzMSLKjOgBWpbMj4SxmxhwLa9S0vO12T5IjK7rHjUMTrcMeE3a/nMTxYvdD0x8g0wzHUcgqQtQeWrqowvQrykpQNopS+NSl4qZqU9aKvFEEsmTgRnhcZVXBk64i0zfpqYS8cfFkhe9UVLVk6niUjRFB7RjHfOOB0C5aCVxX4tQ1apJQbWweyWOiNzI1LG+s8YcN7nSW7T5O0bsDmqlNHt1PDk095jEm7H753DOrvLY+5mr/ZM/r+yB0XdPkuYmymU31MVE2GpVyy+ktckmTOaS5x6jwGexlzFdtNusWYy0odU1PFpxSY5adMUvulJ6XEPuFmeoT1MYx7/OdjvnQQUc+hAEyDBHO/Nu+3o0punN7h5VWyEl2O9v9IZwCfZAc2v8B/98/uIFzyGPC7pcffdrMRPw3r2zw558/WZN67pqmRheNqW/VNQ7geQpZKmr3tkG0cSR1LchcH60FWgsipyCtPdoyo0FS613OOJKezBg1m0ihkUKTVB6uiqkqZTxc0ly/me3kwEz4kUWIUgpVVjS7gyPnmPfG5kjoDSvcYY4cZ+jdoakPWi/WvnMoorU7LfjVL1/lpz9x9lDHhN0vF5ZjPnG+xy+9+CZ/5rlHHg6j6XuhtWmrmSqEUji+i6jcWb68RM8GwIoGSs8hB4bAhtumqBUdZ+nWpQKxQyDgMXdw63OjKsCRDVnloLUgFy5VLtHKDMcQDYBE5Y45khYtpNYg5ZG6o+hNmpnjvUAOE+PFShJjbbDsO4dyPPx/felt0rLmP55TFvz98L/9sSe5Pkj5h//urXkvZa7oqjJzFKcJcpqh0tIMX8jM5BhTgBbIVKAzRZE7jDKfYRGyU8ZsV202qzZj7VACixKW1JRTzogld8qyN6Xj5URBjuNXNGFD7ZujYhUKqtC46KtIUcceOgqQcYRQR+eFxJnWqKQyMTNJhp7aJuiD5J53Wn/tn3z1np/khVc3+OEnV3jyCBpK34/vf3yZH3pyhf/qNy7z+sZk36//f/kT37Xv1zwodJ5TFwVKSVQR4ymJLD1k7RjLQiRuNVjXtWAoNUXl4IiGonHIGtc0VjtDnnGnrKqCttyg0Iqus0CpJa6qcVXDdiOpHYeyceBWcV7iegLRBLhK4ngucjbXsUnmX3f0djNkWiL6Q5rBkOYYuPiPM/csWv/uWzv3/CS90OU/+dFL9/x98+Y//eNP85/8d39wXz/ziUNrdJqBkMgkQjkS5UmcUICEKhdoB9OjmJlUiFHg4zsxnqzYcHsoNEM1wRXQFpolNaHUDptuh6nnU9aKse+Ta0GTK+pKGyEMzFGxCiWycJCVhwpN0CBZbrxPczwqyqRApDk6mU3QPiLH1pPKPYvW7/z1HzmIdRxJnjjV5l/95c/MexlHhmY6ReQ5yvcQjTFJNp6PrOWsOC9ACxpXUdWCgRvd+t6Ok5Fph7ZMecwdsKxcHnOGxKJgGpiseikapqXHSDVMSpNVr4U0+fVKmOlBOGgJft5COgqZZnOPsxG7I3SWUY9Gc1vDw8TRu39sOdLoujZ3FLXGEcLcUawUtWdacQDjmK8FuesxNLcZiZwF8sahq1IAaj1kTUl8kTJqtm655Celj6tqytK0CtXCoSqNqbUojfEU4aCyAEdKZFnRDEfo6fzMm81giK6t0/2wsKJluTe0pkkzhOchfA+ZByglULlGudB4s5wsCVUhKQqHxPGYlD6erNitYjoypSMzHnVqfAE9lbDoTBg3AV0vo2gcAq+k9BV1Jc0EbB8aH+oc6lJQBwpRu4jQR6QuIhVz65CxtobDRWh7/rZYLMeIo9tPY7FYLO+BFS2LxXKssKJlsViOFVa0LBbLseKe7x4KIb4OHPXbJcvA9rwX8SEEWuuPzXsRFstx434sD5nW+nv3fSX7iBDid4/DGue9BovlOGKPhxaL5VhhRctisRwr7ke0/u6+r2L/sWu0WE4o1hFvsViOFfZ4aLFYjhX3JFpCiD8jhPhDIcTXhBBfFEIcuSQ7IcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rsliOE/d0PBRCfD/wqtZ6Vwjx48AvaK2fO7DV3SNCCAW8DnwOuAZ8GfhTWutX5rqwOxBCnAZOa62/IoRoA78H/MxRWqPFcpS5p52W1vqLWuvd2YdfAs7t/5IeiGeBy1rrK1rrAvgV4KfnvKZvQ2t9U2v9ldn7Y+BV4Ox8V2WxHB8epKb154F/vV8L2SfOAlfv+PgaR1gQhBCPAZ8EXprzUiyWY8N9hQAKIX4YI1rP7+9yHh6EEC3gnwJ/RWt9vzm99tbv/iAe5Js/J/+E/Xe4E6mQnotoxbDQRccB+XJI1VLkHUXRFlQRFD1NHWjqdo3TKvGDkld+5hc+9N/iQ0VLCPEXgf9o9uFPYPr6/h7w41rrozbx4Tpw/o6Pz80+d6QQQrgYwfpHWut/Nu/1WCz7iQx8RBTC0gL1QkQVu+SLLmUoKDqCsgVVpCk7DTpocNs5vXZK7BV3df0PFS2t9X8N/NcAQohHgH8G/JzW+vUH+cEOiC8Dl4QQFzBi9SeBPz3fJX07QggB/H3MDY2/Pe/1WCz7hXA9hOciFxfQrZBqIaLoelSxJO8KqlBQtKFsa+pQQ6fECyoW2glr8ZiOl97V89xrTevngSXgvxFC/MFRa/rVWlfAXwJ+DVPg/sda62/Md1XfwaeBnwN+ZPY7/AMhxE8c5BNe3hzzv/7VPyAr7fBQywEhBMJzEWGAbkc0rYCq5VLFkjKSlJE5EtaRpo4adFjjhSVxmNMLUlaCCav++K6e6p5qWlrr/xD4D+/nZzostNb/CvhX817H+6G1fpEHrKHcK3/zf3iFL7yxzU994gw//OSpw3xqy0OAcBxkK0b0ujStiHIloooUeXevfiXIe1AHmqpTI1slQViy2h2z4Cc8Eu/yeLDFonN3Q5GtI/6E88XL23zhDRMt9tKV/pxXYzlpCNdDhCGi3aZpRdRtnzJ2KGM1210JqhDq0OywRFgTRAWtMGcpmLISTDjtDTnj7nLW2f3wJ8SOEDvRaK35P/3aNzndDVhu+bz85lG7b2I57shWjIgjmsU2VS+gjB3ynjT1q72ie2h2WCKsiNo5i3HCYpBwId7hlDfikr/BJXeLRXl35QsrWieYt3cS/uDqgJ//yWfYmuT8P3/7CklREXn2n93yYMggQIQhLC/QxAHFYkjRdahCQd6VVCGUHU3Z0jRhg9vNCcKCldaU09GIVX/EU+FN1twBl9wdAnH3rhF7PDzBvPymOQ5+5tIyz11YpGo0v//OYL6Lshx/pDJHwjiiaYVULY9qdhy8VXAPoQqhmR0Jw7CgE5gj4ao/Ys0fctbdZU2NWJRGiLK71C37knuC+dKbOyzGHk+carHWDZACXrqyw6efWJ730izHFOF6qKUFdLdN3Z6ZRiNJ1pWUrW83jTadCq9VEAUFZ7tDFv0pF6IdHvc3WHOG/BFvFyUEidZcKTvs1C0ev4s1HKudlhDiF4QQf232/t8SQnz2Aa71S0KIzdmgjhPJy2/2efaxRYQQtAOXj57p8qU3bTHecn/IIEB2Wuh2TNMOqFoeZSxvF9xjZrssTRPVOGFlLA1Rymow5nQw4qy3y1l3l7OOaQKZNpr12ud6tcCNcuHu1nGQP+RBorX+ea31Cw9wiV8GfmyflnPkuD5Iubab8tzFxVufe+7CIn9wdUBeWb+W5R4RArnQg4Uu9VKLYjEgX3DJu5K8Kyk6xjRathuaToXTLum0E061JpyNh1yItnk82OQj3jpPukMedQSJho065LX8NJezVd5I786Oc+RFSwjxN4QQrwshXgSevOPzvyyE+NnZ+28JIf7zPcOrEOK7hRC/JoT4lhDiL7zXdbXWvw2c2G3H3p3CZy/cFq1nLyxSVA1fvTqc17Isxw0hkO02zuopmpUe9XKbfNEnW3DIepK8Jyi6UHQ1Zbeh6VZE3ZSF7pSznREX2ztcijd5OrjBR/1rPOMZA+lGXfFaucw38rO8lp7hjekprkzurmxxpEVLCPE9mFacT2D6Hr/vAx7+jtb6E8AXMLuonwU+BfzNA13kEeXlN/t0Aoen1jq3PrcnYNb6YLkrhEB4HjKO0O2YOvapYpcqkt/mwaoi05ajQ3MkbIc5C0HKij9h1Rtx2htw1tllRaW0hEuuYafxWS973Cx7bOQdtrOY3Sy8q2Ud9UL8Z4B/rrVOAIQQ/+IDHrv3ta8BrVlW1VgIkQshelrrwcEu9Wjx0pU+3/fYIkreNt/3Io+n1tq89GafvzTHtVmOPsJxEGGIXFqg6cbULZ9s2aMKTR9h2TKCVfQamlAjOgVxK6cV5Dza2WXZm/JEtMFFb5OzzoCPeYJMSy5XDVfKNdbLLq8kZ9jOW9yYdtmexBT53cnRkd5p3SP57L/NHe/vfXzUxXlf2RxnXNmefls9a4/nLizye2/vUtbNHFZmOfIIcastR7Zimk5E1fYp2y5lLCliQRmbu4RVrKnjBh1VBFFBJ8xYChPWghFn/AHn3T5nnQErqmDYFGzUDTeqLtfLBa4Vi6xnHbayFrtJSJp4lIl7V0s86qL128DPCCHCWTTxT817QceBPX/WsxeWvuNrz15YIilqvn7d1rUs34lQCuF5iIUeerFLtRBS9DzyriLvCMqOoJwlNVTtBtkuCds5S62E0/HoVh/hE8E6l7xNLjoVp1XIjcrhSrnIG/kaV9IV3kqWuDHpsjluMZkENGMXOb67vcWR3oHMctR/FfgqsImJntkXhBD/HfBDwLIQ4hrwf9Ba//39uv48efnNPpGn+OiZznd8ba+u9dKbfT75yN3dYrY8BAiB9H1Eu42IQ+rljsnBWnApWnLmvxLUARSdhiaukVFFt5PQCXLOtQY8EvY57Q15JrjGippyXjUMmoardck3irNslF2upCu8kywwyEK2RzFF5sLYxR1LZH53OQJHWrQAtNa/CPzie3z+z93x/mN3vP/LmEL8d3ztXd//p/ZtkUeMl670+Z5HF3DVd26kV9o+F1diXn6zz1/4wbux8llOPEKYHVbgI+KQphWZWJlQ3XK432p8DjRN1CCjCj8s6YYZC37Cqj/itDdk1R2wpia0ZY0rHMZacqPqcqPssVl02MxbDLKQYRpQpC46dVCJRCUCdXcZgEdftCz3xu604JsbY37qu06/72Oeu7DIv/zDm9SN/rZCveXhREYRIgxgZZGqHVC1PbJFxxTde4IyniWNdmdJo52cbiujF6ZcbG+z4k14IjBF9zU1ZVVJxg18pQi4Upxio+zy2nSNnTzm5rjDcBJQZS5i4OKkAnckcCfg3GUfz1GvaVnukZffMvWs5y5+Zz1rj+cuLDHOKl5bv99oesuJQCqE7yPbLUTHtOVUbY+yrYzTPcYIVqypI42OalRc0Y4zFsOElWDCGX/IOa/PeXeHJZkSCU2/rtmoPa6WS1wrFrme99hI2+ykEePEp0w8dKJwpsK8JeBONe707kTL7rROGC+/2cd3JB8/133fx9yqa13p89Ez7/84y8lGei4ijtALHZrIo+z5FG1FGUmKzsyD1dJUsaYJa7y26SNciU1Swyl/zAV/kzVnyCV3SCAEEsHlss161eXNfIW30yW2shYbkxZJ5pNPfMTYQaUCZyJwUvDGGn/UoO5yp2VF64Tx0ps7fPKRHr6j3vcxZ3oh5xdDXn6zz//8+QuHuDrLkUAqVCtGdNrodkS5FFPFDvmCQxELqti43KtQU3UbiCu8sGSlO6EXpDwW9zkf9Dnt7vJx/zptWRIIwXqtGDQB38jPsll2+NZ0hZtJh0EaMBjGNKmDHCvciUClAn9X46Tgjxq8YYVKq7tb/gH/eiyHyCgreeXG6D2tDu/m2ceWePmtPvcyYdxyAtgb7xVH6Di8I8v9jqTR6A6Xe1DjBLez3Bf9KSvemNOuaXzuyhJfQKY1gybgRrnAzaLH+szlPkgDksynSR1EJnFSI1hOCk4KbtLgTmucSYGa5h++fuxO60Txe2/t0mj41IXvNJW+m+cuLPJPv3KNy5sTLq22D2F1lqOAWugiwpBmuTszjTpkC8qE9/VmSQ2hpurViKCm1U1ZiFIWg4RL7U1OeSOe9G/ymNtnTdXUGgaN5I1ymSvFKW4WPd4Yr7CbR2xPYpKJT5M4OAMHlQncsTkOOgmE/QonqXF3M+TuGJ1kd/UzWNE6Qbz0Zh9XibvyX+255V96s29F6yFgb7yXaLfQUWAEqzPLcm8JquCOHVbUIGMz3qsbZiyHE1aCCWf9XdacIY84u8SiotCwXbts1i2ulktczxfYyNtspy3GmU+a+DRTF5lKnESgslnBfaJxU407qVDTEjlK0NMEnd6daNnj4QnipTd3+Pi5HqH3/vWsPR5ZjFjt+Lxk87UeCkTgI3tdml6LqhdSdF3yjiJvC4q2cblXLU3VrtGtiqiV02slrMUjzkUDLobbXPQ2uehtcsFtiATkGt6qlrhSnOJKusLVdIGbSZf+NGI8DajHLmqscCbG0rC3y9qrYTmDDDmYoneHNMMxzXR6Vz+L3WmdEJKi4mvXhvxHP3Dxrh4vhOC5C0u89OYOWmvMDFnLSUM4DnJhAbotmlZIvhxSh4psQVG8K2m07tSoVkkYFpzpjFgIEi5G2zzi73De2+EZd5tAwLCBG1XIet3l6+k5Nos2b02X2EpixmnAdBBCJnFHCmcscDJmRXeNP6zxBgVqWiA3d9FpSjOZouu7z3izO60Twu+/M6BqNM/dRT1rj2cvLLIxynl7JznAlVnmhXAchO8jWhE6Dm5nuceScpbjXu+53EONjCqT5R5mLAVTlr3prfFea2pEWwqUEAwah/XaND5vFm22Zy73cRqQpR4iVcblnhrBchKNm+hZ0b0ygjVO0dOEJs3QVQX3cEPI7rROCC9d2UEK+J5H776f8FMX9/K1+jy2HB/U0iyHjRAgJGplGR2HVMttypY7K7pLqmBmaWhpqlDT9CqcoKLXSViKpiwHU55qrXPKHfGUf4PHnAmryudapdmqQ14rTvN2vsxG0eFbo2WGeUB/FFFOjWC5AyNY3ghTv0o0wU6Jk1Q4uwn0hzSTKU2S3JNY7WF3WieEl97s87GzXdrB3cV7ADy+0mIp9viSDQU8OQiBDEPUQhfdadF0I8qOR9lxKFrSxMq0ZoIVaZq4xo0KoihnKZpyKhxzNhxwzuvziLvDY84ECfTrnKtVh3eqRa4Vi9zIu6ynbfppxCgJKBMPMVWoqcSZCtyZy92bNHjjGndU4AwzxGCMTlJ0Ud6XYIHdaZ0I8qrm968O+LOfevSevk8IwbMXFu3k6ROEcFxEFEKvQ70QUYcORUdRxEawypYZPFG29tpySrqtjE6QcTYacjowbTl7fYSPOC2uVRNu1B5vlctslF3eTpe4mXbYzcLbfYQThTMxPix3Ak6q8cYN3qjGmVaoYYqYptT9XXRZQXP/cwqsaJ0Avnp1SFE135YHf7c8e2GRf/31da4PUs727i7u1nL02KtfycUFdCem6gQUix5lJMl78lZwX9HV1GED3ZIgKmlHGefaA5b8hI/E6zzqbXPW2eVJNwXg1SLharXAjWqBbyRn2S5avDNZYHsSk6Ye5dBHZnLW9Gz6CP1Bg5tovEGFO8yQ4ww2t2myHF0U973D2sMeD08A7zXE4m55buaet7nxxxghTME9DNCtkDr2qGLjwapCM6L+VpZ7NEtqCCpae1nuwYRVf8SqM2TNGbKiUlwhaYCNusWNaoEbhfFgbWUtBmlAmrmUmYNMTf3KvIFKTQ3LSRqcSYEcZ4hpSpNmNA9wJLwTu9M6Abz0Zp8nV9v0Iu+ev/fJtTadwOGlK33+p588dwCrsxwkt3ZYp5bRUUCxElO2HMqWJOsJ6kBQ9GaCFTeIbkEQlJzqTFgOJ6wFY56M1ll1B3zUW2dFNXSlx6tFw2bd4ZX8LFezRTbyDm+NFhlnPpNxYEyjicQbSJwMvKE2TvdUE2wXqKRA7YzR/QHVZPpAx8F3Y0XrmFPWDb/39i4/+z33JzhKmrrWy9ZkeuwQroeMQ0Rsstzr2LslWEVshk/UvhGsqlVDWBPHObFfsBxOOBMOWfNGPOptsaQmdGVNpmFc5bxTnWK96nE1W+R61mMnixkkIVnm0kxd1MTssNwpqHR2l3Da4E5qnFGGmGbo4Zgmz/dVsMCK1rHnGzdGJEV965h3Pzx7YZEXXt1kc5RxqhPs4+osB4ZUyDBAtFrobouqF1BGDkVnVnCPTR9hHRqXu4gq/KhkIUrpBSnnogFn/V3Ou30ec/v0ZEVPOlytGm7Uba4Up9gsOlzPemwkbYZpQDr1aFIHNZG4E3lLrJwUc5dwWJkj4XCKnkypd3cP5Ee3onXMeemKqUV934X7z3u/Vdd6q89PfvzMvqzLcnDcShpd7FF3Qqq2T77oUoaCvCdvjacveg1N0OB0ClpxRjfMuNDZYdGd8mS0zmPuFmedEYuypga+XrgzS8MSr05Ps1uEXBv3GE5D8tSFgYuTzoruU2MaDXYb3KTB251ZGiYJzdaOKbgf1M9/YFe2HAovv9nn4krMqfb975A+eqZD7ClrfTjqzMZ7iTBARBFNK6Tec7nfGSsTmqSGJmgQYUUY5bSDnMVgyrI3Yc0fctbtc2p2JKyBcSO5Xi3MomW67OQRO1nMJPMpMgedOKhUorJ3udynDU5So6YFYpqiJ1OaNDUu9wPC7rSOMXWjZ7uj98+DvxscJfnuRxdsXeuII30f0YphZZE68siXQ6pYUbRMlnsVziwNkaaJarxuThQUnO6MWA3HnPaHPBXeYM0Z8oy3iycEIHilMEmj38xOcy1bYCtrcXXUJcl8spGPmCrcROIOZ0mjQ40/bnCmDf5OZgRrZ0AzGhuX+0H/Hg78GSwHxmvrI8ZZ9UD1rD0+dXGJb26M6U8PbltvuU+EQMYxotsxptFuSNXxKdtGsIqWuJXlXsXG5a5aJZ04YzFOOB2OOBsMeMTf4TFvmzVnjAIGDVytXN4ql3m7WOZatsDNtMNG0mI8DcinxuXuTOXtpIaJcbm74xpvVBrT6GhKM54c6JHwTqxoHWP2jnP34896N3vX+PJbdrd11BBKmR1WO77VllN0TVtO0b6jLSfW6LjCiUviWVvOajjmXLh7S7AecyYmvA/o1wHfKld4O182dwmTLttJzHAaUk49mDo4E4k7NjUsI1gab1TjjkrUMEMMx+jhiGY6PdAj4Z3Y4+Ex5uU3+5xfDDmzD072j5/r4juSl670+WMfXduH1Vn2A9WbJY2u9KhbPmXHM0mjgSBfuD18ouzWEDRE3ZROZMbTX2xtc8ob83Rwg/PuDmeUiTMeNHClXOKtYpmbZY9vTlbZzSLWx22miU+dOKhZ0qg3EjgT00cYDGqcaY23m6F2p+hpQt0f3FOszH5gd1rHmJff6vPsYw9+NATwHcUnH+nx8lvWGX8kEMKkjYYhuhVRt3yqlksZzxzue8NTZ1nuhDVOUNEOc7p+xkowYc0bccodcdbZpScLAiEYN5J+HXC9XGCj7LKed9jJYoZ5QJp61IljomX2stwTTME91caDNa1MW06Smsbnqtx3H9aHYXdax5j+tLgVm7wfPHthif/bb7zBKCvp3ENahGX/kVGEXOjRLHepY4980aOMTf0q3xtP39XUcY2ITJZ7K8g53x6wFow46w94Orhuiu5uTaKh38A3y1Osl11eS0+znnXYSltsjNpkmUs99ExKQyLwhqYtJxg0eOPG7LC2p4gkh80d6jRDl/Opf9qd1jHnXkL/PoxPXVik0fC7tq41P6RC9brIxQWaXpuq61N0XPKOnMUiz4ZPtDR1u0bEFUErZylOWImmnA93OR/0ueBv8ri7w5rKGTcVNyqHt8oebxXLvJMvcSPtspG06U8j0qlHPXFRE4U7mUXLjGf1q3GDO65wRzlyOIXhhCbPD/1I+G2/ork9s+WBWe34PLIY7dv1PvnIAq4SvPzmwTiZLR/C3nivbgfdiam7AUXbnU18Nm05t+8SNoioIogLunHKcjjhdDjkrD/gUW+bx9xtzjmwKB36jWK97vBWucLVbJFrWY/NmWBNprM+wqnCnQqcKbgTjGBNjGA5wxw5TEzBfTRCH0Brzr1gj4fHmOcuLO1rtnvoKT5+rsdLNvHhcBECoRRyaRERhdTLHaqWS9F1yLrK+K96wvQQxibLXYYVve6UXpixGo65GG+z6o74I8FVzjhjzimXjbqm33i8Vpy+NZ7+ymSZQRayPYopZuPp3aGpYXnDO5JG98Z77UwRw4nxYE0T0M28f1t2p3Wc2Q+rw3td82vXhiTF4dy+fugRwgT3hSEiCmnicFZwdyhDSRVBPYuWqQOowwYVVQRhQSfITZa7P+G0N+CMu8uqmtAWmoaGfuOxVbe5Ufa4WXTZzNvsZiGjzKdIXfQsy91JxGxw6qzgnjQmuG9SIMaJKbhns93VERjua3dax5j/ySf2v0/w2QuL/N//7bf4/XcGfPqJ5X2/vuXbueVyX+hSLbWoIpd80aGMTA2r6Jim56JrkkaduKTXSegEGRfbO6z6Ix7xd3jKv8GamvK4E7JZJ7xSKl7Jz7JRdnl9uspG1mYnjdgZxlSZC0MXNxE4U7PDclJNMDCmUXdS4WyNEUlGvbFl7hAeAbHaw4rWMeYg7vB976MLSGEasa1oHRzCccx4r14XfYdptIokeVtSRSYauWxrGl+jWxVOWNGKM1biCYt+wrlgl3Nen/PuDmfVhEjA9TrhRhXyVrnMm/kKO0WL60mXfhoxTgLKiYfIlEkZnRpLw14OljeqccYlziiD4Zhmz9JwhAQLrGhZ3kU7cPnomS5fsn2IB4ppfA7R3TZNy6fs+BRtM56+nM0jrOJZH2HQ4MYFcVjcastZ9KY86m9zxtnlvDNkWSkarblSBlwtl7haLnIj67GTR7cEK09cxNSYRp29ontiCu5O2hjBGqaIcUI9GO5LNPJBYEXL8h08d2GRf/ilt8nKmsD98GnVlrvnVtLo8iI6CiiXWya4r63Iu4IqFOQ9YxqtYpPl7vkVK90Ji2HC6XDIk9EGy86Ij/vX6cqSnpS8XQn6dcw38rNczxe4lvV4Z7zIOPcYjmLqiYNMFN5QInPwB9rcJZw2+P0SZ1qapNHhiHo4OrSWnPvBFuIt38GzFxYpqoY/vDac91JOFlLNhqfG6DikaQVUsUMVScpIzOJlZoIVanRY4wUlcZizGCas+BPW/BGrrsly78oSV0CmG7bqmOvVAjeLHht5h34eM8p8pqlpyxGZQmUClYGTmDc3bW5nuU9z9HiK3hueeoSxOy3Ld7B3V/KlKzsHcofyoUQIVKeFaLWMaXQppIwcskU1q1/dLrqXvdrkYMUFy+0pvSDlUmuTNX/Io942H/XW6cqaWEjWa7hadXklP8tm0eHydIWttMUgCRmNQpM0OnTMHcJkNp4+g2C3Nh6sSYHaGpo+wu3jYXWxomX5DnqRx1NrbV62zvh9Qfi+uUu40DOWhl5A0XFv3SGsw1nRvaVpwgbZKgnCgl6cshaPWPanPBbscMbd5TF3m0jUNMDbleJ63eVqscRb2TLbeYv1aYdBGpAkPs3ERaby1mgvZzorumcab1TiDHPEJEUPR8bScEywomV5T567sMg/+b1rlHWDq2wV4UGQYWCSGjqm8blsuxStWeNzPEtqiE0fIUFDFBW0Q5PUcCYcsuxOOO/tcMbZ5Ywy/X6ZhndmSaPv5EusZx12s4jdJDSNz7O2HJUbsXKnGndqstydpDYu91GCHk+pR5O5OtzvFStalvfk2QtL/IN/9zZfvz7kk4/cf/78w4zwfWS7BYs9msg3SaMtRd5WFF1BFUDRMykNdavG7RQEYcHp9pilYMr5aJePBOusuQOecbdxBdTAlapFv27x9fQc63mXG2mHa+Me08wjGYaIxAT3eUNTw/IHejaivsbrF6hpjtwa0EymxuV+jAQLbCHe8j7cqmtZ68N9IVxvNi0npokDqpZH1VKUkZxluEMdmhpWHTWIsCYIC9ozl/upYMyqO+Ksu8uKGhMIaIBho9iqOlwvF1jPu2zmLfpZzCT1yTMPkSpkasbTO6kxjd7Kcp/WqHGOnGToNDWWhmMmWGB3Wpb3YaXtc3El5uU3+/yFH3x83ss5XkiFWlowOVi9mGIxoIok6cLMh9WGsqOpA03du20aXW2PWfQTnmxtcNob8Ji7xTPeLrGQjDW3TKPfzE6zVbS5Mlmin0YMp6HJcs8UznCW0jA1Oywnm/UR7plGt3dpkpRmPJ73b+m+saJleV+eu7DEv/zqDepGo+T+NWafZGQcm/H0Cx2aVkDR9Sg6ijKUJlYmMgX3sqXRQY3XKojDnJV4yvl4wKI35YK/yVl3l8ecIQ0waBrerjpcrxZ4O1/mnXSRnTxmY9JimvrkUw85dpCZuBWN7ExNrIyTNLiDHDXOTJb7ZIouynn/mh4Iezy0vC/PXVhknFe8enM076UcD4RARBGi3aJuB5Rtj7LtUMSSMsbEykSaKjJZ7qpV0YoyFqKUlXDCmdnwiUfcPuedEaeVR61h2Li8Uy5yrVjkRt7jZtphO42ZJAFF4iESB5UI3OkdgjU1SaPuuLwtWLNpOfMK79sv7E7L8r7s1bVefrPPx85257yaI8xs2rPsdWkWOmZ46pI/6yMUFN1Z0mhPU4cNulURdTLioOCRzi6nggnn/F2eDG6y5gz4mJeTa7hWl7xRLrNe9ngtPc3NrMtG2mZj3CJLPcqhj0zMcdAbmRqWP2hM0X1U4e2kJml0q0+TpmZE/RFsy7lX7E7L8r6c6YWcXwxtvtaHsFdw162Iuu1TR7dd7lVkBKuKZoIV1rhhSTvMWQhSTgUTVrwxZ7xdzjq7rKiURmumjWarDrleLrJRdtnI2/TziGEWkCY+ZereUXCf+bBuDVCtUUmFmGZmgGqa0hRHr/H5frE7LcsH8tyFJX791Q201vsaOHiSkKsr6NCnXAgpeiapIVsQt4vubSNYolcQhCULrYTT8Yglf8pHonXOuLs87m7xpNvgi4BXy5Ib1QJvFcu8ka6yVbR4a7TEMA3MtJyRh0wl3kCaO4QTk+XuZJpgp0CNCuQ4mXuW+0Fhd1qWD+TZC4vsJiVvbE7mvZQjS70QUy6ElF2XoiPJO4KiMxOslqbq1NCuiNsZC62E1WjMo1Gfi+E2l/x1Hne3eNQpGTQVb1YZV8plvlWc4s18hbeTRa5PeyYaeRJQj12ckTIF9wm4I4031nijBm9Q4Qwy1GACu0OaNJtrlvtBYUXL8oF8aja92vq13p+qvedyn2W5x3dEy8yy3L2ooDdzua+FY876A855fc6qIauqoCsDho3iRtXmarHE9XyBm1mH7bRlXO6JRzM1SQ2mj1Dg7Lncpw3upMIdF8blPhrTDEdmh3UMfVgfhj0eWj6Q84sha52Al67s8HOfenTeyzmSpCsutS/IO5Kyw6zo3tCENapd0mmldIKcC50dTvljLvhbPOXfZEVNedQRDBr4w6LmteIcG2WXV6en2cpabKUxO6OYInNh4OIkEneWNKoyTbDb4E4b3FFpstwnKfXmFrqsTqRY7WFFy/KBCCF49sIiX7qyY+ta70MZSWofs7sKoQ40TVgjwpowLOiGGQt+wil/fCvLfVEltGVNogXDRnG96nGj7LFZdNjKWvSziFEazLLcHZzEFN3VzOXupMbl7k5NUoMYJ7dd7iek4P5+WNGyfCjPXVzkX3z1Bm/tJFxYjue9nCNH3hM0PhRtTdWeJY12c+KwYLk15Vw8YNmb8HR4gzPOLo+7uyxKCUi+VkbcKBd4u1jmcnKKnTzi+rhr2nKmHowcnNT0ETqpmZYTDEzSqN/PUeMcMZyYLPcTVnB/P6xoWT6U5275tXasaL0HZdvsrqqWpmlXqLCi105ZCFLOxEMeC3c47Q245K3TkzltKVivYaxd3sjXuFn2uJotcHXaY5gHDMchZeoipgp3JFGZwBsbS4M30abgnlSo/hQxnpr6VXW8Xe73gi3EWz6Ux1daLMUeL12xxfj3oopNUkMT1bhRSRSZpNGlYMppf2gK7m6fMyphUdZIoN+YLPdrxSLreZeNtEM/jRglAWXiIhJ1a0S9ycIySQ3upJ6N98qNYE2mNEly4o+Ed2J3WpYPxda1PphyoQa/JmgVLLanLAQpT7S3OOWOuehv8pS3zqIqCYRgpxZcrbq8UayxWXb45mSVnSxmexIzGoc0mUINHFRq2nK8gcZJIRjUZrzXuEBtj9BJSrW1cySGpx42dqdluSuev7TMjWHGt7am817KkUOEFW5Y0pq53FeCCae9IWe8Xc64u7RliQQGDWw1ETdmWe7reYedLGaQhiSZR5M4iNRMe97bYbkJuEljdlhJaWJlkhSdpEdmeOphY3dalrviBy6tAPBbr2/xxKnWnFdztGh1U2K/YDWa8Ejc55Q35sngJmecXS46BaU2SaOvFSu3kkavJMvsZhHro/btpNGRGT7hDWfzCCeaYLdGpbXpI5ykMBjRDIZHfvjEQWJFy3JXnF+MeHwl5rde3+LPP39h3ss5Uqy2J7TdjEfiXR4Ptlh1BzzpbhKIhkzDjdpnUEdczte4WXS5mi5wbdxjnPlMxwE6cVATiTuSOBn4Qz0rujd4gxKVFMidkekhHE9OpMv9XrDHQ8td84MfOcVLV3bIyof7j+bdLAVT1sKxORK6e43PmlgKcg1bdZur5RLX8x43sy47mRnvlaYeOnGQiTRHwpRvb3ye1DjjHDnO0JOpeTshSQ0Pgt1pWe6aH3pyhV/6nTf5H756g+9+9OTkxj++8mDH3Y+1b7DgTLnkrXPJ3aUnJYNGs9X4vFUuczlbY73ocHm8wjAP2J1EpKMAMok7UDip6SP0hsY0GvQr3EmFGmbInYEZ7zUeP/RitYcVLctd8+yFRSJP8b/57/9w3kvZV976L/74A33/aXfAkjPhrDO6PTy18VmvutwoF7iR99guYnazkEnmk6fuLMt9VnBPb++unNT0EapJjpwYl/tJycHaL6xoWe6awFX8yn/8Kd7ctncQ7+Qp/wYrKuWCE3CzTtmuXV7LT3OzXODtbIl3pgsMspD+KKZMXZg4OGNzJPRGM9PoWOMPKjPeqz9FjBOa3QFNmp3oPsL7wYqW5Z74+LkeHz/Xm/cyjhRPuikNcK1KebvqsF51eT07zVbR4up0gY1JiyTzKUY+IpU4E4k7Njssb6hxkwZvbNpyZFLA9oAmy6xgvQ+2EG+xPCDLKsYXkmHjslm32ao69MuYnTxmlAekuUeRO4hcojLTluNkJqnByUwdy5lWyKQwSaPTKTpNrWC9D0Lbs7LFYjlG2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrrOXhGCOE+DqQzXsdH8IysD3vRXwIgdb6Y/NehOXusKJ1vMm01t8770V8EEKI3z0Oa5z3Gix3jz0eWiyWY4UVLYvFcqywonW8+bvzXsBdYNdo2VesI95isRwr7E7LYrEcK6xoWSyWY4UVrWOIEOLPCCH+UAjxNSHEF4UQ3zXvNb0bIcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rstwdtqZ1DBFCfD/wqtZ6Vwjx48AvaK2fm/e69hBCKOB14HPANeDLwJ/SWr8y14XdgRDiNHBaa/0VIUQb+D3gZ47SGi3vjd1pHUO01l/UWu/OPvwScG6e63kPngUua62vaK0L4FeAn57zmr4NrfVNrfVXZu+PgVeBs/NdleVusKJ1/PnzwL+e9yLexVng6h0fX+MIC4IQ4jHgk8BLc16K5S6wbTzHGCHED2NE6/l5r+W4IoRoAf8U+Cta69F9XsbWWPYHcTcPsjutY4IQ4i8KIf5g9nZGCPFx4O8BP6213pn3+t7FdeD8HR+fm33uSCGEcDGC9Y+01v9s3uux3B22EH8MEUI8AvwG8Ge11l+c93rejRDCwRTifxQjVl8G/rTW+htzXdgdCCEE8A+Avtb6rzzg5fbtjygtajbHGY8uxft1yeOE3WmdYH4eWAL+m9nO60ilFGitK+AvAb+GKXD/46MkWDM+Dfwc8CN37GB/Yt6L+r/++ht87r/8bW4M0nkv5chid1oWy4Ozb39EP/Z3fpvX1sf82T/6KH/rpx+6iC+707JYjhOb44zX1se0A4dfefkq68Ojnu84H6xoWSxHhN+5bAJe/88/+3Earfl7X7gy5xUdTaxoWSxHhC+8sc1i7PHvPbPGD35khc+/ujHvJR1JrGhZLEcArTUvvrHN9z++hJSCz1xa5u2dhHd2knkv7chhRctiOQK8sTlhc5zzmUvLADx/aQWAL1zemueyjiRWtCz3hRDiF4QQf232/t8SQnz2Pq9j0xYwR0O4LVaPr8Sc7ga8+MZRH2R0+Ng2HssDo7X++Qf49gr4q3emLQghPv+wpS184Y0tLi7HnO2FAAgheP6JZf7NKxvUjUbJu3IDPBTYnZblrhFC/A0hxOtCiBeBJ+/4/C8LIX529v5bQoj/fM/0KoT4biHErwkhviWE+AvvvqZNW4C8qnnpSp/nZ0fDPZ6/tMwwLfna9eGcVnY0saJluSuEEN8D/EngE8BPAN/3AQ9/R2v9CeALwC8DPwt8CvibH/Icj/EQpi185e0BaVnz/BPvEq3Zxy++Yetad2JFy3K3fAb451rrZJaG8C8+4LF7X/sa8JLWeqy13gJyIUTvvb5hn9IWjiUvXt5CScGnHl/6ts8vtXyeOd25Ve+yGKxoWQ6CfPbf5o739z7+jjrqw5628OIb23zifI9O4H7H1z5zaZmvvLPLNK/msLKjiRUty93y28DPCCHCWcH8p/bjorO0hb+PiY/+2/txzePEICn4w+vD7zga7vH8pWXKWvPym/1DXtnRxYqW5a6YFct/FfgqJin1y/t06SOZtnBY/M7lHbSGH/jIe4vW9z22iOdIe0S8A2t5sNw1WutfBH7xPT7/5+54/7E73v9lTCH+O752x+de5C67+08iL17eou07fNe53nt+PXAVzz62yIvWZHoLu9OyWOaE1povvLHNpx5fwlHv/6f4/KVlXt+YsDGyqQ9gRctimRtv7yRc201vte68H7etD/aICFa0LJa58YVZFM37FeH3eOZ0h6XY48XLVrTAipbFMjdefGOLs72QC8sfnAcvpeDTTyzzhTe2sUnDVrQslrlQ1Q1f/NYOzz+xjHF9fDDPX1pme5Lz2vr4EFZ3tLGiZbHMga9eGzLOKj7zPlaHd7NX97J1LStaFstcePGNbYSATz9+d6J1uhvy+Ep8qw72MGNFy2KZAy9e3uJjZ7osxN5df89nLq3w8ps7ZGV9gCs7+ljRslgOmUle8fvvDL4jiubDeP6JZbKy4Stv7x7Qyo4HVrQslkPmS9/aoWo0n/kQq8O7+dTjSzhSPPRHRCtaFssh8+LlbQJX8j2PLdzT97V8h+9+ZIEvPOT5Wla0LJZD5gtvbPHshSV8R93z9z5/aZlv3BjRnxYHsLLjgRUti+UQuTlM+dbW9J6Phns8f2kZrW8Pdn0YsaJlsRwit6fu3J9offxsl3bgPNR+LStaFssh8uIb2yy3fJ5aa9/X9ztK8v2PL/Hi5Ye3pceKlsVySDSN5ncub/P8E0t31brzfjx/aYXrg5Q3t6f7uLrjgxUti+WQeHV9xM60uDWQ9X75gb2Wnoe0rmVFy2I5JPbqUB8WRfNhPLoUc34xfGgjmK1oWSyHxIuXt/nIaou1bvDA13r+iRVjUq2bfVjZ8cKKlsVyCGRlzUtv9nn+iQc7Gu7xmUvLjPOKr14b7Mv1jhNWtCyWQ+DLb/UpquZDo5Xvlu9/fAkheCiPiFa0LJZD4MU3tnGV4NkLi/tyvV7k8fGz3YfSr2VFy2I5BL7wxjbf/cgCsb9/U/uev7TM718dMM7KfbvmccCKlsVywGxPcl65Odq3o+Eezz+xQt1ovnTl4Zo+bUXLYjlg9voEH9Sf9W6++9EeoaseutQHK1oWywHz4hvbdEOXP3K2u6/X9R3FcxcXH7q6lhUti+UA2Zsi/eknllDy/lt33o/nn1jmyvaU64N03699VLGiZbEcIN/amrA+yvbNn/VuPjM7cr74EB0RrWhZLAfIno9qv4vwe3xktcWptv9Q+bWsaFksB8iLb2zz6FLE+cXoQK4vhOD5J5b54rd2aJqHI6rGipbFckCUdcOXruw8cIP0h/H8pWX604JXbo4O9HmOCla0LJYD4vffGTAt6gM7Gu6xl4L62w9JXcuKlsVyQLz4xhZSwB+9yynS98updsBTa+2HxvpgRctiOSC+cHmb7zrfoxu6B/5czz+xzO++tUtanPzp01a0LJYDYJiWfPXq4L6n7twrz19apqgbXn7r5Lf0WNGyWA6Af/etHRq9/60778dzF5bwlHwo/FpWtCyWA+DFy1vEnuKTj/QO5flCT/E9jy48FH4tK1oWywHw4hvbfOriEq46vD+xz3xkmdfWx2yOs0N7znlgRcti2Weu9hPe2knueyDr/fKZWavQFy/vHOrzHjZWtCyWfeagW3fej4+e6bAQuSf+iGhFy2LZZ168vMVaJ+DxldahPq+Ugu9/YpkXL2+d6OnTVrQsln2kbjS/c3mH5y8tP9AU6fvlM08sszHKubw5OfTnPiysaFks+8jXrw8ZpuWhHw332KujneQjohUti2Uf2RtV/+lDMpW+m3MLEReX4xMdwWxFy2LZR77wxhZPn+6w3PLntobnLy3z0ptmzuJJxIqWxbJPJEXF7729O7ej4R6ffmKZpKj5yju7c13HQWFFy2LZJ156s09Z6wPPz/ow/ujjJo/+pKY+WNGyWPaJL7y+jefIfZsifb90ApfvOtflC5etaFkslg/gxctbPPvYIoGr5r0Unr+0wteuDRgmJ2/6tBUti2Uf2BhlvL4xOfTWnffjM5eWaTR88Vsnb7dlRcti2Qf26kfzrmft8YnzPVq+cyKPiFa0LJZ94MXL2yzFHs+c7sx7KQC4SvKpi0snshhvRctieUC01rx4eZvvf2IZeQBTpO+Xz1xa5p1+wts703kvZV+xomWxPCDf3BizNc7n7s96Nye1pceKlsXygLw4pyiaD+PicsyZbnDijohWtCyWB+QLb2zz+ErM6W4476V8G0IInr+0zBe/tU1Zn5yWHitaFssD8vKbfT5zSAMs7pUf/9hpRlnFr7z8zryXsm9Y0bJYHpC0rI+M1eHd/NCTK3zq4iJ/+/OvM0xPhtHUmfcCLJbjjiMFn3p8ad7LeE+EEPxnP/kMP/lfvcjP/f2XON0N5r2k9+W//bnvvavHWdGyWB6QP/G952j5R/dP6aNnuvzvfuwp/j+/f523d5J5L+eBESc5S9piOSTsH9H+cFcmN1vTslgsxworWhaL5VhxdA/iFsvx4ej07jwE2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrbCHeYnlAhBBfB7J5r+NDWAaOetxDoLX+2Ic9yIqWxfLgZFrru+tBmRNCiN89Dmu8m8fZ46HFYjlWWNGyWCzHCitaFsuD83fnvYC74MSs0TZMWyyWY4XdaVkslmOFFS2L5QEQQvwZIcQfCiG+JoT4ohDiu+a9pncjhPgxIcQ3hRCXhRB/fd7reTdCiPNCiN8UQrwihPiGEOIvf+Dj7fHQYrl/hBDfD7yqtd4VQvw48Ata6+fmva49hBAKeB34HHAN+DLwp7TWr8x1YXcghDgNnNZaf0UI0QZ+D/iZ91uj3WlZLA+A1vqLWuvd2YdfAs7Ncz3vwbPAZa31Fa11AfwK8NNzXtO3obW+qbX+yuz9MfAqcPb9Hm9Fy2LZP/488K/nvYh3cRa4esfH1/gAQZg3QojHgE8CL73fY6wj3mLZB4QQP4wRrefnvZbjihCiBfxT4K9orUfv9zi707JY7hEhxF8UQvzB7O2MEOLjwN8DflprvTPv9b2L68D5Oz4+N/vckUII4WIE6x9prf/ZBz7WFuItlvtHCPEI8BvAn9Vaf3He63k3QggHU4j/UYxYfRn401rrb8x1YXcghBDAPwD6Wuu/8qGPt6Jlsdw/Qoi/B/zPgLdnn6qOWmOyEOIngL8DKOCXtNa/ON8VfTtCiOeBLwBfA5rZp//3Wut/9Z6Pt6JlsViOE7amZbFYjhVWtCwWy7HCipbFYjlWWNGyWCzHCitaFovlWGFFy2I5xgghfkEI8ddm7/8tIcRn7/M6gRDiZSHEV2dJC39zf1e6f9g2HovlhKC1/vkH+PYc+BGt9WTmTn9RCPGvtdZf2qfl7Rt2p2WxHDOEEH9DCPG6EOJF4Mk7Pv/LQoifnb3/lhDiP5+1Gv2uEOK7hRC/JoT4lhDiL7z7mtowmX3ozt6OpInTipbFcowQQnwP8CeBTwA/AXzfBzz8Ha31JzBu818Gfhb4FPCeRz8hhBJC/AGwCXxea/2+SQvzxIqWxXK8+Azwz7XWySwJ4V98wGP3vvY14CWt9VhrvQXkQojeux+sta5nIncOeFYI8aGDU+eBFS2L5eSSz/7b3PH+3sfvW8/WWg+A3wR+7MBW9gBY0bJYjhe/DfyMECKcRRP/1H5cVAixsrf7EkKEmHjm1/bj2vuNvXtosRwjZjnqvwp8FVN7+vI+Xfo08A9mmfIS+Mda63+5T9feV2zKg8ViOVbY46HFYjlWWNGyWCzHCitaFovlWGFFy2KxHCusaFkslmOFFS2LxXKssKJlsViOFVa0LBbLseL/D9e1JFwR1+cdAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "condition = posterior.sample((1,))\n", - "\n", - "_ = conditional_pairplot(\n", - " density=posterior,\n", - " condition=condition,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - " figsize=(5,5)\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This plot looks completely different from the marginals obtained with `pairplot()`. As it can be seen on the diagonal plots, if all parameters but one are kept constant, the remaining parameter has to be tuned to a narrow region in parameter space. In addition, the upper diagonal plots show strong correlations: deviations in one parameter can be compensated through changes in another parameter." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can summarize these correlations in a conditional correlation matrix, which computes the Pearson correlation coefficient of each of these pairwise plots. This matrix (below) shows strong correlations between many parameters, which can be interpreted as potential compensation mechansims:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWV0lEQVR4nO3df6xcZZ3H8ffn3tvSdVGLFAEBKYZmlxp2QRvAkCgLiIUYWhG13awUF3JXg7urqEEkkRWXWHYTu/gTG6iAEsCtiDViWOTHglFYKlsoPxapoNJaqZRSJIXC7f3uH+eZMkzv3Dm3c+bcM+d+XuakM+c8M88ZoV+eX+f5KiIwM+u1gcm+ATObGhxszKwUDjZmVgoHGzMrhYONmZXCwcbMSuFgY1ZTklZI2iTpwTbXJekrktZJekDS25quLZH0WDqWFHE/DjZm9XUlMH+c6ycDc9IxDHwTQNIbgAuBo4GjgAsl7dXtzTjYmNVURNwJPDNOkQXA1ZG5G5gpaX/gPcAtEfFMRGwBbmH8oJXLULdfYGbF+fODZsSOF0dzld3+9MsPAS82nVoeEcsnUN0BwJNN79enc+3Od8XBxqxCRl8c5ZDTZuUq+3/LN74YEfN6fEuFcTfKrEoEAwPKdRRgA3BQ0/sD07l257viYGNWMVK+owCrgDPSrNQxwNaI2AjcDJwkaa80MHxSOtcVd6PMKkTAQEFNAEnXAscBsyStJ5thmgYQEZcBNwGnAOuAbcBH0rVnJH0RuDd91UURMd5Acy4ONmZVIhgcKqbZEhGLO1wP4Jw211YAKwq5kcTBxqxiVNPBDQcbswqRYKCgAZmqcbAxqxi3bMysFEUNEFeNg41ZhUhu2ZhZSQYHPWZjZj0muRtlZqUQKuZRhMpxsDGrErdszKwsHiA2s56TPEA8prR94PXAbOA3wAfTzl6t5XYAa9Pb30XEqd3Ua1Znde1GdfuzPgvcGhFzgFvT+7G8EBFHpMOBxqwNARpQrqPfdBtsFgBXpddXAQu7/D6zqS0NEOc5+k23Yzb7ps12AP4A7Num3AxJq4ERYGlE3DhWIUnDZLu8oyG9ffrMqTOkdOCWmZN9C6V56o3PTvYtlOr5jS8/HRH75C1f0+cwOwcbST8F9hvj0gXNbyIiJEWbrzk4IjZIegtwm6S1EfHr1kJps+blADP2mR6zF+bbi7UOvrTytMm+hdIsG1412bdQqru+8ORv85bNNs+qZ7TpGGwi4sR21yQ9JWn/iNiYUkBsavMdG9Kfj0u6AzgS2CXYmE15BW6eVTXd9vxWAY1seUuAH7YWSPuY7pFezwKOBR7usl6zWhJiQPmOftNtsFkKvFvSY8CJ6T2S5km6PJU5DFgt6X7gdrIxGwcbs7EUnF1B0nxJj6YUu7vMFktaJmlNOn4l6dmmazuarnXd9+1qBDYiNgMnjHF+NXB2ev1z4PBu6jGbKoocs5E0CHwdeDdZorl7Ja1q/o99RHyyqfw/kg1xNLwQEUcUcjM4lYtZ5QxoINeRw1HAuoh4PCJeAq4jW67SzmLg2gJ+wpgcbMyqRPm6UDlbP7nT6Eo6GDgEuK3p9AxJqyXdLWnhbv6inabOQhazPiBgaDB3G2BWWr/WMNFc380WASsjYkfTuVxLVvJysDGrkGzzrNzB5ukOub4nkkZ3ES05pIpesuJulFmlFNqNuheYI+kQSdPJAsous0qS/hLYC/hF07nCl6y4ZWNWJQXmjYqIEUkfJ8vTPQisiIiHJF0ErI6IRuBZBFyXMmQ2HAZ8S9IoWaOk6yUrDjZmFVL04woRcRNZTu/mc59vef8vY3yu8CUrDjZmVSIxODg42XfREw42ZhUypR/ENLNyOdiYWc9J5F0d3HccbMwqJf9Dlv3GwcasYvpx+4g8HGzMKkSCoSHPRplZjzU2z6ojBxuzKpFno8ysJA42ZtZzwlPfZlYGeerbzEogxNDgtMm+jZ4opL2WYwf3PSRdn67fI2l2EfWa1Y2AQQ3mOvpN18GmaQf3k4G5wGJJc1uKnQVsiYhDgWXAJd3Wa1ZLEgMDg7mOflNEyybPDu4LgKvS65XACVJNFxOYdWlAg7mOflPEmM1YO7gf3a5M2j1sK7A38HQB9ZvVhtBE9iDuK5UaIJY0DAwDDO3Zf5HbrFuSmDY4fbJvoyeKCDZ5dnBvlFkvaQh4PbC59YtSGorlADP2mR6t183qT7VdZ1PEr8qzg/sqYEl6fTpwW8vmymZGI5VLcQPEOWaKz5T0x6ac3mc3XVsi6bF0LGn97ER13bLJuYP7FcB3JK0DniELSGa2CxU2rZ0n13dyfUR8vOWzbwAuBOYBAfwyfXbL7t5PIWM2nXZwj4gXgQ8UUZdZnRX8uMLOmWIASY2Z4jwpWd4D3BIRz6TP3gLMp4tc4PXsHJr1rQmts5mVcnE3juGWL8ub6/v9kh6QtFJSY/w1d57wvCo1G2U21U1wNqpT+t08fgRcGxHbJf0D2Xq447v8zjG5ZWNWIY1uVJ4jh44zxRGxOSK2p7eXA2/P+9mJcrAxq5JiH1foOFMsaf+mt6cCj6TXNwMnpZzfewEnpXO7zd0os0pRYY8i5Jwp/idJpwIjZDPFZ6bPPiPpi2QBC+CixmDx7nKwMauQLCNmcR2OHDPF5wPnt/nsCmBFUffiYGNWKcWts6kaBxuzCpHEkJ+NMrNe8x7EZlYOicE+3BgrDwcbswrJWjYONmbWc/XdYsLBxqxChBga8ACxmfWahNyNMrMyeMzGzHpOiAEcbMysBG7ZmFnPqcAHMavGwcasUsSgPBtlZj0meZ2NmZWkrt2oQkJoN7lpzKyZnOu7nW5y05jZqwkv6htPN7lpzKyF19m0N1Z+maPHKPd+Se8EfgV8MiKebC2Q8t4MA+yjPfnSytMKuL3+cP7pN0z2LZTmNVvrOQBaBKnYZ6MkzQcuJduD+PKIWNpy/VzgbLI9iP8I/H1E/DZd2wGsTUV/FxGndnMvZf1T/xEwOyL+CriFLDfNLiJieUTMi4h5rxv4s5JuzaxKihuzaRriOBmYCyyWNLel2P8C89LfzZXAvzVdeyEijkhHV4EGigk23eSmMbNXycZs8hw57BziiIiXgMYQx04RcXtEbEtv7yb7+9sTRQSbbnLTmFkTkY3Z5DkoLv1uw1nAT5rez0jfe7ekhd3+tq7HbLrJTWNmrSa0qK+I9LtZrdLfAfOAdzWdPjgiNkh6C3CbpLUR8evdraOQRX3d5KYxs1cUPECcK4WupBOBC4B3NQ13EBEb0p+PS7oDOBLY7WDjaQGzihGDuY4c8gxxHAl8Czg1IjY1nd9L0h7p9SzgWLpczuLHFcwqpMinvnMOcfw7sCfwn5LglSnuw4BvSRola5QsHWOh7oQ42JhVSrFbTOQY4jixzed+Dhxe2I3gYGNWOarp6IaDjVnlaLJvoCccbMwqxHsQm1mJ3I0ysxLI3Sgz6z0hbwtqZuVwy8bMes4DxGZWGnejzKzHhAeIzawU8gpiMyuLWzZmVgK3bMysBMq7V03fcbAxq5BsgNgtGzMrgWejzKz3JPDjCmZWhrq2bAoJoZJWSNok6cE21yXpK5LWSXpA0tuKqNesfrJ1NnmOXN8mzZf0aPq799kxru8h6fp0/R5Js5uunZ/OPyrpPd3+sqLaa1cC88e5fjIwJx3DwDcLqtesdorKrpAz/e5ZwJaIOBRYBlySPjuXLBvDW8n+bn9DOdNwtlNIsImIO8mSz7WzALg6MncDM1uyZJoZrzyukOd/OXRMv5veX5VerwROUJZmYQFwXURsj4gngHXp+3ZbWSNRudKAShpupBJ9bvSFkm7NrEo0gaOQ9Ls7y0TECLAV2DvnZyekUgPEEbEcWA5w6NAbY5Jvx6x8kY58Cku/W4ayWja50oCaWaDId+SQ5+/dzjKShoDXA5tzfnZCygo2q4Az0qzUMcDWiNhYUt1m/WU059FZx/S76f2S9Pp04LaIiHR+UZqtOoRscud/uvhVxXSjJF0LHEfWh1wPXAhMA4iIy8gy8p1CNsi0DfhIEfWa1VK+VkuOr8mVfvcK4DuS1pFN8ixKn31I0vfI8nuPAOdExI5u7qeQYBMRiztcD+CcIuoyq7UAFThamSP97ovAB9p89mLg4qLupVIDxGbGRAaI+4qDjVnVFNSNqhoHG7OqqWescbAxq5Qg77R233GwMauaesYaBxuzynGwMbNSuBtlZmUocp1NlTjYmFXJxB7E7CsONmaVEjBaz2jjYGNWIaK+3ah6buNuZpXjlo1Z1Xg2ysx6zgPEZlYWeYDYzEpRz1jjYGNWKYGnvs2sDEHUdIDYU99mVVPchudtSXqDpFskPZb+3GuMMkdI+oWkh1La7A81XbtS0hOS1qTjiE51OtiYVUgExGjkOrr0WeDWiJgD3Jret9oGnBERjRS8/yFpZtP1z0TEEelY06lCd6PMKiZ2dNlsyWcBWUYUyNLv3gGc96r7iPhV0+vfS9oE7AM8uzsVFtKykbRC0iZJD7a5fpykrU1Nrs+PVc5symsMEOc5OqffHc++Tbnb/gDsO15hSUcB04FfN52+OHWvlknao1OFRbVsrgS+Blw9Tpm7IuK9BdVnVlMTGiAeN/2upJ8C+41x6YJX1RgRUvsnsiTtD3wHWBIRjWbX+WRBajpZyuzzgIvGu9mi8kbdKWl2Ed/V8NQbn2XZcGvyvvp6zdapM3y27flSugn9q6D/eyLixHbXJD0laf+I2JiCyaY25V4H/Bi4ICLubvruRqtou6RvA5/udD9l/hv+Dkn3S/qJpLeOVUDScKNJ+PI2/wtpU1NE5Dq61Jx2dwnww9YCKWXvD4CrI2Jly7X9058CFgJjDqE0KyvY3AccHBF/DXwVuHGsQhGxPCLmRcS8aa+ZOv+lN9tpYmM23VgKvFvSY8CJ6T2S5km6PJX5IPBO4MwxprivkbQWWAvMAv61U4WlzEZFxHNNr2+S9A1JsyLi6TLqN+snZcxGRcRm4IQxzq8Gzk6vvwt8t83nj59onaUEG0n7AU+lgaijyFpUm8uo26yfRBSyhqaSCgk2kq4lm7OfJWk9cCEwDSAiLgNOBz4maQR4AVgUdV2Tbdatmg5XFjUbtbjD9a+RTY2bWQd1/e+wVxCbVYmf+jazspT0uELpHGzMqiTwmI2ZlcGzUWZWFg8Qm1nPpf1s6sjBxqxqHGzMrNciwrNRZlYOBxsz6z2P2ZhZOdyNMrMyBDDqYGNmPRYRjL68Y7JvoyccbMwqxt0oM+u9GnejvNGvWaXky4bZ7YxVnvS7qdyOpv2HVzWdP0TSPZLWSbo+bY4+LgcbsyqJrBuV5+hSnvS7AC80pdg9ten8JcCyiDgU2AKc1alCBxuzCgkgRkdzHV1aQJZ2l/TnwrwfTOlbjgca6V1yfd5jNmZVEkHkn42aJWl10/vlEbE852fzpt+dkeoYAZZGxI3A3sCzETGSyqwHDuhUoYONWZXEhGajyki/e3BEbJD0FuC2lCtqa94bbNZ1sJF0EFmO733JWoHLI+LSljICLgVOAbYBZ0bEfd3WbVY/UUQXKfumAtLvRsSG9Ofjku4AjgS+D8yUNJRaNwcCGzrdTxFjNiPApyJiLnAMcI6kuS1lTgbmpGMY+GYB9ZrVTwA7It/RnTzpd/eStEd6PQs4Fng4pWG6nSxFU9vPt+o62ETExkYrJSL+BDzCrv23BWT5giMlJ5/ZyBVsZq9W0gBxnvS7hwGrJd1PFlyWRsTD6dp5wLmS1pGN4VzRqcJCx2wkzSZrZt3TcukA4Mmm940BpY2Y2U5l7WeTM/3uz4HD23z+ceCoidRZWLCRtCdZX+4Tzbm9J/gdw2TdLPZ4/WBRt2bWP4KJzEb1laLS704jCzTXRMQNYxTZABzU9H7MAaU0bbcc4LVvml7PTT3MxlXcAHHVdD1mk2aargAeiYgvtym2CjhDmWOArU1z/GbWUN4K4tIV0bI5FvgwsFbSmnTuc8CbASLiMuAmsmnvdWRT3x8poF6zGqpvy6brYBMRPwPUoUwA53Rbl1ntNaa+a8griM0qJCIYfWmkc8E+5GBjViUTe1yhrzjYmFVJQIw42JhZzzm7gpmVINyyMbNSRDjYmFkJAka3+3EFM+u1kh7EnAwONmYV4jEbMyuHx2zMrCzuRplZ77kbZWZliAhGt9fz2SgnqTOrkjRmk+foRp70u5L+pin17hpJL0pamK5dKemJpmtHdKrTwcasSiqUfjcibm+k3iXLgLkN+K+mIp9pSs27plOFDjZmVZLGbHrdsmHi6XdPB34SEdt2t0IHG7NKKacbRf70uw2LgGtbzl0s6QFJyxr5pcbjAWKzConRCT2uMG6u74LS75JyvB0O3Nx0+nyyIDWdLEnBecBF492sg41ZpUzocYVxc30XkX43+SDwg4h4uem7G62i7ZK+DXy60826G2VWJeWN2XRMv9tkMS1dqEZG25RdZSHwYKcK3bIxq5LyHldYCnxP0lnAb8laL0iaB3w0Is5O72eT5Xz775bPXyNpH7JkB2uAj3aqsOtgI+kg4GqyAaYg6zde2lLmOLLI+UQ6dUNEjNu/M5uKoqQ9iPOk303vf0OWKru13PETrbOIls0I8KmIuE/Sa4FfSrqlKQF5w10R8d4C6jOrNT8b1UYaKNqYXv9J0iNkkbA12JhZBxHByKg3z+oo9e+OBO4Z4/I7JN0P/B74dEQ8NMbnh4Hh9Pb5u77w5KNF3l9Os4CnJ6HeyTKVfu9k/daDJ1J4R7hlMy5JewLfBz4REc+1XL4PODginpd0CnAjMKf1O9IageWt58skafV404l1M5V+bz/81iAYrWmwKWTqW9I0skBzTUTc0Ho9Ip6LiOfT65uAaZJmFVG3Wd2MRuQ6+k0Rs1ECrgAeiYgvtymzH/BUWql4FFmQ29xt3WZ1VNeWTRHdqGOBDwNrJa1J5z4HvBkgIi4je4jrY5JGgBeARRGVDc2T2o2bBFPp91b+t0bUtxul6v6dN5t6Dh3aN778ug/lKrtgy1d/WfUxqGZeQWxWKeHZKDPrvYC+HPzNww9iNpE0X9KjktZJ2mXnsjqRtELSJkkdH6Drd5IOknS7pIclPSTpnyf7ntqKbIA4z9FvHGwSSYPA14GTgbnAYklzJ/eueupKYP5k30RJGo/UzAWOAc6p7j/bqG2wcTfqFUcB6yLicQBJ15FtnVjLxy4i4s604rv2+umRmgA/rjAFHAA82fR+PXD0JN2L9UiHR2omXXiA2Kz/dXikphrCi/qmgg1kmwQ1HJjOWQ10eqSmKuo8G+Vg84p7gTmSDiELMouAv53cW7Ii5Hmkpjrqu4LYs1FJRIwAHyfbQf4R4HtjbYNRF5KuBX4B/IWk9Wl7yLpqPFJzfFMGx1Mm+6bGkrVsPBtVe+mJ9Jsm+z7KEBGLJ/seyhIRPyPbK7fyIoKXazob5ZaNWcWU0bKR9IG0wHE0bXLertyYC10lHSLpnnT+eknTO9XpYGNWMSXtZ/MgcBpwZ7sCHRa6XgIsi4hDgS1Ax264g41ZhURJK4gj4pGI6LTt7s6FrhHxEnAdsCANuB8PrEzl8uQK95iNWZWs59mbz40b8u5iOWO89LsFaLfQdW/g2TSp0ji/S7qXVg42ZhUSEYU9rzZeru+IGC8DZk842JjV1Hi5vnNqt9B1MzBT0lBq3eRaAOsxGzNrZ+dC1zTbtAhYlbb0vZ1su1/onCsccLAxm5IkvU/SeuAdwI8l3ZzOv0nSTdBxoet5wLmS1pGN4VzRsU7vQWxmZXDLxsxK4WBjZqVwsDGzUjjYmFkpHGzMrBQONmZWCgcbMyvF/wPZqxE8c4CsDAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "cond_coeff_mat = conditional_corrcoeff(\n", - " density=posterior,\n", - " condition=condition,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - ")\n", - "fig, ax = plt.subplots(1,1, figsize=(4,4))\n", - "im = plt.imshow(cond_coeff_mat, clim=[-1, 1], cmap='PiYG')\n", - "_ = fig.colorbar(im)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So far, we have investigated the conditional distribution only at a specific `condition` sampled from the posterior. In many applications, it makes sense to repeat the above analyses with a different `condition` (another sample from the posterior), which can be interpreted as slicing the posterior at a different location. Note that `conditional_corrcoeff()` can directly compute the matrix for several `conditions` and then outputs the average over them. This can be done by passing a batch of $N$ conditions as the `condition` argument." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sampling conditional distributions\n", - "\n", - "So far, we have demonstrated how one can plot 2D conditional distributions with `conditional_pairplot()` and how one can compute the pairwise conditional correlation coefficient with `conditional_corrcoeff()`. In some cases, it can be useful to keep a subset of parameters fixed and to vary **more than two** parameters. This can be done by sampling the conditonal posterior $p(\\theta_i | \\theta_{j \\neq i}, x_o)$. As of `sbi` `v0.18.0`, this functionality requires using the [sampler interface](https://www.mackelab.org/sbi/tutorial/11_sampler_interface/). In this tutorial, we demonstrate this functionality on a linear gaussian simulator with four parameters. We would like to fix the forth parameter to $\\theta_4=0.2$ and sample the first three parameters given that value, i.e. we want to sample $p(\\theta_1, \\theta_2, \\theta_3 | \\theta_4 = 0.2, x_o)$. For an application in neuroscience, see [Deistler, Gonçalves, Macke, 2021](https://www.biorxiv.org/content/10.1101/2021.07.30.454484v4.abstract)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this tutorial, we will use SNPE, but the same also works for SNLE and SNRE. First, we define the prior and the simulator and train the deep neural density estimator:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4683b28ba0614e87b33854d207e30da2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Running 1000 simulations.: 0%| | 0/1000 [00:00" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from sbi.analysis import pairplot\n", - "\n", - "_ = pairplot(cond_samples, limits=[[-2, 2], [-2, 2], [-2, 2], [-2, 2]], figsize=(4, 4))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Analysing variability and compensation mechansims with conditional distributions\n", + "\n", + "A central advantage of `sbi` over parameter search methods such as genetic algorithms is that the posterior captures **all** models that can reproduce experimental data. This allows us to analyse whether parameters can be variable or have to be narrowly tuned, and to analyse compensation mechanisms between different parameters. See also [Marder and Taylor, 2011](https://www.nature.com/articles/nn.2735?page=2) for further motivation to identify all models that capture experimental data. \n", + "\n", + "In this tutorial, we will show how one can use the posterior distribution to identify whether parameters can be variable or have to be finely tuned, and how we can use the posterior to find potential compensation mechanisms between model parameters. To investigate this, we will extract **conditional distributions** from the posterior inferred with `sbi`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note, you can find the original version of this notebook at [https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb](https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb) in the `sbi` repository." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Main syntax" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sbi.analysis import conditional_pairplot, conditional_corrcoeff\n", + "\n", + "# Plot slices through posterior, i.e. conditionals.\n", + "_ = conditional_pairplot(\n", + " density=posterior,\n", + " condition=posterior.sample((1,)),\n", + " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", + ")\n", + "\n", + "# Compute the matrix of correlation coefficients of the slices.\n", + "cond_coeff_mat = conditional_corrcoeff(\n", + " density=posterior,\n", + " condition=posterior.sample((1,)),\n", + " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", + ")\n", + "plt.imshow(cond_coeff_mat, clim=[-1, 1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analysing variability and compensation mechanisms in a toy example\n", + "Below, we use a simple toy example to demonstrate the above described features. For an application of these features to a neuroscience problem, see figure 6 in [Gonçalves, Lueckmann, Deistler et al., 2019](https://arxiv.org/abs/1907.00770)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sbi import utils as utils\n", + "from sbi.analysis import pairplot, conditional_pairplot, conditional_corrcoeff\n", + "import torch\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import animation, rc\n", + "from IPython.display import HTML, Image\n", + "\n", + "_ = torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's say we have used SNPE to obtain a posterior distribution over three parameters. In this tutorial, we just load the posterior from a file:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from toy_posterior_for_07_cc import ExamplePosterior\n", + "posterior = ExamplePosterior()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we specify the experimental observation $x_o$ at which we want to evaluate and sample the posterior $p(\\theta|x_o)$:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "x_o = torch.ones(1, 20) # simulator output was 20-dimensional\n", + "posterior.set_default_x(x_o)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, we can inspect the posterior marginals with the `pairplot()` function:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAACaY0lEQVR4nOz9eZxk6VXfCX/P8zz3xpJZVa3eW2q11m6phQDtrELDgAxYzHhszD4sBmPw6xljjD3GxvYwHuN5PV7GC/BigxnMYoTB7GB4sREWHiEktO9Sa+vWvnR3VWVGxL33WeaP8zz33ojK6s6uyqqs7I7Tn/hEdlZmxM2IG797zu/8zu9ISoltbGMb2zgpYY77ALaxjW1s45HEFrS2sY1tnKjYgtY2trGNExVb0NrGNrZxomILWtvYxjZOVGxBaxvb2MaJCncJv7PVSBxNyOU+wPP//D9NsQI/FcIUooM4SSSrXyebSAaSBUzSZxRIMnoLL3YU+UckyfCOJ71JlHwP0gmSwHQgXjA+fx3AtiA+Ybx+bXzCNgnbJsQn3DJguohZdUgTkM4jTQs+kLoOQoAQSd5DjBACKUQ9lBDyMcV8f3mn5e/GX7jk9+Ol5qsf+sklP7QYxIjeWwPGIM6BtWBHX1cOjCFVjlQ5cIZYW1JliZUhTC3Jgp8ZohNCJYSJvs+hFpKDaCE5SALJMHrvyzGN/l+S/pwFTP6dcu64pN8zeq9vdnmM/GeXc6ScF14g6rkgQfJ5oueN8SAeTADToOfGKuFWCdvBH778ex/2fbgU0NrGNRJ+KsQa/AzCLBFrCJNEcolUJbB6ExsRk5B80gkgkvqTTjZOk/7zn4SUz/KU/x8UJ1IUYhTwBoIoeHnBdDIAWAsmA5lt9GS1jWBbPUGTFYxPOCeYKiCNxRgBH/RD3XYkGxVXUyShn7WUEoLV70WjByRy2cB1ReJigGUt0gOURZwF58DZDFQKUql2RGeI0wxYtSFMheiEbibECmKtoBUdxJp80UoboKUXrdQDzgBYCAMwCXreOD1njEuIJMRErNWvjSnnTSIlISWI0RCjEIMhBiEFQ/D5/IggnUFCwnhR0PJgKunPAwpoHiIeVaDlQ+Te+xc8+YYdPfkf5REnEGoFLD+DWCfSLIBLmDpgXcDahHMBIwlrItboiWdNxGQAAzCy/oGPSfTiWYAqiX4vCSHm+yR0nSNGwXeW2BliZ5BWgcw2+QTtREErCHaVs64WojOYLpGsw1QGW1mwgnRBsxBrkc6DCClEREzOvvSWkiAEBS49avLBXpXX/2FjE7CsRSTfVzmrck4zrAJW1pImTgGrssSJJTpDmGbAqgWfwcrPhFBDrPJFy+VzwGmGlFwaMqMenEYXqvw9MQkRMCYDlUlUlcdKonIBayKVif054yT2f6JPhpiEEA1tsIRoWHWOEAy+s4RgSEFInd7HIEhn9EJmIXVCEs3Ckjnc+/aoAa1//9r7+JeveA/33b/k1tNTvunznsRffMnTHtXgFV2+VflknURkEjBVpKoCVeVxJjIpJ6ANOBMxDKBlJGFImNGJCBDzJTmiYFVuACEZfDR6otaWLhiarqLrLMFbgrPghWAN0mm5kqyCl5args1XVWu1vLQiYASJCTGCycCjWVZCJJByKdhnWzGSxCAmkuI1AlQlDsqwFBn0Zm3OsBSwSlmYrB0AyxlCZTTDmihghVrwU4iV6IWq0uw6TBWo4jTmTCkhrmTYObs2cTgsUaAqmZNIwpmIswpOtVWwmlhPbQNOArUN+VwZXmufz4U2Otpg6aLF2YrWW1rr6DpLDIZgEikYktdzIFotIZMkxCsID7XrQ8ejArR+5tUf5G//ylv57Cdex5/7/Kfw++/+JP/od97FPZ/Y4x9+1WdRu0dnvyE6SPmkTdOATAOTeUdVeeZ1x6zqqE1gp2pwJlKbwMR4jKR8H6lEgQzAEvsTsgBUl3P2mISAXlV90pPTJ8MyVLTBst9NWPqKlXcs2wrvLW3j1rIv8YJdCaEF0wqu1lIyVgY7SbhGiFawrcU6g6kd0npkZREftJzynuQ9YnL2RacZlyRS6F8ZvTvujEvMcF8yrVISlgyrrkguA1atGVaYWFIpBTNv1c2NZtUTwe9AqMDvJGKdlMecRqSKVBOPdRHnApUNWJNwNvRZdXmvpVywJGFF3/faBJzR82FmO2rjmeRbJYHKKGhZiYR8UWuio0uWZahZhoplqDjnpnTBst/VtLXFB0PrNfsKXm/RG4K1pEYfR4JgD/kxPfGg9er3fZof+LW38cXPuIkf/5YXYo3w577gyfzI77+Xf/Q772Kv8fzINz6P6rCvyEkKGRGkNiFW0/qJC0ydZ6dqqY1nt2qoJF818wk4MR6bQauSgJGIHWVcJdMKCDEZQjJEhJAMXbJ0yeKjYWlrmuA0o3Oepa8wkmitRSThrSU4QzSW5A1gSEZINmlJYHNpkIlhCQYMSFSwNCKKQSLIKPtSXgtStEgIJKNNg7WM67h4rhFJKCZnkH2WJUOGZYwCllvPrgpgxcoQKiFWWgaGzF2Fmh6s4kQzbDPzWBeYTjsqG6hdYGJLaZcz7FFWXS5OpdwzkqhMyOdHZGZbKgnM8/34PIHh/FglRxOr/t+MJHwytOKICEYSncnnkjF4Y0AsYiB4Q4qJ5HLzIHKoONGgtWg9f/Xn38gdN8z551//XGwuBUWEv/TFT+fU1PF3f/VtfM/Pv5H/62uf86gDrtIZTC4hVdSTtvLMq47duuF0tWJmO3Zcw8R45rZlIgpWE9P1J+JUWi0fxQNgc7uwAJZ+bQhJiBja5Hrg2gtTumQ562fs+wn7vmZiPU1w7FU1TedovaN1eqWNzhJbg2kNyWjGpSSsnrgkiFUGLCvYLmIBnBk+/NYoYInovTEKXDBkXOmQn4CjjouVhdYqAe9cvh+VhIVwrzPZPrEKVBPBT83AX00gTHKGVSXCToRJwE0C01lL7TynJi0T65m5jqnrcBL77LpkSkAPMCW7NpLWwGluGyoJ7Jgmf89jc5YFShEEhEWcsIoV582UKmfyAEtTAVCZQBctlYmEJLTe0diI95EmQhRL8IJp0O7jIeJEg9YP/d49fOTsil/4rs/j9LS64N+/+fOezKoL/IPfeief2mv4kW98Ptfv1MdwpFc4MplqrfJWE+uZu5Yd1zKzLafdiqnpmJu2B6uptNT9Serz134t24L1jEuzLUObLF0GrnOmpUuWiXj2TMuOU9BahorKBJrK0XjHwlV0wdLYCl8rcCVjMZWAaFkYXebMWkhicZUQS/nQGYwIxhjovHI01iqnJX7oLBIQ7LGXigWw1srCUhJW1UC655Iw1nYAq2mWMdQqZQmVEGZD0yXMEnESkbmnmnimk45T04ZZ1XGmXjK1nh3XMLNdn1VXErQMzKBVwGecYRfAqsUzlXyemJaK0GfloOdEQOiSw5LWvg9aMpZzaBUcPloqE5QDtQFnHa1Vkt6DUgiVxQQOFScWtN77yT1+7A/ex1c973Ze+OTrL/pzf+GLnsaNuxO+75fewhf/49/nq553O896/GkE+Pj5Fas2cGZe85K7buLpN+9evT/gCCPJ0P0pXZ7CX02MZ27aDFoN0x60OqzEfHLqSV0zlIl9tpUzh75MzCdrmywdFiORVayI1uiHIiRikr7DtPJV/3WbNVZiEt5YYhCiMYQwaMGMV3LWeEGSASIhWIwI4iMpWSXnvZ7hEgIpGSQZUoyICAkl8686OT/KsoZvjcpCMb0mC6PZVnJGS0KnZWByCt6xlIWVylpCIdxriJMIdcTVnsmkYz5p2a0b5q7lTM6uZ7ZlN2dLU9P1oGQZLkilRLTEDGixv5AVsCoXM/3d4ZzoksWSCEbL95VUTExHSKbPtryx+T5iQiQag42xf+6mCiqRsKmXaRwmTixo/e+/8XamzvJ9X/HMh/3ZP/O823nGraf4kd9/Lz/1hx/Aj07mQnv8oMCffu7t/G9/6jPYnZyQl2WDvtHKaQCscuKesismpmPHNOyYBoOC1fiErIlU+UpcSzxwVCJCBi1Dlwwthql0dMZRS2A/Tpgb5UAa2zCzLUtX00TLXjehjY79umbZVTSdY99O8J3FW0esDLHWP6J0GWMFtjEkEWwlJCPYJmAar8dXMi4RzbjIpeLFyPkrmW09pB4rl4V11UsbUl31GZZmWSpriE5ULFzrvZ/r6+B3lL8K84jseFztObWzYqfuOD1Zcf1kn13Xcn21z9y0zG3D3LRrWVMBps0oWdc4oyoXsGkGKzM62bT9EWklEKIQxDA1HV2yRGOY27YvOWfR0iUFZp8MTXDUJrAKji5YUoLQqXhWwqO4PPy9d36c33/XJ/nbL7ubm05NDvU7n/H4M/zwNzyPc6uOB/c7QkrcfGrCrLJ84nzDv/mv7+Mn/p8PcN/9C37y217IvD45L41stIoLX1E6Q1r++QxQngq9+pYTswBWVUoFwMqFM14RqEhYcuubRJSWFYmp6ZT3wjC3jXaYMMqVRH0tfVRuxZmIKxyHdTRB0OuIwXTKVRkPIQGIlg1ikICS8ckiPmYiPqpMgkzKMyLnNzOuq0nMmw15g4y+dlY5uky6JyskO2RYoRr0V7FSPV4h3qkirgrUdWBWeeZVy27VsOtadjJQFcA6bZZ91qTva+wBajNKiWiJ/UXMSKIiYvuMTIvvzZkYm39vzIuVk6eSwCRpZhaT6bNuI5E9O6GxEbFJFfj2UQpaqy7wv//GO3jqTTt88+c9+RH//ulpdQH/deuZKd//smfxWbdfx3e//A1818+8nv/7W1/YE/vXakgeq4Hhszh0hQaOopSD5VZJYEfazEdk3ZYkBSSBiqyb4uLAVUmkSxEMVCkQEP1Q5Ct5Ix5DojOWJjpq4/HRMrMd+65m4WqMJJauIiWhM4loHD4ajMudQNGMS6KOpZAMSfTYJCbECiZGKKCUSw+JaeguBrKO6woq5w/MsuyQZfWKd0tyttdixUozkFAPOqxYFcI981l50iHOEmkSsDMl3ec5wzpdrzhTLbnOLdi1Dde7vZ4K2DFNf4GyxAv4ys0otMAYrMpFbAgFLiMJm9LodyK1eKII88xzFq5r3HluoqMygT1fs19NtAPtKjqnwHWYOHGg9S/+83t4/6f2+alve9GR66/+u89+POdWHd//y2/lR//Le/lLX/z0I338I480uqGfxXiAQE91OMpX9ASrhAvAygAWBYXyytqN4USLJhGkhBEIaPmwI22PcAFDnTuRXbJMjWWe9ERexYqdULF0NU4Ci6pGJOWrbiKkiuQMklBphAOS6Bxj0vJPJREOYwQJSUHDZEmEyEDKh5BfFLMOXFciDlK9l7LQWdVjVU67hRM76hSOhKOTLG0oncKpjmjFSSTNAnbqmU47Tk8bdqqW6ycLTlcrTrkVj6v2mYrnlF3pxcm07EhLJZ5p32C5OCgUvqqcE8P/D1HogSI4DqNzw5KYSofJ0oqi6+t/N/OhRR7hJHDOzVj6CucinUmPzjGeN3/oQf7VK9/H17zgdr7orpuuyHN8w4vu4NXvu59/+rvv5nOfegPPf9LjrsjzHGmMgEuV6weDuWEg2cfc1RiwDhMWKWIpqjw5XUnQq7p0dKKn1cR0VMn3ncaA9GRwJQGfDM5EmuD60aBVq7la6KxmUEkwNUAeto165TZd/lB1EWJEQia4kwpQMTpoJzlbY9yZOupsaywiBR1BkhHxnjVZhXhP1qwT7m5UClbSfx1qFY+mWseyqiowqVQ0PHOdEu6m1UaL+NwZ9kxNu9ZgKWU/sMZNXfC+jspA/VmNAvPaQVawGjdmStkZELQ337I5iB9RrV8lIZ8Lhtp6Kht0lrGMFR0iTgxoPbDf8pd/7g3cuFvz/S971hV7HhHhB//0s3nDvQ/wV37+DfzH7/6ia5qY7wft81xgSnol9NHq1S4LQ0uYzF3VEqkephR8uLAieoKnxFQCgQ6MnqBV8liU1+qSzboeQzQrOutYJcfMduyFiWqJ7Jzzled+oK0dQcgfbgDB1mRphGKARINxml2pflMUvIzR8R5AJCrnFcJo3Ofoy8SxvKGIRnvVe1VlLZYbtFgTh88D0H5qetFoN88D8HMI04SfJdKOx0wCO7sr5nXHmcmKG6b77LiWm+rzzG3Lrl1xyi6ZijZbhmzar12kxhnUYaJkViGfUyFnWC2GmAzt6DJXQLI/NzKHNjyWSmUWUTloQ2LHtezZCcbEwYXkEHHtfhpH0frId/3M6/jI2RU/9x2fw5nZhZqso4zT04p/9rXP4Wv+1R/yA7/2Nv7xV3/2FX2+o4riyBDzSdYlm082PclivkeGxP1ipWDoBaYHn+QRCKMPvSFRE4kEulIaSuh5rpCzkJCELgWq5MFp6drWqp4GWNR6rV61lhhBkkHPc0EmJdMSQp3lEK1BvHahpMsCU5fl1SkpgOiLc+XKxAJY1uZBb8PaIPSm6r0aJA5DVqWA1c8S5llSMw1UtWdaeXbqlp2qYce17LiGuS1Z1iAULiC1Jm14iJJvHONX5GJgBfSZEuSsrBD4DIQ+QD1KbwPCKunndj9OWBjfzzE+Uub4mgets4uO/8+/ex1/9P77+edf9xye/6SLa7KOMl7w5Ov5n7746fyL37uHz3vqDXzV82+/Ks97uTEMNeuJVpTrbS7PyglILgm1LBTNVPrHSFj1Txi+N3qOcECGYvOISIWWiQBTkwWpMsr2BAIdMZcK0wxwpRu58hXORoI3dFQEAQmWZHTsRwl6tTZJYpBcMgJIO2i4JDoFsKKUt9r7IuZaURT0jiTbKkp9yZlW8cY6UN4w4rGKgDTzV0rAJ8JUB59lGphMO6Z1x5npilPVisfVS66rFj1gzU2j+ru+HPRrWrtNHmu4WI3fz9HXlJnToskzfSlYMqtN+qGU/AWwxjKJUnKukqXK+pPzpmMaPc6Evsv9SOKaBq3XfuB+/sYvvpn7Hljwj/7sZ/GnnvOEq/r8f/lL7uSPP/gAf/OX38LTb97ls5943VV9/ocNYZjZE53UhxFwZR6hywr2VazppKWVQKe6cSIDl9VnWuWETppplayqXDfXAWz4f328pFdYUe7JSiKgpShrv2f0BDcdoKDlTKCNlomd4oNhXxLeVISQ1fJa82GsmshFKxnITJ5brBR8Q0T67mEYysTSjUxHK4NY6xSOiXdnSZOaNHXEyhKmNvthmUGLNSbd53kAeq7D7/Ws4/R8xazquGG6zynXcF214IxbMjdaFk6lzYJhf4GcQfmmw/9tJbvqMD1YrVLVc1djmsFmYCxTFFPxvWxmKtpxrEX6C1yF/lyXwXXcxQxjE8FDxDU5jPfJ8w1//RfexFf/6B+y6gI/8+2fw1e/4IlX/TicNfzQNzyPm3YnfMdP/TH33b+46sfwsLHxPg8uDWZkKWPWMi7lmGTtCjuOAl7j7KuEAthw60naja7lWGU9lCxDJjaWYJyyS07ZFWfcklNVw07VMK076jpg60CqI6lO2YJnVE7le/WZysryyq6VZGoBMyrdMkHe81CwNuB8SSEydAyzmV9xbsDpiEpyphfQ9kr38vfUw98VJwmZRFwdmNSeWdWxU7Wccs1aSTgxXT+KVWYCx5MM/fuVS7y19+aAPzf29+sZ+nDOODp0CqKQ6vo+p54nnUpgKompCFMRJmKYG8tUjPKno4bA+PhS0qvvYROuay7T+pU3fJi/8ytvZeUD3/mSp/LdX3LnsQo9r9+p+YlvfSFf86/+kG/88T/iF77r87jl9PTYjmccJctKJmUd4+CLVK5khQjvkmOVKlapwqZIkzqsJNpCZCOYCxrc5BNes6wCWIWghQGsYg90mYDN5Z8WZYOViR2duGpx0vXgVklgNamY2Q6fLFYS+zZwPgnBWQJKztui43LDiZ6swQSnn8gYMSnl8lBJ9xSHzOuo+S0Zk+6TWjOsWZ1N/Bxh6oi1wc9Nr3QvpHs3V6DyMwinAtSR6U7LbNJyZrbiptkep1zDTfX5frrhlF322ruSYVUccnDvgBhPOqwyUK1SRZvv10BKIoHItKjoc3dyLoGpwFQME3GZKzWZ5Ux0dDSjjDYmg4/qFBKjqPvtIf+EaybTijHx93797fyVn38jd992mt/+K1/E3/yKu68JZfozbj3Fv/22F/HpvYZv+jd/xAP77XEfksaoPCSXh8V+xBZFPAW8iq2M09nBXAJ0GYRiSsQ18lZPjZJtjQn7vvWdr+IFsApfdpCI0cqFauySIWibXgWRp+yKHduw61SLNK07qtpjJ5pxxTrbStcMIsyJDMLMyhDzEDKV8kvDvJ8M2dZYOCyX+TEo3u7ODVqsyhJzt1A5LMHnW5hk19mJAlaYoOZ9dcROAtO6Y153nKobzlTKZZ1xyx6w+izVtBdcBIqq/eHioL+4cJ4B6WdL+znTwoludKL7LFqgEuVGK7E4LJXYjQthAcfiEpItj6J5ROXh8SMC6o30937j7fzkqz7At37+k/n+l919zdnIPOeJ1/Fj3/ICvvX/fi3f+pOv5d/9+c9h57ilEDLchMEbyZmAO+DkLVlXOQk7Ip1EQs62KiASB8DCZPGoZlub0eWfO0jQCloi5oHANSEi0INaD4RiwNK3xPfrCb64BkwcIolVUD/yIEZ9t0TtnMtCBTs1OFBrX+80o6qcvkTBjVTy5khdT5V0V7AqZn5xkgFranWmMFvNhIl6+/uploRhCmEaSdOIm3omU3VsOD1ZcV296DmsM3aRVe7tgWM5DyVlGPNaB5WGPZeVaYSSkcciV8m2RCqX0y5h/9ii0xHaORQqLAbBypj/EkJMGRBHVEXU+cMQMjd5kvy0fuwP3sdPvuoDfNsXPIW/85V365XwGozPf9qN/PA3PI/v/Ok/5m/+0lv451/3nGM91j7LQkdVrIm9pfJkw3VyfFLryehYkahSZKWFV54fk/wThbOQTKqXqzk5O7tQFV1i7MO19v2+g6jloxl94OpcG5yySwBWVUVIOi7SBIezkRiFFvURCzGLRgtgGsF49eiS4CBqg0F8VN+tMOoU2pFOK5eJlxVVHoSe1sRZRawdYeb6IWhdQCF0O5ueWOB3A0wibubZ3Vkxn7TcONvjTL3ixnqPm+tznDIrHc/JGVY9MuPbjHEXb7N7aDbu9T1hTdpQMvGSDbVJOSzI8pcE8WEy00jqS+4uBToCq6QdxOK/tYrqeOuj0RI9Hlpbevzl4R+979P8f//jO3nZZ97G337ZtQtYJV76rFv43j/xDH7tTR/h377qA8d9OBpjv+9RtlUGWM1G2VAM3AqPEYGO3CnMH+zIZiknByrmxyXFWCl9wc+lC8sEbRaUrC72DgMT0zE1WfFt1cxu6jx17XFVQOpAqnKnrRD0a4pyUdK7MiQ32MCIyXOAZX3XAVYylxRlpjA3AlKeJ4zZ011vI8J9Uu4T1BEzyVqsulMDx6rhlFuxWwags6RBS8LMYxGpKVKD4XapMVa4DzczAFYyFzRb1n9fI/b/JTyBiM6odvlcG3ux+dwQSkn6bPlQL/cl/5VHEPfvt/zll7+BJ92wwz/8s591YpZQ/MWXPI3Xf/AB/sFvvZP/5hk38+Qbd47nQMr73AMWuOzlPfbRGsSHWcOTPb6DqL3MKlkgsCJRC5A04woXIXcPGuU48OeymLX8e+zLyZJxDU6pViIkVVZj4IxdqmSCSDNx1DYQkjqVNtbRRCFZ25eQyeoKM7VvzhyJaKZlclYlKSHGkGJYd4W4TMlDynOFcVrlDMvid5R072bSzxL6+aDDCjvKYVU7LdNcEt4w01nCWybnOeOW3OjOc73bY8c0nDIrakI/hFwcF0x5bTe4poeKoeNLvmAVqyG7lmmFJAdcbEyvvYvZosiSNJvKpHvR98WUWKVIByxixfk4ZREnLKL6yfto8MHqgMJoFO3h4tgyrZQS/8svvokH9jv+5dc/95oeldkMY4T/4898JrUz/K+/9rbLPukvOdLGfQ6TS72SuZSrcy1q6rbpqaTEq363TYmORJeikvP5FkgXQNj4irxeVuhtrO+JbF7FJX8w1jVAhZifmI65bbJddMvctcyrjkmlGZet9UOfCjGfCe1Cyit/ZHoV+iB/MD1xviaDuJwYqd3V232UYU2Uvwq1juaEiTo2MA3YaWA27diZtJzK9tinM+l+xi57p4axO0fRQhkZZCSHjYPkKm1ZAZYnKMbvRcxWQ+U2dlnryfpkWSXLKplcAiaaFGlSZJVi/p6wyoT+QOrrNqc4OoevecnDz/7Rvfynd3yCv/OVz+LZTzhzXIdxyXHz6Snf89K7+N9/4+38zts+xpc/+7ZjOY7Nj1u/Fqz3NvL9/dhSuRC4Ot4TiSJ0qVzFEhbN2LNuvI8w4rNimSfMaX+JcZdwDbA2y0aJgFkDXdV1GabS0khFZ3RkJSZh5Sp8nUdJag+SCEGy86lmM5LAT9UBVaL6rZMS0lkITrMtr1bPmJAdIi7v2q2AZYhlGcWoS6hEu/ph+VkiTXQ3ZTVTHdqpacNu3XDdZMn19T6n3Yozdskpu+S0XbEjOqZT/K3WQGrcH5E4rH3LfGHxvooifeY5frHbbBnTjudD+4vIRbLnJHRYbHIg9KM5TX5cveAVioEsoTA9ub9KdV6KoiNmZV72kVw2jgW03vmxc/z933w7L77zRv7c5z/5OA7hSOJbPu9J/Pxr7+X//J138dJn3XrV/bckkh0edNtziMOgdEzSmwEWwJpKt8YdjeUQZfWwIeUO4oUdqS5dyGGVk73LjFcp6cZRgKt8XUI1PHEYvKYAnqcWy9w0ACxijSVm/k3JeR8MxjhW2UAwGIPxpaM4aH5sYwCnZWLZ5hOz+iy53E28PCJ+PAStCvesdJ9qSeh3EmGSiLsBmQQms46dWcNO3XHjbI/dquH6esGN1R67djUqCZdrGdbw/gwzphcr00OyRBIhl91GEl1af0+HUZ2ReDRnyGNt1vh97KdWY+74GjSDMh1TOmxK/bGWecUuWfZTzSpV+lzFzbQXlnLo0lBfg6sc51Ydf/FnXs+pacU/+ZrPPjE81kHhrOF7vvQu3vfJfX7jzR+5+geQ1oGrODwMJ/VItcy6B/hBpnDFK6lNJhOnrN36Ido0lIUFsIoGLI5KxfH31q7iDF3HoTO1fh4MWaJquLQTGpjaTlehuaCLRV0El5SY7xfXZtV8JapAr3OZWFxDi4NodhS93PIwZY/3YjdTfN1jWfWV7WVkEnql+07d9a6jp10zIt3btXKwdAFhXbrQi3vXCHSzcSszg6YHiWKV3fXfH7RYF2TCG1EAq5SN62BXpBIul4t27fHHHcl+gH8zm7sWy8MQE9/779/Evfcv+Lnv+FxuPnVtKMsvJ77sM27lmbee4p//5/fwlZ/1+KuabUlEW8VBiDFv+g2WJrh+iWYBGLWgyWaArH8YYDzVXyTmsDnIOuY+WorWazjpN7uGB231GUfoP4xmTW9kZRi4BjhjXe/DBOAk0kWLNbG/WgdjCa2O5hgv2W884RujizKKPxdguqzdSulIll/EiW7SUf2VZll+pq6jficRdnWWcH6qYVZ3XDdb8rjJglOu4cbJno4w2SVn7D5T03HKLJV/ZLA9BtYcF9Y4qFHDY3M+EOjV8uVCpa/98HtFSFrKeH1PYj8/qu9fucjY/mJlJRKioRJLJ6638t6kBzpsL3Mo52R53+J4hOdaBK0f/M138Ltv/zg/8N89ixc95eq4NVzpMEb47i+5k7/4s6/nP771o3zlZz3+qj23JJAoECEGIQRDFy1ttDT5BNGUXBdPFBM2GMqE3vc7xzgLYqOcGE/7j6+gIZUPzXpb3F4kkR+XpZslaGkSWBEdvDawSh0BYdc2/Uk/c52WH5XHe6vAVSdi0I01ppUsOch2NnXetVhlM76UEB/yiq/LKzjK+rPoZE3WEGqI04RMA27imU9aduo2uzWoPfIZu2RuG07ZJTum7cdyikyl56VG79HYz2q8Iam8pv1xpaEkt6JFfzc67rUmyUWyrOKJtvb3Jr04hGTy++Xy+6g811jGUrLwnszPAtbCiyayfO5aJOL/9Svfy0/8P+/n277gKXzrFzzlaj3tVYkv+4xbecqNO/zYH7yfl33mbVdNayah3GQAraBK4yY6VtnatogEpxt6pE3g6ku2ET+yfpKbtQ9JKSk2RYglxknMWsdSLuS9Lli8kIljQ2Qnc1udtX22tQyZAPaOtspbXSr14IqN1fIwQKj0NbK5TJSoXT5JSbVb2cLmckI1YdJ3DUs3M07V172e6m7CYi9zw2TBDdU+c9uuebqXtV2bq74uVgqO+cSDwKcfWJc8NL/xPmyW5uu/mwWio2xrDF4hKaB1OGLWiA3HOVx4+g5xoRVKt7IM80cDURXxbJaLF4mrAlo//eoP8g9+65287LNu4/tfdvfVeMqrGsYI3/aFT+Hv/Mpbed0HH+AFD7GH8SijBy0v4A0hGFbesd/V7Luac1bL77nRWcneTdKMMpoLpu6HD8UYvNbb4OM5Rtu3w9eI2hz9h68I10cfmiK/WLsfDeLWKaiHk2Ft+3UlgbZ2OBPwccjuuokjJohTQ/BoR3GqpUdoBTPRTqVptKOIt4gPKjq9jOhFpMVipiykmAXc3LM7X3Fq0nLTdI9T1Yqb6j2ud/vMTcN1djHS0hWn0XFnV2OznCtOoMPrPnRzy+veJd1LaVNaA7CLhT6/IZQLCIY2lXEsHSzffH9LBDTrCsn073VM69ZIY1FpG3WJSQhGh6Uj187A9Mtfcy9/51feypfefTP/7Gufc81vuLnU+KrnPYHr5hU/9gfvu2rPKXG4EYQUBB9U/7LKvFYT1dq48AmFNH0oUSiMwKsMtZKv6KNRj4GMP0DOkOOgk1zHhcrVOK4BVlF6q5By0JYVK+Fev2VaVcvbrJZ3AWsj4hLJJZKD5FBy3kF0uqZLCXOjpLwxw0D1ZYSuABs9V5WIWUdW1b5f9bXjdNv3mnmfGcz7DmqMBGQk+hwyWm2arIPC8F5INn1c57oesgzsu5OxH8JWIXKxvSkXufX7h4txg6C3u8nuDqE4PGSK45qYPXz5a+7l+37pLbzkrpv4oW943jU3BH2UMa8d3/g5d/Ajv/9e7rt/wROvn1/x5zR+uIkXYmdpvWPRRSZ2wtRqdjIxnjh67YMxWJOo8CAPPWw7zqxKOdH2oJfTftbLh0LaXuiaGbP7RFyTYGzuYOyPE12EUZZjlA/PVCZ0lWOS15KpLimx6vR07jpD6JQL0kxLCC14r9IOW1lMTOrIEB4Kug8Xoc66rKn0a7+YBeq5mvhdP1twXb3gplqV7o9z+5w2Sy0LpVkD7OFvz3q0XA73F4mc4Y4zq/IeXEDCJ82QDlzQOnrPy79XEvJ7qRnumJMcP4YZZYPKwQ1mgP3xjzRjIUnPsTbRsQwVq1DReEvwFvHSn8eHiSuGIv/+tff1gPWvvun5TKvLu5qdhPgfP/dJGBF+5o8+eFWeT3zqAUtLRMF7Q+e1g7gKjn0/YRkqzgcdodiPEx1azQT90IE63Ed380p90C7qsYVuObnN6ApdytSxfmwqnmnWI00kMMlrzqbZkrnIAIpK/JTV2bxT1YrdqmHmOqaVp6o9UsVBAjHedNP7sueZRJuFpZdbHpasrhokDraO1NnEb+5adl078nRv16YULgZYpbwKm9lSzx09BGAxbNcZR59FEdfeg3KrxK+V7L07aQEmiWuLfzcFy3YjUxzPGq5ipZl/cDTB4b0l5nN3rK17uLgimdavvekj/I1fejMvvvPGxwxgAdx2ZsaXfcYt/Pxr7+N7vvSuK/53mwAxlEyLDFqW1kaaYFl4XYi6F9Tu5bxM++5hsTkGLihLIMsVksmkquHCEZ6BSzqo5OjLiI0PkJHhg1I+EAdtuB5zOeUY9QH0MXsldtIrt0+Gc9WUEIWmDoTKkLyQKiEG1WzZCqJPWiJm4JLcSbycKNqsUGlpmOo0LKMorqO2Ydeu+mHwIg+42JxgKes2M93SqdXXXQ68aOjLtH6RGLvIbpaA5ef093SWsE3oHCooIme+i9GZ0Ptp5cfuj30EumNNl2ZZtTaKvMN7Q/JGz99weE7ryEHrdR98gL/2C2/ihU+6nh/75hc8ZgCrxDd/3pP5rbd8jF9740f4mhdeWYto04KxCdMKphWSNYTG0gILq6VYSoIzER+HDSorq6VWsToGPcnL9hQFiJhLRy2zaoG2jIIkk1HS9R0moNfZjPkR/f9iexPXruhFQKlZlfrIl2WxxZM+krD4fhNMOb7OLpT4RVSzJYm9iYJz6x0Ln7tUU+1K2QmETiUioTYQE1I7pAvIYbeEXiSKqV+cpN65YVp37NQqHi02yWorM6z1utiK+rFU4aAweWRHX9eQLXyGxxq/3kOmFHsNVfleeS3XfjclyNlfKU3LgP3YkmYMfGuvBYPLR8DQJsci1pwPU/bChH1fc76dsN9W+NZBZzCdYDswHYeKIwWtT55v+M6ffh23np7yo4+hDGscn/OU67nrll3+3WvuvfKgFRJmxAfE3EWMLuKDCk2dcax8RVkcMdHNp+wb3QhdRXWFqIGwcRJa8kBPHgXRcRvV5ZBnFktXKY4EojDwV+Vxen+nUdeyfK+WiBVdhKDjQ6LyKejBq5aYxZUhc12qlJ/GjrltaaJj7lq6YHUBqIuEKhKdQYplTSHkHdmyppSHl5lpWd3HmCwklxAXcTZ7m+WRqZLhjjOSsW6u/x7rZV5MJktSBsGnXjTiaO5wDOj5giGjEm8DsMwFWdZ4njF/L29sAjBpWFAy/vlNrmxzoDqMvOb7plBQ4z8fLMmbviw8tkzrH/3OO3lw0fKzf/7FXL9TH+VDn5gQEb7mBU/k7//mO3j3x89z1y2nrthz2SaRRLAN2JWQJBFXhpgcjY7hEaIgkmjzgCqotYuVSGMqQibop6ZlB3IZMBrMFd+XDHbU0i4q9jbZvk1+wfGNuoTF5K+4OAxZgJ7qFVAhVGIoBoQdgZizPvX7Klogz460WJOIVolpw1AGN0Hb6Usgrmw/TG06QWLCTxRobWOQzvYziZcaw7INoIq4KjBxnon1zGwex2HdkE//DjMabjZrgDMWaOrrV8S4atBYMZjxHcRnjTOhcWdy0yRw0yjQFmNHCTnrKiC1jiib0w5FBBtGI12rVLOINYswYT9MON9N2e8mLNqKVVNBazCNYBvBtGDbw70PR0bEv/lDD/ILr/sQf+4Lnswzbr1yH9STEH/6uU/AGeEX/vi+K/o8mmElTJswHdhWMI0grSG0hra1NF3FyjuWvmKv0xPnfDflrJ9xPkw5H6fsx5r9OOlJ036Uo2RCDG4Rteg2nbG2qurJ2HVDus0refkArvEyhbPqt1wbzehEKPsYbZ+BpdEHbViGUaQDu7bhlGvUNLDyVFX2lZ9EQq1K+bLFZ0zIJ3eZLg8ly7IJbEJMWS5SRmZ00LsbkdLjOcxxOdW/tyO+qF/8wfDalxJbM862/7qXT2xktptxEGDp846WumaQKxeZwkduvtflfV6XYOhURpPJd+0Y6nnYeqddw070Vs7jQ3YPjyzT+sHffAc37NT8z19y51E95ImNG3YnfOndt/DLb/gw/8uXP/OKST1sG3ULTadXqmQ1m0gGkrN4myAJC1P326eNqEvAJGROI5cvEaMGfJSTNfRfA9kFYugOlVIh5Cs/yWElXNCFfCg5RZmJBC0DJ8b0W1wA3ZWYQgYsjTI0bHvA1DKxS+q91aWG3aphUdX4YGjqihTyeI0bBqmNL9otQ7rM92cALRCTVC+WF4yMIyY1Xiz+Y+AVqGQorMZjMyZ/X/mr8bbmMlM4ZFmDXGHMJa6X4uXrzdgc1TKSNMsCkJBL1HUCvv97Dug6Fy6rdAyL6d/C16y8o20tsTNIl6mNbrgAHyaOBLQ+8Kl9/uj99/N9X/FMTk+v7Mr6kxJf/YLb+e23fYxXvvuTfMndt1yR5zCNKpxdpWJKSTr/JgG0ce3oKj2RfTB0WXi6dBUxCUunV8GQDI2t0JGZVk9+01Cn8RU/YEpWMCoZ27zqq5C1RdtzEIkcMZkb0+5UlYneKoNhkxQsq1x+Qrm/kOyw2XalANfcNJxxupdyWdes8pjPclbRkhXx3qhma6qEvJ3qPCKXScTHkmUZzbKGv1fwSQfZV1IxMR3E3NEzI2BJA7CMy8T+b90AnLVZURk6dXDhRaP/+fx+tJLL9tF86UFqlzLaZXsrkQvDSFQLnCzLKCvH9rM7aSHfz3dTzrVT9toJi6bGN460srhGsCvBtrmpdDWJ+F9544cRgT/1nKs3LHytx4vvvIlTE8fvvv3jVw60fCL5hO0S3gumSznTEqxDW/sYfKcfyrI5KyahMsr/GBIzq2fL3DSqx0leAUVgmnmknufKUojN7iIjwr6Q8mPgGpZa0LfTu6SDtp0YuhSpSARJyjEdUh5dSlR1GQjZW16dTpvgqKqA95Ywtq5xQnRJwcYZkrt8p4fxkhGgdzAoZLRLkS46rEm9sh2gGpPcyaxxRcBaqTb8zRolA1IOqjzxhaNXwPB6Zk7Sjl/jAxw9DjqGcWyODY0ticbke5mDbYKjDZauGwh405UxtFweXmx78EZcNmillPjVN36Ez33KDdx2Zna5D/eoidoZXvKMm/hP7/gEMaYr4htmGs1AkqVfEZ9sPhGSXl1jmwhJM64Q8nyiU//vpa9Y1RURYcfqDr0uOULeMzWVDmuGTAtYAy5L6InaQFzrMhXCfhi2LR8kJe675AhGBsdTo3mCTboUoboI3bq5LVl/NWbhqc5YrlzFstJG0N5US+NFawmdIMmoSj4KoRHiSh1OLytGgFWcOH0chteXQY9lmjMtI8WQsKVKo4+g+AFUuLDDd4G8oGjf+gvHaAh6Q4jaJdu/T2XDknYIcxZ7SNzeHKrfXARcysG9nGWd66ac66bstzXLtqJrHDQGs5JMwmtDybYJ21wl0HrTh87y/k/t8xdf8rTLfahHXbz0WbfwG2/+KG/60IM8947HHfnjm05Bw9RGOy+inazc3iMZHURN1pCiXvnbBMEZRBIhf1idBDpnmdlW7UZy2YWBOoVcDq7byPRaIYoTwMVLrE2yuSO35mNNEI9NkaxhZT96KkmMV4VEDioQh1DblchUWqII82zR7JNhXnW03rKqK0JtiV3KJPygkBd/BBeUNNxiVNDySWUn3hp85ndMzrQURBxdGrqqppcyFOAaLhRDxpXWnnPdVmhdiFouCMOY1eD4AAx2zZmLvFhGNY7Ngfri9lH8slZJyfdVrFiGmoWvWPqKZVvRto7UaUk+1mYNo2hXCbT+41s/Sm0NX/6Zt17uQz3q4r+562asEf7TOz5+RUBLuoARwbQR69TjyDZ6JiYj6IVUifkYikLZkapIYxIxGlLeLeiTZWbVo6qsXIeRVOEhuI21f89t76IxKrE5SzcM0oqCFijfYxJTsrNq7hqGAyQJQ9cx5g9SKRNDtnpp8dYycx1N5VhUQXVbffdwGKQ27jJBq3hBRSHlW8iZlk86UmVIzIyhIRsaZvuZAizFc105v9D/jeMc8KBO3yZw6cu4PlA9TBWoti4wUrCXUhFgpNM7CKzK98duE6XjXPisJlYsghLvw4yho8tzhhQCvgBWp/TGVS0P//gDD/CZt5/ZEvAHxJl5xQuf/Dh+9+0f569/2TOP/PFl1UHQleTIcJE2vkzN6weUlG2HfSJ4HWFpo+CrQNdZQhRWtZ4Ky6pSMlgiK1sp2ZpL24oLnQgeSRQ3iL61n6AWPel3TEMnegwrCQTTUaXUb0QOaXA92PxAFcuVSgI7punHRgCuq5fadOgcXet0EcZUOZUwEUKVssvppUcxY5SYtFMZDa23OBtovKM2XpfOxkrvk6NK2r2tcqZVMs3BWkYGQBmByRhUNpfl9rOK2YljFau1cSsrCZtG2rmSv+ZylHShQGI851iAa2yNs8hSmbIabBEmWfk+Ya+bsJfV723rCCuLWZleV2ga9Os2YdqIaQ/LY15GND7wlg+d5flPOvos4tES/80zbubdH9/jU3vN0T94CEgISIhIFzFdzGR8yh2ZlEV7enKYlqzjEmgNsbX4ztJ0jmWrafzCV+z7fALGuidXi9VJiU375E0niLUFFmk02EvxKre5XB1a46tU5QUIrl9LVfzpIxy4IWb8YR60ZGUFmdrXzF3LtPJYF6CKWbleZApqWXNZUZw3Iznb0m+FXCL2y0YYLQKJrnfxHGxlyhjzxT+WDzfcvu7HP7zu4xX3w0yg9Iss+tKyB6nRHgAGi5sxYJXzoifgs8V3Ub+30dLljDN2BnzWZHXa4dbsqqjhE3I1Mq23fvgcbYg87wqUPo+WKID+xnsf5EufdbRdRGk6iEmXkUq+0gva2r8g09JSUQLETj9Qmn0JqwRdZRFJdBNtYVdGea65aXW2zUYt4wTq0bm16Ze1Odx7YZkx4kOwkKATq4tjiw2L6Id4Kh11iv12l4OyCyjOBYOEoByDJXFdtSAm4Xw1ZW/SaTNikhBP3pF4eEuUi74PcXCQJQopNzy8MQNwJbXCdtlJtksWm2Jvk2zT4P5ZiVfdW5Z1DN1CuWiXbww6BaB6gMyvq5HUWy8TVS5RXB1s/j2VXZTnW+9EjheYlA7oIk7okuvlDYtYr2VZi6amaSpSY5HGDOr3kmU1CdtEbBeVoz1EXBZovf6DDwBsM62HiGc//gzOCK+/94EjBy28Vw1D55FWFU0267KsaHkRouqI9HMveWxNu4xFnhCteqyvqgojUNnAuW6KIXHGLbFkz6sU+g8SDFlOKUv66X4G/uOghRab/9b13zPUMRCyB1QUQ5SO8abrgx5zPHJEckylozMKfHPbsrQ1M6eLXtvKsaoiyVm1lLHKa11OSGSNiE+JXsy7GWs8XwYFUwBeVMcWspi0iG9VlHr4xaxDljV4s+v3h+5ry9A46flH8Rc0VMJGVlZAcJUqQjL9WrAmuTXl+6pIHLxm89KZfrC/12X51N+ki4g/XHl4WaD1ug8+wJNumHPTqcnlPMyjOma15e7bTvOGex888sdOMQ52wfkqZZzJ/IjkYWb9YIYkIErQxwjWSi8sDE61213lWJnEwlUsqorKBBaxZmI6VrFixzS0yfYOAQcKGdOF7phjO+YLfj4LHwtwrVLm1FIkogstpnQXqLbHqvHhBSnuEyrXCLmTuHINc9dS24BzAVwiukSykoedL1OnlUvDfqNMGpaQXrAm64DolfJJsFKI9KiKd9bV6OPYJOIvdFwwDEt183H0ZL+usDfkbC8D/qbFzGY3csiyxnqs3C2MajuzCpVuhOocvtPS0HSiw/2ZgLdd0u6hT4hPiL8KoJVS4nX3PsCLn37jpT7EYyaed8d1/MLrPkSI6WjtptuOFNMgEwoOI6LcQEoQLeINkD2LokBKxFqzMAlCSJDEqFVzFlnuSWJeKYycrWYYktodR71EHqTaBoaO1YbwcMxFaYmiH7CxZ1ZRVoOOpIQktMb2wtGLWaHoYw4umkWpX5oHu3ZFl6yaBdYNbbDs15FUm76LmA6pxL5YHFSxiSREEtZEnOitMgFnSmd0GIlat1hez8Q2XSBg05VByfrekYPBYUG5w8HoEegHtEt5GETlMfpcHnLnd7PE3wSr4kS6iDWrWHHeqx5rFVwm32uWTYVvHDRWle+rMtyfcmkYsauIbQKmDcghy8NLJuI/9MCST55veN62NHzYeO4dj2PRBt71sfNH+8ApqpYhBChZl48qhfBRU+8uDml4l/Ksl3I6kl1PjRcIQvK60ccHo1tusqJ5c9B3DEgPFcWv/KGIZVhvp4/J6bH2pzitljb7xQjpMsCrW218bwszMZ7aBiobEBtJNuVbP+FydNEDVsourjo8XWyKy9eXE+Pfv1jZGPqyTi8IXRyyo0G57ta+Lsr2Tf/5gwCrvD+94j3anGVZ2ixxSFniIB5kTZc1Kg1DhBjBX2FO6x0fPQfAs59w5lIf4jETpVHxhvse4FmPP31kj5s6r+Q7OdNyDowg0Q0r4LNvVMz+6MpzSS4Z9ReVCxZ8bYgGOutYdhXWRPZ8TW08czPllF1iUmSaOmqGmbjNKKB2UCcR6AWppszEjQZwlZy3RClLQANtsr0Ga3DgzFlXVpGbNVlA6G1bdkxDY6q1JRjWRS0P++UXR5P99tNMeWja2cDEeqbZomZmWyZGB7wrMzIDvIgTw2YUBwZDyiu+8j/kbCtI7AXDsXQJe75R+nuTTJZX5DI7DtuOygaf8h4WsIpJBv5qBFb7YUITHOf9hHPtlJWv2M/ku185ZGWwS4NdCm6lWZbLN7uKmCYgTUAaj3SH64hcMmi95xN7ADztpp2H+cltPPH6GTfs1Lzh3gf5xs950tE9cFAXKzFCMhbBI14/uiKCGKMLHJyWiNak/AFNWhYZGZwhJCFe2/8hqM6odXrlbKOjSaM9itje1+mCQxq12g/alweFixkbDGZCWobdeWWIedjkMtg0ly3Zxe1gGH/ZtALWTlyxOK6NGh4aG8EkdcMwl59p9Y9h0CzLqjVNZSK1UeCaGL2VzK+31+mbGuvmgIb10nCw90mDueIGp1Uy1rFeq8+0xqBF6nVh2lgx5cqVn2uQSoy5q0WYrEkamlix55XHWviaRVdcHBy+tb1fVpHd9PddtlPyEZMrA11SeYUzrfd+Yo9bT085tRWVPmyICM96/Gne/fGjLQ9TSrraPUREPAmnJSKQMmgBmE4zm2QE22l6ZTpIGcTEq4Gn+LxiKwyK7jbkjlAu06oYCMbQ5VOnuAEcRLQ/XAlZfOjHwDWMmMTe7yXKYJQX8f2adzXGKzOPG4CVP3QlkyklojMRYxRYklETxcsGrUwqJpMGP63sXFpbT20UOAtglVvxq3qoYWW4sOGx2ZQYR9HChVGm1QNWEkD9/kvHsMNiUuy7ivr94eIzLgeb5Oii8lhtLglXWfW+nwFr1Tl854itRVrJtkmDyZ9tswK+U22hdIPWkHCFifh7PrnHnbfsXuqvP+bi6Tfv8vOvve9oh6djoiy1T4CkmEfJ8gZlQKLTDCA67SRmyUN0Rrtco/LQNJCMITlL21qMcez7mtp69u2EhZ1gSZyPM92OY4Z5tfE+vpCKaFHWgGv8AVVeJ2T6OHcPM4D1/Ev+vU5sNqALdGLzmJFoCWgYhrQZ3ChiLh2L39bEdEwygDiXea1eYHp5b0MvVnUgLuFcYOo8M9ex41p2rXp9zW1DWTZ70AYei4EUCSI6VI7Ns59lpGdQw9v+nS/aN+k7ff0255GYtbwX/Wbo0R9tUinXU3/hKTzYKrketPa8Zlplw1MbLefbKUtfsd9WLJtale/7DmkMbmFw+0q+Vwsl390qYZcR20RM43Np2EHnVcJziLika0xKiXs+scfTbtqC1mHjzptPsWgDHzm7PLoHTRFdqZyJzKhlV4p61ZKg5KZ4Fe5JT86n0c7ETMoHMHmzD3kUJYQh22qiLihYpXUytxDj45b4gVf7wnNtSABMXsIwHg9as/LNH8R1Etn2z3GxWHP+LI+Plm1WkqpB5GjKQ6Q8VgKTMGa9a+hMPLAk3NwT2SviS3k3FndmYW0c/X95fcpj9ZumR699efyHy3pjKvKIchsy37JgddPrfeFrdQrxjqar6NqcYfWarEy8t9l+pivnXs60fERyE4kQSFcy0/rI2RWLNvD0m7egddgoWel7PrHH7Y87mkWuKSbERPLFWMtFgGog5yVGsAZSUjmhCBIMzqlNSzK68EGSWhAnA9GNtvpMKyobmPoJO071eGWgep6afvPLeGSkzLyVq3UJkz9QFb4HKwAkf4+Bk+kYAKtXaEfJgtOB7ym6sYr1ZRn6uKWLWDqIyms5GzEmqj6rcFGX8z4YtBPpEtZFnMsEvFPyfb0cXC8FiyJeR5kHcOk3J6VhTjBIGpoX/e8b2qKAZzDkW3vdM4f4cMA1/vcikxh3CEuGtecn6tzQKeneekuzqgkr5bHcvsE04BaCWyr5Xi1Tr343ReLQ+pxhBZLP+/AOEZcEWvdkEn4LWoePp+es9J6P7/HFz7j5aB40RVI0ClxJkBgVuAqhmX3X6byCmYiKT0EdOyWSjOqVQNdskUEsNJZgEstWu4iVCezYtneB6GxDl1z2Dy/KeFkDq1KaQHHotD3Y6HZrQ+81LkMbP2adVRcdjMwElaBOubslvYfVOHrf84wNYy91l/8OayJiMqdlVHB7WW/DiIg3okS5M7H/e0qXNebyt6jO15T9uWtnUyKKKDlu6B0wSjkYCP0W7jHRPp43BPqMzpSxHdXd98cz5tbGSy/WgCtntz6a7As2+LwvuopV51SL5S1hZZGVRRrBrnTG1a5Gmqw2YfNQdK/JKtVACEPVcIi4LNC6cwtah47H7dTcuDvhPZ84eq1WiuqPpQ200GdZB3YUnUFEsG0EMRibsK2q5UOnq7BMB7ETklWnycY6lq7ivNdMa2Y7IkI0LVXmm2D4EPY8CkMHsZQx6nuuRNok298MdsJ630H+8BaCnqHrlaOUQyU2N8wUos9kEWf5gDpRq2NjUvYcy0T65bwF42wta7TM6DYcs6yNyZSsqP8bRqVjftH0NTHQjlXxo+PtdVS5n7sZNhsOGgn9a1hejx7YNvi1YVGsDAsq8mjOKmiGtWx1QYXvHKE10BgFrLxcZdwtVBeHXBp2hbZQXSFxoDOuaPfwnk+c53Hziht2t+M7jyTuvHm3l4ocSaSk2VTKJWISBacYSdHq5zZaBavgSDAo5gUk2n6UR4LkGUXRxzGGGITG1cQw+G4tXE1EVHdkWuWINjpgw1D0uhK+EMAT8VTG912r8Ybiumx6MbEXNRIH0BsDnC0e8VlIOt5UrSMwOgLUiWXHNMxNy8yOxnmKuPRy13PmiSgFLFXDl9fBRxVhYqALwxN1fba18RpJpMml7DRrp9pkwUKH2mDXo07peCZwnNVWBKbiCdmyZ3PUp2xQWivT0Wy3A+U0k/rbl5nC/U7Lwv0mq927TLp3BrMwmlk1QrUAk8l3t8wdw1XQoeg267I6ryDlQ59pHXbT9yVnWtvS8JHHnbfs8suv/7CWcHKZl/cDIsU0ZFwpabotSWcUs2I+2aC8lo+IEYxVRbw1enVMFozTKyRGiI1u9Vm1FftVTUzC1E6UmLXqdrp54m8SzMD6zxggQie5M1bEoZl7GRPVFRCM9FnJsGg0UYxchv1+BbAGHZO6RORSKOuknFFyHJN6Mv5I34c0kOYRdTFdoRq38ppcDLSMpP71weTOHihPKAZMq6M4+bUcG/71j8No9RcGta8uz1FGnoZxorXHGr0YJdPyydBGq7esdve+bNQxWdqAZlmZeLdrpLsa/IlPmmGVxlFfGqahoXSIuCTQet8n93npUTsWPAbizpt3Od94Pn6u4dYz06N50P7qVJYUCERDKuVESio+LTOKIhAjxqqVDaAqeq9yB3UsUJGpBEjWEJKjSXBOEk2lp8zEevWpsjVGkoo2H2I0pYyzdMlSxUBn9EM7NdmHXnQTz7DUVQ3r2uTynJz0c3O6ldr3+/+molnJVHSTT7+WDGglMKVjatqcaan0oXIBcZGUB6ePKorDQxcsrbEs81YglwLLfGSDdmrdsaK8ht4YnBlkCZ3kJRjiCUi/c/Kg0J2Epl9QAlwIaFLGidYfo3dxKBY00fZZ1sLXLLuKRVvRNBVh5aBVpbtpBbfIHFZDn125VcItM481ni/svJaGpWMYAqmA1yHiEYPWXuP59H7LHTccTQfssRRPv1mX2L774+ePDrQ2onQUicN4DJ1+kpNXwakAtF47iiKotYB6p0Nu3zv1l4+VZF7J0ZiE98qfTSpPFy0rqwR3ceccl4ljECug5aNlYnXfX9nBWAZ4VXOVeZb8oTS5VCwEfylFByK5qMxjD1jTsnmZRCfqFaHApsr4qe2obMC4Uh4egctDfuFSNJm4tlT5A69/h9O/t8+0hk5pCZ8bDRGhJjubSui7pWXkpohtx2UilGaHCm0rwsGNChle13EUqUWJIi0pavexz3vMavfBH0t6o8l+SUWrX5u2eGUNEocixUkxFh/wnG1doUzrvvt1t9wd129B65FGGXn6wKf3+SJuOtoHH/NbpaNYpBBhpJvOKnl80KzLBKQ1WCA2Ss4nMyzIsNkRIllDdI4UhZXTGbeUhOAMlQ14YwbyeU1zNRKTilFyHR2mtsS+G2lNUuvhfKBlrrDsUbSYvpwBRtuUw8BloYBVZdAyWcg5zaBVRnrKiI0xEW8vP9MaLGlyaTjyiG+Nza/HOucXudC2ppj8lZ+1RFaifFiTeTGbP7I10ObXaTwDWmYI4cLZ0LXFJCPQKmaMw3GMtkRnX6zeG6sMQWdvrLIouCjetUxkKA2LV1YXwMeBw8raLGLquawrxml98NMKWk+6fjtz+EjjplMTZpXtX8Mjj4cCLujHe4iZ1g6x128lb3X0JyQkGJIY3VotynmJF7oEaWJYJaGtA523NJXH2UjtvJYdmS8yXNhFK2DWGkttdHNx4cQ2SWQrnWYSEqhSWBvxMRL7zKkm9DxWAayJDKNlURp0U49nJ5eIO7ZlYnVwusvZ1uW97sW9VIghz24GqzyeifikjqX9MeXycByGRDShz0hjkjWw67PWEf+m15WIpZR9sR9xKjzfuAQ8mHcswl/93iqVFWB5yWo35Xw7YW81oW0qQmP7IWjTsl4SrhS43FJtv+0q5LLQqybLZwLeB5L3A6dVysMrJXnYZlqXHiLCHdfPrxxowcHAlWSQQpQOVrnSdQJWO42q3cozim2+2jeAyaLTWnQuzek4SJufMpQ1WEZn/lISJGuVUkrENeCSnn6rTOgV1pUEVskxyUZ0OsqiUcqe8YKHujg+SMxtfO2M6qjxmNwWqkQuHwsRrxY1xsQse7j88rC/xVGmFZUX0gMZfrxkWOXeiNaXw/cNURJdMkzSaOC5+JRJ2eQ9GqhemwAYdFdwMeua2FvXjBdhFMuZsk1n6bO0IRj1ee9ErWaCDtqbjt7myOTFwRJSnsIYJjF0MiOsARUhaHZVpjkOGY8YtO69f8HpqePMfDsofSlxxw1zPvjp/Sv7JJvARSCJfpST96qEF6MdRdCfDRaT3UxdTsuMN3l3YlLvc9GV8snYnFEITRS8tYTaqx2LiQQXEElaguVso2RaIoloBJ9Lgc7Y3rUgIpnDUbI55A+hoXhkDWEYabI2wsqI3E42Z2IlO1N7mNpohohNY4ODSwqJjJZaDNt4RBKdyeVw1tKtv00K7n1ZaBIQldtKsS/TTNJdiSamXEYrJ1hsqSHr1ChdVM8F0wE5xhlr6ciuYp0zrAln/ZxzfsrZbsZep2Z+i0bJ97TSIWjbjAz9Mo/lVgnXZNX7SqUNptEMS7ohw6LrNMvyviffU9Ku8WHjkkBrS8Jfejzp+jmvfPcnr/wTPVzGJX4Y9REhpYRZmT4LsiYDmANNE1Im58sWa6NjNUDKBH6wkej0eYyJ4OjJ5wJcjoiPyn/5DC7LUPXlz0Q8WNiPE/WdwlPnxOmgjlmXLFFC3tqjg79dCmvZFuRuZJZVFNmDLbKHy8y01laIRfrFuCEvbS1Z58WeZrOBUaLMAMZs8xNl2JCjw806WF2WrR6k3LhgxVj+/VWs6LCsYs35OGU/Trjf7/CAn3O2m/FgM2PR1eytJn23UFrBZPLd9lue0uDg0KRe8V46hdJkPVbWZQ2ApdTEuCxMV6o8vPf+Bc+67eiM7B5r8aQbd2gO6YV92VGAi0HD1XcURbTsizLY2Vgl542hJ6fVmjl7y+t2d2KTyXmjBH1KghcwI2LZJsEIGRgiJinARRFMtnPyWTjaBIfLpV6THDZqVjRsqEl5i/SmPinm0kY/xF2KgHYLbS4Vdff0EL1zaFauk5Xxl/c60/Na/cLWlIfOk+CKV/xoVOnhooz8wOD1XmYKlROLfdNCrZKH3w1F6DXqBhawKtMK4zX258Nsjcc6301YdHXeV2j7bqHJOqzegXTk927WBqJjXmunJWAvb8jlYcpOpT1gpaiAdchs6xGD1oceWPBln7HdJn2p8aSrzQUmLTkAyl7QXg+T0uB8WjqMMaqdDfTKeeMTEo2O/XSZnM8WuiHq7GKKynXFyhArNdqLSVTIaSPRBqwkUsm6SLQB4mjuzyfNwIqGa5V0U/RU2gu0RTr3aOnE0Ylu4Im0VBJZie6asSKElFglsjuFulIMXFLG9MuUaY3Lw3ILwWCMdln9aLSnjNKUUnGtUSEbkwW9ONWqPU9S7doqOc2sUiLieylEJ7oMhLSelRYHjjLMrtugda9lEys+1e2yDDWfand4sJmx1054cDGjbS3dfo2s1OPdLbIma1kI+OxAmsn3nnhfdpplrdrecia17YW6rEsALLgE0OpC2pLwlxFPOo7SepRx6WBqvoqL5FEKIblsICjKXUmnngvJGZIVrEnYWgtKW6MWv1aUD0oQrSElBcAw0iMlN7T2kxl9kEbKbyOWNpcGrdVTshpN/Mc8iqJzdJnDEeV5OgkEI71v/FQ8XdZt2ZT6TGwRB5/5YUvNUb2++neoMJecadFv5EkouF+sDOz/zqQvSJFEhCR00eoOysxt6Shi6mUQm8PVQYYu7Ngbq2RZSra7fhnvItQ82M3Z9zUPNjPO512FbTO4j0ozMvPrhnlC242yq7IsuGzVKdIG73OGlUHqMgELLlERfywfvEdJPP662dFu5Dls5IwrxVHXST9ZOcPKGVd2Q6XMKBpD2exTWSXn+xnFCCDESiULscpr4YOAS6RaiCFinZoTWjFEF/KqLM26osmllFVwO9dNmRivnuS2Ymo02ypiUhhGUUqpV0ddOLpjWqxE5tL0tsxAv6fvwTDnfJyqgV1wGVQ4cJvOI4mSaUlS2YO+BoZgEsFqJ1FMHHRZolXpmJgvZLyWfwnPwC+Ou41eFHCr5OjyDKd2XqsLZkDHbhvlVjqExWZmGSoebGcsfcW51WRk5FdBJ9h9m8l2zbBMC3aZSfeWXvFuVwGz6pAmQNNqSdh2pK7TkrDz6xzWJmAdUqMFlwha20zr0qOyhidcNzueJy+lopg+40rkWUTQjIsROR8jYgVbxH95tCda039Qk6iGK2tBiV6UB3NJv64iwUVCEIxJ+Giy5XGksrHvMrbR0hiHT5baeJrodOTGeBZZy1WyLVjnhmyWPVQjD/ZKAnXZz5g/uPf7XfbChLPdjH1f03oHfgCHSw0JGbTydqPktDxEEq3Rj1i0w2vrTFzXsOW/pTQmXM6Q1EBQtVvLUKlUQyIT6zEkJqbrX5cybF2i/M0xCU3U17WLOlbkk2HhaxqvNjN7TU0XLKtljW8yf7WwuRRUR1vbgluoeLRfTNFudAqbgLTdoMXquoF07+UNlwdYcAmgZY1w2xUaQXmsxLFnqqOOItGsLciALEQt4z6dgpoxMmyvbqUgm25oDmTXCO2gSRL9XjLaSQuaTZk8LhOtYIwhRgWvZAWbpN8k04rFRx3UbYylSzpL5/KH86AYE+xT0/Vq+FImdcmyFybsh0k/RxeCyWXdZSri4+gWcrblDVEgBJ0lLBunrdHX3koiiWrYoBj16eMV37AOizUqUi0A5kykiQ4jkYlx/aD5GMRLaemj6rvKei8fLaug84XLrtI1cd6yaipC9sTSZRQGu8rDz0XWkMd0TEcvbTDdqFPY6jYd8WFNPHrUgAWXAFp/4YueirNHPBb/GItjzVRHA9Zj1Xz5vmqycqnoTa+ctz4iCUxtkQShLVuDDaFSoaEuPxVCrVxWqFVtHitDqhPBJUJlEZsQGzFW5yStTZjscVWysNoFqixLmFidaxwb69H/FevjMGPl/ZhDikl0GYN3PLCasWhqmmWFNLox5nLCdNpdNY1gspgsWEvwCti+sxirjqbGDBlWWeh60cfNNjflZ4vBoM3c4EO9HiHPQIYk+GAJWfDqg9poe2+J3pB89sLqBLcy2aUhG/l1qnRX1wYtBW2XsrFfUHnDIpPuTYu0nQJW0yhgFfI9ZS5rk7+6BMCCSwCtv/Hlz7ykJ9rGENeMrc/FMi6AYDJwjbb7tB4D2H5PoCGZpNyWDFkGEV3yELKVc0CzLas8lzorGKKLiDGEvMHGSKKzCl6tz7ouG6hs1ZeRMGi/QMEosW7xUn5GYJ3jiQYfLPsr5W3SyqqdyuH2KVw0jEctfbx+4BGIrZCSySR85uyCvtbGDL5bBwlOx6E/M/ysoNmagtj6ccQR+R+y5CLmTmbRjkVvVJbRGcjjWabJK+sbFYwaD3aZFe6rgXR3K7WXsU3uEnYR81DjOUU4GtOFGqxLBCy4jG0827j0+HNf8JTjPoSHnlMs5LzN232iU3I+6XiG3idMZ5BoiZVmXNFBqMDWWgL2a+etarySHbKv5BKx0q5XtCqtCHkFFwJi1RJZDJhsj2yyOrN80AuRHqOow0IGsBLlMz025UtRBt1RXiJqLzPTsi2ArmMr247AEPPfmKqoUwS26MIUpNcOshz4CLTGTV/yqI/ke/27hp8rv5+yTowkEKQfLSplsHj92pZRHK/WyCbk8q8dwMp4JdzLnkK3Ckina+xV5T4i3ZtWu4RF3lBI9+JGehnl4GZsQeuxHBcBruH7Kbufpn7kJ0V1ZxIfkc4iMRGdwXRqbROdECrNqkKlkohoVaSaLIR8n6x2HZPRrKzfjJNBK+WZwGQymAlw0Ac9kX95xE2NifUxHuWft3l+TslldSm4nHCrVFY3IkkIXea18t8YnVVwdsPfdqA8Pv8tsvFnkv9EJGWH1OHHN39PYn7o0t1NQ2eTiA6/56aBCXpvWzSDagexqC32yG0cJA2NR0LErPzgidV2mqEfRLqXkvCIYwtaj/U4ALhKqQjovRGSB7FBl8Nmkt6gingT1AFUgkF0F0WWRWg30ViIIen3csmYbO60FdAqIGWz2l5rO8WjEZBt6hMkyqBID7JmE7P+g/lnyB9YnxeIZnL5csJ4Xfqqzq+5wDYM0pCgf5uE0d8m0idV/Z9U/o6D3qb+j4ALgLg8Rhq0YmW0iFGTgKRApeClNwkpZ1fZWsarut2uIiYU0FJ1u2mzvUyXV9jHOAKqA0j3/hiPLsvSv/WIHmgb29jGNq5GbNuA29jGNk5UbEFrG9vYxomKLWhtYxvbOFGxBa1tbGMbJyoecfdQRN4KrK7AsRxl3Ah86rgP4mFimlJ69nEfxDa2cdLiUiQPq5TSC478SI4wROSPT8IxHvcxbGMbJzG25eE2trGNExVb0NrGNrZxouJSQOtfH/lRHH1sj3Eb23iUxlYRv41tbONExbY83MY2tnGi4hGBloh8o4i8WUTeIiKvEpHPvlIHdqkhIl8uIu8SkXtE5PuO+3g2Q0SeKCKvEJG3i8jbROS7j/uYtrGNkxSPqDwUkc8H3pFSekBEvgL4gZTS51yxo3uEISIWeDfwUuBDwGuBr08pvf1YD2wUInIbcFtK6fUicgp4HfA/XEvHuI1tXMvxiDKtlNKrUkoP5P99NXD70R/SZcWLgHtSSu9LKbXAy4E/dczHtBYppY+mlF6fvz4PvAN4wvEe1Ta2cXLicjitbwf+41EdyBHFE4D7Rv//Ia5hQBCRJwPPBf7omA9lG9s4MXFJJoAi8sUoaH3h0R7OYydEZBf4D8BfSSmdu8SH2bZ+jyYuy2/5pearr/z7IKKr38hbk8Qg1oC1YAziHNh87xw4S6orcJY4qUgTS6wsYWbVXXZqCJVaYvtJ3qpUC9GqKWMsxoy23NLwPZf6f1tzk42og2reFVCcUSXoZh8JjL6n5ovGp37Dj/Hwhy//3od9Lx4WtETkLwHfkf/3T6JzfT8OfEVK6dOX8PJfyfgw8MTR/9+ev3dNhYhUKGD9bErpl477eLZxDUcxgh8DlrWICFTVAFzOIcbApCZVClpxWpMqQ5w5wkS9/Lu5IVaCn+rWpFgJYZKBqspglQEKg1pi2wQWkotgE1IVz/7Bfx+yZ39El2cEIXiD5OUZMvak96LusZ1ucbKNgqE9pIPsw4JWSumHgR/W10/uAH4J+KaU0rsfyWt/leK1wJ0i8hQUrL4O+IbjPaT1EBEB/g3a0Pinx30827iGYwRYJbvCiGZTZsiqxFmoK7CWNK1JlSXVjjB3RGfwc5tBSvCznF3Ny6o3CNOUffwTyWWAqtRsXqqIcbrirao91kYqG7BGV5mVbUchGrpgiNHQeru2pix6A23eMdnpPgHxulDEdDljK2u3DxGPtDz8u8ANwI/oZw9/LQ0mp5S8iPxPwO8AFviJlNLbjvmwNuMLgG8C3iIib8zf+1sppd86vkM6fLQ+8qtv/DC/9ZaPcmZW8ewnnOHrXnQHu5PtuoErEjnD6gHLWsRaXYpYAMu5vhRMtSNOHbGy+Jnrs6sw0YyqmytohRmEOml2NU15vVsElxCbsHVQoKoCzkacDcwqT2UDE+txJuLyRuuI4KOhi1Y3VXtHFwytd3SdJQaDd5bkTc7iDOS1bWV7UaxY20T0kC/JVhF/ouOqvnmfOLfi6/71q3nfp/a54/o5ISY+/OCS63dq/upL7+IbXnQHZnMZ38mIa4vTOiDDKmWg2CGrkrqGypEqR5pphhXmDj+1xNrQ7RhCLXRzCFMFLT/XjCrMErGOUCXM1CM2Udce5wKVDUwrT20DM9cxdy21CZyqVlQSmZgOl3dQxqT7FJtYsQwVTXQsfEUbHUtfZQCzrDpHCIaudfhOAUyWFmkEtxCqPS0T3/JPvufyOa1tbAPg7LLjm3/iNXzs3Iof/+YX8CV334yI8IZ7H+Af/vY7+du/8lZ+880f5e//6WfztJuukWW0JzEuUhL2gFW5gWyvHGlSkSq7ll2FqRBqQzfTfZN+roAVJ2SwSsRZgDpiay37nFOgmji97VQttfGcqhpmtqM2nl3bUElgmtcXGYl00dElSxM9E9PRxIrKBJrgcBKpTKCLFmsirbe6qNakfgM3yRBbbQDIIdFom2md7Lhqb953/fTr+M/v/Dg/8a0v5MV33rR+ECnx8tfexz/4zXew8oH/8XOfxDd97pN46skBr2sj03qoDKtyA+E+nfT8VekM+nml2dWuwU+UaPc7ylv5ncxb1YmwE6GKuJmnqj2VC+xOGyoT2a0bprZjaj3XVUsmpuO0WzG3DVPxzE2DlUglgZBLuVWqiUlYxAmrWNEkx56f0CXLvp/QREcbLXvdhC5all3Foq1ovWO5XxNXDrNnqc4Z7Are8YPbTGsbRxCveNcn+O23fYy//mXPuACwAESEr3/RHXzp3bfwj37nnfzUH36Q//v/+QBPu2mHu287zY27E3YmFitCZQ3TynLdvOKmUxNuf9ycO66fU7vH8BjsZocwc1ZrHcKq7gn3NNGyMM4q4sQRppZuV7uD7a7BT7Uc9HO0JNyJxEkiTSJ2x2NdYD5tmdYdU+c5lcHqTLViZltmtuN6t8/EdJwyK6amYyodlXhs5rFCMkQMdQy0yWIkaRaWLJUEumSZ2Y5lqPDJMrUdbXQsXI0zU1Y+4L2hTULshFiL7oU8RGxBaxsPGasu8AO/9jaedtMO3/Hipz7kz950asL/+Wc/m7/2J57BL7/hw7z2Aw/wpg89yIP7Hfut52LLhp0Rnn7zLi991i38mefdzlNu3LkCf8k1GrL+QRUjA2BZO+KyjJaEuURMlVXAmljCxOitVpAKUyFMwc9GRPskIpNAPemoXGBn0jKrOmau43S1YmY7rqsW7NqGuW04Y5dMpeW0XVGJpyJQiy7wDQgdjkAkZKCNeDBg8mLWLoNXATAngWWoMSR8NIgk9qua4C3ejnRfh4gtaG3jIeNnXv1BPvjpBT/75z/n0NnQzaenfOdLnsZ3vmT9+yklfEwsu8CD+x0fP7/ivvsX3POJPd5w74P88Cvu4f/3++/l+77imXz7Fz4FkRNJ6h8+LiZpsHYAqpJhTWpSXZHqijhXHqvbdYSpwU8N7a7yV92pTLjPEn5HO4Kyo6XgZOI5PVtR28CZesXctey4luuqBXPb8ji3zymzYsc0XGcXVOKZSoclYSQSk77/LRldkqMS3/85pWycSkfAEJMQMJnzqljEmj0/YWI957sJK+9ISfCNJVaJjIkPGycKtETkB4C9lNI/FpG/B7wypfSfLvGxfgL4SuAT2wUTB8ei9fzof3kvX/j0G/mCp9942Y8nIlRWS8TT04o7bpjzwidf3//7x8+t+Lu/+lb+/m++g/d+co9/8Kc/89ELXBcDrHGGNZY0VIV4t8Rab2GaM6yJECaiRPsk81e5HKSOuAxYs7pjXnVMrGe3athxSrKX7OqUWXHKLtkxjfJXRKYjUIpolhWTIaSSYRmsRAICCayAHQlOQxIihn2ZUGVUamJFFy0TG1iYiNiU1fWHe+lOFGiNI6X0dy/zIX4S+CHgpy7/aB6d8bOvvpdP7bV895feeVWe75bTU370f3w+//C338WP/pf38tm3X8fXveiOq/LcVzUeLsMaj+RMapKzpEmtgDXJotFK8FOjhPusSBnAz5OC1iwic4+rAjsz5a9265ZT1Yqp9ZyuVuzYhl3bcL3bZ24abnB7zKVhxyjxrhlWyhmTHnPAKHDl+xKWRF26isSe+wLlv+amYRWrnI0pOn3K7TCpKvZdJNpEOqRc5poHLRH5fuBbgE+gw9Cvy9//SeA3Ukq/KCIfAH4O+ApUtvYXgP8DeDrwj1JKP7r5uCmlV+aB5W0cEMs28K9eqVnWOBu60iEi/PUvewZv+8hZ/u6vvY3n3HEdz7z19FV7/isam2AFw0hOkTRkHZb0koZaRaOzSvmrqcXPjUoadvI4zix3CCcJvxOVv5oGpvOWygVOTRtmrmOnajhTr1TK4FbMTcspq9nV3DScMkum0jEVTyWDDqvFEJNhlSraZOmwPRFfohJPLQFDpJagpP2oub2falamAmARa7pkmVrVghkTCSYdVlt6bTuXisjz0VGc56Bzjy98iB+/N6X0HOAP0CzqzwKfC/xvV/QgH6Xxc6/RLOsvf8nVybLGYY3wz772OexOHP/rr76NR60sR0wPWJg8Q9gPPVvNsCpHrF1fEsZaASvU6Oxgf6+ke6oTMom4OjCptCScOb1Nrac2nonxTE3H3DZMcmdQu4NKnI/BppSDLbYHrC45uuT6jAnIUggFrrkpEomuv+1Iq89jOiZGn8uZgJGkWC6AXJkxnqsdLwZ+OaW0ABCRX3uIny3/9hZgN3tVnReRRkSuSyk9eGUP9dETjdcs63Oecj0vesrVy7LGccPuhL/60rv427/yVn77rR/jKz7ztmM5jiOJi+mvjFnvEFZVn2mlyl1Ausfa4GeGdieT7rtKuod5ottVDsvsdFQTz2zS9RnWqVpLwsJfTYxnblrmpu35q6l01AyA1SUt/1apoksKVKtUEZIS6yZnYlUuBXtQEs/ceCoSU4HCrVdEqhTokuNB0zI1HbVR9b0xKYPW4V7OazrTeoTR5Ps4+rr8/7UOztdU/MIff4iPn2v4n//bq59ljePrXvhEnnnrKX7wt95B4w/ZWrrW4mIK9w3AYqxyz4BVSHd1aMiEe5E1TBS4wjTp4PMki0brQF17zbJcx8R5ahOY5AzLGVWpl6yqkoCVgYMKKH+lQKW3VaqG0nCkS7AkzbAIudPomUhgLokdI0zFMBWhFqGWqBwZEUvEXIYu+loHrVcC/4OIzLI18X933Af0aI/GB37kFffw3Duu4wuefsOxHouzhu9/2d186IElP/2HHzzWY7mkuEiGJZls7yUNVaVZVgGsaTWQ7lPtEvpZVrlPRYWjM5U1hClKuk8DduaZTjrmdcdO3fZzgzOrHcOJyeWZeCoJ1OJ77qlkWDFnUgWoVqliFesMWI7ARkmIjvXsSMfceHZMZEcMc7FMxDEVy0QMlUCVwbHczCHLwc24pkEr2xL/PPAm1CX1tUf12CLyc8AfAs8QkQ+JyLcf1WOf5Hj5a+7jI2dX/NWX3nVNyA1efOdNvPjOG/mhV9zD2eUhDZeuhSimfWKGrmDlVOFeVzr0XNX69aTWknBak2Y1cVoRZhVh5ggzJd67uaGbi5Lvc+0WhnkiziNpFnDTjum0Y3fasFO37FRtHnT2TDKX5YyO4FRGwUqznmwtg9CyAVYZsAqvBWDRxyg82I5p2JGWufHMJXFKDHNTMZGKiTgqsdiNuq+XS2TmPSV0IO2QGHbNl00ppR8EfvCA73/r6Osnj77+SZSIv+DfNn7/64/sIB8lseoCP/yKe3jRk6/nC49Al3VU8Te+/Jl85b/8r/zI79/D3/yKu4/7cB4+RmB/gcK9EO/Wqmmf069TVrmXW6ytEu4TUYfROjuLZvI91jpLmOqImQTqOlA77cYNIBWYmICTgB2VZUXKsK6nMliSloeZzyr3Y8AyOUsqHcKpeKYSmEpiKkIlBv3voS94m8AFHNqa5poHrW1cvfi3r/oAnzjf8C++/rnXRJZV4tlPOMOfed4T+In/+n6++vm38/SbTx33IV08coZ1oGlfXYGxvWCU3B3sx3KmatoX5k4Bayq0O2aNdPez0SzhrsdOQp9hjecIp7ZjZjuqPOA8MV65LDOUhnakp4oY2kSvvyqSBuWtlE8sHcKKwI5Rx4e5eKYSmYswFUuVbwAhRUJKBBJdUnJfO4/KjflkSEmIUZAoh20eXtvl4TauXjy4aPnhV9zDFz/jJj73qcfLZR0Uf+tP3s28dvytX37rtSuBGPm4b5r2rbmMbhLuObOKlSWWWcKp4LNxX+jvIU6y2j3bytS1p3aeqRuVgRIzf6TjN0ZSP9BcSdDv5azpoCgEuyGDVL6VknCaJQtTCdQSqQQsQsmxFKwikUQk0qWYy09Dm6yO+GTjQB+NZlgJbZkdIragtQ0AfvgV97DXeL7vGi2/btyd8H1f8Uxe8/77+alrkZTPmakY0duIcKfKgFVV2aUh30qHcOqUdJ9l0j0DVk+6TyFM1QsrTBNpGpFZYJKdGuZ1x7xqey2WOotqWVht3CwpdwxT380D1u4L11WLDkkX7qpII3akZSeXhROBWgSzkZmPAatLKWdZdsi0osUnq2M+SdSK+dHCaW3jysd99y/4t6/6IF/1vNt5xq3Xbun1tS94Ir/79o/z93/z7Tz7Cad5/pOOR0N2QWyUhNoVrEbzg1nOUFckpy4NsXbgDGFiSVUWjM50Q47aI6tbQ7czHs+JpGnE7nRUVWA+6ZhVCljFXXSSuaxSFjoTezGplold3zUsYtAy9FwGo8dhR2T9AHqRqWgfsRbBQE+2R6LOKKZER2CVIqsE+1nntUoVi5B9toLFB/WRlyCHHpjeZlrb4P/8nXdhDHzvn3jGcR/KQ4Yxwv/1Nc/h8dfN+K6feT333b847kO6MLIGa0y4M1K4673RLTmV6QErFuCqB9Jdt+WMSfeE1FEtkUcuo7UN1EYV5gWw9OvY67BMBrG6lzuEvgQs2VeVea6aQE1YU8mX0Z5yM0B1AO0ZUiKkUZYFdL3ua+CzmqhC1RAFoiCRQ3cPt6D1GI833vcgv/6mj/AdL34qt56ZHvfhPGycmVf82De/gNZHvvHH/4iPnV0d9yENyyaK/ipnVVJVSF2rlGGS5QyzijCvCXOVNHQ7Dj+3dLuqdO92hG5XMyy/A35HLWbiTkDmnnresjNrODVt2K0bdlzLbnZrmNkui0i7Pruamq5Xv6+P7AweWT1XJT7rrTpOGS0BT0nHPItGp/lWkTKPNURAM6uOQJM8qxRYpcR+NCyi43yccS5OOR9m7IUJ+6Fm2amDacprxow/XPNnC1qP4Ugp8Q9+6x3cuFvznS952nEfzqHjrltO8W+/7UV8eq/hm/7NH/Hgoj3uQ1rflDMi3jclDbF2pNroDOHIVqZkWEX1rl8r6R7riNQBVwWqKlC7QG1DHjj2axlWKQk3uaySZRXJQgGsqnwvl32VRB25GZH5/Y2kpeAB2BJT6m8diVVKNAmaZNkvZWGcsIg1TXQ0weGjIQSjmVaCi/QFLnypj/SN28aJit9/1yd5zfvv57u/5M4TtwLsOU+8jh/7lhfwwU8v+I6f+mNW3TGO+RT9VV8SmjVJQ5q4vkOYakMopWAGLSXdycT7yBOrTsTeEytQ1Z5p5Zlnx9GpU2nDZMRljQFr0g8ne+rc/esdGHLJaEnUrINVLVGzqY3bJliUV1x9thSsxoC1nxz7qWI/TvrbItYsQ80quLwfMZeHgW33cBsPHTEm/uFvv5Mn3TA/sZ5Vn/+0G/mnX/vZ/PEHH+B/+/XjW2/ZC0fHXcIiZ6irfhdhmOp6r6Jy9zOhmxWFe/Z0n6rSXT2xEkwiZhqoJ55p3fUWyWU8R2/t2hD03LT9MooiURj4qWFsp4hFTc6ixtnUZozxJCT9/wi0KdGmRJcUrEpJeD5WnI81D8a53sKcs37OWT/jvJ+w8DVdsMRgEZ+7h1vQ2sZDxa+96SO882Pn+d4/8Qwqe3JPg6/8rMfznV/0NH7uNffxO2/72PEchMmODaPtOcmanHEZYqXke6wkE+5CqBiU7pUuK9XV9Hnbc15HLy5ibcSaSJW3O5eRHNcT47kELIPQWUBaiWeszSpC0iJvGM8cmtHX4yjgtPl1SOvf6/L3VklokmWVHPtxwirWrLLVcpMcbe4adlG3UMdQAOvw4tKTVRNs40jCh8g//8/v4Zm3nuIrT7LlS46/+tK7+K/3fJLv+w9v5kVPvp7H7dRX9wDGiyeKLmuUYcVJ9sHKkoYw0fX0oVIDv1ipJ1aYJaLTAehURWQSqSY+yxtapnkfYbGZ2XFNr3afmxabF6kOHb8WK+kCQ74SAcEgVOiW6DYZrCQOKrRDkn7sJ6CAN15UojY2li4Zzscpq1RxLk55MOywiDVn/Zxzfsq5bsrC16y8w3tD8gaT5Q7bTGsbF41ffsOHef+n9vmel951UjdCr0XtDP/kq5/D2WXHv/y9e676869vzNEsKzm9Raeke6wySNXDrTfwK7IGB6lKJJd0Pb2L/cbnysRe8T64NnjmVsvB0iUsncBafC734gWA1Y/qZEfSLt8ioj5aSfrvbf5b+fc2mf62SpZFdKySZT/VeosTzocZi1izCBP2wqTnspqgW6djNH1ZKOM07mFim2k9xqILkX/xe+/h2U84zZ941i3HfThHFs+49RRf84In8tOv/gDf8vlP4kk3XMU1ZGLA2DUH0lTlsZxa9xGOh5/7kZwqDz9XqQcuXIIqIlXEuoCzkcoFJs4ztb5XvM8yUE3Er7mBFh1WP9ycAWu8rxCJmDIgLXEYbk6Hy2HGljLFP35sZ3M+zPQ+TtkLUxahZhkqlqFiFSq6YOm8JQWBoPsO5RFwWlvQeozFL7/hw9x3/5If+JbPuKaGoo8i/upL7+JX3/gR/vH//938y69/7tV7YltU8K7vFsaJywZ+6jYaqjyWUw8dwljlLqFTwEqTCDZhJgHrQr9BZ+o8O67NK78aTjldoHrGLvsu4WYpuLlYordGFlTEme9LxnVQRnbRyD82ztYGK5uKB8OcVao46+d9hnW+m7IKjqWvWHbaOYydRbxBAhgP5pDOQ9vy8DEUPkR++BX38JlPOMN/+8ybj/twjjxuPj3lmz//Sfzmmz9yddXydpRlOUvKRHx0QnJCdNKT7es3XZ1VykFsAhcxJuFcxJlIbbU8rDOPVUSjE/G9pGEqbd8h7LVXF6m1invD+L6o1VssLbZ3Ly3/f9BNbZgdqzh4bxVZw3hUp9zaaGmjloU+KAlPUD/mInfYZlrbuCB+9Y0f4YOfXvBj3/yCR12WVeLPff5T+Dd/8H7+zX99Pz/w33/GVXlOGWmyYq27CcNM/bC6uaGbKX/lZ0VAOsqwJpFUZQ5rEjAuUk+6finF7mhl/Y5rOOOWnLHLvLJ+qYr2rMXazJQCMZeD9N5YACFZbB6NLvbHltRvh96MsnVnvDKsgF6bbJ9p7ccJTaw4G2asYsW+z8r3ULHXTWiCY9lVdN7iOwedYFrBNoJtwDaPjsUW2zii8CHyQ6+4h2fddpovvfvRl2WVuPXMlP/+OY/n3//xfXzPl97FmXl15Z+0bNJxViUOTrOsWA/ZVT9TuEm615phKYelPFbtdJPO1HmmtmPuVIs1N22fYRX9VcmsKsIFJSEofxVHXFX/tcQReEUCEXvAhWy8KqyUmP3C1rw9OmTQWsSaLmr2VTisZahYeV3O2gZL6y2+s0QvSGcwXhAPxieMv+DpD365L+1d2sZJi19/80d4/6f2+ctfcuejNssq8R0vfiqLNvDy1957dZ7Qmr5jGCuTV32Nu4QDYIV6IN01w1LS3YzGdKaZx1IRqUobdAu0ikZ3TFmo2o26hZsjO2HNTnkcCjau94Nv07AWrNxUX1X3PFUpA/djzaKUgfn7w3hOlRXvFU10PWCtgqPxjtY7Om8J3pA6g+lAOuWyTAum22Za28gRYuJf/t49PPPWU4+qjuHF4u7bTvOCJz2On3/tffyFL3rqFQfpNNJlhanLnlimX0ThsxbLzxLJoRYzdeavpgFrIy6DVe0Cu3kpxa5ruK5aMrMtZ9wyq90HX6u1sZyRrUxMhpC/bpOKSQth3uZtOuMN0SXbGm/aKQssSialvz8Q7+V31RvL0WR+q4mOfa981sLXLH1FExyLVoej29YRG4s0FrsS3FJwS3CrhFsdDrS2mdZjIH79TR/hfZ/c57u/5M5HhS7rMPG1L3wi7/vUPq/9wANX/smMIVkhOkNykr9Gb9Vwn4rS3Q0ZlrVaFlZuGISeWN/bJRfHhqJwLyVhuRXAKl5XNfFAFfw4xkS8fi153f3oliRnYYOdzLADcbCXaWLVm/op4T4m3vXWBKsupd4QvQFvkE4wneSuoZaGtt1mWttAdVn/1396N3ffdpov+4xbj/twrlq87LNu4+/9+tt5+WvuveILZ9PEqfp9MkgcysqvMMtK9xrCLGdYdcROAsaqrKF2Ogi9W7VMnOd0Jt13bcPjqgUT07FrV31ZOJemt5QxEvuB5xIGwSahI66tri9lYRhxVJaY5Q/D7yvB7ohJeg+sgNBFl/+9eMiLjuNg+pKwjY6Fr2hjkTdUNJ2jaSp8Z0kri1manGWBW4BbQrWI2OXhht63mdajPH7xdR/ig59e8Nf+xKND/X7YmNeO//45j+c33/JRzq2u7OqxZIuQdDRXuEa85wyrVuGo8leeug5Mq3XnhqLFmo0yLVW6t73baHFoGAPW2o1hCLqELquQtb2FJcbyiD4Lyz/bZ1bRrWVZJbvSjMvhk0oalkEBqwmuJ967kMn31iKtwTTaMTS5Y+iahGkjtjuc5mELWo/iWHWBf/Gf38Pz7rjuUanLerj4quffTuMj/+ntH7+iz5OqMrIjWf0+jOiESVLF+2QMWGULdLFLVsDarRp2bMuu1SzrlF0NoznFqYFMtBPWAWtkIWM3ACtmHqpkXTH7so+38WzGGLCKlisWB9K4Dl4+WpowAFWXb20h3ztL7Ay0BmkF29IDlm3AthHbRqQ9HGhty8NHcfzkqz7AR8+u+Kdf85xHfcfwoHjuE6/jCdfN+M03f5Q/87zbr9jzlKUUfqaaLD9T8j1M1WImzPPK+pnvCff5pKW2oV/5teNadqy6i552K+a2YW7atS5hz2Vl4JqMtuGU7CNCNuLLC1hzdzAwqOJNnkmsRvOJQCbZ8+OkTLiPAGucgfmo3FfJqpahYuFr2mjZ72raYFm2Fc2qIngDS4tpDG4huIVgl1DtQ7WMuEXELj1mdTjNwzbTepTGA/u6Euy/febNfN7Trr2VYFcjRIQ/+Zm38sr3fPKKbqdOToi2KN83VO95nlDqmCUNvtdgzbKJ33TkizWzJavyF0gYhnVeYVgRBnmxhN4MZbwmd/p6/qp0CuMaYBXnUp1TjBdV0gOj7uFwr1mXwedbF5V077xV4j0IqTWqycpCUtPmLKtLSsK3EekCckgjxy1oPUrjh19xD/uN5298+TOP+1CONV72WY+nC4nfvYIlohr7ZfK9mPrNEn6eSDPdnjOZdezOGk5PG66bLnncZMH1kwU3Tva5vl5wnVuo2t0tOGMXzM1IjzVybyj7Biti3upM3uwsWJG8Cac4MmQCPWlBVZatFu/4Ta2XyUD4cFHKS58sTR7PWYWKpa9YdBWLtmLVVnSNIy4csrS4fck3qPaSZll7kWov4PY7zLKD5nC22VvQehTGx86u+KlXf5A/c42vBLsa8dm3n+H2x834zTd/5Io9R6yL7QzESSbeJzoAbaaeulbX0Z26Zbdu2K0aTlWNEu6902i2mMmZ1OA26jOPlbfoZB5rKpEKFKzWxmtydy+DVekQ2rystTzuZtlpD3CEKHGgbKLPqlTp3ngVkK461WJ1rSOuHNIY7NKoHmsFbpmyLitiVwHTBKT10HZIty0PH7PxQ694DyklvvtL7jzuQzn2EBG+9O5beNV7P33FfOQH8l0G8n06eLsXDmsNsArh7poesPoB6DxLON6YMwassg2nANZ4UWqEtSyrlIelJNRZxSFz65+LAlzZ3bSUjL3b6bgTmT22kvSD0E1WvWun0BEaq4C1Mkq6r8Au9d6tIm6ZsKuAXXmk9QpY7eFK+C0R/yiL++5f8POvvY+vfeETeeL18+M+nGsiXnLXTfzkqz7Aa95/P190101H/vhlIYWf5y3Qs4TMAm7asTNrODNbsVO1XD/ZZ2Y7dq1KGioJzG3T81Y7phnd+7WScKfsHSSxYyTvHTRYhEDq/dpXydIWPVYm3ntveInsSIuRyFQuBIiDBqZN9t7ql1gkwecMq5DvS1+x39bsr2ra1hH2HNIa3L7B7eswdH1OO4X1XqTa0yzL7neYpkOWDTQtyW8zrcdk/NQffoCU4C998dOP+1Cumficp15PbQ2vfPcnr8jjR6flYVHBpyphqkhVBWb1+iIK3Us4kOzFp73qZwXjqCQ8OMMaA1aJkBJdImdAw5gNMGRYudycSked5xI3pRHjONAzPknvYlqI9y5YGn+AFmuV3RtW9C4OdqXEu2kz8d558IEUIoSt5OExF40P/OLrPsRLn3ULt52ZHffhXDMxrx0vfMrjeOV7rgxo+alyWWGaVPU+DUxnLfNJy5nJiuvqJaeqFafdam1FfTHwK46jhW8aL0+d5k7hXBK1CBVCJQZTNFfEXuLQIbQYWnQAupR7Fcpl1QTmpssbdxJd7gCOYyyN2IyiyyoZ1sLXnG8mLNuK5WKCXziksbjzqniv9sHtJ9wK6vNRy8L9gF10mMZjFivoPKlpoWlIYds9fMzF77zt4zyw6Pj6E7oS7ErGF915E+/++B4fPbs88sdOI6lDWV1fO1W5T0ebn/sV9Xm4ucgMLGVl/XjjswLWXAJzSUxFmIhhIo5KbG8j0+8bTAOXVexnLKnnxHTNve8Bq0SxmemSUz1XBrGSrRUniKJ8L+4NC1+z6GrtFDYVfqldQruftVhLBaxqUcj3iF1F7MpjVh2y6jTL6jrwXgFrC1qPvXj5a+7l9sfN+MKn33jch3LNReGy/uA9nzryx+6Hoh3gdAB6Uvk8+Kz34xVfMJReRtIgRejLN9+voC+ShqlYKiyVWAyjTCslQkq0ZenEyL3BZMX7sPNw2BpdYtB0Sa+aL49RxKVFCX8BYHUVq5VKG2RlsQuDXa7PFLplpFpE3DL0AlJpPNJ20HUk75XLCoGUti4Pj6n4+LkVr3rvp/maFzzxMTVjeNh45q2neNy84o8/cP+RP/ZYSCp11E3QzjNxClhOAkbSmlf7YAsTs/ZKS8JTZsVcPHMJnDLC3FjmUjGRiom4fglFJNIR8kZnXT/f5nEdUMAqXcfxgouQdAaxQ7foDN7ug3dWcW9YxJq9MMkLVqec7Wbc3+xw/3LOA4sZZ/dmdOcncLbCnTXUZ4X6LNRnE/W5xOR8pD4fqfY8bq/D7rfIokFWDawaLQsLeIW4zbQea/FfMsn80seAX9alhIjw3Dsex+vvffDIHztaSDY7kJqEMQkjWoaVUqxwRxElsoF+AcWYKK9RDdZUUP4KLQULWFkZu5Dm7c6jTGlMwOtzxA2yfSDSh/lBN7KhWR+MLs4Ne75mr5uogDSXhF2jOiyzMheQ7q5J2CZm0n1QvEsh3r2Wg6lkWCmS4taa5jEVr3z3J7np1IRnPsbFpA8Vz33idfzeOz/B2WXHmdnR2TAno15ZySasSZi8/RkUJHw21zMxdwlN6k37xlnWVDxz49kxSrgX/sog62BFpEuBVYprXFbLYOJXnCBg5O0+yvL6DdDZ273fAt27kdY00XG+m7IfFLAeWM1YNDV7+1Ml3VeW6mwm3c9DtZ+wK5icD5gmUS08Zum1U7hoEB+g7UhNozzWiMtKMa3Z4zxUbDOtR0GEmPiv93yKF99542NyMPqw8bwnPQ6AN9734NE+sMl4MHrpfeaCmuBUOZ5sP7NXwhJ7RfpUOiYSqEiDpCFnWGPACikSUiKgm6C7ESel/276W/FxLxlYi2GVXL6tb9FZxAmLoLbJi1hz3k8530052015YDXnwdWMs4sZ+6VLuLDYPTOM5uwn3CJRLSN2lXBZ7W7aMIhHO09qOwiRFKICVkyPCLDyy72Nkx5v/fBZHlx0vOQKCCcfTfFZt59BBF7/waN1M00CG1VZFmGqlqmJg51LCe0eprUh6OLYUCQNZuPjGVIkknKmpVlWmTPU59z4+VwyqgTC9tlYNwatzF+tklomL0LNvp+w52vO+wnn2yl7Xc1+UyvpvsqzhIs8mtN3B9UuWbuEeTynjOjkkpCugxh64p0CVgWwDknEb8vDR0G88t2fRIRt1/Bh4tS04hm3nOINR51pjSIlIQRDGyxGEs5kQ75Rx6509KYyHoRWwLIHPGZIpdQcAKtske9yp08dSYVu9Ahdsv1qsDKOU6QNZTnFIk76lV8PdnOWoWI/1DzYzFj5inOrCYtVrcPP5ytMY6jOD/Yy9Tk18av2s6ShidhFh3QRs2qRptPyr221JGy7nsta47EOCViwzbQeFfGq936aZ912mht2J8d9KNd8PPeOx/GGex8gHpL0PXQkvaUIMRpCvhVHhJDGCnXtJo4HlQ9Un6OZlX6tgNWXhin1JWHJsMalYRnjKfOHZctO2bwzlIdubW39eT9h4Wv22gn7bc2yyYC1cthsk2yXClhulTLhnjBtwvZKdyXeyWR7fx8CxAPKwkcAWLDNtE58xJh4y4fP8qef+4TjPpQTEc994nX83Gvu5X2f2ufpN+8eyWNK0u3IEoQUDMEbumCwxtAFS7RZN5WdFooCfryAYiz4jCkRJSpRJrF3uAopqcyhmPzl0rCY/XW59INCvltsSmsuDatY0SbL+TBT7ipMedBrhvVAM2ffq/7q7GJG21q6/VpFoyuhOmewDVTn6cvBek+7g24RMI3vMyx8UC1W22mnsO20U+i9AlbOtC4ltqB1wuP9n95nr/F85u1njvtQTkQ86/GnAXjnx84dGWgR0UwrQoqaWcU4jMiU8nBs6lfU8cNDCCGBEVW5k7RcLB0/Xaia6FIkoBY0pTQMmB6wCm8WUEuayMBtAX3H8HyYshcm7IUJ57opC19xtp1qdtVWLBe1zhHuq2jU5bEc22TSfZVwjS6jsF3UsZzRPKH4MEgbxor3TdL9EWZZsC0PT3y85UNnASWZt/Hwcectu1gjvOOj547sMftMK0oGMFmb6SuarbXVXge4hMbRTbuDObMiqJNDSms/o6XnYItc9hO2hXQfLWJtR0LSsXB030/Y6ybsd1oOLpqaZlURy1jOwvT8lVuUDmHK23MG0v1gwPJQSPdNPdYllIUltpnWCY83f+gss8ry9JuOKGt4lMfEWZ520w7v+Oj5o3vQVAArQRJSykAlicoGapNHeUygyjOI48WqEUOHySWipm0G6FJYW1WvPBZ0Cdpk+uHoAkZdXphaZBXFG2uVva+65DgbZixCzae7Hc53U/b8hPuXc5adY28xpVs50spiz1tsI7i9oUNYn0/YNpPuTVDh6KpT0WiTZQ0hDCXheEQnxCMBLNiC1omPt3z4QT7j8adxdps0Hzbuvu00r33/0Y3zHORQLJIQhtLQjaxoNqPorDqMlpnZJCaKApWFwc8q/3yX9Vfra+3XZRWRoVQsy1UXQXVYKmuYsJ+HnpvO4VvdS6jmfernrvOD6tTgMuFeVO4luxIfB6I9RAWsuE6+rwHWZcYWtE5whJh464fP8XUveuJxH8qJirtvO82vvvEjPLhouW5eX/4D5s6hJEASImDNkGXVJbvKKnhL1AxLhjX1Gn5tzKcajeAYBsBaJV0msZ90XnCRFe1lJGccIQ87L4Kuqv90t8O+n3B/M+dcM2XRVqpw7yxpz+nQ80qozgt2BfX5QYNV7Wt2ZRcdpg3gI9K0SIhZi+UVsLpWyfbCZV1Gp/Cg2ILWCY73fnKPZRe2fNYjjLtvUzL+HR89f6SbiorIVCRhTcTlWyV6v+m1HpKhE0tJvgp5XqEZWZdMvznakHrv9/GgcyHWV6nqM68SZWdhkxx7fsIy1jzQzln4mrPNlL3VRGcIFxW0BrdnVc6wgmpPTfuUdM8arIXH+Ji7hMpbaUkYe8eGonbvtViPcETnMLEFrRMcb84k/Gc+YQtajyTuvk3nM9/x0XNHu15NAJMQE7EZpNx4PdfYbQGDIarrgwAJbNZU9eAlgZAkzymmPAYkmb+yrGKdSXbX+2iNO4WlXNzzk550P9dOWflqAKzGQT/0rHYyhXS37QBYbhWwTc6uNgh3ipQhxPWSsGRYJY4gy4ItaJ3oeNfHzjFxhqfcuCXhH0nctDvhhp2ad37siDqIkoemDYhJWJuoSpZlQk/Kl1B3UMFiWFFhU6KT2Hu018V6OQ9UWyKkskzVcC5O6ZLjfJzm4eZJn1WVpapj074Hu1nfIbx/OVdJw36tHcLGUJ032EaHnt1icBrVTMv33UGz7IbsqvMqFG11NIcul4IpkTp/IYd1RIAFW9A60fGuj+/1LfxtHD5EhLtvO31kHUQtCxOY1JeHIoM9jeXg0igU4l0iIVnseJhahoypeLcXAem4U1iyrrX19Ul5rP0woQmOvW6incK2ZtFUtE11wXov044I95VmWGWRai9n2FC26/1YzrAhGr0CgAVb0DrR8e6Pnefzn/7Y3B59uXHXLaf4udfcS4zpSEwTez5r008rg9d4ryBkIEqRgFVfG+jnEwPSK9nbZLES+9GcLlnOxZmWfWF6AWD5OGRZZzudH7x/NWe/rVg2Nau9Cakx2D3bj+RU55W/qs8PQ8/VvqrbdY4wl4PZoaEn3McZVgGuTQ7riAELtqB1ouNj51Y845atf9alxF237LLsAh96YMkdN1zmqjUZbpIzLQWr2Ls5jH3hgV4IWgAJII4ysg6wSX8+Rv3Z0iEs5eAi1r17RMmwlqGmiZZVHstZdjr0vFrW+MbBnsuloOl3EdbndX6w3s+C0UZJd3zsXRrW+SvVYJHiUBJeJcCCLWid+LhrC1qXFHfm1+3dHz9/2aBVsqwkqRdtlSyrxAWdw0y2tyOJQtkGPfyMEJOaFSpguTzgPOmzqVIK6teGfV+zCi4PPdesyqacxsLK4vYNps1k+3JkK9Mk7DL2PljSetVfdX7oEHqv4BSzQ0PpEo4Bq39RrgxgwRa0TnzctXUqvaQoc4fv/sR5vvRyLaplVB6yzmlZWee0VAFfrGbsmikgoNYyafjZsil6DFRNdPhks7mg7h5cBbVF3i8LJ9pKDfuK/mo50l81UO2N9Ff90HOnHlhdWNdfZf5q3CHsS8JNpTtcUcCCLWid6NidOB5/Znrch3Ei48ys4tbTU97z8b2r8nzFSRSJMM6uRu6BvcVM/l4XB5V7k78uK+i7ZGijU6PBaFn6ijbafuC5aRzdUvVXRTBqs2mfaVPuEhZJQ8B0EWkCplXDPvFFthAHcCoANS4JrzDpflBsQesEx1237G7tlS8j7rxll3d//Ag6iAe8BWnko9UlS0VgFas173agH3YG1uQKhVRvYkWXDMtQ6Tr6ZGiD0wwr6P833rHsHF2wyl3lcRyzb7Gt4HJ25ZY5w2rSsJq+idhlnh9cdcP8YAatnnCPaT3DSumK6bAeLragdYLjGdvS8LLirltO8TOv/uDlP1Ba/zIlUXDJewIrico9GbvGc4094ws35aPN5d9Q9vl8H5LR7mDQpaqtt/hg8cHQNhXBG9LCIa3gVurfblpVt7uVLp2o9yK21U3PJmdWvf6qdAdj7H2v+hnC3gMr9YB1KVbJRxFb0DrBsSXhLy/uumWXxh/NeIn0zqVCjEKICjAKXLFfeLoJWrEXhAptdIQkrEJFGxxt8ZYPlm7khuqDilO7zhG8IQRDXFnwBrNQoahd6cIJ2w7+V7ZJuEXAjuYHpVNXhrXuYNFgZYK9B6xRhvVIVn4ddWxB6wTHl33Grcd9CCc67jwi0JdscCVeiJ2hs45zqwlNUNA5Z6Y6h5i3S5eRnAJYPoNRmwGqzdmTjwbvLTEKMai9TIpC8oYUBTpBvEE6oVoJ0qHeV81g1mfb7H21StgmYPe7dXfRom4vvFUpBVNcy67UJlkBPpWlqkc0AP1IYwtaJzgef93suA/hRMedR+VcmsAEQbyQOoO3lqZzfWZUWx3lsWbI6sacV/m51qvWyntLCNn91BuIQgqiXjURCIIEQTrBdILxYJd67xYqFLUNVIuE6RJuof5X0uZh5w0pw1opWLKoXCb2gFXAaZN4P4bYgtY2HrNxalrxT776sy/7cUync4e2FVKjWdASMDaxtAFrUy+DAAWslM0CYxRSNHo/yqBICkx40QmhAERBgn4tAUyrQGXavNm5UxmDbVHeahExXcQu/dAZbFotBcfZVRaKEgtftQFWm+XgMfBY49iC1jYe0/FVz7/9sh/DNnqfrE5OxyoRu4pgE8FUdJIG1Xz5nMfcciw2zUEBiKj3peQ0QfLPKFBJUpCUqGBluoTplGg3XkHLdFnVvtKsyiy7QdVeVniNjfouBlawDljHDFYltqC1jW1cZhifSEYwHWCk94tPNvWbp0eW8T1pT9INPgWMxOvvKXgV3/nRLWSw6hISFSyNz6DV6ICzXWVX0S5gVjqK0xPtxahvM7vqOaqN7AqOlXC/WEg6ZtTcxja2sY1HEltj8W1sYxsnKragtY1tbONExRa0trGNbZyo2ILWNraxjRMV2+7hCQ4ReSuwOu7jeJi4EfjUcR/Ew8Q0pfTs4z6IbRwutqB1smOVUnrBcR/EQ4WI/PFJOMbjPoZtHD625eE2trGNExVb0NrGNrZxomILWic7/vVxH8AhYnuM2zjS2Crit7GNbZyo2GZa29jGNk5UbEFrG9vYxomKLWidwBCRbxSRN4vIW0TkVSJy+aZQRxwi8uUi8i4RuUdEvu+4j2czROSJIvIKEXm7iLxNRL77uI9pG4eLLad1AkNEPh94R0rpARH5CuAHUkqfc9zHVUJELPBu4KXAh4DXAl+fUnr7sR7YKETkNuC2lNLrReQU8Drgf7iWjnEbB8c20zqBkVJ6VUrpgfy/rwYu38nuaONFwD0ppfellFrg5cCfOuZjWouU0kdTSq/PX58H3gE84XiPahuHiS1onfz4duA/HvdBbMQTgPtG//8hrmFAEJEnA88F/uiYD2Ubh4jtGM8JDhH5YhS0vvC4j+WkhojsAv8B+CsppXOX+DBbjuVo4lCbh7eZ1gkJEflLIvLGfHu8iHwW8OPAn0opffq4j28jPgw8cfT/t+fvXVMhIhUKWD+bUvql4z6ebRwutkT8CQwRuQP4PeCbU0qvOu7j2QwRcSgR/yUoWL0W+IaU0tuO9cBGISIC/Fvg/pTSX7nMhztRH6IYE/fev+AT5xvOzCruumUXfTmOPQ51EFvQOoEhIj8OfBVQdrr7a81JQUT+JPDPAAv8RErpB4/3iNZDRL4Q+APgLeg2QYC/lVL6rUt4uGv6QxRi4mPnVrzh3gd45bs/ySve9Uk+eb7p//3W01O+6yVP5Zs/78kYc6zgtQWtbWzjKsWxfojOLjped+/9vOOj57nnE3t86IEFn9prOb/yNF1g2QV83qhzaup4yV038YVPv5HHXzfjY2dX/PIbPswfvu/TfMHTb+BHvvH5nJlVx/WnbEFrG9u4SnHVP0QpJV75nk/xo7//Xl79/k/3qwhvOzPljuvn3HRqwqlpxbQyzGvL46+b8cxbT/PZt5/BWXPBY/3ca+7jf/21t/LsJ5zhp7/9c9idHEuPbgta29jGVYqr+iFadYG//otv5tff9BFuPT3la174RD7vqTfw7Cec5tT00rOk337rx/hL/+71vOjJ1/NT3/4iKnvV+3Rb0NrGNq5SXLUP0X7j+eafeA2v++ADfO9L7+IvvOSpTJw9ssf/xdd9iL/2C2/iO178FL7/Zc86ssc9ZBwKtLY6rW1s44RESom/8R/e/P+2d+/BUVV3HMC/v91NSFgSAuQFCSGBhBBEwfCQ8qqCrYBFGcQpYNXWR0eLTp2pb1sHnVqf7ThOHR8VRKc4WkVbtTx8AYoMEEQChCRgHiQgGxOSkISQ969/7GYaIyT7uLt3L34/Mxl2l3PP/U1m8p17zp57Lr6uqMPzK3Jx5UXDDT/H0smpyK+sxz++KENu2hAsuND4cwSK67SILGLtjnJ8uP8E7r4iOyiB1e1PvxiPiSPjcN/6/ThefyZo5/EXQ4v8IiKrRORuz+tHReRyP/vhbgteqKxtxhMbizB3XCJumzMmqOeKdNjw3LJJ6FLgrje/RkdnV/8HhRBDiwKmqg+r6id+Ht4B4A+qOh7AdAArRSTkkynh7rH/FsImgj8vnhCStVSjhjnx58UTkFdeh2c+Ohz08/mCoUVeE5GHROSwiGwHkN3j87UistTzulxEHvfcbrRHRHJFZLOIlIjIbb375G4L/dt+pAabCly4Y24mRsRFh+y8iy9OwXWXpOHFbSXYdPBEyM7bH4YWeUVEJgNYBmASgIUApvbRvEJVJ8G94nwtgKVwX0U90s850sHdFr5HVfH0R8VIiYvGzbMyQn7+hxe557fuemsf8sprQ37+s2FokbdmA3hPVZs9uyG830fb7v87AGCXqjaqajWAVhGJO9sBBu22cN7Zerga+ZX1uHNuJqIijFva4K0BDjtW3zgFI+KicdOredhXWR/yGnpjaFEwdN/Y1tXjdff7Hyyz4W4LZ6eqePaTI0gdEo0luebt8xg/aADW3XIJ4pwRWP7yTnxaWGVaLQBDi7z3OYDFIhLt2Z54kRGdenZbWA339tF/M6LP88UXR2qQX1mPlZdlItJh7p/q8MHRWH/7DGQmDsKtr+/Bm7srTKuFoUVe8UyWvwUgH+6dUvMM6nomgOsBzO2xX9hCg/q2tBe2liA5NgpLcsPje4nEmCi8+dvpmJ2VgPvfPYDnt3xjSh28jYcocIb/Ee2rrMfi57/EQwtzcOuc0UZ3H5D2zi7c83Y+/r3vW6xaNB6/nmnYFwS8jYfIql7cWoLYKAeWX5Jmdik/EGG34ZlrJ6K5rROrPjiExNgoLAzh7T4cHhKFmWJXIzYVuHDjjHSztojpl8Nuw3PLL8bFaXG45+18lNWcDtm5GVpEYea5z47AGWk3ZV2WL6Ii7Pj7ilxEOGxYuW4vWto7Q3JehhZRGDlS1YgNB07gxhnpiBsYaXY5/UqJi8Zfr52IQycaQjYxz9AiCiNPbirGwAg7bpkdXpPvfZmXk4QluSl4YWsJilzBXxfM0CIKE1uKvsMnhVW4c14WhjrD/yqrpz9eOR6x0RF44N0DCPaKBIYWURhoae/EIx8UYHSCEzcZt4QgZIY6I3H/gnH4uqIeH+4P7s3VDC2iMPDExiKUn2zGo1dNMH31u7+uyU3FuOQYPLW5CK0dwZuUt+Zvh+g88lGBC2t3lOOmmRmYlRVvdjl+s9sEDy7MQWXtGazbGbzbfBhaRCY6VteMe97ZjwkpsbhvQXb/B4S5OWMTMH30ULz0eUnQrrYYWkQmae3oxMp1e9GliudX5Br6VB0z3XFZFqoaWvHOV8eC0j9Di8gkT20qRv6xU3h66USMGuY0uxzDzMwchokj4/DitpKg7C/P0CIywe6yWqz5sgy/mp6G+ROSzS7HUCKClZeOQWXtGWw46DK8f4YWUYidaevEve/kI3VINB5YkGN2OUFxeU4SMuKdWLO9zPC+GVpEIfbCthKUn2zGk0sugjNMb4gOlM0m+M3MdOyrrMfeijpj+za0NyLqU2VtM17aVoJFE0dgRqZ1lzd445rcVMREObDa4KsthhZRCP1lg/v5hQ8sGGd2KUHnHODAsqkjsfmgC65TLYb1y9AiCpFdpSex8aALt186JqTPLzTT9dPT0amKNwzcU56hRRQCXV2KxzYUYvjgKNxqoR0cApU2bCDmZifijV0VaOswZvkDQ4soBP6Tfxz7j53CvfOzER15fiwi9dYNM9JR09SKjQY9pZqhRRRkLe2deHpTMS5MGYyrJ4bHk3VCaXZmPNKHDcS6XcYMERlaREG25ssyfHuqBQ8uzIHN5tUDZ84rNpvgl1PTsLusFiXVTYH3Z0BNRHQOtafb8MKWElyek4SfjBlmdjmmuWZyChw2wVt5lQH3xdAiCqLV20vR1NaB++ZbfweHQCTGRGFeTiLWf3Us4Al5hhZRkJw6047XdxzFggnJyEqKMbsc0y2bmoaTp9uw7XB1QP0wtIiC5J87j6KxtQO/uzTT7FLCwqyseMRGObC5ILCbqBlaREHQ0dmFV78sx6XZCZiQMtjscsJChN2GeTlJ+LSwKqAtaxhaREGwo+QkappasXxa+D3W3kxXXJCEuuZ27C6v9bsPhhZREHyQ/y1iBjjw07EJZpcSVuaMTcAAhw0fFVT53QdDi8hgrR2d2FTgws8vSEZUxI9r9Xt/BkY6MGdsAj4+VOX38xEZWkQG21ZcjcaWDlw1aYTZpYSl2VnxOF5/Bsfrz/h1PEOLyGCbC6oQNzACM37Ei0n7kps2BADw1VH/NgdkaBEZLK+8FtPShyLCzj+vsxmXHANnpJ2hRRQOqhpaUFHbjGkZQ80uJWw57DZMSovDnnKGFpHp8jxf5U9NZ2j1ZfKooShyNaCptcPnYxlaRAbKK6tFdIQd40fEml1KWJsyagi6FPjaj4deMLSIDJRXXofcUXGcz+rHpLQ4iMCvISJ/s0QGaWhpR6GrAVNGcWjYn9ioCIxJGISCbxt8PpahRWSQ/Mp6qAJT0oeYXYolZCfH4HBVo8/HMbSIDFJ4wn3VcMEI3iDtjeykGFTUNqO5zbfJeIYWkUGKXI1IiBmAoc5Is0uxhLGePcaOVPm2BTNDi8ggxa5GjEvmZn/eyvb8ropdvg0RGVpEBujo7MKR75qQzR1KvZY2dCCiImwo9nFei6FFZIDyk81o6+jCuOFcn+Utu02Qlej7ZDxDi8gA3UMcDg99MzYphsNDIjMUuxpgEyAzcZDZpVhKdvIgfNfYirrTbV4fw9AiMkCRqxHp8U5u+uej7GT3cNqXeS2GFpEBiqv4zaE/xiQ4AQBlNae9PoahRRSglvZOVNQ2IyuRoeWrEYOjEemwMbSIQunoyWaoAqM9Vw3kPZtNkDHMidJqhhZRyJRWu1d0j47nJLw/MuKdKKvxflU8Q4soQKWeoU0Gr7T8kh7vREVts9ftGVpEASqtPo2k2AEYNMBhdimWNDreifZO7x8nxtAiClBpTROHhgHw9QqVoUUUoNLq05yED0BGPEOLKKROnWn3+Q+P/m+YMxIxUd4PrRlaRAYYk8Dhob9EBKN9CH2GFpEBODwMjC9XqgwtogBF2m1IHTLQ7DIs7aLUOK/bMrSIAjQ1YwjsNjG7DEu7aVaG121F1fv1EUR0VvwjMoZXyc8rLSKyFIYWEVkK7zsgChwntEKIV1pEZCkMLSKyFIYWEVkKQ4uILIUT8UQBEpGDAFrMrqMf8QBqzC6iH1GqOqG/RgwtosC1qOoUs4voi4jssUKN3rTj8JCILIWhRUSWwtAiCtzLZhfghfOmRt4wTUSWwistIrIUhhZRAETkOhHZLyIHRGSHiEw0u6beRGS+iBSLyDcicr/Z9fQmIiNFZIuIHBKRAhH5fZ/tOTwk8p+IzABQqKp1IrIAwCpVvcTsurqJiB3AYQA/A3AMQB6A5ap6yNTCehCR4QCGq+peEYkB8BWAxeeqkVdaRAFQ1R2qWud5uxNAqpn1nMU0AN+oaqmqtgF4E8DVJtf0Pap6QlX3el43AigEkHKu9gwtIuPcDGCj2UX0kgKgssf7Y+gjEMwmIukALgaw61xtuCKeyAAichncoTXL7FqsSkQGAVgP4C5VbThXO15pEflIRFaKyD7PzwgRuQjAKwCuVtWTZtfXy3EAI3u8T/V8FlZEJALuwFqnqu/22ZYT8UT+E5E0AJ8BuEFVd5hdT28i4oB7In4e3GGVB2CFqhaYWlgPIiIAXgNQq6p39dueoUXkPxF5BcA1AI56PuoItxuTRWQhgGcB2AGsUdXHzK3o+0RkFoAvABwA0OX5+EFV3XDW9gwtIrISzmkRkaUwtIjIUhhaRGQpDC0ishSGFhFZCkOLyMJEZJWI3O15/aiIXO5nP1EisltE8j07LTxibKXG4W08ROcJVX04gMNbAcxV1SbP6vTtIrJRVXcaVJ5heKVFZDEi8pCIHBaR7QCye3y+VkSWel6Xi8jjnluN9ohIrohsFpESEbmtd5/q1uR5G+H5CctFnAwtIgsRkckAlgGYBGAhgKl9NK9Q1UlwrzZfC2ApgOkAzjr0ExG7iOwD8B2Aj1X1nDstmImhRWQtswG8p6rNnp0Q3u+jbff/HQCwS1UbVbUaQKuIxPVurKqdnpBLBTBNRPp9cKoZGFpE569Wz79dPV53vz/nfLaq1gPYAmB+0CoLAEOLyFo+B7BYRKI9WxMvMqJTEUnovvoSkWi4t2cuMqJvo/HbQyIL8eyj/haAfLjnnvIM6no4gNc8e8rbAPxLVT80qG9DcZcHIrIUDg+JyFIYWkRkKQwtIrIUhhYRWQpDi4gshaFFRJbC0CIiS2FoEZGl/A/6tLyGhCJesQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "posterior_samples = posterior.sample((5000,))\n", + "\n", + "fig, ax = pairplot(\n", + " samples=posterior_samples,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + " upper=['kde'],\n", + " diag=['kde'],\n", + " figsize=(5,5)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 1D and 2D marginals of the posterior fill almost the entire parameter space! Also, the Pearson correlation coefficient matrix of the marginal shows rather weak interactions (low correlations):" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWYklEQVR4nO3df6xcZZ3H8fen97Y0u6wWKCmI2EJokBpWkAYwRGUBtRBDu4rabpTiQroY3F1FDSKJbHBJym4iiz+xgQooCygi1ohhkR+LRmEpbKH8WOQKKq2VSvmhhFK4vd/94zxThunMnXM7Z849c+7nZU46c84z85yJuV+eX+f5KiIwM+u3aZN9A2Y2NTjYmFkpHGzMrBQONmZWCgcbMyuFg42ZlcLBxqymJK2WtFnSgx2uS9KXJY1IekDS25quLZf0WDqWF3E/DjZm9XUFsGic6ycC89OxAvgGgKQ9gfOBo4AjgfMl7dHrzTjYmNVURNwJPDNOkcXAVZG5C5glaV/gvcAtEfFMRDwL3ML4QSuX4V6/wMyK85f7z4ztL43lKrvt6VceAl5qOrUqIlZNoLr9gCeb3m9I5zqd74mDjVmFjL00xgHvn52r7P+t2vRSRCzs8y0Vxt0osyoRTJumXEcBNgL7N71/YzrX6XxPHGzMKkbKdxRgDXBqmpU6Gng+IjYBNwPvkbRHGhh+TzrXE3ejzCpEwLSCmgCSrgGOBWZL2kA2wzQdICIuBW4CTgJGgBeBj6Vrz0j6InBP+qoLImK8geZcHGzMqkQwNFxMsyUilnW5HsBZHa6tBlYXciOJg41ZxaimgxsONmYVIsG0ggZkqsbBxqxi3LIxs1IUNUBcNQ42ZhUiuWVjZiUZGvKYjZn1meRulJmVQqiYRxEqx8HGrErcsjGzsniA2Mz6TvIAcVtp+8DrgHnAb4APpZ29WsttB9ant7+LiJN7qdeszurajer1Z30OuDUi5gO3pvftbI2Iw9LhQGPWgQBNU65j0PQabBYDV6bXVwJLevw+s6ktDRDnOQZNr2M2c9JmOwB/AOZ0KDdT0lpgFFgZETe2KyRpBdku72hYR8yYNXWGlA7+iwMn+xZKs23W1sm+hVI9+sDvno6IvfOWr+lzmN2DjaSfAvu0uXRe85uICEnR4WvmRsRGSQcCt0laHxG/bi2UNmteBTBz7xkxb0m+vVjr4OYjLp/sWyjNyOK2aYxq6x1vOPO3ectmm2fVM9p0DTYRcUKna5KekrRvRGxKKSA2d/iOjenfxyXdARwO7BRszKa8AjfPqppee35rgEa2vOXAD1sLpH1Md0uvZwPHAA/3WK9ZLQkxTfmOQdNrsFkJvFvSY8AJ6T2SFkq6LJU5BFgr6X7gdrIxGwcbs3YKzq4gaZGkR1OK3Z1miyVdLGldOn4l6bmma9ubrq3p9af1NAIbEVuA49ucXwuckV7/Aji0l3rMpooix2wkDQFfA95NlmjuHklrmv9jHxGfair/j2RDHA1bI+KwQm4Gp3Ixq5xpmpbryOFIYCQiHo+Il4FryZardLIMuKaAn9CWg41ZlShfFypn6yd3Gl1Jc4EDgNuaTs+UtFbSXZKW7OIv2mHqLGQxGwAChodytwFmp/VrDRPN9d1sKXB9RGxvOpdryUpeDjZmFZJtnpU72DzdJdf3RNLoLqUlh1TRS1bcjTKrlEK7UfcA8yUdIGkGWUDZaVZJ0puBPYBfNp0rfMmKWzZmVVJg3qiIGJX0CbI83UPA6oh4SNIFwNqIaASepcC1KUNmwyHANyWNkTVKel6y4mBjViFFP64QETeR5fRuPveFlvf/0uZzhS9ZcbAxqxKJoaGhyb6LvnCwMauQKf0gppmVy8HGzPpOIu/q4IHjYGNWKfkfshw0DjZmFTOI20fk4WBjViESDA97NsrM+qyxeVYdOdiYVYk8G2VmJXGwMbO+E576NrMyyFPfZlYCIYaHpk/2bfRFIe21HDu47ybpunT9bknziqjXrG4EDGko1zFoeg42TTu4nwgsAJZJWtBS7HTg2Yg4CLgYuKjXes1qSWLatKFcx6ApomWTZwf3xcCV6fX1wPFSTRcTmPVomoZyHYOmiDGbdju4H9WpTNo97HlgL+DpAuo3qw2hiexBPFAqNUAsaQWwAmB498GL3Ga9ksT0oRmTfRt9UUSwybODe6PMBknDwOuBLa1flNJQrAKYufeMaL1uVn+q7TqbIn5Vnh3c1wDL0+tTgNtaNlc2MxqpXIobIM4xU3yapD825fQ+o+nackmPpWN562cnqueWTc4d3C8Hvi1pBHiGLCCZ2U5U2LR2nlzfyXUR8YmWz+4JnA8sBAK4N3322V29n0LGbLrt4B4RLwEfLKIuszor+HGFHTPFAJIaM8V5UrK8F7glIp5Jn70FWEQPucDr2Tk0G1gTWmczO+XibhwrWr4sb67vD0h6QNL1khrjr7nzhOdVqdkos6lugrNR3dLv5vEj4JqI2CbpH8jWwx3X43e25ZaNWYU0ulF5jhy6zhRHxJaI2JbeXgYckfezE+VgY1YlxT6u0HWmWNK+TW9PBh5Jr28G3pNyfu8BvCed22XuRplVigp7FCHnTPE/SToZGCWbKT4tffYZSV8kC1gAFzQGi3eVg41ZhWQZMYvrcOSYKT4XOLfDZ1cDq4u6Fwcbs0opbp1N1TjYmFWIJIb9bJSZ9Zv3IDazckgMDeDGWHk42JhVSNaycbAxs76r7xYTDjZmFSLE8DQPEJtZv0nI3SgzK4PHbMys74SYhoONmZXALRsz6zsV+CBm1TjYmFWKGJJno8yszySvszGzktS1G1VICO0lN42ZNZNzfXfSS24aM3st4UV94+klN42ZtfA6m87a5Zc5qk25D0h6J/Ar4FMR8WRrgZT3ZgXAfnvO4Y4jvlfA7Q2GY++dOjn8PnLwoZN9C5UlFftslKRFwCVkexBfFhErW66fDZxBtgfxH4G/j4jfpmvbgfWp6O8i4uRe7qWsYe8fAfMi4q+BW8hy0+wkIlZFxMKIWLjX7rNKujWzKiluzKZpiONEYAGwTNKClmL/CyxMf5vXA//WdG1rRByWjp4CDRQTbHrJTWNmr5GN2eQ5ctgxxBERLwONIY4dIuL2iHgxvb2L7O+3L4oINr3kpjGzJiIbs8lzUFz63YbTgZ80vZ+ZvvcuSUt6/W09j9n0kpvGzFpNaFFfEel3s1qljwALgXc1nZ4bERslHQjcJml9RPx6V+soZFFfL7lpzOxVBQ8Q50qhK+kE4DzgXU3DHUTExvTv45LuAA4HdjnY1HNdtNkAE0O5jhzyDHEcDnwTODkiNjed30PSbun1bOAYelzO4scVzCqkyKe+cw5x/DuwO/A9SfDqFPchwDcljZE1Sla2Wag7IQ42ZpVS7BYTOYY4TujwuV8AhS6IcrAxqxjVdHTDwcascjTZN9AXDjZmFeI9iM2sRO5GmVkJ5G6UmfWfkLcFNbNyuGVjZn3nAWIzK427UWbWZ8IDxGZWCnkFsZmVxS0bMyuBWzZmVgLl3atm4DjYmFVINkDslo2ZlcCzUWbWfxL4cQUzK0NdWzaFhFBJqyVtlvRgh+uS9GVJI5IekPS2Iuo1q59snU2eI9e3SYskPZr+9j7X5vpukq5L1++WNK/p2rnp/KOS3tvrLyuqvXYFsGic6ycC89OxAvhGQfWa1U5R2RVypt89HXg2Ig4CLgYuSp9dQJaN4S1kf9tfV840nJ0UEmwi4k6y5HOdLAauisxdwKyWLJlmxquPK+T5Xw5d0++m91em19cDxytLs7AYuDYitkXEE8BI+r5dVtZIVK40oJJWNFKJbnnhuZJuzaxKNIGjkPS7O8pExCjwPLBXzs9OSKUGiCNiFbAK4K1z3xyTfDtm5Yt05FNY+t0ylNWyyZUG1MwCRb4jhzx/dzvKSBoGXg9syfnZCSkr2KwBTk2zUkcDz0fEppLqNhssYzmP7rqm303vl6fXpwC3RUSk80vTbNUBZJM7/9PDryqmGyXpGuBYsj7kBuB8YDpARFxKlpHvJLJBpheBjxVRr1kt5Wu15PiaXOl3Lwe+LWmEbJJnafrsQ5K+S5bfexQ4KyK293I/hQSbiFjW5XoAZxVRl1mtBajA0coc6XdfAj7Y4bMXAhcWdS+VGiA2MyYyQDxQHGzMqqagblTVONiYVU09Y42DjVmlBHmntQeOg41Z1dQz1jjYmFWOg42ZlcLdKDMrQ5HrbKrEwcasSib2IOZAcbAxq5SAsXpGGwcbswoR9e1G1XMbdzOrHLdszKrGs1Fm1nceIDazssgDxGZWinrGGgcbs0oJPPVtZmUIoqYDxJ76Nqua4jY870jSnpJukfRY+nePNmUOk/RLSQ+ltNkfbrp2haQnJK1Lx2Hd6nSwMauQCIixyHX06HPArRExH7g1vW/1InBqRDRS8P6HpFlN1z8bEYelY123Ct2NMquY2N5jsyWfxWQZUSBLv3sHcM5r7iPiV02vfy9pM7A38NyuVFhIy0bSakmbJT3Y4fqxkp5vanJ9oV05symvMUCc5+iefnc8c5pyt/0BmDNeYUlHAjOAXzedvjB1ry6WtFu3Cotq2VwBfBW4apwyP4uI9xVUn1lNTWiAeNz0u5J+CuzT5tJ5r6kxIqTOT2RJ2hf4NrA8IhrNrnPJgtQMspTZ5wAXjHezReWNulPSvCK+q2HbrK2MLG7bUKqljxx86GTfQmm+85/rJ/sWqq2gXlREnNDpmqSnJO0bEZtSMNncodzrgB8D50XEXU3f3WgVbZP0LeAz3e6nzAHit0u6X9JPJL2lXQFJKxpNwue2vFDirZlVR0TkOnrUnHZ3OfDD1gIpZe8PgKsi4vqWa/umfwUsAbq2DMoKNvcBcyPircBXgBvbFYqIVRGxMCIWztpr95JuzaxCJjZm04uVwLslPQackN4jaaGky1KZDwHvBE5rM8V9taT1wHpgNvCv3SosZTYqIv7U9PomSV+XNDsini6jfrNBUsZsVERsAY5vc34tcEZ6/R3gOx0+f9xE6ywl2EjaB3gqDUQdSdai2lJG3WaDJKKQNTSVVEiwkXQN2Zz9bEkbgPOB6QARcSlwCvBxSaPAVmBp1HVNtlmvSllmU76iZqOWdbn+VbKpcTProq7/HfYKYrMq8VPfZlaWkh5XKJ2DjVmVBB6zMbMyeDbKzMriAWIz67u0n00dOdiYVY2DjZn1W0R4NsrMyuFgY2b95zEbMyuHu1FmVoYAxhxszKzPIoKxV7ZP9m30hYONWcW4G2Vm/VfjbpQzYppVSr5smL3OWOVJv5vKbW/af3hN0/kDJN0taUTSdWlz9HE52JhVSWTdqDxHj/Kk3wXY2pRi9+Sm8xcBF0fEQcCzwOndKnSwMauQAGJsLNfRo8VkaXdJ/y7J+8GUvuU4oJHeJdfnPWZjViURRP7ZqNmS1ja9XxURq3J+Nm/63ZmpjlFgZUTcCOwFPBcRo6nMBmC/bhU62JhVSUxoNqqM9LtzI2KjpAOB21KuqOfz3mCznoONpP3JcnzPIWsFroqIS1rKCLgEOAl4ETgtIu7rtW6z+okiukjZNxWQfjciNqZ/H5d0B3A48H1glqTh1Lp5I7Cx2/0UMWYzCnw6IhYARwNnSVrQUuZEYH46VgDfKKBes/oJYHvkO3qTJ/3uHpJ2S69nA8cAD6c0TLeTpWjq+PlWPQebiNjUaKVExJ+BR9i5/7aYLF9wpOTksxq5gs3stUoaIM6TfvcQYK2k+8mCy8qIeDhdOwc4W9II2RjO5d0qLHTMRtI8smbW3S2X9gOebHrfGFDahJntUNZ+NjnT7/4COLTD5x8HjpxInYUFG0m7k/XlPtmc23uC37GCrJvFnP32LOrWzAZHMJHZqIFSyDobSdPJAs3VEXFDmyIbgf2b3rcdUIqIVRGxMCIWztpr9yJuzWzARFndqNL1HGzSTNPlwCMR8aUOxdYApypzNPB80xy/mTWUt4K4dEV0o44BPgqsl7Qunfs88CaAiLgUuIls2nuEbOr7YwXUa1ZDxU19V03PwSYifg6oS5kAzuq1LrPaa0x915BXEJtVSEQw9vJo94IDyMHGrEom9rjCQHGwMauSgBh1sDGzvnN2BTMrQbhlY2aliHCwMbMSBIxtq+fjCg42ZlVS0oOYk8HBxqxCPGZjZuXwmI2ZlcXdKDPrP3ejzKwMEcHYtno+G+UkdWZVksZs8hy9yJN+V9LfNKXeXSfpJUlL0rUrJD3RdO2wbnU62JhVSYXS70bE7Y3Uu2QZMF8E/qupyGebUvOu61ahg41ZlaQxm363bJh4+t1TgJ9ExIu7WqGDjVmllNONIn/63YalwDUt5y6U9ICkixv5pcbjAWKzComxCT2uMG6u74LS75JyvB0K3Nx0+lyyIDUDWEWWR+qC8W7WwcasUib0uMK4ub6LSL+bfAj4QUS80vTdjVbRNknfAj7T7WbdjTKrkvLGbLqm322yjJYuVCOjbcqusgR4sFuFbtmYVUl5jyusBL4r6XTgt2StFyQtBM6MiDPS+3lkOd/+u+XzV0vamyzZwTrgzG4V9hxsJO0PXEU2wBRk/cZLWsocSxY5n0inboiIcft3ZlNRlLQHcZ70u+n9b8hSZbeWO26idRbRshkFPh0R90n6K+BeSbc0JSBv+FlEvK+A+sxqzc9GdZAGijal13+W9AhZJGwNNmbWRUQwOubNs7pK/bvDgbvbXH67pPuB3wOfiYiH2nx+BbAivX3hHW8489Ei7y+n2cDTk1DvZJlKv3eyfuvciRTeHm7ZjEvS7sD3gU9GxJ9aLt8HzI2IFySdBNwIzG/9jrRGYFXr+TJJWjvedGLdTKXfOwi/NQjGahpsCpn6ljSdLNBcHRE3tF6PiD9FxAvp9U3AdEmzi6jbrG7GInIdg6aI2SgBlwOPRMSXOpTZB3gqrVQ8kizIbem1brM6qmvLpohu1DHAR4H1ktalc58H3gQQEZeSPcT1cUmjwFZgaURlQ/OkduMmwVT6vZX/rRH17Uapun/zZlPPQcNz4kuv+3Cusouf/cq9VR+DauYVxGaVEp6NMrP+CxjIwd88/CBmE0mLJD0qaUTSTjuX1Ymk1ZI2S+r6AN2gk7S/pNslPSzpIUn/PNn31FFkA8R5jkHjYJNIGgK+BpwILACWSVowuXfVV1cAiyb7JkrSeKRmAXA0cFZ1/7+N2gYbd6NedSQwEhGPA0i6lmzrxFo+dhERd6YV37U3SI/UBPhxhSlgP+DJpvcbgKMm6V6sT7o8UjPpwgPEZoOvyyM11RBe1DcVbCTbJKjhjemc1UC3R2qqos6zUQ42r7oHmC/pALIgsxT4u8m9JStCnkdqqqO+K4g9G5VExCjwCbId5B8BvttuG4y6kHQN8EvgYEkb0vaQddV4pOa4pgyOJ032TbWTtWw8G1V76Yn0myb7PsoQEcsm+x7KEhE/J9srt/IigldqOhvllo1ZxZTRspH0wbTAcSxtct6pXNuFrpIOkHR3On+dpBnd6nSwMauYkvazeRB4P3BnpwJdFrpeBFwcEQcBzwJdu+EONmYVEiWtII6IRyKi27a7Oxa6RsTLwLXA4jTgfhxwfSqXJ1e4x2zMqmQDz918dtyQdxfLmeOl3y1Ap4WuewHPpUmVxvmd0r20crAxq5CIKOx5tfFyfUfEeBkw+8LBxqymxsv1nVOnha5bgFmShlPrJtcCWI/ZmFknOxa6ptmmpcCatKXv7WTb/UL3XOGAg43ZlCTpbyVtAN4O/FjSzen8GyTdBF0Xup4DnC1phGwM5/KudXoPYjMrg1s2ZlYKBxszK4WDjZmVwsHGzErhYGNmpXCwMbNSONiYWSn+H4jRLO0e+4NOAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "corr_matrix_marginal = np.corrcoef(posterior_samples.T)\n", + "fig, ax = plt.subplots(1,1, figsize=(4, 4))\n", + "im = plt.imshow(corr_matrix_marginal, clim=[-1, 1], cmap='PiYG')\n", + "_ = fig.colorbar(im)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It might be tempting to conclude that the experimental data barely constrains our parameters and that almost all parameter combinations can reproduce the experimental data. As we will show below, this is not the case." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because our toy posterior has only three parameters, we can plot posterior samples in a 3D plot:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "rc('animation', html='html5')\n", + "\n", + "# First set up the figure, the axis, and the plot element we want to animate\n", + "fig = plt.figure(figsize=(6,6))\n", + "ax = fig.add_subplot(111, projection='3d')\n", + "\n", + "ax.set_xlim((-2, 2))\n", + "ax.set_ylim((-2, 2))\n", + "\n", + "def init():\n", + " line, = ax.plot([], [], lw=2)\n", + " line.set_data([], [])\n", + " return (line,)\n", + "\n", + "def animate(angle):\n", + " num_samples_vis = 1000\n", + " line = ax.scatter(posterior_samples[:num_samples_vis, 0], posterior_samples[:num_samples_vis, 1], posterior_samples[:num_samples_vis, 2], zdir='z', s=15, c='#2171b5', depthshade=False)\n", + " ax.view_init(20, angle)\n", + " return (line,)\n", + "\n", + "anim = animation.FuncAnimation(fig, animate, init_func=init,\n", + " frames=range(0,360,5), interval=150, blit=True)\n", + "\n", + "plt.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "HTML(anim.to_html5_video())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clearly, the range of admissible parameters is constrained to a narrow region in parameter space, which had not been evident from the marginals." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the posterior has more than three dimensions, inspecting all dimensions at once will not be possible anymore. One way to still reveal structures in high-dimensional posteriors is to inspect 2D-slices through the posterior. In `sbi`, this can be done with the `conditional_pairplot()` function, which computes the conditional distributions within the posterior. We can slice (i.e. condition) the posterior at any location, given by the `condition`. In the plot below, for all upper diagonal plots, we keep all but two parameters constant at values sampled from the posterior, and inspect what combinations of the remaining two parameters can reproduce experimental data. For the plots on the diagonal (the 1D conditionals), we keep all but one parameter constant." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABtpElEQVR4nO39eZBl2X3fiX3OOXe/b8utMmvr7qruQi+AQIBLN0Q0uAIjkiJNOkzNaAkqFNbMWLYUI4Ul2wprhqYUZsyE7dDIMZ6xR5YYlMLykNJIsjUayRSapEg0IXSDBAkC6G50F6qX2nJ9+da733v8x3lZVWj0Uktmvsys84nI6Fxe3ncyq/P7zvnd7+/7E1prLBaL5bgg570Ai8ViuResaFkslmOFFS2LxXKssKJlsViOFVa0LBbLscKKlsViOVY49/E91iOxP4gHvcAfa/85rasKnef7sZ6Hms83/+S+/z0+J//E/P4mhAAhUQtdRBjSLHep2j5l2yFbUFShIO8JqgiqSFP1akRQE3dTumHGcjjl8dYWp7wxTwfXOesMWFUFChhrwSvFKm8Vy9wserwxXmE3j9gat0inHk3i4AwcVCZwx+CNNU4CYb/CSWrc3Qw5mKAnU+r+AJr6Q3+cu/l3uB/RshwR5EIPnabUVrQeWoRSCM9DtFvoOKTq+pQth6KlKFuCKhRUsRGsOm6QcYkXVCxEKcvhhLVwzPmgz5oz5DG3z5qqWZAh3ygq1usObxXLXM0W2cg7bCZtxplvBGvqIhOJMxU4mREsd6JxE407qlBJYQSrP6CeTO9KsO6WYy9a46ykaaAbufNeyqHT9NpIz0WVFU2SoKtq3kuyHCLCcZCtGNFq0fRa1JFL0XEpWpIyFhRtQe1D2dLUcY2IaqJWTivIORWNORMOOesPuOhtsuYMedTRJI3mzSrjrWqV9bLLW9kyN9IuO1nMIAnJMpdm4qImCicFdwIqA3ei8UcNzrTGGabISYYeDGnSbF8FC05ATesv/r9/n5/7pZfmvYy50EQuTRQg4gjhOOaoYHk4EMLssIIAHQXUkUsVOVShpAoEdWAEqw41TdCA3+D4FbFf0PFylvwpS+6UZWfMKTVmSeZEwiPTsFWHrJddtqs2/SJimIeMc58sc6lTB5lJVA4qE6gMnFTjphqV1qi0QiQ5pBnNNEVX5b7/6Md6p7U5yvjCG1toDVf7CecXo3kv6VApFgOc0MHBvPqIaUI9Gs17WZaDRiqk5yKXFtGdmKoTUCx4lJEk7wrK2NSwyo6mDhvolARRSTvKONMasuQnPBFtct7tc97d4Uk3RwrF5TLn7WqJ6+UCb6SrbBUtrk4W6E8j0tSjHnnIVOKOJO4UnAT8YYObavzdEmeUI8cZbPdpshxdFgfz4x/IVQ+Jf/W1m+y1Tv76qxvzXcwcqCJJGTk0LQ8dh4goRLie3XGdcGTgI+II3QppIo86cigjSRVIU8MKoQqNYGm/wQ0qoiCnG2Qs+QmL3pQVZ8yaM2RFpdRoxk3NRt1is2qzUXbplzGDImKU+aSZS5nt7bDM7kqloFKNk2lU2qDSCjnJEUmGzvIDLVUc653Wv/zDmzy52qZqGl54dZM/9+kL817SoVK0JI4jkLWHKCOklMgkpcmFvaN4UhEC2WlDFFJ1Q6rYpeg4FC1JHUAZQxVrqkij4xoV1HTijMUwYSWccC7YZdUd8pi7zaPOiFXlcbMu2Gl83iqXeTtfZqPocDPpMMwDxtOAcuohUoUzEeZO4YRbRXdvWOGkNWqYIoZjmvGEJs/hAIMYjq1o3Rik/O7bu/zVz32ESVHxSy++ySgr6QQPT0G+aAtqF0SjEFWAoyRO3kVOE+qy2vcCqGW+yCBAtGJ0t03T8im6HlWsTOG9LYxotbWpY8U1Xqsg8EuWoimnoxGn/DEX/E1OOWMuuiMk0G8K3qq6bFUd3sxXuJH12MjabE1jksynnHiIROHs3SlMZ0X3cYMzbXBHBWpaIHZHRrDS7EAFC47x8fD/9/V1AP74x0/zuadXKWvNb31za86rOlzqAOpAmMJrKKkj1xwTgwDpufaYeJIQAjE7FjYtnzp0qWdF9yqY/b/gQ+NrmrBBBDWBX9IKchb9hEV3yqo74pQzZkWN6UoFwLBR7NQttqoOO0WL3SJklAekuUeRO4hcoVKJyoy1QWXmSOikDU5aI5PSHAmTFF2Uh/JCeWx3Wi+8usGlUy0urrR4dEmzGHv8+qsb/NR3nZn30g6NogPKg73XHteXiCrCcRVSCNjapsmyua7Rsg9IhVpaRLRj6m5MsRBQxYqsq0zRPYaio6kDTd2pUXFJFOWstccsBAkX423OeX3Oezs86e4QS8G40VyrQq5XC7yanmWnjHlrush2EjNOfdJhALkpujsTs8PyBuYuoT+ozQ5rkiO3d9FJeqg3gI7lTmuYlrz8Zp/PPrMKgJKCH3nqFL/x2iZl3cx5dYdHHZijQBUye8UV1JFDHXvoVohoxcggmPcyLQ+AcD1kHCFaEU0roI5d6lDNiu57u+3ZLivQyLDC90vaQc5CkLDoJSy7Y9bcIStqjCug0Jrt2mWzbrNeddkpY7bzFoMsZJp75JkHuby1w1K5sTU4mcbJGpy0MkfCJEcnqalhHSLHUrR+6/Utqkbz2adXb33us0+vMsoqfvet3Tmu7HCpW41pzYigbAmKlqBoK8qOR90OEJ02otuxx8RjjIxDZKdN04moOgFlx6VoS4qWoJrtsqpYU7dqdFwRRgW9OGUpTFgLRpzzd3nM2+a80+cxx7TnJFrwTrXA28UK7+RL3Ei7bKRthmlAMvWppw7ORJka1nRWdJ+CN25wRzXOMEeOEhiMqCfTQ7/pcyyPhy+8ssFyy+MT53u3PveZS8t4juSFVzf4o48vzW9xh0mrpFYOaIFoBFoJVCHRSiAaH9FopOug8hyd5faoeIwQrmdqWAs9dBRQLoSm6N5WFG3TnlO27mjPaZn2nMU4udWe81iwzZoz5KK7TVtUgOBG7bFVt7mcr3E973Ez67I+7TDOfJKJTzN1UdPZkfBd7TnesLzdnjMc0+xze87dcux2WmXd8Jvf3ORHnjqFkrd3ELHv8OnHl3jh1Q0eltx7x68QYUUTNOao6EMVzI6KoaQOXZrIR4QhwrP+rWODEAjXQUQhOvRpIo8qVLfc7sbxbtzudaDRQY3rV4R+QcfPWPBSFl3jxVpxRizK6taxcKeO2azabJetmRfLHAmz3KWZud1lLmaOd/O2V3hXaYlMCvQ0RafpgZlHP4xjt9P68pt9xlnFj95xNNzjs8+s8jf++dd5Y3PCR1bbc1jd4bLYSZjmHlOg1C6NIxGNoHFBC3N3yPUkouwhfQ+pNc00sVaIo4wQqHbbNEB3W5SLEVXskPcUZSQoW4KyY8yjZaeGsMaPC5Y7Uzp+xsXWNmveiEf9bT7q3aArS9pScaPS3KjbfDM/w0bZ4UqyzFbaYjcJGY1DmtRBjRROInASgTfUOCkEgxp3XOOOC9TOGD1JqLe3D9zW8EEcu53W51/dwHckn7m0/B1f+9GnjJC98JC447t+RivIcYMKHTSzV16oQ2auaEkVKeOYjwJkHBkrxOx2t+WIIRXSn7nd45Am9qlihyoyETN1KGb/tmaHRbDndjc7rEU/YcmdsuoOWVEjurIkENCva7aaiPWqx0bZYatosZtFjLKAJPNoUgeRzYruqUClGD9WOrM2JCVymqMnCTpJ5ipYcMx2WlprXnh1g+efWCbyvnPpa92AP3K2ywuvbPC/+qEn5rDCw2UlnOCqmqJSDGtJLTVV7gICUQuKErSQqNRFC4FTN4i8QGiNzu1u66ghAx/h++hOi6YdULY9yraiCm73E1aRpoq1ac+JCuKwYCFKOR2OWPSmPOpvc97d4bwzYlEpSt3wRhVztVziWrHIzazLTh7TTyPGiU+RGPOoSgXOVMx6CjXetMFJZubRYYoYJ9SD4dyOhHdyrETr9Y0JV/vpBwrSZ59e5e/8+utsjXNW2v4hru7weSzaYctp0WhB00im0qMsJVpJ0HJWnAdVOmhHIrRG1TXK96g2t+0x8aggFcJ1kN0OOg6pFyLTntN1yNvydtF9rz2nVaHCim7rdnvOI2GfVXdoYmbUlFUl2agbBo3HG/ka14pFbuRdrk17jAqf4eR2e447fo/2nEGFk1So3cS054zGB5LYcD8cq+Ph3rHvR5869b6P+ewzp9AafvO1zcNa1txYdscsuVN6XkrsFwRBCX5DE5jjQ3WHY94U5h105EMYIFzHHhOPCEKZYyFhgI586tAxhfe9gvu3ebEaVFDj+yUtP6frpyx6U5bdsSm8y4S20CgEg8bcKezXMTtlzG4RMSp8prlHlTuQS2Qmbnuxsju9WDXqDrd7kx1sP+G9cKx2Wp9/ZYPvOtflVOf9DZPPnO5wphvwb17Z4N//vvOHuLrD5yn/JotqghSaBsGOG1NVitx1qfAAiXYEopY0jgbhgo5wPAdV1ejxmHownPeP8VAjXA/ZaSG6HeqlNlXski26lJGkbEHRnfUUdhqaqEa1KjrthE6Qc761y+lgxDlvl6f8G6ypCR9xAzbrhG+Wktfy02xXHV6frrGRtdlJI/qjmDJzYOTiTk0/oTcy5lF/qE3RfVLh9KeISUqztU1zSO05d8ux2WltjjP+4Org2wyl74UQgs8+s8qLl7fIyqPziz4I1tSINWfIqjti2Zuy4CfEQYEXVOiwNrutPcd8CGUoqCJFHbvGMR/PHPPWCnH4CHE7eTQybvcqdm8V3c2/mRGsKpj1E4Y1flDQCXJ6fsqKN2HVHbHqDlhTE7qyZrdJ2Woc1usOG1WXjbLDTh4xyELGmU+ZOejUQSXSFN1nIX5Oqk0NKzFRyWIyszVUFeij1WVybHZav/GqOe7tte58EJ99epV/+O/e5ncub7+nNeKkcM6p8MWQRPuM6wApGkZRgBCauhaUpQChqDKBKc6DKhUIkGmAAkRVIfqDI1FgfagQEuE4iG7b1LHaAWXLMRlpsaCKbhfe67BBRBVBVNCJMpbDCUv+lLP+gHPeDo84fc454OJxuWq4WvW4WixxPV9gO2+xnbYYpAFJ4qMTB5nKmbUB43ifGtFyJxVqVCAnKXo4oskPNhfrfjk2ovXCq5uc7YU8tfbh/qvnLi7S8h1eeHXzRIsWQFs2XHS3KbQiUjnTysdTNVoLdhtJrTRl6aLVTLgaaByFLH20I03qaV2bukWSzPvHeWhQ3Q4iCml6LaqWR9l1ybvGPFq0jbWhjDVVpwa/ptXO6IQZK+GUR6M+p7wxT/jrPOLscs6pGDeacaO5Up7iarHEtWKBa0mP3TwyyaOJRzN1cEYKlXOr6G7SR2tU2uDupsjhFD2emJkD9dE8qRyL42Fa1Lx4eYvPPbOKuIujjO8ofvAjK/z6qxs0zdEoHh4E2awwuigrltSEFWfMojel5yW0/BzfL5FBbRzzvnHMf1thPnBoIs845kN7TDwUhLjVorOX7V5HjmmAvjPbPdA0gQa/xvFrYr+g7eUszLxYC86UU2pMW5YEQjFuJP0mYLPqsF216Bcxw8Jku+eZS5MpRC6Rxe1sd1N8b24lj4okR6eZeavrI1N4fzfHYqf1O5e3ycrmQ+tZd/LZZ07xP37tJl+7PuS77uhRPEl8s+yyoqY84UguOkN6MmXa+LTU7QbWgRMyrAWV44BQCC1oXIGsFI0r0DLArzUi8JBFaaJy7VHxwJBhiFzo0Sx1aCLPZLvHirwtTNHdn2W7Rw06rok6GZFfcqY1ZCWYcD7Y5cngJmvOgCfdnEzDRl1xpVpmvezxRrrKzazLVtpiY9wiSz2qkWfGfSUCb2h6Cv1hgzdpcCY1Xj81wyi2d80O64in3h4L0Xrh1Q3avsOzFxbv+nt++EnTm/j5VzZOrGitVz1qLenJXQIBK6pgzRnQaMHIC5j4PloLksyjaAR1KagKCVpQRoCWyFLhpB4KkHGEFoK6Ko/sq+xxRkYRIo5NDSv2b2W7l3tu9+CObPegwQ1L4qC4le2+5o845Y5YcwYsyowSzVgLtuqQ6+Ui22WbjbxNP4/YzULSxKfKFTKTOKlxuzvpLMhvVnh3kgoxzRBJRpOm6PLo1bDezZEXrabRvPDqJj/wkRU85+5Ps73I43sfXeCFVzf4a3/syQNc4fy4ViySOS6xzHnGzViWPn1nFyUaksZnUhtz7TjymABFbYQLASoXgEbWEpW7aAFup4UARJoZI6EVrv1DKkS7Be2YphOa9pxYUcaScq/oHmtqX6OjGhVVhGHBYpjQ81POBANOuwMecXc47yREQpBoTb82QX43ix5bRZvtrEU/jZikPlXqIDKFSgQqEbiJcbu7qcad1DiTEjXKEePpzIt1PFJAjrxoffXagO1Jzufu4q7hu/ncM6v8H//HV0/seLEr6TJDN0SJBldcZ0WmPOpIIrlDrSWlVoSqJKtdXNWwCxSNmBXixaw4D6JWNI5AVjHKUSig3tm1x8R9QsaxqRkudmlaAUXXo+g5lKGk6MzGfcWasmUSG7x2QRzmLMUJ5+MBi96Ux/0Nzrq7POYMkUCiNVfKDu+Ui1wrlngrXWInj9mYtJgkAUXiIsd7I+tn7TnT2UDVpMHbzVHjDDGa0gyGxot1TDjyhfgXXt1AScEPPblyz9+7d+fwpI4X2y0iBlXEVtU2zufGwxWSSGhOqTHLzphld0LPS2n7OYFfIoIa7d+OsrldmBfUoSnME4WIwLeO+f1AKpOLFd4e91UH73K7+9D4oP07st39gq6X0nNN8ugpZ8ySTGhLQQOMG8Vm3Wa76hi3ex7dynYvcwdydcvt/mHZ7kfNPPphHPmd1guvbPJ9jy3Qi7x7/t4LyzGPr8QndrzY9UmXSelTa4FCM3ZDInGdtoQn3ZyaG7RVSta4hKpEotFA6vhUtY9WEi3FLM5GImuXxpV4QqCaBhEE1FsP17CQ/WTP7c5ijyYOTLZ7y0zPybsmLrnomMjsOm5wOgVBWLDWHrMUTDkbDrgUbsyK7jsEAiSCq5VJbHg9O82NvMd62ubGpMM080hHgZmeM5V4I3OX0Btq3KnGm9R4uwVqmiN3BjSTKc14PO9f0z1zpEXraj/hmxtj/tM//vR9X+Ozz6zy979wMseLjTOfRgsip+Cm00WKhjVnSMOUQDX0ZA7OLhtuj1IrprXHpPTQWjDJHGoNopZUufFwlalANBJZuog0QgqBnEboojiSJsOjjMl2Nzn9TRxQx2bcVxnermHtWRv2zKNhlBP7BQtBwrI/4ZQ34qzbZ0WNCQSUQL+WrFc91qsu63mX7Tymn8VMs1m2eyaRe9nud7jd3aTBSWrUOEcmGc1kis6O9l3C9+NIi9Zeg/T91LP2+NzTq/y3v3WF3/rm1omb1JMmPk0j2XEqIsfUn1Yc88q5KEcsyppIJKy7Jjc/bxwmpY8A8syl1FBXgqowxfkyE4BEVA4y80GCmMQghBWte0EII1hxjG5FVG2Ti1XGctZKZdp09nZYIqzxw9K05wQpa8GI096Q826fs2rIoiqJpGKjbrhedbg6u1O4mbfYyWKGaUCWmFwsNVW33O7G6Q7epDH9hOMCOUnQ0/RY7rD2OPKidelUi0eX4vu+xicfWWAx9njhBI4Xq0YedSnZAqTQZLVLS+VMPR8pGh5zJrSl4JK3SSzNq2rVKDadFnnlMFYNuYCycUxjdWWibEAiag/Xk7h51wQHak2TpvaO4odwZ7Z7E4dUCyFFzzRA552ZYMVmqGoTNsh2SRCaYRRnWkOW/SmPB1uccXd5zN1mVZW4QvBmKbleL/BWscKVdIXtvMWNSZdhGpgXr7GLSiXu+LZgeSONmzZ4gxJnbIZRNDu7x3aHtceRLcQP05KXrvTvqtfwg9gbL/abJ3C8mCgEOlMUucM49xnmoRkHVbbZqjpMG0mpNT1ZsagmrDgjem7CwswxH/olyn9vx3wdSqrQoYl8dOgjwgDhnKzj9b6zN1A1CNBRQBPNxn2F7+V2b9B+g+dXBF5p3O5eMst2H7GkJizKAiUEpdZszgaqbpdt+kXMqAzMuK/coc6MF0sWexEzt7Pd1SzbXSQ5JOmJMA8f2Z3We40Ju18++/Qq//3vXePLb/X5/se/M6b5uOKOJLUvKYEBkJcOvlMxrYw/S4mGM84uH3M1gSjwWKfQirZaIK1dfFUhhGaoI2rHoayNDUILU9tyXYEsAxxXoqRE1jXNlGP/P/2BIBUy8JG9LroV3sp2LzqKvGvuzhYdcySsWg10ZtNz2lMWgpSz0YAnok1WnSGXvA1WVcGqCnm9LNiqY76Zn+Fm0eVa1uPapMc490y2e+IgpwpvJJH5XtEdvGmDv1uiktJku4/G1MPRiTjmH1nReq8xYffLrfFir2yeKNFyEoFoNNpVlJ4DQtNPIySa2MnpOqYBek2t4wKrquTsrL617bdotCSvHbLCJQfqTCK0RNRiZj4Fp6UQjYsoA1SrhRSSemCNp9/GTLBEu4Vu3c52v2UcDe+oYYUaHdZ4s2z3hSBlJZiYmBlnyJoz5IwyLwobdcp63b2V7b6Rd9jOWowy/3a2e6pu9xLeynY3Xixneke2e5Yf2Qboe+VIitbemLAf++jat40Ju1/uHC/2n/3k03fVdH0ccFJAz6bvuA6lFowCHyk0gVPScYzDeU0NWVUpF9wWg8YUYLe9Do0WZLVDWrqMhSYJHSptImyqYnZHMZOIxjEzFLMIIQRiPD7SDbWHjXAdYx5tx7dysYxg3XGncDbyqwmN270VZbT9gpVgwil/zDmvzyNunxU15ZSK2GlSrlYe18sFNqqu6SfMW+xmIdPU5GKJROHMst2dhNt3CacNzqRETjLExIys12V1Yv69jqRo7Y0Je5C7hu/ms8+s8pv//Otc3pxw6YSMF/OGGjkTF4SkqgRTN6CqjCnUEQ1pbdp8pu4OMKQnoS2HTJt1fFkihaZqFJ6qqSpF4biUwpmlnQpkJWZ+LhdRxSjPQZUlzXhCM53O9ec/KqiVZXToUy/EFAs+VaRmPixB2Z55sQJN06lwgop2K2W1NWHRT3g82uKUO+Jxb4Mn3BGxkLxVJazXEW+VK7yenaZfxlyb9tjNQkbTgGLkI/K9oruZAu0NjXnU361wJyVqlMH2gCbL0EVxYgQLjmghfm9M2PPvMSbsftkbL/b5E+SOV7lGZXo2WNO4n5vMochdJrnHoAjpl7FxzFcd+o0x6AZCcEqZQZ4LTsKCn9DxMwK/xPFnjnmfWWrmzDEfSKpQUe855sMA4RzJ17xDR0cm270JHergXQNVfWh8Pct2r/AD43bveSbbfcGZcsoZsaKmBLMTwE7js1V32KraDMqI/izbPZm53cWHZLvLpDBu9ywzdwpPkGDBEdxp7Y0J+/T7jAm7X9a6AR8/d7LGiwWDmiqUmNeemWXBUdS1YESEEJq0Mm74pPapkSjvBsuq5KKbEckNFJpSK2JVkNcOfVUzFiGlnrnkK5MzjwBZOzSeRFZtpKOQQtLs7p6I4u6DUC/G1KFjpud0lIm27sxmT7Ya6naNDCs6rZRumHE6GvFYtMMpb8RT/g3OqDGPOg7bTUW/dngtP83NcoGr2SJvTRcZZCG744gi8WDq4IxNVLI7NtYGd6rxhzXOtMLZTUw/4XB0YgfzHjnR2hsT9r/8wf0Xlh996mSNF3OmNTTguoLGA4S5ra6FpFYOU9f8jJt+C4C2ylhSE2rGnHdKYlFx1tllx2uhRMN2ENNoQVUrqkLR4Bi3fGMEUeUSLcBJXJwmQFU1Ms9p0uyhvqNYxS51sFfDgnqv8B6YXCwZmR1WN8xYCqYs+xNOe0NW3QFn1JhYNkx0yXrts1W32ai6bBbGPLo7y3YvUhedzuYTJqbw7iZ69mZqWGpaIsYJemoK7ydRsOAIitatMWFPv/+YsPvls8+c4r984XV+47UN/oPve2Tfr3/YOJMCUbs0nqD2FKCp/Zm73ZEUnvFV7QTGnBurnK4ydxTPqB0iAWeclJ2mj6Jh02/TaEFZK/LCoWB2R7EB0QjK3Ax/dWOF0B6iapBphATqh1i0ypYy1pPImEf3iu71bBhFEBbEQcFSMOVUMGHNG3HG3WXNGbCmoEQwaDCtOVWP9bzLRt4xTdBpQJZ66NRku985AdqZCZaJmSkQ0ww9Hp/4F5EjJ1p7Y8JWP2BM2P2yN17shVc3T4Roqf4EEQV7p0PK0hTPRW3acUrpkleSLadFUZvivBSacRMQyZwVmXDGEVx0+rRlRtL4prFaaMpGMnF9JpWkVAot5ayxGpN66gi0FHhVg/A9ZFU9tD2KeUdRB1C0TeG9Ck22uwgrwlbOUithMUh4NOqz5g+56G3xlLdBV9ZMNfQbh6tVj8v5GhtlhzenS/SzmH4SMh0H6FThDPfuFII3Mu05waDGmdY4wxy5O0ZPE+rR5MhNz9lvjlQh/m7HhN0ve+PFvvDGCRkvlqTIJEMlFSptbhVkVWYK8zITkEvyzGWS+QyLkH5x2zE/1i6lbohlw4pMWHFGLDpTFr2EjpcTeiVqL8rmlmOe24X5QNFEnnHM+/5DW5jf+53ccrv7e9nuFZFf0vEzOl7Kkjs1A1WdEZGo8YQw2e51NHO735nt7pFlLnqW7a5ykPme0/2ObPekRCY5OjHZ7jQn34pypP4v25sK/aCtOx/ESRovVm1uo1oxCvAEyNKjdl1EBWBqW7ISlMpl0pg7U2pmg4hkwbTxaPQGT3sFq8oh0RsEssQVNWWjCJ2SqpZMZEApPOOYd5jZIEBLhax9HE/hVDVi4CCq6qHbbeVdU1MsO5qq3aADk+0eBwWn4gmPxX1WvPGtbPeLM9PvtNFcqZa4Xi7yTr7ElWSZQR6yNYlNVPLURY0UKhe3ewons6J72uDuZshJCoPxQ3VD5EiJ1udfufsxYffL7fFiG8detNCN8eCMp6jA2Bm8UAIK7UDjGQ9XHSgaAanyGHghAOteByUaYlnQlessqYJVpanZJWtc+r6pgw2LgLqRNI2kKSSVMLfamZlQnVyhBcgkRNUNsqmpd4cntgj8XtQh1L4ZjKvD2nixwpyun7ESTFjzh5xyR5x1dunKHCUEO7Vg2PhcLxe5WfS4mXXZyWLGuW8EK3UQezWsbGYevaOGpdIKOUkR4ynNdHpi3O53w5ERrb0xYX/y+x45UMf67fFimzSNRu6D435uaE1TlMjxGBn4KMCJXbQwdoXaMz9bnQq0UJSOy9gzdxS3/RZSaCJZsKQmwISLrkdDTuIM2XI7AOz4EXUjqRvJNFc0KKpIz0RLzOJsFE7LQzQNsq4Rkyk6f3j+iKrQHAl1YAQrjEzEzKKfcMofc9odsOYOWFXprSC/fhOwXvW4WfTYKDrs5DHDLGCaeTPBUqhkNpAi2xMsjTudteckxUywkoduXuWREa29MWEHcdfw3eyNF/vD68N96W2cK01NkzcwGCKKAtdVyMIHPLQQyApTnK+g0g6JCChL44DPaodGC1xRMXCHBGIdV8CT7ohC36CrpqS1S6AqlGxoGkHuupS1dzv1tJbUnkbULtoROFKi6sYUhXd35/3bORSqjklscFsF3XZKJ8g4Hw9Y9Udc8Le45K+zMotKHjeaK1XIa/kZNsou30qW2c5abCcxg1FEmTnIkWN8WBOBNwKVaoJhgzOtcUclqj9FTFOT418dn2z3/eLIiNbemLDnLiwd+HPtjRf79Vc3jr9oAWiNLgrTF5jkSEfipAonFGhpirhamjt/TaCopGaSe7gqpOXmbFdtXFHTdzx6smBRSpbUlBLFkjslb1ySymPoBUa4fIe6EsjSRNmIBmrfFOZl5CJDH6H1Q9OjqP0G4dczt3tOd+Z2X3ZN0X1JprRlQ6lhrB226g7bVZudMmZQRExmbvdqL9s9Nw3rptNh7+bKLHk0vZ3tfpJtDR/EkRCtvTFhP/jkvY0Ju1/2xot9/pUN/uq/dzLGi+k8py4KlOugZrPrtApnOy1TkDe2dkVdCUZuSFmr2U6rIfE9Yplz3t0hEglnVEFPbjMNfAJZIkVD0Sh23ZDtSlFJKKVC1LPhrzU0Spm007qF8lxknpvBCSf8+OK2c4KgZClOOBsPWfYnPB5sctbtc9Hps6I0rlB8vfBZr3pcKVa4ki6zk8esT9pMUp9s6sHYwUlNtruTzoruIz2bnlOYqOThhGZ759iM+zoIjoRoPciYsPvlRI4X0xo9TUzcr+vgeA40mipwQZhjoinOSyrfJQWGqmbLa9EgWHYnSNEQiIpVVRAJOOvs0mhJ1rgMAlPEn0YeCVA3GMe8EJSRgAZEo1CZMbU6qbmhIur6xDXt3kkryom8kqVgyqo/Ys0f8oi7w4oas6I0idZMa831aoEb5QLX8wW2sha7uZlPmKcuOnVwEpPtvmccdZLZfMJ0lu0+SdDj8UNzl/D9OBI+rVtjwj5y8PWsPU7qeLEmy9CTKWKSoKY5TlKZO06zPwInBZUKRCapU4dp6tNPI3bymJtFlxvlAut1hwZwhWBNJZxxdznj7bLqj1kOJnQCs7MQYU0dNlSBNnfQQpMdVbYUVezStAJEHCF9H8SR+F/tQOgEOYthYtzu/tDkYqkRK6qgKwPGjeJG3eZGucBm0WE7b9GfZbvnqUuTOmZs/d4E6FQbe0OicRIzVFVOU/RkSj2aPPSidSR2WntjwrrR4cX5XliOeeJUi8+/unHixos1eY7e2kZJicgjfCWRpQNamRNiKdCOibIpGsGO1OSVgycr0tqj0RKXmlNqwhOuxhUTPGoK7dBSOUXj4KqaHdkwaQSVqxCNMnUzJczwV1cg6ghHKZRSSK3RaXoi/+DOtQb03JSL4RZP+OusOUM+4goSLXi9LHijXOF6ucDlZJXtImZ92mFrHJNnnsl2TyTuROKOTCaWPzTWBm9Y4fYT5CSj2e6f6H7Ce2HuL3/v7JgxYZ97Zu3Qn/uzT6/y0pU+o+yE3YHRGl1V6CRFJBkqq3DS2hR0Z65qE2siEIWkyB2S3J055iO2yxZbdYedJqLUDS6wolKW1IRld8yil9D1MmK/wPEr8Bsa32RG7Q0gNQNgFU3k0kQBIggQ/vFvUn8vFr0pS96EZWfEKTVmSeY0NCRas1XHbFYddquYQRkyyI3bPc9d6txku6tc3HK734qaSRtUVpls9zQ7UcmjD8rcd1p7DdKfPQSrw7v57NOn+H/81rdO5HgxgGY8RuQ50ndxS1OP0spFlnK2E7rdo5hWkk3VUM56FH1ZMXYD2jLjjMp53G0x1Zu0ZUrSeDiiRoqGqlZMnIasEmip0NI45mvX2C2Q4EqBW5qpPs1eq8kJ4mK4xaKa8BFvgyfcjLb0uVxWrNddvlWs8q3sFNt5i+tTMz1nPAmpR94syE+aovsYM7I+1fiD0hwJhwnsDGjSh/dO4XtxJETrI6sPNibsfjnJ48UA88pcGBOiABxX4foKNNSB6bLWamZE1ZD6HlJoXFXTdjMaBFedJWAHT0zpSY1yRmy5AxotqbRiXAYIoSkLM/y10qbtRAsocoFoFDSg0gApBSrp0CQJOj/eY6zu5Iy7yyk15oyTUmrYqHOuVwusV12uFwvczDr085hBEjJNjXl0b6CqcbrfNo86SWPmE44zxGhCk87G1ltuMVfRGiYlL73Z53/xAxfn8vx748X+zTfWKesGV839tLy/7B0TxxMzLdp1cAIHcKhCgZbQuAKVznZcgctUaFynZsczLyKn3C6uqIjFDmeUJlINZ51dSq1IGo9+YO68TgOPtBY0jaDKZu1DmaCsQTSSKvVwANVpIeua+gSJ1llnl0WZsSw9NuqCYeOyXnW5WS6wVbSNYGUhSeZRZe7tYRSzwruTatxUm8SGpEZOc8Q0NZHWRXnidqYPylxF69++vknd6ANtkP4wTup4sTtp0hSaBlE3uFIic4/GCZGVBA0IQVUKtHIoK8FAC5RsmJQmqiZpPErtIL112lLzhDsiELcbq2NVUNaKvmpIVEBVm92bqE1hvlFylvsl8WqNkhLlOiemR/FJN6XUmu2m4ErVZavq8Hp2mo28w/Wky8akRZL5Jts9lTiTWb57Ohv5lTR44wa/nyOTwmS7J7P2nBNqE3kQ5rq1eOHVTTMm7FxvbmvYGy/2+VdOlvXh29jbcWUZIsmQWTkrzDe3BnuqHGRmCvNV7jDNPSaFPyvMt9mq2vSbgEQLAiFoy4IVNWLZnbDoTen4pjDv+SXab8wA2FuF+VnOfChpIhcdBYgwNJOrpZr3b+eBUQhqoF+77MyGqvbLmN0iZJQHpLlHMct2V9nsWJjddrs7qcZJK8Se2z3L0IUd0/Z+zG2nVVQN//abm/z4x9bm2rR853ixn//JZ07MeLF3Y46JY4TjIMsK11WI2tzN01IhKmF6FLWk0g5TZab6+E6PojE9ipHMmTpDev6ARVkTubtk2iWSOWltivNKaDYrSem4lJWLFhizayloHIUszeccJRBliUxSmvF4vr+cB2SsG/q1y+vlKS5na2yWbd6ZLjDIQvqTiHTsQ6ZM0X02PccdmSOhPzQ+LGeUIXdH6On02P8+Dpq5idaX3zJjwg4q8O9eMOPFtnhjc8JHTsh4sfejSRJkXSNDHwfQSlB7clacN7UoLSSV65Jpwe4sysaTFQvOEqV2OKUmdGVJWwrWnCFKNPQr46pvtCQpXKYCikJSNwqhMcV5qXFycyNAaHDSFlLK27fzj+lR8WoVsVO3uFYscSPvsV3E7KQRk8y43UXiIGfzCVVmhlF4U2NrcMclapIjRwl6PDF3Vy0fyNxE6/Ov7P+YsPvlR59a5W/w9dmdzJMtWjrPqcsKJ46QUqI8BzcwGVxVahqrtSNoMkkjFEngoaTGVy02PdNDeN3t4opdVoRgReYoNOvukFw75LVzq92nzB3q2phNqxBAUIYgaomsHVQcgNaIiQ9Zjj6morVe9dipW2yUHbaLmN3MCFaWejSJcysTy0TMzPLdU5PaoKazbPfJ9MRnu+8XcxEtrTW//toGz+/zmLD75SSOF/tAmpp6u48sSxytQbSQpUvjOMjKDLHQUlCVDrnyqStF3Qgc2TD2AwJZknkemd7mCVexqGoyfYNAlviiIm8c+m5E1UhTmFcaGueWN0wriVYCUYU4gYNTN+jBiGbcHEvH/Dez0+xWEW8ni9yYdBlnPtNRgE4d1FiZonsG/tC0UnmTBr9fopICuTMy7TnD0bHdaR42cynE740JO0rJoZ99epXfvzpga3xybsV/ELoq0bkZ6qmSEpVUd+TLc2sQqMgVVa5Ic49x4bNbhMYxX7XZqWOSpqbRmhWVzgrzYxa9KT0vJfYLXK9C+A1NcNsxf2uYaSipA8cU5qPQOOaPYU1xp4xNtnseMs09svzbs91vdyHc4XZPTZCf3vOsWcG6a+ayzZmnC/79+OzTq/ztz7/Ob762yb//fefnvZyDR2szzLMskY7CqSI8TyFqhdASrYxlQTuSqnHIgG0npqgVkVNSakWNJJAla2rKo46Hyy5KNCSNZ6wSlTGrDmRDMhskKyqJFgKkMFN93NmOSwhT38rzY7fbupouMMxDtqYx42lAnTioiboV5OeOTfOzP65xJjXesED1J5CkVP1de5fwHpmLaO2NCTt1AGPC7penT7c52wv5/KsbD4doAeiGpigRkymybnBDF/DQUpg0Um2ibDSSWjqknk/TCLZcM/zVFTU9lVBqRVsMcQU85gwZuDEKzSAMZ8V5QZG7VBLqXCA0gJg55yUydxGN+cOVaRedZjTT6dx+LffKZtJmWnhMEp966hov1nRmHp2CN5kV3Uc1TlIiRyl6NEYnqRWs++DQRWtvTNhf/dxHDvupPxAhBJ99+hS/+rtXycqawD3+/qEPRWvQ9a0/HjmNcJQJ9XMC04qjQtPqox1B7StyPIZ5gJINoSrZcLsoGgZqTFs2LErJmjOgRrDhdSgah6JWTEKfDKhCaXZzNVShAA1VZFIoRNXgtGITY3OMjJWjzCcrXON2zyRqNpDCyZh5sBozBTopkZPCuN0n04c6yO9BOHTR+o1XD35M2P3y2WdW+QcnZLzYvdCkKaIokI6DKlv4dYyWAlWaY6KsTBZ8Lh3qUtKXMUWlqBuJK2vGtSnOn3f6nFEF552EtiwotUMgSzxZUTaKkRvQbwSVZHb8nDVuNxItnZmLvkH6Hqqqjk2P4mgcUecKMXZwx2aH5c1iZrxxg79b4SQVameCmCTU/V0Timi5Lw5dtF54dePAx4TdL89dWDo548XuBa3RtdlxScdBei5O6oIEJ5NopU2PYi5AGMd86niM3Ip+EeGIms2qQyRy2rKkLTRtUbGiRoydgMT12PFjGi2Y+D55JWlqQR1oRGOK8rKUyBrq0DUN1nFkehTL6sgXqetMQSFntoZvz3Y3O60alczc7mmKLqtjs4s8ihyqaJkxYdv8B997/kg6zz1H8oMfWeGFVzf5xeM+Xuxe0do4sZsGCWaqT+lRexIxG/TaKDPVp3QdMg27GnxVk1UuoTLF+QbJx7wd2lLwuGsK866oSRoPT9YUtWJXaDKhqUoP7TCLyDFDOGTh4boSUbaRgNSaZjI50n/kYuKYgaoTgTuZBfmNZ9nugwJnkJoj4U7f3GQ44iJ81DlU0dobEzaPwL+75USNF7sPdJ7T7A5MX2DT4AUKoRVaKmqPWXFeUuNQCOh7IVUj6XgdzFTEhp5MWFQZK1KSqQm1K+n7Ma6omVQ+jTYimBZqJlQSoU3WvJNLtHKQRWDmKAKU5ZFOO3CS2wNV3eks231c4yYVzjhHjKbGPFqUoJt5L/fYc6iitTcm7NkLi4f5tPfEiRsvdo/o2Vh7kaRIIVCxT+MIHFfgBBJmE6a1lNSOIstcBNAPYnxZ48uKdbeLFA3nlGZRFtTOmHVnCJghsXnlUNWKPHRptEMdSEQFooYyFKAlVWjmN8q6QUzMSLKjOgBWpbMj4SxmxhwLa9S0vO12T5IjK7rHjUMTrcMeE3a/nMTxYvdD0x8g0wzHUcgqQtQeWrqowvQrykpQNopS+NSl4qZqU9aKvFEEsmTgRnhcZVXBk64i0zfpqYS8cfFkhe9UVLVk6niUjRFB7RjHfOOB0C5aCVxX4tQ1apJQbWweyWOiNzI1LG+s8YcN7nSW7T5O0bsDmqlNHt1PDk095jEm7H753DOrvLY+5mr/ZM/r+yB0XdPkuYmymU31MVE2GpVyy+ktckmTOaS5x6jwGexlzFdtNusWYy0odU1PFpxSY5adMUvulJ6XEPuFmeoT1MYx7/OdjvnQQUc+hAEyDBHO/Nu+3o0punN7h5VWyEl2O9v9IZwCfZAc2v8B/98/uIFzyGPC7pcffdrMRPw3r2zw558/WZN67pqmRheNqW/VNQ7geQpZKmr3tkG0cSR1LchcH60FWgsipyCtPdoyo0FS613OOJKezBg1m0ihkUKTVB6uiqkqZTxc0ly/me3kwEz4kUWIUgpVVjS7gyPnmPfG5kjoDSvcYY4cZ+jdoakPWi/WvnMoorU7LfjVL1/lpz9x9lDHhN0vF5ZjPnG+xy+9+CZ/5rlHHg6j6XuhtWmrmSqEUji+i6jcWb68RM8GwIoGSs8hB4bAhtumqBUdZ+nWpQKxQyDgMXdw63OjKsCRDVnloLUgFy5VLtHKDMcQDYBE5Y45khYtpNYg5ZG6o+hNmpnjvUAOE+PFShJjbbDsO4dyPPx/felt0rLmP55TFvz98L/9sSe5Pkj5h//urXkvZa7oqjJzFKcJcpqh0tIMX8jM5BhTgBbIVKAzRZE7jDKfYRGyU8ZsV202qzZj7VACixKW1JRTzogld8qyN6Xj5URBjuNXNGFD7ZujYhUKqtC46KtIUcceOgqQcYRQR+eFxJnWqKQyMTNJhp7aJuiD5J53Wn/tn3z1np/khVc3+OEnV3jyCBpK34/vf3yZH3pyhf/qNy7z+sZk36//f/kT37Xv1zwodJ5TFwVKSVQR4ymJLD1k7RjLQiRuNVjXtWAoNUXl4IiGonHIGtc0VjtDnnGnrKqCttyg0Iqus0CpJa6qcVXDdiOpHYeyceBWcV7iegLRBLhK4ngucjbXsUnmX3f0djNkWiL6Q5rBkOYYuPiPM/csWv/uWzv3/CS90OU/+dFL9/x98+Y//eNP85/8d39wXz/ziUNrdJqBkMgkQjkS5UmcUICEKhdoB9OjmJlUiFHg4zsxnqzYcHsoNEM1wRXQFpolNaHUDptuh6nnU9aKse+Ta0GTK+pKGyEMzFGxCiWycJCVhwpN0CBZbrxPczwqyqRApDk6mU3QPiLH1pPKPYvW7/z1HzmIdRxJnjjV5l/95c/MexlHhmY6ReQ5yvcQjTFJNp6PrOWsOC9ACxpXUdWCgRvd+t6Ok5Fph7ZMecwdsKxcHnOGxKJgGpiseikapqXHSDVMSpNVr4U0+fVKmOlBOGgJft5COgqZZnOPsxG7I3SWUY9Gc1vDw8TRu39sOdLoujZ3FLXGEcLcUawUtWdacQDjmK8FuesxNLcZiZwF8sahq1IAaj1kTUl8kTJqtm655Celj6tqytK0CtXCoSqNqbUojfEU4aCyAEdKZFnRDEfo6fzMm81giK6t0/2wsKJluTe0pkkzhOchfA+ZByglULlGudB4s5wsCVUhKQqHxPGYlD6erNitYjoypSMzHnVqfAE9lbDoTBg3AV0vo2gcAq+k9BV1Jc0EbB8aH+oc6lJQBwpRu4jQR6QuIhVz65CxtobDRWh7/rZYLMeIo9tPY7FYLO+BFS2LxXKssKJlsViOFVa0LBbLseKe7x4KIb4OHPXbJcvA9rwX8SEEWuuPzXsRFstx434sD5nW+nv3fSX7iBDid4/DGue9BovlOGKPhxaL5VhhRctisRwr7ke0/u6+r2L/sWu0WE4o1hFvsViOFfZ4aLFYjhX3JFpCiD8jhPhDIcTXhBBfFEIcuSQ7IcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rsliOE/d0PBRCfD/wqtZ6Vwjx48AvaK2fO7DV3SNCCAW8DnwOuAZ8GfhTWutX5rqwOxBCnAZOa62/IoRoA78H/MxRWqPFcpS5p52W1vqLWuvd2YdfAs7t/5IeiGeBy1rrK1rrAvgV4KfnvKZvQ2t9U2v9ldn7Y+BV4Ox8V2WxHB8epKb154F/vV8L2SfOAlfv+PgaR1gQhBCPAZ8EXprzUiyWY8N9hQAKIX4YI1rP7+9yHh6EEC3gnwJ/RWt9vzm99tbv/iAe5Js/J/+E/Xe4E6mQnotoxbDQRccB+XJI1VLkHUXRFlQRFD1NHWjqdo3TKvGDkld+5hc+9N/iQ0VLCPEXgf9o9uFPYPr6/h7w41rrozbx4Tpw/o6Pz80+d6QQQrgYwfpHWut/Nu/1WCz7iQx8RBTC0gL1QkQVu+SLLmUoKDqCsgVVpCk7DTpocNs5vXZK7BV3df0PFS2t9X8N/NcAQohHgH8G/JzW+vUH+cEOiC8Dl4QQFzBi9SeBPz3fJX07QggB/H3MDY2/Pe/1WCz7hXA9hOciFxfQrZBqIaLoelSxJO8KqlBQtKFsa+pQQ6fECyoW2glr8ZiOl97V89xrTevngSXgvxFC/MFRa/rVWlfAXwJ+DVPg/sda62/Md1XfwaeBnwN+ZPY7/AMhxE8c5BNe3hzzv/7VPyAr7fBQywEhBMJzEWGAbkc0rYCq5VLFkjKSlJE5EtaRpo4adFjjhSVxmNMLUlaCCav++K6e6p5qWlrr/xD4D+/nZzostNb/CvhX817H+6G1fpEHrKHcK3/zf3iFL7yxzU994gw//OSpw3xqy0OAcBxkK0b0ujStiHIloooUeXevfiXIe1AHmqpTI1slQViy2h2z4Cc8Eu/yeLDFonN3Q5GtI/6E88XL23zhDRMt9tKV/pxXYzlpCNdDhCGi3aZpRdRtnzJ2KGM1210JqhDq0OywRFgTRAWtMGcpmLISTDjtDTnj7nLW2f3wJ8SOEDvRaK35P/3aNzndDVhu+bz85lG7b2I57shWjIgjmsU2VS+gjB3ynjT1q72ie2h2WCKsiNo5i3HCYpBwId7hlDfikr/BJXeLRXl35QsrWieYt3cS/uDqgJ//yWfYmuT8P3/7CklREXn2n93yYMggQIQhLC/QxAHFYkjRdahCQd6VVCGUHU3Z0jRhg9vNCcKCldaU09GIVX/EU+FN1twBl9wdAnH3rhF7PDzBvPymOQ5+5tIyz11YpGo0v//OYL6Lshx/pDJHwjiiaYVULY9qdhy8VXAPoQqhmR0Jw7CgE5gj4ao/Ys0fctbdZU2NWJRGiLK71C37knuC+dKbOyzGHk+carHWDZACXrqyw6efWJ730izHFOF6qKUFdLdN3Z6ZRiNJ1pWUrW83jTadCq9VEAUFZ7tDFv0pF6IdHvc3WHOG/BFvFyUEidZcKTvs1C0ev4s1HKudlhDiF4QQf232/t8SQnz2Aa71S0KIzdmgjhPJy2/2efaxRYQQtAOXj57p8qU3bTHecn/IIEB2Wuh2TNMOqFoeZSxvF9xjZrssTRPVOGFlLA1Rymow5nQw4qy3y1l3l7OOaQKZNpr12ud6tcCNcuHu1nGQP+RBorX+ea31Cw9wiV8GfmyflnPkuD5Iubab8tzFxVufe+7CIn9wdUBeWb+W5R4RArnQg4Uu9VKLYjEgX3DJu5K8Kyk6xjRathuaToXTLum0E061JpyNh1yItnk82OQj3jpPukMedQSJho065LX8NJezVd5I786Oc+RFSwjxN4QQrwshXgSevOPzvyyE+NnZ+28JIf7zPcOrEOK7hRC/JoT4lhDiL7zXdbXWvw2c2G3H3p3CZy/cFq1nLyxSVA1fvTqc17Isxw0hkO02zuopmpUe9XKbfNEnW3DIepK8Jyi6UHQ1Zbeh6VZE3ZSF7pSznREX2ztcijd5OrjBR/1rPOMZA+lGXfFaucw38rO8lp7hjekprkzurmxxpEVLCPE9mFacT2D6Hr/vAx7+jtb6E8AXMLuonwU+BfzNA13kEeXlN/t0Aoen1jq3PrcnYNb6YLkrhEB4HjKO0O2YOvapYpcqkt/mwaoi05ajQ3MkbIc5C0HKij9h1Rtx2htw1tllRaW0hEuuYafxWS973Cx7bOQdtrOY3Sy8q2Ud9UL8Z4B/rrVOAIQQ/+IDHrv3ta8BrVlW1VgIkQshelrrwcEu9Wjx0pU+3/fYIkreNt/3Io+n1tq89GafvzTHtVmOPsJxEGGIXFqg6cbULZ9s2aMKTR9h2TKCVfQamlAjOgVxK6cV5Dza2WXZm/JEtMFFb5OzzoCPeYJMSy5XDVfKNdbLLq8kZ9jOW9yYdtmexBT53cnRkd5p3SP57L/NHe/vfXzUxXlf2RxnXNmefls9a4/nLizye2/vUtbNHFZmOfIIcastR7Zimk5E1fYp2y5lLCliQRmbu4RVrKnjBh1VBFFBJ8xYChPWghFn/AHn3T5nnQErqmDYFGzUDTeqLtfLBa4Vi6xnHbayFrtJSJp4lIl7V0s86qL128DPCCHCWTTxT817QceBPX/WsxeWvuNrz15YIilqvn7d1rUs34lQCuF5iIUeerFLtRBS9DzyriLvCMqOoJwlNVTtBtkuCds5S62E0/HoVh/hE8E6l7xNLjoVp1XIjcrhSrnIG/kaV9IV3kqWuDHpsjluMZkENGMXOb67vcWR3oHMctR/FfgqsImJntkXhBD/HfBDwLIQ4hrwf9Ba//39uv48efnNPpGn+OiZznd8ba+u9dKbfT75yN3dYrY8BAiB9H1Eu42IQ+rljsnBWnApWnLmvxLUARSdhiaukVFFt5PQCXLOtQY8EvY57Q15JrjGippyXjUMmoardck3irNslF2upCu8kywwyEK2RzFF5sLYxR1LZH53OQJHWrQAtNa/CPzie3z+z93x/mN3vP/LmEL8d3ztXd//p/ZtkUeMl670+Z5HF3DVd26kV9o+F1diXn6zz1/4wbux8llOPEKYHVbgI+KQphWZWJlQ3XK432p8DjRN1CCjCj8s6YYZC37Cqj/itDdk1R2wpia0ZY0rHMZacqPqcqPssVl02MxbDLKQYRpQpC46dVCJRCUCdXcZgEdftCz3xu604JsbY37qu06/72Oeu7DIv/zDm9SN/rZCveXhREYRIgxgZZGqHVC1PbJFxxTde4IyniWNdmdJo52cbiujF6ZcbG+z4k14IjBF9zU1ZVVJxg18pQi4Upxio+zy2nSNnTzm5rjDcBJQZS5i4OKkAnckcCfg3GUfz1GvaVnukZffMvWs5y5+Zz1rj+cuLDHOKl5bv99oesuJQCqE7yPbLUTHtOVUbY+yrYzTPcYIVqypI42OalRc0Y4zFsOElWDCGX/IOa/PeXeHJZkSCU2/rtmoPa6WS1wrFrme99hI2+ykEePEp0w8dKJwpsK8JeBONe707kTL7rROGC+/2cd3JB8/133fx9yqa13p89Ez7/84y8lGei4ijtALHZrIo+z5FG1FGUmKzsyD1dJUsaYJa7y26SNciU1Swyl/zAV/kzVnyCV3SCAEEsHlss161eXNfIW30yW2shYbkxZJ5pNPfMTYQaUCZyJwUvDGGn/UoO5yp2VF64Tx0ps7fPKRHr6j3vcxZ3oh5xdDXn6zz//8+QuHuDrLkUAqVCtGdNrodkS5FFPFDvmCQxELqti43KtQU3UbiCu8sGSlO6EXpDwW9zkf9Dnt7vJx/zptWRIIwXqtGDQB38jPsll2+NZ0hZtJh0EaMBjGNKmDHCvciUClAn9X46Tgjxq8YYVKq7tb/gH/eiyHyCgreeXG6D2tDu/m2ceWePmtPvcyYdxyAtgb7xVH6Di8I8v9jqTR6A6Xe1DjBLez3Bf9KSvemNOuaXzuyhJfQKY1gybgRrnAzaLH+szlPkgDksynSR1EJnFSI1hOCk4KbtLgTmucSYGa5h++fuxO60Txe2/t0mj41IXvNJW+m+cuLPJPv3KNy5sTLq22D2F1lqOAWugiwpBmuTszjTpkC8qE9/VmSQ2hpurViKCm1U1ZiFIWg4RL7U1OeSOe9G/ymNtnTdXUGgaN5I1ymSvFKW4WPd4Yr7CbR2xPYpKJT5M4OAMHlQncsTkOOgmE/QonqXF3M+TuGJ1kd/UzWNE6Qbz0Zh9XibvyX+255V96s29F6yFgb7yXaLfQUWAEqzPLcm8JquCOHVbUIGMz3qsbZiyHE1aCCWf9XdacIY84u8SiotCwXbts1i2ulktczxfYyNtspy3GmU+a+DRTF5lKnESgslnBfaJxU407qVDTEjlK0NMEnd6daNnj4QnipTd3+Pi5HqH3/vWsPR5ZjFjt+Lxk87UeCkTgI3tdml6LqhdSdF3yjiJvC4q2cblXLU3VrtGtiqiV02slrMUjzkUDLobbXPQ2uehtcsFtiATkGt6qlrhSnOJKusLVdIGbSZf+NGI8DajHLmqscCbG0rC3y9qrYTmDDDmYoneHNMMxzXR6Vz+L3WmdEJKi4mvXhvxHP3Dxrh4vhOC5C0u89OYOWmvMDFnLSUM4DnJhAbotmlZIvhxSh4psQVG8K2m07tSoVkkYFpzpjFgIEi5G2zzi73De2+EZd5tAwLCBG1XIet3l6+k5Nos2b02X2EpixmnAdBBCJnFHCmcscDJmRXeNP6zxBgVqWiA3d9FpSjOZouu7z3izO60Twu+/M6BqNM/dRT1rj2cvLLIxynl7JznAlVnmhXAchO8jWhE6Dm5nuceScpbjXu+53EONjCqT5R5mLAVTlr3prfFea2pEWwqUEAwah/XaND5vFm22Zy73cRqQpR4iVcblnhrBchKNm+hZ0b0ygjVO0dOEJs3QVQX3cEPI7rROCC9d2UEK+J5H776f8FMX9/K1+jy2HB/U0iyHjRAgJGplGR2HVMttypY7K7pLqmBmaWhpqlDT9CqcoKLXSViKpiwHU55qrXPKHfGUf4PHnAmryudapdmqQ14rTvN2vsxG0eFbo2WGeUB/FFFOjWC5AyNY3ghTv0o0wU6Jk1Q4uwn0hzSTKU2S3JNY7WF3WieEl97s87GzXdrB3cV7ADy+0mIp9viSDQU8OQiBDEPUQhfdadF0I8qOR9lxKFrSxMq0ZoIVaZq4xo0KoihnKZpyKhxzNhxwzuvziLvDY84ECfTrnKtVh3eqRa4Vi9zIu6ynbfppxCgJKBMPMVWoqcSZCtyZy92bNHjjGndU4AwzxGCMTlJ0Ud6XYIHdaZ0I8qrm968O+LOfevSevk8IwbMXFu3k6ROEcFxEFEKvQ70QUYcORUdRxEawypYZPFG29tpySrqtjE6QcTYacjowbTl7fYSPOC2uVRNu1B5vlctslF3eTpe4mXbYzcLbfYQThTMxPix3Ak6q8cYN3qjGmVaoYYqYptT9XXRZQXP/cwqsaJ0Avnp1SFE135YHf7c8e2GRf/31da4PUs727i7u1nL02KtfycUFdCem6gQUix5lJMl78lZwX9HV1GED3ZIgKmlHGefaA5b8hI/E6zzqbXPW2eVJNwXg1SLharXAjWqBbyRn2S5avDNZYHsSk6Ye5dBHZnLW9Gz6CP1Bg5tovEGFO8yQ4ww2t2myHF0U973D2sMeD08A7zXE4m55buaet7nxxxghTME9DNCtkDr2qGLjwapCM6L+VpZ7NEtqCCpae1nuwYRVf8SqM2TNGbKiUlwhaYCNusWNaoEbhfFgbWUtBmlAmrmUmYNMTf3KvIFKTQ3LSRqcSYEcZ4hpSpNmNA9wJLwTu9M6Abz0Zp8nV9v0Iu+ev/fJtTadwOGlK33+p588dwCrsxwkt3ZYp5bRUUCxElO2HMqWJOsJ6kBQ9GaCFTeIbkEQlJzqTFgOJ6wFY56M1ll1B3zUW2dFNXSlx6tFw2bd4ZX8LFezRTbyDm+NFhlnPpNxYEyjicQbSJwMvKE2TvdUE2wXqKRA7YzR/QHVZPpAx8F3Y0XrmFPWDb/39i4/+z33JzhKmrrWy9ZkeuwQroeMQ0Rsstzr2LslWEVshk/UvhGsqlVDWBPHObFfsBxOOBMOWfNGPOptsaQmdGVNpmFc5bxTnWK96nE1W+R61mMnixkkIVnm0kxd1MTssNwpqHR2l3Da4E5qnFGGmGbo4Zgmz/dVsMCK1rHnGzdGJEV965h3Pzx7YZEXXt1kc5RxqhPs4+osB4ZUyDBAtFrobouqF1BGDkVnVnCPTR9hHRqXu4gq/KhkIUrpBSnnogFn/V3Ou30ec/v0ZEVPOlytGm7Uba4Up9gsOlzPemwkbYZpQDr1aFIHNZG4E3lLrJwUc5dwWJkj4XCKnkypd3cP5Ee3onXMeemKqUV934X7z3u/Vdd6q89PfvzMvqzLcnDcShpd7FF3Qqq2T77oUoaCvCdvjacveg1N0OB0ClpxRjfMuNDZYdGd8mS0zmPuFmedEYuypga+XrgzS8MSr05Ps1uEXBv3GE5D8tSFgYuTzoruU2MaDXYb3KTB251ZGiYJzdaOKbgf1M9/YFe2HAovv9nn4krMqfb975A+eqZD7ClrfTjqzMZ7iTBARBFNK6Tec7nfGSsTmqSGJmgQYUUY5bSDnMVgyrI3Yc0fctbtc2p2JKyBcSO5Xi3MomW67OQRO1nMJPMpMgedOKhUorJ3udynDU5So6YFYpqiJ1OaNDUu9wPC7rSOMXWjZ7uj98+DvxscJfnuRxdsXeuII30f0YphZZE68siXQ6pYUbRMlnsVziwNkaaJarxuThQUnO6MWA3HnPaHPBXeYM0Z8oy3iycEIHilMEmj38xOcy1bYCtrcXXUJcl8spGPmCrcROIOZ0mjQ40/bnCmDf5OZgRrZ0AzGhuX+0H/Hg78GSwHxmvrI8ZZ9UD1rD0+dXGJb26M6U8PbltvuU+EQMYxotsxptFuSNXxKdtGsIqWuJXlXsXG5a5aJZ04YzFOOB2OOBsMeMTf4TFvmzVnjAIGDVytXN4ql3m7WOZatsDNtMNG0mI8DcinxuXuTOXtpIaJcbm74xpvVBrT6GhKM54c6JHwTqxoHWP2jnP34896N3vX+PJbdrd11BBKmR1WO77VllN0TVtO0b6jLSfW6LjCiUviWVvOajjmXLh7S7AecyYmvA/o1wHfKld4O182dwmTLttJzHAaUk49mDo4E4k7NjUsI1gab1TjjkrUMEMMx+jhiGY6PdAj4Z3Y4+Ex5uU3+5xfDDmzD072j5/r4juSl670+WMfXduH1Vn2A9WbJY2u9KhbPmXHM0mjgSBfuD18ouzWEDRE3ZROZMbTX2xtc8ob83Rwg/PuDmeUiTMeNHClXOKtYpmbZY9vTlbZzSLWx22miU+dOKhZ0qg3EjgT00cYDGqcaY23m6F2p+hpQt0f3FOszH5gd1rHmJff6vPsYw9+NATwHcUnH+nx8lvWGX8kEMKkjYYhuhVRt3yqlksZzxzue8NTZ1nuhDVOUNEOc7p+xkowYc0bccodcdbZpScLAiEYN5J+HXC9XGCj7LKed9jJYoZ5QJp61IljomX2stwTTME91caDNa1MW06Smsbnqtx3H9aHYXdax5j+tLgVm7wfPHthif/bb7zBKCvp3ENahGX/kVGEXOjRLHepY4980aOMTf0q3xtP39XUcY2ITJZ7K8g53x6wFow46w94Orhuiu5uTaKh38A3y1Osl11eS0+znnXYSltsjNpkmUs99ExKQyLwhqYtJxg0eOPG7LC2p4gkh80d6jRDl/Opf9qd1jHnXkL/PoxPXVik0fC7tq41P6RC9brIxQWaXpuq61N0XPKOnMUiz4ZPtDR1u0bEFUErZylOWImmnA93OR/0ueBv8ri7w5rKGTcVNyqHt8oebxXLvJMvcSPtspG06U8j0qlHPXFRE4U7mUXLjGf1q3GDO65wRzlyOIXhhCbPD/1I+G2/ork9s+WBWe34PLIY7dv1PvnIAq4SvPzmwTiZLR/C3nivbgfdiam7AUXbnU18Nm05t+8SNoioIogLunHKcjjhdDjkrD/gUW+bx9xtzjmwKB36jWK97vBWucLVbJFrWY/NmWBNprM+wqnCnQqcKbgTjGBNjGA5wxw5TEzBfTRCH0Brzr1gj4fHmOcuLO1rtnvoKT5+rsdLNvHhcBECoRRyaRERhdTLHaqWS9F1yLrK+K96wvQQxibLXYYVve6UXpixGo65GG+z6o74I8FVzjhjzimXjbqm33i8Vpy+NZ7+ymSZQRayPYopZuPp3aGpYXnDO5JG98Z77UwRw4nxYE0T0M28f1t2p3Wc2Q+rw3td82vXhiTF4dy+fugRwgT3hSEiCmnicFZwdyhDSRVBPYuWqQOowwYVVQRhQSfITZa7P+G0N+CMu8uqmtAWmoaGfuOxVbe5Ufa4WXTZzNvsZiGjzKdIXfQsy91JxGxw6qzgnjQmuG9SIMaJKbhns93VERjua3dax5j/ySf2v0/w2QuL/N//7bf4/XcGfPqJ5X2/vuXbueVyX+hSLbWoIpd80aGMTA2r6Jim56JrkkaduKTXSegEGRfbO6z6Ix7xd3jKv8GamvK4E7JZJ7xSKl7Jz7JRdnl9uspG1mYnjdgZxlSZC0MXNxE4U7PDclJNMDCmUXdS4WyNEUlGvbFl7hAeAbHaw4rWMeYg7vB976MLSGEasa1oHRzCccx4r14XfYdptIokeVtSRSYauWxrGl+jWxVOWNGKM1biCYt+wrlgl3Nen/PuDmfVhEjA9TrhRhXyVrnMm/kKO0WL60mXfhoxTgLKiYfIlEkZnRpLw14OljeqccYlziiD4Zhmz9JwhAQLrGhZ3kU7cPnomS5fsn2IB4ppfA7R3TZNy6fs+BRtM56+nM0jrOJZH2HQ4MYFcVjcastZ9KY86m9zxtnlvDNkWSkarblSBlwtl7haLnIj67GTR7cEK09cxNSYRp29ontiCu5O2hjBGqaIcUI9GO5LNPJBYEXL8h08d2GRf/ilt8nKmsD98GnVlrvnVtLo8iI6CiiXWya4r63Iu4IqFOQ9YxqtYpPl7vkVK90Ji2HC6XDIk9EGy86Ij/vX6cqSnpS8XQn6dcw38rNczxe4lvV4Z7zIOPcYjmLqiYNMFN5QInPwB9rcJZw2+P0SZ1qapNHhiHo4OrSWnPvBFuIt38GzFxYpqoY/vDac91JOFlLNhqfG6DikaQVUsUMVScpIzOJlZoIVanRY4wUlcZizGCas+BPW/BGrrsly78oSV0CmG7bqmOvVAjeLHht5h34eM8p8pqlpyxGZQmUClYGTmDc3bW5nuU9z9HiK3hueeoSxOy3Ld7B3V/KlKzsHcofyoUQIVKeFaLWMaXQppIwcskU1q1/dLrqXvdrkYMUFy+0pvSDlUmuTNX/Io942H/XW6cqaWEjWa7hadXklP8tm0eHydIWttMUgCRmNQpM0OnTMHcJkNp4+g2C3Nh6sSYHaGpo+wu3jYXWxomX5DnqRx1NrbV62zvh9Qfi+uUu40DOWhl5A0XFv3SGsw1nRvaVpwgbZKgnCgl6cshaPWPanPBbscMbd5TF3m0jUNMDbleJ63eVqscRb2TLbeYv1aYdBGpAkPs3ERaby1mgvZzorumcab1TiDHPEJEUPR8bScEywomV5T567sMg/+b1rlHWDq2wV4UGQYWCSGjqm8blsuxStWeNzPEtqiE0fIUFDFBW0Q5PUcCYcsuxOOO/tcMbZ5Ywy/X6ZhndmSaPv5EusZx12s4jdJDSNz7O2HJUbsXKnGndqstydpDYu91GCHk+pR5O5OtzvFStalvfk2QtL/IN/9zZfvz7kk4/cf/78w4zwfWS7BYs9msg3SaMtRd5WFF1BFUDRMykNdavG7RQEYcHp9pilYMr5aJePBOusuQOecbdxBdTAlapFv27x9fQc63mXG2mHa+Me08wjGYaIxAT3eUNTw/IHejaivsbrF6hpjtwa0EymxuV+jAQLbCHe8j7cqmtZ68N9IVxvNi0npokDqpZH1VKUkZxluEMdmhpWHTWIsCYIC9ozl/upYMyqO+Ksu8uKGhMIaIBho9iqOlwvF1jPu2zmLfpZzCT1yTMPkSpkasbTO6kxjd7Kcp/WqHGOnGToNDWWhmMmWGB3Wpb3YaXtc3El5uU3+/yFH3x83ss5XkiFWlowOVi9mGIxoIok6cLMh9WGsqOpA03du20aXW2PWfQTnmxtcNob8Ji7xTPeLrGQjDW3TKPfzE6zVbS5Mlmin0YMp6HJcs8UznCW0jA1Oywnm/UR7plGt3dpkpRmPJ73b+m+saJleV+eu7DEv/zqDepGo+T+NWafZGQcm/H0Cx2aVkDR9Sg6ijKUJlYmMgX3sqXRQY3XKojDnJV4yvl4wKI35YK/yVl3l8ecIQ0waBrerjpcrxZ4O1/mnXSRnTxmY9JimvrkUw85dpCZuBWN7ExNrIyTNLiDHDXOTJb7ZIouynn/mh4Iezy0vC/PXVhknFe8enM076UcD4RARBGi3aJuB5Rtj7LtUMSSMsbEykSaKjJZ7qpV0YoyFqKUlXDCmdnwiUfcPuedEaeVR61h2Li8Uy5yrVjkRt7jZtphO42ZJAFF4iESB5UI3OkdgjU1SaPuuLwtWLNpOfMK79sv7E7L8r7s1bVefrPPx85257yaI8xs2rPsdWkWOmZ46pI/6yMUFN1Z0mhPU4cNulURdTLioOCRzi6nggnn/F2eDG6y5gz4mJeTa7hWl7xRLrNe9ngtPc3NrMtG2mZj3CJLPcqhj0zMcdAbmRqWP2hM0X1U4e2kJml0q0+TpmZE/RFsy7lX7E7L8r6c6YWcXwxtvtaHsFdw162Iuu1TR7dd7lVkBKuKZoIV1rhhSTvMWQhSTgUTVrwxZ7xdzjq7rKiURmumjWarDrleLrJRdtnI2/TziGEWkCY+ZereUXCf+bBuDVCtUUmFmGZmgGqa0hRHr/H5frE7LcsH8tyFJX791Q201vsaOHiSkKsr6NCnXAgpeiapIVsQt4vubSNYolcQhCULrYTT8Yglf8pHonXOuLs87m7xpNvgi4BXy5Ib1QJvFcu8ka6yVbR4a7TEMA3MtJyRh0wl3kCaO4QTk+XuZJpgp0CNCuQ4mXuW+0Fhd1qWD+TZC4vsJiVvbE7mvZQjS70QUy6ElF2XoiPJO4KiMxOslqbq1NCuiNsZC62E1WjMo1Gfi+E2l/x1Hne3eNQpGTQVb1YZV8plvlWc4s18hbeTRa5PeyYaeRJQj12ckTIF9wm4I4031nijBm9Q4Qwy1GACu0OaNJtrlvtBYUXL8oF8aja92vq13p+qvedyn2W5x3dEy8yy3L2ooDdzua+FY876A855fc6qIauqoCsDho3iRtXmarHE9XyBm1mH7bRlXO6JRzM1SQ2mj1Dg7Lncpw3upMIdF8blPhrTDEdmh3UMfVgfhj0eWj6Q84sha52Al67s8HOfenTeyzmSpCsutS/IO5Kyw6zo3tCENapd0mmldIKcC50dTvljLvhbPOXfZEVNedQRDBr4w6LmteIcG2WXV6en2cpabKUxO6OYInNh4OIkEneWNKoyTbDb4E4b3FFpstwnKfXmFrqsTqRY7WFFy/KBCCF49sIiX7qyY+ta70MZSWofs7sKoQ40TVgjwpowLOiGGQt+wil/fCvLfVEltGVNogXDRnG96nGj7LFZdNjKWvSziFEazLLcHZzEFN3VzOXupMbl7k5NUoMYJ7dd7iek4P5+WNGyfCjPXVzkX3z1Bm/tJFxYjue9nCNH3hM0PhRtTdWeJY12c+KwYLk15Vw8YNmb8HR4gzPOLo+7uyxKCUi+VkbcKBd4u1jmcnKKnTzi+rhr2nKmHowcnNT0ETqpmZYTDEzSqN/PUeMcMZyYLPcTVnB/P6xoWT6U5275tXasaL0HZdvsrqqWpmlXqLCi105ZCFLOxEMeC3c47Q245K3TkzltKVivYaxd3sjXuFn2uJotcHXaY5gHDMchZeoipgp3JFGZwBsbS4M30abgnlSo/hQxnpr6VXW8Xe73gi3EWz6Ux1daLMUeL12xxfj3oopNUkMT1bhRSRSZpNGlYMppf2gK7m6fMyphUdZIoN+YLPdrxSLreZeNtEM/jRglAWXiIhJ1a0S9ycIySQ3upJ6N98qNYE2mNEly4o+Ed2J3WpYPxda1PphyoQa/JmgVLLanLAQpT7S3OOWOuehv8pS3zqIqCYRgpxZcrbq8UayxWXb45mSVnSxmexIzGoc0mUINHFRq2nK8gcZJIRjUZrzXuEBtj9BJSrW1cySGpx42dqdluSuev7TMjWHGt7am817KkUOEFW5Y0pq53FeCCae9IWe8Xc64u7RliQQGDWw1ETdmWe7reYedLGaQhiSZR5M4iNRMe97bYbkJuEljdlhJaWJlkhSdpEdmeOphY3dalrviBy6tAPBbr2/xxKnWnFdztGh1U2K/YDWa8Ejc55Q35sngJmecXS46BaU2SaOvFSu3kkavJMvsZhHro/btpNGRGT7hDWfzCCeaYLdGpbXpI5ykMBjRDIZHfvjEQWJFy3JXnF+MeHwl5rde3+LPP39h3ss5Uqy2J7TdjEfiXR4Ptlh1BzzpbhKIhkzDjdpnUEdczte4WXS5mi5wbdxjnPlMxwE6cVATiTuSOBn4Qz0rujd4gxKVFMidkekhHE9OpMv9XrDHQ8td84MfOcVLV3bIyof7j+bdLAVT1sKxORK6e43PmlgKcg1bdZur5RLX8x43sy47mRnvlaYeOnGQiTRHwpRvb3ye1DjjHDnO0JOpeTshSQ0Pgt1pWe6aH3pyhV/6nTf5H756g+9+9OTkxj++8mDH3Y+1b7DgTLnkrXPJ3aUnJYNGs9X4vFUuczlbY73ocHm8wjAP2J1EpKMAMok7UDip6SP0hsY0GvQr3EmFGmbInYEZ7zUeP/RitYcVLctd8+yFRSJP8b/57/9w3kvZV976L/74A33/aXfAkjPhrDO6PTy18VmvutwoF7iR99guYnazkEnmk6fuLMt9VnBPb++unNT0EapJjpwYl/tJycHaL6xoWe6awFX8yn/8Kd7ctncQ7+Qp/wYrKuWCE3CzTtmuXV7LT3OzXODtbIl3pgsMspD+KKZMXZg4OGNzJPRGM9PoWOMPKjPeqz9FjBOa3QFNmp3oPsL7wYqW5Z74+LkeHz/Xm/cyjhRPuikNcK1KebvqsF51eT07zVbR4up0gY1JiyTzKUY+IpU4E4k7Njssb6hxkwZvbNpyZFLA9oAmy6xgvQ+2EG+xPCDLKsYXkmHjslm32ao69MuYnTxmlAekuUeRO4hcojLTluNkJqnByUwdy5lWyKQwSaPTKTpNrWC9D0Lbs7LFYjlG2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrrOXhGCOE+DqQzXsdH8IysD3vRXwIgdb6Y/NehOXusKJ1vMm01t8770V8EEKI3z0Oa5z3Gix3jz0eWiyWY4UVLYvFcqywonW8+bvzXsBdYNdo2VesI95isRwr7E7LYrEcK6xoWSyWY4UVrWOIEOLPCCH+UAjxNSHEF4UQ3zXvNb0bIcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rstwdtqZ1DBFCfD/wqtZ6Vwjx48AvaK2fm/e69hBCKOB14HPANeDLwJ/SWr8y14XdgRDiNHBaa/0VIUQb+D3gZ47SGi3vjd1pHUO01l/UWu/OPvwScG6e63kPngUua62vaK0L4FeAn57zmr4NrfVNrfVXZu+PgVeBs/NdleVusKJ1/PnzwL+e9yLexVng6h0fX+MIC4IQ4jHgk8BLc16K5S6wbTzHGCHED2NE6/l5r+W4IoRoAf8U+Cta69F9XsbWWPYHcTcPsjutY4IQ4i8KIf5g9nZGCPFx4O8BP6213pn3+t7FdeD8HR+fm33uSCGEcDGC9Y+01v9s3uux3B22EH8MEUI8AvwG8Ge11l+c93rejRDCwRTifxQjVl8G/rTW+htzXdgdCCEE8A+Avtb6rzzg5fbtjygtajbHGY8uxft1yeOE3WmdYH4eWAL+m9nO60ilFGitK+AvAb+GKXD/46MkWDM+Dfwc8CN37GB/Yt6L+r/++ht87r/8bW4M0nkv5chid1oWy4Ozb39EP/Z3fpvX1sf82T/6KH/rpx+6iC+707JYjhOb44zX1se0A4dfefkq68Ojnu84H6xoWSxHhN+5bAJe/88/+3Earfl7X7gy5xUdTaxoWSxHhC+8sc1i7PHvPbPGD35khc+/ujHvJR1JrGhZLEcArTUvvrHN9z++hJSCz1xa5u2dhHd2knkv7chhRctiOQK8sTlhc5zzmUvLADx/aQWAL1zemueyjiRWtCz3hRDiF4QQf232/t8SQnz2Pq9j0xYwR0O4LVaPr8Sc7ga8+MZRH2R0+Ng2HssDo7X++Qf49gr4q3emLQghPv+wpS184Y0tLi7HnO2FAAgheP6JZf7NKxvUjUbJu3IDPBTYnZblrhFC/A0hxOtCiBeBJ+/4/C8LIX529v5bQoj/fM/0KoT4biHErwkhviWE+AvvvqZNW4C8qnnpSp/nZ0fDPZ6/tMwwLfna9eGcVnY0saJluSuEEN8D/EngE8BPAN/3AQ9/R2v9CeALwC8DPwt8CvibH/Icj/EQpi185e0BaVnz/BPvEq3Zxy++Yetad2JFy3K3fAb451rrZJaG8C8+4LF7X/sa8JLWeqy13gJyIUTvvb5hn9IWjiUvXt5CScGnHl/6ts8vtXyeOd25Ve+yGKxoWQ6CfPbf5o739z7+jjrqw5628OIb23zifI9O4H7H1z5zaZmvvLPLNK/msLKjiRUty93y28DPCCHCWcH8p/bjorO0hb+PiY/+2/txzePEICn4w+vD7zga7vH8pWXKWvPym/1DXtnRxYqW5a6YFct/FfgqJin1y/t06SOZtnBY/M7lHbSGH/jIe4vW9z22iOdIe0S8A2t5sNw1WutfBH7xPT7/5+54/7E73v9lTCH+O752x+de5C67+08iL17eou07fNe53nt+PXAVzz62yIvWZHoLu9OyWOaE1povvLHNpx5fwlHv/6f4/KVlXt+YsDGyqQ9gRctimRtv7yRc201vte68H7etD/aICFa0LJa58YVZFM37FeH3eOZ0h6XY48XLVrTAipbFMjdefGOLs72QC8sfnAcvpeDTTyzzhTe2sUnDVrQslrlQ1Q1f/NYOzz+xjHF9fDDPX1pme5Lz2vr4EFZ3tLGiZbHMga9eGzLOKj7zPlaHd7NX97J1LStaFstcePGNbYSATz9+d6J1uhvy+Ep8qw72MGNFy2KZAy9e3uJjZ7osxN5df89nLq3w8ps7ZGV9gCs7+ljRslgOmUle8fvvDL4jiubDeP6JZbKy4Stv7x7Qyo4HVrQslkPmS9/aoWo0n/kQq8O7+dTjSzhSPPRHRCtaFssh8+LlbQJX8j2PLdzT97V8h+9+ZIEvPOT5Wla0LJZD5gtvbPHshSV8R93z9z5/aZlv3BjRnxYHsLLjgRUti+UQuTlM+dbW9J6Phns8f2kZrW8Pdn0YsaJlsRwit6fu3J9offxsl3bgPNR+LStaFssh8uIb2yy3fJ5aa9/X9ztK8v2PL/Hi5Ye3pceKlsVySDSN5ncub/P8E0t31brzfjx/aYXrg5Q3t6f7uLrjgxUti+WQeHV9xM60uDWQ9X75gb2Wnoe0rmVFy2I5JPbqUB8WRfNhPLoUc34xfGgjmK1oWSyHxIuXt/nIaou1bvDA13r+iRVjUq2bfVjZ8cKKlsVyCGRlzUtv9nn+iQc7Gu7xmUvLjPOKr14b7Mv1jhNWtCyWQ+DLb/UpquZDo5Xvlu9/fAkheCiPiFa0LJZD4MU3tnGV4NkLi/tyvV7k8fGz3YfSr2VFy2I5BL7wxjbf/cgCsb9/U/uev7TM718dMM7KfbvmccCKlsVywGxPcl65Odq3o+Eezz+xQt1ovnTl4Zo+bUXLYjlg9voEH9Sf9W6++9EeoaseutQHK1oWywHz4hvbdEOXP3K2u6/X9R3FcxcXH7q6lhUti+UA2Zsi/eknllDy/lt33o/nn1jmyvaU64N03699VLGiZbEcIN/amrA+yvbNn/VuPjM7cr74EB0RrWhZLAfIno9qv4vwe3xktcWptv9Q+bWsaFksB8iLb2zz6FLE+cXoQK4vhOD5J5b54rd2aJqHI6rGipbFckCUdcOXruw8cIP0h/H8pWX604JXbo4O9HmOCla0LJYD4vffGTAt6gM7Gu6xl4L62w9JXcuKlsVyQLz4xhZSwB+9yynS98updsBTa+2HxvpgRctiOSC+cHmb7zrfoxu6B/5czz+xzO++tUtanPzp01a0LJYDYJiWfPXq4L6n7twrz19apqgbXn7r5Lf0WNGyWA6Af/etHRq9/60778dzF5bwlHwo/FpWtCyWA+DFy1vEnuKTj/QO5flCT/E9jy48FH4tK1oWywHw4hvbfOriEq46vD+xz3xkmdfWx2yOs0N7znlgRcti2Weu9hPe2knueyDr/fKZWavQFy/vHOrzHjZWtCyWfeagW3fej4+e6bAQuSf+iGhFy2LZZ168vMVaJ+DxldahPq+Ugu9/YpkXL2+d6OnTVrQsln2kbjS/c3mH5y8tP9AU6fvlM08sszHKubw5OfTnPiysaFks+8jXrw8ZpuWhHw332KujneQjohUti2Uf2RtV/+lDMpW+m3MLEReX4xMdwWxFy2LZR77wxhZPn+6w3PLntobnLy3z0ptmzuJJxIqWxbJPJEXF7729O7ej4R6ffmKZpKj5yju7c13HQWFFy2LZJ156s09Z6wPPz/ow/ujjJo/+pKY+WNGyWPaJL7y+jefIfZsifb90ApfvOtflC5etaFkslg/gxctbPPvYIoGr5r0Unr+0wteuDRgmJ2/6tBUti2Uf2BhlvL4xOfTWnffjM5eWaTR88Vsnb7dlRcti2Qf26kfzrmft8YnzPVq+cyKPiFa0LJZ94MXL2yzFHs+c7sx7KQC4SvKpi0snshhvRctieUC01rx4eZvvf2IZeQBTpO+Xz1xa5p1+wts703kvZV+xomWxPCDf3BizNc7n7s96Nye1pceKlsXygLw4pyiaD+PicsyZbnDijohWtCyWB+QLb2zz+ErM6W4476V8G0IInr+0zBe/tU1Zn5yWHitaFssD8vKbfT5zSAMs7pUf/9hpRlnFr7z8zryXsm9Y0bJYHpC0rI+M1eHd/NCTK3zq4iJ/+/OvM0xPhtHUmfcCLJbjjiMFn3p8ad7LeE+EEPxnP/kMP/lfvcjP/f2XON0N5r2k9+W//bnvvavHWdGyWB6QP/G952j5R/dP6aNnuvzvfuwp/j+/f523d5J5L+eBESc5S9piOSTsH9H+cFcmN1vTslgsxworWhaL5VhxdA/iFsvx4ej07jwE2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrbCHeYnlAhBBfB7J5r+NDWAaOetxDoLX+2Ic9yIqWxfLgZFrru+tBmRNCiN89Dmu8m8fZ46HFYjlWWNGyWCzHCitaFsuD83fnvYC74MSs0TZMWyyWY4XdaVkslmOFFS2L5QEQQvwZIcQfCiG+JoT4ohDiu+a9pncjhPgxIcQ3hRCXhRB/fd7reTdCiPNCiN8UQrwihPiGEOIvf+Dj7fHQYrl/hBDfD7yqtd4VQvw48Ata6+fmva49hBAKeB34HHAN+DLwp7TWr8x1YXcghDgNnNZaf0UI0QZ+D/iZ91uj3WlZLA+A1vqLWuvd2YdfAs7Ncz3vwbPAZa31Fa11AfwK8NNzXtO3obW+qbX+yuz9MfAqcPb9Hm9Fy2LZP/488K/nvYh3cRa4esfH1/gAQZg3QojHgE8CL73fY6wj3mLZB4QQP4wRrefnvZbjihCiBfxT4K9orUfv9zi707JY7hEhxF8UQvzB7O2MEOLjwN8DflprvTPv9b2L68D5Oz4+N/vckUII4WIE6x9prf/ZBz7WFuItlvtHCPEI8BvAn9Vaf3He63k3QggHU4j/UYxYfRn401rrb8x1YXcghBDAPwD6Wuu/8qGPt6Jlsdw/Qoi/B/zPgLdnn6qOWmOyEOIngL8DKOCXtNa/ON8VfTtCiOeBLwBfA5rZp//3Wut/9Z6Pt6JlsViOE7amZbFYjhVWtCwWy7HCipbFYjlWWNGyWCzHCitaFovlWGFFy2I5xgghfkEI8ddm7/8tIcRn7/M6gRDiZSHEV2dJC39zf1e6f9g2HovlhKC1/vkH+PYc+BGt9WTmTn9RCPGvtdZf2qfl7Rt2p2WxHDOEEH9DCPG6EOJF4Mk7Pv/LQoifnb3/lhDiP5+1Gv2uEOK7hRC/JoT4lhDiL7z7mtowmX3ozt6OpInTipbFcowQQnwP8CeBTwA/AXzfBzz8Ha31JzBu818Gfhb4FPCeRz8hhBJC/AGwCXxea/2+SQvzxIqWxXK8+Azwz7XWySwJ4V98wGP3vvY14CWt9VhrvQXkQojeux+sta5nIncOeFYI8aGDU+eBFS2L5eSSz/7b3PH+3sfvW8/WWg+A3wR+7MBW9gBY0bJYjhe/DfyMECKcRRP/1H5cVAixsrf7EkKEmHjm1/bj2vuNvXtosRwjZjnqvwp8FVN7+vI+Xfo08A9mmfIS+Mda63+5T9feV2zKg8ViOVbY46HFYjlWWNGyWCzHCitaFovlWGFFy2KxHCusaFkslmOFFS2LxXKssKJlsViOFVa0LBbLseL/D9e1JFwR1+cdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "condition = posterior.sample((1,))\n", + "\n", + "_ = conditional_pairplot(\n", + " density=posterior,\n", + " condition=condition,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + " figsize=(5,5)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This plot looks completely different from the marginals obtained with `pairplot()`. As it can be seen on the diagonal plots, if all parameters but one are kept constant, the remaining parameter has to be tuned to a narrow region in parameter space. In addition, the upper diagonal plots show strong correlations: deviations in one parameter can be compensated through changes in another parameter." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can summarize these correlations in a conditional correlation matrix, which computes the Pearson correlation coefficient of each of these pairwise plots. This matrix (below) shows strong correlations between many parameters, which can be interpreted as potential compensation mechansims:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWV0lEQVR4nO3df6xcZZ3H8ffn3tvSdVGLFAEBKYZmlxp2QRvAkCgLiIUYWhG13awUF3JXg7urqEEkkRWXWHYTu/gTG6iAEsCtiDViWOTHglFYKlsoPxapoNJaqZRSJIXC7f3uH+eZMkzv3Dm3c+bcM+d+XuakM+c8M88ZoV+eX+f5KiIwM+u1gcm+ATObGhxszKwUDjZmVgoHGzMrhYONmZXCwcbMSuFgY1ZTklZI2iTpwTbXJekrktZJekDS25quLZH0WDqWFHE/DjZm9XUlMH+c6ycDc9IxDHwTQNIbgAuBo4GjgAsl7dXtzTjYmNVURNwJPDNOkQXA1ZG5G5gpaX/gPcAtEfFMRGwBbmH8oJXLULdfYGbF+fODZsSOF0dzld3+9MsPAS82nVoeEcsnUN0BwJNN79enc+3Od8XBxqxCRl8c5ZDTZuUq+3/LN74YEfN6fEuFcTfKrEoEAwPKdRRgA3BQ0/sD07l257viYGNWMVK+owCrgDPSrNQxwNaI2AjcDJwkaa80MHxSOtcVd6PMKkTAQEFNAEnXAscBsyStJ5thmgYQEZcBNwGnAOuAbcBH0rVnJH0RuDd91UURMd5Acy4ONmZVIhgcKqbZEhGLO1wP4Jw211YAKwq5kcTBxqxiVNPBDQcbswqRYKCgAZmqcbAxqxi3bMysFEUNEFeNg41ZhUhu2ZhZSQYHPWZjZj0muRtlZqUQKuZRhMpxsDGrErdszKwsHiA2s56TPEA8prR94PXAbOA3wAfTzl6t5XYAa9Pb30XEqd3Ua1Znde1GdfuzPgvcGhFzgFvT+7G8EBFHpMOBxqwNARpQrqPfdBtsFgBXpddXAQu7/D6zqS0NEOc5+k23Yzb7ps12AP4A7Num3AxJq4ERYGlE3DhWIUnDZLu8oyG9ffrMqTOkdOCWmZN9C6V56o3PTvYtlOr5jS8/HRH75C1f0+cwOwcbST8F9hvj0gXNbyIiJEWbrzk4IjZIegtwm6S1EfHr1kJps+blADP2mR6zF+bbi7UOvrTytMm+hdIsG1412bdQqru+8ORv85bNNs+qZ7TpGGwi4sR21yQ9JWn/iNiYUkBsavMdG9Kfj0u6AzgS2CXYmE15BW6eVTXd9vxWAY1seUuAH7YWSPuY7pFezwKOBR7usl6zWhJiQPmOftNtsFkKvFvSY8CJ6T2S5km6PJU5DFgt6X7gdrIxGwcbs7EUnF1B0nxJj6YUu7vMFktaJmlNOn4l6dmmazuarnXd9+1qBDYiNgMnjHF+NXB2ev1z4PBu6jGbKoocs5E0CHwdeDdZorl7Ja1q/o99RHyyqfw/kg1xNLwQEUcUcjM4lYtZ5QxoINeRw1HAuoh4PCJeAq4jW67SzmLg2gJ+wpgcbMyqRPm6UDlbP7nT6Eo6GDgEuK3p9AxJqyXdLWnhbv6inabOQhazPiBgaDB3G2BWWr/WMNFc380WASsjYkfTuVxLVvJysDGrkGzzrNzB5ukOub4nkkZ3ES05pIpesuJulFmlFNqNuheYI+kQSdPJAsous0qS/hLYC/hF07nCl6y4ZWNWJQXmjYqIEUkfJ8vTPQisiIiHJF0ErI6IRuBZBFyXMmQ2HAZ8S9IoWaOk6yUrDjZmFVL04woRcRNZTu/mc59vef8vY3yu8CUrDjZmVSIxODg42XfREw42ZhUypR/ENLNyOdiYWc9J5F0d3HccbMwqJf9Dlv3GwcasYvpx+4g8HGzMKkSCoSHPRplZjzU2z6ojBxuzKpFno8ysJA42ZtZzwlPfZlYGeerbzEogxNDgtMm+jZ4opL2WYwf3PSRdn67fI2l2EfWa1Y2AQQ3mOvpN18GmaQf3k4G5wGJJc1uKnQVsiYhDgWXAJd3Wa1ZLEgMDg7mOflNEyybPDu4LgKvS65XACVJNFxOYdWlAg7mOflPEmM1YO7gf3a5M2j1sK7A38HQB9ZvVhtBE9iDuK5UaIJY0DAwDDO3Zf5HbrFuSmDY4fbJvoyeKCDZ5dnBvlFkvaQh4PbC59YtSGorlADP2mR6t183qT7VdZ1PEr8qzg/sqYEl6fTpwW8vmymZGI5VLcQPEOWaKz5T0x6ac3mc3XVsi6bF0LGn97ER13bLJuYP7FcB3JK0DniELSGa2CxU2rZ0n13dyfUR8vOWzbwAuBOYBAfwyfXbL7t5PIWM2nXZwj4gXgQ8UUZdZnRX8uMLOmWIASY2Z4jwpWd4D3BIRz6TP3gLMp4tc4PXsHJr1rQmts5mVcnE3juGWL8ub6/v9kh6QtFJSY/w1d57wvCo1G2U21U1wNqpT+t08fgRcGxHbJf0D2Xq447v8zjG5ZWNWIY1uVJ4jh44zxRGxOSK2p7eXA2/P+9mJcrAxq5JiH1foOFMsaf+mt6cCj6TXNwMnpZzfewEnpXO7zd0os0pRYY8i5Jwp/idJpwIjZDPFZ6bPPiPpi2QBC+CixmDx7nKwMauQLCNmcR2OHDPF5wPnt/nsCmBFUffiYGNWKcWts6kaBxuzCpHEkJ+NMrNe8x7EZlYOicE+3BgrDwcbswrJWjYONmbWc/XdYsLBxqxChBga8ACxmfWahNyNMrMyeMzGzHpOiAEcbMysBG7ZmFnPqcAHMavGwcasUsSgPBtlZj0meZ2NmZWkrt2oQkJoN7lpzKyZnOu7nW5y05jZqwkv6htPN7lpzKyF19m0N1Z+maPHKPd+Se8EfgV8MiKebC2Q8t4MA+yjPfnSytMKuL3+cP7pN0z2LZTmNVvrOQBaBKnYZ6MkzQcuJduD+PKIWNpy/VzgbLI9iP8I/H1E/DZd2wGsTUV/FxGndnMvZf1T/xEwOyL+CriFLDfNLiJieUTMi4h5rxv4s5JuzaxKihuzaRriOBmYCyyWNLel2P8C89LfzZXAvzVdeyEijkhHV4EGigk23eSmMbNXycZs8hw57BziiIiXgMYQx04RcXtEbEtv7yb7+9sTRQSbbnLTmFkTkY3Z5DkoLv1uw1nAT5rez0jfe7ekhd3+tq7HbLrJTWNmrSa0qK+I9LtZrdLfAfOAdzWdPjgiNkh6C3CbpLUR8evdraOQRX3d5KYxs1cUPECcK4WupBOBC4B3NQ13EBEb0p+PS7oDOBLY7WDjaQGzihGDuY4c8gxxHAl8Czg1IjY1nd9L0h7p9SzgWLpczuLHFcwqpMinvnMOcfw7sCfwn5LglSnuw4BvSRola5QsHWOh7oQ42JhVSrFbTOQY4jixzed+Dhxe2I3gYGNWOarp6IaDjVnlaLJvoCccbMwqxHsQm1mJ3I0ysxLI3Sgz6z0hbwtqZuVwy8bMes4DxGZWGnejzKzHhAeIzawU8gpiMyuLWzZmVgK3bMysBMq7V03fcbAxq5BsgNgtGzMrgWejzKz3JPDjCmZWhrq2bAoJoZJWSNok6cE21yXpK5LWSXpA0tuKqNesfrJ1NnmOXN8mzZf0aPq799kxru8h6fp0/R5Js5uunZ/OPyrpPd3+sqLaa1cC88e5fjIwJx3DwDcLqtesdorKrpAz/e5ZwJaIOBRYBlySPjuXLBvDW8n+bn9DOdNwtlNIsImIO8mSz7WzALg6MncDM1uyZJoZrzyukOd/OXRMv5veX5VerwROUJZmYQFwXURsj4gngHXp+3ZbWSNRudKAShpupBJ9bvSFkm7NrEo0gaOQ9Ls7y0TECLAV2DvnZyekUgPEEbEcWA5w6NAbY5Jvx6x8kY58Cku/W4ayWja50oCaWaDId+SQ5+/dzjKShoDXA5tzfnZCygo2q4Az0qzUMcDWiNhYUt1m/WU059FZx/S76f2S9Pp04LaIiHR+UZqtOoRscud/uvhVxXSjJF0LHEfWh1wPXAhMA4iIy8gy8p1CNsi0DfhIEfWa1VK+VkuOr8mVfvcK4DuS1pFN8ixKn31I0vfI8nuPAOdExI5u7qeQYBMRiztcD+CcIuoyq7UAFThamSP97ovAB9p89mLg4qLupVIDxGbGRAaI+4qDjVnVFNSNqhoHG7OqqWescbAxq5Qg77R233GwMauaesYaBxuzynGwMbNSuBtlZmUocp1NlTjYmFXJxB7E7CsONmaVEjBaz2jjYGNWIaK+3ah6buNuZpXjlo1Z1Xg2ysx6zgPEZlYWeYDYzEpRz1jjYGNWKYGnvs2sDEHUdIDYU99mVVPchudtSXqDpFskPZb+3GuMMkdI+oWkh1La7A81XbtS0hOS1qTjiE51OtiYVUgExGjkOrr0WeDWiJgD3Jret9oGnBERjRS8/yFpZtP1z0TEEelY06lCd6PMKiZ2dNlsyWcBWUYUyNLv3gGc96r7iPhV0+vfS9oE7AM8uzsVFtKykbRC0iZJD7a5fpykrU1Nrs+PVc5symsMEOc5OqffHc++Tbnb/gDsO15hSUcB04FfN52+OHWvlknao1OFRbVsrgS+Blw9Tpm7IuK9BdVnVlMTGiAeN/2upJ8C+41x6YJX1RgRUvsnsiTtD3wHWBIRjWbX+WRBajpZyuzzgIvGu9mi8kbdKWl2Ed/V8NQbn2XZcGvyvvp6zdapM3y27flSugn9q6D/eyLixHbXJD0laf+I2JiCyaY25V4H/Bi4ICLubvruRqtou6RvA5/udD9l/hv+Dkn3S/qJpLeOVUDScKNJ+PI2/wtpU1NE5Dq61Jx2dwnww9YCKWXvD4CrI2Jly7X9058CFgJjDqE0KyvY3AccHBF/DXwVuHGsQhGxPCLmRcS8aa+ZOv+lN9tpYmM23VgKvFvSY8CJ6T2S5km6PJX5IPBO4MwxprivkbQWWAvMAv61U4WlzEZFxHNNr2+S9A1JsyLi6TLqN+snZcxGRcRm4IQxzq8Gzk6vvwt8t83nj59onaUEG0n7AU+lgaijyFpUm8uo26yfRBSyhqaSCgk2kq4lm7OfJWk9cCEwDSAiLgNOBz4maQR4AVgUdV2Tbdatmg5XFjUbtbjD9a+RTY2bWQd1/e+wVxCbVYmf+jazspT0uELpHGzMqiTwmI2ZlcGzUWZWFg8Qm1nPpf1s6sjBxqxqHGzMrNciwrNRZlYOBxsz6z2P2ZhZOdyNMrMyBDDqYGNmPRYRjL68Y7JvoyccbMwqxt0oM+u9GnejvNGvWaXky4bZ7YxVnvS7qdyOpv2HVzWdP0TSPZLWSbo+bY4+LgcbsyqJrBuV5+hSnvS7AC80pdg9ten8JcCyiDgU2AKc1alCBxuzCgkgRkdzHV1aQJZ2l/TnwrwfTOlbjgca6V1yfd5jNmZVEkHkn42aJWl10/vlEbE852fzpt+dkeoYAZZGxI3A3sCzETGSyqwHDuhUoYONWZXEhGajyki/e3BEbJD0FuC2lCtqa94bbNZ1sJF0EFmO733JWoHLI+LSljICLgVOAbYBZ0bEfd3WbVY/UUQXKfumAtLvRsSG9Ofjku4AjgS+D8yUNJRaNwcCGzrdTxFjNiPApyJiLnAMcI6kuS1lTgbmpGMY+GYB9ZrVTwA7It/RnTzpd/eStEd6PQs4Fng4pWG6nSxFU9vPt+o62ETExkYrJSL+BDzCrv23BWT5giMlJ5/ZyBVsZq9W0gBxnvS7hwGrJd1PFlyWRsTD6dp5wLmS1pGN4VzRqcJCx2wkzSZrZt3TcukA4Mmm940BpY2Y2U5l7WeTM/3uz4HD23z+ceCoidRZWLCRtCdZX+4Tzbm9J/gdw2TdLPZ4/WBRt2bWP4KJzEb1laLS704jCzTXRMQNYxTZABzU9H7MAaU0bbcc4LVvml7PTT3MxlXcAHHVdD1mk2aargAeiYgvtym2CjhDmWOArU1z/GbWUN4K4tIV0bI5FvgwsFbSmnTuc8CbASLiMuAmsmnvdWRT3x8poF6zGqpvy6brYBMRPwPUoUwA53Rbl1ntNaa+a8griM0qJCIYfWmkc8E+5GBjViUTe1yhrzjYmFVJQIw42JhZzzm7gpmVINyyMbNSRDjYmFkJAka3+3EFM+u1kh7EnAwONmYV4jEbMyuHx2zMrCzuRplZ77kbZWZliAhGt9fz2SgnqTOrkjRmk+foRp70u5L+pin17hpJL0pamK5dKemJpmtHdKrTwcasSiqUfjcibm+k3iXLgLkN+K+mIp9pSs27plOFDjZmVZLGbHrdsmHi6XdPB34SEdt2t0IHG7NKKacbRf70uw2LgGtbzl0s6QFJyxr5pcbjAWKzConRCT2uMG6u74LS75JyvB0O3Nx0+nyyIDWdLEnBecBF492sg41ZpUzocYVxc30XkX43+SDwg4h4uem7G62i7ZK+DXy60826G2VWJeWN2XRMv9tkMS1dqEZG25RdZSHwYKcK3bIxq5LyHldYCnxP0lnAb8laL0iaB3w0Is5O72eT5Xz775bPXyNpH7JkB2uAj3aqsOtgI+kg4GqyAaYg6zde2lLmOLLI+UQ6dUNEjNu/M5uKoqQ9iPOk303vf0OWKru13PETrbOIls0I8KmIuE/Sa4FfSrqlKQF5w10R8d4C6jOrNT8b1UYaKNqYXv9J0iNkkbA12JhZBxHByKg3z+oo9e+OBO4Z4/I7JN0P/B74dEQ8NMbnh4Hh9Pb5u77w5KNF3l9Os4CnJ6HeyTKVfu9k/daDJ1J4R7hlMy5JewLfBz4REc+1XL4PODginpd0CnAjMKf1O9IageWt58skafV404l1M5V+bz/81iAYrWmwKWTqW9I0skBzTUTc0Ho9Ip6LiOfT65uAaZJmFVG3Wd2MRuQ6+k0Rs1ECrgAeiYgvtymzH/BUWql4FFmQ29xt3WZ1VNeWTRHdqGOBDwNrJa1J5z4HvBkgIi4je4jrY5JGgBeARRGVDc2T2o2bBFPp91b+t0bUtxul6v6dN5t6Dh3aN778ug/lKrtgy1d/WfUxqGZeQWxWKeHZKDPrvYC+HPzNww9iNpE0X9KjktZJ2mXnsjqRtELSJkkdH6Drd5IOknS7pIclPSTpnyf7ntqKbIA4z9FvHGwSSYPA14GTgbnAYklzJ/eueupKYP5k30RJGo/UzAWOAc6p7j/bqG2wcTfqFUcB6yLicQBJ15FtnVjLxy4i4s604rv2+umRmgA/rjAFHAA82fR+PXD0JN2L9UiHR2omXXiA2Kz/dXikphrCi/qmgg1kmwQ1HJjOWQ10eqSmKuo8G+Vg84p7gTmSDiELMouAv53cW7Ii5Hmkpjrqu4LYs1FJRIwAHyfbQf4R4HtjbYNRF5KuBX4B/IWk9Wl7yLpqPFJzfFMGx1Mm+6bGkrVsPBtVe+mJ9Jsm+z7KEBGLJ/seyhIRPyPbK7fyIoKXazob5ZaNWcWU0bKR9IG0wHE0bXLertyYC10lHSLpnnT+eknTO9XpYGNWMSXtZ/MgcBpwZ7sCHRa6XgIsi4hDgS1Ax264g41ZhURJK4gj4pGI6LTt7s6FrhHxEnAdsCANuB8PrEzl8uQK95iNWZWs59mbz40b8u5iOWO89LsFaLfQdW/g2TSp0ji/S7qXVg42ZhUSEYU9rzZeru+IGC8DZk842JjV1Hi5vnNqt9B1MzBT0lBq3eRaAOsxGzNrZ+dC1zTbtAhYlbb0vZ1su1/onCsccLAxm5IkvU/SeuAdwI8l3ZzOv0nSTdBxoet5wLmS1pGN4VzRsU7vQWxmZXDLxsxK4WBjZqVwsDGzUjjYmFkpHGzMrBQONmZWCgcbMyvF/wPZqxE8c4CsDAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "cond_coeff_mat = conditional_corrcoeff(\n", + " density=posterior,\n", + " condition=condition,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + ")\n", + "fig, ax = plt.subplots(1,1, figsize=(4,4))\n", + "im = plt.imshow(cond_coeff_mat, clim=[-1, 1], cmap='PiYG')\n", + "_ = fig.colorbar(im)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So far, we have investigated the conditional distribution only at a specific `condition` sampled from the posterior. In many applications, it makes sense to repeat the above analyses with a different `condition` (another sample from the posterior), which can be interpreted as slicing the posterior at a different location. Note that `conditional_corrcoeff()` can directly compute the matrix for several `conditions` and then outputs the average over them. This can be done by passing a batch of $N$ conditions as the `condition` argument." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sampling conditional distributions\n", + "\n", + "So far, we have demonstrated how one can plot 2D conditional distributions with `conditional_pairplot()` and how one can compute the pairwise conditional correlation coefficient with `conditional_corrcoeff()`. In some cases, it can be useful to keep a subset of parameters fixed and to vary **more than two** parameters. This can be done by sampling the conditonal posterior $p(\\theta_i | \\theta_{j \\neq i}, x_o)$. As of `sbi` `v0.18.0`, this functionality requires using the [sampler interface](https://www.mackelab.org/sbi/tutorial/11_sampler_interface/). In this tutorial, we demonstrate this functionality on a linear gaussian simulator with four parameters. We would like to fix the forth parameter to $\\theta_4=0.2$ and sample the first three parameters given that value, i.e. we want to sample $p(\\theta_1, \\theta_2, \\theta_3 | \\theta_4 = 0.2, x_o)$. For an application in neuroscience, see [Deistler, Gonçalves, Macke, 2021](https://www.biorxiv.org/content/10.1101/2021.07.30.454484v4.abstract)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, we will use SNPE, but the same also works for SNLE and SNRE. First, we define the prior and the simulator and train the deep neural density estimator:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4683b28ba0614e87b33854d207e30da2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Running 1000 simulations.: 0%| | 0/1000 [00:00" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from sbi.analysis import pairplot\n", + "\n", + "_ = pairplot(cond_samples, limits=[[-2, 2], [-2, 2], [-2, 2], [-2, 2]], figsize=(4, 4))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 713398b63a24c5033761fed3fae90dde61a1f1e9 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 21 Apr 2022 15:09:29 -0400 Subject: [PATCH 07/15] Trying to fix whitespace issue --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9d9fc26e5..bff20f8dc 100644 --- a/README.md +++ b/README.md @@ -117,3 +117,4 @@ If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.211 journal = {Journal of Open Source Software} } ``` + From 0d167662bd7803074f76969b145f08fce8bb04c1 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 21 Apr 2022 15:12:07 -0400 Subject: [PATCH 08/15] new_branch --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index bff20f8dc..9d9fc26e5 100644 --- a/README.md +++ b/README.md @@ -117,4 +117,3 @@ If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.211 journal = {Journal of Open Source Software} } ``` - From 1767ec89a0684a22a26a2a0eb46a429530e792bf Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 21 Apr 2022 15:18:54 -0400 Subject: [PATCH 09/15] Renormalize - fixes line endings --- README.md | 238 +- sbi/inference/snpe/snpe_a.py | 1718 +- sbi/inference/snpe/snpe_base.py | 1194 +- sbi/inference/snpe/snpe_c.py | 1250 +- tests/linearGaussian_snpe_test.py | 1166 +- tutorials/07_conditional_distributions.ipynb | 33176 ++++++++--------- 6 files changed, 19371 insertions(+), 19371 deletions(-) diff --git a/README.md b/README.md index 9d9fc26e5..96012b76c 100644 --- a/README.md +++ b/README.md @@ -1,119 +1,119 @@ -[![PyPI version](https://badge.fury.io/py/sbi.svg)](https://badge.fury.io/py/sbi) -[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/mackelab/sbi/blob/master/CONTRIBUTING.md) -[![Tests](https://github.com/mackelab/sbi/workflows/Tests/badge.svg?branch=main)](https://github.com/mackelab/sbi/actions) -[![codecov](https://codecov.io/gh/mackelab/sbi/branch/main/graph/badge.svg)](https://codecov.io/gh/mackelab/sbi) -[![GitHub license](https://img.shields.io/github/license/mackelab/sbi)](https://github.com/mackelab/sbi/blob/master/LICENSE.txt) -[![DOI](https://joss.theoj.org/papers/10.21105/joss.02505/status.svg)](https://doi.org/10.21105/joss.02505) - -## sbi: simulation-based inference -[Getting Started](https://www.mackelab.org/sbi/tutorial/00_getting_started/) | [Documentation](https://www.mackelab.org/sbi/) - -`sbi` is a PyTorch package for simulation-based inference. Simulation-based inference is -the process of finding parameters of a simulator from observations. - -`sbi` takes a Bayesian approach and returns a full posterior distribution -over the parameters, conditional on the observations. This posterior can be amortized (i.e. -useful for any observation) or focused (i.e. tailored to a particular observation), with different -computational trade-offs. - -`sbi` offers a simple interface for one-line posterior inference. - -```python -from sbi.inference import infer -# import your simulator, define your prior over the parameters -parameter_posterior = infer(simulator, prior, method='SNPE', num_simulations=100) -``` -See below for the available methods of inference, `SNPE`, `SNRE` and `SNLE`. - - -## Installation - -`sbi` requires Python 3.6 or higher. We recommend to use a [`conda`](https://docs.conda.io/en/latest/miniconda.html) virtual -environment ([Miniconda installation instructions](https://docs.conda.io/en/latest/miniconda.html])). If `conda` is installed on the system, an environment for -installing `sbi` can be created as follows: -```commandline -# Create an environment for sbi (indicate Python 3.6 or higher); activate it -$ conda create -n sbi_env python=3.7 && conda activate sbi_env -``` - -Independent of whether you are using `conda` or not, `sbi` can be installed using `pip`: -```commandline -$ pip install sbi -``` - -To test the installation, drop into a python prompt and run -```python -from sbi.examples.minimal import simple -posterior = simple() -print(posterior) -``` - -## Inference Algorithms - -The following algorithms are currently available: - -#### Sequential Neural Posterior Estimation (SNPE) - -* [`SNPE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_a.SNPE_A) from Papamakarios G and Murray I [_Fast ε-free Inference of Simulation Models with Bayesian Conditional Density Estimation_](https://proceedings.neurips.cc/paper/2016/hash/6aca97005c68f1206823815f66102863-Abstract.html) (NeurIPS 2016). - -* [`SNPE_C`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_c.SNPE_C) or `APT` from Greenberg D, Nonnenmacher M, and Macke J [_Automatic - Posterior Transformation for likelihood-free - inference_](https://arxiv.org/abs/1905.07488) (ICML 2019). - - -#### Sequential Neural Likelihood Estimation (SNLE) -* [`SNLE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snle.snle_a.SNLE_A) or just `SNL` from Papamakarios G, Sterrat DC and Murray I [_Sequential - Neural Likelihood_](https://arxiv.org/abs/1805.07226) (AISTATS 2019). - - -#### Sequential Neural Ratio Estimation (SNRE) - -* [`SNRE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_a.SNRE_A) or `AALR` from Hermans J, Begy V, and Louppe G. [_Likelihood-free Inference with Amortized Approximate Likelihood Ratios_](https://arxiv.org/abs/1903.04057) (ICML 2020). - -* [`SNRE_B`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_b.SNRE_B) or `SRE` from Durkan C, Murray I, and Papamakarios G. [_On Contrastive Learning for Likelihood-free Inference_](https://arxiv.org/abs/2002.03712) (ICML 2020). - -#### Sequential Neural Variational Inference (SNVI) - -* [`SNVI`](https://www.mackelab.org/sbi/reference/#sbi.inference.posteriors.vi_posterior) from Glöckler M, Deistler M, Macke J, [_Variational methods for simulation-based inference_](https://openreview.net/forum?id=kZ0UYdhqkNY) (ICLR 2022). - -## Feedback and Contributions - -We would like to hear how `sbi` is working for your inference problems as well as receive bug reports, pull requests and other feedback (see -[contribute](http://www.mackelab.org/sbi/contribute/)). - - -## Acknowledgements - -`sbi` is the successor (using PyTorch) of the -[`delfi`](https://github.com/mackelab/delfi) package. It was started as a fork of Conor -M. Durkan's `lfi`. `sbi` runs as a community project; development is coordinated at the -[mackelab](https://uni-tuebingen.de/en/research/core-research/cluster-of-excellence-machine-learning/research/research/cluster-research-groups/professorships/machine-learning-in-science/). See also [credits](https://github.com/mackelab/sbi/blob/master/docs/docs/credits.md). - - -## Support - -`sbi` has been supported by the German Federal Ministry of Education and Research (BMBF) through the project ADIMEM, FKZ 01IS18052 A-D). [ADIMEM](https://fit.uni-tuebingen.de/Project/Details?id=9199) is a collaborative project between the groups of Jakob Macke (Uni Tübingen), Philipp Berens (Uni Tübingen), Philipp Hennig (Uni Tübingen) and Marcel Oberlaender (caesar Bonn) which aims to develop inference methods for mechanistic models. - - -## License - -[Affero General Public License v3 (AGPLv3)](https://www.gnu.org/licenses/) - - -## Citation -If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.21105/joss.02505), in addition to the original research articles describing the specifc sbi-algorithm(s) you are using: - -``` -@article{tejero-cantero2020sbi, - doi = {10.21105/joss.02505}, - url = {https://doi.org/10.21105/joss.02505}, - year = {2020}, - publisher = {The Open Journal}, - volume = {5}, - number = {52}, - pages = {2505}, - author = {Alvaro Tejero-Cantero and Jan Boelts and Michael Deistler and Jan-Matthis Lueckmann and Conor Durkan and Pedro J. Gonçalves and David S. Greenberg and Jakob H. Macke}, - title = {sbi: A toolkit for simulation-based inference}, - journal = {Journal of Open Source Software} -} -``` +[![PyPI version](https://badge.fury.io/py/sbi.svg)](https://badge.fury.io/py/sbi) +[![Contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/mackelab/sbi/blob/master/CONTRIBUTING.md) +[![Tests](https://github.com/mackelab/sbi/workflows/Tests/badge.svg?branch=main)](https://github.com/mackelab/sbi/actions) +[![codecov](https://codecov.io/gh/mackelab/sbi/branch/main/graph/badge.svg)](https://codecov.io/gh/mackelab/sbi) +[![GitHub license](https://img.shields.io/github/license/mackelab/sbi)](https://github.com/mackelab/sbi/blob/master/LICENSE.txt) +[![DOI](https://joss.theoj.org/papers/10.21105/joss.02505/status.svg)](https://doi.org/10.21105/joss.02505) + +## sbi: simulation-based inference +[Getting Started](https://www.mackelab.org/sbi/tutorial/00_getting_started/) | [Documentation](https://www.mackelab.org/sbi/) + +`sbi` is a PyTorch package for simulation-based inference. Simulation-based inference is +the process of finding parameters of a simulator from observations. + +`sbi` takes a Bayesian approach and returns a full posterior distribution +over the parameters, conditional on the observations. This posterior can be amortized (i.e. +useful for any observation) or focused (i.e. tailored to a particular observation), with different +computational trade-offs. + +`sbi` offers a simple interface for one-line posterior inference. + +```python +from sbi.inference import infer +# import your simulator, define your prior over the parameters +parameter_posterior = infer(simulator, prior, method='SNPE', num_simulations=100) +``` +See below for the available methods of inference, `SNPE`, `SNRE` and `SNLE`. + + +## Installation + +`sbi` requires Python 3.6 or higher. We recommend to use a [`conda`](https://docs.conda.io/en/latest/miniconda.html) virtual +environment ([Miniconda installation instructions](https://docs.conda.io/en/latest/miniconda.html])). If `conda` is installed on the system, an environment for +installing `sbi` can be created as follows: +```commandline +# Create an environment for sbi (indicate Python 3.6 or higher); activate it +$ conda create -n sbi_env python=3.7 && conda activate sbi_env +``` + +Independent of whether you are using `conda` or not, `sbi` can be installed using `pip`: +```commandline +$ pip install sbi +``` + +To test the installation, drop into a python prompt and run +```python +from sbi.examples.minimal import simple +posterior = simple() +print(posterior) +``` + +## Inference Algorithms + +The following algorithms are currently available: + +#### Sequential Neural Posterior Estimation (SNPE) + +* [`SNPE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_a.SNPE_A) from Papamakarios G and Murray I [_Fast ε-free Inference of Simulation Models with Bayesian Conditional Density Estimation_](https://proceedings.neurips.cc/paper/2016/hash/6aca97005c68f1206823815f66102863-Abstract.html) (NeurIPS 2016). + +* [`SNPE_C`](https://www.mackelab.org/sbi/reference/#sbi.inference.snpe.snpe_c.SNPE_C) or `APT` from Greenberg D, Nonnenmacher M, and Macke J [_Automatic + Posterior Transformation for likelihood-free + inference_](https://arxiv.org/abs/1905.07488) (ICML 2019). + + +#### Sequential Neural Likelihood Estimation (SNLE) +* [`SNLE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snle.snle_a.SNLE_A) or just `SNL` from Papamakarios G, Sterrat DC and Murray I [_Sequential + Neural Likelihood_](https://arxiv.org/abs/1805.07226) (AISTATS 2019). + + +#### Sequential Neural Ratio Estimation (SNRE) + +* [`SNRE_A`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_a.SNRE_A) or `AALR` from Hermans J, Begy V, and Louppe G. [_Likelihood-free Inference with Amortized Approximate Likelihood Ratios_](https://arxiv.org/abs/1903.04057) (ICML 2020). + +* [`SNRE_B`](https://www.mackelab.org/sbi/reference/#sbi.inference.snre.snre_b.SNRE_B) or `SRE` from Durkan C, Murray I, and Papamakarios G. [_On Contrastive Learning for Likelihood-free Inference_](https://arxiv.org/abs/2002.03712) (ICML 2020). + +#### Sequential Neural Variational Inference (SNVI) + +* [`SNVI`](https://www.mackelab.org/sbi/reference/#sbi.inference.posteriors.vi_posterior) from Glöckler M, Deistler M, Macke J, [_Variational methods for simulation-based inference_](https://openreview.net/forum?id=kZ0UYdhqkNY) (ICLR 2022). + +## Feedback and Contributions + +We would like to hear how `sbi` is working for your inference problems as well as receive bug reports, pull requests and other feedback (see +[contribute](http://www.mackelab.org/sbi/contribute/)). + + +## Acknowledgements + +`sbi` is the successor (using PyTorch) of the +[`delfi`](https://github.com/mackelab/delfi) package. It was started as a fork of Conor +M. Durkan's `lfi`. `sbi` runs as a community project; development is coordinated at the +[mackelab](https://uni-tuebingen.de/en/research/core-research/cluster-of-excellence-machine-learning/research/research/cluster-research-groups/professorships/machine-learning-in-science/). See also [credits](https://github.com/mackelab/sbi/blob/master/docs/docs/credits.md). + + +## Support + +`sbi` has been supported by the German Federal Ministry of Education and Research (BMBF) through the project ADIMEM, FKZ 01IS18052 A-D). [ADIMEM](https://fit.uni-tuebingen.de/Project/Details?id=9199) is a collaborative project between the groups of Jakob Macke (Uni Tübingen), Philipp Berens (Uni Tübingen), Philipp Hennig (Uni Tübingen) and Marcel Oberlaender (caesar Bonn) which aims to develop inference methods for mechanistic models. + + +## License + +[Affero General Public License v3 (AGPLv3)](https://www.gnu.org/licenses/) + + +## Citation +If you use `sbi` consider citing the [sbi software paper](https://doi.org/10.21105/joss.02505), in addition to the original research articles describing the specifc sbi-algorithm(s) you are using: + +``` +@article{tejero-cantero2020sbi, + doi = {10.21105/joss.02505}, + url = {https://doi.org/10.21105/joss.02505}, + year = {2020}, + publisher = {The Open Journal}, + volume = {5}, + number = {52}, + pages = {2505}, + author = {Alvaro Tejero-Cantero and Jan Boelts and Michael Deistler and Jan-Matthis Lueckmann and Conor Durkan and Pedro J. Gonçalves and David S. Greenberg and Jakob H. Macke}, + title = {sbi: A toolkit for simulation-based inference}, + journal = {Journal of Open Source Software} +} +``` diff --git a/sbi/inference/snpe/snpe_a.py b/sbi/inference/snpe/snpe_a.py index 664e62fff..d480e6997 100644 --- a/sbi/inference/snpe/snpe_a.py +++ b/sbi/inference/snpe/snpe_a.py @@ -1,859 +1,859 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - -import warnings -from copy import deepcopy -from functools import partial -from typing import Any, Callable, Dict, Optional, Union - -import torch -import torch.nn as nn -from pyknos.mdn.mdn import MultivariateGaussianMDN -from pyknos.nflows import flows -from pyknos.nflows.transforms import CompositeTransform -from torch import Tensor -from torch.distributions import Distribution, MultivariateNormal - -import sbi.utils as utils -from sbi.inference.posteriors.direct_posterior import DirectPosterior -from sbi.inference.snpe.snpe_base import PosteriorEstimator -from sbi.types import TensorboardSummaryWriter, TorchModule -from sbi.utils import torchutils - - -class SNPE_A(PosteriorEstimator): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "mdn_snpe_a", - num_components: int = 10, - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[TensorboardSummaryWriter] = None, - show_progress_bars: bool = True, - ): - r"""SNPE-A [1]. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - - This class implements SNPE-A. SNPE-A trains across multiple rounds with a - maximum-likelihood-loss. This will make training converge to the proposal - posterior instead of the true posterior. To correct for this, SNPE-A applies a - post-hoc correction after training. This correction has to be performed - analytically. Thus, SNPE-A is limited to Gaussian distributions for all but the - last round. In the last round, SNPE-A can use a Mixture of Gaussians. - - Args: - prior: A probability distribution that expresses prior knowledge about the - parameters, e.g. which ranges are meaningful for them. Any - object with `.log_prob()`and `.sample()` (for example, a PyTorch - distribution) can be used. - density_estimator: If it is a string (only "mdn_snpe_a" is valid), use a - pre-configured mixture of densities network. Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. Note that until the last round only a - single (multivariate) Gaussian component is used for training (see - Algorithm 1 in [1]). In the last round, this component is replicated - `num_components` times, its parameters are perturbed with a very small - noise, and then the last training round is done with the expanded - Gaussian mixture as estimator for the proposal posterior. - num_components: Number of components of the mixture of Gaussians in the - last round. This overrides the `num_components` value passed to - `posterior_nn()`. - device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". - logging_level: Minimum severity of messages to log. One of the strings - INFO, WARNING, DEBUG, ERROR and CRITICAL. - summary_writer: A tensorboard `SummaryWriter` to control, among others, log - file location (default is `/logs`.) - show_progress_bars: Whether to show a progressbar during training. - """ - - # Catch invalid inputs. - if not ((density_estimator == "mdn_snpe_a") or callable(density_estimator)): - raise TypeError( - "The `density_estimator` passed to SNPE_A needs to be a " - "callable or the string 'mdn_snpe_a'!" - ) - - # `num_components` will be used to replicate the Gaussian in the last round. - self._num_components = num_components - self._ran_final_round = False - - # WARNING: sneaky trick ahead. We proxy the parent's `train` here, - # requiring the signature to have `num_atoms`, save it for use below, and - # continue. It's sneaky because we are using the object (self) as a namespace - # to pass arguments between functions, and that's implicit state management. - kwargs = utils.del_entries( - locals(), - entries=("self", "__class__", "num_components"), - ) - super().__init__(**kwargs) - - def train( - self, - final_round: bool = False, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - resume_training: bool = False, - force_first_round_loss: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - dataloader_kwargs: Optional[Dict] = None, - component_perturbation: float = 5e-3, - ) -> nn.Module: - r"""Return density estimator that approximates the proposal posterior. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - - Training is performed with maximum likelihood on samples from the latest round, - which leads the algorithm to converge to the proposal posterior. - - Args: - final_round: Whether we are in the last round of training or not. For all - but the last round, Algorithm 1 from [1] is executed. In last the - round, Algorithm 2 from [1] is executed once. - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - force_first_round_loss: If `True`, train with maximum likelihood, - regardless of the proposal distribution. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. Not supported for - SNPE-A. - show_train_summary: Whether to print the number of epochs and validation - loss and leakage after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - component_perturbation: The standard deviation applied to all weights and - biases when, in the last round, the Mixture of Gaussians is build from - a single Gaussian. This value can be problem-specific and also depends - on the number of mixture components. - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - - assert not retrain_from_scratch, """Retraining from scratch is not supported in SNPE-A yet. The reason for - this is that, if we reininitialized the density estimator, the z-scoring would - change, which would break the posthoc correction. This is a pure implementation - issue.""" - - kwargs = utils.del_entries( - locals(), - entries=("self", "__class__", "final_round", "component_perturbation"), - ) - - # SNPE-A always discards the prior samples. - kwargs["discard_prior_samples"] = True - - self._round = max(self._data_round_index) - - if final_round: - # If there is (will be) only one round, train with Algorithm 2 from [1]. - if self._round == 0: - self._build_neural_net = partial( - self._build_neural_net, num_components=self._num_components - ) - # Run Algorithm 2 from [1]. - elif not self._ran_final_round: - # Now switch to the specified number of components. This method will - # only be used if `retrain_from_scratch=True`. Otherwise, - # the MDN will be built from replicating the single-component net for - # `num_component` times (via `_expand_mog()`). - self._build_neural_net = partial( - self._build_neural_net, num_components=self._num_components - ) - - # Extend the MDN to the originally desired number of components. - self._expand_mog(eps=component_perturbation) - else: - warnings.warn( - "You have already run SNPE-A with `final_round=True`. Running it" - "again with this setting will not allow computing the posthoc" - "correction applied in SNPE-A. Thus, you will get an error when " - "calling `.build_posterior()` after training.", - UserWarning, - ) - else: - # Run Algorithm 1 from [1]. - # Wrap the function that builds the MDN such that we can make - # sure that there is only one component when running. - self._build_neural_net = partial(self._build_neural_net, num_components=1) - - if final_round: - self._ran_final_round = True - - return super().train(**kwargs) - - def correct_for_proposal( - self, - density_estimator: Optional[TorchModule] = None, - ) -> "SNPE_A_MDN": - r"""Build mixture of Gaussians that approximates the posterior. - - Returns a `SNPE_A_MDN` object, which applies the posthoc-correction required in - SNPE-A. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. - """ - if density_estimator is None: - density_estimator = deepcopy( - self._neural_net - ) # PosteriorEstimator.train() also returns a deepcopy, mimic this here - # If internal net is used device is defined. - device = self._device - else: - # Otherwise, infer it from the device of the net parameters. - device = str(next(density_estimator.parameters()).device) - - # Set proposal of the density estimator. - # This also evokes the z-scoring correction if necessary. - if ( - self._proposal_roundwise[-1] is self._prior - or self._proposal_roundwise[-1] is None - ): - proposal = self._prior - assert isinstance( - proposal, (MultivariateNormal, utils.BoxUniform) - ), """Prior must be `torch.distributions.MultivariateNormal` or `sbi.utils. - BoxUniform`""" - else: - assert isinstance( - self._proposal_roundwise[-1], DirectPosterior - ), """The proposal you passed to `append_simulations` is neither the prior - nor a `DirectPosterior`. SNPE-A currently only supports these scenarios. - """ - proposal = self._proposal_roundwise[-1] - - # Create the SNPE_A_MDN - wrapped_density_estimator = SNPE_A_MDN( - flow=density_estimator, proposal=proposal, prior=self._prior, device=device - ) - return wrapped_density_estimator - - def build_posterior( - self, - density_estimator: Optional[TorchModule] = None, - prior: Optional[Distribution] = None, - ) -> "DirectPosterior": - r"""Build posterior from the neural density estimator. - - This method first corrects the estimated density with `correct_for_proposal` - and then returns a `DirectPosterior`. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - prior: Prior distribution. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. - """ - if prior is None: - assert ( - self._prior is not None - ), """You did not pass a prior. You have to pass the prior either at - initialization `inference = SNPE_A(prior)` or to `.build_posterior - (prior=prior)`.""" - prior = self._prior - - wrapped_density_estimator = self.correct_for_proposal( - density_estimator=density_estimator - ) - self._posterior = DirectPosterior( - posterior_estimator=wrapped_density_estimator, - prior=prior, - ) - return deepcopy(self._posterior) - - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - ) -> Tensor: - """Return the log-probability of the proposal posterior. - - For SNPE-A this is the same as `self._neural_net.log_prob(theta, x)` in - `_loss()` to be found in `snpe_base.py`. - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - proposal: Proposal distribution. - - Returns: Log-probability of the proposal posterior. - """ - return self._neural_net.log_prob(theta, x) - - def _expand_mog(self, eps: float = 1e-5): - """ - Replicate a singe Gaussian trained with Algorithm 1 before continuing - with Algorithm 2. The weights and biases of the associated MDN layers - are repeated `num_components` times, slightly perturbed to break the - symmetry such that the gradients in the subsequent training are not - all identical. - - Args: - eps: Standard deviation for the random perturbation. - """ - assert isinstance(self._neural_net._distribution, MultivariateGaussianMDN) - - # Increase the number of components - self._neural_net._distribution._num_components = self._num_components - - # Expand the 1-dim Gaussian. - for name, param in self._neural_net.named_parameters(): - if any( - key in name for key in ["logits", "means", "unconstrained", "upper"] - ): - if "bias" in name: - param.data = param.data.repeat(self._num_components) - param.data.add_(torch.randn_like(param.data) * eps) - param.grad = None # let autograd construct a new gradient - elif "weight" in name: - param.data = param.data.repeat(self._num_components, 1) - param.data.add_(torch.randn_like(param.data) * eps) - param.grad = None # let autograd construct a new gradient - - -class SNPE_A_MDN(nn.Module): - """Generates a posthoc-corrected MDN which approximates the posterior. - - This class takes as input the density estimator (abbreviated with `_d` suffix, aka - the proposal posterior) and the proposal prior (abbreviated with `_pp` suffix) from - which the simulations were drawn. It uses the algorithm presented in SNPE-A [1] to - compute the approximate posterior (abbreviated with `_p` suffix) from the two. The - approximate posterior is a MoG. This class also implements log-prob calculation - sampling from the approximate posterior. It inherits from `nn.Module` since the - constructor of `DirectPosterior` expects the argument `neural_net` to be a - `nn.Module`. - - [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional - Density Estimation_, Papamakarios et al., NeurIPS 2016, - https://arxiv.org/abs/1605.06376. - """ - - def __init__( - self, - flow: flows.Flow, - proposal: Union["utils.BoxUniform", "MultivariateNormal", "DirectPosterior"], - prior: Distribution, - device: str, - ): - """Constructor. - - Args: - flow: The trained normalizing flow, passed when building the posterior. - proposal: The proposal distribution. - prior: The prior distribution. - """ - # Call nn.Module's constructor. - super().__init__() - - self._neural_net = flow - self._prior = prior - self._device = device - - # Set the proposal using the `default_x`. - if isinstance(proposal, (utils.BoxUniform, MultivariateNormal)): - self._apply_correction = False - else: - self._apply_correction = True - logits_pp, m_pp, prec_pp = proposal.posterior_estimator._posthoc_correction( - proposal.default_x - ) - self._logits_pp, self._m_pp, self._prec_pp = ( - logits_pp.detach(), - m_pp.detach(), - prec_pp.detach(), - ) - - # Take care of z-scoring, pre-compute and store prior terms. - self._set_state_for_mog_proposal() - - def log_prob(self, inputs: Tensor, context: Tensor) -> Tensor: - inputs, context = inputs.to(self._device), context.to(self._device) - - if not self._apply_correction: - return self._neural_net.log_prob(inputs, context) - else: - # When we want to compute the approx. posterior, a proposal prior \tilde{p} - # has already been observed. To analytically calculate the log-prob of the - # Gaussian, we first need to compute the mixture components. - - # Compute the mixture components of the proposal posterior. - logits_pp, m_pp, prec_pp = self._posthoc_correction(context) - - # z-score theta if it z-scoring had been requested. - theta = self._maybe_z_score_theta(inputs) - - # Compute the log_prob of theta under the product. - log_prob_proposal_posterior = utils.mog_log_prob( - theta, logits_pp, m_pp, prec_pp - ) - utils.assert_all_finite( - log_prob_proposal_posterior, "proposal posterior eval" - ) - return log_prob_proposal_posterior # \hat{p} from eq (3) in [1] - - def sample(self, num_samples: int, context: Tensor, batch_size: int = 1) -> Tensor: - context = context.to(self._device) - - if not self._apply_correction: - return self._neural_net.sample(num_samples, context, batch_size) - else: - # When we want to sample from the approx. posterior, a proposal prior - # \tilde{p} has already been observed. To analytically calculate the - # log-prob of the Gaussian, we first need to compute the mixture components. - return self._sample_approx_posterior_mog(num_samples, context, batch_size) - - def _sample_approx_posterior_mog( - self, num_samples, x: Tensor, batch_size: int - ) -> Tensor: - r"""Sample from the approximate posterior. - - Args: - num_samples: Desired number of samples. - x: Conditioning context for posterior $p(\theta|x)$. - batch_size: Batch size for sampling. - - Returns: - Samples from the approximate mixture of Gaussians posterior. - """ - - # Compute the mixture components of the posterior. - logits_p, m_p, prec_p = self._posthoc_correction(x) - - # Compute the precision factors which represent the upper triangular matrix - # of the cholesky decomposition of the prec_p. - prec_factors_p = torch.linalg.cholesky(prec_p, upper=True) - - assert logits_p.ndim == 2 - assert m_p.ndim == 3 - assert prec_p.ndim == 4 - assert prec_factors_p.ndim == 4 - - # Replicate to use batched sampling from pyknos. - if batch_size is not None and batch_size > 1: - logits_p = logits_p.repeat(batch_size, 1) - m_p = m_p.repeat(batch_size, 1, 1) - prec_factors_p = prec_factors_p.repeat(batch_size, 1, 1, 1) - - # Get (optionally z-scored) MoG samples. - theta = MultivariateGaussianMDN.sample_mog( - num_samples, logits_p, m_p, prec_factors_p - ) - - embedded_context = self._neural_net._embedding_net(x) - if embedded_context is not None: - # Merge the context dimension with sample dimension in order to - # apply the transform. - theta = torchutils.merge_leading_dims(theta, num_dims=2) - embedded_context = torchutils.repeat_rows( - embedded_context, num_reps=num_samples - ) - - theta, _ = self._neural_net._transform.inverse(theta, context=embedded_context) - - if embedded_context is not None: - # Split the context dimension from sample dimension. - theta = torchutils.split_leading_dim(theta, shape=[-1, num_samples]) - - return theta - - def _posthoc_correction(self, x: Tensor): - """ - Compute the mixture components of the posterior given the current density - estimator and the proposal. - - Args: - x: Conditioning context for posterior. - - Returns: - Mixture components of the posterior. - """ - - # Evaluate the density estimator. - encoded_x = self._neural_net._embedding_net(x) - dist = self._neural_net._distribution # defined to avoid black formatting. - logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) - - # The following if case is needed because, in the constructor, we call - # `_posthoc_correction` regardless of whether the `proposal` itself had a - # `proposal` or not. - if not self._apply_correction: - return norm_logits_d, m_d, prec_d - else: - logits_pp, m_pp, prec_pp = self._logits_pp, self._m_pp, self._prec_pp - - # Compute the MoG parameters of the posterior. - logits_p, m_p, prec_p, cov_p = self._proposal_posterior_transformation( - logits_pp, m_pp, prec_pp, norm_logits_d, m_d, prec_d - ) - return logits_p, m_p, prec_p - - def _proposal_posterior_transformation( - self, - logits_pp: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Transforms the proposal posterior (the MDN) into the posterior. - - The approximate posterior is: - $p(\theta|x) = 1/Z * q(\theta|x) * p(\theta) / prop(\theta)$ - In words: posterior = proposal posterior estimate * prior / proposal. - - Since the proposal posterior estimate and the proposal are MoG, and the - prior is either Gaussian or uniform, we can solve this in closed-form. - - This function implements Appendix C from [1], and is highly similar to - `SNPE_C._automatic_posterior_transformation()`. - - Args: - logits_pp: Component weight of each Gaussian of the proposal prior. - means_pp: Mean of each Gaussian of the proposal prior. - precisions_pp: Precision matrix of each Gaussian of the proposal prior. - logits_d: Component weight for each Gaussian of the density estimator. - means_d: Mean of each Gaussian of the density estimator. - precisions_d: Precision matrix of each Gaussian of the density estimator. - - Returns: (Component weight, mean, precision matrix, covariance matrix) of each - Gaussian of the approximate posterior. - """ - - precisions_post, covariances_post = self._precisions_posterior( - precisions_pp, precisions_d - ) - - means_post = self._means_posterior( - covariances_post, means_pp, precisions_pp, means_d, precisions_d - ) - - logits_post = SNPE_A_MDN._logits_posterior( - means_post, - precisions_post, - covariances_post, - logits_pp, - means_pp, - precisions_pp, - logits_d, - means_d, - precisions_d, - ) - - return logits_post, means_post, precisions_post, covariances_post - - def _set_state_for_mog_proposal(self) -> None: - """ - Set state variables of the SNPE_A_MDN instance every time `set_proposal()` - is called, i.e. every time a posterior is build using - `SNPE_A.build_posterior()`. - - This function is almost identical to `SNPE_C._set_state_for_mog_proposal()`. - - Three things are computed: - 1) Check if z-scoring was requested. To do so, we check if the `_transform` - argument of the net had been a `CompositeTransform`. See pyknos mdn.py. - 2) Define a (potentially standardized) prior. It's standardized if z-scoring - had been requested. - 3) Compute (Precision * mean) for the prior. This quantity is used at every - training step if the prior is Gaussian. - """ - - self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) - - self._set_maybe_z_scored_prior() - - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - self.prec_m_prod_prior = torch.mv( - self._maybe_z_scored_prior.precision_matrix, # type: ignore - self._maybe_z_scored_prior.loc, # type: ignore - ) - - def _set_maybe_z_scored_prior(self) -> None: - r""" - Compute and store potentially standardized prior (if z-scoring was requested). - - This function is highly similar to `SNPE_C._set_maybe_z_scored_prior()`. - - The proposal posterior is: - $p(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - - Let's denote z-scored theta by `a`: a = (theta - mean) / std - Then $p'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ - - The ' indicates that the evaluation occurs in standardized space. The constant - scaling factor has been absorbed into $Z_2$. - From the above equation, we see that we need to evaluate the prior **in - standardized space**. We build the standardized prior in this function. - - The standardize transform that is applied to the samples theta does not use - the exact prior mean and std (due to implementation issues). Hence, the z-scored - prior will not be exactly have mean=0 and std=1. - """ - if self.z_score_theta: - scale = self._neural_net._transform._transforms[0]._scale - shift = self._neural_net._transform._transforms[0]._shift - - # Following the definition of the linear transform in - # `standardizing_transform` in `sbiutils.py`: - # shift=-mean / std - # scale=1 / std - # Solving these equations for mean and std: - estim_prior_std = 1 / scale - estim_prior_mean = -shift * estim_prior_std - - # Compute the discrepancy of the true prior mean and std and the mean and - # std that was empirically estimated from samples. - # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) - # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean - # and std (estimated from samples and used to build standardize transform). - almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std - almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std - - if isinstance(self._prior, MultivariateNormal): - self._maybe_z_scored_prior = MultivariateNormal( - almost_zero_mean, torch.diag(almost_one_std) - ) - else: - range_ = torch.sqrt(almost_one_std * 3.0) - self._maybe_z_scored_prior = utils.BoxUniform( - almost_zero_mean - range_, almost_zero_mean + range_ - ) - else: - self._maybe_z_scored_prior = self._prior - - def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: - """Return potentially standardized theta if z-scoring was requested.""" - - if self.z_score_theta: - theta, _ = self._neural_net._transform(theta) - - return theta - - def _precisions_posterior(self, precisions_pp: Tensor, precisions_d: Tensor): - r"""Return the precisions and covariances of the MoG posterior. - - As described at the end of Appendix C in [1], it can happen that the - proposal's precision matrix is not positive definite. - - $S_k^\prime = ( S_k^{-1} - S_0^{-1} )^{-1}$ - (see eq (23) in Appendix C of [1]) - - Args: - precisions_pp: Precision matrices of the proposal prior. - precisions_d: Precision matrices of the density estimator. - - Returns: (Precisions, Covariances) of the MoG posterior. - """ - - num_comps_p = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Check if precision matrices are positive definite. - for batches in precisions_pp: - for pprior in batches: - eig_pprior = torch.linalg.eigvalsh(pprior, UPLO="U") - if not (eig_pprior > 0).all(): - raise AssertionError( - "The precision matrix of the proposal is not positive definite!" - ) - for batches in precisions_d: - for d in batches: - eig_d = torch.linalg.eigvalsh(d, UPLO="U") - if not (eig_d > 0).all(): - raise AssertionError( - "The precision matrix of the density estimator is not " - "positive definite!" - ) - - precisions_pp_rep = precisions_pp.repeat_interleave(num_comps_d, dim=1) - precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) - - precisions_p = precisions_d_rep - precisions_pp_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - precisions_p += self._maybe_z_scored_prior.precision_matrix - - # Check if precision matrix is positive definite. - for idx_batch, batches in enumerate(precisions_p): - for idx_comp, pp in enumerate(batches): - eig_pp = torch.symeig(pp, eigenvectors=False).eigenvalues - if not (eig_pp > 0).all(): - raise AssertionError( - "The precision matrix of a posterior is not positive " - "definite! This is a known issue for SNPE-A. Either try a " - "different parameter setting, e.g. a different number of " - "mixture components (when contracting SNPE-A), or a different " - "value for the parameter perturbation (when building the " - "posterior)." - ) - - covariances_p = torch.inverse(precisions_p) - return precisions_p, covariances_p - - def _means_posterior( - self, - covariances_p: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Return the means of the MoG posterior. - - $m_k^\prime = S_k^\prime ( S_k^{-1} m_k - S_0^{-1} m_0 )$ - (see eq (24) in Appendix C of [1]) - - Args: - covariances_post: Covariance matrices of the MoG posterior. - means_pp: Means of the proposal prior. - precisions_pp: Precision matrices of the proposal prior. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Means of the MoG posterior. - """ - - num_comps_pp = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute the products P_k * m_k and P_0 * m_0. - prec_m_prod_pp = utils.batched_mixture_mv(precisions_pp, means_pp) - prec_m_prod_d = utils.batched_mixture_mv(precisions_d, means_d) - - # Repeat them to allow for matrix operations: same trick as for the precisions. - prec_m_prod_pp_rep = prec_m_prod_pp.repeat_interleave(num_comps_d, dim=1) - prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_pp, 1) - - # Compute the means P_k^prime * (P_k * m_k - P_0 * m_0). - summed_cov_m_prod_rep = prec_m_prod_d_rep - prec_m_prod_pp_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - summed_cov_m_prod_rep += self.prec_m_prod_prior - - means_p = utils.batched_mixture_mv(covariances_p, summed_cov_m_prod_rep) - return means_p - - @staticmethod - def _logits_posterior( - means_post: Tensor, - precisions_post: Tensor, - covariances_post: Tensor, - logits_pp: Tensor, - means_pp: Tensor, - precisions_pp: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Return the component weights (i.e. logits) of the MoG posterior. - - $\alpha_k^\prime = \frac{ \alpha_k exp(-0.5 c_k) }{ \sum{j} \alpha_j exp(-0.5 - c_j) } $ - with - $c_k = logdet(S_k) - logdet(S_0) - logdet(S_k^\prime) + - + m_k^T P_k m_k - m_0^T P_0 m_0 - m_k^\prime^T P_k^\prime m_k^\prime$ - (see eqs. (25, 26) in Appendix C of [1]) - - Args: - means_post: Means of the posterior. - precisions_post: Precision matrices of the posterior. - covariances_post: Covariance matrices of the posterior. - logits_pp: Component weights (i.e. logits) of the proposal prior. - means_pp: Means of the proposal prior. - precisions_pp: Precision matrices of the proposal prior. - logits_d: Component weights (i.e. logits) of the density estimator. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Component weights of the proposal posterior. - """ - - num_comps_pp = precisions_pp.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute the ratio of the logits similar to eq (10) in Appendix A.1 of [2] - logits_pp_rep = logits_pp.repeat_interleave(num_comps_d, dim=1) - logits_d_rep = logits_d.repeat(1, num_comps_pp) - logit_factors = logits_d_rep - logits_pp_rep - - # Compute the log-determinants - logdet_covariances_post = torch.logdet(covariances_post) - logdet_covariances_pp = -torch.logdet(precisions_pp) - logdet_covariances_d = -torch.logdet(precisions_d) - - # Repeat the proposal and density estimator terms such that there are LK terms. - # Same trick as has been used above. - logdet_covariances_pp_rep = logdet_covariances_pp.repeat_interleave( - num_comps_d, dim=1 - ) - logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_pp) - - log_sqrt_det_ratio = 0.5 * ( # similar to eq (14) in Appendix A.1 of [2] - logdet_covariances_post - + logdet_covariances_pp_rep - - logdet_covariances_d_rep - ) - - # Compute for proposal, density estimator, and proposal posterior: - exponent_pp = utils.batched_mixture_vmv( - precisions_pp, means_pp # m_0 in eq (26) in Appendix C of [1] - ) - exponent_d = utils.batched_mixture_vmv( - precisions_d, means_d # m_k in eq (26) in Appendix C of [1] - ) - exponent_post = utils.batched_mixture_vmv( - precisions_post, means_post # m_k^\prime in eq (26) in Appendix C of [1] - ) - - # Extend proposal and density estimator exponents to get LK terms. - exponent_pp_rep = exponent_pp.repeat_interleave(num_comps_d, dim=1) - exponent_d_rep = exponent_d.repeat(1, num_comps_pp) - exponent = -0.5 * ( - exponent_d_rep - exponent_pp_rep - exponent_post # eq (26) in [1] - ) - - logits_post = logit_factors + log_sqrt_det_ratio + exponent - return logits_post +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + +import warnings +from copy import deepcopy +from functools import partial +from typing import Any, Callable, Dict, Optional, Union + +import torch +import torch.nn as nn +from pyknos.mdn.mdn import MultivariateGaussianMDN +from pyknos.nflows import flows +from pyknos.nflows.transforms import CompositeTransform +from torch import Tensor +from torch.distributions import Distribution, MultivariateNormal + +import sbi.utils as utils +from sbi.inference.posteriors.direct_posterior import DirectPosterior +from sbi.inference.snpe.snpe_base import PosteriorEstimator +from sbi.types import TensorboardSummaryWriter, TorchModule +from sbi.utils import torchutils + + +class SNPE_A(PosteriorEstimator): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "mdn_snpe_a", + num_components: int = 10, + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[TensorboardSummaryWriter] = None, + show_progress_bars: bool = True, + ): + r"""SNPE-A [1]. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + + This class implements SNPE-A. SNPE-A trains across multiple rounds with a + maximum-likelihood-loss. This will make training converge to the proposal + posterior instead of the true posterior. To correct for this, SNPE-A applies a + post-hoc correction after training. This correction has to be performed + analytically. Thus, SNPE-A is limited to Gaussian distributions for all but the + last round. In the last round, SNPE-A can use a Mixture of Gaussians. + + Args: + prior: A probability distribution that expresses prior knowledge about the + parameters, e.g. which ranges are meaningful for them. Any + object with `.log_prob()`and `.sample()` (for example, a PyTorch + distribution) can be used. + density_estimator: If it is a string (only "mdn_snpe_a" is valid), use a + pre-configured mixture of densities network. Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. Note that until the last round only a + single (multivariate) Gaussian component is used for training (see + Algorithm 1 in [1]). In the last round, this component is replicated + `num_components` times, its parameters are perturbed with a very small + noise, and then the last training round is done with the expanded + Gaussian mixture as estimator for the proposal posterior. + num_components: Number of components of the mixture of Gaussians in the + last round. This overrides the `num_components` value passed to + `posterior_nn()`. + device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". + logging_level: Minimum severity of messages to log. One of the strings + INFO, WARNING, DEBUG, ERROR and CRITICAL. + summary_writer: A tensorboard `SummaryWriter` to control, among others, log + file location (default is `/logs`.) + show_progress_bars: Whether to show a progressbar during training. + """ + + # Catch invalid inputs. + if not ((density_estimator == "mdn_snpe_a") or callable(density_estimator)): + raise TypeError( + "The `density_estimator` passed to SNPE_A needs to be a " + "callable or the string 'mdn_snpe_a'!" + ) + + # `num_components` will be used to replicate the Gaussian in the last round. + self._num_components = num_components + self._ran_final_round = False + + # WARNING: sneaky trick ahead. We proxy the parent's `train` here, + # requiring the signature to have `num_atoms`, save it for use below, and + # continue. It's sneaky because we are using the object (self) as a namespace + # to pass arguments between functions, and that's implicit state management. + kwargs = utils.del_entries( + locals(), + entries=("self", "__class__", "num_components"), + ) + super().__init__(**kwargs) + + def train( + self, + final_round: bool = False, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[Dict] = None, + component_perturbation: float = 5e-3, + ) -> nn.Module: + r"""Return density estimator that approximates the proposal posterior. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + + Training is performed with maximum likelihood on samples from the latest round, + which leads the algorithm to converge to the proposal posterior. + + Args: + final_round: Whether we are in the last round of training or not. For all + but the last round, Algorithm 1 from [1] is executed. In last the + round, Algorithm 2 from [1] is executed once. + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + force_first_round_loss: If `True`, train with maximum likelihood, + regardless of the proposal distribution. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. Not supported for + SNPE-A. + show_train_summary: Whether to print the number of epochs and validation + loss and leakage after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + component_perturbation: The standard deviation applied to all weights and + biases when, in the last round, the Mixture of Gaussians is build from + a single Gaussian. This value can be problem-specific and also depends + on the number of mixture components. + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + + assert not retrain_from_scratch, """Retraining from scratch is not supported in SNPE-A yet. The reason for + this is that, if we reininitialized the density estimator, the z-scoring would + change, which would break the posthoc correction. This is a pure implementation + issue.""" + + kwargs = utils.del_entries( + locals(), + entries=("self", "__class__", "final_round", "component_perturbation"), + ) + + # SNPE-A always discards the prior samples. + kwargs["discard_prior_samples"] = True + + self._round = max(self._data_round_index) + + if final_round: + # If there is (will be) only one round, train with Algorithm 2 from [1]. + if self._round == 0: + self._build_neural_net = partial( + self._build_neural_net, num_components=self._num_components + ) + # Run Algorithm 2 from [1]. + elif not self._ran_final_round: + # Now switch to the specified number of components. This method will + # only be used if `retrain_from_scratch=True`. Otherwise, + # the MDN will be built from replicating the single-component net for + # `num_component` times (via `_expand_mog()`). + self._build_neural_net = partial( + self._build_neural_net, num_components=self._num_components + ) + + # Extend the MDN to the originally desired number of components. + self._expand_mog(eps=component_perturbation) + else: + warnings.warn( + "You have already run SNPE-A with `final_round=True`. Running it" + "again with this setting will not allow computing the posthoc" + "correction applied in SNPE-A. Thus, you will get an error when " + "calling `.build_posterior()` after training.", + UserWarning, + ) + else: + # Run Algorithm 1 from [1]. + # Wrap the function that builds the MDN such that we can make + # sure that there is only one component when running. + self._build_neural_net = partial(self._build_neural_net, num_components=1) + + if final_round: + self._ran_final_round = True + + return super().train(**kwargs) + + def correct_for_proposal( + self, + density_estimator: Optional[TorchModule] = None, + ) -> "SNPE_A_MDN": + r"""Build mixture of Gaussians that approximates the posterior. + + Returns a `SNPE_A_MDN` object, which applies the posthoc-correction required in + SNPE-A. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. + """ + if density_estimator is None: + density_estimator = deepcopy( + self._neural_net + ) # PosteriorEstimator.train() also returns a deepcopy, mimic this here + # If internal net is used device is defined. + device = self._device + else: + # Otherwise, infer it from the device of the net parameters. + device = str(next(density_estimator.parameters()).device) + + # Set proposal of the density estimator. + # This also evokes the z-scoring correction if necessary. + if ( + self._proposal_roundwise[-1] is self._prior + or self._proposal_roundwise[-1] is None + ): + proposal = self._prior + assert isinstance( + proposal, (MultivariateNormal, utils.BoxUniform) + ), """Prior must be `torch.distributions.MultivariateNormal` or `sbi.utils. + BoxUniform`""" + else: + assert isinstance( + self._proposal_roundwise[-1], DirectPosterior + ), """The proposal you passed to `append_simulations` is neither the prior + nor a `DirectPosterior`. SNPE-A currently only supports these scenarios. + """ + proposal = self._proposal_roundwise[-1] + + # Create the SNPE_A_MDN + wrapped_density_estimator = SNPE_A_MDN( + flow=density_estimator, proposal=proposal, prior=self._prior, device=device + ) + return wrapped_density_estimator + + def build_posterior( + self, + density_estimator: Optional[TorchModule] = None, + prior: Optional[Distribution] = None, + ) -> "DirectPosterior": + r"""Build posterior from the neural density estimator. + + This method first corrects the estimated density with `correct_for_proposal` + and then returns a `DirectPosterior`. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + prior: Prior distribution. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods. + """ + if prior is None: + assert ( + self._prior is not None + ), """You did not pass a prior. You have to pass the prior either at + initialization `inference = SNPE_A(prior)` or to `.build_posterior + (prior=prior)`.""" + prior = self._prior + + wrapped_density_estimator = self.correct_for_proposal( + density_estimator=density_estimator + ) + self._posterior = DirectPosterior( + posterior_estimator=wrapped_density_estimator, + prior=prior, + ) + return deepcopy(self._posterior) + + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + ) -> Tensor: + """Return the log-probability of the proposal posterior. + + For SNPE-A this is the same as `self._neural_net.log_prob(theta, x)` in + `_loss()` to be found in `snpe_base.py`. + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + proposal: Proposal distribution. + + Returns: Log-probability of the proposal posterior. + """ + return self._neural_net.log_prob(theta, x) + + def _expand_mog(self, eps: float = 1e-5): + """ + Replicate a singe Gaussian trained with Algorithm 1 before continuing + with Algorithm 2. The weights and biases of the associated MDN layers + are repeated `num_components` times, slightly perturbed to break the + symmetry such that the gradients in the subsequent training are not + all identical. + + Args: + eps: Standard deviation for the random perturbation. + """ + assert isinstance(self._neural_net._distribution, MultivariateGaussianMDN) + + # Increase the number of components + self._neural_net._distribution._num_components = self._num_components + + # Expand the 1-dim Gaussian. + for name, param in self._neural_net.named_parameters(): + if any( + key in name for key in ["logits", "means", "unconstrained", "upper"] + ): + if "bias" in name: + param.data = param.data.repeat(self._num_components) + param.data.add_(torch.randn_like(param.data) * eps) + param.grad = None # let autograd construct a new gradient + elif "weight" in name: + param.data = param.data.repeat(self._num_components, 1) + param.data.add_(torch.randn_like(param.data) * eps) + param.grad = None # let autograd construct a new gradient + + +class SNPE_A_MDN(nn.Module): + """Generates a posthoc-corrected MDN which approximates the posterior. + + This class takes as input the density estimator (abbreviated with `_d` suffix, aka + the proposal posterior) and the proposal prior (abbreviated with `_pp` suffix) from + which the simulations were drawn. It uses the algorithm presented in SNPE-A [1] to + compute the approximate posterior (abbreviated with `_p` suffix) from the two. The + approximate posterior is a MoG. This class also implements log-prob calculation + sampling from the approximate posterior. It inherits from `nn.Module` since the + constructor of `DirectPosterior` expects the argument `neural_net` to be a + `nn.Module`. + + [1] _Fast epsilon-free Inference of Simulation Models with Bayesian Conditional + Density Estimation_, Papamakarios et al., NeurIPS 2016, + https://arxiv.org/abs/1605.06376. + """ + + def __init__( + self, + flow: flows.Flow, + proposal: Union["utils.BoxUniform", "MultivariateNormal", "DirectPosterior"], + prior: Distribution, + device: str, + ): + """Constructor. + + Args: + flow: The trained normalizing flow, passed when building the posterior. + proposal: The proposal distribution. + prior: The prior distribution. + """ + # Call nn.Module's constructor. + super().__init__() + + self._neural_net = flow + self._prior = prior + self._device = device + + # Set the proposal using the `default_x`. + if isinstance(proposal, (utils.BoxUniform, MultivariateNormal)): + self._apply_correction = False + else: + self._apply_correction = True + logits_pp, m_pp, prec_pp = proposal.posterior_estimator._posthoc_correction( + proposal.default_x + ) + self._logits_pp, self._m_pp, self._prec_pp = ( + logits_pp.detach(), + m_pp.detach(), + prec_pp.detach(), + ) + + # Take care of z-scoring, pre-compute and store prior terms. + self._set_state_for_mog_proposal() + + def log_prob(self, inputs: Tensor, context: Tensor) -> Tensor: + inputs, context = inputs.to(self._device), context.to(self._device) + + if not self._apply_correction: + return self._neural_net.log_prob(inputs, context) + else: + # When we want to compute the approx. posterior, a proposal prior \tilde{p} + # has already been observed. To analytically calculate the log-prob of the + # Gaussian, we first need to compute the mixture components. + + # Compute the mixture components of the proposal posterior. + logits_pp, m_pp, prec_pp = self._posthoc_correction(context) + + # z-score theta if it z-scoring had been requested. + theta = self._maybe_z_score_theta(inputs) + + # Compute the log_prob of theta under the product. + log_prob_proposal_posterior = utils.mog_log_prob( + theta, logits_pp, m_pp, prec_pp + ) + utils.assert_all_finite( + log_prob_proposal_posterior, "proposal posterior eval" + ) + return log_prob_proposal_posterior # \hat{p} from eq (3) in [1] + + def sample(self, num_samples: int, context: Tensor, batch_size: int = 1) -> Tensor: + context = context.to(self._device) + + if not self._apply_correction: + return self._neural_net.sample(num_samples, context, batch_size) + else: + # When we want to sample from the approx. posterior, a proposal prior + # \tilde{p} has already been observed. To analytically calculate the + # log-prob of the Gaussian, we first need to compute the mixture components. + return self._sample_approx_posterior_mog(num_samples, context, batch_size) + + def _sample_approx_posterior_mog( + self, num_samples, x: Tensor, batch_size: int + ) -> Tensor: + r"""Sample from the approximate posterior. + + Args: + num_samples: Desired number of samples. + x: Conditioning context for posterior $p(\theta|x)$. + batch_size: Batch size for sampling. + + Returns: + Samples from the approximate mixture of Gaussians posterior. + """ + + # Compute the mixture components of the posterior. + logits_p, m_p, prec_p = self._posthoc_correction(x) + + # Compute the precision factors which represent the upper triangular matrix + # of the cholesky decomposition of the prec_p. + prec_factors_p = torch.linalg.cholesky(prec_p, upper=True) + + assert logits_p.ndim == 2 + assert m_p.ndim == 3 + assert prec_p.ndim == 4 + assert prec_factors_p.ndim == 4 + + # Replicate to use batched sampling from pyknos. + if batch_size is not None and batch_size > 1: + logits_p = logits_p.repeat(batch_size, 1) + m_p = m_p.repeat(batch_size, 1, 1) + prec_factors_p = prec_factors_p.repeat(batch_size, 1, 1, 1) + + # Get (optionally z-scored) MoG samples. + theta = MultivariateGaussianMDN.sample_mog( + num_samples, logits_p, m_p, prec_factors_p + ) + + embedded_context = self._neural_net._embedding_net(x) + if embedded_context is not None: + # Merge the context dimension with sample dimension in order to + # apply the transform. + theta = torchutils.merge_leading_dims(theta, num_dims=2) + embedded_context = torchutils.repeat_rows( + embedded_context, num_reps=num_samples + ) + + theta, _ = self._neural_net._transform.inverse(theta, context=embedded_context) + + if embedded_context is not None: + # Split the context dimension from sample dimension. + theta = torchutils.split_leading_dim(theta, shape=[-1, num_samples]) + + return theta + + def _posthoc_correction(self, x: Tensor): + """ + Compute the mixture components of the posterior given the current density + estimator and the proposal. + + Args: + x: Conditioning context for posterior. + + Returns: + Mixture components of the posterior. + """ + + # Evaluate the density estimator. + encoded_x = self._neural_net._embedding_net(x) + dist = self._neural_net._distribution # defined to avoid black formatting. + logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) + + # The following if case is needed because, in the constructor, we call + # `_posthoc_correction` regardless of whether the `proposal` itself had a + # `proposal` or not. + if not self._apply_correction: + return norm_logits_d, m_d, prec_d + else: + logits_pp, m_pp, prec_pp = self._logits_pp, self._m_pp, self._prec_pp + + # Compute the MoG parameters of the posterior. + logits_p, m_p, prec_p, cov_p = self._proposal_posterior_transformation( + logits_pp, m_pp, prec_pp, norm_logits_d, m_d, prec_d + ) + return logits_p, m_p, prec_p + + def _proposal_posterior_transformation( + self, + logits_pp: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Transforms the proposal posterior (the MDN) into the posterior. + + The approximate posterior is: + $p(\theta|x) = 1/Z * q(\theta|x) * p(\theta) / prop(\theta)$ + In words: posterior = proposal posterior estimate * prior / proposal. + + Since the proposal posterior estimate and the proposal are MoG, and the + prior is either Gaussian or uniform, we can solve this in closed-form. + + This function implements Appendix C from [1], and is highly similar to + `SNPE_C._automatic_posterior_transformation()`. + + Args: + logits_pp: Component weight of each Gaussian of the proposal prior. + means_pp: Mean of each Gaussian of the proposal prior. + precisions_pp: Precision matrix of each Gaussian of the proposal prior. + logits_d: Component weight for each Gaussian of the density estimator. + means_d: Mean of each Gaussian of the density estimator. + precisions_d: Precision matrix of each Gaussian of the density estimator. + + Returns: (Component weight, mean, precision matrix, covariance matrix) of each + Gaussian of the approximate posterior. + """ + + precisions_post, covariances_post = self._precisions_posterior( + precisions_pp, precisions_d + ) + + means_post = self._means_posterior( + covariances_post, means_pp, precisions_pp, means_d, precisions_d + ) + + logits_post = SNPE_A_MDN._logits_posterior( + means_post, + precisions_post, + covariances_post, + logits_pp, + means_pp, + precisions_pp, + logits_d, + means_d, + precisions_d, + ) + + return logits_post, means_post, precisions_post, covariances_post + + def _set_state_for_mog_proposal(self) -> None: + """ + Set state variables of the SNPE_A_MDN instance every time `set_proposal()` + is called, i.e. every time a posterior is build using + `SNPE_A.build_posterior()`. + + This function is almost identical to `SNPE_C._set_state_for_mog_proposal()`. + + Three things are computed: + 1) Check if z-scoring was requested. To do so, we check if the `_transform` + argument of the net had been a `CompositeTransform`. See pyknos mdn.py. + 2) Define a (potentially standardized) prior. It's standardized if z-scoring + had been requested. + 3) Compute (Precision * mean) for the prior. This quantity is used at every + training step if the prior is Gaussian. + """ + + self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) + + self._set_maybe_z_scored_prior() + + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + self.prec_m_prod_prior = torch.mv( + self._maybe_z_scored_prior.precision_matrix, # type: ignore + self._maybe_z_scored_prior.loc, # type: ignore + ) + + def _set_maybe_z_scored_prior(self) -> None: + r""" + Compute and store potentially standardized prior (if z-scoring was requested). + + This function is highly similar to `SNPE_C._set_maybe_z_scored_prior()`. + + The proposal posterior is: + $p(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + + Let's denote z-scored theta by `a`: a = (theta - mean) / std + Then $p'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ + + The ' indicates that the evaluation occurs in standardized space. The constant + scaling factor has been absorbed into $Z_2$. + From the above equation, we see that we need to evaluate the prior **in + standardized space**. We build the standardized prior in this function. + + The standardize transform that is applied to the samples theta does not use + the exact prior mean and std (due to implementation issues). Hence, the z-scored + prior will not be exactly have mean=0 and std=1. + """ + if self.z_score_theta: + scale = self._neural_net._transform._transforms[0]._scale + shift = self._neural_net._transform._transforms[0]._shift + + # Following the definition of the linear transform in + # `standardizing_transform` in `sbiutils.py`: + # shift=-mean / std + # scale=1 / std + # Solving these equations for mean and std: + estim_prior_std = 1 / scale + estim_prior_mean = -shift * estim_prior_std + + # Compute the discrepancy of the true prior mean and std and the mean and + # std that was empirically estimated from samples. + # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) + # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean + # and std (estimated from samples and used to build standardize transform). + almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std + almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std + + if isinstance(self._prior, MultivariateNormal): + self._maybe_z_scored_prior = MultivariateNormal( + almost_zero_mean, torch.diag(almost_one_std) + ) + else: + range_ = torch.sqrt(almost_one_std * 3.0) + self._maybe_z_scored_prior = utils.BoxUniform( + almost_zero_mean - range_, almost_zero_mean + range_ + ) + else: + self._maybe_z_scored_prior = self._prior + + def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: + """Return potentially standardized theta if z-scoring was requested.""" + + if self.z_score_theta: + theta, _ = self._neural_net._transform(theta) + + return theta + + def _precisions_posterior(self, precisions_pp: Tensor, precisions_d: Tensor): + r"""Return the precisions and covariances of the MoG posterior. + + As described at the end of Appendix C in [1], it can happen that the + proposal's precision matrix is not positive definite. + + $S_k^\prime = ( S_k^{-1} - S_0^{-1} )^{-1}$ + (see eq (23) in Appendix C of [1]) + + Args: + precisions_pp: Precision matrices of the proposal prior. + precisions_d: Precision matrices of the density estimator. + + Returns: (Precisions, Covariances) of the MoG posterior. + """ + + num_comps_p = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Check if precision matrices are positive definite. + for batches in precisions_pp: + for pprior in batches: + eig_pprior = torch.linalg.eigvalsh(pprior, UPLO="U") + if not (eig_pprior > 0).all(): + raise AssertionError( + "The precision matrix of the proposal is not positive definite!" + ) + for batches in precisions_d: + for d in batches: + eig_d = torch.linalg.eigvalsh(d, UPLO="U") + if not (eig_d > 0).all(): + raise AssertionError( + "The precision matrix of the density estimator is not " + "positive definite!" + ) + + precisions_pp_rep = precisions_pp.repeat_interleave(num_comps_d, dim=1) + precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) + + precisions_p = precisions_d_rep - precisions_pp_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + precisions_p += self._maybe_z_scored_prior.precision_matrix + + # Check if precision matrix is positive definite. + for idx_batch, batches in enumerate(precisions_p): + for idx_comp, pp in enumerate(batches): + eig_pp = torch.symeig(pp, eigenvectors=False).eigenvalues + if not (eig_pp > 0).all(): + raise AssertionError( + "The precision matrix of a posterior is not positive " + "definite! This is a known issue for SNPE-A. Either try a " + "different parameter setting, e.g. a different number of " + "mixture components (when contracting SNPE-A), or a different " + "value for the parameter perturbation (when building the " + "posterior)." + ) + + covariances_p = torch.inverse(precisions_p) + return precisions_p, covariances_p + + def _means_posterior( + self, + covariances_p: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Return the means of the MoG posterior. + + $m_k^\prime = S_k^\prime ( S_k^{-1} m_k - S_0^{-1} m_0 )$ + (see eq (24) in Appendix C of [1]) + + Args: + covariances_post: Covariance matrices of the MoG posterior. + means_pp: Means of the proposal prior. + precisions_pp: Precision matrices of the proposal prior. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Means of the MoG posterior. + """ + + num_comps_pp = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute the products P_k * m_k and P_0 * m_0. + prec_m_prod_pp = utils.batched_mixture_mv(precisions_pp, means_pp) + prec_m_prod_d = utils.batched_mixture_mv(precisions_d, means_d) + + # Repeat them to allow for matrix operations: same trick as for the precisions. + prec_m_prod_pp_rep = prec_m_prod_pp.repeat_interleave(num_comps_d, dim=1) + prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_pp, 1) + + # Compute the means P_k^prime * (P_k * m_k - P_0 * m_0). + summed_cov_m_prod_rep = prec_m_prod_d_rep - prec_m_prod_pp_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + summed_cov_m_prod_rep += self.prec_m_prod_prior + + means_p = utils.batched_mixture_mv(covariances_p, summed_cov_m_prod_rep) + return means_p + + @staticmethod + def _logits_posterior( + means_post: Tensor, + precisions_post: Tensor, + covariances_post: Tensor, + logits_pp: Tensor, + means_pp: Tensor, + precisions_pp: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Return the component weights (i.e. logits) of the MoG posterior. + + $\alpha_k^\prime = \frac{ \alpha_k exp(-0.5 c_k) }{ \sum{j} \alpha_j exp(-0.5 + c_j) } $ + with + $c_k = logdet(S_k) - logdet(S_0) - logdet(S_k^\prime) + + + m_k^T P_k m_k - m_0^T P_0 m_0 - m_k^\prime^T P_k^\prime m_k^\prime$ + (see eqs. (25, 26) in Appendix C of [1]) + + Args: + means_post: Means of the posterior. + precisions_post: Precision matrices of the posterior. + covariances_post: Covariance matrices of the posterior. + logits_pp: Component weights (i.e. logits) of the proposal prior. + means_pp: Means of the proposal prior. + precisions_pp: Precision matrices of the proposal prior. + logits_d: Component weights (i.e. logits) of the density estimator. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Component weights of the proposal posterior. + """ + + num_comps_pp = precisions_pp.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute the ratio of the logits similar to eq (10) in Appendix A.1 of [2] + logits_pp_rep = logits_pp.repeat_interleave(num_comps_d, dim=1) + logits_d_rep = logits_d.repeat(1, num_comps_pp) + logit_factors = logits_d_rep - logits_pp_rep + + # Compute the log-determinants + logdet_covariances_post = torch.logdet(covariances_post) + logdet_covariances_pp = -torch.logdet(precisions_pp) + logdet_covariances_d = -torch.logdet(precisions_d) + + # Repeat the proposal and density estimator terms such that there are LK terms. + # Same trick as has been used above. + logdet_covariances_pp_rep = logdet_covariances_pp.repeat_interleave( + num_comps_d, dim=1 + ) + logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_pp) + + log_sqrt_det_ratio = 0.5 * ( # similar to eq (14) in Appendix A.1 of [2] + logdet_covariances_post + + logdet_covariances_pp_rep + - logdet_covariances_d_rep + ) + + # Compute for proposal, density estimator, and proposal posterior: + exponent_pp = utils.batched_mixture_vmv( + precisions_pp, means_pp # m_0 in eq (26) in Appendix C of [1] + ) + exponent_d = utils.batched_mixture_vmv( + precisions_d, means_d # m_k in eq (26) in Appendix C of [1] + ) + exponent_post = utils.batched_mixture_vmv( + precisions_post, means_post # m_k^\prime in eq (26) in Appendix C of [1] + ) + + # Extend proposal and density estimator exponents to get LK terms. + exponent_pp_rep = exponent_pp.repeat_interleave(num_comps_d, dim=1) + exponent_d_rep = exponent_d.repeat(1, num_comps_pp) + exponent = -0.5 * ( + exponent_d_rep - exponent_pp_rep - exponent_post # eq (26) in [1] + ) + + logits_post = logit_factors + log_sqrt_det_ratio + exponent + return logits_post diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 1a56f9c75..6d7b6ccef 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -1,597 +1,597 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . -import time -from abc import ABC, abstractmethod -from copy import deepcopy -from typing import Any, Callable, Dict, Optional, Union -from warnings import warn - -import torch -from torch import Tensor, nn, ones, optim -from torch.distributions import Distribution -from torch.nn.utils.clip_grad import clip_grad_norm_ -from torch.utils import data -from torch.utils.tensorboard.writer import SummaryWriter - -from sbi import utils as utils -from sbi.inference import NeuralInference, check_if_proposal_has_default_x -from sbi.inference.posteriors import ( - DirectPosterior, - MCMCPosterior, - RejectionPosterior, - VIPosterior, -) -from sbi.inference.posteriors.base_posterior import NeuralPosterior -from sbi.inference.potentials import posterior_estimator_based_potential -from sbi.utils import ( - RestrictedPrior, - check_estimator_arg, - test_posterior_net_for_multi_d_x, - validate_theta_and_x, - x_shape_from_simulation, - handle_invalid_x, - warn_if_zscoring_changes_data, - warn_on_invalid_x, - warn_on_invalid_x_for_snpec_leakage, -) -from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior - - -class PosteriorEstimator(NeuralInference, ABC): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "maf", - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[SummaryWriter] = None, - show_progress_bars: bool = True, - ): - """Base class for Sequential Neural Posterior Estimation methods. - - Args: - density_estimator: If it is a string, use a pre-configured network of the - provided type (one of nsf, maf, mdn, made). Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. - - See docstring of `NeuralInference` class for all other arguments. - """ - - super().__init__( - prior=prior, - device=device, - logging_level=logging_level, - summary_writer=summary_writer, - show_progress_bars=show_progress_bars, - ) - - # As detailed in the docstring, `density_estimator` is either a string or - # a callable. The function creating the neural network is attached to - # `_build_neural_net`. It will be called in the first round and receive - # thetas and xs as inputs, so that they can be used for shape inference and - # potentially for z-scoring. - check_estimator_arg(density_estimator) - if isinstance(density_estimator, str): - self._build_neural_net = utils.posterior_nn(model=density_estimator) - else: - self._build_neural_net = density_estimator - - self._proposal_roundwise = [] - self.use_non_atomic_loss = False - - # Extra SNPE-specific fields summary_writer. - self._summary.update({"rejection_sampling_acceptance_rates": []}) # type:ignore - - def append_simulations( - self, - theta: Tensor, - x: Tensor, - proposal: Optional[DirectPosterior] = None, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: bool = True, - return_self: bool = True, - data_device: str = None, - ) -> "PosteriorEstimator": - r"""Store parameters and simulation outputs to use them for later training. - - Data are stored as entries in lists for each type of variable (parameter/data). - - Stores $\theta$, $x$, prior_masks (indicating if simulations are coming from the - prior or not) and an index indicating which round the batch of simulations came - from. - - Args: - theta: Parameter sets. - x: Simulation outputs. - proposal: The distribution that the parameters $\theta$ were sampled from. - Pass `None` if the parameters were sampled from the prior. If not - `None`, it will trigger a different loss-function. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - warn_on_invalid: Whether to warn if data is invalid - warn_if_zscoring: Whether to test if z-scoring causes duplicates - return_self: Whether to return a instance of the class, allows chaining - with `.train()`. Setting `False` decreases memory overhead. - data_device: Where to store the data, default is on the same device where - the training is happening. If training a large dataset on a GPU with not - much VRAM can set to 'cpu' to store data on system memory instead. - - Returns: - NeuralInference object (returned so that this function is chainable). - """ - - # Add ability to specify device data is saved on - if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) - - - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) - warn_on_invalid_x_for_snpec_leakage( - num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round - ) - - x = x[is_valid_x] - theta = theta[is_valid_x] - - - self._check_proposal(proposal) - - if ( - proposal is None - or proposal is self._prior - or ( - isinstance(proposal, RestrictedPrior) and proposal._prior is self._prior - ) - ): - # The `_data_round_index` will later be used to infer if one should train - # with MLE loss or with atomic loss (see, in `train()`: - # self._round = max(self._data_round_index)) - self._data_round_index.append(0) - prior_masks = mask_sims_from_prior(0, theta.size(0)) - else: - if not self._data_round_index: - # This catches a pretty specific case: if, in the first round, one - # passes data that does not come from the prior. - self._data_round_index.append(1) - else: - self._data_round_index.append(max(self._data_round_index) + 1) - prior_masks = mask_sims_from_prior(1, theta.size(0)) - - - if self._dataset is None: - #If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) - else: - #Otherwise append to Dataset - self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) - - self._num_sims_per_round.append(theta.size(0)) - self._proposal_roundwise.append(proposal) - - if self._prior is None or isinstance(self._prior, ImproperEmpirical): - if proposal is not None: - raise ValueError( - "You had not passed a prior at initialization, but now you " - "passed a proposal. If you want to run multi-round SNPE, you have " - "to specify a prior (set the `.prior` argument or re-initialize " - "the object with a prior distribution). If the samples you passed " - "to `append_simulations()` were sampled from the prior, you can " - "run single-round inference with " - "`append_simulations(..., proposal=None)`." - ) - theta_prior = self.get_simulations()[0] - self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) - - #Add ability to not return self - if return_self: - return self - - def train( - self, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - resume_training: bool = False, - force_first_round_loss: bool = False, - discard_prior_samples: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - dataloader_kwargs: Optional[dict] = None, - ) -> nn.Module: - r"""Return density estimator that approximates the distribution $p(\theta|x)$. - - Args: - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - discard_prior_samples: Whether to discard samples simulated in round 1, i.e. - from the prior. Training may be sped up by ignoring such less targeted - samples. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. - show_train_summary: Whether to print the number of epochs and validation - loss after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - if self._round == 0 and self._neural_net is not None: - assert force_first_round_loss, ( - "You have already trained this neural network. After you had trained " - "the network, you again appended simulations with `append_simulations" - "(theta, x)`, but you did not provide a proposal. If the new " - "simulations are sampled from the prior, you can set " - "`.train(..., force_first_round_loss=True`). However, if the new " - "simulations were not sampled from the prior, you should pass the " - "proposal, i.e. `append_simulations(theta, x, proposal)`. If " - "your samples are not sampled from the prior and you do not pass a " - "proposal and you set `force_first_round_loss=True`, the result of " - "SNPE will not be the true posterior. Instead, it will be the proposal " - "posterior, which (usually) is more narrow than the true posterior." - ) - - # Calibration kernels proposed in Lueckmann, Gonçalves et al., 2017. - if calibration_kernel is None: - calibration_kernel = lambda x: ones([len(x)], device=self._device) - - # Starting index for the training set (1 = discard round-0 samples). - start_idx = int(discard_prior_samples and self._round > 0) - - # For non-atomic loss, we can not reuse samples from previous rounds as of now. - # SNPE-A can, by construction of the algorithm, only use samples from the last - # round. SNPE-A is the only algorithm that has an attribute `_ran_final_round`, - # so this is how we check for whether or not we are using SNPE-A. - if self.use_non_atomic_loss or hasattr(self, "_ran_final_round"): - start_idx = self._round - - # Set the proposal to the last proposal that was passed by the user. For - # atomic SNPE, it does not matter what the proposal is. For non-atomic - # SNPE, we only use the latest data that was passed, i.e. the one from the - # last proposal. - proposal = self._proposal_roundwise[-1] - - train_loader, val_loader = self.get_dataloaders( - start_idx, - training_batch_size, - validation_fraction, - resume_training, - dataloader_kwargs=dataloader_kwargs, - ) - # First round or if retraining from scratch: - # Call the `self._build_neural_net` with the rounds' thetas and xs as - # arguments, which will build the neural network. - # This is passed into NeuralPosterior, to create a neural posterior which - # can `sample()` and `log_prob()`. The network is accessible via `.net`. - if self._neural_net is None or retrain_from_scratch: - - #Get theta,x from dataset to initialize NN - test_theta = self._dataset.datasets[0].tensors[0][:100] - test_x = self._dataset.datasets[0].tensors[1][:100] - - self._neural_net = self._build_neural_net( - test_theta, test_x - ) - # If data on training device already move net as well. - if ( - not self._device == "cpu" - and f"{test_x.device.type}:{test_x.device.index}" == self._device - ): - self._neural_net.to(self._device) - - test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) - self._x_shape = x_shape_from_simulation(test_x) - - # Move entire net to device for training. - self._neural_net.to(self._device) - - if not resume_training: - self.optimizer = optim.Adam( - list(self._neural_net.parameters()), lr=learning_rate - ) - self.epoch, self._val_log_prob = 0, float("-Inf") - - while self.epoch <= max_num_epochs and not self._converged( - self.epoch, stop_after_epochs - ): - - # Train for a single epoch. - self._neural_net.train() - train_log_probs_sum = 0 - epoch_start_time = time.time() - for batch in train_loader: - self.optimizer.zero_grad() - # Get batches on current device. - theta_batch, x_batch, masks_batch = ( - batch[0].to(self._device), - batch[1].to(self._device), - batch[2].to(self._device), - ) - - train_losses = self._loss( - theta_batch, x_batch, masks_batch, proposal, calibration_kernel - ) - train_loss = torch.mean(train_losses) - train_log_probs_sum -= train_losses.sum().item() - - train_loss.backward() - if clip_max_norm is not None: - clip_grad_norm_( - self._neural_net.parameters(), max_norm=clip_max_norm - ) - self.optimizer.step() - - self.epoch += 1 - - train_log_prob_average = train_log_probs_sum / ( - len(train_loader) * train_loader.batch_size # type: ignore - ) - self._summary["train_log_probs"].append(train_log_prob_average) - - # Calculate validation performance. - self._neural_net.eval() - val_log_prob_sum = 0 - - with torch.no_grad(): - for batch in val_loader: - theta_batch, x_batch, masks_batch = ( - batch[0].to(self._device), - batch[1].to(self._device), - batch[2].to(self._device), - ) - # Take negative loss here to get validation log_prob. - val_losses = self._loss( - theta_batch, - x_batch, - masks_batch, - proposal, - calibration_kernel, - ) - val_log_prob_sum -= val_losses.sum().item() - - # Take mean over all validation samples. - self._val_log_prob = val_log_prob_sum / ( - len(val_loader) * val_loader.batch_size # type: ignore - ) - # Log validation log prob for every epoch. - self._summary["validation_log_probs"].append(self._val_log_prob) - self._summary["epoch_durations_sec"].append(time.time() - epoch_start_time) - - self._maybe_show_progress(self._show_progress_bars, self.epoch) - - self._report_convergence_at_end(self.epoch, stop_after_epochs, max_num_epochs) - - # Update summary. - self._summary["epochs"].append(self.epoch) - self._summary["best_validation_log_probs"].append(self._best_val_log_prob) - - # Update tensorboard and summary dict. - self._summarize(round_=self._round, x_o=None, theta_bank=None, x_bank=None) - - # Update description for progress bar. - if show_train_summary: - print(self._describe_round(self._round, self._summary)) - - # Avoid keeping the gradients in the resulting network, which can - # cause memory leakage when benchmarking. - self._neural_net.zero_grad(set_to_none=True) - - return deepcopy(self._neural_net) - - def build_posterior( - self, - density_estimator: Optional[nn.Module] = None, - prior: Optional[Distribution] = None, - sample_with: str = "rejection", - mcmc_method: str = "slice_np", - vi_method: str = "rKL", - mcmc_parameters: Dict[str, Any] = {}, - vi_parameters: Dict[str, Any] = {}, - rejection_sampling_parameters: Dict[str, Any] = {}, - ) -> Union[MCMCPosterior, RejectionPosterior, VIPosterior, DirectPosterior]: - r"""Build posterior from the neural density estimator. - - For SNPE, the posterior distribution that is returned here implements the - following functionality over the raw neural density estimator: - - correct the calculation of the log probability such that it compensates for - the leakage. - - reject samples that lie outside of the prior bounds. - - alternatively, if leakage is very high (which can happen for multi-round - SNPE), sample from the posterior with MCMC. - - Args: - density_estimator: The density estimator that the posterior is based on. - If `None`, use the latest neural density estimator that was trained. - prior: Prior distribution. - sample_with: Method to use for sampling from the posterior. Must be one of - [`mcmc` | `rejection` | `vi`]. - mcmc_method: Method used for MCMC sampling, one of `slice_np`, `slice`, - `hmc`, `nuts`. Currently defaults to `slice_np` for a custom numpy - implementation of slice sampling; select `hmc`, `nuts` or `slice` for - Pyro-based sampling. - vi_method: Method used for VI, one of [`rKL`, `fKL`, `IW`, `alpha`]. Note - some of the methods admit a `mode seeking` property (e.g. rKL) whereas - some admit a `mass covering` one (e.g fKL). - mcmc_parameters: Additional kwargs passed to `MCMCPosterior`. - vi_parameters: Additional kwargs passed to `VIPosterior`. - rejection_sampling_parameters: Additional kwargs passed to - `RejectionPosterior` or `DirectPosterior`. By default, - `DirectPosterior` is used. Only if `rejection_sampling_parameters` - contains `proposal`, a `RejectionPosterior` is instantiated. - - Returns: - Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods - (the returned log-probability is unnormalized). - """ - if prior is None: - assert self._prior is not None, ( - "You did not pass a prior. You have to pass the prior either at " - "initialization `inference = SNPE(prior)` or to " - "`.build_posterior(prior=prior)`." - ) - prior = self._prior - else: - utils.check_prior(prior) - - if density_estimator is None: - posterior_estimator = self._neural_net - # If internal net is used device is defined. - device = self._device - else: - posterior_estimator = density_estimator - # Otherwise, infer it from the device of the net parameters. - device = next(density_estimator.parameters()).device.type - - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator=posterior_estimator, prior=prior, x_o=None - ) - - if sample_with == "rejection": - if "proposal" in rejection_sampling_parameters.keys(): - self._posterior = RejectionPosterior( - potential_fn=potential_fn, - device=device, - x_shape=self._x_shape, - **rejection_sampling_parameters, - ) - else: - self._posterior = DirectPosterior( - posterior_estimator=posterior_estimator, - prior=prior, - x_shape=self._x_shape, - device=device, - ) - elif sample_with == "mcmc": - self._posterior = MCMCPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - proposal=prior, - method=mcmc_method, - device=device, - x_shape=self._x_shape, - **mcmc_parameters, - ) - elif sample_with == "vi": - self._posterior = VIPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - prior=prior, # type: ignore - vi_method=vi_method, - device=device, - x_shape=self._x_shape, - **vi_parameters, - ) - else: - raise NotImplementedError - - # Store models at end of each round. - self._model_bank.append(deepcopy(self._posterior)) - - return deepcopy(self._posterior) - - @abstractmethod - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - ) -> Tensor: - raise NotImplementedError - - def _loss( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: Optional[Any], - calibration_kernel: Callable, - ) -> Tensor: - """Return loss with proposal correction (`round_>0`) or without it (`round_=0`). - - The loss is the negative log prob. Irrespective of the round or SNPE method - (A, B, or C), it can be weighted with a calibration kernel. - - Returns: - Calibration kernel-weighted negative log prob. - """ - if self._round == 0: - # Use posterior log prob (without proposal correction) for first round. - log_prob = self._neural_net.log_prob(theta, x) - else: - log_prob = self._log_prob_proposal_posterior(theta, x, masks, proposal) - - return -(calibration_kernel(x) * log_prob) - - def _check_proposal(self, proposal): - """ - Check for validity of the provided proposal distribution. - - If the proposal is a `NeuralPosterior`, we check if the default_x is set. - If the proposal is **not** a `NeuralPosterior`, we warn since it is likely that - the user simply passed the prior, but this would still trigger atomic loss. - """ - if proposal is not None: - check_if_proposal_has_default_x(proposal) - - if isinstance(proposal, RestrictedPrior): - if proposal._prior is not self._prior: - warn( - "The proposal you passed is a `RestrictedPrior`, but the " - "proposal distribution it uses is not the prior (it can be " - "accessed via `RestrictedPrior._prior`). We do not " - "recommend to mix the `RestrictedPrior` with multi-round " - "SNPE." - ) - elif ( - not isinstance(proposal, NeuralPosterior) - and proposal is not self._prior - ): - warn( - "The proposal you passed is neither the prior nor a " - "`NeuralPosterior` object. If you are an expert user and did so " - "for research purposes, this is fine. If not, you might be doing " - "something wrong: feel free to create an issue on Github." - ) - elif self._round > 0: - raise ValueError( - "A proposal was passed but no prior was passed at initialisation. When " - "running multi-round inference, a prior needs to be specified upon " - "initialisation. Potential fix: setting the `._prior` attribute or " - "re-initialisation. If the samples passed to `append_simulations()` " - "were sampled from the prior, single-round inference can be performed " - "with `append_simulations(..., proprosal=None)`." - ) +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . +import time +from abc import ABC, abstractmethod +from copy import deepcopy +from typing import Any, Callable, Dict, Optional, Union +from warnings import warn + +import torch +from torch import Tensor, nn, ones, optim +from torch.distributions import Distribution +from torch.nn.utils.clip_grad import clip_grad_norm_ +from torch.utils import data +from torch.utils.tensorboard.writer import SummaryWriter + +from sbi import utils as utils +from sbi.inference import NeuralInference, check_if_proposal_has_default_x +from sbi.inference.posteriors import ( + DirectPosterior, + MCMCPosterior, + RejectionPosterior, + VIPosterior, +) +from sbi.inference.posteriors.base_posterior import NeuralPosterior +from sbi.inference.potentials import posterior_estimator_based_potential +from sbi.utils import ( + RestrictedPrior, + check_estimator_arg, + test_posterior_net_for_multi_d_x, + validate_theta_and_x, + x_shape_from_simulation, + handle_invalid_x, + warn_if_zscoring_changes_data, + warn_on_invalid_x, + warn_on_invalid_x_for_snpec_leakage, +) +from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior + + +class PosteriorEstimator(NeuralInference, ABC): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "maf", + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[SummaryWriter] = None, + show_progress_bars: bool = True, + ): + """Base class for Sequential Neural Posterior Estimation methods. + + Args: + density_estimator: If it is a string, use a pre-configured network of the + provided type (one of nsf, maf, mdn, made). Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. + + See docstring of `NeuralInference` class for all other arguments. + """ + + super().__init__( + prior=prior, + device=device, + logging_level=logging_level, + summary_writer=summary_writer, + show_progress_bars=show_progress_bars, + ) + + # As detailed in the docstring, `density_estimator` is either a string or + # a callable. The function creating the neural network is attached to + # `_build_neural_net`. It will be called in the first round and receive + # thetas and xs as inputs, so that they can be used for shape inference and + # potentially for z-scoring. + check_estimator_arg(density_estimator) + if isinstance(density_estimator, str): + self._build_neural_net = utils.posterior_nn(model=density_estimator) + else: + self._build_neural_net = density_estimator + + self._proposal_roundwise = [] + self.use_non_atomic_loss = False + + # Extra SNPE-specific fields summary_writer. + self._summary.update({"rejection_sampling_acceptance_rates": []}) # type:ignore + + def append_simulations( + self, + theta: Tensor, + x: Tensor, + proposal: Optional[DirectPosterior] = None, + exclude_invalid_x: bool = True, + warn_on_invalid: bool = True, + warn_if_zscoring: bool = True, + return_self: bool = True, + data_device: str = None, + ) -> "PosteriorEstimator": + r"""Store parameters and simulation outputs to use them for later training. + + Data are stored as entries in lists for each type of variable (parameter/data). + + Stores $\theta$, $x$, prior_masks (indicating if simulations are coming from the + prior or not) and an index indicating which round the batch of simulations came + from. + + Args: + theta: Parameter sets. + x: Simulation outputs. + proposal: The distribution that the parameters $\theta$ were sampled from. + Pass `None` if the parameters were sampled from the prior. If not + `None`, it will trigger a different loss-function. + exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` + during training. Expect errors, silent or explicit, when `False`. + warn_on_invalid: Whether to warn if data is invalid + warn_if_zscoring: Whether to test if z-scoring causes duplicates + return_self: Whether to return a instance of the class, allows chaining + with `.train()`. Setting `False` decreases memory overhead. + data_device: Where to store the data, default is on the same device where + the training is happening. If training a large dataset on a GPU with not + much VRAM can set to 'cpu' to store data on system memory instead. + + Returns: + NeuralInference object (returned so that this function is chainable). + """ + + # Add ability to specify device data is saved on + if data_device is None: data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) + + + is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) + + # Check for problematic z-scoring + if warn_if_zscoring: + warn_if_zscoring_changes_data(x[is_valid_x]) + if warn_on_invalid: + warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + warn_on_invalid_x_for_snpec_leakage( + num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round + ) + + x = x[is_valid_x] + theta = theta[is_valid_x] + + + self._check_proposal(proposal) + + if ( + proposal is None + or proposal is self._prior + or ( + isinstance(proposal, RestrictedPrior) and proposal._prior is self._prior + ) + ): + # The `_data_round_index` will later be used to infer if one should train + # with MLE loss or with atomic loss (see, in `train()`: + # self._round = max(self._data_round_index)) + self._data_round_index.append(0) + prior_masks = mask_sims_from_prior(0, theta.size(0)) + else: + if not self._data_round_index: + # This catches a pretty specific case: if, in the first round, one + # passes data that does not come from the prior. + self._data_round_index.append(1) + else: + self._data_round_index.append(max(self._data_round_index) + 1) + prior_masks = mask_sims_from_prior(1, theta.size(0)) + + + if self._dataset is None: + #If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + else: + #Otherwise append to Dataset + self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) + + self._num_sims_per_round.append(theta.size(0)) + self._proposal_roundwise.append(proposal) + + if self._prior is None or isinstance(self._prior, ImproperEmpirical): + if proposal is not None: + raise ValueError( + "You had not passed a prior at initialization, but now you " + "passed a proposal. If you want to run multi-round SNPE, you have " + "to specify a prior (set the `.prior` argument or re-initialize " + "the object with a prior distribution). If the samples you passed " + "to `append_simulations()` were sampled from the prior, you can " + "run single-round inference with " + "`append_simulations(..., proposal=None)`." + ) + theta_prior = self.get_simulations()[0] + self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) + + #Add ability to not return self + if return_self: + return self + + def train( + self, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + discard_prior_samples: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[dict] = None, + ) -> nn.Module: + r"""Return density estimator that approximates the distribution $p(\theta|x)$. + + Args: + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + discard_prior_samples: Whether to discard samples simulated in round 1, i.e. + from the prior. Training may be sped up by ignoring such less targeted + samples. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. + show_train_summary: Whether to print the number of epochs and validation + loss after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + if self._round == 0 and self._neural_net is not None: + assert force_first_round_loss, ( + "You have already trained this neural network. After you had trained " + "the network, you again appended simulations with `append_simulations" + "(theta, x)`, but you did not provide a proposal. If the new " + "simulations are sampled from the prior, you can set " + "`.train(..., force_first_round_loss=True`). However, if the new " + "simulations were not sampled from the prior, you should pass the " + "proposal, i.e. `append_simulations(theta, x, proposal)`. If " + "your samples are not sampled from the prior and you do not pass a " + "proposal and you set `force_first_round_loss=True`, the result of " + "SNPE will not be the true posterior. Instead, it will be the proposal " + "posterior, which (usually) is more narrow than the true posterior." + ) + + # Calibration kernels proposed in Lueckmann, Gonçalves et al., 2017. + if calibration_kernel is None: + calibration_kernel = lambda x: ones([len(x)], device=self._device) + + # Starting index for the training set (1 = discard round-0 samples). + start_idx = int(discard_prior_samples and self._round > 0) + + # For non-atomic loss, we can not reuse samples from previous rounds as of now. + # SNPE-A can, by construction of the algorithm, only use samples from the last + # round. SNPE-A is the only algorithm that has an attribute `_ran_final_round`, + # so this is how we check for whether or not we are using SNPE-A. + if self.use_non_atomic_loss or hasattr(self, "_ran_final_round"): + start_idx = self._round + + # Set the proposal to the last proposal that was passed by the user. For + # atomic SNPE, it does not matter what the proposal is. For non-atomic + # SNPE, we only use the latest data that was passed, i.e. the one from the + # last proposal. + proposal = self._proposal_roundwise[-1] + + train_loader, val_loader = self.get_dataloaders( + start_idx, + training_batch_size, + validation_fraction, + resume_training, + dataloader_kwargs=dataloader_kwargs, + ) + # First round or if retraining from scratch: + # Call the `self._build_neural_net` with the rounds' thetas and xs as + # arguments, which will build the neural network. + # This is passed into NeuralPosterior, to create a neural posterior which + # can `sample()` and `log_prob()`. The network is accessible via `.net`. + if self._neural_net is None or retrain_from_scratch: + + #Get theta,x from dataset to initialize NN + test_theta = self._dataset.datasets[0].tensors[0][:100] + test_x = self._dataset.datasets[0].tensors[1][:100] + + self._neural_net = self._build_neural_net( + test_theta, test_x + ) + # If data on training device already move net as well. + if ( + not self._device == "cpu" + and f"{test_x.device.type}:{test_x.device.index}" == self._device + ): + self._neural_net.to(self._device) + + test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) + self._x_shape = x_shape_from_simulation(test_x) + + # Move entire net to device for training. + self._neural_net.to(self._device) + + if not resume_training: + self.optimizer = optim.Adam( + list(self._neural_net.parameters()), lr=learning_rate + ) + self.epoch, self._val_log_prob = 0, float("-Inf") + + while self.epoch <= max_num_epochs and not self._converged( + self.epoch, stop_after_epochs + ): + + # Train for a single epoch. + self._neural_net.train() + train_log_probs_sum = 0 + epoch_start_time = time.time() + for batch in train_loader: + self.optimizer.zero_grad() + # Get batches on current device. + theta_batch, x_batch, masks_batch = ( + batch[0].to(self._device), + batch[1].to(self._device), + batch[2].to(self._device), + ) + + train_losses = self._loss( + theta_batch, x_batch, masks_batch, proposal, calibration_kernel + ) + train_loss = torch.mean(train_losses) + train_log_probs_sum -= train_losses.sum().item() + + train_loss.backward() + if clip_max_norm is not None: + clip_grad_norm_( + self._neural_net.parameters(), max_norm=clip_max_norm + ) + self.optimizer.step() + + self.epoch += 1 + + train_log_prob_average = train_log_probs_sum / ( + len(train_loader) * train_loader.batch_size # type: ignore + ) + self._summary["train_log_probs"].append(train_log_prob_average) + + # Calculate validation performance. + self._neural_net.eval() + val_log_prob_sum = 0 + + with torch.no_grad(): + for batch in val_loader: + theta_batch, x_batch, masks_batch = ( + batch[0].to(self._device), + batch[1].to(self._device), + batch[2].to(self._device), + ) + # Take negative loss here to get validation log_prob. + val_losses = self._loss( + theta_batch, + x_batch, + masks_batch, + proposal, + calibration_kernel, + ) + val_log_prob_sum -= val_losses.sum().item() + + # Take mean over all validation samples. + self._val_log_prob = val_log_prob_sum / ( + len(val_loader) * val_loader.batch_size # type: ignore + ) + # Log validation log prob for every epoch. + self._summary["validation_log_probs"].append(self._val_log_prob) + self._summary["epoch_durations_sec"].append(time.time() - epoch_start_time) + + self._maybe_show_progress(self._show_progress_bars, self.epoch) + + self._report_convergence_at_end(self.epoch, stop_after_epochs, max_num_epochs) + + # Update summary. + self._summary["epochs"].append(self.epoch) + self._summary["best_validation_log_probs"].append(self._best_val_log_prob) + + # Update tensorboard and summary dict. + self._summarize(round_=self._round, x_o=None, theta_bank=None, x_bank=None) + + # Update description for progress bar. + if show_train_summary: + print(self._describe_round(self._round, self._summary)) + + # Avoid keeping the gradients in the resulting network, which can + # cause memory leakage when benchmarking. + self._neural_net.zero_grad(set_to_none=True) + + return deepcopy(self._neural_net) + + def build_posterior( + self, + density_estimator: Optional[nn.Module] = None, + prior: Optional[Distribution] = None, + sample_with: str = "rejection", + mcmc_method: str = "slice_np", + vi_method: str = "rKL", + mcmc_parameters: Dict[str, Any] = {}, + vi_parameters: Dict[str, Any] = {}, + rejection_sampling_parameters: Dict[str, Any] = {}, + ) -> Union[MCMCPosterior, RejectionPosterior, VIPosterior, DirectPosterior]: + r"""Build posterior from the neural density estimator. + + For SNPE, the posterior distribution that is returned here implements the + following functionality over the raw neural density estimator: + - correct the calculation of the log probability such that it compensates for + the leakage. + - reject samples that lie outside of the prior bounds. + - alternatively, if leakage is very high (which can happen for multi-round + SNPE), sample from the posterior with MCMC. + + Args: + density_estimator: The density estimator that the posterior is based on. + If `None`, use the latest neural density estimator that was trained. + prior: Prior distribution. + sample_with: Method to use for sampling from the posterior. Must be one of + [`mcmc` | `rejection` | `vi`]. + mcmc_method: Method used for MCMC sampling, one of `slice_np`, `slice`, + `hmc`, `nuts`. Currently defaults to `slice_np` for a custom numpy + implementation of slice sampling; select `hmc`, `nuts` or `slice` for + Pyro-based sampling. + vi_method: Method used for VI, one of [`rKL`, `fKL`, `IW`, `alpha`]. Note + some of the methods admit a `mode seeking` property (e.g. rKL) whereas + some admit a `mass covering` one (e.g fKL). + mcmc_parameters: Additional kwargs passed to `MCMCPosterior`. + vi_parameters: Additional kwargs passed to `VIPosterior`. + rejection_sampling_parameters: Additional kwargs passed to + `RejectionPosterior` or `DirectPosterior`. By default, + `DirectPosterior` is used. Only if `rejection_sampling_parameters` + contains `proposal`, a `RejectionPosterior` is instantiated. + + Returns: + Posterior $p(\theta|x)$ with `.sample()` and `.log_prob()` methods + (the returned log-probability is unnormalized). + """ + if prior is None: + assert self._prior is not None, ( + "You did not pass a prior. You have to pass the prior either at " + "initialization `inference = SNPE(prior)` or to " + "`.build_posterior(prior=prior)`." + ) + prior = self._prior + else: + utils.check_prior(prior) + + if density_estimator is None: + posterior_estimator = self._neural_net + # If internal net is used device is defined. + device = self._device + else: + posterior_estimator = density_estimator + # Otherwise, infer it from the device of the net parameters. + device = next(density_estimator.parameters()).device.type + + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator=posterior_estimator, prior=prior, x_o=None + ) + + if sample_with == "rejection": + if "proposal" in rejection_sampling_parameters.keys(): + self._posterior = RejectionPosterior( + potential_fn=potential_fn, + device=device, + x_shape=self._x_shape, + **rejection_sampling_parameters, + ) + else: + self._posterior = DirectPosterior( + posterior_estimator=posterior_estimator, + prior=prior, + x_shape=self._x_shape, + device=device, + ) + elif sample_with == "mcmc": + self._posterior = MCMCPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + proposal=prior, + method=mcmc_method, + device=device, + x_shape=self._x_shape, + **mcmc_parameters, + ) + elif sample_with == "vi": + self._posterior = VIPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + prior=prior, # type: ignore + vi_method=vi_method, + device=device, + x_shape=self._x_shape, + **vi_parameters, + ) + else: + raise NotImplementedError + + # Store models at end of each round. + self._model_bank.append(deepcopy(self._posterior)) + + return deepcopy(self._posterior) + + @abstractmethod + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + ) -> Tensor: + raise NotImplementedError + + def _loss( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: Optional[Any], + calibration_kernel: Callable, + ) -> Tensor: + """Return loss with proposal correction (`round_>0`) or without it (`round_=0`). + + The loss is the negative log prob. Irrespective of the round or SNPE method + (A, B, or C), it can be weighted with a calibration kernel. + + Returns: + Calibration kernel-weighted negative log prob. + """ + if self._round == 0: + # Use posterior log prob (without proposal correction) for first round. + log_prob = self._neural_net.log_prob(theta, x) + else: + log_prob = self._log_prob_proposal_posterior(theta, x, masks, proposal) + + return -(calibration_kernel(x) * log_prob) + + def _check_proposal(self, proposal): + """ + Check for validity of the provided proposal distribution. + + If the proposal is a `NeuralPosterior`, we check if the default_x is set. + If the proposal is **not** a `NeuralPosterior`, we warn since it is likely that + the user simply passed the prior, but this would still trigger atomic loss. + """ + if proposal is not None: + check_if_proposal_has_default_x(proposal) + + if isinstance(proposal, RestrictedPrior): + if proposal._prior is not self._prior: + warn( + "The proposal you passed is a `RestrictedPrior`, but the " + "proposal distribution it uses is not the prior (it can be " + "accessed via `RestrictedPrior._prior`). We do not " + "recommend to mix the `RestrictedPrior` with multi-round " + "SNPE." + ) + elif ( + not isinstance(proposal, NeuralPosterior) + and proposal is not self._prior + ): + warn( + "The proposal you passed is neither the prior nor a " + "`NeuralPosterior` object. If you are an expert user and did so " + "for research purposes, this is fine. If not, you might be doing " + "something wrong: feel free to create an issue on Github." + ) + elif self._round > 0: + raise ValueError( + "A proposal was passed but no prior was passed at initialisation. When " + "running multi-round inference, a prior needs to be specified upon " + "initialisation. Potential fix: setting the `._prior` attribute or " + "re-initialisation. If the samples passed to `append_simulations()` " + "were sampled from the prior, single-round inference can be performed " + "with `append_simulations(..., proprosal=None)`." + ) diff --git a/sbi/inference/snpe/snpe_c.py b/sbi/inference/snpe/snpe_c.py index b9cd11984..34c53ab4b 100644 --- a/sbi/inference/snpe/snpe_c.py +++ b/sbi/inference/snpe/snpe_c.py @@ -1,625 +1,625 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - - -from typing import Callable, Dict, Optional, Union - -import torch -from pyknos.mdn.mdn import MultivariateGaussianMDN as mdn -from pyknos.nflows.transforms import CompositeTransform -from torch import Tensor, eye, nn, ones -from torch.distributions import Distribution, MultivariateNormal, Uniform - -from sbi import utils as utils -from sbi.inference.posteriors.direct_posterior import DirectPosterior -from sbi.inference.snpe.snpe_base import PosteriorEstimator -from sbi.types import TensorboardSummaryWriter -from sbi.utils import ( - batched_mixture_mv, - batched_mixture_vmv, - check_dist_class, - clamp_and_warn, - del_entries, - repeat_rows, -) - - -class SNPE_C(PosteriorEstimator): - def __init__( - self, - prior: Optional[Distribution] = None, - density_estimator: Union[str, Callable] = "maf", - device: str = "cpu", - logging_level: Union[int, str] = "WARNING", - summary_writer: Optional[TensorboardSummaryWriter] = None, - show_progress_bars: bool = True, - ): - r"""SNPE-C / APT [1]. - - [1] _Automatic Posterior Transformation for Likelihood-free Inference_, - Greenberg et al., ICML 2019, https://arxiv.org/abs/1905.07488. - - This class implements two loss variants of SNPE-C: the non-atomic and the atomic - version. The atomic loss of SNPE-C can be used for any density estimator, - i.e. also for normalizing flows. However, it suffers from leakage issues. On - the other hand, the non-atomic loss can only be used only if the proposal - distribution is a mixture of Gaussians, the density estimator is a mixture of - Gaussians, and the prior is either Gaussian or Uniform. It does not suffer from - leakage issues. At the beginning of each round, we print whether the non-atomic - or the atomic version is used. - - In this codebase, we will automatically switch to the non-atomic loss if the - following criteria are fulfilled:
- - proposal is a `DirectPosterior` with density_estimator `mdn`, as built - with `utils.sbi.posterior_nn()`.
- - the density estimator is a `mdn`, as built with - `utils.sbi.posterior_nn()`.
- - `isinstance(prior, MultivariateNormal)` (from `torch.distributions`) or - `isinstance(prior, sbi.utils.BoxUniform)` - - Note that custom implementations of any of these densities (or estimators) will - not trigger the non-atomic loss, and the algorithm will fall back onto using - the atomic loss. - - Args: - prior: A probability distribution that expresses prior knowledge about the - parameters, e.g. which ranges are meaningful for them. - density_estimator: If it is a string, use a pre-configured network of the - provided type (one of nsf, maf, mdn, made). Alternatively, a function - that builds a custom neural network can be provided. The function will - be called with the first batch of simulations (theta, x), which can - thus be used for shape inference and potentially for z-scoring. It - needs to return a PyTorch `nn.Module` implementing the density - estimator. The density estimator needs to provide the methods - `.log_prob` and `.sample()`. - device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". - logging_level: Minimum severity of messages to log. One of the strings - INFO, WARNING, DEBUG, ERROR and CRITICAL. - summary_writer: A tensorboard `SummaryWriter` to control, among others, log - file location (default is `/logs`.) - show_progress_bars: Whether to show a progressbar during training. - """ - - kwargs = del_entries(locals(), entries=("self", "__class__")) - super().__init__(**kwargs) - - def train( - self, - num_atoms: int = 10, - training_batch_size: int = 50, - learning_rate: float = 5e-4, - validation_fraction: float = 0.1, - stop_after_epochs: int = 20, - max_num_epochs: int = 2**31 - 1, - clip_max_norm: Optional[float] = 5.0, - calibration_kernel: Optional[Callable] = None, - resume_training: bool = False, - force_first_round_loss: bool = False, - discard_prior_samples: bool = False, - use_combined_loss: bool = False, - retrain_from_scratch: bool = False, - show_train_summary: bool = False, - dataloader_kwargs: Optional[Dict] = None, - ) -> nn.Module: - r"""Return density estimator that approximates the distribution $p(\theta|x)$. - - Args: - num_atoms: Number of atoms to use for classification. - training_batch_size: Training batch size. - learning_rate: Learning rate for Adam optimizer. - validation_fraction: The fraction of data to use for validation. - stop_after_epochs: The number of epochs to wait for improvement on the - validation set before terminating training. - max_num_epochs: Maximum number of epochs to run. If reached, we stop - training even when the validation loss is still decreasing. Otherwise, - we train until validation loss increases (see also `stop_after_epochs`). - clip_max_norm: Value at which to clip the total gradient norm in order to - prevent exploding gradients. Use None for no clipping. - calibration_kernel: A function to calibrate the loss with respect to the - simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. - resume_training: Can be used in case training time is limited, e.g. on a - cluster. If `True`, the split between train and validation set, the - optimizer, the number of epochs, and the best validation log-prob will - be restored from the last time `.train()` was called. - force_first_round_loss: If `True`, train with maximum likelihood, - i.e., potentially ignoring the correction for using a proposal - distribution different from the prior. - discard_prior_samples: Whether to discard samples simulated in round 1, i.e. - from the prior. Training may be sped up by ignoring such less targeted - samples. - use_combined_loss: Whether to train the neural net also on prior samples - using maximum likelihood in addition to training it on all samples using - atomic loss. The extra MLE loss helps prevent density leaking with - bounded priors. - retrain_from_scratch: Whether to retrain the conditional density - estimator for the posterior from scratch each round. - show_train_summary: Whether to print the number of epochs and validation - loss and leakage after the training. - dataloader_kwargs: Additional or updated kwargs to be passed to the training - and validation dataloaders (like, e.g., a collate_fn) - - Returns: - Density estimator that approximates the distribution $p(\theta|x)$. - """ - - # WARNING: sneaky trick ahead. We proxy the parent's `train` here, - # requiring the signature to have `num_atoms`, save it for use below, and - # continue. It's sneaky because we are using the object (self) as a namespace - # to pass arguments between functions, and that's implicit state management. - self._num_atoms = num_atoms - self._use_combined_loss = use_combined_loss - kwargs = del_entries( - locals(), entries=("self", "__class__", "num_atoms", "use_combined_loss") - ) - - self._round = max(self._data_round_index) - - if self._round > 0: - # Set the proposal to the last proposal that was passed by the user. For - # atomic SNPE, it does not matter what the proposal is. For non-atomic - # SNPE, we only use the latest data that was passed, i.e. the one from the - # last proposal. - proposal = self._proposal_roundwise[-1] - self.use_non_atomic_loss = ( - isinstance(proposal.posterior_estimator._distribution, mdn) - and isinstance(self._neural_net._distribution, mdn) - and check_dist_class( - self._prior, class_to_check=(Uniform, MultivariateNormal) - )[0] - ) - - algorithm = "non-atomic" if self.use_non_atomic_loss else "atomic" - print(f"Using SNPE-C with {algorithm} loss") - - if self.use_non_atomic_loss: - # Take care of z-scoring, pre-compute and store prior terms. - self._set_state_for_mog_proposal() - - return super().train(**kwargs) - - def _set_state_for_mog_proposal(self) -> None: - """Set state variables that are used at each training step of non-atomic SNPE-C. - - Three things are computed: - 1) Check if z-scoring was requested. To do so, we check if the `_transform` - argument of the net had been a `CompositeTransform`. See pyknos mdn.py. - 2) Define a (potentially standardized) prior. It's standardized if z-scoring - had been requested. - 3) Compute (Precision * mean) for the prior. This quantity is used at every - training step if the prior is Gaussian. - """ - - self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) - - self._set_maybe_z_scored_prior() - - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - self.prec_m_prod_prior = torch.mv( - self._maybe_z_scored_prior.precision_matrix, # type: ignore - self._maybe_z_scored_prior.loc, # type: ignore - ) - - def _set_maybe_z_scored_prior(self) -> None: - r"""Compute and store potentially standardized prior (if z-scoring was done). - - The proposal posterior is: - $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - - Let's denote z-scored theta by `a`: a = (theta - mean) / std - Then pp'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ - - The ' indicates that the evaluation occurs in standardized space. The constant - scaling factor has been absorbed into Z_2. - From the above equation, we see that we need to evaluate the prior **in - standardized space**. We build the standardized prior in this function. - - The standardize transform that is applied to the samples theta does not use - the exact prior mean and std (due to implementation issues). Hence, the z-scored - prior will not be exactly have mean=0 and std=1. - """ - - if self.z_score_theta: - scale = self._neural_net._transform._transforms[0]._scale - shift = self._neural_net._transform._transforms[0]._shift - - # Following the definintion of the linear transform in - # `standardizing_transform` in `sbiutils.py`: - # shift=-mean / std - # scale=1 / std - # Solving these equations for mean and std: - estim_prior_std = 1 / scale - estim_prior_mean = -shift * estim_prior_std - - # Compute the discrepancy of the true prior mean and std and the mean and - # std that was empirically estimated from samples. - # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) - # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean - # and std (estimated from samples and used to build standardize transform). - almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std - almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std - - if isinstance(self._prior, MultivariateNormal): - self._maybe_z_scored_prior = MultivariateNormal( - almost_zero_mean, torch.diag(almost_one_std) - ) - else: - range_ = torch.sqrt(almost_one_std * 3.0) - self._maybe_z_scored_prior = utils.BoxUniform( - almost_zero_mean - range_, almost_zero_mean + range_ - ) - else: - self._maybe_z_scored_prior = self._prior - - def _log_prob_proposal_posterior( - self, - theta: Tensor, - x: Tensor, - masks: Tensor, - proposal: DirectPosterior, - ) -> Tensor: - """Return the log-probability of the proposal posterior. - - If the proposal is a MoG, the density estimator is a MoG, and the prior is - either Gaussian or uniform, we use non-atomic loss. Else, use atomic loss (which - suffers from leakage). - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - proposal: Proposal distribution. - - Returns: Log-probability of the proposal posterior. - """ - - if self.use_non_atomic_loss: - return self._log_prob_proposal_posterior_mog(theta, x, proposal) - else: - return self._log_prob_proposal_posterior_atomic(theta, x, masks) - - def _log_prob_proposal_posterior_atomic( - self, theta: Tensor, x: Tensor, masks: Tensor - ): - """Return log probability of the proposal posterior for atomic proposals. - - We have two main options when evaluating the proposal posterior. - (1) Generate atoms from the proposal prior. - (2) Generate atoms from a more targeted distribution, such as the most - recent posterior. - If we choose the latter, it is likely beneficial not to do this in the first - round, since we would be sampling from a randomly-initialized neural density - estimator. - - Args: - theta: Batch of parameters θ. - x: Batch of data. - masks: Mask that is True for prior samples in the batch in order to train - them with prior loss. - - Returns: - Log-probability of the proposal posterior. - """ - - batch_size = theta.shape[0] - - num_atoms = int( - clamp_and_warn("num_atoms", self._num_atoms, min_val=2, max_val=batch_size) - ) - - # Each set of parameter atoms is evaluated using the same x, - # so we repeat rows of the data x, e.g. [1, 2] -> [1, 1, 2, 2] - repeated_x = repeat_rows(x, num_atoms) - - # To generate the full set of atoms for a given item in the batch, - # we sample without replacement num_atoms - 1 times from the rest - # of the theta in the batch. - probs = ones(batch_size, batch_size) * (1 - eye(batch_size)) / (batch_size - 1) - - choices = torch.multinomial(probs, num_samples=num_atoms - 1, replacement=False) - contrasting_theta = theta[choices] - - # We can now create our sets of atoms from the contrasting parameter sets - # we have generated. - atomic_theta = torch.cat((theta[:, None, :], contrasting_theta), dim=1).reshape( - batch_size * num_atoms, -1 - ) - - # Evaluate large batch giving (batch_size * num_atoms) log prob posterior evals. - log_prob_posterior = self._neural_net.log_prob(atomic_theta, repeated_x) - utils.assert_all_finite(log_prob_posterior, "posterior eval") - log_prob_posterior = log_prob_posterior.reshape(batch_size, num_atoms) - - # Get (batch_size * num_atoms) log prob prior evals. - log_prob_prior = self._prior.log_prob(atomic_theta) - log_prob_prior = log_prob_prior.reshape(batch_size, num_atoms) - utils.assert_all_finite(log_prob_prior, "prior eval") - - # Compute unnormalized proposal posterior. - unnormalized_log_prob = log_prob_posterior - log_prob_prior - - # Normalize proposal posterior across discrete set of atoms. - log_prob_proposal_posterior = unnormalized_log_prob[:, 0] - torch.logsumexp( - unnormalized_log_prob, dim=-1 - ) - utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") - - # XXX This evaluates the posterior on _all_ prior samples - if self._use_combined_loss: - log_prob_posterior_non_atomic = self._neural_net.log_prob(theta, x) - masks = masks.reshape(-1) - log_prob_proposal_posterior = ( - masks * log_prob_posterior_non_atomic + log_prob_proposal_posterior - ) - - return log_prob_proposal_posterior - - def _log_prob_proposal_posterior_mog( - self, theta: Tensor, x: Tensor, proposal: DirectPosterior - ) -> Tensor: - """Return log-probability of the proposal posterior for MoG proposal. - - For MoG proposals and MoG density estimators, this can be done in closed form - and does not require atomic loss (i.e. there will be no leakage issues). - - Notation: - - m are mean vectors. - prec are precision matrices. - cov are covariance matrices. - - _p at the end indicates that it is the proposal. - _d indicates that it is the density estimator. - _pp indicates the proposal posterior. - - All tensors will have shapes (batch_dim, num_components, ...) - - Args: - theta: Batch of parameters θ. - x: Batch of data. - proposal: Proposal distribution. - - Returns: - Log-probability of the proposal posterior. - """ - - # Evaluate the proposal. MDNs do not have functionality to run the embedding_net - # and then get the mixture_components (**without** calling log_prob()). Hence, - # we call them separately here. - encoded_x = proposal.posterior_estimator._embedding_net(proposal.default_x) - dist = ( - proposal.posterior_estimator._distribution - ) # defined to avoid ugly black formatting. - logits_p, m_p, prec_p, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_p = logits_p - torch.logsumexp(logits_p, dim=-1, keepdim=True) - - # Evaluate the density estimator. - encoded_x = self._neural_net._embedding_net(x) - dist = self._neural_net._distribution # defined to avoid black formatting. - logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) - norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) - - # z-score theta if it z-scoring had been requested. - theta = self._maybe_z_score_theta(theta) - - # Compute the MoG parameters of the proposal posterior. - logits_pp, m_pp, prec_pp, cov_pp = self._automatic_posterior_transformation( - norm_logits_p, m_p, prec_p, norm_logits_d, m_d, prec_d - ) - - # Compute the log_prob of theta under the product. - log_prob_proposal_posterior = utils.mog_log_prob( - theta, logits_pp, m_pp, prec_pp - ) - utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") - - return log_prob_proposal_posterior - - def _automatic_posterior_transformation( - self, - logits_p: Tensor, - means_p: Tensor, - precisions_p: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - r"""Returns the MoG parameters of the proposal posterior. - - The proposal posterior is: - $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ - In words: proposal posterior = posterior estimate * proposal / prior. - - If the posterior estimate and the proposal are MoG and the prior is either - Gaussian or uniform, we can solve this in closed-form. The is implemented in - this function. - - This function implements Appendix A1 from Greenberg et al. 2019. - - We have to build L*K components. How do we do this? - Example: proposal has two components, density estimator has three components. - Let's call the two components of the proposal i,j and the three components - of the density estimator x,y,z. We have to multiply every component of the - proposal with every component of the density estimator. So, what we do is: - 1) for the proposal, build: i,i,i,j,j,j. Done with torch.repeat_interleave() - 2) for the density estimator, build: x,y,z,x,y,z. Done with torch.repeat() - 3) Multiply them with simple matrix operations. - - Args: - logits_p: Component weight of each Gaussian of the proposal. - means_p: Mean of each Gaussian of the proposal. - precisions_p: Precision matrix of each Gaussian of the proposal. - logits_d: Component weight for each Gaussian of the density estimator. - means_d: Mean of each Gaussian of the density estimator. - precisions_d: Precision matrix of each Gaussian of the density estimator. - - Returns: (Component weight, mean, precision matrix, covariance matrix) of each - Gaussian of the proposal posterior. Has L*K terms (proposal has L terms, - density estimator has K terms). - """ - - precisions_pp, covariances_pp = self._precisions_proposal_posterior( - precisions_p, precisions_d - ) - - means_pp = self._means_proposal_posterior( - covariances_pp, means_p, precisions_p, means_d, precisions_d - ) - - logits_pp = self._logits_proposal_posterior( - means_pp, - precisions_pp, - covariances_pp, - logits_p, - means_p, - precisions_p, - logits_d, - means_d, - precisions_d, - ) - - return logits_pp, means_pp, precisions_pp, covariances_pp - - def _precisions_proposal_posterior( - self, precisions_p: Tensor, precisions_d: Tensor - ): - """Return the precisions and covariances of the proposal posterior. - - Args: - precisions_p: Precision matrices of the proposal distribution. - precisions_d: Precision matrices of the density estimator. - - Returns: (Precisions, Covariances) of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - precisions_p_rep = precisions_p.repeat_interleave(num_comps_d, dim=1) - precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) - - precisions_pp = precisions_p_rep + precisions_d_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - precisions_pp -= self._maybe_z_scored_prior.precision_matrix - - covariances_pp = torch.inverse(precisions_pp) - - return precisions_pp, covariances_pp - - def _means_proposal_posterior( - self, - covariances_pp: Tensor, - means_p: Tensor, - precisions_p: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - """Return the means of the proposal posterior. - - means_pp = C_ix * (P_i * m_i + P_x * m_x - P_o * m_o). - - Args: - covariances_pp: Covariance matrices of the proposal posterior. - means_p: Means of the proposal distribution. - precisions_p: Precision matrices of the proposal distribution. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Means of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - # First, compute the product P_i * m_i and P_j * m_j - prec_m_prod_p = batched_mixture_mv(precisions_p, means_p) - prec_m_prod_d = batched_mixture_mv(precisions_d, means_d) - - # Repeat them to allow for matrix operations: same trick as for the precisions. - prec_m_prod_p_rep = prec_m_prod_p.repeat_interleave(num_comps_d, dim=1) - prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_p, 1) - - # Means = C_ij * (P_i * m_i + P_x * m_x - P_o * m_o). - summed_cov_m_prod_rep = prec_m_prod_p_rep + prec_m_prod_d_rep - if isinstance(self._maybe_z_scored_prior, MultivariateNormal): - summed_cov_m_prod_rep -= self.prec_m_prod_prior - - means_pp = batched_mixture_mv(covariances_pp, summed_cov_m_prod_rep) - - return means_pp - - @staticmethod - def _logits_proposal_posterior( - means_pp: Tensor, - precisions_pp: Tensor, - covariances_pp: Tensor, - logits_p: Tensor, - means_p: Tensor, - precisions_p: Tensor, - logits_d: Tensor, - means_d: Tensor, - precisions_d: Tensor, - ): - """Return the component weights (i.e. logits) of the proposal posterior. - - Args: - means_pp: Means of the proposal posterior. - precisions_pp: Precision matrices of the proposal posterior. - covariances_pp: Covariance matrices of the proposal posterior. - logits_p: Component weights (i.e. logits) of the proposal distribution. - means_p: Means of the proposal distribution. - precisions_p: Precision matrices of the proposal distribution. - logits_d: Component weights (i.e. logits) of the density estimator. - means_d: Means of the density estimator. - precisions_d: Precision matrices of the density estimator. - - Returns: Component weights of the proposal posterior. L*K terms. - """ - - num_comps_p = precisions_p.shape[1] - num_comps_d = precisions_d.shape[1] - - # Compute log(alpha_i * beta_j) - logits_p_rep = logits_p.repeat_interleave(num_comps_d, dim=1) - logits_d_rep = logits_d.repeat(1, num_comps_p) - logit_factors = logits_p_rep + logits_d_rep - - # Compute sqrt(det()/(det()*det())) - logdet_covariances_pp = torch.logdet(covariances_pp) - logdet_covariances_p = -torch.logdet(precisions_p) - logdet_covariances_d = -torch.logdet(precisions_d) - - # Repeat the proposal and density estimator terms such that there are LK terms. - # Same trick as has been used above. - logdet_covariances_p_rep = logdet_covariances_p.repeat_interleave( - num_comps_d, dim=1 - ) - logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_p) - - log_sqrt_det_ratio = 0.5 * ( - logdet_covariances_pp - - (logdet_covariances_p_rep + logdet_covariances_d_rep) - ) - - # Compute for proposal, density estimator, and proposal posterior: - # mu_i.T * P_i * mu_i - exponent_p = batched_mixture_vmv(precisions_p, means_p) - exponent_d = batched_mixture_vmv(precisions_d, means_d) - exponent_pp = batched_mixture_vmv(precisions_pp, means_pp) - - # Extend proposal and density estimator exponents to get LK terms. - exponent_p_rep = exponent_p.repeat_interleave(num_comps_d, dim=1) - exponent_d_rep = exponent_d.repeat(1, num_comps_p) - exponent = -0.5 * (exponent_p_rep + exponent_d_rep - exponent_pp) - - logits_pp = logit_factors + log_sqrt_det_ratio + exponent - - return logits_pp - - def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: - """Return potentially standardized theta if z-scoring was requested.""" - - if self.z_score_theta: - theta, _ = self._neural_net._transform(theta) - - return theta +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + + +from typing import Callable, Dict, Optional, Union + +import torch +from pyknos.mdn.mdn import MultivariateGaussianMDN as mdn +from pyknos.nflows.transforms import CompositeTransform +from torch import Tensor, eye, nn, ones +from torch.distributions import Distribution, MultivariateNormal, Uniform + +from sbi import utils as utils +from sbi.inference.posteriors.direct_posterior import DirectPosterior +from sbi.inference.snpe.snpe_base import PosteriorEstimator +from sbi.types import TensorboardSummaryWriter +from sbi.utils import ( + batched_mixture_mv, + batched_mixture_vmv, + check_dist_class, + clamp_and_warn, + del_entries, + repeat_rows, +) + + +class SNPE_C(PosteriorEstimator): + def __init__( + self, + prior: Optional[Distribution] = None, + density_estimator: Union[str, Callable] = "maf", + device: str = "cpu", + logging_level: Union[int, str] = "WARNING", + summary_writer: Optional[TensorboardSummaryWriter] = None, + show_progress_bars: bool = True, + ): + r"""SNPE-C / APT [1]. + + [1] _Automatic Posterior Transformation for Likelihood-free Inference_, + Greenberg et al., ICML 2019, https://arxiv.org/abs/1905.07488. + + This class implements two loss variants of SNPE-C: the non-atomic and the atomic + version. The atomic loss of SNPE-C can be used for any density estimator, + i.e. also for normalizing flows. However, it suffers from leakage issues. On + the other hand, the non-atomic loss can only be used only if the proposal + distribution is a mixture of Gaussians, the density estimator is a mixture of + Gaussians, and the prior is either Gaussian or Uniform. It does not suffer from + leakage issues. At the beginning of each round, we print whether the non-atomic + or the atomic version is used. + + In this codebase, we will automatically switch to the non-atomic loss if the + following criteria are fulfilled:
+ - proposal is a `DirectPosterior` with density_estimator `mdn`, as built + with `utils.sbi.posterior_nn()`.
+ - the density estimator is a `mdn`, as built with + `utils.sbi.posterior_nn()`.
+ - `isinstance(prior, MultivariateNormal)` (from `torch.distributions`) or + `isinstance(prior, sbi.utils.BoxUniform)` + + Note that custom implementations of any of these densities (or estimators) will + not trigger the non-atomic loss, and the algorithm will fall back onto using + the atomic loss. + + Args: + prior: A probability distribution that expresses prior knowledge about the + parameters, e.g. which ranges are meaningful for them. + density_estimator: If it is a string, use a pre-configured network of the + provided type (one of nsf, maf, mdn, made). Alternatively, a function + that builds a custom neural network can be provided. The function will + be called with the first batch of simulations (theta, x), which can + thus be used for shape inference and potentially for z-scoring. It + needs to return a PyTorch `nn.Module` implementing the density + estimator. The density estimator needs to provide the methods + `.log_prob` and `.sample()`. + device: Training device, e.g., "cpu", "cuda" or "cuda:{0, 1, ...}". + logging_level: Minimum severity of messages to log. One of the strings + INFO, WARNING, DEBUG, ERROR and CRITICAL. + summary_writer: A tensorboard `SummaryWriter` to control, among others, log + file location (default is `/logs`.) + show_progress_bars: Whether to show a progressbar during training. + """ + + kwargs = del_entries(locals(), entries=("self", "__class__")) + super().__init__(**kwargs) + + def train( + self, + num_atoms: int = 10, + training_batch_size: int = 50, + learning_rate: float = 5e-4, + validation_fraction: float = 0.1, + stop_after_epochs: int = 20, + max_num_epochs: int = 2**31 - 1, + clip_max_norm: Optional[float] = 5.0, + calibration_kernel: Optional[Callable] = None, + resume_training: bool = False, + force_first_round_loss: bool = False, + discard_prior_samples: bool = False, + use_combined_loss: bool = False, + retrain_from_scratch: bool = False, + show_train_summary: bool = False, + dataloader_kwargs: Optional[Dict] = None, + ) -> nn.Module: + r"""Return density estimator that approximates the distribution $p(\theta|x)$. + + Args: + num_atoms: Number of atoms to use for classification. + training_batch_size: Training batch size. + learning_rate: Learning rate for Adam optimizer. + validation_fraction: The fraction of data to use for validation. + stop_after_epochs: The number of epochs to wait for improvement on the + validation set before terminating training. + max_num_epochs: Maximum number of epochs to run. If reached, we stop + training even when the validation loss is still decreasing. Otherwise, + we train until validation loss increases (see also `stop_after_epochs`). + clip_max_norm: Value at which to clip the total gradient norm in order to + prevent exploding gradients. Use None for no clipping. + calibration_kernel: A function to calibrate the loss with respect to the + simulations `x`. See Lueckmann, Gonçalves et al., NeurIPS 2017. + resume_training: Can be used in case training time is limited, e.g. on a + cluster. If `True`, the split between train and validation set, the + optimizer, the number of epochs, and the best validation log-prob will + be restored from the last time `.train()` was called. + force_first_round_loss: If `True`, train with maximum likelihood, + i.e., potentially ignoring the correction for using a proposal + distribution different from the prior. + discard_prior_samples: Whether to discard samples simulated in round 1, i.e. + from the prior. Training may be sped up by ignoring such less targeted + samples. + use_combined_loss: Whether to train the neural net also on prior samples + using maximum likelihood in addition to training it on all samples using + atomic loss. The extra MLE loss helps prevent density leaking with + bounded priors. + retrain_from_scratch: Whether to retrain the conditional density + estimator for the posterior from scratch each round. + show_train_summary: Whether to print the number of epochs and validation + loss and leakage after the training. + dataloader_kwargs: Additional or updated kwargs to be passed to the training + and validation dataloaders (like, e.g., a collate_fn) + + Returns: + Density estimator that approximates the distribution $p(\theta|x)$. + """ + + # WARNING: sneaky trick ahead. We proxy the parent's `train` here, + # requiring the signature to have `num_atoms`, save it for use below, and + # continue. It's sneaky because we are using the object (self) as a namespace + # to pass arguments between functions, and that's implicit state management. + self._num_atoms = num_atoms + self._use_combined_loss = use_combined_loss + kwargs = del_entries( + locals(), entries=("self", "__class__", "num_atoms", "use_combined_loss") + ) + + self._round = max(self._data_round_index) + + if self._round > 0: + # Set the proposal to the last proposal that was passed by the user. For + # atomic SNPE, it does not matter what the proposal is. For non-atomic + # SNPE, we only use the latest data that was passed, i.e. the one from the + # last proposal. + proposal = self._proposal_roundwise[-1] + self.use_non_atomic_loss = ( + isinstance(proposal.posterior_estimator._distribution, mdn) + and isinstance(self._neural_net._distribution, mdn) + and check_dist_class( + self._prior, class_to_check=(Uniform, MultivariateNormal) + )[0] + ) + + algorithm = "non-atomic" if self.use_non_atomic_loss else "atomic" + print(f"Using SNPE-C with {algorithm} loss") + + if self.use_non_atomic_loss: + # Take care of z-scoring, pre-compute and store prior terms. + self._set_state_for_mog_proposal() + + return super().train(**kwargs) + + def _set_state_for_mog_proposal(self) -> None: + """Set state variables that are used at each training step of non-atomic SNPE-C. + + Three things are computed: + 1) Check if z-scoring was requested. To do so, we check if the `_transform` + argument of the net had been a `CompositeTransform`. See pyknos mdn.py. + 2) Define a (potentially standardized) prior. It's standardized if z-scoring + had been requested. + 3) Compute (Precision * mean) for the prior. This quantity is used at every + training step if the prior is Gaussian. + """ + + self.z_score_theta = isinstance(self._neural_net._transform, CompositeTransform) + + self._set_maybe_z_scored_prior() + + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + self.prec_m_prod_prior = torch.mv( + self._maybe_z_scored_prior.precision_matrix, # type: ignore + self._maybe_z_scored_prior.loc, # type: ignore + ) + + def _set_maybe_z_scored_prior(self) -> None: + r"""Compute and store potentially standardized prior (if z-scoring was done). + + The proposal posterior is: + $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + + Let's denote z-scored theta by `a`: a = (theta - mean) / std + Then pp'(a|x) = 1/Z_2 * q'(a|x) * prop'(a) / p'(a)$ + + The ' indicates that the evaluation occurs in standardized space. The constant + scaling factor has been absorbed into Z_2. + From the above equation, we see that we need to evaluate the prior **in + standardized space**. We build the standardized prior in this function. + + The standardize transform that is applied to the samples theta does not use + the exact prior mean and std (due to implementation issues). Hence, the z-scored + prior will not be exactly have mean=0 and std=1. + """ + + if self.z_score_theta: + scale = self._neural_net._transform._transforms[0]._scale + shift = self._neural_net._transform._transforms[0]._shift + + # Following the definintion of the linear transform in + # `standardizing_transform` in `sbiutils.py`: + # shift=-mean / std + # scale=1 / std + # Solving these equations for mean and std: + estim_prior_std = 1 / scale + estim_prior_mean = -shift * estim_prior_std + + # Compute the discrepancy of the true prior mean and std and the mean and + # std that was empirically estimated from samples. + # N(theta|m,s) = N((theta-m_e)/s_e|(m-m_e)/s_e, s/s_e) + # Above: m,s are true prior mean and std. m_e,s_e are estimated prior mean + # and std (estimated from samples and used to build standardize transform). + almost_zero_mean = (self._prior.mean - estim_prior_mean) / estim_prior_std + almost_one_std = torch.sqrt(self._prior.variance) / estim_prior_std + + if isinstance(self._prior, MultivariateNormal): + self._maybe_z_scored_prior = MultivariateNormal( + almost_zero_mean, torch.diag(almost_one_std) + ) + else: + range_ = torch.sqrt(almost_one_std * 3.0) + self._maybe_z_scored_prior = utils.BoxUniform( + almost_zero_mean - range_, almost_zero_mean + range_ + ) + else: + self._maybe_z_scored_prior = self._prior + + def _log_prob_proposal_posterior( + self, + theta: Tensor, + x: Tensor, + masks: Tensor, + proposal: DirectPosterior, + ) -> Tensor: + """Return the log-probability of the proposal posterior. + + If the proposal is a MoG, the density estimator is a MoG, and the prior is + either Gaussian or uniform, we use non-atomic loss. Else, use atomic loss (which + suffers from leakage). + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + proposal: Proposal distribution. + + Returns: Log-probability of the proposal posterior. + """ + + if self.use_non_atomic_loss: + return self._log_prob_proposal_posterior_mog(theta, x, proposal) + else: + return self._log_prob_proposal_posterior_atomic(theta, x, masks) + + def _log_prob_proposal_posterior_atomic( + self, theta: Tensor, x: Tensor, masks: Tensor + ): + """Return log probability of the proposal posterior for atomic proposals. + + We have two main options when evaluating the proposal posterior. + (1) Generate atoms from the proposal prior. + (2) Generate atoms from a more targeted distribution, such as the most + recent posterior. + If we choose the latter, it is likely beneficial not to do this in the first + round, since we would be sampling from a randomly-initialized neural density + estimator. + + Args: + theta: Batch of parameters θ. + x: Batch of data. + masks: Mask that is True for prior samples in the batch in order to train + them with prior loss. + + Returns: + Log-probability of the proposal posterior. + """ + + batch_size = theta.shape[0] + + num_atoms = int( + clamp_and_warn("num_atoms", self._num_atoms, min_val=2, max_val=batch_size) + ) + + # Each set of parameter atoms is evaluated using the same x, + # so we repeat rows of the data x, e.g. [1, 2] -> [1, 1, 2, 2] + repeated_x = repeat_rows(x, num_atoms) + + # To generate the full set of atoms for a given item in the batch, + # we sample without replacement num_atoms - 1 times from the rest + # of the theta in the batch. + probs = ones(batch_size, batch_size) * (1 - eye(batch_size)) / (batch_size - 1) + + choices = torch.multinomial(probs, num_samples=num_atoms - 1, replacement=False) + contrasting_theta = theta[choices] + + # We can now create our sets of atoms from the contrasting parameter sets + # we have generated. + atomic_theta = torch.cat((theta[:, None, :], contrasting_theta), dim=1).reshape( + batch_size * num_atoms, -1 + ) + + # Evaluate large batch giving (batch_size * num_atoms) log prob posterior evals. + log_prob_posterior = self._neural_net.log_prob(atomic_theta, repeated_x) + utils.assert_all_finite(log_prob_posterior, "posterior eval") + log_prob_posterior = log_prob_posterior.reshape(batch_size, num_atoms) + + # Get (batch_size * num_atoms) log prob prior evals. + log_prob_prior = self._prior.log_prob(atomic_theta) + log_prob_prior = log_prob_prior.reshape(batch_size, num_atoms) + utils.assert_all_finite(log_prob_prior, "prior eval") + + # Compute unnormalized proposal posterior. + unnormalized_log_prob = log_prob_posterior - log_prob_prior + + # Normalize proposal posterior across discrete set of atoms. + log_prob_proposal_posterior = unnormalized_log_prob[:, 0] - torch.logsumexp( + unnormalized_log_prob, dim=-1 + ) + utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") + + # XXX This evaluates the posterior on _all_ prior samples + if self._use_combined_loss: + log_prob_posterior_non_atomic = self._neural_net.log_prob(theta, x) + masks = masks.reshape(-1) + log_prob_proposal_posterior = ( + masks * log_prob_posterior_non_atomic + log_prob_proposal_posterior + ) + + return log_prob_proposal_posterior + + def _log_prob_proposal_posterior_mog( + self, theta: Tensor, x: Tensor, proposal: DirectPosterior + ) -> Tensor: + """Return log-probability of the proposal posterior for MoG proposal. + + For MoG proposals and MoG density estimators, this can be done in closed form + and does not require atomic loss (i.e. there will be no leakage issues). + + Notation: + + m are mean vectors. + prec are precision matrices. + cov are covariance matrices. + + _p at the end indicates that it is the proposal. + _d indicates that it is the density estimator. + _pp indicates the proposal posterior. + + All tensors will have shapes (batch_dim, num_components, ...) + + Args: + theta: Batch of parameters θ. + x: Batch of data. + proposal: Proposal distribution. + + Returns: + Log-probability of the proposal posterior. + """ + + # Evaluate the proposal. MDNs do not have functionality to run the embedding_net + # and then get the mixture_components (**without** calling log_prob()). Hence, + # we call them separately here. + encoded_x = proposal.posterior_estimator._embedding_net(proposal.default_x) + dist = ( + proposal.posterior_estimator._distribution + ) # defined to avoid ugly black formatting. + logits_p, m_p, prec_p, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_p = logits_p - torch.logsumexp(logits_p, dim=-1, keepdim=True) + + # Evaluate the density estimator. + encoded_x = self._neural_net._embedding_net(x) + dist = self._neural_net._distribution # defined to avoid black formatting. + logits_d, m_d, prec_d, _, _ = dist.get_mixture_components(encoded_x) + norm_logits_d = logits_d - torch.logsumexp(logits_d, dim=-1, keepdim=True) + + # z-score theta if it z-scoring had been requested. + theta = self._maybe_z_score_theta(theta) + + # Compute the MoG parameters of the proposal posterior. + logits_pp, m_pp, prec_pp, cov_pp = self._automatic_posterior_transformation( + norm_logits_p, m_p, prec_p, norm_logits_d, m_d, prec_d + ) + + # Compute the log_prob of theta under the product. + log_prob_proposal_posterior = utils.mog_log_prob( + theta, logits_pp, m_pp, prec_pp + ) + utils.assert_all_finite(log_prob_proposal_posterior, "proposal posterior eval") + + return log_prob_proposal_posterior + + def _automatic_posterior_transformation( + self, + logits_p: Tensor, + means_p: Tensor, + precisions_p: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + r"""Returns the MoG parameters of the proposal posterior. + + The proposal posterior is: + $pp(\theta|x) = 1/Z * q(\theta|x) * prop(\theta) / p(\theta)$ + In words: proposal posterior = posterior estimate * proposal / prior. + + If the posterior estimate and the proposal are MoG and the prior is either + Gaussian or uniform, we can solve this in closed-form. The is implemented in + this function. + + This function implements Appendix A1 from Greenberg et al. 2019. + + We have to build L*K components. How do we do this? + Example: proposal has two components, density estimator has three components. + Let's call the two components of the proposal i,j and the three components + of the density estimator x,y,z. We have to multiply every component of the + proposal with every component of the density estimator. So, what we do is: + 1) for the proposal, build: i,i,i,j,j,j. Done with torch.repeat_interleave() + 2) for the density estimator, build: x,y,z,x,y,z. Done with torch.repeat() + 3) Multiply them with simple matrix operations. + + Args: + logits_p: Component weight of each Gaussian of the proposal. + means_p: Mean of each Gaussian of the proposal. + precisions_p: Precision matrix of each Gaussian of the proposal. + logits_d: Component weight for each Gaussian of the density estimator. + means_d: Mean of each Gaussian of the density estimator. + precisions_d: Precision matrix of each Gaussian of the density estimator. + + Returns: (Component weight, mean, precision matrix, covariance matrix) of each + Gaussian of the proposal posterior. Has L*K terms (proposal has L terms, + density estimator has K terms). + """ + + precisions_pp, covariances_pp = self._precisions_proposal_posterior( + precisions_p, precisions_d + ) + + means_pp = self._means_proposal_posterior( + covariances_pp, means_p, precisions_p, means_d, precisions_d + ) + + logits_pp = self._logits_proposal_posterior( + means_pp, + precisions_pp, + covariances_pp, + logits_p, + means_p, + precisions_p, + logits_d, + means_d, + precisions_d, + ) + + return logits_pp, means_pp, precisions_pp, covariances_pp + + def _precisions_proposal_posterior( + self, precisions_p: Tensor, precisions_d: Tensor + ): + """Return the precisions and covariances of the proposal posterior. + + Args: + precisions_p: Precision matrices of the proposal distribution. + precisions_d: Precision matrices of the density estimator. + + Returns: (Precisions, Covariances) of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + precisions_p_rep = precisions_p.repeat_interleave(num_comps_d, dim=1) + precisions_d_rep = precisions_d.repeat(1, num_comps_p, 1, 1) + + precisions_pp = precisions_p_rep + precisions_d_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + precisions_pp -= self._maybe_z_scored_prior.precision_matrix + + covariances_pp = torch.inverse(precisions_pp) + + return precisions_pp, covariances_pp + + def _means_proposal_posterior( + self, + covariances_pp: Tensor, + means_p: Tensor, + precisions_p: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + """Return the means of the proposal posterior. + + means_pp = C_ix * (P_i * m_i + P_x * m_x - P_o * m_o). + + Args: + covariances_pp: Covariance matrices of the proposal posterior. + means_p: Means of the proposal distribution. + precisions_p: Precision matrices of the proposal distribution. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Means of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + # First, compute the product P_i * m_i and P_j * m_j + prec_m_prod_p = batched_mixture_mv(precisions_p, means_p) + prec_m_prod_d = batched_mixture_mv(precisions_d, means_d) + + # Repeat them to allow for matrix operations: same trick as for the precisions. + prec_m_prod_p_rep = prec_m_prod_p.repeat_interleave(num_comps_d, dim=1) + prec_m_prod_d_rep = prec_m_prod_d.repeat(1, num_comps_p, 1) + + # Means = C_ij * (P_i * m_i + P_x * m_x - P_o * m_o). + summed_cov_m_prod_rep = prec_m_prod_p_rep + prec_m_prod_d_rep + if isinstance(self._maybe_z_scored_prior, MultivariateNormal): + summed_cov_m_prod_rep -= self.prec_m_prod_prior + + means_pp = batched_mixture_mv(covariances_pp, summed_cov_m_prod_rep) + + return means_pp + + @staticmethod + def _logits_proposal_posterior( + means_pp: Tensor, + precisions_pp: Tensor, + covariances_pp: Tensor, + logits_p: Tensor, + means_p: Tensor, + precisions_p: Tensor, + logits_d: Tensor, + means_d: Tensor, + precisions_d: Tensor, + ): + """Return the component weights (i.e. logits) of the proposal posterior. + + Args: + means_pp: Means of the proposal posterior. + precisions_pp: Precision matrices of the proposal posterior. + covariances_pp: Covariance matrices of the proposal posterior. + logits_p: Component weights (i.e. logits) of the proposal distribution. + means_p: Means of the proposal distribution. + precisions_p: Precision matrices of the proposal distribution. + logits_d: Component weights (i.e. logits) of the density estimator. + means_d: Means of the density estimator. + precisions_d: Precision matrices of the density estimator. + + Returns: Component weights of the proposal posterior. L*K terms. + """ + + num_comps_p = precisions_p.shape[1] + num_comps_d = precisions_d.shape[1] + + # Compute log(alpha_i * beta_j) + logits_p_rep = logits_p.repeat_interleave(num_comps_d, dim=1) + logits_d_rep = logits_d.repeat(1, num_comps_p) + logit_factors = logits_p_rep + logits_d_rep + + # Compute sqrt(det()/(det()*det())) + logdet_covariances_pp = torch.logdet(covariances_pp) + logdet_covariances_p = -torch.logdet(precisions_p) + logdet_covariances_d = -torch.logdet(precisions_d) + + # Repeat the proposal and density estimator terms such that there are LK terms. + # Same trick as has been used above. + logdet_covariances_p_rep = logdet_covariances_p.repeat_interleave( + num_comps_d, dim=1 + ) + logdet_covariances_d_rep = logdet_covariances_d.repeat(1, num_comps_p) + + log_sqrt_det_ratio = 0.5 * ( + logdet_covariances_pp + - (logdet_covariances_p_rep + logdet_covariances_d_rep) + ) + + # Compute for proposal, density estimator, and proposal posterior: + # mu_i.T * P_i * mu_i + exponent_p = batched_mixture_vmv(precisions_p, means_p) + exponent_d = batched_mixture_vmv(precisions_d, means_d) + exponent_pp = batched_mixture_vmv(precisions_pp, means_pp) + + # Extend proposal and density estimator exponents to get LK terms. + exponent_p_rep = exponent_p.repeat_interleave(num_comps_d, dim=1) + exponent_d_rep = exponent_d.repeat(1, num_comps_p) + exponent = -0.5 * (exponent_p_rep + exponent_d_rep - exponent_pp) + + logits_pp = logit_factors + log_sqrt_det_ratio + exponent + + return logits_pp + + def _maybe_z_score_theta(self, theta: Tensor) -> Tensor: + """Return potentially standardized theta if z-scoring was requested.""" + + if self.z_score_theta: + theta, _ = self._neural_net._transform(theta) + + return theta diff --git a/tests/linearGaussian_snpe_test.py b/tests/linearGaussian_snpe_test.py index b43bd0144..2d84aa759 100644 --- a/tests/linearGaussian_snpe_test.py +++ b/tests/linearGaussian_snpe_test.py @@ -1,583 +1,583 @@ -# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed -# under the Affero General Public License v3, see . - -from __future__ import annotations - -import numpy as np -import pytest -import torch -from scipy.stats import gaussian_kde -from torch import eye, ones, zeros -from torch.distributions import MultivariateNormal - -from sbi import analysis as analysis -from sbi import utils as utils -from sbi.analysis import ConditionedMDN, conditonal_potential -from sbi.inference import ( - SNPE_A, - SNPE_B, - SNPE_C, - DirectPosterior, - MCMCPosterior, - RejectionPosterior, - posterior_estimator_based_potential, - prepare_for_sbi, - simulate_for_sbi, -) -from sbi.simulators.linear_gaussian import ( - linear_gaussian, - samples_true_posterior_linear_gaussian_mvn_prior_different_dims, - samples_true_posterior_linear_gaussian_uniform_prior, - true_posterior_linear_gaussian_mvn_prior, -) -from tests.sbiutils_test import conditional_of_mvn -from tests.test_utils import ( - check_c2st, - get_dkl_gaussian_prior, - get_normalization_uniform_prior, - get_prob_outside_uniform_prior, -) - - -@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) -@pytest.mark.parametrize( - "num_dim, prior_str, num_trials", - ( - (2, "gaussian", 1), - (2, "uniform", 1), - (1, "gaussian", 1), - # no iid x in snpe. - pytest.param(1, "gaussian", 2, marks=pytest.mark.xfail), - pytest.param(2, "gaussian", 2, marks=pytest.mark.xfail), - ), -) -def test_c2st_snpe_on_linearGaussian( - snpe_method, num_dim: int, prior_str: str, num_trials: int -): - """Test whether SNPE infers well a simple example with available ground truth.""" - - x_o = zeros(num_trials, num_dim) - num_samples = 1000 - num_simulations = 2600 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - if prior_str == "gaussian": - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - gt_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - target_samples = gt_posterior.sample((num_samples,)) - else: - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - target_samples = samples_true_posterior_linear_gaussian_uniform_prior( - x_o, likelihood_shift, likelihood_cov, prior=prior, num_samples=num_samples - ) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - - inference = snpe_method(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, num_simulations, simulation_batch_size=1000 - ) - posterior_estimator = inference.append_simulations(theta, x).train( - training_batch_size=100 - ) - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg="snpe_c") - - map_ = posterior.map(num_init_samples=1_000, show_progress_bars=False) - - # Checks for log_prob() - if prior_str == "gaussian": - # For the Gaussian prior, we compute the KLd between ground truth and posterior. - dkl = get_dkl_gaussian_prior( - posterior, x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - - max_dkl = 0.15 - - assert ( - dkl < max_dkl - ), f"D-KL={dkl} is more than 2 stds above the average performance." - - assert ((map_ - gt_posterior.mean) ** 2).sum() < 0.5 - - elif prior_str == "uniform": - # Check whether the returned probability outside of the support is zero. - posterior_prob = get_prob_outside_uniform_prior(posterior, prior, num_dim) - assert ( - posterior_prob == 0.0 - ), "The posterior probability outside of the prior support is not zero" - - # Check whether normalization (i.e. scaling up the density due - # to leakage into regions without prior support) scales up the density by the - # correct factor. - ( - posterior_likelihood_unnorm, - posterior_likelihood_norm, - acceptance_prob, - ) = get_normalization_uniform_prior(posterior, prior, x=x_o) - # The acceptance probability should be *exactly* the ratio of the unnormalized - # and the normalized likelihood. However, we allow for an error margin of 1%, - # since the estimation of the acceptance probability is random (based on - # rejection sampling). - assert ( - acceptance_prob * 0.99 - < posterior_likelihood_unnorm / posterior_likelihood_norm - < acceptance_prob * 1.01 - ), "Normalizing the posterior density using the acceptance probability failed." - - assert ((map_ - ones(num_dim)) ** 2).sum() < 0.5 - - -def test_c2st_snpe_on_linearGaussian_different_dims(): - """Test whether SNPE B/C infer well a simple example with available ground truth. - - This example has different number of parameters theta than number of x. Also - this implicitly tests simulation_batch_size=1. It also impleictly tests whether the - prior can be `None` and whether we can stop and resume training. - - """ - - theta_dim = 3 - x_dim = 2 - discard_dims = theta_dim - x_dim - - x_o = zeros(1, x_dim) - num_samples = 1000 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(x_dim) - likelihood_cov = 0.3 * eye(x_dim) - - prior_mean = zeros(theta_dim) - prior_cov = eye(theta_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - target_samples = samples_true_posterior_linear_gaussian_mvn_prior_different_dims( - x_o, - likelihood_shift, - likelihood_cov, - prior_mean, - prior_cov, - num_discarded_dims=discard_dims, - num_samples=num_samples, - ) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian( - theta, likelihood_shift, likelihood_cov, num_discarded_dims=discard_dims - ), - prior, - ) - # Test whether prior can be `None`. - inference = SNPE_C(prior=None, density_estimator="maf", show_progress_bars=False) - - # type: ignore - theta, x = simulate_for_sbi(simulator, prior, 2000, simulation_batch_size=1) - - inference = inference.append_simulations(theta, x) - posterior_estimator = inference.train( - max_num_epochs=10 - ) # Test whether we can stop and resume. - posterior_estimator = inference.train( - resume_training=True, force_first_round_loss=True - ) - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg="snpe_c") - - -# Test multi-round SNPE. -@pytest.mark.slow -@pytest.mark.parametrize( - "method_str", - ( - "snpe_a", - pytest.param( - "snpe_b", - marks=pytest.mark.xfail( - raises=NotImplementedError, reason="""SNPE-B not implemented""" - ), - ), - "snpe_c", - "snpe_c_non_atomic", - ), -) -def test_c2st_multi_round_snpe_on_linearGaussian(method_str: str): - """Test whether SNPE B/C infer well a simple example with available ground truth. - . - """ - - num_dim = 2 - x_o = zeros((1, num_dim)) - num_samples = 1000 - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - gt_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - target_samples = gt_posterior.sample((num_samples,)) - - if method_str == "snpe_c_non_atomic": - # Test whether SNPE works properly with structured z-scoring. - density_estimator = utils.posterior_nn( - "mdn", z_score_x="structured", num_components=5 - ) - method_str = "snpe_c" - elif method_str == "snpe_a": - density_estimator = "mdn_snpe_a" - else: - density_estimator = "maf" - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - creation_args = dict( - prior=prior, - density_estimator=density_estimator, - show_progress_bars=False, - ) - - if method_str == "snpe_b": - inference = SNPE_B(**creation_args) - theta, x = simulate_for_sbi(simulator, prior, 500, simulation_batch_size=10) - posterior_estimator = inference.append_simulations(theta, x).train() - posterior1 = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - theta, x = simulate_for_sbi( - simulator, posterior1, 1000, simulation_batch_size=10 - ) - posterior_estimator = inference.append_simulations( - theta, x, proposal=posterior1 - ).train() - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - elif method_str == "snpe_c": - inference = SNPE_C(**creation_args) - theta, x = simulate_for_sbi(simulator, prior, 900, simulation_batch_size=50) - posterior_estimator = inference.append_simulations(theta, x).train() - posterior1 = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - theta = posterior1.sample((1000,)) - x = simulator(theta) - _ = inference.append_simulations(theta, x, proposal=posterior1).train() - posterior = inference.build_posterior().set_default_x(x_o) - elif method_str == "snpe_a": - inference = SNPE_A(**creation_args) - proposal = prior - final_round = False - num_rounds = 3 - for r in range(num_rounds): - if r == 2: - final_round = True - theta, x = simulate_for_sbi( - simulator, proposal, 500, simulation_batch_size=50 - ) - inference = inference.append_simulations(theta, x, proposal=proposal) - _ = inference.train(max_num_epochs=200, final_round=final_round) - posterior = inference.build_posterior().set_default_x(x_o) - proposal = posterior - - samples = posterior.sample((num_samples,)) - - # Compute the c2st and assert it is near chance level of 0.5. - check_c2st(samples, target_samples, alg=method_str) - - -# Testing rejection and mcmc sampling methods. -@pytest.mark.slow -@pytest.mark.parametrize( - "sample_with, mcmc_method, prior_str", - ( - ("mcmc", "slice_np", "gaussian"), - ("mcmc", "slice", "gaussian"), - # XXX (True, "slice", "uniform"), - # XXX takes very long. fix when refactoring pyro sampling - ("rejection", "rejection", "uniform"), - ), -) -def test_api_snpe_c_posterior_correction(sample_with, mcmc_method, prior_str): - """Test that leakage correction applied to sampling works, with both MCMC and - rejection. - - """ - - num_dim = 2 - x_o = zeros(1, num_dim) - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - if prior_str == "gaussian": - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - else: - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - inference = SNPE_C(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi(simulator, prior, 1000) - posterior_estimator = inference.append_simulations(theta, x).train() - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator, prior, x_o - ) - if sample_with == "mcmc": - posterior = MCMCPosterior( - potential_fn=potential_fn, - theta_transform=theta_transform, - proposal=prior, - method=mcmc_method, - ) - elif sample_with == "rejection": - posterior = RejectionPosterior( - potential_fn=potential_fn, - proposal=prior, - theta_transform=theta_transform, - ) - - # Posterior should be corrected for leakage even if num_rounds just 1. - samples = posterior.sample((10,)) - - # Evaluate the samples to check correction factor. - _ = posterior.log_prob(samples) - - -@pytest.mark.slow -def test_sample_conditional(): - """ - Test whether sampling from the conditional gives the same results as evaluating. - - This compares samples that get smoothed with a Gaussian kde to evaluating the - conditional log-probability with `eval_conditional_density`. - - `eval_conditional_density` is itself tested in `sbiutils_test.py`. Here, we use - a bimodal posterior to test the conditional. - """ - - num_dim = 3 - dim_to_sample_1 = 0 - dim_to_sample_2 = 2 - - x_o = zeros(1, num_dim) - - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.1 * eye(num_dim) - - prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) - - def simulator(theta): - if torch.rand(1) > 0.5: - return linear_gaussian(theta, likelihood_shift, likelihood_cov) - else: - return linear_gaussian(theta, -likelihood_shift, likelihood_cov) - - # Test whether SNPE works properly with structured z-scoring. - net = utils.posterior_nn("maf", z_score_x="structured", hidden_features=20) - - simulator, prior = prepare_for_sbi(simulator, prior) - - inference = SNPE_C(prior, density_estimator=net, show_progress_bars=False) - - # We need a pretty big dataset to properly model the bimodality. - theta, x = simulate_for_sbi(simulator, prior, 10000) - posterior_estimator = inference.append_simulations(theta, x).train( - max_num_epochs=60 - ) - - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - samples = posterior.sample((50,)) - - # Evaluate the conditional density be drawing samples and smoothing with a Gaussian - # kde. - potential_fn, theta_transform = posterior_estimator_based_potential( - posterior_estimator, prior=prior, x_o=x_o - ) - (conditioned_potential_fn, restricted_tf, restricted_prior,) = conditonal_potential( - potential_fn=potential_fn, - theta_transform=theta_transform, - prior=prior, - condition=samples[0], - dims_to_sample=[dim_to_sample_1, dim_to_sample_2], - ) - mcmc_posterior = MCMCPosterior( - potential_fn=conditioned_potential_fn, - theta_transform=restricted_tf, - proposal=restricted_prior, - ) - cond_samples = mcmc_posterior.sample((500,)) - - _ = analysis.pairplot( - cond_samples, - limits=[[-2, 2], [-2, 2], [-2, 2]], - figsize=(2, 2), - diag="kde", - upper="kde", - ) - - limits = [[-2, 2], [-2, 2], [-2, 2]] - - density = gaussian_kde(cond_samples.numpy().T, bw_method="scott") - - X, Y = np.meshgrid( - np.linspace(limits[0][0], limits[0][1], 50), - np.linspace(limits[1][0], limits[1][1], 50), - ) - positions = np.vstack([X.ravel(), Y.ravel()]) - sample_kde_grid = np.reshape(density(positions).T, X.shape) - - # Evaluate the conditional with eval_conditional_density. - eval_grid = analysis.eval_conditional_density( - posterior, - condition=samples[0], - dim1=dim_to_sample_1, - dim2=dim_to_sample_2, - limits=torch.tensor([[-2, 2], [-2, 2], [-2, 2]]), - ) - - # Compare the two densities. - sample_kde_grid = sample_kde_grid / np.sum(sample_kde_grid) - eval_grid = eval_grid / torch.sum(eval_grid) - - error = np.abs(sample_kde_grid - eval_grid.numpy()) - - max_err = np.max(error) - assert max_err < 0.0027 - - -def test_mdn_conditional_density(num_dim: int = 3, cond_dim: int = 1): - """Test whether the conditional density infered from MDN parameters of a - `DirectPosterior` matches analytical results for MVN. This uses a n-D joint and - conditions on the last m values to generate a conditional. - - Gaussian prior used for easier ground truthing of conditional posterior. - - Args: - num_dim: Dimensionality of the MVM. - cond_dim: Dimensionality of the condition. - """ - - assert ( - num_dim > cond_dim - ), "The number of dimensions needs to be greater than that of the condition!" - - x_o = zeros(1, num_dim) - num_samples = 1000 - num_simulations = 2700 - condition = 0.1 * ones(1, num_dim) - - dims = list(range(num_dim)) - dims2sample = dims[-cond_dim:] - dims2condition = dims[:-cond_dim] - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - joint_posterior = true_posterior_linear_gaussian_mvn_prior( - x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov - ) - joint_cov = joint_posterior.covariance_matrix - joint_mean = joint_posterior.loc - - conditional_mean, conditional_cov = conditional_of_mvn( - joint_mean, joint_cov, condition[0, dims2condition] - ) - conditional_dist_gt = MultivariateNormal(conditional_mean, conditional_cov) - - conditional_samples_gt = conditional_dist_gt.sample((num_samples,)) - - def simulator(theta): - return linear_gaussian(theta, likelihood_shift, likelihood_cov) - - simulator, prior = prepare_for_sbi(simulator, prior) - inference = SNPE_C(density_estimator="mdn", show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, num_simulations, simulation_batch_size=1000 - ) - posterior_mdn = inference.append_simulations(theta, x).train( - training_batch_size=100 - ) - conditioned_mdn = ConditionedMDN( - posterior_mdn, x_o, condition=condition, dims_to_sample=[0] - ) - conditional_samples_sbi = conditioned_mdn.sample((num_samples,)) - check_c2st( - conditional_samples_sbi, - conditional_samples_gt, - alg="analytic_mdn_conditioning_of_direct_posterior", - ) - - -@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) -def test_example_posterior(snpe_method: type): - """Return an inferred `NeuralPosterior` for interactive examination.""" - num_dim = 2 - x_o = zeros(1, num_dim) - - # likelihood_mean will be likelihood_shift+theta - likelihood_shift = -1.0 * ones(num_dim) - likelihood_cov = 0.3 * eye(num_dim) - - prior_mean = zeros(num_dim) - prior_cov = eye(num_dim) - prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) - - if snpe_method == SNPE_A: - extra_kwargs = dict(final_round=True) - else: - extra_kwargs = dict() - - simulator, prior = prepare_for_sbi( - lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior - ) - inference = snpe_method(prior, show_progress_bars=False) - - theta, x = simulate_for_sbi( - simulator, prior, 1000, simulation_batch_size=10, num_workers=6 - ) - posterior_estimator = inference.append_simulations(theta, x).train(**extra_kwargs) - if snpe_method == SNPE_A: - posterior_estimator = inference.correct_for_proposal() - posterior = DirectPosterior( - prior=prior, posterior_estimator=posterior_estimator - ).set_default_x(x_o) - assert posterior is not None +# This file is part of sbi, a toolkit for simulation-based inference. sbi is licensed +# under the Affero General Public License v3, see . + +from __future__ import annotations + +import numpy as np +import pytest +import torch +from scipy.stats import gaussian_kde +from torch import eye, ones, zeros +from torch.distributions import MultivariateNormal + +from sbi import analysis as analysis +from sbi import utils as utils +from sbi.analysis import ConditionedMDN, conditonal_potential +from sbi.inference import ( + SNPE_A, + SNPE_B, + SNPE_C, + DirectPosterior, + MCMCPosterior, + RejectionPosterior, + posterior_estimator_based_potential, + prepare_for_sbi, + simulate_for_sbi, +) +from sbi.simulators.linear_gaussian import ( + linear_gaussian, + samples_true_posterior_linear_gaussian_mvn_prior_different_dims, + samples_true_posterior_linear_gaussian_uniform_prior, + true_posterior_linear_gaussian_mvn_prior, +) +from tests.sbiutils_test import conditional_of_mvn +from tests.test_utils import ( + check_c2st, + get_dkl_gaussian_prior, + get_normalization_uniform_prior, + get_prob_outside_uniform_prior, +) + + +@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) +@pytest.mark.parametrize( + "num_dim, prior_str, num_trials", + ( + (2, "gaussian", 1), + (2, "uniform", 1), + (1, "gaussian", 1), + # no iid x in snpe. + pytest.param(1, "gaussian", 2, marks=pytest.mark.xfail), + pytest.param(2, "gaussian", 2, marks=pytest.mark.xfail), + ), +) +def test_c2st_snpe_on_linearGaussian( + snpe_method, num_dim: int, prior_str: str, num_trials: int +): + """Test whether SNPE infers well a simple example with available ground truth.""" + + x_o = zeros(num_trials, num_dim) + num_samples = 1000 + num_simulations = 2600 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + if prior_str == "gaussian": + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + gt_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + target_samples = gt_posterior.sample((num_samples,)) + else: + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + target_samples = samples_true_posterior_linear_gaussian_uniform_prior( + x_o, likelihood_shift, likelihood_cov, prior=prior, num_samples=num_samples + ) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + + inference = snpe_method(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, num_simulations, simulation_batch_size=1000 + ) + posterior_estimator = inference.append_simulations(theta, x).train( + training_batch_size=100 + ) + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg="snpe_c") + + map_ = posterior.map(num_init_samples=1_000, show_progress_bars=False) + + # Checks for log_prob() + if prior_str == "gaussian": + # For the Gaussian prior, we compute the KLd between ground truth and posterior. + dkl = get_dkl_gaussian_prior( + posterior, x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + + max_dkl = 0.15 + + assert ( + dkl < max_dkl + ), f"D-KL={dkl} is more than 2 stds above the average performance." + + assert ((map_ - gt_posterior.mean) ** 2).sum() < 0.5 + + elif prior_str == "uniform": + # Check whether the returned probability outside of the support is zero. + posterior_prob = get_prob_outside_uniform_prior(posterior, prior, num_dim) + assert ( + posterior_prob == 0.0 + ), "The posterior probability outside of the prior support is not zero" + + # Check whether normalization (i.e. scaling up the density due + # to leakage into regions without prior support) scales up the density by the + # correct factor. + ( + posterior_likelihood_unnorm, + posterior_likelihood_norm, + acceptance_prob, + ) = get_normalization_uniform_prior(posterior, prior, x=x_o) + # The acceptance probability should be *exactly* the ratio of the unnormalized + # and the normalized likelihood. However, we allow for an error margin of 1%, + # since the estimation of the acceptance probability is random (based on + # rejection sampling). + assert ( + acceptance_prob * 0.99 + < posterior_likelihood_unnorm / posterior_likelihood_norm + < acceptance_prob * 1.01 + ), "Normalizing the posterior density using the acceptance probability failed." + + assert ((map_ - ones(num_dim)) ** 2).sum() < 0.5 + + +def test_c2st_snpe_on_linearGaussian_different_dims(): + """Test whether SNPE B/C infer well a simple example with available ground truth. + + This example has different number of parameters theta than number of x. Also + this implicitly tests simulation_batch_size=1. It also impleictly tests whether the + prior can be `None` and whether we can stop and resume training. + + """ + + theta_dim = 3 + x_dim = 2 + discard_dims = theta_dim - x_dim + + x_o = zeros(1, x_dim) + num_samples = 1000 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(x_dim) + likelihood_cov = 0.3 * eye(x_dim) + + prior_mean = zeros(theta_dim) + prior_cov = eye(theta_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + target_samples = samples_true_posterior_linear_gaussian_mvn_prior_different_dims( + x_o, + likelihood_shift, + likelihood_cov, + prior_mean, + prior_cov, + num_discarded_dims=discard_dims, + num_samples=num_samples, + ) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian( + theta, likelihood_shift, likelihood_cov, num_discarded_dims=discard_dims + ), + prior, + ) + # Test whether prior can be `None`. + inference = SNPE_C(prior=None, density_estimator="maf", show_progress_bars=False) + + # type: ignore + theta, x = simulate_for_sbi(simulator, prior, 2000, simulation_batch_size=1) + + inference = inference.append_simulations(theta, x) + posterior_estimator = inference.train( + max_num_epochs=10 + ) # Test whether we can stop and resume. + posterior_estimator = inference.train( + resume_training=True, force_first_round_loss=True + ) + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg="snpe_c") + + +# Test multi-round SNPE. +@pytest.mark.slow +@pytest.mark.parametrize( + "method_str", + ( + "snpe_a", + pytest.param( + "snpe_b", + marks=pytest.mark.xfail( + raises=NotImplementedError, reason="""SNPE-B not implemented""" + ), + ), + "snpe_c", + "snpe_c_non_atomic", + ), +) +def test_c2st_multi_round_snpe_on_linearGaussian(method_str: str): + """Test whether SNPE B/C infer well a simple example with available ground truth. + . + """ + + num_dim = 2 + x_o = zeros((1, num_dim)) + num_samples = 1000 + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + gt_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o, likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + target_samples = gt_posterior.sample((num_samples,)) + + if method_str == "snpe_c_non_atomic": + # Test whether SNPE works properly with structured z-scoring. + density_estimator = utils.posterior_nn( + "mdn", z_score_x="structured", num_components=5 + ) + method_str = "snpe_c" + elif method_str == "snpe_a": + density_estimator = "mdn_snpe_a" + else: + density_estimator = "maf" + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + creation_args = dict( + prior=prior, + density_estimator=density_estimator, + show_progress_bars=False, + ) + + if method_str == "snpe_b": + inference = SNPE_B(**creation_args) + theta, x = simulate_for_sbi(simulator, prior, 500, simulation_batch_size=10) + posterior_estimator = inference.append_simulations(theta, x).train() + posterior1 = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + theta, x = simulate_for_sbi( + simulator, posterior1, 1000, simulation_batch_size=10 + ) + posterior_estimator = inference.append_simulations( + theta, x, proposal=posterior1 + ).train() + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + elif method_str == "snpe_c": + inference = SNPE_C(**creation_args) + theta, x = simulate_for_sbi(simulator, prior, 900, simulation_batch_size=50) + posterior_estimator = inference.append_simulations(theta, x).train() + posterior1 = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + theta = posterior1.sample((1000,)) + x = simulator(theta) + _ = inference.append_simulations(theta, x, proposal=posterior1).train() + posterior = inference.build_posterior().set_default_x(x_o) + elif method_str == "snpe_a": + inference = SNPE_A(**creation_args) + proposal = prior + final_round = False + num_rounds = 3 + for r in range(num_rounds): + if r == 2: + final_round = True + theta, x = simulate_for_sbi( + simulator, proposal, 500, simulation_batch_size=50 + ) + inference = inference.append_simulations(theta, x, proposal=proposal) + _ = inference.train(max_num_epochs=200, final_round=final_round) + posterior = inference.build_posterior().set_default_x(x_o) + proposal = posterior + + samples = posterior.sample((num_samples,)) + + # Compute the c2st and assert it is near chance level of 0.5. + check_c2st(samples, target_samples, alg=method_str) + + +# Testing rejection and mcmc sampling methods. +@pytest.mark.slow +@pytest.mark.parametrize( + "sample_with, mcmc_method, prior_str", + ( + ("mcmc", "slice_np", "gaussian"), + ("mcmc", "slice", "gaussian"), + # XXX (True, "slice", "uniform"), + # XXX takes very long. fix when refactoring pyro sampling + ("rejection", "rejection", "uniform"), + ), +) +def test_api_snpe_c_posterior_correction(sample_with, mcmc_method, prior_str): + """Test that leakage correction applied to sampling works, with both MCMC and + rejection. + + """ + + num_dim = 2 + x_o = zeros(1, num_dim) + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + if prior_str == "gaussian": + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + else: + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + inference = SNPE_C(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi(simulator, prior, 1000) + posterior_estimator = inference.append_simulations(theta, x).train() + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator, prior, x_o + ) + if sample_with == "mcmc": + posterior = MCMCPosterior( + potential_fn=potential_fn, + theta_transform=theta_transform, + proposal=prior, + method=mcmc_method, + ) + elif sample_with == "rejection": + posterior = RejectionPosterior( + potential_fn=potential_fn, + proposal=prior, + theta_transform=theta_transform, + ) + + # Posterior should be corrected for leakage even if num_rounds just 1. + samples = posterior.sample((10,)) + + # Evaluate the samples to check correction factor. + _ = posterior.log_prob(samples) + + +@pytest.mark.slow +def test_sample_conditional(): + """ + Test whether sampling from the conditional gives the same results as evaluating. + + This compares samples that get smoothed with a Gaussian kde to evaluating the + conditional log-probability with `eval_conditional_density`. + + `eval_conditional_density` is itself tested in `sbiutils_test.py`. Here, we use + a bimodal posterior to test the conditional. + """ + + num_dim = 3 + dim_to_sample_1 = 0 + dim_to_sample_2 = 2 + + x_o = zeros(1, num_dim) + + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.1 * eye(num_dim) + + prior = utils.BoxUniform(-2.0 * ones(num_dim), 2.0 * ones(num_dim)) + + def simulator(theta): + if torch.rand(1) > 0.5: + return linear_gaussian(theta, likelihood_shift, likelihood_cov) + else: + return linear_gaussian(theta, -likelihood_shift, likelihood_cov) + + # Test whether SNPE works properly with structured z-scoring. + net = utils.posterior_nn("maf", z_score_x="structured", hidden_features=20) + + simulator, prior = prepare_for_sbi(simulator, prior) + + inference = SNPE_C(prior, density_estimator=net, show_progress_bars=False) + + # We need a pretty big dataset to properly model the bimodality. + theta, x = simulate_for_sbi(simulator, prior, 10000) + posterior_estimator = inference.append_simulations(theta, x).train( + max_num_epochs=60 + ) + + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + samples = posterior.sample((50,)) + + # Evaluate the conditional density be drawing samples and smoothing with a Gaussian + # kde. + potential_fn, theta_transform = posterior_estimator_based_potential( + posterior_estimator, prior=prior, x_o=x_o + ) + (conditioned_potential_fn, restricted_tf, restricted_prior,) = conditonal_potential( + potential_fn=potential_fn, + theta_transform=theta_transform, + prior=prior, + condition=samples[0], + dims_to_sample=[dim_to_sample_1, dim_to_sample_2], + ) + mcmc_posterior = MCMCPosterior( + potential_fn=conditioned_potential_fn, + theta_transform=restricted_tf, + proposal=restricted_prior, + ) + cond_samples = mcmc_posterior.sample((500,)) + + _ = analysis.pairplot( + cond_samples, + limits=[[-2, 2], [-2, 2], [-2, 2]], + figsize=(2, 2), + diag="kde", + upper="kde", + ) + + limits = [[-2, 2], [-2, 2], [-2, 2]] + + density = gaussian_kde(cond_samples.numpy().T, bw_method="scott") + + X, Y = np.meshgrid( + np.linspace(limits[0][0], limits[0][1], 50), + np.linspace(limits[1][0], limits[1][1], 50), + ) + positions = np.vstack([X.ravel(), Y.ravel()]) + sample_kde_grid = np.reshape(density(positions).T, X.shape) + + # Evaluate the conditional with eval_conditional_density. + eval_grid = analysis.eval_conditional_density( + posterior, + condition=samples[0], + dim1=dim_to_sample_1, + dim2=dim_to_sample_2, + limits=torch.tensor([[-2, 2], [-2, 2], [-2, 2]]), + ) + + # Compare the two densities. + sample_kde_grid = sample_kde_grid / np.sum(sample_kde_grid) + eval_grid = eval_grid / torch.sum(eval_grid) + + error = np.abs(sample_kde_grid - eval_grid.numpy()) + + max_err = np.max(error) + assert max_err < 0.0027 + + +def test_mdn_conditional_density(num_dim: int = 3, cond_dim: int = 1): + """Test whether the conditional density infered from MDN parameters of a + `DirectPosterior` matches analytical results for MVN. This uses a n-D joint and + conditions on the last m values to generate a conditional. + + Gaussian prior used for easier ground truthing of conditional posterior. + + Args: + num_dim: Dimensionality of the MVM. + cond_dim: Dimensionality of the condition. + """ + + assert ( + num_dim > cond_dim + ), "The number of dimensions needs to be greater than that of the condition!" + + x_o = zeros(1, num_dim) + num_samples = 1000 + num_simulations = 2700 + condition = 0.1 * ones(1, num_dim) + + dims = list(range(num_dim)) + dims2sample = dims[-cond_dim:] + dims2condition = dims[:-cond_dim] + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + joint_posterior = true_posterior_linear_gaussian_mvn_prior( + x_o[0], likelihood_shift, likelihood_cov, prior_mean, prior_cov + ) + joint_cov = joint_posterior.covariance_matrix + joint_mean = joint_posterior.loc + + conditional_mean, conditional_cov = conditional_of_mvn( + joint_mean, joint_cov, condition[0, dims2condition] + ) + conditional_dist_gt = MultivariateNormal(conditional_mean, conditional_cov) + + conditional_samples_gt = conditional_dist_gt.sample((num_samples,)) + + def simulator(theta): + return linear_gaussian(theta, likelihood_shift, likelihood_cov) + + simulator, prior = prepare_for_sbi(simulator, prior) + inference = SNPE_C(density_estimator="mdn", show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, num_simulations, simulation_batch_size=1000 + ) + posterior_mdn = inference.append_simulations(theta, x).train( + training_batch_size=100 + ) + conditioned_mdn = ConditionedMDN( + posterior_mdn, x_o, condition=condition, dims_to_sample=[0] + ) + conditional_samples_sbi = conditioned_mdn.sample((num_samples,)) + check_c2st( + conditional_samples_sbi, + conditional_samples_gt, + alg="analytic_mdn_conditioning_of_direct_posterior", + ) + + +@pytest.mark.parametrize("snpe_method", [SNPE_A, SNPE_C]) +def test_example_posterior(snpe_method: type): + """Return an inferred `NeuralPosterior` for interactive examination.""" + num_dim = 2 + x_o = zeros(1, num_dim) + + # likelihood_mean will be likelihood_shift+theta + likelihood_shift = -1.0 * ones(num_dim) + likelihood_cov = 0.3 * eye(num_dim) + + prior_mean = zeros(num_dim) + prior_cov = eye(num_dim) + prior = MultivariateNormal(loc=prior_mean, covariance_matrix=prior_cov) + + if snpe_method == SNPE_A: + extra_kwargs = dict(final_round=True) + else: + extra_kwargs = dict() + + simulator, prior = prepare_for_sbi( + lambda theta: linear_gaussian(theta, likelihood_shift, likelihood_cov), prior + ) + inference = snpe_method(prior, show_progress_bars=False) + + theta, x = simulate_for_sbi( + simulator, prior, 1000, simulation_batch_size=10, num_workers=6 + ) + posterior_estimator = inference.append_simulations(theta, x).train(**extra_kwargs) + if snpe_method == SNPE_A: + posterior_estimator = inference.correct_for_proposal() + posterior = DirectPosterior( + prior=prior, posterior_estimator=posterior_estimator + ).set_default_x(x_o) + assert posterior is not None diff --git a/tutorials/07_conditional_distributions.ipynb b/tutorials/07_conditional_distributions.ipynb index 2e46f8cd0..f84014a88 100644 --- a/tutorials/07_conditional_distributions.ipynb +++ b/tutorials/07_conditional_distributions.ipynb @@ -1,16588 +1,16588 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Analysing variability and compensation mechansims with conditional distributions\n", - "\n", - "A central advantage of `sbi` over parameter search methods such as genetic algorithms is that the posterior captures **all** models that can reproduce experimental data. This allows us to analyse whether parameters can be variable or have to be narrowly tuned, and to analyse compensation mechanisms between different parameters. See also [Marder and Taylor, 2011](https://www.nature.com/articles/nn.2735?page=2) for further motivation to identify all models that capture experimental data. \n", - "\n", - "In this tutorial, we will show how one can use the posterior distribution to identify whether parameters can be variable or have to be finely tuned, and how we can use the posterior to find potential compensation mechanisms between model parameters. To investigate this, we will extract **conditional distributions** from the posterior inferred with `sbi`." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note, you can find the original version of this notebook at [https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb](https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb) in the `sbi` repository." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Main syntax" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sbi.analysis import conditional_pairplot, conditional_corrcoeff\n", - "\n", - "# Plot slices through posterior, i.e. conditionals.\n", - "_ = conditional_pairplot(\n", - " density=posterior,\n", - " condition=posterior.sample((1,)),\n", - " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", - ")\n", - "\n", - "# Compute the matrix of correlation coefficients of the slices.\n", - "cond_coeff_mat = conditional_corrcoeff(\n", - " density=posterior,\n", - " condition=posterior.sample((1,)),\n", - " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", - ")\n", - "plt.imshow(cond_coeff_mat, clim=[-1, 1])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Analysing variability and compensation mechanisms in a toy example\n", - "Below, we use a simple toy example to demonstrate the above described features. For an application of these features to a neuroscience problem, see figure 6 in [Gonçalves, Lueckmann, Deistler et al., 2019](https://arxiv.org/abs/1907.00770)." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "from sbi import utils as utils\n", - "from sbi.analysis import pairplot, conditional_pairplot, conditional_corrcoeff\n", - "import torch\n", - "import numpy as np\n", - "\n", - "import matplotlib.pyplot as plt\n", - "from mpl_toolkits.mplot3d import Axes3D\n", - "from matplotlib import animation, rc\n", - "from IPython.display import HTML, Image\n", - "\n", - "_ = torch.manual_seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's say we have used SNPE to obtain a posterior distribution over three parameters. In this tutorial, we just load the posterior from a file:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from toy_posterior_for_07_cc import ExamplePosterior\n", - "posterior = ExamplePosterior()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we specify the experimental observation $x_o$ at which we want to evaluate and sample the posterior $p(\\theta|x_o)$:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "x_o = torch.ones(1, 20) # simulator output was 20-dimensional\n", - "posterior.set_default_x(x_o)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As always, we can inspect the posterior marginals with the `pairplot()` function:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAACaY0lEQVR4nOz9eZxk6VXfCX/P8zz3xpJZVa3eW2q11m6phQDtrELDgAxYzHhszD4sBmPw6xljjD3GxvYwHuN5PV7GC/BigxnMYoTB7GB4sREWHiEktO9Sa+vWvnR3VWVGxL33WeaP8zz33ojK6s6uyqqs7I7Tn/hEdlZmxM2IG797zu/8zu9ISoltbGMb2zgpYY77ALaxjW1s45HEFrS2sY1tnKjYgtY2trGNExVb0NrGNrZxomILWtvYxjZOVGxBaxvb2MaJCncJv7PVSBxNyOU+wPP//D9NsQI/FcIUooM4SSSrXyebSAaSBUzSZxRIMnoLL3YU+UckyfCOJ71JlHwP0gmSwHQgXjA+fx3AtiA+Ybx+bXzCNgnbJsQn3DJguohZdUgTkM4jTQs+kLoOQoAQSd5DjBACKUQ9lBDyMcV8f3mn5e/GX7jk9+Ol5qsf+sklP7QYxIjeWwPGIM6BtWBHX1cOjCFVjlQ5cIZYW1JliZUhTC3Jgp8ZohNCJYSJvs+hFpKDaCE5SALJMHrvyzGN/l+S/pwFTP6dcu64pN8zeq9vdnmM/GeXc6ScF14g6rkgQfJ5oueN8SAeTADToOfGKuFWCdvBH778ex/2fbgU0NrGNRJ+KsQa/AzCLBFrCJNEcolUJbB6ExsRk5B80gkgkvqTTjZOk/7zn4SUz/KU/x8UJ1IUYhTwBoIoeHnBdDIAWAsmA5lt9GS1jWBbPUGTFYxPOCeYKiCNxRgBH/RD3XYkGxVXUyShn7WUEoLV70WjByRy2cB1ReJigGUt0gOURZwF58DZDFQKUql2RGeI0wxYtSFMheiEbibECmKtoBUdxJp80UoboKUXrdQDzgBYCAMwCXreOD1njEuIJMRErNWvjSnnTSIlISWI0RCjEIMhBiEFQ/D5/IggnUFCwnhR0PJgKunPAwpoHiIeVaDlQ+Te+xc8+YYdPfkf5REnEGoFLD+DWCfSLIBLmDpgXcDahHMBIwlrItboiWdNxGQAAzCy/oGPSfTiWYAqiX4vCSHm+yR0nSNGwXeW2BliZ5BWgcw2+QTtREErCHaVs64WojOYLpGsw1QGW1mwgnRBsxBrkc6DCClEREzOvvSWkiAEBS49avLBXpXX/2FjE7CsRSTfVzmrck4zrAJW1pImTgGrssSJJTpDmGbAqgWfwcrPhFBDrPJFy+VzwGmGlFwaMqMenEYXqvw9MQkRMCYDlUlUlcdKonIBayKVif054yT2f6JPhpiEEA1tsIRoWHWOEAy+s4RgSEFInd7HIEhn9EJmIXVCEs3Ckjnc+/aoAa1//9r7+JeveA/33b/k1tNTvunznsRffMnTHtXgFV2+VflknURkEjBVpKoCVeVxJjIpJ6ANOBMxDKBlJGFImNGJCBDzJTmiYFVuACEZfDR6otaWLhiarqLrLMFbgrPghWAN0mm5kqyCl5args1XVWu1vLQiYASJCTGCycCjWVZCJJByKdhnWzGSxCAmkuI1AlQlDsqwFBn0Zm3OsBSwSlmYrB0AyxlCZTTDmihghVrwU4iV6IWq0uw6TBWo4jTmTCkhrmTYObs2cTgsUaAqmZNIwpmIswpOtVWwmlhPbQNOArUN+VwZXmufz4U2Otpg6aLF2YrWW1rr6DpLDIZgEikYktdzIFotIZMkxCsID7XrQ8ejArR+5tUf5G//ylv57Cdex5/7/Kfw++/+JP/od97FPZ/Y4x9+1WdRu0dnvyE6SPmkTdOATAOTeUdVeeZ1x6zqqE1gp2pwJlKbwMR4jKR8H6lEgQzAEvsTsgBUl3P2mISAXlV90pPTJ8MyVLTBst9NWPqKlXcs2wrvLW3j1rIv8YJdCaEF0wqu1lIyVgY7SbhGiFawrcU6g6kd0npkZREftJzynuQ9YnL2RacZlyRS6F8ZvTvujEvMcF8yrVISlgyrrkguA1atGVaYWFIpBTNv1c2NZtUTwe9AqMDvJGKdlMecRqSKVBOPdRHnApUNWJNwNvRZdXmvpVywJGFF3/faBJzR82FmO2rjmeRbJYHKKGhZiYR8UWuio0uWZahZhoplqDjnpnTBst/VtLXFB0PrNfsKXm/RG4K1pEYfR4JgD/kxPfGg9er3fZof+LW38cXPuIkf/5YXYo3w577gyfzI77+Xf/Q772Kv8fzINz6P6rCvyEkKGRGkNiFW0/qJC0ydZ6dqqY1nt2qoJF818wk4MR6bQauSgJGIHWVcJdMKCDEZQjJEhJAMXbJ0yeKjYWlrmuA0o3Oepa8wkmitRSThrSU4QzSW5A1gSEZINmlJYHNpkIlhCQYMSFSwNCKKQSLIKPtSXgtStEgIJKNNg7WM67h4rhFJKCZnkH2WJUOGZYwCllvPrgpgxcoQKiFWWgaGzF2Fmh6s4kQzbDPzWBeYTjsqG6hdYGJLaZcz7FFWXS5OpdwzkqhMyOdHZGZbKgnM8/34PIHh/FglRxOr/t+MJHwytOKICEYSncnnkjF4Y0AsYiB4Q4qJ5HLzIHKoONGgtWg9f/Xn38gdN8z551//XGwuBUWEv/TFT+fU1PF3f/VtfM/Pv5H/62uf86gDrtIZTC4hVdSTtvLMq47duuF0tWJmO3Zcw8R45rZlIgpWE9P1J+JUWi0fxQNgc7uwAJZ+bQhJiBja5Hrg2gtTumQ562fs+wn7vmZiPU1w7FU1TedovaN1eqWNzhJbg2kNyWjGpSSsnrgkiFUGLCvYLmIBnBk+/NYoYInovTEKXDBkXOmQn4CjjouVhdYqAe9cvh+VhIVwrzPZPrEKVBPBT83AX00gTHKGVSXCToRJwE0C01lL7TynJi0T65m5jqnrcBL77LpkSkAPMCW7NpLWwGluGyoJ7Jgmf89jc5YFShEEhEWcsIoV582UKmfyAEtTAVCZQBctlYmEJLTe0diI95EmQhRL8IJp0O7jIeJEg9YP/d49fOTsil/4rs/j9LS64N+/+fOezKoL/IPfeief2mv4kW98Ptfv1MdwpFc4MplqrfJWE+uZu5Yd1zKzLafdiqnpmJu2B6uptNT9Serz134t24L1jEuzLUObLF0GrnOmpUuWiXj2TMuOU9BahorKBJrK0XjHwlV0wdLYCl8rcCVjMZWAaFkYXebMWkhicZUQS/nQGYwIxhjovHI01iqnJX7oLBIQ7LGXigWw1srCUhJW1UC655Iw1nYAq2mWMdQqZQmVEGZD0yXMEnESkbmnmnimk45T04ZZ1XGmXjK1nh3XMLNdn1VXErQMzKBVwGecYRfAqsUzlXyemJaK0GfloOdEQOiSw5LWvg9aMpZzaBUcPloqE5QDtQFnHa1Vkt6DUgiVxQQOFScWtN77yT1+7A/ex1c973Ze+OTrL/pzf+GLnsaNuxO+75fewhf/49/nq553O896/GkE+Pj5Fas2cGZe85K7buLpN+9evT/gCCPJ0P0pXZ7CX02MZ27aDFoN0x60OqzEfHLqSV0zlIl9tpUzh75MzCdrmywdFiORVayI1uiHIiRikr7DtPJV/3WbNVZiEt5YYhCiMYQwaMGMV3LWeEGSASIhWIwI4iMpWSXnvZ7hEgIpGSQZUoyICAkl8686OT/KsoZvjcpCMb0mC6PZVnJGS0KnZWByCt6xlIWVylpCIdxriJMIdcTVnsmkYz5p2a0b5q7lTM6uZ7ZlN2dLU9P1oGQZLkilRLTEDGixv5AVsCoXM/3d4ZzoksWSCEbL95VUTExHSKbPtryx+T5iQiQag42xf+6mCiqRsKmXaRwmTixo/e+/8XamzvJ9X/HMh/3ZP/O823nGraf4kd9/Lz/1hx/Aj07mQnv8oMCffu7t/G9/6jPYnZyQl2WDvtHKaQCscuKesismpmPHNOyYBoOC1fiErIlU+UpcSzxwVCJCBi1Dlwwthql0dMZRS2A/Tpgb5UAa2zCzLUtX00TLXjehjY79umbZVTSdY99O8J3FW0esDLHWP6J0GWMFtjEkEWwlJCPYJmAar8dXMi4RzbjIpeLFyPkrmW09pB4rl4V11UsbUl31GZZmWSpriE5ULFzrvZ/r6+B3lL8K84jseFztObWzYqfuOD1Zcf1kn13Xcn21z9y0zG3D3LRrWVMBps0oWdc4oyoXsGkGKzM62bT9EWklEKIQxDA1HV2yRGOY27YvOWfR0iUFZp8MTXDUJrAKji5YUoLQqXhWwqO4PPy9d36c33/XJ/nbL7ubm05NDvU7n/H4M/zwNzyPc6uOB/c7QkrcfGrCrLJ84nzDv/mv7+Mn/p8PcN/9C37y217IvD45L41stIoLX1E6Q1r++QxQngq9+pYTswBWVUoFwMqFM14RqEhYcuubRJSWFYmp6ZT3wjC3jXaYMMqVRH0tfVRuxZmIKxyHdTRB0OuIwXTKVRkPIQGIlg1ikICS8ckiPmYiPqpMgkzKMyLnNzOuq0nMmw15g4y+dlY5uky6JyskO2RYoRr0V7FSPV4h3qkirgrUdWBWeeZVy27VsOtadjJQFcA6bZZ91qTva+wBajNKiWiJ/UXMSKIiYvuMTIvvzZkYm39vzIuVk6eSwCRpZhaT6bNuI5E9O6GxEbFJFfj2UQpaqy7wv//GO3jqTTt88+c9+RH//ulpdQH/deuZKd//smfxWbdfx3e//A1818+8nv/7W1/YE/vXakgeq4Hhszh0hQaOopSD5VZJYEfazEdk3ZYkBSSBiqyb4uLAVUmkSxEMVCkQEP1Q5Ct5Ix5DojOWJjpq4/HRMrMd+65m4WqMJJauIiWhM4loHD4ajMudQNGMS6KOpZAMSfTYJCbECiZGKKCUSw+JaeguBrKO6woq5w/MsuyQZfWKd0tyttdixUozkFAPOqxYFcI981l50iHOEmkSsDMl3ec5wzpdrzhTLbnOLdi1Dde7vZ4K2DFNf4GyxAv4ys0otMAYrMpFbAgFLiMJm9LodyK1eKII88xzFq5r3HluoqMygT1fs19NtAPtKjqnwHWYOHGg9S/+83t4/6f2+alve9GR66/+u89+POdWHd//y2/lR//Le/lLX/z0I338I480uqGfxXiAQE91OMpX9ASrhAvAygAWBYXyytqN4USLJhGkhBEIaPmwI22PcAFDnTuRXbJMjWWe9ERexYqdULF0NU4Ci6pGJOWrbiKkiuQMklBphAOS6Bxj0vJPJREOYwQJSUHDZEmEyEDKh5BfFLMOXFciDlK9l7LQWdVjVU67hRM76hSOhKOTLG0oncKpjmjFSSTNAnbqmU47Tk8bdqqW6ycLTlcrTrkVj6v2mYrnlF3pxcm07EhLJZ5p32C5OCgUvqqcE8P/D1HogSI4DqNzw5KYSofJ0oqi6+t/N/OhRR7hJHDOzVj6CucinUmPzjGeN3/oQf7VK9/H17zgdr7orpuuyHN8w4vu4NXvu59/+rvv5nOfegPPf9LjrsjzHGmMgEuV6weDuWEg2cfc1RiwDhMWKWIpqjw5XUnQq7p0dKKn1cR0VMn3ncaA9GRwJQGfDM5EmuD60aBVq7la6KxmUEkwNUAeto165TZd/lB1EWJEQia4kwpQMTpoJzlbY9yZOupsaywiBR1BkhHxnjVZhXhP1qwT7m5UClbSfx1qFY+mWseyqiowqVQ0PHOdEu6m1UaL+NwZ9kxNu9ZgKWU/sMZNXfC+jspA/VmNAvPaQVawGjdmStkZELQ337I5iB9RrV8lIZ8Lhtp6Kht0lrGMFR0iTgxoPbDf8pd/7g3cuFvz/S971hV7HhHhB//0s3nDvQ/wV37+DfzH7/6ia5qY7wft81xgSnol9NHq1S4LQ0uYzF3VEqkephR8uLAieoKnxFQCgQ6MnqBV8liU1+qSzboeQzQrOutYJcfMduyFiWqJ7Jzzled+oK0dQcgfbgDB1mRphGKARINxml2pflMUvIzR8R5AJCrnFcJo3Ofoy8SxvKGIRnvVe1VlLZYbtFgTh88D0H5qetFoN88D8HMI04SfJdKOx0wCO7sr5nXHmcmKG6b77LiWm+rzzG3Lrl1xyi6ZijZbhmzar12kxhnUYaJkViGfUyFnWC2GmAzt6DJXQLI/NzKHNjyWSmUWUTloQ2LHtezZCcbEwYXkEHHtfhpH0frId/3M6/jI2RU/9x2fw5nZhZqso4zT04p/9rXP4Wv+1R/yA7/2Nv7xV3/2FX2+o4riyBDzSdYlm082PclivkeGxP1ipWDoBaYHn+QRCKMPvSFRE4kEulIaSuh5rpCzkJCELgWq5MFp6drWqp4GWNR6rV61lhhBkkHPc0EmJdMSQp3lEK1BvHahpMsCU5fl1SkpgOiLc+XKxAJY1uZBb8PaIPSm6r0aJA5DVqWA1c8S5llSMw1UtWdaeXbqlp2qYce17LiGuS1Z1iAULiC1Jm14iJJvHONX5GJgBfSZEuSsrBD4DIQ+QD1KbwPCKunndj9OWBjfzzE+Uub4mgets4uO/8+/ex1/9P77+edf9xye/6SLa7KOMl7w5Ov5n7746fyL37uHz3vqDXzV82+/Ks97uTEMNeuJVpTrbS7PyglILgm1LBTNVPrHSFj1Txi+N3qOcECGYvOISIWWiQBTkwWpMsr2BAIdMZcK0wxwpRu58hXORoI3dFQEAQmWZHTsRwl6tTZJYpBcMgJIO2i4JDoFsKKUt9r7IuZaURT0jiTbKkp9yZlW8cY6UN4w4rGKgDTzV0rAJ8JUB59lGphMO6Z1x5npilPVisfVS66rFj1gzU2j+ru+HPRrWrtNHmu4WI3fz9HXlJnToskzfSlYMqtN+qGU/AWwxjKJUnKukqXK+pPzpmMaPc6Evsv9SOKaBq3XfuB+/sYvvpn7Hljwj/7sZ/GnnvOEq/r8f/lL7uSPP/gAf/OX38LTb97ls5943VV9/ocNYZjZE53UhxFwZR6hywr2VazppKWVQKe6cSIDl9VnWuWETppplayqXDfXAWz4f328pFdYUe7JSiKgpShrv2f0BDcdoKDlTKCNlomd4oNhXxLeVISQ1fJa82GsmshFKxnITJ5brBR8Q0T67mEYysTSjUxHK4NY6xSOiXdnSZOaNHXEyhKmNvthmUGLNSbd53kAeq7D7/Ws4/R8xazquGG6zynXcF214IxbMjdaFk6lzYJhf4GcQfmmw/9tJbvqMD1YrVLVc1djmsFmYCxTFFPxvWxmKtpxrEX6C1yF/lyXwXXcxQxjE8FDxDU5jPfJ8w1//RfexFf/6B+y6gI/8+2fw1e/4IlX/TicNfzQNzyPm3YnfMdP/TH33b+46sfwsLHxPg8uDWZkKWPWMi7lmGTtCjuOAl7j7KuEAthw60naja7lWGU9lCxDJjaWYJyyS07ZFWfcklNVw07VMK076jpg60CqI6lO2YJnVE7le/WZysryyq6VZGoBMyrdMkHe81CwNuB8SSEydAyzmV9xbsDpiEpyphfQ9kr38vfUw98VJwmZRFwdmNSeWdWxU7Wccs1aSTgxXT+KVWYCx5MM/fuVS7y19+aAPzf29+sZ+nDOODp0CqKQ6vo+p54nnUpgKompCFMRJmKYG8tUjPKno4bA+PhS0qvvYROuay7T+pU3fJi/8ytvZeUD3/mSp/LdX3LnsQo9r9+p+YlvfSFf86/+kG/88T/iF77r87jl9PTYjmccJctKJmUd4+CLVK5khQjvkmOVKlapwqZIkzqsJNpCZCOYCxrc5BNes6wCWIWghQGsYg90mYDN5Z8WZYOViR2duGpx0vXgVklgNamY2Q6fLFYS+zZwPgnBWQJKztui43LDiZ6swQSnn8gYMSnl8lBJ9xSHzOuo+S0Zk+6TWjOsWZ1N/Bxh6oi1wc9Nr3QvpHs3V6DyMwinAtSR6U7LbNJyZrbiptkep1zDTfX5frrhlF322ruSYVUccnDvgBhPOqwyUK1SRZvv10BKIoHItKjoc3dyLoGpwFQME3GZKzWZ5Ux0dDSjjDYmg4/qFBKjqPvtIf+EaybTijHx93797fyVn38jd992mt/+K1/E3/yKu68JZfozbj3Fv/22F/HpvYZv+jd/xAP77XEfksaoPCSXh8V+xBZFPAW8iq2M09nBXAJ0GYRiSsQ18lZPjZJtjQn7vvWdr+IFsApfdpCI0cqFauySIWibXgWRp+yKHduw61SLNK07qtpjJ5pxxTrbStcMIsyJDMLMyhDzEDKV8kvDvJ8M2dZYOCyX+TEo3u7ODVqsyhJzt1A5LMHnW5hk19mJAlaYoOZ9dcROAtO6Y153nKobzlTKZZ1xyx6w+izVtBdcBIqq/eHioL+4cJ4B6WdL+znTwoludKL7LFqgEuVGK7E4LJXYjQthAcfiEpItj6J5ROXh8SMC6o30937j7fzkqz7At37+k/n+l919zdnIPOeJ1/Fj3/ICvvX/fi3f+pOv5d/9+c9h57ilEDLchMEbyZmAO+DkLVlXOQk7Ip1EQs62KiASB8DCZPGoZlub0eWfO0jQCloi5oHANSEi0INaD4RiwNK3xPfrCb64BkwcIolVUD/yIEZ9t0TtnMtCBTs1OFBrX+80o6qcvkTBjVTy5khdT5V0V7AqZn5xkgFranWmMFvNhIl6+/uploRhCmEaSdOIm3omU3VsOD1ZcV296DmsM3aRVe7tgWM5DyVlGPNaB5WGPZeVaYSSkcciV8m2RCqX0y5h/9ii0xHaORQqLAbBypj/EkJMGRBHVEXU+cMQMjd5kvy0fuwP3sdPvuoDfNsXPIW/85V365XwGozPf9qN/PA3PI/v/Ok/5m/+0lv451/3nGM91j7LQkdVrIm9pfJkw3VyfFLryehYkahSZKWFV54fk/wThbOQTKqXqzk5O7tQFV1i7MO19v2+g6jloxl94OpcG5yySwBWVUVIOi7SBIezkRiFFvURCzGLRgtgGsF49eiS4CBqg0F8VN+tMOoU2pFOK5eJlxVVHoSe1sRZRawdYeb6IWhdQCF0O5ueWOB3A0wibubZ3Vkxn7TcONvjTL3ixnqPm+tznDIrHc/JGVY9MuPbjHEXb7N7aDbu9T1hTdpQMvGSDbVJOSzI8pcE8WEy00jqS+4uBToCq6QdxOK/tYrqeOuj0RI9Hlpbevzl4R+979P8f//jO3nZZ97G337ZtQtYJV76rFv43j/xDH7tTR/h377qA8d9OBpjv+9RtlUGWM1G2VAM3AqPEYGO3CnMH+zIZiknByrmxyXFWCl9wc+lC8sEbRaUrC72DgMT0zE1WfFt1cxu6jx17XFVQOpAqnKnrRD0a4pyUdK7MiQ32MCIyXOAZX3XAVYylxRlpjA3AlKeJ4zZ011vI8J9Uu4T1BEzyVqsulMDx6rhlFuxWwags6RBS8LMYxGpKVKD4XapMVa4DzczAFYyFzRb1n9fI/b/JTyBiM6odvlcG3ux+dwQSkn6bPlQL/cl/5VHEPfvt/zll7+BJ92wwz/8s591YpZQ/MWXPI3Xf/AB/sFvvZP/5hk38+Qbd47nQMr73AMWuOzlPfbRGsSHWcOTPb6DqL3MKlkgsCJRC5A04woXIXcPGuU48OeymLX8e+zLyZJxDU6pViIkVVZj4IxdqmSCSDNx1DYQkjqVNtbRRCFZ25eQyeoKM7VvzhyJaKZlclYlKSHGkGJYd4W4TMlDynOFcVrlDMvid5R072bSzxL6+aDDCjvKYVU7LdNcEt4w01nCWybnOeOW3OjOc73bY8c0nDIrakI/hFwcF0x5bTe4poeKoeNLvmAVqyG7lmmFJAdcbEyvvYvZosiSNJvKpHvR98WUWKVIByxixfk4ZREnLKL6yfto8MHqgMJoFO3h4tgyrZQS/8svvokH9jv+5dc/95oeldkMY4T/4898JrUz/K+/9rbLPukvOdLGfQ6TS72SuZSrcy1q6rbpqaTEq363TYmORJeikvP5FkgXQNj4irxeVuhtrO+JbF7FJX8w1jVAhZifmI65bbJddMvctcyrjkmlGZet9UOfCjGfCe1Cyit/ZHoV+iB/MD1xviaDuJwYqd3V232UYU2Uvwq1juaEiTo2MA3YaWA27diZtJzK9tinM+l+xi57p4axO0fRQhkZZCSHjYPkKm1ZAZYnKMbvRcxWQ+U2dlnryfpkWSXLKplcAiaaFGlSZJVi/p6wyoT+QOrrNqc4OoevecnDz/7Rvfynd3yCv/OVz+LZTzhzXIdxyXHz6Snf89K7+N9/4+38zts+xpc/+7ZjOY7Nj1u/Fqz3NvL9/dhSuRC4Ot4TiSJ0qVzFEhbN2LNuvI8w4rNimSfMaX+JcZdwDbA2y0aJgFkDXdV1GabS0khFZ3RkJSZh5Sp8nUdJag+SCEGy86lmM5LAT9UBVaL6rZMS0lkITrMtr1bPmJAdIi7v2q2AZYhlGcWoS6hEu/ph+VkiTXQ3ZTVTHdqpacNu3XDdZMn19T6n3Yozdskpu+S0XbEjOqZT/K3WQGrcH5E4rH3LfGHxvooifeY5frHbbBnTjudD+4vIRbLnJHRYbHIg9KM5TX5cveAVioEsoTA9ub9KdV6KoiNmZV72kVw2jgW03vmxc/z933w7L77zRv7c5z/5OA7hSOJbPu9J/Pxr7+X//J138dJn3XrV/bckkh0edNtziMOgdEzSmwEWwJpKt8YdjeUQZfWwIeUO4oUdqS5dyGGVk73LjFcp6cZRgKt8XUI1PHEYvKYAnqcWy9w0ACxijSVm/k3JeR8MxjhW2UAwGIPxpaM4aH5sYwCnZWLZ5hOz+iy53E28PCJ+PAStCvesdJ9qSeh3EmGSiLsBmQQms46dWcNO3XHjbI/dquH6esGN1R67djUqCZdrGdbw/gwzphcr00OyRBIhl91GEl1af0+HUZ2ReDRnyGNt1vh97KdWY+74GjSDMh1TOmxK/bGWecUuWfZTzSpV+lzFzbQXlnLo0lBfg6sc51Ydf/FnXs+pacU/+ZrPPjE81kHhrOF7vvQu3vfJfX7jzR+5+geQ1oGrODwMJ/VItcy6B/hBpnDFK6lNJhOnrN36Ido0lIUFsIoGLI5KxfH31q7iDF3HoTO1fh4MWaJquLQTGpjaTlehuaCLRV0El5SY7xfXZtV8JapAr3OZWFxDi4NodhS93PIwZY/3YjdTfN1jWfWV7WVkEnql+07d9a6jp10zIt3btXKwdAFhXbrQi3vXCHSzcSszg6YHiWKV3fXfH7RYF2TCG1EAq5SN62BXpBIul4t27fHHHcl+gH8zm7sWy8MQE9/779/Evfcv+Lnv+FxuPnVtKMsvJ77sM27lmbee4p//5/fwlZ/1+KuabUlEW8VBiDFv+g2WJrh+iWYBGLWgyWaArH8YYDzVXyTmsDnIOuY+WorWazjpN7uGB231GUfoP4xmTW9kZRi4BjhjXe/DBOAk0kWLNbG/WgdjCa2O5hgv2W884RujizKKPxdguqzdSulIll/EiW7SUf2VZll+pq6jficRdnWWcH6qYVZ3XDdb8rjJglOu4cbJno4w2SVn7D5T03HKLJV/ZLA9BtYcF9Y4qFHDY3M+EOjV8uVCpa/98HtFSFrKeH1PYj8/qu9fucjY/mJlJRKioRJLJ6638t6kBzpsL3Mo52R53+J4hOdaBK0f/M138Ltv/zg/8N89ixc95eq4NVzpMEb47i+5k7/4s6/nP771o3zlZz3+qj23JJAoECEGIQRDFy1ttDT5BNGUXBdPFBM2GMqE3vc7xzgLYqOcGE/7j6+gIZUPzXpb3F4kkR+XpZslaGkSWBEdvDawSh0BYdc2/Uk/c52WH5XHe6vAVSdi0I01ppUsOch2NnXetVhlM76UEB/yiq/LKzjK+rPoZE3WEGqI04RMA27imU9aduo2uzWoPfIZu2RuG07ZJTum7cdyikyl56VG79HYz2q8Iam8pv1xpaEkt6JFfzc67rUmyUWyrOKJtvb3Jr04hGTy++Xy+6g811jGUrLwnszPAtbCiyayfO5aJOL/9Svfy0/8P+/n277gKXzrFzzlaj3tVYkv+4xbecqNO/zYH7yfl33mbVdNayah3GQAraBK4yY6VtnatogEpxt6pE3g6ku2ET+yfpKbtQ9JKSk2RYglxknMWsdSLuS9Lli8kIljQ2Qnc1udtX22tQyZAPaOtspbXSr14IqN1fIwQKj0NbK5TJSoXT5JSbVb2cLmckI1YdJ3DUs3M07V172e6m7CYi9zw2TBDdU+c9uuebqXtV2bq74uVgqO+cSDwKcfWJc8NL/xPmyW5uu/mwWio2xrDF4hKaB1OGLWiA3HOVx4+g5xoRVKt7IM80cDURXxbJaLF4mrAlo//eoP8g9+65287LNu4/tfdvfVeMqrGsYI3/aFT+Hv/Mpbed0HH+AFD7GH8SijBy0v4A0hGFbesd/V7Luac1bL77nRWcneTdKMMpoLpu6HD8UYvNbb4OM5Rtu3w9eI2hz9h68I10cfmiK/WLsfDeLWKaiHk2Ft+3UlgbZ2OBPwccjuuokjJohTQ/BoR3GqpUdoBTPRTqVptKOIt4gPKjq9jOhFpMVipiykmAXc3LM7X3Fq0nLTdI9T1Yqb6j2ud/vMTcN1djHS0hWn0XFnV2OznCtOoMPrPnRzy+veJd1LaVNaA7CLhT6/IZQLCIY2lXEsHSzffH9LBDTrCsn073VM69ZIY1FpG3WJSQhGh6Uj187A9Mtfcy9/51feypfefTP/7Gufc81vuLnU+KrnPYHr5hU/9gfvu2rPKXG4EYQUBB9U/7LKvFYT1dq48AmFNH0oUSiMwKsMtZKv6KNRj4GMP0DOkOOgk1zHhcrVOK4BVlF6q5By0JYVK+Fev2VaVcvbrJZ3AWsj4hLJJZKD5FBy3kF0uqZLCXOjpLwxw0D1ZYSuABs9V5WIWUdW1b5f9bXjdNv3mnmfGcz7DmqMBGQk+hwyWm2arIPC8F5INn1c57oesgzsu5OxH8JWIXKxvSkXufX7h4txg6C3u8nuDqE4PGSK45qYPXz5a+7l+37pLbzkrpv4oW943jU3BH2UMa8d3/g5d/Ajv/9e7rt/wROvn1/x5zR+uIkXYmdpvWPRRSZ2wtRqdjIxnjh67YMxWJOo8CAPPWw7zqxKOdH2oJfTftbLh0LaXuiaGbP7RFyTYGzuYOyPE12EUZZjlA/PVCZ0lWOS15KpLimx6vR07jpD6JQL0kxLCC14r9IOW1lMTOrIEB4Kug8Xoc66rKn0a7+YBeq5mvhdP1twXb3gplqV7o9z+5w2Sy0LpVkD7OFvz3q0XA73F4mc4Y4zq/IeXEDCJ82QDlzQOnrPy79XEvJ7qRnumJMcP4YZZYPKwQ1mgP3xjzRjIUnPsTbRsQwVq1DReEvwFvHSn8eHiSuGIv/+tff1gPWvvun5TKvLu5qdhPgfP/dJGBF+5o8+eFWeT3zqAUtLRMF7Q+e1g7gKjn0/YRkqzgcdodiPEx1azQT90IE63Ed380p90C7qsYVuObnN6ApdytSxfmwqnmnWI00kMMlrzqbZkrnIAIpK/JTV2bxT1YrdqmHmOqaVp6o9UsVBAjHedNP7sueZRJuFpZdbHpasrhokDraO1NnEb+5adl078nRv16YULgZYpbwKm9lSzx09BGAxbNcZR59FEdfeg3KrxK+V7L07aQEmiWuLfzcFy3YjUxzPGq5ipZl/cDTB4b0l5nN3rK17uLgimdavvekj/I1fejMvvvPGxwxgAdx2ZsaXfcYt/Pxr7+N7vvSuK/53mwAxlEyLDFqW1kaaYFl4XYi6F9Tu5bxM++5hsTkGLihLIMsVksmkquHCEZ6BSzqo5OjLiI0PkJHhg1I+EAdtuB5zOeUY9QH0MXsldtIrt0+Gc9WUEIWmDoTKkLyQKiEG1WzZCqJPWiJm4JLcSbycKNqsUGlpmOo0LKMorqO2Ydeu+mHwIg+42JxgKes2M93SqdXXXQ68aOjLtH6RGLvIbpaA5ef093SWsE3oHCooIme+i9GZ0Ptp5cfuj30EumNNl2ZZtTaKvMN7Q/JGz99weE7ryEHrdR98gL/2C2/ihU+6nh/75hc8ZgCrxDd/3pP5rbd8jF9740f4mhdeWYto04KxCdMKphWSNYTG0gILq6VYSoIzER+HDSorq6VWsToGPcnL9hQFiJhLRy2zaoG2jIIkk1HS9R0moNfZjPkR/f9iexPXruhFQKlZlfrIl2WxxZM+krD4fhNMOb7OLpT4RVSzJYm9iYJz6x0Ln7tUU+1K2QmETiUioTYQE1I7pAvIYbeEXiSKqV+cpN65YVp37NQqHi02yWorM6z1utiK+rFU4aAweWRHX9eQLXyGxxq/3kOmFHsNVfleeS3XfjclyNlfKU3LgP3YkmYMfGuvBYPLR8DQJsci1pwPU/bChH1fc76dsN9W+NZBZzCdYDswHYeKIwWtT55v+M6ffh23np7yo4+hDGscn/OU67nrll3+3WvuvfKgFRJmxAfE3EWMLuKDCk2dcax8RVkcMdHNp+wb3QhdRXWFqIGwcRJa8kBPHgXRcRvV5ZBnFktXKY4EojDwV+Vxen+nUdeyfK+WiBVdhKDjQ6LyKejBq5aYxZUhc12qlJ/GjrltaaJj7lq6YHUBqIuEKhKdQYplTSHkHdmyppSHl5lpWd3HmCwklxAXcTZ7m+WRqZLhjjOSsW6u/x7rZV5MJktSBsGnXjTiaO5wDOj5giGjEm8DsMwFWdZ4njF/L29sAjBpWFAy/vlNrmxzoDqMvOb7plBQ4z8fLMmbviw8tkzrH/3OO3lw0fKzf/7FXL9TH+VDn5gQEb7mBU/k7//mO3j3x89z1y2nrthz2SaRRLAN2JWQJBFXhpgcjY7hEaIgkmjzgCqotYuVSGMqQibop6ZlB3IZMBrMFd+XDHbU0i4q9jbZvk1+wfGNuoTF5K+4OAxZgJ7qFVAhVGIoBoQdgZizPvX7Klogz460WJOIVolpw1AGN0Hb6Usgrmw/TG06QWLCTxRobWOQzvYziZcaw7INoIq4KjBxnon1zGwex2HdkE//DjMabjZrgDMWaOrrV8S4atBYMZjxHcRnjTOhcWdy0yRw0yjQFmNHCTnrKiC1jiib0w5FBBtGI12rVLOINYswYT9MON9N2e8mLNqKVVNBazCNYBvBtGDbw70PR0bEv/lDD/ILr/sQf+4Lnswzbr1yH9STEH/6uU/AGeEX/vi+K/o8mmElTJswHdhWMI0grSG0hra1NF3FyjuWvmKv0xPnfDflrJ9xPkw5H6fsx5r9OOlJ036Uo2RCDG4Rteg2nbG2qurJ2HVDus0refkArvEyhbPqt1wbzehEKPsYbZ+BpdEHbViGUaQDu7bhlGvUNLDyVFX2lZ9EQq1K+bLFZ0zIJ3eZLg8ly7IJbEJMWS5SRmZ00LsbkdLjOcxxOdW/tyO+qF/8wfDalxJbM862/7qXT2xktptxEGDp846WumaQKxeZwkduvtflfV6XYOhURpPJd+0Y6nnYeqddw070Vs7jQ3YPjyzT+sHffAc37NT8z19y51E95ImNG3YnfOndt/DLb/gw/8uXP/OKST1sG3ULTadXqmQ1m0gGkrN4myAJC1P326eNqEvAJGROI5cvEaMGfJSTNfRfA9kFYugOlVIh5Cs/yWElXNCFfCg5RZmJBC0DJ8b0W1wA3ZWYQgYsjTI0bHvA1DKxS+q91aWG3aphUdX4YGjqihTyeI0bBqmNL9otQ7rM92cALRCTVC+WF4yMIyY1Xiz+Y+AVqGQorMZjMyZ/X/mr8bbmMlM4ZFmDXGHMJa6X4uXrzdgc1TKSNMsCkJBL1HUCvv97Dug6Fy6rdAyL6d/C16y8o20tsTNIl6mNbrgAHyaOBLQ+8Kl9/uj99/N9X/FMTk+v7Mr6kxJf/YLb+e23fYxXvvuTfMndt1yR5zCNKpxdpWJKSTr/JgG0ce3oKj2RfTB0WXi6dBUxCUunV8GQDI2t0JGZVk9+01Cn8RU/YEpWMCoZ27zqq5C1RdtzEIkcMZkb0+5UlYneKoNhkxQsq1x+Qrm/kOyw2XalANfcNJxxupdyWdes8pjPclbRkhXx3qhma6qEvJ3qPCKXScTHkmUZzbKGv1fwSQfZV1IxMR3E3NEzI2BJA7CMy8T+b90AnLVZURk6dXDhRaP/+fx+tJLL9tF86UFqlzLaZXsrkQvDSFQLnCzLKCvH9rM7aSHfz3dTzrVT9toJi6bGN460srhGsCvBtrmpdDWJ+F9544cRgT/1nKs3LHytx4vvvIlTE8fvvv3jVw60fCL5hO0S3gumSznTEqxDW/sYfKcfyrI5KyahMsr/GBIzq2fL3DSqx0leAUVgmnmknufKUojN7iIjwr6Q8mPgGpZa0LfTu6SDtp0YuhSpSARJyjEdUh5dSlR1GQjZW16dTpvgqKqA95Ywtq5xQnRJwcYZkrt8p4fxkhGgdzAoZLRLkS46rEm9sh2gGpPcyaxxRcBaqTb8zRolA1IOqjzxhaNXwPB6Zk7Sjl/jAxw9DjqGcWyODY0ticbke5mDbYKjDZauGwh405UxtFweXmx78EZcNmillPjVN36Ez33KDdx2Zna5D/eoidoZXvKMm/hP7/gEMaYr4htmGs1AkqVfEZ9sPhGSXl1jmwhJM64Q8nyiU//vpa9Y1RURYcfqDr0uOULeMzWVDmuGTAtYAy5L6InaQFzrMhXCfhi2LR8kJe675AhGBsdTo3mCTboUoboI3bq5LVl/NWbhqc5YrlzFstJG0N5US+NFawmdIMmoSj4KoRHiSh1OLytGgFWcOH0chteXQY9lmjMtI8WQsKVKo4+g+AFUuLDDd4G8oGjf+gvHaAh6Q4jaJdu/T2XDknYIcxZ7SNzeHKrfXARcysG9nGWd66ac66bstzXLtqJrHDQGs5JMwmtDybYJ21wl0HrTh87y/k/t8xdf8rTLfahHXbz0WbfwG2/+KG/60IM8947HHfnjm05Bw9RGOy+inazc3iMZHURN1pCiXvnbBMEZRBIhf1idBDpnmdlW7UZy2YWBOoVcDq7byPRaIYoTwMVLrE2yuSO35mNNEI9NkaxhZT96KkmMV4VEDioQh1DblchUWqII82zR7JNhXnW03rKqK0JtiV3KJPygkBd/BBeUNNxiVNDySWUn3hp85ndMzrQURBxdGrqqppcyFOAaLhRDxpXWnnPdVmhdiFouCMOY1eD4AAx2zZmLvFhGNY7Ngfri9lH8slZJyfdVrFiGmoWvWPqKZVvRto7UaUk+1mYNo2hXCbT+41s/Sm0NX/6Zt17uQz3q4r+562asEf7TOz5+RUBLuoARwbQR69TjyDZ6JiYj6IVUifkYikLZkapIYxIxGlLeLeiTZWbVo6qsXIeRVOEhuI21f89t76IxKrE5SzcM0oqCFijfYxJTsrNq7hqGAyQJQ9cx5g9SKRNDtnpp8dYycx1N5VhUQXVbffdwGKQ27jJBq3hBRSHlW8iZlk86UmVIzIyhIRsaZvuZAizFc105v9D/jeMc8KBO3yZw6cu4PlA9TBWoti4wUrCXUhFgpNM7CKzK98duE6XjXPisJlYsghLvw4yho8tzhhQCvgBWp/TGVS0P//gDD/CZt5/ZEvAHxJl5xQuf/Dh+9+0f569/2TOP/PFl1UHQleTIcJE2vkzN6weUlG2HfSJ4HWFpo+CrQNdZQhRWtZ4Ky6pSMlgiK1sp2ZpL24oLnQgeSRQ3iL61n6AWPel3TEMnegwrCQTTUaXUb0QOaXA92PxAFcuVSgI7punHRgCuq5fadOgcXet0EcZUOZUwEUKVssvppUcxY5SYtFMZDa23OBtovKM2XpfOxkrvk6NK2r2tcqZVMs3BWkYGQBmByRhUNpfl9rOK2YljFau1cSsrCZtG2rmSv+ZylHShQGI851iAa2yNs8hSmbIabBEmWfk+Ya+bsJfV723rCCuLWZleV2ga9Os2YdqIaQ/LY15GND7wlg+d5flPOvos4tES/80zbubdH9/jU3vN0T94CEgISIhIFzFdzGR8yh2ZlEV7enKYlqzjEmgNsbX4ztJ0jmWrafzCV+z7fALGuidXi9VJiU375E0niLUFFmk02EvxKre5XB1a46tU5QUIrl9LVfzpIxy4IWb8YR60ZGUFmdrXzF3LtPJYF6CKWbleZApqWXNZUZw3Iznb0m+FXCL2y0YYLQKJrnfxHGxlyhjzxT+WDzfcvu7HP7zu4xX3w0yg9Iss+tKyB6nRHgAGi5sxYJXzoifgs8V3Ub+30dLljDN2BnzWZHXa4dbsqqjhE3I1Mq23fvgcbYg87wqUPo+WKID+xnsf5EufdbRdRGk6iEmXkUq+0gva2r8g09JSUQLETj9Qmn0JqwRdZRFJdBNtYVdGea65aXW2zUYt4wTq0bm16Ze1Odx7YZkx4kOwkKATq4tjiw2L6Id4Kh11iv12l4OyCyjOBYOEoByDJXFdtSAm4Xw1ZW/SaTNikhBP3pF4eEuUi74PcXCQJQopNzy8MQNwJbXCdtlJtksWm2Jvk2zT4P5ZiVfdW5Z1DN1CuWiXbww6BaB6gMyvq5HUWy8TVS5RXB1s/j2VXZTnW+9EjheYlA7oIk7okuvlDYtYr2VZi6amaSpSY5HGDOr3kmU1CdtEbBeVoz1EXBZovf6DDwBsM62HiGc//gzOCK+/94EjBy28Vw1D55FWFU0267KsaHkRouqI9HMveWxNu4xFnhCteqyvqgojUNnAuW6KIXHGLbFkz6sU+g8SDFlOKUv66X4G/uOghRab/9b13zPUMRCyB1QUQ5SO8abrgx5zPHJEckylozMKfHPbsrQ1M6eLXtvKsaoiyVm1lLHKa11OSGSNiE+JXsy7GWs8XwYFUwBeVMcWspi0iG9VlHr4xaxDljV4s+v3h+5ry9A46flH8Rc0VMJGVlZAcJUqQjL9WrAmuTXl+6pIHLxm89KZfrC/12X51N+ki4g/XHl4WaD1ug8+wJNumHPTqcnlPMyjOma15e7bTvOGex888sdOMQ52wfkqZZzJ/IjkYWb9YIYkIErQxwjWSi8sDE61213lWJnEwlUsqorKBBaxZmI6VrFixzS0yfYOAQcKGdOF7phjO+YLfj4LHwtwrVLm1FIkogstpnQXqLbHqvHhBSnuEyrXCLmTuHINc9dS24BzAVwiukSykoedL1OnlUvDfqNMGpaQXrAm64DolfJJsFKI9KiKd9bV6OPYJOIvdFwwDEt183H0ZL+usDfkbC8D/qbFzGY3csiyxnqs3C2MajuzCpVuhOocvtPS0HSiw/2ZgLdd0u6hT4hPiL8KoJVS4nX3PsCLn37jpT7EYyaed8d1/MLrPkSI6WjtptuOFNMgEwoOI6LcQEoQLeINkD2LokBKxFqzMAlCSJDEqFVzFlnuSWJeKYycrWYYktodR71EHqTaBoaO1YbwcMxFaYmiH7CxZ1ZRVoOOpIQktMb2wtGLWaHoYw4umkWpX5oHu3ZFl6yaBdYNbbDs15FUm76LmA6pxL5YHFSxiSREEtZEnOitMgFnSmd0GIlat1hez8Q2XSBg05VByfrekYPBYUG5w8HoEegHtEt5GETlMfpcHnLnd7PE3wSr4kS6iDWrWHHeqx5rFVwm32uWTYVvHDRWle+rMtyfcmkYsauIbQKmDcghy8NLJuI/9MCST55veN62NHzYeO4dj2PRBt71sfNH+8ApqpYhBChZl48qhfBRU+8uDml4l/Ksl3I6kl1PjRcIQvK60ccHo1tusqJ5c9B3DEgPFcWv/KGIZVhvp4/J6bH2pzitljb7xQjpMsCrW218bwszMZ7aBiobEBtJNuVbP+FydNEDVsourjo8XWyKy9eXE+Pfv1jZGPqyTi8IXRyyo0G57ta+Lsr2Tf/5gwCrvD+94j3anGVZ2ixxSFniIB5kTZc1Kg1DhBjBX2FO6x0fPQfAs59w5lIf4jETpVHxhvse4FmPP31kj5s6r+Q7OdNyDowg0Q0r4LNvVMz+6MpzSS4Z9ReVCxZ8bYgGOutYdhXWRPZ8TW08czPllF1iUmSaOmqGmbjNKKB2UCcR6AWppszEjQZwlZy3RClLQANtsr0Ga3DgzFlXVpGbNVlA6G1bdkxDY6q1JRjWRS0P++UXR5P99tNMeWja2cDEeqbZomZmWyZGB7wrMzIDvIgTw2YUBwZDyiu+8j/kbCtI7AXDsXQJe75R+nuTTJZX5DI7DtuOygaf8h4WsIpJBv5qBFb7YUITHOf9hHPtlJWv2M/ku185ZGWwS4NdCm6lWZbLN7uKmCYgTUAaj3SH64hcMmi95xN7ADztpp2H+cltPPH6GTfs1Lzh3gf5xs950tE9cFAXKzFCMhbBI14/uiKCGKMLHJyWiNak/AFNWhYZGZwhJCFe2/8hqM6odXrlbKOjSaM9itje1+mCQxq12g/alweFixkbDGZCWobdeWWIedjkMtg0ly3Zxe1gGH/ZtALWTlyxOK6NGh4aG8EkdcMwl59p9Y9h0CzLqjVNZSK1UeCaGL2VzK+31+mbGuvmgIb10nCw90mDueIGp1Uy1rFeq8+0xqBF6nVh2lgx5cqVn2uQSoy5q0WYrEkamlix55XHWviaRVdcHBy+tb1fVpHd9PddtlPyEZMrA11SeYUzrfd+Yo9bT085tRWVPmyICM96/Gne/fGjLQ9TSrraPUREPAmnJSKQMmgBmE4zm2QE22l6ZTpIGcTEq4Gn+LxiKwyK7jbkjlAu06oYCMbQ5VOnuAEcRLQ/XAlZfOjHwDWMmMTe7yXKYJQX8f2adzXGKzOPG4CVP3QlkyklojMRYxRYklETxcsGrUwqJpMGP63sXFpbT20UOAtglVvxq3qoYWW4sOGx2ZQYR9HChVGm1QNWEkD9/kvHsMNiUuy7ivr94eIzLgeb5Oii8lhtLglXWfW+nwFr1Tl854itRVrJtkmDyZ9tswK+U22hdIPWkHCFifh7PrnHnbfsXuqvP+bi6Tfv8vOvve9oh6djoiy1T4CkmEfJ8gZlQKLTDCA67SRmyUN0Rrtco/LQNJCMITlL21qMcez7mtp69u2EhZ1gSZyPM92OY4Z5tfE+vpCKaFHWgGv8AVVeJ2T6OHcPM4D1/Ev+vU5sNqALdGLzmJFoCWgYhrQZ3ChiLh2L39bEdEwygDiXea1eYHp5b0MvVnUgLuFcYOo8M9ex41p2rXp9zW1DWTZ70AYei4EUCSI6VI7Ns59lpGdQw9v+nS/aN+k7ff0255GYtbwX/Wbo0R9tUinXU3/hKTzYKrketPa8Zlplw1MbLefbKUtfsd9WLJtale/7DmkMbmFw+0q+Vwsl390qYZcR20RM43Np2EHnVcJziLika0xKiXs+scfTbtqC1mHjzptPsWgDHzm7PLoHTRFdqZyJzKhlV4p61ZKg5KZ4Fe5JT86n0c7ETMoHMHmzD3kUJYQh22qiLihYpXUytxDj45b4gVf7wnNtSABMXsIwHg9as/LNH8R1Etn2z3GxWHP+LI+Plm1WkqpB5GjKQ6Q8VgKTMGa9a+hMPLAk3NwT2SviS3k3FndmYW0c/X95fcpj9ZumR699efyHy3pjKvKIchsy37JgddPrfeFrdQrxjqar6NqcYfWarEy8t9l+pivnXs60fERyE4kQSFcy0/rI2RWLNvD0m7egddgoWel7PrHH7Y87mkWuKSbERPLFWMtFgGog5yVGsAZSUjmhCBIMzqlNSzK68EGSWhAnA9GNtvpMKyobmPoJO071eGWgep6afvPLeGSkzLyVq3UJkz9QFb4HKwAkf4+Bk+kYAKtXaEfJgtOB7ym6sYr1ZRn6uKWLWDqIyms5GzEmqj6rcFGX8z4YtBPpEtZFnMsEvFPyfb0cXC8FiyJeR5kHcOk3J6VhTjBIGpoX/e8b2qKAZzDkW3vdM4f4cMA1/vcikxh3CEuGtecn6tzQKeneekuzqgkr5bHcvsE04BaCWyr5Xi1Tr343ReLQ+pxhBZLP+/AOEZcEWvdkEn4LWoePp+es9J6P7/HFz7j5aB40RVI0ClxJkBgVuAqhmX3X6byCmYiKT0EdOyWSjOqVQNdskUEsNJZgEstWu4iVCezYtneB6GxDl1z2Dy/KeFkDq1KaQHHotD3Y6HZrQ+81LkMbP2adVRcdjMwElaBOubslvYfVOHrf84wNYy91l/8OayJiMqdlVHB7WW/DiIg3okS5M7H/e0qXNebyt6jO15T9uWtnUyKKKDlu6B0wSjkYCP0W7jHRPp43BPqMzpSxHdXd98cz5tbGSy/WgCtntz6a7As2+LwvuopV51SL5S1hZZGVRRrBrnTG1a5Gmqw2YfNQdK/JKtVACEPVcIi4LNC6cwtah47H7dTcuDvhPZ84eq1WiuqPpQ200GdZB3YUnUFEsG0EMRibsK2q5UOnq7BMB7ETklWnycY6lq7ivNdMa2Y7IkI0LVXmm2D4EPY8CkMHsZQx6nuuRNok298MdsJ630H+8BaCnqHrlaOUQyU2N8wUos9kEWf5gDpRq2NjUvYcy0T65bwF42wta7TM6DYcs6yNyZSsqP8bRqVjftH0NTHQjlXxo+PtdVS5n7sZNhsOGgn9a1hejx7YNvi1YVGsDAsq8mjOKmiGtWx1QYXvHKE10BgFrLxcZdwtVBeHXBp2hbZQXSFxoDOuaPfwnk+c53Hziht2t+M7jyTuvHm3l4ocSaSk2VTKJWISBacYSdHq5zZaBavgSDAo5gUk2n6UR4LkGUXRxzGGGITG1cQw+G4tXE1EVHdkWuWINjpgw1D0uhK+EMAT8VTG912r8Ybiumx6MbEXNRIH0BsDnC0e8VlIOt5UrSMwOgLUiWXHNMxNy8yOxnmKuPRy13PmiSgFLFXDl9fBRxVhYqALwxN1fba18RpJpMml7DRrp9pkwUKH2mDXo07peCZwnNVWBKbiCdmyZ3PUp2xQWivT0Wy3A+U0k/rbl5nC/U7Lwv0mq927TLp3BrMwmlk1QrUAk8l3t8wdw1XQoeg267I6ryDlQ59pHXbT9yVnWtvS8JHHnbfs8suv/7CWcHKZl/cDIsU0ZFwpabotSWcUs2I+2aC8lo+IEYxVRbw1enVMFozTKyRGiI1u9Vm1FftVTUzC1E6UmLXqdrp54m8SzMD6zxggQie5M1bEoZl7GRPVFRCM9FnJsGg0UYxchv1+BbAGHZO6RORSKOuknFFyHJN6Mv5I34c0kOYRdTFdoRq38ppcDLSMpP71weTOHihPKAZMq6M4+bUcG/71j8No9RcGta8uz1FGnoZxorXHGr0YJdPyydBGq7esdve+bNQxWdqAZlmZeLdrpLsa/IlPmmGVxlFfGqahoXSIuCTQet8n93npUTsWPAbizpt3Od94Pn6u4dYz06N50P7qVJYUCERDKuVESio+LTOKIhAjxqqVDaAqeq9yB3UsUJGpBEjWEJKjSXBOEk2lp8zEevWpsjVGkoo2H2I0pYyzdMlSxUBn9EM7NdmHXnQTz7DUVQ3r2uTynJz0c3O6ldr3+/+molnJVHSTT7+WDGglMKVjatqcaan0oXIBcZGUB6ePKorDQxcsrbEs81YglwLLfGSDdmrdsaK8ht4YnBlkCZ3kJRjiCUi/c/Kg0J2Epl9QAlwIaFLGidYfo3dxKBY00fZZ1sLXLLuKRVvRNBVh5aBVpbtpBbfIHFZDn125VcItM481ni/svJaGpWMYAqmA1yHiEYPWXuP59H7LHTccTQfssRRPv1mX2L774+ePDrQ2onQUicN4DJ1+kpNXwakAtF47iiKotYB6p0Nu3zv1l4+VZF7J0ZiE98qfTSpPFy0rqwR3ceccl4ljECug5aNlYnXfX9nBWAZ4VXOVeZb8oTS5VCwEfylFByK5qMxjD1jTsnmZRCfqFaHApsr4qe2obMC4Uh4egctDfuFSNJm4tlT5A69/h9O/t8+0hk5pCZ8bDRGhJjubSui7pWXkpohtx2UilGaHCm0rwsGNChle13EUqUWJIi0pavexz3vMavfBH0t6o8l+SUWrX5u2eGUNEocixUkxFh/wnG1doUzrvvt1t9wd129B65FGGXn6wKf3+SJuOtoHH/NbpaNYpBBhpJvOKnl80KzLBKQ1WCA2Ss4nMyzIsNkRIllDdI4UhZXTGbeUhOAMlQ14YwbyeU1zNRKTilFyHR2mtsS+G2lNUuvhfKBlrrDsUbSYvpwBRtuUw8BloYBVZdAyWcg5zaBVRnrKiI0xEW8vP9MaLGlyaTjyiG+Nza/HOucXudC2ppj8lZ+1RFaifFiTeTGbP7I10ObXaTwDWmYI4cLZ0LXFJCPQKmaMw3GMtkRnX6zeG6sMQWdvrLIouCjetUxkKA2LV1YXwMeBw8raLGLquawrxml98NMKWk+6fjtz+EjjplMTZpXtX8Mjj4cCLujHe4iZ1g6x128lb3X0JyQkGJIY3VotynmJF7oEaWJYJaGtA523NJXH2UjtvJYdmS8yXNhFK2DWGkttdHNx4cQ2SWQrnWYSEqhSWBvxMRL7zKkm9DxWAayJDKNlURp0U49nJ5eIO7ZlYnVwusvZ1uW97sW9VIghz24GqzyeifikjqX9MeXycByGRDShz0hjkjWw67PWEf+m15WIpZR9sR9xKjzfuAQ8mHcswl/93iqVFWB5yWo35Xw7YW81oW0qQmP7IWjTsl4SrhS43FJtv+0q5LLQqybLZwLeB5L3A6dVysMrJXnYZlqXHiLCHdfPrxxowcHAlWSQQpQOVrnSdQJWO42q3cozim2+2jeAyaLTWnQuzek4SJufMpQ1WEZn/lISJGuVUkrENeCSnn6rTOgV1pUEVskxyUZ0OsqiUcqe8YKHujg+SMxtfO2M6qjxmNwWqkQuHwsRrxY1xsQse7j88rC/xVGmFZUX0gMZfrxkWOXeiNaXw/cNURJdMkzSaOC5+JRJ2eQ9GqhemwAYdFdwMeua2FvXjBdhFMuZsk1n6bO0IRj1ee9ErWaCDtqbjt7myOTFwRJSnsIYJjF0MiOsARUhaHZVpjkOGY8YtO69f8HpqePMfDsofSlxxw1zPvjp/Sv7JJvARSCJfpST96qEF6MdRdCfDRaT3UxdTsuMN3l3YlLvc9GV8snYnFEITRS8tYTaqx2LiQQXEElaguVso2RaIoloBJ9Lgc7Y3rUgIpnDUbI55A+hoXhkDWEYabI2wsqI3E42Z2IlO1N7mNpohohNY4ODSwqJjJZaDNt4RBKdyeVw1tKtv00K7n1ZaBIQldtKsS/TTNJdiSamXEYrJ1hsqSHr1ChdVM8F0wE5xhlr6ciuYp0zrAln/ZxzfsrZbsZep2Z+i0bJ97TSIWjbjAz9Mo/lVgnXZNX7SqUNptEMS7ohw6LrNMvyviffU9Ku8WHjkkBrS8Jfejzp+jmvfPcnr/wTPVzGJX4Y9REhpYRZmT4LsiYDmANNE1Im58sWa6NjNUDKBH6wkej0eYyJ4OjJ5wJcjoiPyn/5DC7LUPXlz0Q8WNiPE/WdwlPnxOmgjlmXLFFC3tqjg79dCmvZFuRuZJZVFNmDLbKHy8y01laIRfrFuCEvbS1Z58WeZrOBUaLMAMZs8xNl2JCjw806WF2WrR6k3LhgxVj+/VWs6LCsYs35OGU/Trjf7/CAn3O2m/FgM2PR1eytJn23UFrBZPLd9lue0uDg0KRe8V46hdJkPVbWZQ2ApdTEuCxMV6o8vPf+Bc+67eiM7B5r8aQbd2gO6YV92VGAi0HD1XcURbTsizLY2Vgl542hJ6fVmjl7y+t2d2KTyXmjBH1KghcwI2LZJsEIGRgiJinARRFMtnPyWTjaBIfLpV6THDZqVjRsqEl5i/SmPinm0kY/xF2KgHYLbS4Vdff0EL1zaFauk5Xxl/c60/Na/cLWlIfOk+CKV/xoVOnhooz8wOD1XmYKlROLfdNCrZKH3w1F6DXqBhawKtMK4zX258Nsjcc6301YdHXeV2j7bqHJOqzegXTk927WBqJjXmunJWAvb8jlYcpOpT1gpaiAdchs6xGD1oceWPBln7HdJn2p8aSrzQUmLTkAyl7QXg+T0uB8WjqMMaqdDfTKeeMTEo2O/XSZnM8WuiHq7GKKynXFyhArNdqLSVTIaSPRBqwkUsm6SLQB4mjuzyfNwIqGa5V0U/RU2gu0RTr3aOnE0Ylu4Im0VBJZie6asSKElFglsjuFulIMXFLG9MuUaY3Lw3ILwWCMdln9aLSnjNKUUnGtUSEbkwW9ONWqPU9S7doqOc2sUiLieylEJ7oMhLSelRYHjjLMrtugda9lEys+1e2yDDWfand4sJmx1054cDGjbS3dfo2s1OPdLbIma1kI+OxAmsn3nnhfdpplrdrecia17YW6rEsALLgE0OpC2pLwlxFPOo7SepRx6WBqvoqL5FEKIblsICjKXUmnngvJGZIVrEnYWgtKW6MWv1aUD0oQrSElBcAw0iMlN7T2kxl9kEbKbyOWNpcGrdVTshpN/Mc8iqJzdJnDEeV5OgkEI71v/FQ8XdZt2ZT6TGwRB5/5YUvNUb2++neoMJecadFv5EkouF+sDOz/zqQvSJFEhCR00eoOysxt6Shi6mUQm8PVQYYu7Ngbq2RZSra7fhnvItQ82M3Z9zUPNjPO512FbTO4j0ozMvPrhnlC242yq7IsuGzVKdIG73OGlUHqMgELLlERfywfvEdJPP662dFu5Dls5IwrxVHXST9ZOcPKGVd2Q6XMKBpD2exTWSXn+xnFCCDESiULscpr4YOAS6RaiCFinZoTWjFEF/KqLM26osmllFVwO9dNmRivnuS2Ymo02ypiUhhGUUqpV0ddOLpjWqxE5tL0tsxAv6fvwTDnfJyqgV1wGVQ4cJvOI4mSaUlS2YO+BoZgEsFqJ1FMHHRZolXpmJgvZLyWfwnPwC+Ou41eFHCr5OjyDKd2XqsLZkDHbhvlVjqExWZmGSoebGcsfcW51WRk5FdBJ9h9m8l2zbBMC3aZSfeWXvFuVwGz6pAmQNNqSdh2pK7TkrDz6xzWJmAdUqMFlwha20zr0qOyhidcNzueJy+lopg+40rkWUTQjIsROR8jYgVbxH95tCda039Qk6iGK2tBiV6UB3NJv64iwUVCEIxJ+Giy5XGksrHvMrbR0hiHT5baeJrodOTGeBZZy1WyLVjnhmyWPVQjD/ZKAnXZz5g/uPf7XfbChLPdjH1f03oHfgCHSw0JGbTydqPktDxEEq3Rj1i0w2vrTFzXsOW/pTQmXM6Q1EBQtVvLUKlUQyIT6zEkJqbrX5cybF2i/M0xCU3U17WLOlbkk2HhaxqvNjN7TU0XLKtljW8yf7WwuRRUR1vbgluoeLRfTNFudAqbgLTdoMXquoF07+UNlwdYcAmgZY1w2xUaQXmsxLFnqqOOItGsLciALEQt4z6dgpoxMmyvbqUgm25oDmTXCO2gSRL9XjLaSQuaTZk8LhOtYIwhRgWvZAWbpN8k04rFRx3UbYylSzpL5/KH86AYE+xT0/Vq+FImdcmyFybsh0k/RxeCyWXdZSri4+gWcrblDVEgBJ0lLBunrdHX3koiiWrYoBj16eMV37AOizUqUi0A5kykiQ4jkYlx/aD5GMRLaemj6rvKei8fLaug84XLrtI1cd6yaipC9sTSZRQGu8rDz0XWkMd0TEcvbTDdqFPY6jYd8WFNPHrUgAWXAFp/4YueirNHPBb/GItjzVRHA9Zj1Xz5vmqycqnoTa+ctz4iCUxtkQShLVuDDaFSoaEuPxVCrVxWqFVtHitDqhPBJUJlEZsQGzFW5yStTZjscVWysNoFqixLmFidaxwb69H/FevjMGPl/ZhDikl0GYN3PLCasWhqmmWFNLox5nLCdNpdNY1gspgsWEvwCti+sxirjqbGDBlWWeh60cfNNjflZ4vBoM3c4EO9HiHPQIYk+GAJWfDqg9poe2+J3pB89sLqBLcy2aUhG/l1qnRX1wYtBW2XsrFfUHnDIpPuTYu0nQJW0yhgFfI9ZS5rk7+6BMCCSwCtv/Hlz7ykJ9rGENeMrc/FMi6AYDJwjbb7tB4D2H5PoCGZpNyWDFkGEV3yELKVc0CzLas8lzorGKKLiDGEvMHGSKKzCl6tz7ouG6hs1ZeRMGi/QMEosW7xUn5GYJ3jiQYfLPsr5W3SyqqdyuH2KVw0jEctfbx+4BGIrZCSySR85uyCvtbGDL5bBwlOx6E/M/ysoNmagtj6ccQR+R+y5CLmTmbRjkVvVJbRGcjjWabJK+sbFYwaD3aZFe6rgXR3K7WXsU3uEnYR81DjOUU4GtOFGqxLBCy4jG0827j0+HNf8JTjPoSHnlMs5LzN232iU3I+6XiG3idMZ5BoiZVmXNFBqMDWWgL2a+etarySHbKv5BKx0q5XtCqtCHkFFwJi1RJZDJhsj2yyOrN80AuRHqOow0IGsBLlMz025UtRBt1RXiJqLzPTsi2ArmMr247AEPPfmKqoUwS26MIUpNcOshz4CLTGTV/yqI/ke/27hp8rv5+yTowkEKQfLSplsHj92pZRHK/WyCbk8q8dwMp4JdzLnkK3Ckina+xV5T4i3ZtWu4RF3lBI9+JGehnl4GZsQeuxHBcBruH7Kbufpn7kJ0V1ZxIfkc4iMRGdwXRqbROdECrNqkKlkohoVaSaLIR8n6x2HZPRrKzfjJNBK+WZwGQymAlw0Ac9kX95xE2NifUxHuWft3l+TslldSm4nHCrVFY3IkkIXea18t8YnVVwdsPfdqA8Pv8tsvFnkv9EJGWH1OHHN39PYn7o0t1NQ2eTiA6/56aBCXpvWzSDagexqC32yG0cJA2NR0LErPzgidV2mqEfRLqXkvCIYwtaj/U4ALhKqQjovRGSB7FBl8Nmkt6gingT1AFUgkF0F0WWRWg30ViIIen3csmYbO60FdAqIGWz2l5rO8WjEZBt6hMkyqBID7JmE7P+g/lnyB9YnxeIZnL5csJ4Xfqqzq+5wDYM0pCgf5uE0d8m0idV/Z9U/o6D3qb+j4ALgLg8Rhq0YmW0iFGTgKRApeClNwkpZ1fZWsarut2uIiYU0FJ1u2mzvUyXV9jHOAKqA0j3/hiPLsvSv/WIHmgb29jGNq5GbNuA29jGNk5UbEFrG9vYxomKLWhtYxvbOFGxBa1tbGMbJyoecfdQRN4KrK7AsRxl3Ah86rgP4mFimlJ69nEfxDa2cdLiUiQPq5TSC478SI4wROSPT8IxHvcxbGMbJzG25eE2trGNExVb0NrGNrZxouJSQOtfH/lRHH1sj3Eb23iUxlYRv41tbONExbY83MY2tnGi4hGBloh8o4i8WUTeIiKvEpHPvlIHdqkhIl8uIu8SkXtE5PuO+3g2Q0SeKCKvEJG3i8jbROS7j/uYtrGNkxSPqDwUkc8H3pFSekBEvgL4gZTS51yxo3uEISIWeDfwUuBDwGuBr08pvf1YD2wUInIbcFtK6fUicgp4HfA/XEvHuI1tXMvxiDKtlNKrUkoP5P99NXD70R/SZcWLgHtSSu9LKbXAy4E/dczHtBYppY+mlF6fvz4PvAN4wvEe1Ta2cXLicjitbwf+41EdyBHFE4D7Rv//Ia5hQBCRJwPPBf7omA9lG9s4MXFJJoAi8sUoaH3h0R7OYydEZBf4D8BfSSmdu8SH2bZ+jyYuy2/5pearr/z7IKKr38hbk8Qg1oC1YAziHNh87xw4S6orcJY4qUgTS6wsYWbVXXZqCJVaYvtJ3qpUC9GqKWMsxoy23NLwPZf6f1tzk42og2reFVCcUSXoZh8JjL6n5ovGp37Dj/Hwhy//3od9Lx4WtETkLwHfkf/3T6JzfT8OfEVK6dOX8PJfyfgw8MTR/9+ev3dNhYhUKGD9bErpl477eLZxDUcxgh8DlrWICFTVAFzOIcbApCZVClpxWpMqQ5w5wkS9/Lu5IVaCn+rWpFgJYZKBqspglQEKg1pi2wQWkotgE1IVz/7Bfx+yZ39El2cEIXiD5OUZMvak96LusZ1ucbKNgqE9pIPsw4JWSumHgR/W10/uAH4J+KaU0rsfyWt/leK1wJ0i8hQUrL4O+IbjPaT1EBEB/g3a0Pinx30827iGYwRYJbvCiGZTZsiqxFmoK7CWNK1JlSXVjjB3RGfwc5tBSvCznF3Ny6o3CNOUffwTyWWAqtRsXqqIcbrirao91kYqG7BGV5mVbUchGrpgiNHQeru2pix6A23eMdnpPgHxulDEdDljK2u3DxGPtDz8u8ANwI/oZw9/LQ0mp5S8iPxPwO8AFviJlNLbjvmwNuMLgG8C3iIib8zf+1sppd86vkM6fLQ+8qtv/DC/9ZaPcmZW8ewnnOHrXnQHu5PtuoErEjnD6gHLWsRaXYpYAMu5vhRMtSNOHbGy+Jnrs6sw0YyqmytohRmEOml2NU15vVsElxCbsHVQoKoCzkacDcwqT2UDE+txJuLyRuuI4KOhi1Y3VXtHFwytd3SdJQaDd5bkTc7iDOS1bWV7UaxY20T0kC/JVhF/ouOqvnmfOLfi6/71q3nfp/a54/o5ISY+/OCS63dq/upL7+IbXnQHZnMZ38mIa4vTOiDDKmWg2CGrkrqGypEqR5pphhXmDj+1xNrQ7RhCLXRzCFMFLT/XjCrMErGOUCXM1CM2Udce5wKVDUwrT20DM9cxdy21CZyqVlQSmZgOl3dQxqT7FJtYsQwVTXQsfEUbHUtfZQCzrDpHCIaudfhOAUyWFmkEtxCqPS0T3/JPvufyOa1tbAPg7LLjm3/iNXzs3Iof/+YX8CV334yI8IZ7H+Af/vY7+du/8lZ+880f5e//6WfztJuukWW0JzEuUhL2gFW5gWyvHGlSkSq7ll2FqRBqQzfTfZN+roAVJ2SwSsRZgDpiay37nFOgmji97VQttfGcqhpmtqM2nl3bUElgmtcXGYl00dElSxM9E9PRxIrKBJrgcBKpTKCLFmsirbe6qNakfgM3yRBbbQDIIdFom2md7Lhqb953/fTr+M/v/Dg/8a0v5MV33rR+ECnx8tfexz/4zXew8oH/8XOfxDd97pN46skBr2sj03qoDKtyA+E+nfT8VekM+nml2dWuwU+UaPc7ylv5ncxb1YmwE6GKuJmnqj2VC+xOGyoT2a0bprZjaj3XVUsmpuO0WzG3DVPxzE2DlUglgZBLuVWqiUlYxAmrWNEkx56f0CXLvp/QREcbLXvdhC5all3Foq1ovWO5XxNXDrNnqc4Z7Are8YPbTGsbRxCveNcn+O23fYy//mXPuACwAESEr3/RHXzp3bfwj37nnfzUH36Q//v/+QBPu2mHu287zY27E3YmFitCZQ3TynLdvOKmUxNuf9ycO66fU7vH8BjsZocwc1ZrHcKq7gn3NNGyMM4q4sQRppZuV7uD7a7BT7Uc9HO0JNyJxEkiTSJ2x2NdYD5tmdYdU+c5lcHqTLViZltmtuN6t8/EdJwyK6amYyodlXhs5rFCMkQMdQy0yWIkaRaWLJUEumSZ2Y5lqPDJMrUdbXQsXI0zU1Y+4L2hTULshFiL7oU8RGxBaxsPGasu8AO/9jaedtMO3/Hipz7kz950asL/+Wc/m7/2J57BL7/hw7z2Aw/wpg89yIP7Hfut52LLhp0Rnn7zLi991i38mefdzlNu3LkCf8k1GrL+QRUjA2BZO+KyjJaEuURMlVXAmljCxOitVpAKUyFMwc9GRPskIpNAPemoXGBn0jKrOmau43S1YmY7rqsW7NqGuW04Y5dMpeW0XVGJpyJQiy7wDQgdjkAkZKCNeDBg8mLWLoNXATAngWWoMSR8NIgk9qua4C3ejnRfh4gtaG3jIeNnXv1BPvjpBT/75z/n0NnQzaenfOdLnsZ3vmT9+yklfEwsu8CD+x0fP7/ivvsX3POJPd5w74P88Cvu4f/3++/l+77imXz7Fz4FkRNJ6h8+LiZpsHYAqpJhTWpSXZHqijhXHqvbdYSpwU8N7a7yV92pTLjPEn5HO4Kyo6XgZOI5PVtR28CZesXctey4luuqBXPb8ji3zymzYsc0XGcXVOKZSoclYSQSk77/LRldkqMS3/85pWycSkfAEJMQMJnzqljEmj0/YWI957sJK+9ISfCNJVaJjIkPGycKtETkB4C9lNI/FpG/B7wypfSfLvGxfgL4SuAT2wUTB8ei9fzof3kvX/j0G/mCp9942Y8nIlRWS8TT04o7bpjzwidf3//7x8+t+Lu/+lb+/m++g/d+co9/8Kc/89ELXBcDrHGGNZY0VIV4t8Rab2GaM6yJECaiRPsk81e5HKSOuAxYs7pjXnVMrGe3athxSrKX7OqUWXHKLtkxjfJXRKYjUIpolhWTIaSSYRmsRAICCayAHQlOQxIihn2ZUGVUamJFFy0TG1iYiNiU1fWHe+lOFGiNI6X0dy/zIX4S+CHgpy7/aB6d8bOvvpdP7bV895feeVWe75bTU370f3w+//C338WP/pf38tm3X8fXveiOq/LcVzUeLsMaj+RMapKzpEmtgDXJotFK8FOjhPusSBnAz5OC1iwic4+rAjsz5a9265ZT1Yqp9ZyuVuzYhl3bcL3bZ24abnB7zKVhxyjxrhlWyhmTHnPAKHDl+xKWRF26isSe+wLlv+amYRWrnI0pOn3K7TCpKvZdJNpEOqRc5poHLRH5fuBbgE+gw9Cvy9//SeA3Ukq/KCIfAH4O+ApUtvYXgP8DeDrwj1JKP7r5uCmlV+aB5W0cEMs28K9eqVnWOBu60iEi/PUvewZv+8hZ/u6vvY3n3HEdz7z19FV7/isam2AFw0hOkTRkHZb0koZaRaOzSvmrqcXPjUoadvI4zix3CCcJvxOVv5oGpvOWygVOTRtmrmOnajhTr1TK4FbMTcspq9nV3DScMkum0jEVTyWDDqvFEJNhlSraZOmwPRFfohJPLQFDpJagpP2oub2falamAmARa7pkmVrVghkTCSYdVlt6bTuXisjz0VGc56Bzjy98iB+/N6X0HOAP0CzqzwKfC/xvV/QgH6Xxc6/RLOsvf8nVybLGYY3wz772OexOHP/rr76NR60sR0wPWJg8Q9gPPVvNsCpHrF1fEsZaASvU6Oxgf6+ke6oTMom4OjCptCScOb1Nrac2nonxTE3H3DZMcmdQu4NKnI/BppSDLbYHrC45uuT6jAnIUggFrrkpEomuv+1Iq89jOiZGn8uZgJGkWC6AXJkxnqsdLwZ+OaW0ABCRX3uIny3/9hZgN3tVnReRRkSuSyk9eGUP9dETjdcs63Oecj0vesrVy7LGccPuhL/60rv427/yVn77rR/jKz7ztmM5jiOJi+mvjFnvEFZVn2mlyl1Ausfa4GeGdieT7rtKuod5ottVDsvsdFQTz2zS9RnWqVpLwsJfTYxnblrmpu35q6l01AyA1SUt/1apoksKVKtUEZIS6yZnYlUuBXtQEs/ceCoSU4HCrVdEqhTokuNB0zI1HbVR9b0xKYPW4V7OazrTeoTR5Ps4+rr8/7UOztdU/MIff4iPn2v4n//bq59ljePrXvhEnnnrKX7wt95B4w/ZWrrW4mIK9w3AYqxyz4BVSHd1aMiEe5E1TBS4wjTp4PMki0brQF17zbJcx8R5ahOY5AzLGVWpl6yqkoCVgYMKKH+lQKW3VaqG0nCkS7AkzbAIudPomUhgLokdI0zFMBWhFqGWqBwZEUvEXIYu+loHrVcC/4OIzLI18X933Af0aI/GB37kFffw3Duu4wuefsOxHouzhu9/2d186IElP/2HHzzWY7mkuEiGJZls7yUNVaVZVgGsaTWQ7lPtEvpZVrlPRYWjM5U1hClKuk8DduaZTjrmdcdO3fZzgzOrHcOJyeWZeCoJ1OJ77qlkWDFnUgWoVqliFesMWI7ARkmIjvXsSMfceHZMZEcMc7FMxDEVy0QMlUCVwbHczCHLwc24pkEr2xL/PPAm1CX1tUf12CLyc8AfAs8QkQ+JyLcf1WOf5Hj5a+7jI2dX/NWX3nVNyA1efOdNvPjOG/mhV9zD2eUhDZeuhSimfWKGrmDlVOFeVzr0XNX69aTWknBak2Y1cVoRZhVh5ggzJd67uaGbi5Lvc+0WhnkiziNpFnDTjum0Y3fasFO37FRtHnT2TDKX5YyO4FRGwUqznmwtg9CyAVYZsAqvBWDRxyg82I5p2JGWufHMJXFKDHNTMZGKiTgqsdiNuq+XS2TmPSV0IO2QGHbNl00ppR8EfvCA73/r6Osnj77+SZSIv+DfNn7/64/sIB8lseoCP/yKe3jRk6/nC49Al3VU8Te+/Jl85b/8r/zI79/D3/yKu4/7cB4+RmB/gcK9EO/Wqmmf069TVrmXW6ytEu4TUYfROjuLZvI91jpLmOqImQTqOlA77cYNIBWYmICTgB2VZUXKsK6nMliSloeZzyr3Y8AyOUsqHcKpeKYSmEpiKkIlBv3voS94m8AFHNqa5poHrW1cvfi3r/oAnzjf8C++/rnXRJZV4tlPOMOfed4T+In/+n6++vm38/SbTx33IV08coZ1oGlfXYGxvWCU3B3sx3KmatoX5k4Bayq0O2aNdPez0SzhrsdOQp9hjecIp7ZjZjuqPOA8MV65LDOUhnakp4oY2kSvvyqSBuWtlE8sHcKKwI5Rx4e5eKYSmYswFUuVbwAhRUJKBBJdUnJfO4/KjflkSEmIUZAoh20eXtvl4TauXjy4aPnhV9zDFz/jJj73qcfLZR0Uf+tP3s28dvytX37rtSuBGPm4b5r2rbmMbhLuObOKlSWWWcKp4LNxX+jvIU6y2j3bytS1p3aeqRuVgRIzf6TjN0ZSP9BcSdDv5azpoCgEuyGDVL6VknCaJQtTCdQSqQQsQsmxFKwikUQk0qWYy09Dm6yO+GTjQB+NZlgJbZkdIragtQ0AfvgV97DXeL7vGi2/btyd8H1f8Uxe8/77+alrkZTPmakY0duIcKfKgFVV2aUh30qHcOqUdJ9l0j0DVk+6TyFM1QsrTBNpGpFZYJKdGuZ1x7xqey2WOotqWVht3CwpdwxT380D1u4L11WLDkkX7qpII3akZSeXhROBWgSzkZmPAatLKWdZdsi0osUnq2M+SdSK+dHCaW3jysd99y/4t6/6IF/1vNt5xq3Xbun1tS94Ir/79o/z93/z7Tz7Cad5/pOOR0N2QWyUhNoVrEbzg1nOUFckpy4NsXbgDGFiSVUWjM50Q47aI6tbQ7czHs+JpGnE7nRUVWA+6ZhVCljFXXSSuaxSFjoTezGplold3zUsYtAy9FwGo8dhR2T9AHqRqWgfsRbBQE+2R6LOKKZER2CVIqsE+1nntUoVi5B9toLFB/WRlyCHHpjeZlrb4P/8nXdhDHzvn3jGcR/KQ4Yxwv/1Nc/h8dfN+K6feT333b847kO6MLIGa0y4M1K4673RLTmV6QErFuCqB9Jdt+WMSfeE1FEtkUcuo7UN1EYV5gWw9OvY67BMBrG6lzuEvgQs2VeVea6aQE1YU8mX0Z5yM0B1AO0ZUiKkUZYFdL3ua+CzmqhC1RAFoiCRQ3cPt6D1GI833vcgv/6mj/AdL34qt56ZHvfhPGycmVf82De/gNZHvvHH/4iPnV0d9yENyyaK/ipnVVJVSF2rlGGS5QyzijCvCXOVNHQ7Dj+3dLuqdO92hG5XMyy/A35HLWbiTkDmnnresjNrODVt2K0bdlzLbnZrmNkui0i7Pruamq5Xv6+P7AweWT1XJT7rrTpOGS0BT0nHPItGp/lWkTKPNURAM6uOQJM8qxRYpcR+NCyi43yccS5OOR9m7IUJ+6Fm2amDacprxow/XPNnC1qP4Ugp8Q9+6x3cuFvznS952nEfzqHjrltO8W+/7UV8eq/hm/7NH/Hgoj3uQ1rflDMi3jclDbF2pNroDOHIVqZkWEX1rl8r6R7riNQBVwWqKlC7QG1DHjj2axlWKQk3uaySZRXJQgGsqnwvl32VRB25GZH5/Y2kpeAB2BJT6m8diVVKNAmaZNkvZWGcsIg1TXQ0weGjIQSjmVaCi/QFLnypj/SN28aJit9/1yd5zfvv57u/5M4TtwLsOU+8jh/7lhfwwU8v+I6f+mNW3TGO+RT9VV8SmjVJQ5q4vkOYakMopWAGLSXdycT7yBOrTsTeEytQ1Z5p5Zlnx9GpU2nDZMRljQFr0g8ne+rc/esdGHLJaEnUrINVLVGzqY3bJliUV1x9thSsxoC1nxz7qWI/TvrbItYsQ80quLwfMZeHgW33cBsPHTEm/uFvv5Mn3TA/sZ5Vn/+0G/mnX/vZ/PEHH+B/+/XjW2/ZC0fHXcIiZ6irfhdhmOp6r6Jy9zOhmxWFe/Z0n6rSXT2xEkwiZhqoJ55p3fUWyWU8R2/t2hD03LT9MooiURj4qWFsp4hFTc6ixtnUZozxJCT9/wi0KdGmRJcUrEpJeD5WnI81D8a53sKcs37OWT/jvJ+w8DVdsMRgEZ+7h1vQ2sZDxa+96SO882Pn+d4/8Qwqe3JPg6/8rMfznV/0NH7uNffxO2/72PEchMmODaPtOcmanHEZYqXke6wkE+5CqBiU7pUuK9XV9Hnbc15HLy5ibcSaSJW3O5eRHNcT47kELIPQWUBaiWeszSpC0iJvGM8cmtHX4yjgtPl1SOvf6/L3VklokmWVHPtxwirWrLLVcpMcbe4adlG3UMdQAOvw4tKTVRNs40jCh8g//8/v4Zm3nuIrT7LlS46/+tK7+K/3fJLv+w9v5kVPvp7H7dRX9wDGiyeKLmuUYcVJ9sHKkoYw0fX0oVIDv1ipJ1aYJaLTAehURWQSqSY+yxtapnkfYbGZ2XFNr3afmxabF6kOHb8WK+kCQ74SAcEgVOiW6DYZrCQOKrRDkn7sJ6CAN15UojY2li4Zzscpq1RxLk55MOywiDVn/Zxzfsq5bsrC16y8w3tD8gaT5Q7bTGsbF41ffsOHef+n9vmel951UjdCr0XtDP/kq5/D2WXHv/y9e676869vzNEsKzm9Raeke6wySNXDrTfwK7IGB6lKJJd0Pb2L/cbnysRe8T64NnjmVsvB0iUsncBafC734gWA1Y/qZEfSLt8ioj5aSfrvbf5b+fc2mf62SpZFdKySZT/VeosTzocZi1izCBP2wqTnspqgW6djNH1ZKOM07mFim2k9xqILkX/xe+/h2U84zZ941i3HfThHFs+49RRf84In8tOv/gDf8vlP4kk3XMU1ZGLA2DUH0lTlsZxa9xGOh5/7kZwqDz9XqQcuXIIqIlXEuoCzkcoFJs4ztb5XvM8yUE3Er7mBFh1WP9ycAWu8rxCJmDIgLXEYbk6Hy2HGljLFP35sZ3M+zPQ+TtkLUxahZhkqlqFiFSq6YOm8JQWBoPsO5RFwWlvQeozFL7/hw9x3/5If+JbPuKaGoo8i/upL7+JX3/gR/vH//938y69/7tV7YltU8K7vFsaJywZ+6jYaqjyWUw8dwljlLqFTwEqTCDZhJgHrQr9BZ+o8O67NK78aTjldoHrGLvsu4WYpuLlYordGFlTEme9LxnVQRnbRyD82ztYGK5uKB8OcVao46+d9hnW+m7IKjqWvWHbaOYydRbxBAhgP5pDOQ9vy8DEUPkR++BX38JlPOMN/+8ybj/twjjxuPj3lmz//Sfzmmz9yddXydpRlOUvKRHx0QnJCdNKT7es3XZ1VykFsAhcxJuFcxJlIbbU8rDOPVUSjE/G9pGEqbd8h7LVXF6m1invD+L6o1VssLbZ3Ly3/f9BNbZgdqzh4bxVZw3hUp9zaaGmjloU+KAlPUD/mInfYZlrbuCB+9Y0f4YOfXvBj3/yCR12WVeLPff5T+Dd/8H7+zX99Pz/w33/GVXlOGWmyYq27CcNM/bC6uaGbKX/lZ0VAOsqwJpFUZQ5rEjAuUk+6finF7mhl/Y5rOOOWnLHLvLJ+qYr2rMXazJQCMZeD9N5YACFZbB6NLvbHltRvh96MsnVnvDKsgF6bbJ9p7ccJTaw4G2asYsW+z8r3ULHXTWiCY9lVdN7iOwedYFrBNoJtwDaPjsUW2zii8CHyQ6+4h2fddpovvfvRl2WVuPXMlP/+OY/n3//xfXzPl97FmXl15Z+0bNJxViUOTrOsWA/ZVT9TuEm615phKYelPFbtdJPO1HmmtmPuVIs1N22fYRX9VcmsKsIFJSEofxVHXFX/tcQReEUCEXvAhWy8KqyUmP3C1rw9OmTQWsSaLmr2VTisZahYeV3O2gZL6y2+s0QvSGcwXhAPxieMv+DpD365L+1d2sZJi19/80d4/6f2+ctfcuejNssq8R0vfiqLNvDy1957dZ7Qmr5jGCuTV32Nu4QDYIV6IN01w1LS3YzGdKaZx1IRqUobdAu0ikZ3TFmo2o26hZsjO2HNTnkcCjau94Nv07AWrNxUX1X3PFUpA/djzaKUgfn7w3hOlRXvFU10PWCtgqPxjtY7Om8J3pA6g+lAOuWyTAum22Za28gRYuJf/t49PPPWU4+qjuHF4u7bTvOCJz2On3/tffyFL3rqFQfpNNJlhanLnlimX0ThsxbLzxLJoRYzdeavpgFrIy6DVe0Cu3kpxa5ruK5aMrMtZ9wyq90HX6u1sZyRrUxMhpC/bpOKSQth3uZtOuMN0SXbGm/aKQssSialvz8Q7+V31RvL0WR+q4mOfa981sLXLH1FExyLVoej29YRG4s0FrsS3FJwS3CrhFsdDrS2mdZjIH79TR/hfZ/c57u/5M5HhS7rMPG1L3wi7/vUPq/9wANX/smMIVkhOkNykr9Gb9Vwn4rS3Q0ZlrVaFlZuGISeWN/bJRfHhqJwLyVhuRXAKl5XNfFAFfw4xkS8fi153f3oliRnYYOdzLADcbCXaWLVm/op4T4m3vXWBKsupd4QvQFvkE4wneSuoZaGtt1mWttAdVn/1396N3ffdpov+4xbj/twrlq87LNu4+/9+tt5+WvuveILZ9PEqfp9MkgcysqvMMtK9xrCLGdYdcROAsaqrKF2Ogi9W7VMnOd0Jt13bcPjqgUT07FrV31ZOJemt5QxEvuB5xIGwSahI66tri9lYRhxVJaY5Q/D7yvB7ohJeg+sgNBFl/+9eMiLjuNg+pKwjY6Fr2hjkTdUNJ2jaSp8Z0kri1manGWBW4BbQrWI2OXhht63mdajPH7xdR/ig59e8Nf+xKND/X7YmNeO//45j+c33/JRzq2u7OqxZIuQdDRXuEa85wyrVuGo8leeug5Mq3XnhqLFmo0yLVW6t73baHFoGAPW2o1hCLqELquQtb2FJcbyiD4Lyz/bZ1bRrWVZJbvSjMvhk0oalkEBqwmuJ967kMn31iKtwTTaMTS5Y+iahGkjtjuc5mELWo/iWHWBf/Gf38Pz7rjuUanLerj4quffTuMj/+ntH7+iz5OqMrIjWf0+jOiESVLF+2QMWGULdLFLVsDarRp2bMuu1SzrlF0NoznFqYFMtBPWAWtkIWM3ACtmHqpkXTH7so+38WzGGLCKlisWB9K4Dl4+WpowAFWXb20h3ztL7Ay0BmkF29IDlm3AthHbRqQ9HGhty8NHcfzkqz7AR8+u+Kdf85xHfcfwoHjuE6/jCdfN+M03f5Q/87zbr9jzlKUUfqaaLD9T8j1M1WImzPPK+pnvCff5pKW2oV/5teNadqy6i552K+a2YW7atS5hz2Vl4JqMtuGU7CNCNuLLC1hzdzAwqOJNnkmsRvOJQCbZ8+OkTLiPAGucgfmo3FfJqpahYuFr2mjZ72raYFm2Fc2qIngDS4tpDG4huIVgl1DtQ7WMuEXELj1mdTjNwzbTepTGA/u6Euy/febNfN7Trr2VYFcjRIQ/+Zm38sr3fPKKbqdOToi2KN83VO95nlDqmCUNvtdgzbKJ33TkizWzJavyF0gYhnVeYVgRBnmxhN4MZbwmd/p6/qp0CuMaYBXnUp1TjBdV0gOj7uFwr1mXwedbF5V077xV4j0IqTWqycpCUtPmLKtLSsK3EekCckgjxy1oPUrjh19xD/uN5298+TOP+1CONV72WY+nC4nfvYIlohr7ZfK9mPrNEn6eSDPdnjOZdezOGk5PG66bLnncZMH1kwU3Tva5vl5wnVuo2t0tOGMXzM1IjzVybyj7Biti3upM3uwsWJG8Cac4MmQCPWlBVZatFu/4Ta2XyUD4cFHKS58sTR7PWYWKpa9YdBWLtmLVVnSNIy4csrS4fck3qPaSZll7kWov4PY7zLKD5nC22VvQehTGx86u+KlXf5A/c42vBLsa8dm3n+H2x834zTd/5Io9R6yL7QzESSbeJzoAbaaeulbX0Z26Zbdu2K0aTlWNEu6902i2mMmZ1OA26jOPlbfoZB5rKpEKFKzWxmtydy+DVekQ2rystTzuZtlpD3CEKHGgbKLPqlTp3ngVkK461WJ1rSOuHNIY7NKoHmsFbpmyLitiVwHTBKT10HZIty0PH7PxQ694DyklvvtL7jzuQzn2EBG+9O5beNV7P33FfOQH8l0G8n06eLsXDmsNsArh7poesPoB6DxLON6YMwassg2nANZ4UWqEtSyrlIelJNRZxSFz65+LAlzZ3bSUjL3b6bgTmT22kvSD0E1WvWun0BEaq4C1Mkq6r8Au9d6tIm6ZsKuAXXmk9QpY7eFK+C0R/yiL++5f8POvvY+vfeETeeL18+M+nGsiXnLXTfzkqz7Aa95/P190101H/vhlIYWf5y3Qs4TMAm7asTNrODNbsVO1XD/ZZ2Y7dq1KGioJzG3T81Y7phnd+7WScKfsHSSxYyTvHTRYhEDq/dpXydIWPVYm3ntveInsSIuRyFQuBIiDBqZN9t7ql1gkwecMq5DvS1+x39bsr2ra1hH2HNIa3L7B7eswdH1OO4X1XqTa0yzL7neYpkOWDTQtyW8zrcdk/NQffoCU4C998dOP+1Cumficp15PbQ2vfPcnr8jjR6flYVHBpyphqkhVBWb1+iIK3Us4kOzFp73qZwXjqCQ8OMMaA1aJkBJdImdAw5gNMGRYudycSked5xI3pRHjONAzPknvYlqI9y5YGn+AFmuV3RtW9C4OdqXEu2kz8d558IEUIoSt5OExF40P/OLrPsRLn3ULt52ZHffhXDMxrx0vfMrjeOV7rgxo+alyWWGaVPU+DUxnLfNJy5nJiuvqJaeqFafdam1FfTHwK46jhW8aL0+d5k7hXBK1CBVCJQZTNFfEXuLQIbQYWnQAupR7Fcpl1QTmpssbdxJd7gCOYyyN2IyiyyoZ1sLXnG8mLNuK5WKCXziksbjzqniv9sHtJ9wK6vNRy8L9gF10mMZjFivoPKlpoWlIYds9fMzF77zt4zyw6Pj6E7oS7ErGF915E+/++B4fPbs88sdOI6lDWV1fO1W5T0ebn/sV9Xm4ucgMLGVl/XjjswLWXAJzSUxFmIhhIo5KbG8j0+8bTAOXVexnLKnnxHTNve8Bq0SxmemSUz1XBrGSrRUniKJ8L+4NC1+z6GrtFDYVfqldQruftVhLBaxqUcj3iF1F7MpjVh2y6jTL6jrwXgFrC1qPvXj5a+7l9sfN+MKn33jch3LNReGy/uA9nzryx+6Hoh3gdAB6Uvk8+Kz34xVfMJReRtIgRejLN9+voC+ShqlYKiyVWAyjTCslQkq0ZenEyL3BZMX7sPNw2BpdYtB0Sa+aL49RxKVFCX8BYHUVq5VKG2RlsQuDXa7PFLplpFpE3DL0AlJpPNJ20HUk75XLCoGUti4Pj6n4+LkVr3rvp/maFzzxMTVjeNh45q2neNy84o8/cP+RP/ZYSCp11E3QzjNxClhOAkbSmlf7YAsTs/ZKS8JTZsVcPHMJnDLC3FjmUjGRiom4fglFJNIR8kZnXT/f5nEdUMAqXcfxgouQdAaxQ7foDN7ug3dWcW9YxJq9MMkLVqec7Wbc3+xw/3LOA4sZZ/dmdOcncLbCnTXUZ4X6LNRnE/W5xOR8pD4fqfY8bq/D7rfIokFWDawaLQsLeIW4zbQea/FfMsn80seAX9alhIjw3Dsex+vvffDIHztaSDY7kJqEMQkjWoaVUqxwRxElsoF+AcWYKK9RDdZUUP4KLQULWFkZu5Dm7c6jTGlMwOtzxA2yfSDSh/lBN7KhWR+MLs4Ne75mr5uogDSXhF2jOiyzMheQ7q5J2CZm0n1QvEsh3r2Wg6lkWCmS4taa5jEVr3z3J7np1IRnPsbFpA8Vz33idfzeOz/B2WXHmdnR2TAno15ZySasSZi8/RkUJHw21zMxdwlN6k37xlnWVDxz49kxSrgX/sog62BFpEuBVYprXFbLYOJXnCBg5O0+yvL6DdDZ273fAt27kdY00XG+m7IfFLAeWM1YNDV7+1Ml3VeW6mwm3c9DtZ+wK5icD5gmUS08Zum1U7hoEB+g7UhNozzWiMtKMa3Z4zxUbDOtR0GEmPiv93yKF99542NyMPqw8bwnPQ6AN9734NE+sMl4MHrpfeaCmuBUOZ5sP7NXwhJ7RfpUOiYSqEiDpCFnWGPACikSUiKgm6C7ESel/276W/FxLxlYi2GVXL6tb9FZxAmLoLbJi1hz3k8530052015YDXnwdWMs4sZ+6VLuLDYPTOM5uwn3CJRLSN2lXBZ7W7aMIhHO09qOwiRFKICVkyPCLDyy72Nkx5v/fBZHlx0vOQKCCcfTfFZt59BBF7/waN1M00CG1VZFmGqlqmJg51LCe0eprUh6OLYUCQNZuPjGVIkknKmpVlWmTPU59z4+VwyqgTC9tlYNwatzF+tklomL0LNvp+w52vO+wnn2yl7Xc1+UyvpvsqzhIs8mtN3B9UuWbuEeTynjOjkkpCugxh64p0CVgWwDknEb8vDR0G88t2fRIRt1/Bh4tS04hm3nOINR51pjSIlIQRDGyxGEs5kQ75Rx6509KYyHoRWwLIHPGZIpdQcAKtske9yp08dSYVu9Ahdsv1qsDKOU6QNZTnFIk76lV8PdnOWoWI/1DzYzFj5inOrCYtVrcPP5ytMY6jOD/Yy9Tk18av2s6ShidhFh3QRs2qRptPyr221JGy7nsta47EOCViwzbQeFfGq936aZ912mht2J8d9KNd8PPeOx/GGex8gHpL0PXQkvaUIMRpCvhVHhJDGCnXtJo4HlQ9Un6OZlX6tgNWXhin1JWHJsMalYRnjKfOHZctO2bwzlIdubW39eT9h4Wv22gn7bc2yyYC1cthsk2yXClhulTLhnjBtwvZKdyXeyWR7fx8CxAPKwkcAWLDNtE58xJh4y4fP8qef+4TjPpQTEc994nX83Gvu5X2f2ufpN+8eyWNK0u3IEoQUDMEbumCwxtAFS7RZN5WdFooCfryAYiz4jCkRJSpRJrF3uAopqcyhmPzl0rCY/XW59INCvltsSmsuDatY0SbL+TBT7ipMedBrhvVAM2ffq/7q7GJG21q6/VpFoyuhOmewDVTn6cvBek+7g24RMI3vMyx8UC1W22mnsO20U+i9AlbOtC4ltqB1wuP9n95nr/F85u1njvtQTkQ86/GnAXjnx84dGWgR0UwrQoqaWcU4jMiU8nBs6lfU8cNDCCGBEVW5k7RcLB0/Xaia6FIkoBY0pTQMmB6wCm8WUEuayMBtAX3H8HyYshcm7IUJ57opC19xtp1qdtVWLBe1zhHuq2jU5bEc22TSfZVwjS6jsF3UsZzRPKH4MEgbxor3TdL9EWZZsC0PT3y85UNnASWZt/Hwcectu1gjvOOj547sMftMK0oGMFmb6SuarbXVXge4hMbRTbuDObMiqJNDSms/o6XnYItc9hO2hXQfLWJtR0LSsXB030/Y6ybsd1oOLpqaZlURy1jOwvT8lVuUDmHK23MG0v1gwPJQSPdNPdYllIUltpnWCY83f+gss8ry9JuOKGt4lMfEWZ520w7v+Oj5o3vQVAArQRJSykAlicoGapNHeUygyjOI48WqEUOHySWipm0G6FJYW1WvPBZ0Cdpk+uHoAkZdXphaZBXFG2uVva+65DgbZixCzae7Hc53U/b8hPuXc5adY28xpVs50spiz1tsI7i9oUNYn0/YNpPuTVDh6KpT0WiTZQ0hDCXheEQnxCMBLNiC1omPt3z4QT7j8adxdps0Hzbuvu00r33/0Y3zHORQLJIQhtLQjaxoNqPorDqMlpnZJCaKApWFwc8q/3yX9Vfra+3XZRWRoVQsy1UXQXVYKmuYsJ+HnpvO4VvdS6jmfernrvOD6tTgMuFeVO4luxIfB6I9RAWsuE6+rwHWZcYWtE5whJh464fP8XUveuJxH8qJirtvO82vvvEjPLhouW5eX/4D5s6hJEASImDNkGXVJbvKKnhL1AxLhjX1Gn5tzKcajeAYBsBaJV0msZ90XnCRFe1lJGccIQ87L4Kuqv90t8O+n3B/M+dcM2XRVqpw7yxpz+nQ80qozgt2BfX5QYNV7Wt2ZRcdpg3gI9K0SIhZi+UVsLpWyfbCZV1Gp/Cg2ILWCY73fnKPZRe2fNYjjLtvUzL+HR89f6SbiorIVCRhTcTlWyV6v+m1HpKhE0tJvgp5XqEZWZdMvznakHrv9/GgcyHWV6nqM68SZWdhkxx7fsIy1jzQzln4mrPNlL3VRGcIFxW0BrdnVc6wgmpPTfuUdM8arIXH+Ji7hMpbaUkYe8eGonbvtViPcETnMLEFrRMcb84k/Gc+YQtajyTuvk3nM9/x0XNHu15NAJMQE7EZpNx4PdfYbQGDIarrgwAJbNZU9eAlgZAkzymmPAYkmb+yrGKdSXbX+2iNO4WlXNzzk550P9dOWflqAKzGQT/0rHYyhXS37QBYbhWwTc6uNgh3ipQhxPWSsGRYJY4gy4ItaJ3oeNfHzjFxhqfcuCXhH0nctDvhhp2ad37siDqIkoemDYhJWJuoSpZlQk/Kl1B3UMFiWFFhU6KT2Hu018V6OQ9UWyKkskzVcC5O6ZLjfJzm4eZJn1WVpapj074Hu1nfIbx/OVdJw36tHcLGUJ032EaHnt1icBrVTMv33UGz7IbsqvMqFG11NIcul4IpkTp/IYd1RIAFW9A60fGuj+/1LfxtHD5EhLtvO31kHUQtCxOY1JeHIoM9jeXg0igU4l0iIVnseJhahoypeLcXAem4U1iyrrX19Ul5rP0woQmOvW6incK2ZtFUtE11wXov044I95VmWGWRai9n2FC26/1YzrAhGr0CgAVb0DrR8e6Pnefzn/7Y3B59uXHXLaf4udfcS4zpSEwTez5r008rg9d4ryBkIEqRgFVfG+jnEwPSK9nbZLES+9GcLlnOxZmWfWF6AWD5OGRZZzudH7x/NWe/rVg2Nau9Cakx2D3bj+RU55W/qs8PQ8/VvqrbdY4wl4PZoaEn3McZVgGuTQ7riAELtqB1ouNj51Y845atf9alxF237LLsAh96YMkdN1zmqjUZbpIzLQWr2Ls5jH3hgV4IWgAJII4ysg6wSX8+Rv3Z0iEs5eAi1r17RMmwlqGmiZZVHstZdjr0vFrW+MbBnsuloOl3EdbndX6w3s+C0UZJd3zsXRrW+SvVYJHiUBJeJcCCLWid+LhrC1qXFHfm1+3dHz9/2aBVsqwkqRdtlSyrxAWdw0y2tyOJQtkGPfyMEJOaFSpguTzgPOmzqVIK6teGfV+zCi4PPdesyqacxsLK4vYNps1k+3JkK9Mk7DL2PljSetVfdX7oEHqv4BSzQ0PpEo4Bq39RrgxgwRa0TnzctXUqvaQoc4fv/sR5vvRyLaplVB6yzmlZWee0VAFfrGbsmikgoNYyafjZsil6DFRNdPhks7mg7h5cBbVF3i8LJ9pKDfuK/mo50l81UO2N9Ff90HOnHlhdWNdfZf5q3CHsS8JNpTtcUcCCLWid6NidOB5/Znrch3Ei48ys4tbTU97z8b2r8nzFSRSJMM6uRu6BvcVM/l4XB5V7k78uK+i7ZGijU6PBaFn6ijbafuC5aRzdUvVXRTBqs2mfaVPuEhZJQ8B0EWkCplXDPvFFthAHcCoANS4JrzDpflBsQesEx1237G7tlS8j7rxll3d//Ag6iAe8BWnko9UlS0VgFas173agH3YG1uQKhVRvYkWXDMtQ6Tr6ZGiD0wwr6P833rHsHF2wyl3lcRyzb7Gt4HJ25ZY5w2rSsJq+idhlnh9cdcP8YAatnnCPaT3DSumK6bAeLragdYLjGdvS8LLirltO8TOv/uDlP1Ba/zIlUXDJewIrico9GbvGc4094ws35aPN5d9Q9vl8H5LR7mDQpaqtt/hg8cHQNhXBG9LCIa3gVurfblpVt7uVLp2o9yK21U3PJmdWvf6qdAdj7H2v+hnC3gMr9YB1KVbJRxFb0DrBsSXhLy/uumWXxh/NeIn0zqVCjEKICjAKXLFfeLoJWrEXhAptdIQkrEJFGxxt8ZYPlm7khuqDilO7zhG8IQRDXFnwBrNQoahd6cIJ2w7+V7ZJuEXAjuYHpVNXhrXuYNFgZYK9B6xRhvVIVn4ddWxB6wTHl33Grcd9CCc67jwi0JdscCVeiJ2hs45zqwlNUNA5Z6Y6h5i3S5eRnAJYPoNRmwGqzdmTjwbvLTEKMai9TIpC8oYUBTpBvEE6oVoJ0qHeV81g1mfb7H21StgmYPe7dXfRom4vvFUpBVNcy67UJlkBPpWlqkc0AP1IYwtaJzgef93suA/hRMedR+VcmsAEQbyQOoO3lqZzfWZUWx3lsWbI6sacV/m51qvWyntLCNn91BuIQgqiXjURCIIEQTrBdILxYJd67xYqFLUNVIuE6RJuof5X0uZh5w0pw1opWLKoXCb2gFXAaZN4P4bYgtY2HrNxalrxT776sy/7cUync4e2FVKjWdASMDaxtAFrUy+DAAWslM0CYxRSNHo/yqBICkx40QmhAERBgn4tAUyrQGXavNm5UxmDbVHeahExXcQu/dAZbFotBcfZVRaKEgtftQFWm+XgMfBY49iC1jYe0/FVz7/9sh/DNnqfrE5OxyoRu4pgE8FUdJIG1Xz5nMfcciw2zUEBiKj3peQ0QfLPKFBJUpCUqGBluoTplGg3XkHLdFnVvtKsyiy7QdVeVniNjfouBlawDljHDFYltqC1jW1cZhifSEYwHWCk94tPNvWbp0eW8T1pT9INPgWMxOvvKXgV3/nRLWSw6hISFSyNz6DV6ICzXWVX0S5gVjqK0xPtxahvM7vqOaqN7AqOlXC/WEg6ZtTcxja2sY1HEltj8W1sYxsnKragtY1tbONExRa0trGNbZyo2ILWNraxjRMV2+7hCQ4ReSuwOu7jeJi4EfjUcR/Ew8Q0pfTs4z6IbRwutqB1smOVUnrBcR/EQ4WI/PFJOMbjPoZtHD625eE2trGNExVb0NrGNrZxomILWic7/vVxH8AhYnuM2zjS2Crit7GNbZyo2GZa29jGNk5UbEFrG9vYxomKLWidwBCRbxSRN4vIW0TkVSJy+aZQRxwi8uUi8i4RuUdEvu+4j2czROSJIvIKEXm7iLxNRL77uI9pG4eLLad1AkNEPh94R0rpARH5CuAHUkqfc9zHVUJELPBu4KXAh4DXAl+fUnr7sR7YKETkNuC2lNLrReQU8Drgf7iWjnEbB8c20zqBkVJ6VUrpgfy/rwYu38nuaONFwD0ppfellFrg5cCfOuZjWouU0kdTSq/PX58H3gE84XiPahuHiS1onfz4duA/HvdBbMQTgPtG//8hrmFAEJEnA88F/uiYD2Ubh4jtGM8JDhH5YhS0vvC4j+WkhojsAv8B+CsppXOX+DBbjuVo4lCbh7eZ1gkJEflLIvLGfHu8iHwW8OPAn0opffq4j28jPgw8cfT/t+fvXVMhIhUKWD+bUvql4z6ebRwutkT8CQwRuQP4PeCbU0qvOu7j2QwRcSgR/yUoWL0W+IaU0tuO9cBGISIC/Fvg/pTSX7nMhztRH6IYE/fev+AT5xvOzCruumUXfTmOPQ51EFvQOoEhIj8OfBVQdrr7a81JQUT+JPDPAAv8RErpB4/3iNZDRL4Q+APgLeg2QYC/lVL6rUt4uGv6QxRi4mPnVrzh3gd45bs/ySve9Uk+eb7p//3W01O+6yVP5Zs/78kYc6zgtQWtbWzjKsWxfojOLjped+/9vOOj57nnE3t86IEFn9prOb/yNF1g2QV83qhzaup4yV038YVPv5HHXzfjY2dX/PIbPswfvu/TfMHTb+BHvvH5nJlVx/WnbEFrG9u4SnHVP0QpJV75nk/xo7//Xl79/k/3qwhvOzPljuvn3HRqwqlpxbQyzGvL46+b8cxbT/PZt5/BWXPBY/3ca+7jf/21t/LsJ5zhp7/9c9idHEuPbgta29jGVYqr+iFadYG//otv5tff9BFuPT3la174RD7vqTfw7Cec5tT00rOk337rx/hL/+71vOjJ1/NT3/4iKnvV+3Rb0NrGNq5SXLUP0X7j+eafeA2v++ADfO9L7+IvvOSpTJw9ssf/xdd9iL/2C2/iO178FL7/Zc86ssc9ZBwKtLY6rW1s44RESom/8R/e/P+2d+/BUVV3HMC/v91NSFgSAuQFCSGBhBBEwfCQ8qqCrYBFGcQpYNXWR0eLTp2pb1sHnVqf7ThOHR8VRKc4WkVbtTx8AYoMEEQChCRgHiQgGxOSkISQ969/7GYaIyT7uLt3L34/Mxl2l3PP/U1m8p17zp57Lr6uqMPzK3Jx5UXDDT/H0smpyK+sxz++KENu2hAsuND4cwSK67SILGLtjnJ8uP8E7r4iOyiB1e1PvxiPiSPjcN/6/ThefyZo5/EXQ4v8IiKrRORuz+tHReRyP/vhbgteqKxtxhMbizB3XCJumzMmqOeKdNjw3LJJ6FLgrje/RkdnV/8HhRBDiwKmqg+r6id+Ht4B4A+qOh7AdAArRSTkkynh7rH/FsImgj8vnhCStVSjhjnx58UTkFdeh2c+Ohz08/mCoUVeE5GHROSwiGwHkN3j87UistTzulxEHvfcbrRHRHJFZLOIlIjIbb375G4L/dt+pAabCly4Y24mRsRFh+y8iy9OwXWXpOHFbSXYdPBEyM7bH4YWeUVEJgNYBmASgIUApvbRvEJVJ8G94nwtgKVwX0U90s850sHdFr5HVfH0R8VIiYvGzbMyQn7+hxe557fuemsf8sprQ37+s2FokbdmA3hPVZs9uyG830fb7v87AGCXqjaqajWAVhGJO9sBBu22cN7Zerga+ZX1uHNuJqIijFva4K0BDjtW3zgFI+KicdOredhXWR/yGnpjaFEwdN/Y1tXjdff7Hyyz4W4LZ6eqePaTI0gdEo0luebt8xg/aADW3XIJ4pwRWP7yTnxaWGVaLQBDi7z3OYDFIhLt2Z54kRGdenZbWA339tF/M6LP88UXR2qQX1mPlZdlItJh7p/q8MHRWH/7DGQmDsKtr+/Bm7srTKuFoUVe8UyWvwUgH+6dUvMM6nomgOsBzO2xX9hCg/q2tBe2liA5NgpLcsPje4nEmCi8+dvpmJ2VgPvfPYDnt3xjSh28jYcocIb/Ee2rrMfi57/EQwtzcOuc0UZ3H5D2zi7c83Y+/r3vW6xaNB6/nmnYFwS8jYfIql7cWoLYKAeWX5Jmdik/EGG34ZlrJ6K5rROrPjiExNgoLAzh7T4cHhKFmWJXIzYVuHDjjHSztojpl8Nuw3PLL8bFaXG45+18lNWcDtm5GVpEYea5z47AGWk3ZV2WL6Ii7Pj7ilxEOGxYuW4vWto7Q3JehhZRGDlS1YgNB07gxhnpiBsYaXY5/UqJi8Zfr52IQycaQjYxz9AiCiNPbirGwAg7bpkdXpPvfZmXk4QluSl4YWsJilzBXxfM0CIKE1uKvsMnhVW4c14WhjrD/yqrpz9eOR6x0RF44N0DCPaKBIYWURhoae/EIx8UYHSCEzcZt4QgZIY6I3H/gnH4uqIeH+4P7s3VDC2iMPDExiKUn2zGo1dNMH31u7+uyU3FuOQYPLW5CK0dwZuUt+Zvh+g88lGBC2t3lOOmmRmYlRVvdjl+s9sEDy7MQWXtGazbGbzbfBhaRCY6VteMe97ZjwkpsbhvQXb/B4S5OWMTMH30ULz0eUnQrrYYWkQmae3oxMp1e9GliudX5Br6VB0z3XFZFqoaWvHOV8eC0j9Di8gkT20qRv6xU3h66USMGuY0uxzDzMwchokj4/DitpKg7C/P0CIywe6yWqz5sgy/mp6G+ROSzS7HUCKClZeOQWXtGWw46DK8f4YWUYidaevEve/kI3VINB5YkGN2OUFxeU4SMuKdWLO9zPC+GVpEIfbCthKUn2zGk0sugjNMb4gOlM0m+M3MdOyrrMfeijpj+za0NyLqU2VtM17aVoJFE0dgRqZ1lzd445rcVMREObDa4KsthhZRCP1lg/v5hQ8sGGd2KUHnHODAsqkjsfmgC65TLYb1y9AiCpFdpSex8aALt186JqTPLzTT9dPT0amKNwzcU56hRRQCXV2KxzYUYvjgKNxqoR0cApU2bCDmZifijV0VaOswZvkDQ4soBP6Tfxz7j53CvfOzER15fiwi9dYNM9JR09SKjQY9pZqhRRRkLe2deHpTMS5MGYyrJ4bHk3VCaXZmPNKHDcS6XcYMERlaREG25ssyfHuqBQ8uzIHN5tUDZ84rNpvgl1PTsLusFiXVTYH3Z0BNRHQOtafb8MKWElyek4SfjBlmdjmmuWZyChw2wVt5lQH3xdAiCqLV20vR1NaB++ZbfweHQCTGRGFeTiLWf3Us4Al5hhZRkJw6047XdxzFggnJyEqKMbsc0y2bmoaTp9uw7XB1QP0wtIiC5J87j6KxtQO/uzTT7FLCwqyseMRGObC5ILCbqBlaREHQ0dmFV78sx6XZCZiQMtjscsJChN2GeTlJ+LSwKqAtaxhaREGwo+QkappasXxa+D3W3kxXXJCEuuZ27C6v9bsPhhZREHyQ/y1iBjjw07EJZpcSVuaMTcAAhw0fFVT53QdDi8hgrR2d2FTgws8vSEZUxI9r9Xt/BkY6MGdsAj4+VOX38xEZWkQG21ZcjcaWDlw1aYTZpYSl2VnxOF5/Bsfrz/h1PEOLyGCbC6oQNzACM37Ei0n7kps2BADw1VH/NgdkaBEZLK+8FtPShyLCzj+vsxmXHANnpJ2hRRQOqhpaUFHbjGkZQ80uJWw57DZMSovDnnKGFpHp8jxf5U9NZ2j1ZfKooShyNaCptcPnYxlaRAbKK6tFdIQd40fEml1KWJsyagi6FPjaj4deMLSIDJRXXofcUXGcz+rHpLQ4iMCvISJ/s0QGaWhpR6GrAVNGcWjYn9ioCIxJGISCbxt8PpahRWSQ/Mp6qAJT0oeYXYolZCfH4HBVo8/HMbSIDFJ4wn3VcMEI3iDtjeykGFTUNqO5zbfJeIYWkUGKXI1IiBmAoc5Is0uxhLGePcaOVPm2BTNDi8ggxa5GjEvmZn/eyvb8ropdvg0RGVpEBujo7MKR75qQzR1KvZY2dCCiImwo9nFei6FFZIDyk81o6+jCuOFcn+Utu02Qlej7ZDxDi8gA3UMcDg99MzYphsNDIjMUuxpgEyAzcZDZpVhKdvIgfNfYirrTbV4fw9AiMkCRqxHp8U5u+uej7GT3cNqXeS2GFpEBiqv4zaE/xiQ4AQBlNae9PoahRRSglvZOVNQ2IyuRoeWrEYOjEemwMbSIQunoyWaoAqM9Vw3kPZtNkDHMidJqhhZRyJRWu1d0j47nJLw/MuKdKKvxflU8Q4soQKWeoU0Gr7T8kh7vREVts9ftGVpEASqtPo2k2AEYNMBhdimWNDreifZO7x8nxtAiClBpTROHhgHw9QqVoUUUoNLq05yED0BGPEOLKKROnWn3+Q+P/m+YMxIxUd4PrRlaRAYYk8Dhob9EBKN9CH2GFpEBODwMjC9XqgwtogBF2m1IHTLQ7DIs7aLUOK/bMrSIAjQ1YwjsNjG7DEu7aVaG121F1fv1EUR0VvwjMoZXyc8rLSKyFIYWEVkK7zsgChwntEKIV1pEZCkMLSKyFIYWEVkKQ4uILIUT8UQBEpGDAFrMrqMf8QBqzC6iH1GqOqG/RgwtosC1qOoUs4voi4jssUKN3rTj8JCILIWhRUSWwtAiCtzLZhfghfOmRt4wTUSWwistIrIUhhZRAETkOhHZLyIHRGSHiEw0u6beRGS+iBSLyDcicr/Z9fQmIiNFZIuIHBKRAhH5fZ/tOTwk8p+IzABQqKp1IrIAwCpVvcTsurqJiB3AYQA/A3AMQB6A5ap6yNTCehCR4QCGq+peEYkB8BWAxeeqkVdaRAFQ1R2qWud5uxNAqpn1nMU0AN+oaqmqtgF4E8DVJtf0Pap6QlX3el43AigEkHKu9gwtIuPcDGCj2UX0kgKgssf7Y+gjEMwmIukALgaw61xtuCKeyAAichncoTXL7FqsSkQGAVgP4C5VbThXO15pEflIRFaKyD7PzwgRuQjAKwCuVtWTZtfXy3EAI3u8T/V8FlZEJALuwFqnqu/22ZYT8UT+E5E0AJ8BuEFVd5hdT28i4oB7In4e3GGVB2CFqhaYWlgPIiIAXgNQq6p39dueoUXkPxF5BcA1AI56PuoItxuTRWQhgGcB2AGsUdXHzK3o+0RkFoAvABwA0OX5+EFV3XDW9gwtIrISzmkRkaUwtIjIUhhaRGQpDC0ishSGFhFZCkOLyMJEZJWI3O15/aiIXO5nP1EisltE8j07LTxibKXG4W08ROcJVX04gMNbAcxV1SbP6vTtIrJRVXcaVJ5heKVFZDEi8pCIHBaR7QCye3y+VkSWel6Xi8jjnluN9ohIrohsFpESEbmtd5/q1uR5G+H5CctFnAwtIgsRkckAlgGYBGAhgKl9NK9Q1UlwrzZfC2ApgOkAzjr0ExG7iOwD8B2Aj1X1nDstmImhRWQtswG8p6rNnp0Q3u+jbff/HQCwS1UbVbUaQKuIxPVurKqdnpBLBTBNRPp9cKoZGFpE569Wz79dPV53vz/nfLaq1gPYAmB+0CoLAEOLyFo+B7BYRKI9WxMvMqJTEUnovvoSkWi4t2cuMqJvo/HbQyIL8eyj/haAfLjnnvIM6no4gNc8e8rbAPxLVT80qG9DcZcHIrIUDg+JyFIYWkRkKQwtIrIUhhYRWQpDi4gshaFFRJbC0CIiS2FoEZGl/A/6tLyGhCJesQAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "posterior_samples = posterior.sample((5000,))\n", - "\n", - "fig, ax = pairplot(\n", - " samples=posterior_samples,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - " upper=['kde'],\n", - " diag=['kde'],\n", - " figsize=(5,5)\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The 1D and 2D marginals of the posterior fill almost the entire parameter space! Also, the Pearson correlation coefficient matrix of the marginal shows rather weak interactions (low correlations):" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWYklEQVR4nO3df6xcZZ3H8fen97Y0u6wWKCmI2EJokBpWkAYwRGUBtRBDu4rabpTiQroY3F1FDSKJbHBJym4iiz+xgQooCygi1ohhkR+LRmEpbKH8WOQKKq2VSvmhhFK4vd/94zxThunMnXM7Z849c+7nZU46c84z85yJuV+eX+f5KiIwM+u3aZN9A2Y2NTjYmFkpHGzMrBQONmZWCgcbMyuFg42ZlcLBxqymJK2WtFnSgx2uS9KXJY1IekDS25quLZf0WDqWF3E/DjZm9XUFsGic6ycC89OxAvgGgKQ9gfOBo4AjgfMl7dHrzTjYmNVURNwJPDNOkcXAVZG5C5glaV/gvcAtEfFMRDwL3ML4QSuX4V6/wMyK85f7z4ztL43lKrvt6VceAl5qOrUqIlZNoLr9gCeb3m9I5zqd74mDjVmFjL00xgHvn52r7P+t2vRSRCzs8y0Vxt0osyoRTJumXEcBNgL7N71/YzrX6XxPHGzMKkbKdxRgDXBqmpU6Gng+IjYBNwPvkbRHGhh+TzrXE3ejzCpEwLSCmgCSrgGOBWZL2kA2wzQdICIuBW4CTgJGgBeBj6Vrz0j6InBP+qoLImK8geZcHGzMqkQwNFxMsyUilnW5HsBZHa6tBlYXciOJg41ZxaimgxsONmYVIsG0ggZkqsbBxqxi3LIxs1IUNUBcNQ42ZhUiuWVjZiUZGvKYjZn1meRulJmVQqiYRxEqx8HGrErcsjGzsniA2Mz6TvIAcVtp+8DrgHnAb4APpZ29WsttB9ant7+LiJN7qdeszurajer1Z30OuDUi5gO3pvftbI2Iw9LhQGPWgQBNU65j0PQabBYDV6bXVwJLevw+s6ktDRDnOQZNr2M2c9JmOwB/AOZ0KDdT0lpgFFgZETe2KyRpBdku72hYR8yYNXWGlA7+iwMn+xZKs23W1sm+hVI9+sDvno6IvfOWr+lzmN2DjaSfAvu0uXRe85uICEnR4WvmRsRGSQcCt0laHxG/bi2UNmteBTBz7xkxb0m+vVjr4OYjLp/sWyjNyOK2aYxq6x1vOPO3ectmm2fVM9p0DTYRcUKna5KekrRvRGxKKSA2d/iOjenfxyXdARwO7BRszKa8AjfPqppee35rgEa2vOXAD1sLpH1Md0uvZwPHAA/3WK9ZLQkxTfmOQdNrsFkJvFvSY8AJ6T2SFkq6LJU5BFgr6X7gdrIxGwcbs3YKzq4gaZGkR1OK3Z1miyVdLGldOn4l6bmma9ubrq3p9af1NAIbEVuA49ucXwuckV7/Aji0l3rMpooix2wkDQFfA95NlmjuHklrmv9jHxGfair/j2RDHA1bI+KwQm4Gp3Ixq5xpmpbryOFIYCQiHo+Il4FryZardLIMuKaAn9CWg41ZlShfFypn6yd3Gl1Jc4EDgNuaTs+UtFbSXZKW7OIv2mHqLGQxGwAChodytwFmp/VrDRPN9d1sKXB9RGxvOpdryUpeDjZmFZJtnpU72DzdJdf3RNLoLqUlh1TRS1bcjTKrlEK7UfcA8yUdIGkGWUDZaVZJ0puBPYBfNp0rfMmKWzZmVVJg3qiIGJX0CbI83UPA6oh4SNIFwNqIaASepcC1KUNmwyHANyWNkTVKel6y4mBjViFFP64QETeR5fRuPveFlvf/0uZzhS9ZcbAxqxKJoaGhyb6LvnCwMauQKf0gppmVy8HGzPpOIu/q4IHjYGNWKfkfshw0DjZmFTOI20fk4WBjViESDA97NsrM+qyxeVYdOdiYVYk8G2VmJXGwMbO+E576NrMyyFPfZlYCIYaHpk/2bfRFIe21HDu47ybpunT9bknziqjXrG4EDGko1zFoeg42TTu4nwgsAJZJWtBS7HTg2Yg4CLgYuKjXes1qSWLatKFcx6ApomWTZwf3xcCV6fX1wPFSTRcTmPVomoZyHYOmiDGbdju4H9WpTNo97HlgL+DpAuo3qw2hiexBPFAqNUAsaQWwAmB498GL3Ga9ksT0oRmTfRt9UUSwybODe6PMBknDwOuBLa1flNJQrAKYufeMaL1uVn+q7TqbIn5Vnh3c1wDL0+tTgNtaNlc2MxqpXIobIM4xU3yapD825fQ+o+nackmPpWN562cnqueWTc4d3C8Hvi1pBHiGLCCZ2U5U2LR2nlzfyXUR8YmWz+4JnA8sBAK4N3322V29n0LGbLrt4B4RLwEfLKIuszor+HGFHTPFAJIaM8V5UrK8F7glIp5Jn70FWEQPucDr2Tk0G1gTWmczO+XibhwrWr4sb67vD0h6QNL1khrjr7nzhOdVqdkos6lugrNR3dLv5vEj4JqI2CbpH8jWwx3X43e25ZaNWYU0ulF5jhy6zhRHxJaI2JbeXgYckfezE+VgY1YlxT6u0HWmWNK+TW9PBh5Jr28G3pNyfu8BvCed22XuRplVigp7FCHnTPE/SToZGCWbKT4tffYZSV8kC1gAFzQGi3eVg41ZhWQZMYvrcOSYKT4XOLfDZ1cDq4u6Fwcbs0opbp1N1TjYmFWIJIb9bJSZ9Zv3IDazckgMDeDGWHk42JhVSNaycbAxs76r7xYTDjZmFSLE8DQPEJtZv0nI3SgzK4PHbMys74SYhoONmZXALRsz6zsV+CBm1TjYmFWKGJJno8yszySvszGzktS1G1VICO0lN42ZNZNzfXfSS24aM3st4UV94+klN42ZtfA6m87a5Zc5qk25D0h6J/Ar4FMR8WRrgZT3ZgXAfnvO4Y4jvlfA7Q2GY++dOjn8PnLwoZN9C5UlFftslKRFwCVkexBfFhErW66fDZxBtgfxH4G/j4jfpmvbgfWp6O8i4uRe7qWsYe8fAfMi4q+BW8hy0+wkIlZFxMKIWLjX7rNKujWzKiluzKZpiONEYAGwTNKClmL/CyxMf5vXA//WdG1rRByWjp4CDRQTbHrJTWNmr5GN2eQ5ctgxxBERLwONIY4dIuL2iHgxvb2L7O+3L4oINr3kpjGzJiIbs8lzUFz63YbTgZ80vZ+ZvvcuSUt6/W09j9n0kpvGzFpNaFFfEel3s1qljwALgXc1nZ4bERslHQjcJml9RPx6V+soZFFfL7lpzOxVBQ8Q50qhK+kE4DzgXU3DHUTExvTv45LuAA4HdjnY1HNdtNkAE0O5jhzyDHEcDnwTODkiNjed30PSbun1bOAYelzO4scVzCqkyKe+cw5x/DuwO/A9SfDqFPchwDcljZE1Sla2Wag7IQ42ZpVS7BYTOYY4TujwuV8AhS6IcrAxqxjVdHTDwcascjTZN9AXDjZmFeI9iM2sRO5GmVkJ5G6UmfWfkLcFNbNyuGVjZn3nAWIzK427UWbWZ8IDxGZWCnkFsZmVxS0bMyuBWzZmVgLl3atm4DjYmFVINkDslo2ZlcCzUWbWfxL4cQUzK0NdWzaFhFBJqyVtlvRgh+uS9GVJI5IekPS2Iuo1q59snU2eI9e3SYskPZr+9j7X5vpukq5L1++WNK/p2rnp/KOS3tvrLyuqvXYFsGic6ycC89OxAvhGQfWa1U5R2RVypt89HXg2Ig4CLgYuSp9dQJaN4S1kf9tfV840nJ0UEmwi4k6y5HOdLAauisxdwKyWLJlmxquPK+T5Xw5d0++m91em19cDxytLs7AYuDYitkXEE8BI+r5dVtZIVK40oJJWNFKJbnnhuZJuzaxKNIGjkPS7O8pExCjwPLBXzs9OSKUGiCNiFbAK4K1z3xyTfDtm5Yt05FNY+t0ylNWyyZUG1MwCRb4jhzx/dzvKSBoGXg9syfnZCSkr2KwBTk2zUkcDz0fEppLqNhssYzmP7rqm303vl6fXpwC3RUSk80vTbNUBZJM7/9PDryqmGyXpGuBYsj7kBuB8YDpARFxKlpHvJLJBpheBjxVRr1kt5Wu15PiaXOl3Lwe+LWmEbJJnafrsQ5K+S5bfexQ4KyK293I/hQSbiFjW5XoAZxVRl1mtBajA0coc6XdfAj7Y4bMXAhcWdS+VGiA2MyYyQDxQHGzMqqagblTVONiYVU09Y42DjVmlBHmntQeOg41Z1dQz1jjYmFWOg42ZlcLdKDMrQ5HrbKrEwcasSib2IOZAcbAxq5SAsXpGGwcbswoR9e1G1XMbdzOrHLdszKrGs1Fm1nceIDazssgDxGZWinrGGgcbs0oJPPVtZmUIoqYDxJ76Nqua4jY870jSnpJukfRY+nePNmUOk/RLSQ+ltNkfbrp2haQnJK1Lx2Hd6nSwMauQCIixyHX06HPArRExH7g1vW/1InBqRDRS8P6HpFlN1z8bEYelY123Ct2NMquY2N5jsyWfxWQZUSBLv3sHcM5r7iPiV02vfy9pM7A38NyuVFhIy0bSakmbJT3Y4fqxkp5vanJ9oV05symvMUCc5+iefnc8c5pyt/0BmDNeYUlHAjOAXzedvjB1ry6WtFu3Cotq2VwBfBW4apwyP4uI9xVUn1lNTWiAeNz0u5J+CuzT5tJ5r6kxIqTOT2RJ2hf4NrA8IhrNrnPJgtQMspTZ5wAXjHezReWNulPSvCK+q2HbrK2MLG7bUKqljxx86GTfQmm+85/rJ/sWqq2gXlREnNDpmqSnJO0bEZtSMNncodzrgB8D50XEXU3f3WgVbZP0LeAz3e6nzAHit0u6X9JPJL2lXQFJKxpNwue2vFDirZlVR0TkOnrUnHZ3OfDD1gIpZe8PgKsi4vqWa/umfwUsAbq2DMoKNvcBcyPircBXgBvbFYqIVRGxMCIWztpr95JuzaxCJjZm04uVwLslPQackN4jaaGky1KZDwHvBE5rM8V9taT1wHpgNvCv3SosZTYqIv7U9PomSV+XNDsini6jfrNBUsZsVERsAY5vc34tcEZ6/R3gOx0+f9xE6ywl2EjaB3gqDUQdSdai2lJG3WaDJKKQNTSVVEiwkXQN2Zz9bEkbgPOB6QARcSlwCvBxSaPAVmBp1HVNtlmvSllmU76iZqOWdbn+VbKpcTProq7/HfYKYrMq8VPfZlaWkh5XKJ2DjVmVBB6zMbMyeDbKzMriAWIz67u0n00dOdiYVY2DjZn1W0R4NsrMyuFgY2b95zEbMyuHu1FmVoYAxhxszKzPIoKxV7ZP9m30hYONWcW4G2Vm/VfjbpQzYppVSr5smL3OWOVJv5vKbW/af3hN0/kDJN0taUTSdWlz9HE52JhVSWTdqDxHj/Kk3wXY2pRi9+Sm8xcBF0fEQcCzwOndKnSwMauQAGJsLNfRo8VkaXdJ/y7J+8GUvuU4oJHeJdfnPWZjViURRP7ZqNmS1ja9XxURq3J+Nm/63ZmpjlFgZUTcCOwFPBcRo6nMBmC/bhU62JhVSUxoNqqM9LtzI2KjpAOB21KuqOfz3mCznoONpP3JcnzPIWsFroqIS1rKCLgEOAl4ETgtIu7rtW6z+okiukjZNxWQfjciNqZ/H5d0B3A48H1glqTh1Lp5I7Cx2/0UMWYzCnw6IhYARwNnSVrQUuZEYH46VgDfKKBes/oJYHvkO3qTJ/3uHpJ2S69nA8cAD6c0TLeTpWjq+PlWPQebiNjUaKVExJ+BR9i5/7aYLF9wpOTksxq5gs3stUoaIM6TfvcQYK2k+8mCy8qIeDhdOwc4W9II2RjO5d0qLHTMRtI8smbW3S2X9gOebHrfGFDahJntUNZ+NjnT7/4COLTD5x8HjpxInYUFG0m7k/XlPtmc23uC37GCrJvFnP32LOrWzAZHMJHZqIFSyDobSdPJAs3VEXFDmyIbgf2b3rcdUIqIVRGxMCIWztpr9yJuzWzARFndqNL1HGzSTNPlwCMR8aUOxdYApypzNPB80xy/mTWUt4K4dEV0o44BPgqsl7Qunfs88CaAiLgUuIls2nuEbOr7YwXUa1ZDxU19V03PwSYifg6oS5kAzuq1LrPaa0x915BXEJtVSEQw9vJo94IDyMHGrEom9rjCQHGwMauSgBh1sDGzvnN2BTMrQbhlY2aliHCwMbMSBIxtq+fjCg42ZlVS0oOYk8HBxqxCPGZjZuXwmI2ZlcXdKDPrP3ejzKwMEcHYtno+G+UkdWZVksZs8hy9yJN+V9LfNKXeXSfpJUlL0rUrJD3RdO2wbnU62JhVSYXS70bE7Y3Uu2QZMF8E/qupyGebUvOu61ahg41ZlaQxm363bJh4+t1TgJ9ExIu7WqGDjVmllNONIn/63YalwDUt5y6U9ICkixv5pcbjAWKzComxCT2uMG6u74LS75JyvB0K3Nx0+lyyIDUDWEWWR+qC8W7WwcasUib0uMK4ub6LSL+bfAj4QUS80vTdjVbRNknfAj7T7WbdjTKrkvLGbLqm322yjJYuVCOjbcqusgR4sFuFbtmYVUl5jyusBL4r6XTgt2StFyQtBM6MiDPS+3lkOd/+u+XzV0vamyzZwTrgzG4V9hxsJO0PXEU2wBRk/cZLWsocSxY5n0inboiIcft3ZlNRlLQHcZ70u+n9b8hSZbeWO26idRbRshkFPh0R90n6K+BeSbc0JSBv+FlEvK+A+sxqzc9GdZAGijal13+W9AhZJGwNNmbWRUQwOubNs7pK/bvDgbvbXH67pPuB3wOfiYiH2nx+BbAivX3hHW8489Ei7y+n2cDTk1DvZJlKv3eyfuvciRTeHm7ZjEvS7sD3gU9GxJ9aLt8HzI2IFySdBNwIzG/9jrRGYFXr+TJJWjvedGLdTKXfOwi/NQjGahpsCpn6ljSdLNBcHRE3tF6PiD9FxAvp9U3AdEmzi6jbrG7GInIdg6aI2SgBlwOPRMSXOpTZB3gqrVQ8kizIbem1brM6qmvLpohu1DHAR4H1ktalc58H3gQQEZeSPcT1cUmjwFZgaURlQ/OkduMmwVT6vZX/rRH17Uapun/zZlPPQcNz4kuv+3Cusouf/cq9VR+DauYVxGaVEp6NMrP+CxjIwd88/CBmE0mLJD0qaUTSTjuX1Ymk1ZI2S+r6AN2gk7S/pNslPSzpIUn/PNn31FFkA8R5jkHjYJNIGgK+BpwILACWSVowuXfVV1cAiyb7JkrSeKRmAXA0cFZ1/7+N2gYbd6NedSQwEhGPA0i6lmzrxFo+dhERd6YV37U3SI/UBPhxhSlgP+DJpvcbgKMm6V6sT7o8UjPpwgPEZoOvyyM11RBe1DcVbCTbJKjhjemc1UC3R2qqos6zUQ42r7oHmC/pALIgsxT4u8m9JStCnkdqqqO+K4g9G5VExCjwCbId5B8BvttuG4y6kHQN8EvgYEkb0vaQddV4pOa4pgyOJ032TbWTtWw8G1V76Yn0myb7PsoQEcsm+x7KEhE/J9srt/IigldqOhvllo1ZxZTRspH0wbTAcSxtct6pXNuFrpIOkHR3On+dpBnd6nSwMauYkvazeRB4P3BnpwJdFrpeBFwcEQcBzwJdu+EONmYVEiWtII6IRyKi27a7Oxa6RsTLwLXA4jTgfhxwfSqXJ1e4x2zMqmQDz918dtyQdxfLmeOl3y1Ap4WuewHPpUmVxvmd0r20crAxq5CIKOx5tfFyfUfEeBkw+8LBxqymxsv1nVOnha5bgFmShlPrJtcCWI/ZmFknOxa6ptmmpcCatKXv7WTb/UL3XOGAg43ZlCTpbyVtAN4O/FjSzen8GyTdBF0Xup4DnC1phGwM5/KudXoPYjMrg1s2ZlYKBxszK4WDjZmVwsHGzErhYGNmpXCwMbNSONiYWSn+H4jRLO0e+4NOAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "corr_matrix_marginal = np.corrcoef(posterior_samples.T)\n", - "fig, ax = plt.subplots(1,1, figsize=(4, 4))\n", - "im = plt.imshow(corr_matrix_marginal, clim=[-1, 1], cmap='PiYG')\n", - "_ = fig.colorbar(im)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It might be tempting to conclude that the experimental data barely constrains our parameters and that almost all parameter combinations can reproduce the experimental data. As we will show below, this is not the case." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Because our toy posterior has only three parameters, we can plot posterior samples in a 3D plot:" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "rc('animation', html='html5')\n", - "\n", - "# First set up the figure, the axis, and the plot element we want to animate\n", - "fig = plt.figure(figsize=(6,6))\n", - "ax = fig.add_subplot(111, projection='3d')\n", - "\n", - "ax.set_xlim((-2, 2))\n", - "ax.set_ylim((-2, 2))\n", - "\n", - "def init():\n", - " line, = ax.plot([], [], lw=2)\n", - " line.set_data([], [])\n", - " return (line,)\n", - "\n", - "def animate(angle):\n", - " num_samples_vis = 1000\n", - " line = ax.scatter(posterior_samples[:num_samples_vis, 0], posterior_samples[:num_samples_vis, 1], posterior_samples[:num_samples_vis, 2], zdir='z', s=15, c='#2171b5', depthshade=False)\n", - " ax.view_init(20, angle)\n", - " return (line,)\n", - "\n", - "anim = animation.FuncAnimation(fig, animate, init_func=init,\n", - " frames=range(0,360,5), interval=150, blit=True)\n", - "\n", - "plt.close()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "HTML(anim.to_html5_video())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Clearly, the range of admissible parameters is constrained to a narrow region in parameter space, which had not been evident from the marginals." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the posterior has more than three dimensions, inspecting all dimensions at once will not be possible anymore. One way to still reveal structures in high-dimensional posteriors is to inspect 2D-slices through the posterior. In `sbi`, this can be done with the `conditional_pairplot()` function, which computes the conditional distributions within the posterior. We can slice (i.e. condition) the posterior at any location, given by the `condition`. In the plot below, for all upper diagonal plots, we keep all but two parameters constant at values sampled from the posterior, and inspect what combinations of the remaining two parameters can reproduce experimental data. For the plots on the diagonal (the 1D conditionals), we keep all but one parameter constant." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABtpElEQVR4nO39eZBl2X3fiX3OOXe/b8utMmvr7qruQi+AQIBLN0Q0uAIjkiJNOkzNaAkqFNbMWLYUI4Ul2wprhqYUZsyE7dDIMZ6xR5YYlMLykNJIsjUayRSapEg0IXSDBAkC6G50F6qX2nJ9+da733v8x3lZVWj0Uktmvsys84nI6Fxe3ncyq/P7zvnd7+/7E1prLBaL5bgg570Ai8ViuResaFkslmOFFS2LxXKssKJlsViOFVa0LBbLscKKlsViOVY49/E91iOxP4gHvcAfa/85rasKnef7sZ6Hms83/+S+/z0+J//E/P4mhAAhUQtdRBjSLHep2j5l2yFbUFShIO8JqgiqSFP1akRQE3dTumHGcjjl8dYWp7wxTwfXOesMWFUFChhrwSvFKm8Vy9wserwxXmE3j9gat0inHk3i4AwcVCZwx+CNNU4CYb/CSWrc3Qw5mKAnU+r+AJr6Q3+cu/l3uB/RshwR5EIPnabUVrQeWoRSCM9DtFvoOKTq+pQth6KlKFuCKhRUsRGsOm6QcYkXVCxEKcvhhLVwzPmgz5oz5DG3z5qqWZAh3ygq1usObxXLXM0W2cg7bCZtxplvBGvqIhOJMxU4mREsd6JxE407qlBJYQSrP6CeTO9KsO6WYy9a46ykaaAbufNeyqHT9NpIz0WVFU2SoKtq3kuyHCLCcZCtGNFq0fRa1JFL0XEpWpIyFhRtQe1D2dLUcY2IaqJWTivIORWNORMOOesPuOhtsuYMedTRJI3mzSrjrWqV9bLLW9kyN9IuO1nMIAnJMpdm4qImCicFdwIqA3ei8UcNzrTGGabISYYeDGnSbF8FC05ATesv/r9/n5/7pZfmvYy50EQuTRQg4gjhOOaoYHk4EMLssIIAHQXUkUsVOVShpAoEdWAEqw41TdCA3+D4FbFf0PFylvwpS+6UZWfMKTVmSeZEwiPTsFWHrJddtqs2/SJimIeMc58sc6lTB5lJVA4qE6gMnFTjphqV1qi0QiQ5pBnNNEVX5b7/6Md6p7U5yvjCG1toDVf7CecXo3kv6VApFgOc0MHBvPqIaUI9Gs17WZaDRiqk5yKXFtGdmKoTUCx4lJEk7wrK2NSwyo6mDhvolARRSTvKONMasuQnPBFtct7tc97d4Uk3RwrF5TLn7WqJ6+UCb6SrbBUtrk4W6E8j0tSjHnnIVOKOJO4UnAT8YYObavzdEmeUI8cZbPdpshxdFgfz4x/IVQ+Jf/W1m+y1Tv76qxvzXcwcqCJJGTk0LQ8dh4goRLie3XGdcGTgI+II3QppIo86cigjSRVIU8MKoQqNYGm/wQ0qoiCnG2Qs+QmL3pQVZ8yaM2RFpdRoxk3NRt1is2qzUXbplzGDImKU+aSZS5nt7bDM7kqloFKNk2lU2qDSCjnJEUmGzvIDLVUc653Wv/zDmzy52qZqGl54dZM/9+kL817SoVK0JI4jkLWHKCOklMgkpcmFvaN4UhEC2WlDFFJ1Q6rYpeg4FC1JHUAZQxVrqkij4xoV1HTijMUwYSWccC7YZdUd8pi7zaPOiFXlcbMu2Gl83iqXeTtfZqPocDPpMMwDxtOAcuohUoUzEeZO4YRbRXdvWOGkNWqYIoZjmvGEJs/hAIMYjq1o3Rik/O7bu/zVz32ESVHxSy++ySgr6QQPT0G+aAtqF0SjEFWAoyRO3kVOE+qy2vcCqGW+yCBAtGJ0t03T8im6HlWsTOG9LYxotbWpY8U1Xqsg8EuWoimnoxGn/DEX/E1OOWMuuiMk0G8K3qq6bFUd3sxXuJH12MjabE1jksynnHiIROHs3SlMZ0X3cYMzbXBHBWpaIHZHRrDS7EAFC47x8fD/9/V1AP74x0/zuadXKWvNb31za86rOlzqAOpAmMJrKKkj1xwTgwDpufaYeJIQAjE7FjYtnzp0qWdF9yqY/b/gQ+NrmrBBBDWBX9IKchb9hEV3yqo74pQzZkWN6UoFwLBR7NQttqoOO0WL3SJklAekuUeRO4hcoVKJyoy1QWXmSOikDU5aI5PSHAmTFF2Uh/JCeWx3Wi+8usGlUy0urrR4dEmzGHv8+qsb/NR3nZn30g6NogPKg73XHteXiCrCcRVSCNjapsmyua7Rsg9IhVpaRLRj6m5MsRBQxYqsq0zRPYaio6kDTd2pUXFJFOWstccsBAkX423OeX3Oezs86e4QS8G40VyrQq5XC7yanmWnjHlrush2EjNOfdJhALkpujsTs8PyBuYuoT+ozQ5rkiO3d9FJeqg3gI7lTmuYlrz8Zp/PPrMKgJKCH3nqFL/x2iZl3cx5dYdHHZijQBUye8UV1JFDHXvoVohoxcggmPcyLQ+AcD1kHCFaEU0roI5d6lDNiu57u+3ZLivQyLDC90vaQc5CkLDoJSy7Y9bcIStqjCug0Jrt2mWzbrNeddkpY7bzFoMsZJp75JkHuby1w1K5sTU4mcbJGpy0MkfCJEcnqalhHSLHUrR+6/Utqkbz2adXb33us0+vMsoqfvet3Tmu7HCpW41pzYigbAmKlqBoK8qOR90OEJ02otuxx8RjjIxDZKdN04moOgFlx6VoS4qWoJrtsqpYU7dqdFwRRgW9OGUpTFgLRpzzd3nM2+a80+cxx7TnJFrwTrXA28UK7+RL3Ei7bKRthmlAMvWppw7ORJka1nRWdJ+CN25wRzXOMEeOEhiMqCfTQ7/pcyyPhy+8ssFyy+MT53u3PveZS8t4juSFVzf4o48vzW9xh0mrpFYOaIFoBFoJVCHRSiAaH9FopOug8hyd5faoeIwQrmdqWAs9dBRQLoSm6N5WFG3TnlO27mjPaZn2nMU4udWe81iwzZoz5KK7TVtUgOBG7bFVt7mcr3E973Ez67I+7TDOfJKJTzN1UdPZkfBd7TnesLzdnjMc0+xze87dcux2WmXd8Jvf3ORHnjqFkrd3ELHv8OnHl3jh1Q0eltx7x68QYUUTNOao6EMVzI6KoaQOXZrIR4QhwrP+rWODEAjXQUQhOvRpIo8qVLfc7sbxbtzudaDRQY3rV4R+QcfPWPBSFl3jxVpxRizK6taxcKeO2azabJetmRfLHAmz3KWZud1lLmaOd/O2V3hXaYlMCvQ0RafpgZlHP4xjt9P68pt9xlnFj95xNNzjs8+s8jf++dd5Y3PCR1bbc1jd4bLYSZjmHlOg1C6NIxGNoHFBC3N3yPUkouwhfQ+pNc00sVaIo4wQqHbbNEB3W5SLEVXskPcUZSQoW4KyY8yjZaeGsMaPC5Y7Uzp+xsXWNmveiEf9bT7q3aArS9pScaPS3KjbfDM/w0bZ4UqyzFbaYjcJGY1DmtRBjRROInASgTfUOCkEgxp3XOOOC9TOGD1JqLe3D9zW8EEcu53W51/dwHckn7m0/B1f+9GnjJC98JC447t+RivIcYMKHTSzV16oQ2auaEkVKeOYjwJkHBkrxOx2t+WIIRXSn7nd45Am9qlihyoyETN1KGb/tmaHRbDndjc7rEU/YcmdsuoOWVEjurIkENCva7aaiPWqx0bZYatosZtFjLKAJPNoUgeRzYruqUClGD9WOrM2JCVymqMnCTpJ5ipYcMx2WlprXnh1g+efWCbyvnPpa92AP3K2ywuvbPC/+qEn5rDCw2UlnOCqmqJSDGtJLTVV7gICUQuKErSQqNRFC4FTN4i8QGiNzu1u66ghAx/h++hOi6YdULY9yraiCm73E1aRpoq1ac+JCuKwYCFKOR2OWPSmPOpvc97d4bwzYlEpSt3wRhVztVziWrHIzazLTh7TTyPGiU+RGPOoSgXOVMx6CjXetMFJZubRYYoYJ9SD4dyOhHdyrETr9Y0JV/vpBwrSZ59e5e/8+utsjXNW2v4hru7weSzaYctp0WhB00im0qMsJVpJ0HJWnAdVOmhHIrRG1TXK96g2t+0x8aggFcJ1kN0OOg6pFyLTntN1yNvydtF9rz2nVaHCim7rdnvOI2GfVXdoYmbUlFUl2agbBo3HG/ka14pFbuRdrk17jAqf4eR2e447fo/2nEGFk1So3cS054zGB5LYcD8cq+Ph3rHvR5869b6P+ewzp9AafvO1zcNa1txYdscsuVN6XkrsFwRBCX5DE5jjQ3WHY94U5h105EMYIFzHHhOPCEKZYyFhgI586tAxhfe9gvu3ebEaVFDj+yUtP6frpyx6U5bdsSm8y4S20CgEg8bcKezXMTtlzG4RMSp8prlHlTuQS2Qmbnuxsju9WDXqDrd7kx1sP+G9cKx2Wp9/ZYPvOtflVOf9DZPPnO5wphvwb17Z4N//vvOHuLrD5yn/JotqghSaBsGOG1NVitx1qfAAiXYEopY0jgbhgo5wPAdV1ejxmHownPeP8VAjXA/ZaSG6HeqlNlXski26lJGkbEHRnfUUdhqaqEa1KjrthE6Qc761y+lgxDlvl6f8G6ypCR9xAzbrhG+Wktfy02xXHV6frrGRtdlJI/qjmDJzYOTiTk0/oTcy5lF/qE3RfVLh9KeISUqztU1zSO05d8ux2WltjjP+4Org2wyl74UQgs8+s8qLl7fIyqPziz4I1tSINWfIqjti2Zuy4CfEQYEXVOiwNrutPcd8CGUoqCJFHbvGMR/PHPPWCnH4CHE7eTQybvcqdm8V3c2/mRGsKpj1E4Y1flDQCXJ6fsqKN2HVHbHqDlhTE7qyZrdJ2Woc1usOG1WXjbLDTh4xyELGmU+ZOejUQSXSFN1nIX5Oqk0NKzFRyWIyszVUFeij1WVybHZav/GqOe7tte58EJ99epV/+O/e5ncub7+nNeKkcM6p8MWQRPuM6wApGkZRgBCauhaUpQChqDKBKc6DKhUIkGmAAkRVIfqDI1FgfagQEuE4iG7b1LHaAWXLMRlpsaCKbhfe67BBRBVBVNCJMpbDCUv+lLP+gHPeDo84fc454OJxuWq4WvW4WixxPV9gO2+xnbYYpAFJ4qMTB5nKmbUB43ifGtFyJxVqVCAnKXo4oskPNhfrfjk2ovXCq5uc7YU8tfbh/qvnLi7S8h1eeHXzRIsWQFs2XHS3KbQiUjnTysdTNVoLdhtJrTRl6aLVTLgaaByFLH20I03qaV2bukWSzPvHeWhQ3Q4iCml6LaqWR9l1ybvGPFq0jbWhjDVVpwa/ptXO6IQZK+GUR6M+p7wxT/jrPOLscs6pGDeacaO5Up7iarHEtWKBa0mP3TwyyaOJRzN1cEYKlXOr6G7SR2tU2uDupsjhFD2emJkD9dE8qRyL42Fa1Lx4eYvPPbOKuIujjO8ofvAjK/z6qxs0zdEoHh4E2awwuigrltSEFWfMojel5yW0/BzfL5FBbRzzvnHMf1thPnBoIs845kN7TDwUhLjVorOX7V5HjmmAvjPbPdA0gQa/xvFrYr+g7eUszLxYC86UU2pMW5YEQjFuJP0mYLPqsF216Bcxw8Jku+eZS5MpRC6Rxe1sd1N8b24lj4okR6eZeavrI1N4fzfHYqf1O5e3ycrmQ+tZd/LZZ07xP37tJl+7PuS77uhRPEl8s+yyoqY84UguOkN6MmXa+LTU7QbWgRMyrAWV44BQCC1oXIGsFI0r0DLArzUi8JBFaaJy7VHxwJBhiFzo0Sx1aCLPZLvHirwtTNHdn2W7Rw06rok6GZFfcqY1ZCWYcD7Y5cngJmvOgCfdnEzDRl1xpVpmvezxRrrKzazLVtpiY9wiSz2qkWfGfSUCb2h6Cv1hgzdpcCY1Xj81wyi2d80O64in3h4L0Xrh1Q3avsOzFxbv+nt++EnTm/j5VzZOrGitVz1qLenJXQIBK6pgzRnQaMHIC5j4PloLksyjaAR1KagKCVpQRoCWyFLhpB4KkHGEFoK6Ko/sq+xxRkYRIo5NDSv2b2W7l3tu9+CObPegwQ1L4qC4le2+5o845Y5YcwYsyowSzVgLtuqQ6+Ui22WbjbxNP4/YzULSxKfKFTKTOKlxuzvpLMhvVnh3kgoxzRBJRpOm6PLo1bDezZEXrabRvPDqJj/wkRU85+5Ps73I43sfXeCFVzf4a3/syQNc4fy4ViySOS6xzHnGzViWPn1nFyUaksZnUhtz7TjymABFbYQLASoXgEbWEpW7aAFup4UARJoZI6EVrv1DKkS7Be2YphOa9pxYUcaScq/oHmtqX6OjGhVVhGHBYpjQ81POBANOuwMecXc47yREQpBoTb82QX43ix5bRZvtrEU/jZikPlXqIDKFSgQqEbiJcbu7qcad1DiTEjXKEePpzIt1PFJAjrxoffXagO1Jzufu4q7hu/ncM6v8H//HV0/seLEr6TJDN0SJBldcZ0WmPOpIIrlDrSWlVoSqJKtdXNWwCxSNmBXixaw4D6JWNI5AVjHKUSig3tm1x8R9QsaxqRkudmlaAUXXo+g5lKGk6MzGfcWasmUSG7x2QRzmLMUJ5+MBi96Ux/0Nzrq7POYMkUCiNVfKDu+Ui1wrlngrXWInj9mYtJgkAUXiIsd7I+tn7TnT2UDVpMHbzVHjDDGa0gyGxot1TDjyhfgXXt1AScEPPblyz9+7d+fwpI4X2y0iBlXEVtU2zufGwxWSSGhOqTHLzphld0LPS2n7OYFfIoIa7d+OsrldmBfUoSnME4WIwLeO+f1AKpOLFd4e91UH73K7+9D4oP07st39gq6X0nNN8ugpZ8ySTGhLQQOMG8Vm3Wa76hi3ex7dynYvcwdydcvt/mHZ7kfNPPphHPmd1guvbPJ9jy3Qi7x7/t4LyzGPr8QndrzY9UmXSelTa4FCM3ZDInGdtoQn3ZyaG7RVSta4hKpEotFA6vhUtY9WEi3FLM5GImuXxpV4QqCaBhEE1FsP17CQ/WTP7c5ijyYOTLZ7y0zPybsmLrnomMjsOm5wOgVBWLDWHrMUTDkbDrgUbsyK7jsEAiSCq5VJbHg9O82NvMd62ubGpMM080hHgZmeM5V4I3OX0Btq3KnGm9R4uwVqmiN3BjSTKc14PO9f0z1zpEXraj/hmxtj/tM//vR9X+Ozz6zy979wMseLjTOfRgsip+Cm00WKhjVnSMOUQDX0ZA7OLhtuj1IrprXHpPTQWjDJHGoNopZUufFwlalANBJZuog0QgqBnEboojiSJsOjjMl2Nzn9TRxQx2bcVxnermHtWRv2zKNhlBP7BQtBwrI/4ZQ34qzbZ0WNCQSUQL+WrFc91qsu63mX7Tymn8VMs1m2eyaRe9nud7jd3aTBSWrUOEcmGc1kis6O9l3C9+NIi9Zeg/T91LP2+NzTq/y3v3WF3/rm1omb1JMmPk0j2XEqIsfUn1Yc88q5KEcsyppIJKy7Jjc/bxwmpY8A8syl1FBXgqowxfkyE4BEVA4y80GCmMQghBWte0EII1hxjG5FVG2Ti1XGctZKZdp09nZYIqzxw9K05wQpa8GI096Q826fs2rIoiqJpGKjbrhedbg6u1O4mbfYyWKGaUCWmFwsNVW33O7G6Q7epDH9hOMCOUnQ0/RY7rD2OPKidelUi0eX4vu+xicfWWAx9njhBI4Xq0YedSnZAqTQZLVLS+VMPR8pGh5zJrSl4JK3SSzNq2rVKDadFnnlMFYNuYCycUxjdWWibEAiag/Xk7h51wQHak2TpvaO4odwZ7Z7E4dUCyFFzzRA552ZYMVmqGoTNsh2SRCaYRRnWkOW/SmPB1uccXd5zN1mVZW4QvBmKbleL/BWscKVdIXtvMWNSZdhGpgXr7GLSiXu+LZgeSONmzZ4gxJnbIZRNDu7x3aHtceRLcQP05KXrvTvqtfwg9gbL/abJ3C8mCgEOlMUucM49xnmoRkHVbbZqjpMG0mpNT1ZsagmrDgjem7CwswxH/olyn9vx3wdSqrQoYl8dOgjwgDhnKzj9b6zN1A1CNBRQBPNxn2F7+V2b9B+g+dXBF5p3O5eMst2H7GkJizKAiUEpdZszgaqbpdt+kXMqAzMuK/coc6MF0sWexEzt7Pd1SzbXSQ5JOmJMA8f2Z3We40Ju18++/Qq//3vXePLb/X5/se/M6b5uOKOJLUvKYEBkJcOvlMxrYw/S4mGM84uH3M1gSjwWKfQirZaIK1dfFUhhGaoI2rHoayNDUILU9tyXYEsAxxXoqRE1jXNlGP/P/2BIBUy8JG9LroV3sp2LzqKvGvuzhYdcySsWg10ZtNz2lMWgpSz0YAnok1WnSGXvA1WVcGqCnm9LNiqY76Zn+Fm0eVa1uPapMc490y2e+IgpwpvJJH5XtEdvGmDv1uiktJku4/G1MPRiTjmH1nReq8xYffLrfFir2yeKNFyEoFoNNpVlJ4DQtNPIySa2MnpOqYBek2t4wKrquTsrL617bdotCSvHbLCJQfqTCK0RNRiZj4Fp6UQjYsoA1SrhRSSemCNp9/GTLBEu4Vu3c52v2UcDe+oYYUaHdZ4s2z3hSBlJZiYmBlnyJoz5IwyLwobdcp63b2V7b6Rd9jOWowy/3a2e6pu9xLeynY3Xixneke2e5Yf2Qboe+VIitbemLAf++jat40Ju1/uHC/2n/3k03fVdH0ccFJAz6bvuA6lFowCHyk0gVPScYzDeU0NWVUpF9wWg8YUYLe9Do0WZLVDWrqMhSYJHSptImyqYnZHMZOIxjEzFLMIIQRiPD7SDbWHjXAdYx5tx7dysYxg3XGncDbyqwmN270VZbT9gpVgwil/zDmvzyNunxU15ZSK2GlSrlYe18sFNqqu6SfMW+xmIdPU5GKJROHMst2dhNt3CacNzqRETjLExIys12V1Yv69jqRo7Y0Je5C7hu/ms8+s8pv//Otc3pxw6YSMF/OGGjkTF4SkqgRTN6CqjCnUEQ1pbdp8pu4OMKQnoS2HTJt1fFkihaZqFJ6qqSpF4biUwpmlnQpkJWZ+LhdRxSjPQZUlzXhCM53O9ec/KqiVZXToUy/EFAs+VaRmPixB2Z55sQJN06lwgop2K2W1NWHRT3g82uKUO+Jxb4Mn3BGxkLxVJazXEW+VK7yenaZfxlyb9tjNQkbTgGLkI/K9oruZAu0NjXnU361wJyVqlMH2gCbL0EVxYgQLjmghfm9M2PPvMSbsftkbL/b5E+SOV7lGZXo2WNO4n5vMochdJrnHoAjpl7FxzFcd+o0x6AZCcEqZQZ4LTsKCn9DxMwK/xPFnjnmfWWrmzDEfSKpQUe855sMA4RzJ17xDR0cm270JHergXQNVfWh8Pct2r/AD43bveSbbfcGZcsoZsaKmBLMTwE7js1V32KraDMqI/izbPZm53cWHZLvLpDBu9ywzdwpPkGDBEdxp7Y0J+/T7jAm7X9a6AR8/d7LGiwWDmiqUmNeemWXBUdS1YESEEJq0Mm74pPapkSjvBsuq5KKbEckNFJpSK2JVkNcOfVUzFiGlnrnkK5MzjwBZOzSeRFZtpKOQQtLs7p6I4u6DUC/G1KFjpud0lIm27sxmT7Ya6naNDCs6rZRumHE6GvFYtMMpb8RT/g3OqDGPOg7bTUW/dngtP83NcoGr2SJvTRcZZCG744gi8WDq4IxNVLI7NtYGd6rxhzXOtMLZTUw/4XB0YgfzHjnR2hsT9r/8wf0Xlh996mSNF3OmNTTguoLGA4S5ra6FpFYOU9f8jJt+C4C2ylhSE2rGnHdKYlFx1tllx2uhRMN2ENNoQVUrqkLR4Bi3fGMEUeUSLcBJXJwmQFU1Ms9p0uyhvqNYxS51sFfDgnqv8B6YXCwZmR1WN8xYCqYs+xNOe0NW3QFn1JhYNkx0yXrts1W32ai6bBbGPLo7y3YvUhedzuYTJqbw7iZ69mZqWGpaIsYJemoK7ydRsOAIitatMWFPv/+YsPvls8+c4r984XV+47UN/oPve2Tfr3/YOJMCUbs0nqD2FKCp/Zm73ZEUnvFV7QTGnBurnK4ydxTPqB0iAWeclJ2mj6Jh02/TaEFZK/LCoWB2R7EB0QjK3Ax/dWOF0B6iapBphATqh1i0ypYy1pPImEf3iu71bBhFEBbEQcFSMOVUMGHNG3HG3WXNGbCmoEQwaDCtOVWP9bzLRt4xTdBpQJZ66NRku985AdqZCZaJmSkQ0ww9Hp/4F5EjJ1p7Y8JWP2BM2P2yN17shVc3T4Roqf4EEQV7p0PK0hTPRW3acUrpkleSLadFUZvivBSacRMQyZwVmXDGEVx0+rRlRtL4prFaaMpGMnF9JpWkVAot5ayxGpN66gi0FHhVg/A9ZFU9tD2KeUdRB1C0TeG9Ck22uwgrwlbOUithMUh4NOqz5g+56G3xlLdBV9ZMNfQbh6tVj8v5GhtlhzenS/SzmH4SMh0H6FThDPfuFII3Mu05waDGmdY4wxy5O0ZPE+rR5MhNz9lvjlQh/m7HhN0ve+PFvvDGCRkvlqTIJEMlFSptbhVkVWYK8zITkEvyzGWS+QyLkH5x2zE/1i6lbohlw4pMWHFGLDpTFr2EjpcTeiVqL8rmlmOe24X5QNFEnnHM+/5DW5jf+53ccrv7e9nuFZFf0vEzOl7Kkjs1A1WdEZGo8YQw2e51NHO735nt7pFlLnqW7a5ykPme0/2ObPekRCY5OjHZ7jQn34pypP4v25sK/aCtOx/ESRovVm1uo1oxCvAEyNKjdl1EBWBqW7ISlMpl0pg7U2pmg4hkwbTxaPQGT3sFq8oh0RsEssQVNWWjCJ2SqpZMZEApPOOYd5jZIEBLhax9HE/hVDVi4CCq6qHbbeVdU1MsO5qq3aADk+0eBwWn4gmPxX1WvPGtbPeLM9PvtNFcqZa4Xi7yTr7ElWSZQR6yNYlNVPLURY0UKhe3ewons6J72uDuZshJCoPxQ3VD5EiJ1udfufsxYffL7fFiG8detNCN8eCMp6jA2Bm8UAIK7UDjGQ9XHSgaAanyGHghAOteByUaYlnQlessqYJVpanZJWtc+r6pgw2LgLqRNI2kKSSVMLfamZlQnVyhBcgkRNUNsqmpd4cntgj8XtQh1L4ZjKvD2nixwpyun7ESTFjzh5xyR5x1dunKHCUEO7Vg2PhcLxe5WfS4mXXZyWLGuW8EK3UQezWsbGYevaOGpdIKOUkR4ynNdHpi3O53w5ERrb0xYX/y+x45UMf67fFimzSNRu6D435uaE1TlMjxGBn4KMCJXbQwdoXaMz9bnQq0UJSOy9gzdxS3/RZSaCJZsKQmwISLrkdDTuIM2XI7AOz4EXUjqRvJNFc0KKpIz0RLzOJsFE7LQzQNsq4Rkyk6f3j+iKrQHAl1YAQrjEzEzKKfcMofc9odsOYOWFXprSC/fhOwXvW4WfTYKDrs5DHDLGCaeTPBUqhkNpAi2xMsjTudteckxUywkoduXuWREa29MWEHcdfw3eyNF/vD68N96W2cK01NkzcwGCKKAtdVyMIHPLQQyApTnK+g0g6JCChL44DPaodGC1xRMXCHBGIdV8CT7ohC36CrpqS1S6AqlGxoGkHuupS1dzv1tJbUnkbULtoROFKi6sYUhXd35/3bORSqjklscFsF3XZKJ8g4Hw9Y9Udc8Le45K+zMotKHjeaK1XIa/kZNsou30qW2c5abCcxg1FEmTnIkWN8WBOBNwKVaoJhgzOtcUclqj9FTFOT418dn2z3/eLIiNbemLDnLiwd+HPtjRf79Vc3jr9oAWiNLgrTF5jkSEfipAonFGhpirhamjt/TaCopGaSe7gqpOXmbFdtXFHTdzx6smBRSpbUlBLFkjslb1ySymPoBUa4fIe6EsjSRNmIBmrfFOZl5CJDH6H1Q9OjqP0G4dczt3tOd+Z2X3ZN0X1JprRlQ6lhrB226g7bVZudMmZQRExmbvdqL9s9Nw3rptNh7+bKLHk0vZ3tfpJtDR/EkRCtvTFhP/jkvY0Ju1/2xot9/pUN/uq/dzLGi+k8py4KlOugZrPrtApnOy1TkDe2dkVdCUZuSFmr2U6rIfE9Yplz3t0hEglnVEFPbjMNfAJZIkVD0Sh23ZDtSlFJKKVC1LPhrzU0Spm007qF8lxknpvBCSf8+OK2c4KgZClOOBsPWfYnPB5sctbtc9Hps6I0rlB8vfBZr3pcKVa4ki6zk8esT9pMUp9s6sHYwUlNtruTzoruIz2bnlOYqOThhGZ759iM+zoIjoRoPciYsPvlRI4X0xo9TUzcr+vgeA40mipwQZhjoinOSyrfJQWGqmbLa9EgWHYnSNEQiIpVVRAJOOvs0mhJ1rgMAlPEn0YeCVA3GMe8EJSRgAZEo1CZMbU6qbmhIur6xDXt3kkryom8kqVgyqo/Ys0f8oi7w4oas6I0idZMa831aoEb5QLX8wW2sha7uZlPmKcuOnVwEpPtvmccdZLZfMJ0lu0+SdDj8UNzl/D9OBI+rVtjwj5y8PWsPU7qeLEmy9CTKWKSoKY5TlKZO06zPwInBZUKRCapU4dp6tNPI3bymJtFlxvlAut1hwZwhWBNJZxxdznj7bLqj1kOJnQCs7MQYU0dNlSBNnfQQpMdVbYUVezStAJEHCF9H8SR+F/tQOgEOYthYtzu/tDkYqkRK6qgKwPGjeJG3eZGucBm0WE7b9GfZbvnqUuTOmZs/d4E6FQbe0OicRIzVFVOU/RkSj2aPPSidSR2WntjwrrR4cX5XliOeeJUi8+/unHixos1eY7e2kZJicgjfCWRpQNamRNiKdCOibIpGsGO1OSVgycr0tqj0RKXmlNqwhOuxhUTPGoK7dBSOUXj4KqaHdkwaQSVqxCNMnUzJczwV1cg6ghHKZRSSK3RaXoi/+DOtQb03JSL4RZP+OusOUM+4goSLXi9LHijXOF6ucDlZJXtImZ92mFrHJNnnsl2TyTuROKOTCaWPzTWBm9Y4fYT5CSj2e6f6H7Ce2HuL3/v7JgxYZ97Zu3Qn/uzT6/y0pU+o+yE3YHRGl1V6CRFJBkqq3DS2hR0Z65qE2siEIWkyB2S3J055iO2yxZbdYedJqLUDS6wolKW1IRld8yil9D1MmK/wPEr8Bsa32RG7Q0gNQNgFU3k0kQBIggQ/vFvUn8vFr0pS96EZWfEKTVmSeY0NCRas1XHbFYddquYQRkyyI3bPc9d6txku6tc3HK734qaSRtUVpls9zQ7UcmjD8rcd1p7DdKfPQSrw7v57NOn+H/81rdO5HgxgGY8RuQ50ndxS1OP0spFlnK2E7rdo5hWkk3VUM56FH1ZMXYD2jLjjMp53G0x1Zu0ZUrSeDiiRoqGqlZMnIasEmip0NI45mvX2C2Q4EqBW5qpPs1eq8kJ4mK4xaKa8BFvgyfcjLb0uVxWrNddvlWs8q3sFNt5i+tTMz1nPAmpR94syE+aovsYM7I+1fiD0hwJhwnsDGjSh/dO4XtxJETrI6sPNibsfjnJ48UA88pcGBOiABxX4foKNNSB6bLWamZE1ZD6HlJoXFXTdjMaBFedJWAHT0zpSY1yRmy5AxotqbRiXAYIoSkLM/y10qbtRAsocoFoFDSg0gApBSrp0CQJOj/eY6zu5Iy7yyk15oyTUmrYqHOuVwusV12uFwvczDr085hBEjJNjXl0b6CqcbrfNo86SWPmE44zxGhCk87G1ltuMVfRGiYlL73Z53/xAxfn8vx748X+zTfWKesGV839tLy/7B0TxxMzLdp1cAIHcKhCgZbQuAKVznZcgctUaFynZsczLyKn3C6uqIjFDmeUJlINZ51dSq1IGo9+YO68TgOPtBY0jaDKZu1DmaCsQTSSKvVwANVpIeua+gSJ1llnl0WZsSw9NuqCYeOyXnW5WS6wVbSNYGUhSeZRZe7tYRSzwruTatxUm8SGpEZOc8Q0NZHWRXnidqYPylxF69++vknd6ANtkP4wTup4sTtp0hSaBlE3uFIic4/GCZGVBA0IQVUKtHIoK8FAC5RsmJQmqiZpPErtIL112lLzhDsiELcbq2NVUNaKvmpIVEBVm92bqE1hvlFylvsl8WqNkhLlOiemR/FJN6XUmu2m4ErVZavq8Hp2mo28w/Wky8akRZL5Jts9lTiTWb57Ohv5lTR44wa/nyOTwmS7J7P2nBNqE3kQ5rq1eOHVTTMm7FxvbmvYGy/2+VdOlvXh29jbcWUZIsmQWTkrzDe3BnuqHGRmCvNV7jDNPSaFPyvMt9mq2vSbgEQLAiFoy4IVNWLZnbDoTen4pjDv+SXab8wA2FuF+VnOfChpIhcdBYgwNJOrpZr3b+eBUQhqoF+77MyGqvbLmN0iZJQHpLlHMct2V9nsWJjddrs7qcZJK8Se2z3L0IUd0/Z+zG2nVVQN//abm/z4x9bm2rR853ixn//JZ07MeLF3Y46JY4TjIMsK11WI2tzN01IhKmF6FLWk0g5TZab6+E6PojE9ipHMmTpDev6ARVkTubtk2iWSOWltivNKaDYrSem4lJWLFhizayloHIUszeccJRBliUxSmvF4vr+cB2SsG/q1y+vlKS5na2yWbd6ZLjDIQvqTiHTsQ6ZM0X02PccdmSOhPzQ+LGeUIXdH6On02P8+Dpq5idaX3zJjwg4q8O9eMOPFtnhjc8JHTsh4sfejSRJkXSNDHwfQSlB7clacN7UoLSSV65Jpwe4sysaTFQvOEqV2OKUmdGVJWwrWnCFKNPQr46pvtCQpXKYCikJSNwqhMcV5qXFycyNAaHDSFlLK27fzj+lR8WoVsVO3uFYscSPvsV3E7KQRk8y43UXiIGfzCVVmhlF4U2NrcMclapIjRwl6PDF3Vy0fyNxE6/Ov7P+YsPvlR59a5W/w9dmdzJMtWjrPqcsKJ46QUqI8BzcwGVxVahqrtSNoMkkjFEngoaTGVy02PdNDeN3t4opdVoRgReYoNOvukFw75LVzq92nzB3q2phNqxBAUIYgaomsHVQcgNaIiQ9Zjj6morVe9dipW2yUHbaLmN3MCFaWejSJcysTy0TMzPLdU5PaoKazbPfJ9MRnu+8XcxEtrTW//toGz+/zmLD75SSOF/tAmpp6u48sSxytQbSQpUvjOMjKDLHQUlCVDrnyqStF3Qgc2TD2AwJZknkemd7mCVexqGoyfYNAlviiIm8c+m5E1UhTmFcaGueWN0wriVYCUYU4gYNTN+jBiGbcHEvH/Dez0+xWEW8ni9yYdBlnPtNRgE4d1FiZonsG/tC0UnmTBr9fopICuTMy7TnD0bHdaR42cynE740JO0rJoZ99epXfvzpga3xybsV/ELoq0bkZ6qmSEpVUd+TLc2sQqMgVVa5Ic49x4bNbhMYxX7XZqWOSpqbRmhWVzgrzYxa9KT0vJfYLXK9C+A1NcNsxf2uYaSipA8cU5qPQOOaPYU1xp4xNtnseMs09svzbs91vdyHc4XZPTZCf3vOsWcG6a+ayzZmnC/79+OzTq/ztz7/Ob762yb//fefnvZyDR2szzLMskY7CqSI8TyFqhdASrYxlQTuSqnHIgG0npqgVkVNSakWNJJAla2rKo46Hyy5KNCSNZ6wSlTGrDmRDMhskKyqJFgKkMFN93NmOSwhT38rzY7fbupouMMxDtqYx42lAnTioiboV5OeOTfOzP65xJjXesED1J5CkVP1de5fwHpmLaO2NCTt1AGPC7penT7c52wv5/KsbD4doAeiGpigRkymybnBDF/DQUpg0Um2ibDSSWjqknk/TCLZcM/zVFTU9lVBqRVsMcQU85gwZuDEKzSAMZ8V5QZG7VBLqXCA0gJg55yUydxGN+cOVaRedZjTT6dx+LffKZtJmWnhMEp966hov1nRmHp2CN5kV3Uc1TlIiRyl6NEYnqRWs++DQRWtvTNhf/dxHDvupPxAhBJ99+hS/+rtXycqawD3+/qEPRWvQ9a0/HjmNcJQJ9XMC04qjQtPqox1B7StyPIZ5gJINoSrZcLsoGgZqTFs2LErJmjOgRrDhdSgah6JWTEKfDKhCaXZzNVShAA1VZFIoRNXgtGITY3OMjJWjzCcrXON2zyRqNpDCyZh5sBozBTopkZPCuN0n04c6yO9BOHTR+o1XD35M2P3y2WdW+QcnZLzYvdCkKaIokI6DKlv4dYyWAlWaY6KsTBZ8Lh3qUtKXMUWlqBuJK2vGtSnOn3f6nFEF552EtiwotUMgSzxZUTaKkRvQbwSVZHb8nDVuNxItnZmLvkH6Hqqqjk2P4mgcUecKMXZwx2aH5c1iZrxxg79b4SQVameCmCTU/V0Timi5Lw5dtF54dePAx4TdL89dWDo548XuBa3RtdlxScdBei5O6oIEJ5NopU2PYi5AGMd86niM3Ip+EeGIms2qQyRy2rKkLTRtUbGiRoydgMT12PFjGi2Y+D55JWlqQR1oRGOK8rKUyBrq0DUN1nFkehTL6sgXqetMQSFntoZvz3Y3O60alczc7mmKLqtjs4s8ihyqaJkxYdv8B997/kg6zz1H8oMfWeGFVzf5xeM+Xuxe0do4sZsGCWaqT+lRexIxG/TaKDPVp3QdMg27GnxVk1UuoTLF+QbJx7wd2lLwuGsK866oSRoPT9YUtWJXaDKhqUoP7TCLyDFDOGTh4boSUbaRgNSaZjI50n/kYuKYgaoTgTuZBfmNZ9nugwJnkJoj4U7f3GQ44iJ81DlU0dobEzaPwL+75USNF7sPdJ7T7A5MX2DT4AUKoRVaKmqPWXFeUuNQCOh7IVUj6XgdzFTEhp5MWFQZK1KSqQm1K+n7Ma6omVQ+jTYimBZqJlQSoU3WvJNLtHKQRWDmKAKU5ZFOO3CS2wNV3eks231c4yYVzjhHjKbGPFqUoJt5L/fYc6iitTcm7NkLi4f5tPfEiRsvdo/o2Vh7kaRIIVCxT+MIHFfgBBJmE6a1lNSOIstcBNAPYnxZ48uKdbeLFA3nlGZRFtTOmHVnCJghsXnlUNWKPHRptEMdSEQFooYyFKAlVWjmN8q6QUzMSLKjOgBWpbMj4SxmxhwLa9S0vO12T5IjK7rHjUMTrcMeE3a/nMTxYvdD0x8g0wzHUcgqQtQeWrqowvQrykpQNopS+NSl4qZqU9aKvFEEsmTgRnhcZVXBk64i0zfpqYS8cfFkhe9UVLVk6niUjRFB7RjHfOOB0C5aCVxX4tQ1apJQbWweyWOiNzI1LG+s8YcN7nSW7T5O0bsDmqlNHt1PDk095jEm7H753DOrvLY+5mr/ZM/r+yB0XdPkuYmymU31MVE2GpVyy+ktckmTOaS5x6jwGexlzFdtNusWYy0odU1PFpxSY5adMUvulJ6XEPuFmeoT1MYx7/OdjvnQQUc+hAEyDBHO/Nu+3o0punN7h5VWyEl2O9v9IZwCfZAc2v8B/98/uIFzyGPC7pcffdrMRPw3r2zw558/WZN67pqmRheNqW/VNQ7geQpZKmr3tkG0cSR1LchcH60FWgsipyCtPdoyo0FS613OOJKezBg1m0ihkUKTVB6uiqkqZTxc0ly/me3kwEz4kUWIUgpVVjS7gyPnmPfG5kjoDSvcYY4cZ+jdoakPWi/WvnMoorU7LfjVL1/lpz9x9lDHhN0vF5ZjPnG+xy+9+CZ/5rlHHg6j6XuhtWmrmSqEUji+i6jcWb68RM8GwIoGSs8hB4bAhtumqBUdZ+nWpQKxQyDgMXdw63OjKsCRDVnloLUgFy5VLtHKDMcQDYBE5Y45khYtpNYg5ZG6o+hNmpnjvUAOE+PFShJjbbDsO4dyPPx/felt0rLmP55TFvz98L/9sSe5Pkj5h//urXkvZa7oqjJzFKcJcpqh0tIMX8jM5BhTgBbIVKAzRZE7jDKfYRGyU8ZsV202qzZj7VACixKW1JRTzogld8qyN6Xj5URBjuNXNGFD7ZujYhUKqtC46KtIUcceOgqQcYRQR+eFxJnWqKQyMTNJhp7aJuiD5J53Wn/tn3z1np/khVc3+OEnV3jyCBpK34/vf3yZH3pyhf/qNy7z+sZk36//f/kT37Xv1zwodJ5TFwVKSVQR4ymJLD1k7RjLQiRuNVjXtWAoNUXl4IiGonHIGtc0VjtDnnGnrKqCttyg0Iqus0CpJa6qcVXDdiOpHYeyceBWcV7iegLRBLhK4ngucjbXsUnmX3f0djNkWiL6Q5rBkOYYuPiPM/csWv/uWzv3/CS90OU/+dFL9/x98+Y//eNP85/8d39wXz/ziUNrdJqBkMgkQjkS5UmcUICEKhdoB9OjmJlUiFHg4zsxnqzYcHsoNEM1wRXQFpolNaHUDptuh6nnU9aKse+Ta0GTK+pKGyEMzFGxCiWycJCVhwpN0CBZbrxPczwqyqRApDk6mU3QPiLH1pPKPYvW7/z1HzmIdRxJnjjV5l/95c/MexlHhmY6ReQ5yvcQjTFJNp6PrOWsOC9ACxpXUdWCgRvd+t6Ok5Fph7ZMecwdsKxcHnOGxKJgGpiseikapqXHSDVMSpNVr4U0+fVKmOlBOGgJft5COgqZZnOPsxG7I3SWUY9Gc1vDw8TRu39sOdLoujZ3FLXGEcLcUawUtWdacQDjmK8FuesxNLcZiZwF8sahq1IAaj1kTUl8kTJqtm655Celj6tqytK0CtXCoSqNqbUojfEU4aCyAEdKZFnRDEfo6fzMm81giK6t0/2wsKJluTe0pkkzhOchfA+ZByglULlGudB4s5wsCVUhKQqHxPGYlD6erNitYjoypSMzHnVqfAE9lbDoTBg3AV0vo2gcAq+k9BV1Jc0EbB8aH+oc6lJQBwpRu4jQR6QuIhVz65CxtobDRWh7/rZYLMeIo9tPY7FYLO+BFS2LxXKssKJlsViOFVa0LBbLseKe7x4KIb4OHPXbJcvA9rwX8SEEWuuPzXsRFstx434sD5nW+nv3fSX7iBDid4/DGue9BovlOGKPhxaL5VhhRctisRwr7ke0/u6+r2L/sWu0WE4o1hFvsViOFfZ4aLFYjhX3JFpCiD8jhPhDIcTXhBBfFEIcuSQ7IcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rsliOE/d0PBRCfD/wqtZ6Vwjx48AvaK2fO7DV3SNCCAW8DnwOuAZ8GfhTWutX5rqwOxBCnAZOa62/IoRoA78H/MxRWqPFcpS5p52W1vqLWuvd2YdfAs7t/5IeiGeBy1rrK1rrAvgV4KfnvKZvQ2t9U2v9ldn7Y+BV4Ox8V2WxHB8epKb154F/vV8L2SfOAlfv+PgaR1gQhBCPAZ8EXprzUiyWY8N9hQAKIX4YI1rP7+9yHh6EEC3gnwJ/RWt9vzm99tbv/iAe5Js/J/+E/Xe4E6mQnotoxbDQRccB+XJI1VLkHUXRFlQRFD1NHWjqdo3TKvGDkld+5hc+9N/iQ0VLCPEXgf9o9uFPYPr6/h7w41rrozbx4Tpw/o6Pz80+d6QQQrgYwfpHWut/Nu/1WCz7iQx8RBTC0gL1QkQVu+SLLmUoKDqCsgVVpCk7DTpocNs5vXZK7BV3df0PFS2t9X8N/NcAQohHgH8G/JzW+vUH+cEOiC8Dl4QQFzBi9SeBPz3fJX07QggB/H3MDY2/Pe/1WCz7hXA9hOciFxfQrZBqIaLoelSxJO8KqlBQtKFsa+pQQ6fECyoW2glr8ZiOl97V89xrTevngSXgvxFC/MFRa/rVWlfAXwJ+DVPg/sda62/Md1XfwaeBnwN+ZPY7/AMhxE8c5BNe3hzzv/7VPyAr7fBQywEhBMJzEWGAbkc0rYCq5VLFkjKSlJE5EtaRpo4adFjjhSVxmNMLUlaCCav++K6e6p5qWlrr/xD4D+/nZzostNb/CvhX817H+6G1fpEHrKHcK3/zf3iFL7yxzU994gw//OSpw3xqy0OAcBxkK0b0ujStiHIloooUeXevfiXIe1AHmqpTI1slQViy2h2z4Cc8Eu/yeLDFonN3Q5GtI/6E88XL23zhDRMt9tKV/pxXYzlpCNdDhCGi3aZpRdRtnzJ2KGM1210JqhDq0OywRFgTRAWtMGcpmLISTDjtDTnj7nLW2f3wJ8SOEDvRaK35P/3aNzndDVhu+bz85lG7b2I57shWjIgjmsU2VS+gjB3ynjT1q72ie2h2WCKsiNo5i3HCYpBwId7hlDfikr/BJXeLRXl35QsrWieYt3cS/uDqgJ//yWfYmuT8P3/7CklREXn2n93yYMggQIQhLC/QxAHFYkjRdahCQd6VVCGUHU3Z0jRhg9vNCcKCldaU09GIVX/EU+FN1twBl9wdAnH3rhF7PDzBvPymOQ5+5tIyz11YpGo0v//OYL6Lshx/pDJHwjiiaYVULY9qdhy8VXAPoQqhmR0Jw7CgE5gj4ao/Ys0fctbdZU2NWJRGiLK71C37knuC+dKbOyzGHk+carHWDZACXrqyw6efWJ730izHFOF6qKUFdLdN3Z6ZRiNJ1pWUrW83jTadCq9VEAUFZ7tDFv0pF6IdHvc3WHOG/BFvFyUEidZcKTvs1C0ev4s1HKudlhDiF4QQf232/t8SQnz2Aa71S0KIzdmgjhPJy2/2efaxRYQQtAOXj57p8qU3bTHecn/IIEB2Wuh2TNMOqFoeZSxvF9xjZrssTRPVOGFlLA1Rymow5nQw4qy3y1l3l7OOaQKZNpr12ud6tcCNcuHu1nGQP+RBorX+ea31Cw9wiV8GfmyflnPkuD5Iubab8tzFxVufe+7CIn9wdUBeWb+W5R4RArnQg4Uu9VKLYjEgX3DJu5K8Kyk6xjRathuaToXTLum0E061JpyNh1yItnk82OQj3jpPukMedQSJho065LX8NJezVd5I786Oc+RFSwjxN4QQrwshXgSevOPzvyyE+NnZ+28JIf7zPcOrEOK7hRC/JoT4lhDiL7zXdbXWvw2c2G3H3p3CZy/cFq1nLyxSVA1fvTqc17Isxw0hkO02zuopmpUe9XKbfNEnW3DIepK8Jyi6UHQ1Zbeh6VZE3ZSF7pSznREX2ztcijd5OrjBR/1rPOMZA+lGXfFaucw38rO8lp7hjekprkzurmxxpEVLCPE9mFacT2D6Hr/vAx7+jtb6E8AXMLuonwU+BfzNA13kEeXlN/t0Aoen1jq3PrcnYNb6YLkrhEB4HjKO0O2YOvapYpcqkt/mwaoi05ajQ3MkbIc5C0HKij9h1Rtx2htw1tllRaW0hEuuYafxWS973Cx7bOQdtrOY3Sy8q2Ud9UL8Z4B/rrVOAIQQ/+IDHrv3ta8BrVlW1VgIkQshelrrwcEu9Wjx0pU+3/fYIkreNt/3Io+n1tq89GafvzTHtVmOPsJxEGGIXFqg6cbULZ9s2aMKTR9h2TKCVfQamlAjOgVxK6cV5Dza2WXZm/JEtMFFb5OzzoCPeYJMSy5XDVfKNdbLLq8kZ9jOW9yYdtmexBT53cnRkd5p3SP57L/NHe/vfXzUxXlf2RxnXNmefls9a4/nLizye2/vUtbNHFZmOfIIcastR7Zimk5E1fYp2y5lLCliQRmbu4RVrKnjBh1VBFFBJ8xYChPWghFn/AHn3T5nnQErqmDYFGzUDTeqLtfLBa4Vi6xnHbayFrtJSJp4lIl7V0s86qL128DPCCHCWTTxT817QceBPX/WsxeWvuNrz15YIilqvn7d1rUs34lQCuF5iIUeerFLtRBS9DzyriLvCMqOoJwlNVTtBtkuCds5S62E0/HoVh/hE8E6l7xNLjoVp1XIjcrhSrnIG/kaV9IV3kqWuDHpsjluMZkENGMXOb67vcWR3oHMctR/FfgqsImJntkXhBD/HfBDwLIQ4hrwf9Ba//39uv48efnNPpGn+OiZznd8ba+u9dKbfT75yN3dYrY8BAiB9H1Eu42IQ+rljsnBWnApWnLmvxLUARSdhiaukVFFt5PQCXLOtQY8EvY57Q15JrjGippyXjUMmoardck3irNslF2upCu8kywwyEK2RzFF5sLYxR1LZH53OQJHWrQAtNa/CPzie3z+z93x/mN3vP/LmEL8d3ztXd//p/ZtkUeMl670+Z5HF3DVd26kV9o+F1diXn6zz1/4wbux8llOPEKYHVbgI+KQphWZWJlQ3XK432p8DjRN1CCjCj8s6YYZC37Cqj/itDdk1R2wpia0ZY0rHMZacqPqcqPssVl02MxbDLKQYRpQpC46dVCJRCUCdXcZgEdftCz3xu604JsbY37qu06/72Oeu7DIv/zDm9SN/rZCveXhREYRIgxgZZGqHVC1PbJFxxTde4IyniWNdmdJo52cbiujF6ZcbG+z4k14IjBF9zU1ZVVJxg18pQi4Upxio+zy2nSNnTzm5rjDcBJQZS5i4OKkAnckcCfg3GUfz1GvaVnukZffMvWs5y5+Zz1rj+cuLDHOKl5bv99oesuJQCqE7yPbLUTHtOVUbY+yrYzTPcYIVqypI42OalRc0Y4zFsOElWDCGX/IOa/PeXeHJZkSCU2/rtmoPa6WS1wrFrme99hI2+ykEePEp0w8dKJwpsK8JeBONe707kTL7rROGC+/2cd3JB8/133fx9yqa13p89Ez7/84y8lGei4ijtALHZrIo+z5FG1FGUmKzsyD1dJUsaYJa7y26SNciU1Swyl/zAV/kzVnyCV3SCAEEsHlss161eXNfIW30yW2shYbkxZJ5pNPfMTYQaUCZyJwUvDGGn/UoO5yp2VF64Tx0ps7fPKRHr6j3vcxZ3oh5xdDXn6zz//8+QuHuDrLkUAqVCtGdNrodkS5FFPFDvmCQxELqti43KtQU3UbiCu8sGSlO6EXpDwW9zkf9Dnt7vJx/zptWRIIwXqtGDQB38jPsll2+NZ0hZtJh0EaMBjGNKmDHCvciUClAn9X46Tgjxq8YYVKq7tb/gH/eiyHyCgreeXG6D2tDu/m2ceWePmtPvcyYdxyAtgb7xVH6Di8I8v9jqTR6A6Xe1DjBLez3Bf9KSvemNOuaXzuyhJfQKY1gybgRrnAzaLH+szlPkgDksynSR1EJnFSI1hOCk4KbtLgTmucSYGa5h++fuxO60Txe2/t0mj41IXvNJW+m+cuLPJPv3KNy5sTLq22D2F1lqOAWugiwpBmuTszjTpkC8qE9/VmSQ2hpurViKCm1U1ZiFIWg4RL7U1OeSOe9G/ymNtnTdXUGgaN5I1ymSvFKW4WPd4Yr7CbR2xPYpKJT5M4OAMHlQncsTkOOgmE/QonqXF3M+TuGJ1kd/UzWNE6Qbz0Zh9XibvyX+255V96s29F6yFgb7yXaLfQUWAEqzPLcm8JquCOHVbUIGMz3qsbZiyHE1aCCWf9XdacIY84u8SiotCwXbts1i2ulktczxfYyNtspy3GmU+a+DRTF5lKnESgslnBfaJxU407qVDTEjlK0NMEnd6daNnj4QnipTd3+Pi5HqH3/vWsPR5ZjFjt+Lxk87UeCkTgI3tdml6LqhdSdF3yjiJvC4q2cblXLU3VrtGtiqiV02slrMUjzkUDLobbXPQ2uehtcsFtiATkGt6qlrhSnOJKusLVdIGbSZf+NGI8DajHLmqscCbG0rC3y9qrYTmDDDmYoneHNMMxzXR6Vz+L3WmdEJKi4mvXhvxHP3Dxrh4vhOC5C0u89OYOWmvMDFnLSUM4DnJhAbotmlZIvhxSh4psQVG8K2m07tSoVkkYFpzpjFgIEi5G2zzi73De2+EZd5tAwLCBG1XIet3l6+k5Nos2b02X2EpixmnAdBBCJnFHCmcscDJmRXeNP6zxBgVqWiA3d9FpSjOZouu7z3izO60Twu+/M6BqNM/dRT1rj2cvLLIxynl7JznAlVnmhXAchO8jWhE6Dm5nuceScpbjXu+53EONjCqT5R5mLAVTlr3prfFea2pEWwqUEAwah/XaND5vFm22Zy73cRqQpR4iVcblnhrBchKNm+hZ0b0ygjVO0dOEJs3QVQX3cEPI7rROCC9d2UEK+J5H776f8FMX9/K1+jy2HB/U0iyHjRAgJGplGR2HVMttypY7K7pLqmBmaWhpqlDT9CqcoKLXSViKpiwHU55qrXPKHfGUf4PHnAmryudapdmqQ14rTvN2vsxG0eFbo2WGeUB/FFFOjWC5AyNY3ghTv0o0wU6Jk1Q4uwn0hzSTKU2S3JNY7WF3WieEl97s87GzXdrB3cV7ADy+0mIp9viSDQU8OQiBDEPUQhfdadF0I8qOR9lxKFrSxMq0ZoIVaZq4xo0KoihnKZpyKhxzNhxwzuvziLvDY84ECfTrnKtVh3eqRa4Vi9zIu6ynbfppxCgJKBMPMVWoqcSZCtyZy92bNHjjGndU4AwzxGCMTlJ0Ud6XYIHdaZ0I8qrm968O+LOfevSevk8IwbMXFu3k6ROEcFxEFEKvQ70QUYcORUdRxEawypYZPFG29tpySrqtjE6QcTYacjowbTl7fYSPOC2uVRNu1B5vlctslF3eTpe4mXbYzcLbfYQThTMxPix3Ak6q8cYN3qjGmVaoYYqYptT9XXRZQXP/cwqsaJ0Avnp1SFE135YHf7c8e2GRf/31da4PUs727i7u1nL02KtfycUFdCem6gQUix5lJMl78lZwX9HV1GED3ZIgKmlHGefaA5b8hI/E6zzqbXPW2eVJNwXg1SLharXAjWqBbyRn2S5avDNZYHsSk6Ye5dBHZnLW9Gz6CP1Bg5tovEGFO8yQ4ww2t2myHF0U973D2sMeD08A7zXE4m55buaet7nxxxghTME9DNCtkDr2qGLjwapCM6L+VpZ7NEtqCCpae1nuwYRVf8SqM2TNGbKiUlwhaYCNusWNaoEbhfFgbWUtBmlAmrmUmYNMTf3KvIFKTQ3LSRqcSYEcZ4hpSpNmNA9wJLwTu9M6Abz0Zp8nV9v0Iu+ev/fJtTadwOGlK33+p588dwCrsxwkt3ZYp5bRUUCxElO2HMqWJOsJ6kBQ9GaCFTeIbkEQlJzqTFgOJ6wFY56M1ll1B3zUW2dFNXSlx6tFw2bd4ZX8LFezRTbyDm+NFhlnPpNxYEyjicQbSJwMvKE2TvdUE2wXqKRA7YzR/QHVZPpAx8F3Y0XrmFPWDb/39i4/+z33JzhKmrrWy9ZkeuwQroeMQ0Rsstzr2LslWEVshk/UvhGsqlVDWBPHObFfsBxOOBMOWfNGPOptsaQmdGVNpmFc5bxTnWK96nE1W+R61mMnixkkIVnm0kxd1MTssNwpqHR2l3Da4E5qnFGGmGbo4Zgmz/dVsMCK1rHnGzdGJEV965h3Pzx7YZEXXt1kc5RxqhPs4+osB4ZUyDBAtFrobouqF1BGDkVnVnCPTR9hHRqXu4gq/KhkIUrpBSnnogFn/V3Ou30ec/v0ZEVPOlytGm7Uba4Up9gsOlzPemwkbYZpQDr1aFIHNZG4E3lLrJwUc5dwWJkj4XCKnkypd3cP5Ee3onXMeemKqUV934X7z3u/Vdd6q89PfvzMvqzLcnDcShpd7FF3Qqq2T77oUoaCvCdvjacveg1N0OB0ClpxRjfMuNDZYdGd8mS0zmPuFmedEYuypga+XrgzS8MSr05Ps1uEXBv3GE5D8tSFgYuTzoruU2MaDXYb3KTB251ZGiYJzdaOKbgf1M9/YFe2HAovv9nn4krMqfb975A+eqZD7ClrfTjqzMZ7iTBARBFNK6Tec7nfGSsTmqSGJmgQYUUY5bSDnMVgyrI3Yc0fctbtc2p2JKyBcSO5Xi3MomW67OQRO1nMJPMpMgedOKhUorJ3udynDU5So6YFYpqiJ1OaNDUu9wPC7rSOMXWjZ7uj98+DvxscJfnuRxdsXeuII30f0YphZZE68siXQ6pYUbRMlnsVziwNkaaJarxuThQUnO6MWA3HnPaHPBXeYM0Z8oy3iycEIHilMEmj38xOcy1bYCtrcXXUJcl8spGPmCrcROIOZ0mjQ40/bnCmDf5OZgRrZ0AzGhuX+0H/Hg78GSwHxmvrI8ZZ9UD1rD0+dXGJb26M6U8PbltvuU+EQMYxotsxptFuSNXxKdtGsIqWuJXlXsXG5a5aJZ04YzFOOB2OOBsMeMTf4TFvmzVnjAIGDVytXN4ql3m7WOZatsDNtMNG0mI8DcinxuXuTOXtpIaJcbm74xpvVBrT6GhKM54c6JHwTqxoHWP2jnP34896N3vX+PJbdrd11BBKmR1WO77VllN0TVtO0b6jLSfW6LjCiUviWVvOajjmXLh7S7AecyYmvA/o1wHfKld4O182dwmTLttJzHAaUk49mDo4E4k7NjUsI1gab1TjjkrUMEMMx+jhiGY6PdAj4Z3Y4+Ex5uU3+5xfDDmzD072j5/r4juSl670+WMfXduH1Vn2A9WbJY2u9KhbPmXHM0mjgSBfuD18ouzWEDRE3ZROZMbTX2xtc8ob83Rwg/PuDmeUiTMeNHClXOKtYpmbZY9vTlbZzSLWx22miU+dOKhZ0qg3EjgT00cYDGqcaY23m6F2p+hpQt0f3FOszH5gd1rHmJff6vPsYw9+NATwHcUnH+nx8lvWGX8kEMKkjYYhuhVRt3yqlksZzxzue8NTZ1nuhDVOUNEOc7p+xkowYc0bccodcdbZpScLAiEYN5J+HXC9XGCj7LKed9jJYoZ5QJp61IljomX2stwTTME91caDNa1MW06Smsbnqtx3H9aHYXdax5j+tLgVm7wfPHthif/bb7zBKCvp3ENahGX/kVGEXOjRLHepY4980aOMTf0q3xtP39XUcY2ITJZ7K8g53x6wFow46w94Orhuiu5uTaKh38A3y1Osl11eS0+znnXYSltsjNpkmUs99ExKQyLwhqYtJxg0eOPG7LC2p4gkh80d6jRDl/Opf9qd1jHnXkL/PoxPXVik0fC7tq41P6RC9brIxQWaXpuq61N0XPKOnMUiz4ZPtDR1u0bEFUErZylOWImmnA93OR/0ueBv8ri7w5rKGTcVNyqHt8oebxXLvJMvcSPtspG06U8j0qlHPXFRE4U7mUXLjGf1q3GDO65wRzlyOIXhhCbPD/1I+G2/ork9s+WBWe34PLIY7dv1PvnIAq4SvPzmwTiZLR/C3nivbgfdiam7AUXbnU18Nm05t+8SNoioIogLunHKcjjhdDjkrD/gUW+bx9xtzjmwKB36jWK97vBWucLVbJFrWY/NmWBNprM+wqnCnQqcKbgTjGBNjGA5wxw5TEzBfTRCH0Brzr1gj4fHmOcuLO1rtnvoKT5+rsdLNvHhcBECoRRyaRERhdTLHaqWS9F1yLrK+K96wvQQxibLXYYVve6UXpixGo65GG+z6o74I8FVzjhjzimXjbqm33i8Vpy+NZ7+ymSZQRayPYopZuPp3aGpYXnDO5JG98Z77UwRw4nxYE0T0M28f1t2p3Wc2Q+rw3td82vXhiTF4dy+fugRwgT3hSEiCmnicFZwdyhDSRVBPYuWqQOowwYVVQRhQSfITZa7P+G0N+CMu8uqmtAWmoaGfuOxVbe5Ufa4WXTZzNvsZiGjzKdIXfQsy91JxGxw6qzgnjQmuG9SIMaJKbhns93VERjua3dax5j/ySf2v0/w2QuL/N//7bf4/XcGfPqJ5X2/vuXbueVyX+hSLbWoIpd80aGMTA2r6Jim56JrkkaduKTXSegEGRfbO6z6Ix7xd3jKv8GamvK4E7JZJ7xSKl7Jz7JRdnl9uspG1mYnjdgZxlSZC0MXNxE4U7PDclJNMDCmUXdS4WyNEUlGvbFl7hAeAbHaw4rWMeYg7vB976MLSGEasa1oHRzCccx4r14XfYdptIokeVtSRSYauWxrGl+jWxVOWNGKM1biCYt+wrlgl3Nen/PuDmfVhEjA9TrhRhXyVrnMm/kKO0WL60mXfhoxTgLKiYfIlEkZnRpLw14OljeqccYlziiD4Zhmz9JwhAQLrGhZ3kU7cPnomS5fsn2IB4ppfA7R3TZNy6fs+BRtM56+nM0jrOJZH2HQ4MYFcVjcastZ9KY86m9zxtnlvDNkWSkarblSBlwtl7haLnIj67GTR7cEK09cxNSYRp29ontiCu5O2hjBGqaIcUI9GO5LNPJBYEXL8h08d2GRf/ilt8nKmsD98GnVlrvnVtLo8iI6CiiXWya4r63Iu4IqFOQ9YxqtYpPl7vkVK90Ji2HC6XDIk9EGy86Ij/vX6cqSnpS8XQn6dcw38rNczxe4lvV4Z7zIOPcYjmLqiYNMFN5QInPwB9rcJZw2+P0SZ1qapNHhiHo4OrSWnPvBFuIt38GzFxYpqoY/vDac91JOFlLNhqfG6DikaQVUsUMVScpIzOJlZoIVanRY4wUlcZizGCas+BPW/BGrrsly78oSV0CmG7bqmOvVAjeLHht5h34eM8p8pqlpyxGZQmUClYGTmDc3bW5nuU9z9HiK3hueeoSxOy3Ld7B3V/KlKzsHcofyoUQIVKeFaLWMaXQppIwcskU1q1/dLrqXvdrkYMUFy+0pvSDlUmuTNX/Io942H/XW6cqaWEjWa7hadXklP8tm0eHydIWttMUgCRmNQpM0OnTMHcJkNp4+g2C3Nh6sSYHaGpo+wu3jYXWxomX5DnqRx1NrbV62zvh9Qfi+uUu40DOWhl5A0XFv3SGsw1nRvaVpwgbZKgnCgl6cshaPWPanPBbscMbd5TF3m0jUNMDbleJ63eVqscRb2TLbeYv1aYdBGpAkPs3ERaby1mgvZzorumcab1TiDHPEJEUPR8bScEywomV5T567sMg/+b1rlHWDq2wV4UGQYWCSGjqm8blsuxStWeNzPEtqiE0fIUFDFBW0Q5PUcCYcsuxOOO/tcMbZ5Ywy/X6ZhndmSaPv5EusZx12s4jdJDSNz7O2HJUbsXKnGndqstydpDYu91GCHk+pR5O5OtzvFStalvfk2QtL/IN/9zZfvz7kk4/cf/78w4zwfWS7BYs9msg3SaMtRd5WFF1BFUDRMykNdavG7RQEYcHp9pilYMr5aJePBOusuQOecbdxBdTAlapFv27x9fQc63mXG2mHa+Me08wjGYaIxAT3eUNTw/IHejaivsbrF6hpjtwa0EymxuV+jAQLbCHe8j7cqmtZ68N9IVxvNi0npokDqpZH1VKUkZxluEMdmhpWHTWIsCYIC9ozl/upYMyqO+Ksu8uKGhMIaIBho9iqOlwvF1jPu2zmLfpZzCT1yTMPkSpkasbTO6kxjd7Kcp/WqHGOnGToNDWWhmMmWGB3Wpb3YaXtc3El5uU3+/yFH3x83ss5XkiFWlowOVi9mGIxoIok6cLMh9WGsqOpA03du20aXW2PWfQTnmxtcNob8Ji7xTPeLrGQjDW3TKPfzE6zVbS5Mlmin0YMp6HJcs8UznCW0jA1Oywnm/UR7plGt3dpkpRmPJ73b+m+saJleV+eu7DEv/zqDepGo+T+NWafZGQcm/H0Cx2aVkDR9Sg6ijKUJlYmMgX3sqXRQY3XKojDnJV4yvl4wKI35YK/yVl3l8ecIQ0waBrerjpcrxZ4O1/mnXSRnTxmY9JimvrkUw85dpCZuBWN7ExNrIyTNLiDHDXOTJb7ZIouynn/mh4Iezy0vC/PXVhknFe8enM076UcD4RARBGi3aJuB5Rtj7LtUMSSMsbEykSaKjJZ7qpV0YoyFqKUlXDCmdnwiUfcPuedEaeVR61h2Li8Uy5yrVjkRt7jZtphO42ZJAFF4iESB5UI3OkdgjU1SaPuuLwtWLNpOfMK79sv7E7L8r7s1bVefrPPx85257yaI8xs2rPsdWkWOmZ46pI/6yMUFN1Z0mhPU4cNulURdTLioOCRzi6nggnn/F2eDG6y5gz4mJeTa7hWl7xRLrNe9ngtPc3NrMtG2mZj3CJLPcqhj0zMcdAbmRqWP2hM0X1U4e2kJml0q0+TpmZE/RFsy7lX7E7L8r6c6YWcXwxtvtaHsFdw162Iuu1TR7dd7lVkBKuKZoIV1rhhSTvMWQhSTgUTVrwxZ7xdzjq7rKiURmumjWarDrleLrJRdtnI2/TziGEWkCY+ZereUXCf+bBuDVCtUUmFmGZmgGqa0hRHr/H5frE7LcsH8tyFJX791Q201vsaOHiSkKsr6NCnXAgpeiapIVsQt4vubSNYolcQhCULrYTT8Yglf8pHonXOuLs87m7xpNvgi4BXy5Ib1QJvFcu8ka6yVbR4a7TEMA3MtJyRh0wl3kCaO4QTk+XuZJpgp0CNCuQ4mXuW+0Fhd1qWD+TZC4vsJiVvbE7mvZQjS70QUy6ElF2XoiPJO4KiMxOslqbq1NCuiNsZC62E1WjMo1Gfi+E2l/x1Hne3eNQpGTQVb1YZV8plvlWc4s18hbeTRa5PeyYaeRJQj12ckTIF9wm4I4031nijBm9Q4Qwy1GACu0OaNJtrlvtBYUXL8oF8aja92vq13p+qvedyn2W5x3dEy8yy3L2ooDdzua+FY876A855fc6qIauqoCsDho3iRtXmarHE9XyBm1mH7bRlXO6JRzM1SQ2mj1Dg7Lncpw3upMIdF8blPhrTDEdmh3UMfVgfhj0eWj6Q84sha52Al67s8HOfenTeyzmSpCsutS/IO5Kyw6zo3tCENapd0mmldIKcC50dTvljLvhbPOXfZEVNedQRDBr4w6LmteIcG2WXV6en2cpabKUxO6OYInNh4OIkEneWNKoyTbDb4E4b3FFpstwnKfXmFrqsTqRY7WFFy/KBCCF49sIiX7qyY+ta70MZSWofs7sKoQ40TVgjwpowLOiGGQt+wil/fCvLfVEltGVNogXDRnG96nGj7LFZdNjKWvSziFEazLLcHZzEFN3VzOXupMbl7k5NUoMYJ7dd7iek4P5+WNGyfCjPXVzkX3z1Bm/tJFxYjue9nCNH3hM0PhRtTdWeJY12c+KwYLk15Vw8YNmb8HR4gzPOLo+7uyxKCUi+VkbcKBd4u1jmcnKKnTzi+rhr2nKmHowcnNT0ETqpmZYTDEzSqN/PUeMcMZyYLPcTVnB/P6xoWT6U5275tXasaL0HZdvsrqqWpmlXqLCi105ZCFLOxEMeC3c47Q245K3TkzltKVivYaxd3sjXuFn2uJotcHXaY5gHDMchZeoipgp3JFGZwBsbS4M30abgnlSo/hQxnpr6VXW8Xe73gi3EWz6Ux1daLMUeL12xxfj3oopNUkMT1bhRSRSZpNGlYMppf2gK7m6fMyphUdZIoN+YLPdrxSLreZeNtEM/jRglAWXiIhJ1a0S9ycIySQ3upJ6N98qNYE2mNEly4o+Ed2J3WpYPxda1PphyoQa/JmgVLLanLAQpT7S3OOWOuehv8pS3zqIqCYRgpxZcrbq8UayxWXb45mSVnSxmexIzGoc0mUINHFRq2nK8gcZJIRjUZrzXuEBtj9BJSrW1cySGpx42dqdluSuev7TMjWHGt7am817KkUOEFW5Y0pq53FeCCae9IWe8Xc64u7RliQQGDWw1ETdmWe7reYedLGaQhiSZR5M4iNRMe97bYbkJuEljdlhJaWJlkhSdpEdmeOphY3dalrviBy6tAPBbr2/xxKnWnFdztGh1U2K/YDWa8Ejc55Q35sngJmecXS46BaU2SaOvFSu3kkavJMvsZhHro/btpNGRGT7hDWfzCCeaYLdGpbXpI5ykMBjRDIZHfvjEQWJFy3JXnF+MeHwl5rde3+LPP39h3ss5Uqy2J7TdjEfiXR4Ptlh1BzzpbhKIhkzDjdpnUEdczte4WXS5mi5wbdxjnPlMxwE6cVATiTuSOBn4Qz0rujd4gxKVFMidkekhHE9OpMv9XrDHQ8td84MfOcVLV3bIyof7j+bdLAVT1sKxORK6e43PmlgKcg1bdZur5RLX8x43sy47mRnvlaYeOnGQiTRHwpRvb3ye1DjjHDnO0JOpeTshSQ0Pgt1pWe6aH3pyhV/6nTf5H756g+9+9OTkxj++8mDH3Y+1b7DgTLnkrXPJ3aUnJYNGs9X4vFUuczlbY73ocHm8wjAP2J1EpKMAMok7UDip6SP0hsY0GvQr3EmFGmbInYEZ7zUeP/RitYcVLctd8+yFRSJP8b/57/9w3kvZV976L/74A33/aXfAkjPhrDO6PTy18VmvutwoF7iR99guYnazkEnmk6fuLMt9VnBPb++unNT0EapJjpwYl/tJycHaL6xoWe6awFX8yn/8Kd7ctncQ7+Qp/wYrKuWCE3CzTtmuXV7LT3OzXODtbIl3pgsMspD+KKZMXZg4OGNzJPRGM9PoWOMPKjPeqz9FjBOa3QFNmp3oPsL7wYqW5Z74+LkeHz/Xm/cyjhRPuikNcK1KebvqsF51eT07zVbR4up0gY1JiyTzKUY+IpU4E4k7Njssb6hxkwZvbNpyZFLA9oAmy6xgvQ+2EG+xPCDLKsYXkmHjslm32ao69MuYnTxmlAekuUeRO4hcojLTluNkJqnByUwdy5lWyKQwSaPTKTpNrWC9D0Lbs7LFYjlG2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrrOXhGCOE+DqQzXsdH8IysD3vRXwIgdb6Y/NehOXusKJ1vMm01t8770V8EEKI3z0Oa5z3Gix3jz0eWiyWY4UVLYvFcqywonW8+bvzXsBdYNdo2VesI95isRwr7E7LYrEcK6xoWSyWY4UVrWOIEOLPCCH+UAjxNSHEF4UQ3zXvNb0bIcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rstwdtqZ1DBFCfD/wqtZ6Vwjx48AvaK2fm/e69hBCKOB14HPANeDLwJ/SWr8y14XdgRDiNHBaa/0VIUQb+D3gZ47SGi3vjd1pHUO01l/UWu/OPvwScG6e63kPngUua62vaK0L4FeAn57zmr4NrfVNrfVXZu+PgVeBs/NdleVusKJ1/PnzwL+e9yLexVng6h0fX+MIC4IQ4jHgk8BLc16K5S6wbTzHGCHED2NE6/l5r+W4IoRoAf8U+Cta69F9XsbWWPYHcTcPsjutY4IQ4i8KIf5g9nZGCPFx4O8BP6213pn3+t7FdeD8HR+fm33uSCGEcDGC9Y+01v9s3uux3B22EH8MEUI8AvwG8Ge11l+c93rejRDCwRTifxQjVl8G/rTW+htzXdgdCCEE8A+Avtb6rzzg5fbtjygtajbHGY8uxft1yeOE3WmdYH4eWAL+m9nO60ilFGitK+AvAb+GKXD/46MkWDM+Dfwc8CN37GB/Yt6L+r/++ht87r/8bW4M0nkv5chid1oWy4Ozb39EP/Z3fpvX1sf82T/6KH/rpx+6iC+707JYjhOb44zX1se0A4dfefkq68Ojnu84H6xoWSxHhN+5bAJe/88/+3Earfl7X7gy5xUdTaxoWSxHhC+8sc1i7PHvPbPGD35khc+/ujHvJR1JrGhZLEcArTUvvrHN9z++hJSCz1xa5u2dhHd2knkv7chhRctiOQK8sTlhc5zzmUvLADx/aQWAL1zemueyjiRWtCz3hRDiF4QQf232/t8SQnz2Pq9j0xYwR0O4LVaPr8Sc7ga8+MZRH2R0+Ng2HssDo7X++Qf49gr4q3emLQghPv+wpS184Y0tLi7HnO2FAAgheP6JZf7NKxvUjUbJu3IDPBTYnZblrhFC/A0hxOtCiBeBJ+/4/C8LIX529v5bQoj/fM/0KoT4biHErwkhviWE+AvvvqZNW4C8qnnpSp/nZ0fDPZ6/tMwwLfna9eGcVnY0saJluSuEEN8D/EngE8BPAN/3AQ9/R2v9CeALwC8DPwt8CvibH/Icj/EQpi185e0BaVnz/BPvEq3Zxy++Yetad2JFy3K3fAb451rrZJaG8C8+4LF7X/sa8JLWeqy13gJyIUTvvb5hn9IWjiUvXt5CScGnHl/6ts8vtXyeOd25Ve+yGKxoWQ6CfPbf5o739z7+jjrqw5628OIb23zifI9O4H7H1z5zaZmvvLPLNK/msLKjiRUty93y28DPCCHCWcH8p/bjorO0hb+PiY/+2/txzePEICn4w+vD7zga7vH8pWXKWvPym/1DXtnRxYqW5a6YFct/FfgqJin1y/t06SOZtnBY/M7lHbSGH/jIe4vW9z22iOdIe0S8A2t5sNw1WutfBH7xPT7/5+54/7E73v9lTCH+O752x+de5C67+08iL17eou07fNe53nt+PXAVzz62yIvWZHoLu9OyWOaE1povvLHNpx5fwlHv/6f4/KVlXt+YsDGyqQ9gRctimRtv7yRc201vte68H7etD/aICFa0LJa58YVZFM37FeH3eOZ0h6XY48XLVrTAipbFMjdefGOLs72QC8sfnAcvpeDTTyzzhTe2sUnDVrQslrlQ1Q1f/NYOzz+xjHF9fDDPX1pme5Lz2vr4EFZ3tLGiZbHMga9eGzLOKj7zPlaHd7NX97J1LStaFstcePGNbYSATz9+d6J1uhvy+Ep8qw72MGNFy2KZAy9e3uJjZ7osxN5df89nLq3w8ps7ZGV9gCs7+ljRslgOmUle8fvvDL4jiubDeP6JZbKy4Stv7x7Qyo4HVrQslkPmS9/aoWo0n/kQq8O7+dTjSzhSPPRHRCtaFssh8+LlbQJX8j2PLdzT97V8h+9+ZIEvPOT5Wla0LJZD5gtvbPHshSV8R93z9z5/aZlv3BjRnxYHsLLjgRUti+UQuTlM+dbW9J6Phns8f2kZrW8Pdn0YsaJlsRwit6fu3J9offxsl3bgPNR+LStaFssh8uIb2yy3fJ5aa9/X9ztK8v2PL/Hi5Ye3pceKlsVySDSN5ncub/P8E0t31brzfjx/aYXrg5Q3t6f7uLrjgxUti+WQeHV9xM60uDWQ9X75gb2Wnoe0rmVFy2I5JPbqUB8WRfNhPLoUc34xfGgjmK1oWSyHxIuXt/nIaou1bvDA13r+iRVjUq2bfVjZ8cKKlsVyCGRlzUtv9nn+iQc7Gu7xmUvLjPOKr14b7Mv1jhNWtCyWQ+DLb/UpquZDo5Xvlu9/fAkheCiPiFa0LJZD4MU3tnGV4NkLi/tyvV7k8fGz3YfSr2VFy2I5BL7wxjbf/cgCsb9/U/uev7TM718dMM7KfbvmccCKlsVywGxPcl65Odq3o+Eezz+xQt1ovnTl4Zo+bUXLYjlg9voEH9Sf9W6++9EeoaseutQHK1oWywHz4hvbdEOXP3K2u6/X9R3FcxcXH7q6lhUti+UA2Zsi/eknllDy/lt33o/nn1jmyvaU64N03699VLGiZbEcIN/amrA+yvbNn/VuPjM7cr74EB0RrWhZLAfIno9qv4vwe3xktcWptv9Q+bWsaFksB8iLb2zz6FLE+cXoQK4vhOD5J5b54rd2aJqHI6rGipbFckCUdcOXruw8cIP0h/H8pWX604JXbo4O9HmOCla0LJYD4vffGTAt6gM7Gu6xl4L62w9JXcuKlsVyQLz4xhZSwB+9yynS98updsBTa+2HxvpgRctiOSC+cHmb7zrfoxu6B/5czz+xzO++tUtanPzp01a0LJYDYJiWfPXq4L6n7twrz19apqgbXn7r5Lf0WNGyWA6Af/etHRq9/60778dzF5bwlHwo/FpWtCyWA+DFy1vEnuKTj/QO5flCT/E9jy48FH4tK1oWywHw4hvbfOriEq46vD+xz3xkmdfWx2yOs0N7znlgRcti2Weu9hPe2knueyDr/fKZWavQFy/vHOrzHjZWtCyWfeagW3fej4+e6bAQuSf+iGhFy2LZZ168vMVaJ+DxldahPq+Ugu9/YpkXL2+d6OnTVrQsln2kbjS/c3mH5y8tP9AU6fvlM08sszHKubw5OfTnPiysaFks+8jXrw8ZpuWhHw332KujneQjohUti2Uf2RtV/+lDMpW+m3MLEReX4xMdwWxFy2LZR77wxhZPn+6w3PLntobnLy3z0ptmzuJJxIqWxbJPJEXF7729O7ej4R6ffmKZpKj5yju7c13HQWFFy2LZJ156s09Z6wPPz/ow/ujjJo/+pKY+WNGyWPaJL7y+jefIfZsifb90ApfvOtflC5etaFkslg/gxctbPPvYIoGr5r0Unr+0wteuDRgmJ2/6tBUti2Uf2BhlvL4xOfTWnffjM5eWaTR88Vsnb7dlRcti2Qf26kfzrmft8YnzPVq+cyKPiFa0LJZ94MXL2yzFHs+c7sx7KQC4SvKpi0snshhvRctieUC01rx4eZvvf2IZeQBTpO+Xz1xa5p1+wts703kvZV+xomWxPCDf3BizNc7n7s96Nye1pceKlsXygLw4pyiaD+PicsyZbnDijohWtCyWB+QLb2zz+ErM6W4476V8G0IInr+0zBe/tU1Zn5yWHitaFssD8vKbfT5zSAMs7pUf/9hpRlnFr7z8zryXsm9Y0bJYHpC0rI+M1eHd/NCTK3zq4iJ/+/OvM0xPhtHUmfcCLJbjjiMFn3p8ad7LeE+EEPxnP/kMP/lfvcjP/f2XON0N5r2k9+W//bnvvavHWdGyWB6QP/G952j5R/dP6aNnuvzvfuwp/j+/f523d5J5L+eBESc5S9piOSTsH9H+cFcmN1vTslgsxworWhaL5VhxdA/iFsvx4ej07jwE2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrbCHeYnlAhBBfB7J5r+NDWAaOetxDoLX+2Ic9yIqWxfLgZFrru+tBmRNCiN89Dmu8m8fZ46HFYjlWWNGyWCzHCitaFsuD83fnvYC74MSs0TZMWyyWY4XdaVkslmOFFS2L5QEQQvwZIcQfCiG+JoT4ohDiu+a9pncjhPgxIcQ3hRCXhRB/fd7reTdCiPNCiN8UQrwihPiGEOIvf+Dj7fHQYrl/hBDfD7yqtd4VQvw48Ata6+fmva49hBAKeB34HHAN+DLwp7TWr8x1YXcghDgNnNZaf0UI0QZ+D/iZ91uj3WlZLA+A1vqLWuvd2YdfAs7Ncz3vwbPAZa31Fa11AfwK8NNzXtO3obW+qbX+yuz9MfAqcPb9Hm9Fy2LZP/488K/nvYh3cRa4esfH1/gAQZg3QojHgE8CL73fY6wj3mLZB4QQP4wRrefnvZbjihCiBfxT4K9orUfv9zi707JY7hEhxF8UQvzB7O2MEOLjwN8DflprvTPv9b2L68D5Oz4+N/vckUII4WIE6x9prf/ZBz7WFuItlvtHCPEI8BvAn9Vaf3He63k3QggHU4j/UYxYfRn401rrb8x1YXcghBDAPwD6Wuu/8qGPt6Jlsdw/Qoi/B/zPgLdnn6qOWmOyEOIngL8DKOCXtNa/ON8VfTtCiOeBLwBfA5rZp//3Wut/9Z6Pt6JlsViOE7amZbFYjhVWtCwWy7HCipbFYjlWWNGyWCzHCitaFovlWGFFy2I5xgghfkEI8ddm7/8tIcRn7/M6gRDiZSHEV2dJC39zf1e6f9g2HovlhKC1/vkH+PYc+BGt9WTmTn9RCPGvtdZf2qfl7Rt2p2WxHDOEEH9DCPG6EOJF4Mk7Pv/LQoifnb3/lhDiP5+1Gv2uEOK7hRC/JoT4lhDiL7z7mtowmX3ozt6OpInTipbFcowQQnwP8CeBTwA/AXzfBzz8Ha31JzBu818Gfhb4FPCeRz8hhBJC/AGwCXxea/2+SQvzxIqWxXK8+Azwz7XWySwJ4V98wGP3vvY14CWt9VhrvQXkQojeux+sta5nIncOeFYI8aGDU+eBFS2L5eSSz/7b3PH+3sfvW8/WWg+A3wR+7MBW9gBY0bJYjhe/DfyMECKcRRP/1H5cVAixsrf7EkKEmHjm1/bj2vuNvXtosRwjZjnqvwp8FVN7+vI+Xfo08A9mmfIS+Mda63+5T9feV2zKg8ViOVbY46HFYjlWWNGyWCzHCitaFovlWGFFy2KxHCusaFkslmOFFS2LxXKssKJlsViOFVa0LBbLseL/D9e1JFwR1+cdAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "condition = posterior.sample((1,))\n", - "\n", - "_ = conditional_pairplot(\n", - " density=posterior,\n", - " condition=condition,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - " figsize=(5,5)\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This plot looks completely different from the marginals obtained with `pairplot()`. As it can be seen on the diagonal plots, if all parameters but one are kept constant, the remaining parameter has to be tuned to a narrow region in parameter space. In addition, the upper diagonal plots show strong correlations: deviations in one parameter can be compensated through changes in another parameter." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can summarize these correlations in a conditional correlation matrix, which computes the Pearson correlation coefficient of each of these pairwise plots. This matrix (below) shows strong correlations between many parameters, which can be interpreted as potential compensation mechansims:" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWV0lEQVR4nO3df6xcZZ3H8ffn3tvSdVGLFAEBKYZmlxp2QRvAkCgLiIUYWhG13awUF3JXg7urqEEkkRWXWHYTu/gTG6iAEsCtiDViWOTHglFYKlsoPxapoNJaqZRSJIXC7f3uH+eZMkzv3Dm3c+bcM+d+XuakM+c8M88ZoV+eX+f5KiIwM+u1gcm+ATObGhxszKwUDjZmVgoHGzMrhYONmZXCwcbMSuFgY1ZTklZI2iTpwTbXJekrktZJekDS25quLZH0WDqWFHE/DjZm9XUlMH+c6ycDc9IxDHwTQNIbgAuBo4GjgAsl7dXtzTjYmNVURNwJPDNOkQXA1ZG5G5gpaX/gPcAtEfFMRGwBbmH8oJXLULdfYGbF+fODZsSOF0dzld3+9MsPAS82nVoeEcsnUN0BwJNN79enc+3Od8XBxqxCRl8c5ZDTZuUq+3/LN74YEfN6fEuFcTfKrEoEAwPKdRRgA3BQ0/sD07l257viYGNWMVK+owCrgDPSrNQxwNaI2AjcDJwkaa80MHxSOtcVd6PMKkTAQEFNAEnXAscBsyStJ5thmgYQEZcBNwGnAOuAbcBH0rVnJH0RuDd91UURMd5Acy4ONmZVIhgcKqbZEhGLO1wP4Jw211YAKwq5kcTBxqxiVNPBDQcbswqRYKCgAZmqcbAxqxi3bMysFEUNEFeNg41ZhUhu2ZhZSQYHPWZjZj0muRtlZqUQKuZRhMpxsDGrErdszKwsHiA2s56TPEA8prR94PXAbOA3wAfTzl6t5XYAa9Pb30XEqd3Ua1Znde1GdfuzPgvcGhFzgFvT+7G8EBFHpMOBxqwNARpQrqPfdBtsFgBXpddXAQu7/D6zqS0NEOc5+k23Yzb7ps12AP4A7Num3AxJq4ERYGlE3DhWIUnDZLu8oyG9ffrMqTOkdOCWmZN9C6V56o3PTvYtlOr5jS8/HRH75C1f0+cwOwcbST8F9hvj0gXNbyIiJEWbrzk4IjZIegtwm6S1EfHr1kJps+blADP2mR6zF+bbi7UOvrTytMm+hdIsG1412bdQqru+8ORv85bNNs+qZ7TpGGwi4sR21yQ9JWn/iNiYUkBsavMdG9Kfj0u6AzgS2CXYmE15BW6eVTXd9vxWAY1seUuAH7YWSPuY7pFezwKOBR7usl6zWhJiQPmOftNtsFkKvFvSY8CJ6T2S5km6PJU5DFgt6X7gdrIxGwcbs7EUnF1B0nxJj6YUu7vMFktaJmlNOn4l6dmmazuarnXd9+1qBDYiNgMnjHF+NXB2ev1z4PBu6jGbKoocs5E0CHwdeDdZorl7Ja1q/o99RHyyqfw/kg1xNLwQEUcUcjM4lYtZ5QxoINeRw1HAuoh4PCJeAq4jW67SzmLg2gJ+wpgcbMyqRPm6UDlbP7nT6Eo6GDgEuK3p9AxJqyXdLWnhbv6inabOQhazPiBgaDB3G2BWWr/WMNFc380WASsjYkfTuVxLVvJysDGrkGzzrNzB5ukOub4nkkZ3ES05pIpesuJulFmlFNqNuheYI+kQSdPJAsous0qS/hLYC/hF07nCl6y4ZWNWJQXmjYqIEUkfJ8vTPQisiIiHJF0ErI6IRuBZBFyXMmQ2HAZ8S9IoWaOk6yUrDjZmFVL04woRcRNZTu/mc59vef8vY3yu8CUrDjZmVSIxODg42XfREw42ZhUypR/ENLNyOdiYWc9J5F0d3HccbMwqJf9Dlv3GwcasYvpx+4g8HGzMKkSCoSHPRplZjzU2z6ojBxuzKpFno8ysJA42ZtZzwlPfZlYGeerbzEogxNDgtMm+jZ4opL2WYwf3PSRdn67fI2l2EfWa1Y2AQQ3mOvpN18GmaQf3k4G5wGJJc1uKnQVsiYhDgWXAJd3Wa1ZLEgMDg7mOflNEyybPDu4LgKvS65XACVJNFxOYdWlAg7mOflPEmM1YO7gf3a5M2j1sK7A38HQB9ZvVhtBE9iDuK5UaIJY0DAwDDO3Zf5HbrFuSmDY4fbJvoyeKCDZ5dnBvlFkvaQh4PbC59YtSGorlADP2mR6t183qT7VdZ1PEr8qzg/sqYEl6fTpwW8vmymZGI5VLcQPEOWaKz5T0x6ac3mc3XVsi6bF0LGn97ER13bLJuYP7FcB3JK0DniELSGa2CxU2rZ0n13dyfUR8vOWzbwAuBOYBAfwyfXbL7t5PIWM2nXZwj4gXgQ8UUZdZnRX8uMLOmWIASY2Z4jwpWd4D3BIRz6TP3gLMp4tc4PXsHJr1rQmts5mVcnE3juGWL8ub6/v9kh6QtFJSY/w1d57wvCo1G2U21U1wNqpT+t08fgRcGxHbJf0D2Xq447v8zjG5ZWNWIY1uVJ4jh44zxRGxOSK2p7eXA2/P+9mJcrAxq5JiH1foOFMsaf+mt6cCj6TXNwMnpZzfewEnpXO7zd0os0pRYY8i5Jwp/idJpwIjZDPFZ6bPPiPpi2QBC+CixmDx7nKwMauQLCNmcR2OHDPF5wPnt/nsCmBFUffiYGNWKcWts6kaBxuzCpHEkJ+NMrNe8x7EZlYOicE+3BgrDwcbswrJWjYONmbWc/XdYsLBxqxChBga8ACxmfWahNyNMrMyeMzGzHpOiAEcbMysBG7ZmFnPqcAHMavGwcasUsSgPBtlZj0meZ2NmZWkrt2oQkJoN7lpzKyZnOu7nW5y05jZqwkv6htPN7lpzKyF19m0N1Z+maPHKPd+Se8EfgV8MiKebC2Q8t4MA+yjPfnSytMKuL3+cP7pN0z2LZTmNVvrOQBaBKnYZ6MkzQcuJduD+PKIWNpy/VzgbLI9iP8I/H1E/DZd2wGsTUV/FxGndnMvZf1T/xEwOyL+CriFLDfNLiJieUTMi4h5rxv4s5JuzaxKihuzaRriOBmYCyyWNLel2P8C89LfzZXAvzVdeyEijkhHV4EGigk23eSmMbNXycZs8hw57BziiIiXgMYQx04RcXtEbEtv7yb7+9sTRQSbbnLTmFkTkY3Z5DkoLv1uw1nAT5rez0jfe7ekhd3+tq7HbLrJTWNmrSa0qK+I9LtZrdLfAfOAdzWdPjgiNkh6C3CbpLUR8evdraOQRX3d5KYxs1cUPECcK4WupBOBC4B3NQ13EBEb0p+PS7oDOBLY7WDjaQGzihGDuY4c8gxxHAl8Czg1IjY1nd9L0h7p9SzgWLpczuLHFcwqpMinvnMOcfw7sCfwn5LglSnuw4BvSRola5QsHWOh7oQ42JhVSrFbTOQY4jixzed+Dhxe2I3gYGNWOarp6IaDjVnlaLJvoCccbMwqxHsQm1mJ3I0ysxLI3Sgz6z0hbwtqZuVwy8bMes4DxGZWGnejzKzHhAeIzawU8gpiMyuLWzZmVgK3bMysBMq7V03fcbAxq5BsgNgtGzMrgWejzKz3JPDjCmZWhrq2bAoJoZJWSNok6cE21yXpK5LWSXpA0tuKqNesfrJ1NnmOXN8mzZf0aPq799kxru8h6fp0/R5Js5uunZ/OPyrpPd3+sqLaa1cC88e5fjIwJx3DwDcLqtesdorKrpAz/e5ZwJaIOBRYBlySPjuXLBvDW8n+bn9DOdNwtlNIsImIO8mSz7WzALg6MncDM1uyZJoZrzyukOd/OXRMv5veX5VerwROUJZmYQFwXURsj4gngHXp+3ZbWSNRudKAShpupBJ9bvSFkm7NrEo0gaOQ9Ls7y0TECLAV2DvnZyekUgPEEbEcWA5w6NAbY5Jvx6x8kY58Cku/W4ayWja50oCaWaDId+SQ5+/dzjKShoDXA5tzfnZCygo2q4Az0qzUMcDWiNhYUt1m/WU059FZx/S76f2S9Pp04LaIiHR+UZqtOoRscud/uvhVxXSjJF0LHEfWh1wPXAhMA4iIy8gy8p1CNsi0DfhIEfWa1VK+VkuOr8mVfvcK4DuS1pFN8ixKn31I0vfI8nuPAOdExI5u7qeQYBMRiztcD+CcIuoyq7UAFThamSP97ovAB9p89mLg4qLupVIDxGbGRAaI+4qDjVnVFNSNqhoHG7OqqWescbAxq5Qg77R233GwMauaesYaBxuzynGwMbNSuBtlZmUocp1NlTjYmFXJxB7E7CsONmaVEjBaz2jjYGNWIaK+3ah6buNuZpXjlo1Z1Xg2ysx6zgPEZlYWeYDYzEpRz1jjYGNWKYGnvs2sDEHUdIDYU99mVVPchudtSXqDpFskPZb+3GuMMkdI+oWkh1La7A81XbtS0hOS1qTjiE51OtiYVUgExGjkOrr0WeDWiJgD3Jret9oGnBERjRS8/yFpZtP1z0TEEelY06lCd6PMKiZ2dNlsyWcBWUYUyNLv3gGc96r7iPhV0+vfS9oE7AM8uzsVFtKykbRC0iZJD7a5fpykrU1Nrs+PVc5symsMEOc5OqffHc++Tbnb/gDsO15hSUcB04FfN52+OHWvlknao1OFRbVsrgS+Blw9Tpm7IuK9BdVnVlMTGiAeN/2upJ8C+41x6YJX1RgRUvsnsiTtD3wHWBIRjWbX+WRBajpZyuzzgIvGu9mi8kbdKWl2Ed/V8NQbn2XZcGvyvvp6zdapM3y27flSugn9q6D/eyLixHbXJD0laf+I2JiCyaY25V4H/Bi4ICLubvruRqtou6RvA5/udD9l/hv+Dkn3S/qJpLeOVUDScKNJ+PI2/wtpU1NE5Dq61Jx2dwnww9YCKWXvD4CrI2Jly7X9058CFgJjDqE0KyvY3AccHBF/DXwVuHGsQhGxPCLmRcS8aa+ZOv+lN9tpYmM23VgKvFvSY8CJ6T2S5km6PJX5IPBO4MwxprivkbQWWAvMAv61U4WlzEZFxHNNr2+S9A1JsyLi6TLqN+snZcxGRcRm4IQxzq8Gzk6vvwt8t83nj59onaUEG0n7AU+lgaijyFpUm8uo26yfRBSyhqaSCgk2kq4lm7OfJWk9cCEwDSAiLgNOBz4maQR4AVgUdV2Tbdatmg5XFjUbtbjD9a+RTY2bWQd1/e+wVxCbVYmf+jazspT0uELpHGzMqiTwmI2ZlcGzUWZWFg8Qm1nPpf1s6sjBxqxqHGzMrNciwrNRZlYOBxsz6z2P2ZhZOdyNMrMyBDDqYGNmPRYRjL68Y7JvoyccbMwqxt0oM+u9GnejvNGvWaXky4bZ7YxVnvS7qdyOpv2HVzWdP0TSPZLWSbo+bY4+LgcbsyqJrBuV5+hSnvS7AC80pdg9ten8JcCyiDgU2AKc1alCBxuzCgkgRkdzHV1aQJZ2l/TnwrwfTOlbjgca6V1yfd5jNmZVEkHkn42aJWl10/vlEbE852fzpt+dkeoYAZZGxI3A3sCzETGSyqwHDuhUoYONWZXEhGajyki/e3BEbJD0FuC2lCtqa94bbNZ1sJF0EFmO733JWoHLI+LSljICLgVOAbYBZ0bEfd3WbVY/UUQXKfumAtLvRsSG9Ofjku4AjgS+D8yUNJRaNwcCGzrdTxFjNiPApyJiLnAMcI6kuS1lTgbmpGMY+GYB9ZrVTwA7It/RnTzpd/eStEd6PQs4Fng4pWG6nSxFU9vPt+o62ETExkYrJSL+BDzCrv23BWT5giMlJ5/ZyBVsZq9W0gBxnvS7hwGrJd1PFlyWRsTD6dp5wLmS1pGN4VzRqcJCx2wkzSZrZt3TcukA4Mmm940BpY2Y2U5l7WeTM/3uz4HD23z+ceCoidRZWLCRtCdZX+4Tzbm9J/gdw2TdLPZ4/WBRt2bWP4KJzEb1laLS704jCzTXRMQNYxTZABzU9H7MAaU0bbcc4LVvml7PTT3MxlXcAHHVdD1mk2aargAeiYgvtym2CjhDmWOArU1z/GbWUN4K4tIV0bI5FvgwsFbSmnTuc8CbASLiMuAmsmnvdWRT3x8poF6zGqpvy6brYBMRPwPUoUwA53Rbl1ntNaa+a8griM0qJCIYfWmkc8E+5GBjViUTe1yhrzjYmFVJQIw42JhZzzm7gpmVINyyMbNSRDjYmFkJAka3+3EFM+u1kh7EnAwONmYV4jEbMyuHx2zMrCzuRplZ77kbZWZliAhGt9fz2SgnqTOrkjRmk+foRp70u5L+pin17hpJL0pamK5dKemJpmtHdKrTwcasSiqUfjcibm+k3iXLgLkN+K+mIp9pSs27plOFDjZmVZLGbHrdsmHi6XdPB34SEdt2t0IHG7NKKacbRf70uw2LgGtbzl0s6QFJyxr5pcbjAWKzConRCT2uMG6u74LS75JyvB0O3Nx0+nyyIDWdLEnBecBF492sg41ZpUzocYVxc30XkX43+SDwg4h4uem7G62i7ZK+DXy60826G2VWJeWN2XRMv9tkMS1dqEZG25RdZSHwYKcK3bIxq5LyHldYCnxP0lnAb8laL0iaB3w0Is5O72eT5Xz775bPXyNpH7JkB2uAj3aqsOtgI+kg4GqyAaYg6zde2lLmOLLI+UQ6dUNEjNu/M5uKoqQ9iPOk303vf0OWKru13PETrbOIls0I8KmIuE/Sa4FfSrqlKQF5w10R8d4C6jOrNT8b1UYaKNqYXv9J0iNkkbA12JhZBxHByKg3z+oo9e+OBO4Z4/I7JN0P/B74dEQ8NMbnh4Hh9Pb5u77w5KNF3l9Os4CnJ6HeyTKVfu9k/daDJ1J4R7hlMy5JewLfBz4REc+1XL4PODginpd0CnAjMKf1O9IageWt58skafV404l1M5V+bz/81iAYrWmwKWTqW9I0skBzTUTc0Ho9Ip6LiOfT65uAaZJmFVG3Wd2MRuQ6+k0Rs1ECrgAeiYgvtymzH/BUWql4FFmQ29xt3WZ1VNeWTRHdqGOBDwNrJa1J5z4HvBkgIi4je4jrY5JGgBeARRGVDc2T2o2bBFPp91b+t0bUtxul6v6dN5t6Dh3aN778ug/lKrtgy1d/WfUxqGZeQWxWKeHZKDPrvYC+HPzNww9iNpE0X9KjktZJ2mXnsjqRtELSJkkdH6Drd5IOknS7pIclPSTpnyf7ntqKbIA4z9FvHGwSSYPA14GTgbnAYklzJ/eueupKYP5k30RJGo/UzAWOAc6p7j/bqG2wcTfqFUcB6yLicQBJ15FtnVjLxy4i4s604rv2+umRmgA/rjAFHAA82fR+PXD0JN2L9UiHR2omXXiA2Kz/dXikphrCi/qmgg1kmwQ1HJjOWQ10eqSmKuo8G+Vg84p7gTmSDiELMouAv53cW7Ii5Hmkpjrqu4LYs1FJRIwAHyfbQf4R4HtjbYNRF5KuBX4B/IWk9Wl7yLpqPFJzfFMGx1Mm+6bGkrVsPBtVe+mJ9Jsm+z7KEBGLJ/seyhIRPyPbK7fyIoKXazob5ZaNWcWU0bKR9IG0wHE0bXLertyYC10lHSLpnnT+eknTO9XpYGNWMSXtZ/MgcBpwZ7sCHRa6XgIsi4hDgS1Ax264g41ZhURJK4gj4pGI6LTt7s6FrhHxEnAdsCANuB8PrEzl8uQK95iNWZWs59mbz40b8u5iOWO89LsFaLfQdW/g2TSp0ji/S7qXVg42ZhUSEYU9rzZeru+IGC8DZk842JjV1Hi5vnNqt9B1MzBT0lBq3eRaAOsxGzNrZ+dC1zTbtAhYlbb0vZ1su1/onCsccLAxm5IkvU/SeuAdwI8l3ZzOv0nSTdBxoet5wLmS1pGN4VzRsU7vQWxmZXDLxsxK4WBjZqVwsDGzUjjYmFkpHGzMrBQONmZWCgcbMyvF/wPZqxE8c4CsDAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "cond_coeff_mat = conditional_corrcoeff(\n", - " density=posterior,\n", - " condition=condition,\n", - " limits=torch.tensor([[-2., 2.]]*3),\n", - ")\n", - "fig, ax = plt.subplots(1,1, figsize=(4,4))\n", - "im = plt.imshow(cond_coeff_mat, clim=[-1, 1], cmap='PiYG')\n", - "_ = fig.colorbar(im)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "So far, we have investigated the conditional distribution only at a specific `condition` sampled from the posterior. In many applications, it makes sense to repeat the above analyses with a different `condition` (another sample from the posterior), which can be interpreted as slicing the posterior at a different location. Note that `conditional_corrcoeff()` can directly compute the matrix for several `conditions` and then outputs the average over them. This can be done by passing a batch of $N$ conditions as the `condition` argument." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Sampling conditional distributions\n", - "\n", - "So far, we have demonstrated how one can plot 2D conditional distributions with `conditional_pairplot()` and how one can compute the pairwise conditional correlation coefficient with `conditional_corrcoeff()`. In some cases, it can be useful to keep a subset of parameters fixed and to vary **more than two** parameters. This can be done by sampling the conditonal posterior $p(\\theta_i | \\theta_{j \\neq i}, x_o)$. As of `sbi` `v0.18.0`, this functionality requires using the [sampler interface](https://www.mackelab.org/sbi/tutorial/11_sampler_interface/). In this tutorial, we demonstrate this functionality on a linear gaussian simulator with four parameters. We would like to fix the forth parameter to $\\theta_4=0.2$ and sample the first three parameters given that value, i.e. we want to sample $p(\\theta_1, \\theta_2, \\theta_3 | \\theta_4 = 0.2, x_o)$. For an application in neuroscience, see [Deistler, Gonçalves, Macke, 2021](https://www.biorxiv.org/content/10.1101/2021.07.30.454484v4.abstract)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this tutorial, we will use SNPE, but the same also works for SNLE and SNRE. First, we define the prior and the simulator and train the deep neural density estimator:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4683b28ba0614e87b33854d207e30da2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Running 1000 simulations.: 0%| | 0/1000 [00:00" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "from sbi.analysis import pairplot\n", - "\n", - "_ = pairplot(cond_samples, limits=[[-2, 2], [-2, 2], [-2, 2], [-2, 2]], figsize=(4, 4))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Analysing variability and compensation mechansims with conditional distributions\n", + "\n", + "A central advantage of `sbi` over parameter search methods such as genetic algorithms is that the posterior captures **all** models that can reproduce experimental data. This allows us to analyse whether parameters can be variable or have to be narrowly tuned, and to analyse compensation mechanisms between different parameters. See also [Marder and Taylor, 2011](https://www.nature.com/articles/nn.2735?page=2) for further motivation to identify all models that capture experimental data. \n", + "\n", + "In this tutorial, we will show how one can use the posterior distribution to identify whether parameters can be variable or have to be finely tuned, and how we can use the posterior to find potential compensation mechanisms between model parameters. To investigate this, we will extract **conditional distributions** from the posterior inferred with `sbi`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note, you can find the original version of this notebook at [https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb](https://github.com/mackelab/sbi/blob/main/tutorials/07_conditional_distributions.ipynb) in the `sbi` repository." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Main syntax" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sbi.analysis import conditional_pairplot, conditional_corrcoeff\n", + "\n", + "# Plot slices through posterior, i.e. conditionals.\n", + "_ = conditional_pairplot(\n", + " density=posterior,\n", + " condition=posterior.sample((1,)),\n", + " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", + ")\n", + "\n", + "# Compute the matrix of correlation coefficients of the slices.\n", + "cond_coeff_mat = conditional_corrcoeff(\n", + " density=posterior,\n", + " condition=posterior.sample((1,)),\n", + " limits=torch.tensor([[-2., 2.], [-2., 2.]]),\n", + ")\n", + "plt.imshow(cond_coeff_mat, clim=[-1, 1])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analysing variability and compensation mechanisms in a toy example\n", + "Below, we use a simple toy example to demonstrate the above described features. For an application of these features to a neuroscience problem, see figure 6 in [Gonçalves, Lueckmann, Deistler et al., 2019](https://arxiv.org/abs/1907.00770)." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sbi import utils as utils\n", + "from sbi.analysis import pairplot, conditional_pairplot, conditional_corrcoeff\n", + "import torch\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "from matplotlib import animation, rc\n", + "from IPython.display import HTML, Image\n", + "\n", + "_ = torch.manual_seed(0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's say we have used SNPE to obtain a posterior distribution over three parameters. In this tutorial, we just load the posterior from a file:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from toy_posterior_for_07_cc import ExamplePosterior\n", + "posterior = ExamplePosterior()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we specify the experimental observation $x_o$ at which we want to evaluate and sample the posterior $p(\\theta|x_o)$:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "x_o = torch.ones(1, 20) # simulator output was 20-dimensional\n", + "posterior.set_default_x(x_o)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As always, we can inspect the posterior marginals with the `pairplot()` function:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAACaY0lEQVR4nOz9eZxk6VXfCX/P8zz3xpJZVa3eW2q11m6phQDtrELDgAxYzHhszD4sBmPw6xljjD3GxvYwHuN5PV7GC/BigxnMYoTB7GB4sREWHiEktO9Sa+vWvnR3VWVGxL33WeaP8zz33ojK6s6uyqqs7I7Tn/hEdlZmxM2IG797zu/8zu9ISoltbGMb2zgpYY77ALaxjW1s45HEFrS2sY1tnKjYgtY2trGNExVb0NrGNrZxomILWtvYxjZOVGxBaxvb2MaJCncJv7PVSBxNyOU+wPP//D9NsQI/FcIUooM4SSSrXyebSAaSBUzSZxRIMnoLL3YU+UckyfCOJ71JlHwP0gmSwHQgXjA+fx3AtiA+Ybx+bXzCNgnbJsQn3DJguohZdUgTkM4jTQs+kLoOQoAQSd5DjBACKUQ9lBDyMcV8f3mn5e/GX7jk9+Ol5qsf+sklP7QYxIjeWwPGIM6BtWBHX1cOjCFVjlQ5cIZYW1JliZUhTC3Jgp8ZohNCJYSJvs+hFpKDaCE5SALJMHrvyzGN/l+S/pwFTP6dcu64pN8zeq9vdnmM/GeXc6ScF14g6rkgQfJ5oueN8SAeTADToOfGKuFWCdvBH778ex/2fbgU0NrGNRJ+KsQa/AzCLBFrCJNEcolUJbB6ExsRk5B80gkgkvqTTjZOk/7zn4SUz/KU/x8UJ1IUYhTwBoIoeHnBdDIAWAsmA5lt9GS1jWBbPUGTFYxPOCeYKiCNxRgBH/RD3XYkGxVXUyShn7WUEoLV70WjByRy2cB1ReJigGUt0gOURZwF58DZDFQKUql2RGeI0wxYtSFMheiEbibECmKtoBUdxJp80UoboKUXrdQDzgBYCAMwCXreOD1njEuIJMRErNWvjSnnTSIlISWI0RCjEIMhBiEFQ/D5/IggnUFCwnhR0PJgKunPAwpoHiIeVaDlQ+Te+xc8+YYdPfkf5REnEGoFLD+DWCfSLIBLmDpgXcDahHMBIwlrItboiWdNxGQAAzCy/oGPSfTiWYAqiX4vCSHm+yR0nSNGwXeW2BliZ5BWgcw2+QTtREErCHaVs64WojOYLpGsw1QGW1mwgnRBsxBrkc6DCClEREzOvvSWkiAEBS49avLBXpXX/2FjE7CsRSTfVzmrck4zrAJW1pImTgGrssSJJTpDmGbAqgWfwcrPhFBDrPJFy+VzwGmGlFwaMqMenEYXqvw9MQkRMCYDlUlUlcdKonIBayKVif054yT2f6JPhpiEEA1tsIRoWHWOEAy+s4RgSEFInd7HIEhn9EJmIXVCEs3Ckjnc+/aoAa1//9r7+JeveA/33b/k1tNTvunznsRffMnTHtXgFV2+VflknURkEjBVpKoCVeVxJjIpJ6ANOBMxDKBlJGFImNGJCBDzJTmiYFVuACEZfDR6otaWLhiarqLrLMFbgrPghWAN0mm5kqyCl5args1XVWu1vLQiYASJCTGCycCjWVZCJJByKdhnWzGSxCAmkuI1AlQlDsqwFBn0Zm3OsBSwSlmYrB0AyxlCZTTDmihghVrwU4iV6IWq0uw6TBWo4jTmTCkhrmTYObs2cTgsUaAqmZNIwpmIswpOtVWwmlhPbQNOArUN+VwZXmufz4U2Otpg6aLF2YrWW1rr6DpLDIZgEikYktdzIFotIZMkxCsID7XrQ8ejArR+5tUf5G//ylv57Cdex5/7/Kfw++/+JP/od97FPZ/Y4x9+1WdRu0dnvyE6SPmkTdOATAOTeUdVeeZ1x6zqqE1gp2pwJlKbwMR4jKR8H6lEgQzAEvsTsgBUl3P2mISAXlV90pPTJ8MyVLTBst9NWPqKlXcs2wrvLW3j1rIv8YJdCaEF0wqu1lIyVgY7SbhGiFawrcU6g6kd0npkZREftJzynuQ9YnL2RacZlyRS6F8ZvTvujEvMcF8yrVISlgyrrkguA1atGVaYWFIpBTNv1c2NZtUTwe9AqMDvJGKdlMecRqSKVBOPdRHnApUNWJNwNvRZdXmvpVywJGFF3/faBJzR82FmO2rjmeRbJYHKKGhZiYR8UWuio0uWZahZhoplqDjnpnTBst/VtLXFB0PrNfsKXm/RG4K1pEYfR4JgD/kxPfGg9er3fZof+LW38cXPuIkf/5YXYo3w577gyfzI77+Xf/Q772Kv8fzINz6P6rCvyEkKGRGkNiFW0/qJC0ydZ6dqqY1nt2qoJF818wk4MR6bQauSgJGIHWVcJdMKCDEZQjJEhJAMXbJ0yeKjYWlrmuA0o3Oepa8wkmitRSThrSU4QzSW5A1gSEZINmlJYHNpkIlhCQYMSFSwNCKKQSLIKPtSXgtStEgIJKNNg7WM67h4rhFJKCZnkH2WJUOGZYwCllvPrgpgxcoQKiFWWgaGzF2Fmh6s4kQzbDPzWBeYTjsqG6hdYGJLaZcz7FFWXS5OpdwzkqhMyOdHZGZbKgnM8/34PIHh/FglRxOr/t+MJHwytOKICEYSncnnkjF4Y0AsYiB4Q4qJ5HLzIHKoONGgtWg9f/Xn38gdN8z551//XGwuBUWEv/TFT+fU1PF3f/VtfM/Pv5H/62uf86gDrtIZTC4hVdSTtvLMq47duuF0tWJmO3Zcw8R45rZlIgpWE9P1J+JUWi0fxQNgc7uwAJZ+bQhJiBja5Hrg2gtTumQ562fs+wn7vmZiPU1w7FU1TedovaN1eqWNzhJbg2kNyWjGpSSsnrgkiFUGLCvYLmIBnBk+/NYoYInovTEKXDBkXOmQn4CjjouVhdYqAe9cvh+VhIVwrzPZPrEKVBPBT83AX00gTHKGVSXCToRJwE0C01lL7TynJi0T65m5jqnrcBL77LpkSkAPMCW7NpLWwGluGyoJ7Jgmf89jc5YFShEEhEWcsIoV582UKmfyAEtTAVCZQBctlYmEJLTe0diI95EmQhRL8IJp0O7jIeJEg9YP/d49fOTsil/4rs/j9LS64N+/+fOezKoL/IPfeief2mv4kW98Ptfv1MdwpFc4MplqrfJWE+uZu5Yd1zKzLafdiqnpmJu2B6uptNT9Serz134t24L1jEuzLUObLF0GrnOmpUuWiXj2TMuOU9BahorKBJrK0XjHwlV0wdLYCl8rcCVjMZWAaFkYXebMWkhicZUQS/nQGYwIxhjovHI01iqnJX7oLBIQ7LGXigWw1srCUhJW1UC655Iw1nYAq2mWMdQqZQmVEGZD0yXMEnESkbmnmnimk45T04ZZ1XGmXjK1nh3XMLNdn1VXErQMzKBVwGecYRfAqsUzlXyemJaK0GfloOdEQOiSw5LWvg9aMpZzaBUcPloqE5QDtQFnHa1Vkt6DUgiVxQQOFScWtN77yT1+7A/ex1c973Ze+OTrL/pzf+GLnsaNuxO+75fewhf/49/nq553O896/GkE+Pj5Fas2cGZe85K7buLpN+9evT/gCCPJ0P0pXZ7CX02MZ27aDFoN0x60OqzEfHLqSV0zlIl9tpUzh75MzCdrmywdFiORVayI1uiHIiRikr7DtPJV/3WbNVZiEt5YYhCiMYQwaMGMV3LWeEGSASIhWIwI4iMpWSXnvZ7hEgIpGSQZUoyICAkl8686OT/KsoZvjcpCMb0mC6PZVnJGS0KnZWByCt6xlIWVylpCIdxriJMIdcTVnsmkYz5p2a0b5q7lTM6uZ7ZlN2dLU9P1oGQZLkilRLTEDGixv5AVsCoXM/3d4ZzoksWSCEbL95VUTExHSKbPtryx+T5iQiQag42xf+6mCiqRsKmXaRwmTixo/e+/8XamzvJ9X/HMh/3ZP/O823nGraf4kd9/Lz/1hx/Aj07mQnv8oMCffu7t/G9/6jPYnZyQl2WDvtHKaQCscuKesismpmPHNOyYBoOC1fiErIlU+UpcSzxwVCJCBi1Dlwwthql0dMZRS2A/Tpgb5UAa2zCzLUtX00TLXjehjY79umbZVTSdY99O8J3FW0esDLHWP6J0GWMFtjEkEWwlJCPYJmAar8dXMi4RzbjIpeLFyPkrmW09pB4rl4V11UsbUl31GZZmWSpriE5ULFzrvZ/r6+B3lL8K84jseFztObWzYqfuOD1Zcf1kn13Xcn21z9y0zG3D3LRrWVMBps0oWdc4oyoXsGkGKzM62bT9EWklEKIQxDA1HV2yRGOY27YvOWfR0iUFZp8MTXDUJrAKji5YUoLQqXhWwqO4PPy9d36c33/XJ/nbL7ubm05NDvU7n/H4M/zwNzyPc6uOB/c7QkrcfGrCrLJ84nzDv/mv7+Mn/p8PcN/9C37y217IvD45L41stIoLX1E6Q1r++QxQngq9+pYTswBWVUoFwMqFM14RqEhYcuubRJSWFYmp6ZT3wjC3jXaYMMqVRH0tfVRuxZmIKxyHdTRB0OuIwXTKVRkPIQGIlg1ikICS8ckiPmYiPqpMgkzKMyLnNzOuq0nMmw15g4y+dlY5uky6JyskO2RYoRr0V7FSPV4h3qkirgrUdWBWeeZVy27VsOtadjJQFcA6bZZ91qTva+wBajNKiWiJ/UXMSKIiYvuMTIvvzZkYm39vzIuVk6eSwCRpZhaT6bNuI5E9O6GxEbFJFfj2UQpaqy7wv//GO3jqTTt88+c9+RH//ulpdQH/deuZKd//smfxWbdfx3e//A1818+8nv/7W1/YE/vXakgeq4Hhszh0hQaOopSD5VZJYEfazEdk3ZYkBSSBiqyb4uLAVUmkSxEMVCkQEP1Q5Ct5Ix5DojOWJjpq4/HRMrMd+65m4WqMJJauIiWhM4loHD4ajMudQNGMS6KOpZAMSfTYJCbECiZGKKCUSw+JaeguBrKO6woq5w/MsuyQZfWKd0tyttdixUozkFAPOqxYFcI981l50iHOEmkSsDMl3ec5wzpdrzhTLbnOLdi1Dde7vZ4K2DFNf4GyxAv4ys0otMAYrMpFbAgFLiMJm9LodyK1eKII88xzFq5r3HluoqMygT1fs19NtAPtKjqnwHWYOHGg9S/+83t4/6f2+alve9GR66/+u89+POdWHd//y2/lR//Le/lLX/z0I338I480uqGfxXiAQE91OMpX9ASrhAvAygAWBYXyytqN4USLJhGkhBEIaPmwI22PcAFDnTuRXbJMjWWe9ERexYqdULF0NU4Ci6pGJOWrbiKkiuQMklBphAOS6Bxj0vJPJREOYwQJSUHDZEmEyEDKh5BfFLMOXFciDlK9l7LQWdVjVU67hRM76hSOhKOTLG0oncKpjmjFSSTNAnbqmU47Tk8bdqqW6ycLTlcrTrkVj6v2mYrnlF3pxcm07EhLJZ5p32C5OCgUvqqcE8P/D1HogSI4DqNzw5KYSofJ0oqi6+t/N/OhRR7hJHDOzVj6CucinUmPzjGeN3/oQf7VK9/H17zgdr7orpuuyHN8w4vu4NXvu59/+rvv5nOfegPPf9LjrsjzHGmMgEuV6weDuWEg2cfc1RiwDhMWKWIpqjw5XUnQq7p0dKKn1cR0VMn3ncaA9GRwJQGfDM5EmuD60aBVq7la6KxmUEkwNUAeto165TZd/lB1EWJEQia4kwpQMTpoJzlbY9yZOupsaywiBR1BkhHxnjVZhXhP1qwT7m5UClbSfx1qFY+mWseyqiowqVQ0PHOdEu6m1UaL+NwZ9kxNu9ZgKWU/sMZNXfC+jspA/VmNAvPaQVawGjdmStkZELQ337I5iB9RrV8lIZ8Lhtp6Kht0lrGMFR0iTgxoPbDf8pd/7g3cuFvz/S971hV7HhHhB//0s3nDvQ/wV37+DfzH7/6ia5qY7wft81xgSnol9NHq1S4LQ0uYzF3VEqkephR8uLAieoKnxFQCgQ6MnqBV8liU1+qSzboeQzQrOutYJcfMduyFiWqJ7Jzzled+oK0dQcgfbgDB1mRphGKARINxml2pflMUvIzR8R5AJCrnFcJo3Ofoy8SxvKGIRnvVe1VlLZYbtFgTh88D0H5qetFoN88D8HMI04SfJdKOx0wCO7sr5nXHmcmKG6b77LiWm+rzzG3Lrl1xyi6ZijZbhmzar12kxhnUYaJkViGfUyFnWC2GmAzt6DJXQLI/NzKHNjyWSmUWUTloQ2LHtezZCcbEwYXkEHHtfhpH0frId/3M6/jI2RU/9x2fw5nZhZqso4zT04p/9rXP4Wv+1R/yA7/2Nv7xV3/2FX2+o4riyBDzSdYlm082PclivkeGxP1ipWDoBaYHn+QRCKMPvSFRE4kEulIaSuh5rpCzkJCELgWq5MFp6drWqp4GWNR6rV61lhhBkkHPc0EmJdMSQp3lEK1BvHahpMsCU5fl1SkpgOiLc+XKxAJY1uZBb8PaIPSm6r0aJA5DVqWA1c8S5llSMw1UtWdaeXbqlp2qYce17LiGuS1Z1iAULiC1Jm14iJJvHONX5GJgBfSZEuSsrBD4DIQ+QD1KbwPCKunndj9OWBjfzzE+Uub4mgets4uO/8+/ex1/9P77+edf9xye/6SLa7KOMl7w5Ov5n7746fyL37uHz3vqDXzV82+/Ks97uTEMNeuJVpTrbS7PyglILgm1LBTNVPrHSFj1Txi+N3qOcECGYvOISIWWiQBTkwWpMsr2BAIdMZcK0wxwpRu58hXORoI3dFQEAQmWZHTsRwl6tTZJYpBcMgJIO2i4JDoFsKKUt9r7IuZaURT0jiTbKkp9yZlW8cY6UN4w4rGKgDTzV0rAJ8JUB59lGphMO6Z1x5npilPVisfVS66rFj1gzU2j+ru+HPRrWrtNHmu4WI3fz9HXlJnToskzfSlYMqtN+qGU/AWwxjKJUnKukqXK+pPzpmMaPc6Evsv9SOKaBq3XfuB+/sYvvpn7Hljwj/7sZ/GnnvOEq/r8f/lL7uSPP/gAf/OX38LTb97ls5943VV9/ocNYZjZE53UhxFwZR6hywr2VazppKWVQKe6cSIDl9VnWuWETppplayqXDfXAWz4f328pFdYUe7JSiKgpShrv2f0BDcdoKDlTKCNlomd4oNhXxLeVISQ1fJa82GsmshFKxnITJ5brBR8Q0T67mEYysTSjUxHK4NY6xSOiXdnSZOaNHXEyhKmNvthmUGLNSbd53kAeq7D7/Ws4/R8xazquGG6zynXcF214IxbMjdaFk6lzYJhf4GcQfmmw/9tJbvqMD1YrVLVc1djmsFmYCxTFFPxvWxmKtpxrEX6C1yF/lyXwXXcxQxjE8FDxDU5jPfJ8w1//RfexFf/6B+y6gI/8+2fw1e/4IlX/TicNfzQNzyPm3YnfMdP/TH33b+46sfwsLHxPg8uDWZkKWPWMi7lmGTtCjuOAl7j7KuEAthw60naja7lWGU9lCxDJjaWYJyyS07ZFWfcklNVw07VMK076jpg60CqI6lO2YJnVE7le/WZysryyq6VZGoBMyrdMkHe81CwNuB8SSEydAyzmV9xbsDpiEpyphfQ9kr38vfUw98VJwmZRFwdmNSeWdWxU7Wccs1aSTgxXT+KVWYCx5MM/fuVS7y19+aAPzf29+sZ+nDOODp0CqKQ6vo+p54nnUpgKompCFMRJmKYG8tUjPKno4bA+PhS0qvvYROuay7T+pU3fJi/8ytvZeUD3/mSp/LdX3LnsQo9r9+p+YlvfSFf86/+kG/88T/iF77r87jl9PTYjmccJctKJmUd4+CLVK5khQjvkmOVKlapwqZIkzqsJNpCZCOYCxrc5BNes6wCWIWghQGsYg90mYDN5Z8WZYOViR2duGpx0vXgVklgNamY2Q6fLFYS+zZwPgnBWQJKztui43LDiZ6swQSnn8gYMSnl8lBJ9xSHzOuo+S0Zk+6TWjOsWZ1N/Bxh6oi1wc9Nr3QvpHs3V6DyMwinAtSR6U7LbNJyZrbiptkep1zDTfX5frrhlF322ruSYVUccnDvgBhPOqwyUK1SRZvv10BKIoHItKjoc3dyLoGpwFQME3GZKzWZ5Ux0dDSjjDYmg4/qFBKjqPvtIf+EaybTijHx93797fyVn38jd992mt/+K1/E3/yKu68JZfozbj3Fv/22F/HpvYZv+jd/xAP77XEfksaoPCSXh8V+xBZFPAW8iq2M09nBXAJ0GYRiSsQ18lZPjZJtjQn7vvWdr+IFsApfdpCI0cqFauySIWibXgWRp+yKHduw61SLNK07qtpjJ5pxxTrbStcMIsyJDMLMyhDzEDKV8kvDvJ8M2dZYOCyX+TEo3u7ODVqsyhJzt1A5LMHnW5hk19mJAlaYoOZ9dcROAtO6Y153nKobzlTKZZ1xyx6w+izVtBdcBIqq/eHioL+4cJ4B6WdL+znTwoludKL7LFqgEuVGK7E4LJXYjQthAcfiEpItj6J5ROXh8SMC6o30937j7fzkqz7At37+k/n+l919zdnIPOeJ1/Fj3/ICvvX/fi3f+pOv5d/9+c9h57ilEDLchMEbyZmAO+DkLVlXOQk7Ip1EQs62KiASB8DCZPGoZlub0eWfO0jQCloi5oHANSEi0INaD4RiwNK3xPfrCb64BkwcIolVUD/yIEZ9t0TtnMtCBTs1OFBrX+80o6qcvkTBjVTy5khdT5V0V7AqZn5xkgFranWmMFvNhIl6+/uploRhCmEaSdOIm3omU3VsOD1ZcV296DmsM3aRVe7tgWM5DyVlGPNaB5WGPZeVaYSSkcciV8m2RCqX0y5h/9ii0xHaORQqLAbBypj/EkJMGRBHVEXU+cMQMjd5kvy0fuwP3sdPvuoDfNsXPIW/85V365XwGozPf9qN/PA3PI/v/Ok/5m/+0lv451/3nGM91j7LQkdVrIm9pfJkw3VyfFLryehYkahSZKWFV54fk/wThbOQTKqXqzk5O7tQFV1i7MO19v2+g6jloxl94OpcG5yySwBWVUVIOi7SBIezkRiFFvURCzGLRgtgGsF49eiS4CBqg0F8VN+tMOoU2pFOK5eJlxVVHoSe1sRZRawdYeb6IWhdQCF0O5ueWOB3A0wibubZ3Vkxn7TcONvjTL3ixnqPm+tznDIrHc/JGVY9MuPbjHEXb7N7aDbu9T1hTdpQMvGSDbVJOSzI8pcE8WEy00jqS+4uBToCq6QdxOK/tYrqeOuj0RI9Hlpbevzl4R+979P8f//jO3nZZ97G337ZtQtYJV76rFv43j/xDH7tTR/h377qA8d9OBpjv+9RtlUGWM1G2VAM3AqPEYGO3CnMH+zIZiknByrmxyXFWCl9wc+lC8sEbRaUrC72DgMT0zE1WfFt1cxu6jx17XFVQOpAqnKnrRD0a4pyUdK7MiQ32MCIyXOAZX3XAVYylxRlpjA3AlKeJ4zZ011vI8J9Uu4T1BEzyVqsulMDx6rhlFuxWwags6RBS8LMYxGpKVKD4XapMVa4DzczAFYyFzRb1n9fI/b/JTyBiM6odvlcG3ux+dwQSkn6bPlQL/cl/5VHEPfvt/zll7+BJ92wwz/8s591YpZQ/MWXPI3Xf/AB/sFvvZP/5hk38+Qbd47nQMr73AMWuOzlPfbRGsSHWcOTPb6DqL3MKlkgsCJRC5A04woXIXcPGuU48OeymLX8e+zLyZJxDU6pViIkVVZj4IxdqmSCSDNx1DYQkjqVNtbRRCFZ25eQyeoKM7VvzhyJaKZlclYlKSHGkGJYd4W4TMlDynOFcVrlDMvid5R072bSzxL6+aDDCjvKYVU7LdNcEt4w01nCWybnOeOW3OjOc73bY8c0nDIrakI/hFwcF0x5bTe4poeKoeNLvmAVqyG7lmmFJAdcbEyvvYvZosiSNJvKpHvR98WUWKVIByxixfk4ZREnLKL6yfto8MHqgMJoFO3h4tgyrZQS/8svvokH9jv+5dc/95oeldkMY4T/4898JrUz/K+/9rbLPukvOdLGfQ6TS72SuZSrcy1q6rbpqaTEq363TYmORJeikvP5FkgXQNj4irxeVuhtrO+JbF7FJX8w1jVAhZifmI65bbJddMvctcyrjkmlGZet9UOfCjGfCe1Cyit/ZHoV+iB/MD1xviaDuJwYqd3V232UYU2Uvwq1juaEiTo2MA3YaWA27diZtJzK9tinM+l+xi57p4axO0fRQhkZZCSHjYPkKm1ZAZYnKMbvRcxWQ+U2dlnryfpkWSXLKplcAiaaFGlSZJVi/p6wyoT+QOrrNqc4OoevecnDz/7Rvfynd3yCv/OVz+LZTzhzXIdxyXHz6Snf89K7+N9/4+38zts+xpc/+7ZjOY7Nj1u/Fqz3NvL9/dhSuRC4Ot4TiSJ0qVzFEhbN2LNuvI8w4rNimSfMaX+JcZdwDbA2y0aJgFkDXdV1GabS0khFZ3RkJSZh5Sp8nUdJag+SCEGy86lmM5LAT9UBVaL6rZMS0lkITrMtr1bPmJAdIi7v2q2AZYhlGcWoS6hEu/ph+VkiTXQ3ZTVTHdqpacNu3XDdZMn19T6n3Yozdskpu+S0XbEjOqZT/K3WQGrcH5E4rH3LfGHxvooifeY5frHbbBnTjudD+4vIRbLnJHRYbHIg9KM5TX5cveAVioEsoTA9ub9KdV6KoiNmZV72kVw2jgW03vmxc/z933w7L77zRv7c5z/5OA7hSOJbPu9J/Pxr7+X//J138dJn3XrV/bckkh0edNtziMOgdEzSmwEWwJpKt8YdjeUQZfWwIeUO4oUdqS5dyGGVk73LjFcp6cZRgKt8XUI1PHEYvKYAnqcWy9w0ACxijSVm/k3JeR8MxjhW2UAwGIPxpaM4aH5sYwCnZWLZ5hOz+iy53E28PCJ+PAStCvesdJ9qSeh3EmGSiLsBmQQms46dWcNO3XHjbI/dquH6esGN1R67djUqCZdrGdbw/gwzphcr00OyRBIhl91GEl1af0+HUZ2ReDRnyGNt1vh97KdWY+74GjSDMh1TOmxK/bGWecUuWfZTzSpV+lzFzbQXlnLo0lBfg6sc51Ydf/FnXs+pacU/+ZrPPjE81kHhrOF7vvQu3vfJfX7jzR+5+geQ1oGrODwMJ/VItcy6B/hBpnDFK6lNJhOnrN36Ido0lIUFsIoGLI5KxfH31q7iDF3HoTO1fh4MWaJquLQTGpjaTlehuaCLRV0El5SY7xfXZtV8JapAr3OZWFxDi4NodhS93PIwZY/3YjdTfN1jWfWV7WVkEnql+07d9a6jp10zIt3btXKwdAFhXbrQi3vXCHSzcSszg6YHiWKV3fXfH7RYF2TCG1EAq5SN62BXpBIul4t27fHHHcl+gH8zm7sWy8MQE9/779/Evfcv+Lnv+FxuPnVtKMsvJ77sM27lmbee4p//5/fwlZ/1+KuabUlEW8VBiDFv+g2WJrh+iWYBGLWgyWaArH8YYDzVXyTmsDnIOuY+WorWazjpN7uGB231GUfoP4xmTW9kZRi4BjhjXe/DBOAk0kWLNbG/WgdjCa2O5hgv2W884RujizKKPxdguqzdSulIll/EiW7SUf2VZll+pq6jficRdnWWcH6qYVZ3XDdb8rjJglOu4cbJno4w2SVn7D5T03HKLJV/ZLA9BtYcF9Y4qFHDY3M+EOjV8uVCpa/98HtFSFrKeH1PYj8/qu9fucjY/mJlJRKioRJLJ6638t6kBzpsL3Mo52R53+J4hOdaBK0f/M138Ltv/zg/8N89ixc95eq4NVzpMEb47i+5k7/4s6/nP771o3zlZz3+qj23JJAoECEGIQRDFy1ttDT5BNGUXBdPFBM2GMqE3vc7xzgLYqOcGE/7j6+gIZUPzXpb3F4kkR+XpZslaGkSWBEdvDawSh0BYdc2/Uk/c52WH5XHe6vAVSdi0I01ppUsOch2NnXetVhlM76UEB/yiq/LKzjK+rPoZE3WEGqI04RMA27imU9aduo2uzWoPfIZu2RuG07ZJTum7cdyikyl56VG79HYz2q8Iam8pv1xpaEkt6JFfzc67rUmyUWyrOKJtvb3Jr04hGTy++Xy+6g811jGUrLwnszPAtbCiyayfO5aJOL/9Svfy0/8P+/n277gKXzrFzzlaj3tVYkv+4xbecqNO/zYH7yfl33mbVdNayah3GQAraBK4yY6VtnatogEpxt6pE3g6ku2ET+yfpKbtQ9JKSk2RYglxknMWsdSLuS9Lli8kIljQ2Qnc1udtX22tQyZAPaOtspbXSr14IqN1fIwQKj0NbK5TJSoXT5JSbVb2cLmckI1YdJ3DUs3M07V172e6m7CYi9zw2TBDdU+c9uuebqXtV2bq74uVgqO+cSDwKcfWJc8NL/xPmyW5uu/mwWio2xrDF4hKaB1OGLWiA3HOVx4+g5xoRVKt7IM80cDURXxbJaLF4mrAlo//eoP8g9+65287LNu4/tfdvfVeMqrGsYI3/aFT+Hv/Mpbed0HH+AFD7GH8SijBy0v4A0hGFbesd/V7Luac1bL77nRWcneTdKMMpoLpu6HD8UYvNbb4OM5Rtu3w9eI2hz9h68I10cfmiK/WLsfDeLWKaiHk2Ft+3UlgbZ2OBPwccjuuokjJohTQ/BoR3GqpUdoBTPRTqVptKOIt4gPKjq9jOhFpMVipiykmAXc3LM7X3Fq0nLTdI9T1Yqb6j2ud/vMTcN1djHS0hWn0XFnV2OznCtOoMPrPnRzy+veJd1LaVNaA7CLhT6/IZQLCIY2lXEsHSzffH9LBDTrCsn073VM69ZIY1FpG3WJSQhGh6Uj187A9Mtfcy9/51feypfefTP/7Gufc81vuLnU+KrnPYHr5hU/9gfvu2rPKXG4EYQUBB9U/7LKvFYT1dq48AmFNH0oUSiMwKsMtZKv6KNRj4GMP0DOkOOgk1zHhcrVOK4BVlF6q5By0JYVK+Fev2VaVcvbrJZ3AWsj4hLJJZKD5FBy3kF0uqZLCXOjpLwxw0D1ZYSuABs9V5WIWUdW1b5f9bXjdNv3mnmfGcz7DmqMBGQk+hwyWm2arIPC8F5INn1c57oesgzsu5OxH8JWIXKxvSkXufX7h4txg6C3u8nuDqE4PGSK45qYPXz5a+7l+37pLbzkrpv4oW943jU3BH2UMa8d3/g5d/Ajv/9e7rt/wROvn1/x5zR+uIkXYmdpvWPRRSZ2wtRqdjIxnjh67YMxWJOo8CAPPWw7zqxKOdH2oJfTftbLh0LaXuiaGbP7RFyTYGzuYOyPE12EUZZjlA/PVCZ0lWOS15KpLimx6vR07jpD6JQL0kxLCC14r9IOW1lMTOrIEB4Kug8Xoc66rKn0a7+YBeq5mvhdP1twXb3gplqV7o9z+5w2Sy0LpVkD7OFvz3q0XA73F4mc4Y4zq/IeXEDCJ82QDlzQOnrPy79XEvJ7qRnumJMcP4YZZYPKwQ1mgP3xjzRjIUnPsTbRsQwVq1DReEvwFvHSn8eHiSuGIv/+tff1gPWvvun5TKvLu5qdhPgfP/dJGBF+5o8+eFWeT3zqAUtLRMF7Q+e1g7gKjn0/YRkqzgcdodiPEx1azQT90IE63Ed380p90C7qsYVuObnN6ApdytSxfmwqnmnWI00kMMlrzqbZkrnIAIpK/JTV2bxT1YrdqmHmOqaVp6o9UsVBAjHedNP7sueZRJuFpZdbHpasrhokDraO1NnEb+5adl078nRv16YULgZYpbwKm9lSzx09BGAxbNcZR59FEdfeg3KrxK+V7L07aQEmiWuLfzcFy3YjUxzPGq5ipZl/cDTB4b0l5nN3rK17uLgimdavvekj/I1fejMvvvPGxwxgAdx2ZsaXfcYt/Pxr7+N7vvSuK/53mwAxlEyLDFqW1kaaYFl4XYi6F9Tu5bxM++5hsTkGLihLIMsVksmkquHCEZ6BSzqo5OjLiI0PkJHhg1I+EAdtuB5zOeUY9QH0MXsldtIrt0+Gc9WUEIWmDoTKkLyQKiEG1WzZCqJPWiJm4JLcSbycKNqsUGlpmOo0LKMorqO2Ydeu+mHwIg+42JxgKes2M93SqdXXXQ68aOjLtH6RGLvIbpaA5ef093SWsE3oHCooIme+i9GZ0Ptp5cfuj30EumNNl2ZZtTaKvMN7Q/JGz99weE7ryEHrdR98gL/2C2/ihU+6nh/75hc8ZgCrxDd/3pP5rbd8jF9740f4mhdeWYto04KxCdMKphWSNYTG0gILq6VYSoIzER+HDSorq6VWsToGPcnL9hQFiJhLRy2zaoG2jIIkk1HS9R0moNfZjPkR/f9iexPXruhFQKlZlfrIl2WxxZM+krD4fhNMOb7OLpT4RVSzJYm9iYJz6x0Ln7tUU+1K2QmETiUioTYQE1I7pAvIYbeEXiSKqV+cpN65YVp37NQqHi02yWorM6z1utiK+rFU4aAweWRHX9eQLXyGxxq/3kOmFHsNVfleeS3XfjclyNlfKU3LgP3YkmYMfGuvBYPLR8DQJsci1pwPU/bChH1fc76dsN9W+NZBZzCdYDswHYeKIwWtT55v+M6ffh23np7yo4+hDGscn/OU67nrll3+3WvuvfKgFRJmxAfE3EWMLuKDCk2dcax8RVkcMdHNp+wb3QhdRXWFqIGwcRJa8kBPHgXRcRvV5ZBnFktXKY4EojDwV+Vxen+nUdeyfK+WiBVdhKDjQ6LyKejBq5aYxZUhc12qlJ/GjrltaaJj7lq6YHUBqIuEKhKdQYplTSHkHdmyppSHl5lpWd3HmCwklxAXcTZ7m+WRqZLhjjOSsW6u/x7rZV5MJktSBsGnXjTiaO5wDOj5giGjEm8DsMwFWdZ4njF/L29sAjBpWFAy/vlNrmxzoDqMvOb7plBQ4z8fLMmbviw8tkzrH/3OO3lw0fKzf/7FXL9TH+VDn5gQEb7mBU/k7//mO3j3x89z1y2nrthz2SaRRLAN2JWQJBFXhpgcjY7hEaIgkmjzgCqotYuVSGMqQibop6ZlB3IZMBrMFd+XDHbU0i4q9jbZvk1+wfGNuoTF5K+4OAxZgJ7qFVAhVGIoBoQdgZizPvX7Klogz460WJOIVolpw1AGN0Hb6Usgrmw/TG06QWLCTxRobWOQzvYziZcaw7INoIq4KjBxnon1zGwex2HdkE//DjMabjZrgDMWaOrrV8S4atBYMZjxHcRnjTOhcWdy0yRw0yjQFmNHCTnrKiC1jiib0w5FBBtGI12rVLOINYswYT9MON9N2e8mLNqKVVNBazCNYBvBtGDbw70PR0bEv/lDD/ILr/sQf+4Lnswzbr1yH9STEH/6uU/AGeEX/vi+K/o8mmElTJswHdhWMI0grSG0hra1NF3FyjuWvmKv0xPnfDflrJ9xPkw5H6fsx5r9OOlJ036Uo2RCDG4Rteg2nbG2qurJ2HVDus0refkArvEyhbPqt1wbzehEKPsYbZ+BpdEHbViGUaQDu7bhlGvUNLDyVFX2lZ9EQq1K+bLFZ0zIJ3eZLg8ly7IJbEJMWS5SRmZ00LsbkdLjOcxxOdW/tyO+qF/8wfDalxJbM862/7qXT2xktptxEGDp846WumaQKxeZwkduvtflfV6XYOhURpPJd+0Y6nnYeqddw070Vs7jQ3YPjyzT+sHffAc37NT8z19y51E95ImNG3YnfOndt/DLb/gw/8uXP/OKST1sG3ULTadXqmQ1m0gGkrN4myAJC1P326eNqEvAJGROI5cvEaMGfJSTNfRfA9kFYugOlVIh5Cs/yWElXNCFfCg5RZmJBC0DJ8b0W1wA3ZWYQgYsjTI0bHvA1DKxS+q91aWG3aphUdX4YGjqihTyeI0bBqmNL9otQ7rM92cALRCTVC+WF4yMIyY1Xiz+Y+AVqGQorMZjMyZ/X/mr8bbmMlM4ZFmDXGHMJa6X4uXrzdgc1TKSNMsCkJBL1HUCvv97Dug6Fy6rdAyL6d/C16y8o20tsTNIl6mNbrgAHyaOBLQ+8Kl9/uj99/N9X/FMTk+v7Mr6kxJf/YLb+e23fYxXvvuTfMndt1yR5zCNKpxdpWJKSTr/JgG0ce3oKj2RfTB0WXi6dBUxCUunV8GQDI2t0JGZVk9+01Cn8RU/YEpWMCoZ27zqq5C1RdtzEIkcMZkb0+5UlYneKoNhkxQsq1x+Qrm/kOyw2XalANfcNJxxupdyWdes8pjPclbRkhXx3qhma6qEvJ3qPCKXScTHkmUZzbKGv1fwSQfZV1IxMR3E3NEzI2BJA7CMy8T+b90AnLVZURk6dXDhRaP/+fx+tJLL9tF86UFqlzLaZXsrkQvDSFQLnCzLKCvH9rM7aSHfz3dTzrVT9toJi6bGN460srhGsCvBtrmpdDWJ+F9544cRgT/1nKs3LHytx4vvvIlTE8fvvv3jVw60fCL5hO0S3gumSznTEqxDW/sYfKcfyrI5KyahMsr/GBIzq2fL3DSqx0leAUVgmnmknufKUojN7iIjwr6Q8mPgGpZa0LfTu6SDtp0YuhSpSARJyjEdUh5dSlR1GQjZW16dTpvgqKqA95Ywtq5xQnRJwcYZkrt8p4fxkhGgdzAoZLRLkS46rEm9sh2gGpPcyaxxRcBaqTb8zRolA1IOqjzxhaNXwPB6Zk7Sjl/jAxw9DjqGcWyODY0ticbke5mDbYKjDZauGwh405UxtFweXmx78EZcNmillPjVN36Ez33KDdx2Zna5D/eoidoZXvKMm/hP7/gEMaYr4htmGs1AkqVfEZ9sPhGSXl1jmwhJM64Q8nyiU//vpa9Y1RURYcfqDr0uOULeMzWVDmuGTAtYAy5L6InaQFzrMhXCfhi2LR8kJe675AhGBsdTo3mCTboUoboI3bq5LVl/NWbhqc5YrlzFstJG0N5US+NFawmdIMmoSj4KoRHiSh1OLytGgFWcOH0chteXQY9lmjMtI8WQsKVKo4+g+AFUuLDDd4G8oGjf+gvHaAh6Q4jaJdu/T2XDknYIcxZ7SNzeHKrfXARcysG9nGWd66ac66bstzXLtqJrHDQGs5JMwmtDybYJ21wl0HrTh87y/k/t8xdf8rTLfahHXbz0WbfwG2/+KG/60IM8947HHfnjm05Bw9RGOy+inazc3iMZHURN1pCiXvnbBMEZRBIhf1idBDpnmdlW7UZy2YWBOoVcDq7byPRaIYoTwMVLrE2yuSO35mNNEI9NkaxhZT96KkmMV4VEDioQh1DblchUWqII82zR7JNhXnW03rKqK0JtiV3KJPygkBd/BBeUNNxiVNDySWUn3hp85ndMzrQURBxdGrqqppcyFOAaLhRDxpXWnnPdVmhdiFouCMOY1eD4AAx2zZmLvFhGNY7Ngfri9lH8slZJyfdVrFiGmoWvWPqKZVvRto7UaUk+1mYNo2hXCbT+41s/Sm0NX/6Zt17uQz3q4r+562asEf7TOz5+RUBLuoARwbQR69TjyDZ6JiYj6IVUifkYikLZkapIYxIxGlLeLeiTZWbVo6qsXIeRVOEhuI21f89t76IxKrE5SzcM0oqCFijfYxJTsrNq7hqGAyQJQ9cx5g9SKRNDtnpp8dYycx1N5VhUQXVbffdwGKQ27jJBq3hBRSHlW8iZlk86UmVIzIyhIRsaZvuZAizFc105v9D/jeMc8KBO3yZw6cu4PlA9TBWoti4wUrCXUhFgpNM7CKzK98duE6XjXPisJlYsghLvw4yho8tzhhQCvgBWp/TGVS0P//gDD/CZt5/ZEvAHxJl5xQuf/Dh+9+0f569/2TOP/PFl1UHQleTIcJE2vkzN6weUlG2HfSJ4HWFpo+CrQNdZQhRWtZ4Ky6pSMlgiK1sp2ZpL24oLnQgeSRQ3iL61n6AWPel3TEMnegwrCQTTUaXUb0QOaXA92PxAFcuVSgI7punHRgCuq5fadOgcXet0EcZUOZUwEUKVssvppUcxY5SYtFMZDa23OBtovKM2XpfOxkrvk6NK2r2tcqZVMs3BWkYGQBmByRhUNpfl9rOK2YljFau1cSsrCZtG2rmSv+ZylHShQGI851iAa2yNs8hSmbIabBEmWfk+Ya+bsJfV723rCCuLWZleV2ga9Os2YdqIaQ/LY15GND7wlg+d5flPOvos4tES/80zbubdH9/jU3vN0T94CEgISIhIFzFdzGR8yh2ZlEV7enKYlqzjEmgNsbX4ztJ0jmWrafzCV+z7fALGuidXi9VJiU375E0niLUFFmk02EvxKre5XB1a46tU5QUIrl9LVfzpIxy4IWb8YR60ZGUFmdrXzF3LtPJYF6CKWbleZApqWXNZUZw3Iznb0m+FXCL2y0YYLQKJrnfxHGxlyhjzxT+WDzfcvu7HP7zu4xX3w0yg9Iss+tKyB6nRHgAGi5sxYJXzoifgs8V3Ub+30dLljDN2BnzWZHXa4dbsqqjhE3I1Mq23fvgcbYg87wqUPo+WKID+xnsf5EufdbRdRGk6iEmXkUq+0gva2r8g09JSUQLETj9Qmn0JqwRdZRFJdBNtYVdGea65aXW2zUYt4wTq0bm16Ze1Odx7YZkx4kOwkKATq4tjiw2L6Id4Kh11iv12l4OyCyjOBYOEoByDJXFdtSAm4Xw1ZW/SaTNikhBP3pF4eEuUi74PcXCQJQopNzy8MQNwJbXCdtlJtksWm2Jvk2zT4P5ZiVfdW5Z1DN1CuWiXbww6BaB6gMyvq5HUWy8TVS5RXB1s/j2VXZTnW+9EjheYlA7oIk7okuvlDYtYr2VZi6amaSpSY5HGDOr3kmU1CdtEbBeVoz1EXBZovf6DDwBsM62HiGc//gzOCK+/94EjBy28Vw1D55FWFU0267KsaHkRouqI9HMveWxNu4xFnhCteqyvqgojUNnAuW6KIXHGLbFkz6sU+g8SDFlOKUv66X4G/uOghRab/9b13zPUMRCyB1QUQ5SO8abrgx5zPHJEckylozMKfHPbsrQ1M6eLXtvKsaoiyVm1lLHKa11OSGSNiE+JXsy7GWs8XwYFUwBeVMcWspi0iG9VlHr4xaxDljV4s+v3h+5ry9A46flH8Rc0VMJGVlZAcJUqQjL9WrAmuTXl+6pIHLxm89KZfrC/12X51N+ki4g/XHl4WaD1ug8+wJNumHPTqcnlPMyjOma15e7bTvOGex888sdOMQ52wfkqZZzJ/IjkYWb9YIYkIErQxwjWSi8sDE61213lWJnEwlUsqorKBBaxZmI6VrFixzS0yfYOAQcKGdOF7phjO+YLfj4LHwtwrVLm1FIkogstpnQXqLbHqvHhBSnuEyrXCLmTuHINc9dS24BzAVwiukSykoedL1OnlUvDfqNMGpaQXrAm64DolfJJsFKI9KiKd9bV6OPYJOIvdFwwDEt183H0ZL+usDfkbC8D/qbFzGY3csiyxnqs3C2MajuzCpVuhOocvtPS0HSiw/2ZgLdd0u6hT4hPiL8KoJVS4nX3PsCLn37jpT7EYyaed8d1/MLrPkSI6WjtptuOFNMgEwoOI6LcQEoQLeINkD2LokBKxFqzMAlCSJDEqFVzFlnuSWJeKYycrWYYktodR71EHqTaBoaO1YbwcMxFaYmiH7CxZ1ZRVoOOpIQktMb2wtGLWaHoYw4umkWpX5oHu3ZFl6yaBdYNbbDs15FUm76LmA6pxL5YHFSxiSREEtZEnOitMgFnSmd0GIlat1hez8Q2XSBg05VByfrekYPBYUG5w8HoEegHtEt5GETlMfpcHnLnd7PE3wSr4kS6iDWrWHHeqx5rFVwm32uWTYVvHDRWle+rMtyfcmkYsauIbQKmDcghy8NLJuI/9MCST55veN62NHzYeO4dj2PRBt71sfNH+8ApqpYhBChZl48qhfBRU+8uDml4l/Ksl3I6kl1PjRcIQvK60ccHo1tusqJ5c9B3DEgPFcWv/KGIZVhvp4/J6bH2pzitljb7xQjpMsCrW218bwszMZ7aBiobEBtJNuVbP+FydNEDVsourjo8XWyKy9eXE+Pfv1jZGPqyTi8IXRyyo0G57ta+Lsr2Tf/5gwCrvD+94j3anGVZ2ixxSFniIB5kTZc1Kg1DhBjBX2FO6x0fPQfAs59w5lIf4jETpVHxhvse4FmPP31kj5s6r+Q7OdNyDowg0Q0r4LNvVMz+6MpzSS4Z9ReVCxZ8bYgGOutYdhXWRPZ8TW08czPllF1iUmSaOmqGmbjNKKB2UCcR6AWppszEjQZwlZy3RClLQANtsr0Ga3DgzFlXVpGbNVlA6G1bdkxDY6q1JRjWRS0P++UXR5P99tNMeWja2cDEeqbZomZmWyZGB7wrMzIDvIgTw2YUBwZDyiu+8j/kbCtI7AXDsXQJe75R+nuTTJZX5DI7DtuOygaf8h4WsIpJBv5qBFb7YUITHOf9hHPtlJWv2M/ku185ZGWwS4NdCm6lWZbLN7uKmCYgTUAaj3SH64hcMmi95xN7ADztpp2H+cltPPH6GTfs1Lzh3gf5xs950tE9cFAXKzFCMhbBI14/uiKCGKMLHJyWiNak/AFNWhYZGZwhJCFe2/8hqM6odXrlbKOjSaM9itje1+mCQxq12g/alweFixkbDGZCWobdeWWIedjkMtg0ly3Zxe1gGH/ZtALWTlyxOK6NGh4aG8EkdcMwl59p9Y9h0CzLqjVNZSK1UeCaGL2VzK+31+mbGuvmgIb10nCw90mDueIGp1Uy1rFeq8+0xqBF6nVh2lgx5cqVn2uQSoy5q0WYrEkamlix55XHWviaRVdcHBy+tb1fVpHd9PddtlPyEZMrA11SeYUzrfd+Yo9bT085tRWVPmyICM96/Gne/fGjLQ9TSrraPUREPAmnJSKQMmgBmE4zm2QE22l6ZTpIGcTEq4Gn+LxiKwyK7jbkjlAu06oYCMbQ5VOnuAEcRLQ/XAlZfOjHwDWMmMTe7yXKYJQX8f2adzXGKzOPG4CVP3QlkyklojMRYxRYklETxcsGrUwqJpMGP63sXFpbT20UOAtglVvxq3qoYWW4sOGx2ZQYR9HChVGm1QNWEkD9/kvHsMNiUuy7ivr94eIzLgeb5Oii8lhtLglXWfW+nwFr1Tl854itRVrJtkmDyZ9tswK+U22hdIPWkHCFifh7PrnHnbfsXuqvP+bi6Tfv8vOvve9oh6djoiy1T4CkmEfJ8gZlQKLTDCA67SRmyUN0Rrtco/LQNJCMITlL21qMcez7mtp69u2EhZ1gSZyPM92OY4Z5tfE+vpCKaFHWgGv8AVVeJ2T6OHcPM4D1/Ev+vU5sNqALdGLzmJFoCWgYhrQZ3ChiLh2L39bEdEwygDiXea1eYHp5b0MvVnUgLuFcYOo8M9ex41p2rXp9zW1DWTZ70AYei4EUCSI6VI7Ns59lpGdQw9v+nS/aN+k7ff0255GYtbwX/Wbo0R9tUinXU3/hKTzYKrketPa8Zlplw1MbLefbKUtfsd9WLJtale/7DmkMbmFw+0q+Vwsl390qYZcR20RM43Np2EHnVcJziLika0xKiXs+scfTbtqC1mHjzptPsWgDHzm7PLoHTRFdqZyJzKhlV4p61ZKg5KZ4Fe5JT86n0c7ETMoHMHmzD3kUJYQh22qiLihYpXUytxDj45b4gVf7wnNtSABMXsIwHg9as/LNH8R1Etn2z3GxWHP+LI+Plm1WkqpB5GjKQ6Q8VgKTMGa9a+hMPLAk3NwT2SviS3k3FndmYW0c/X95fcpj9ZumR699efyHy3pjKvKIchsy37JgddPrfeFrdQrxjqar6NqcYfWarEy8t9l+pivnXs60fERyE4kQSFcy0/rI2RWLNvD0m7egddgoWel7PrHH7Y87mkWuKSbERPLFWMtFgGog5yVGsAZSUjmhCBIMzqlNSzK68EGSWhAnA9GNtvpMKyobmPoJO071eGWgep6afvPLeGSkzLyVq3UJkz9QFb4HKwAkf4+Bk+kYAKtXaEfJgtOB7ym6sYr1ZRn6uKWLWDqIyms5GzEmqj6rcFGX8z4YtBPpEtZFnMsEvFPyfb0cXC8FiyJeR5kHcOk3J6VhTjBIGpoX/e8b2qKAZzDkW3vdM4f4cMA1/vcikxh3CEuGtecn6tzQKeneekuzqgkr5bHcvsE04BaCWyr5Xi1Tr343ReLQ+pxhBZLP+/AOEZcEWvdkEn4LWoePp+es9J6P7/HFz7j5aB40RVI0ClxJkBgVuAqhmX3X6byCmYiKT0EdOyWSjOqVQNdskUEsNJZgEstWu4iVCezYtneB6GxDl1z2Dy/KeFkDq1KaQHHotD3Y6HZrQ+81LkMbP2adVRcdjMwElaBOubslvYfVOHrf84wNYy91l/8OayJiMqdlVHB7WW/DiIg3okS5M7H/e0qXNebyt6jO15T9uWtnUyKKKDlu6B0wSjkYCP0W7jHRPp43BPqMzpSxHdXd98cz5tbGSy/WgCtntz6a7As2+LwvuopV51SL5S1hZZGVRRrBrnTG1a5Gmqw2YfNQdK/JKtVACEPVcIi4LNC6cwtah47H7dTcuDvhPZ84eq1WiuqPpQ200GdZB3YUnUFEsG0EMRibsK2q5UOnq7BMB7ETklWnycY6lq7ivNdMa2Y7IkI0LVXmm2D4EPY8CkMHsZQx6nuuRNok298MdsJ630H+8BaCnqHrlaOUQyU2N8wUos9kEWf5gDpRq2NjUvYcy0T65bwF42wta7TM6DYcs6yNyZSsqP8bRqVjftH0NTHQjlXxo+PtdVS5n7sZNhsOGgn9a1hejx7YNvi1YVGsDAsq8mjOKmiGtWx1QYXvHKE10BgFrLxcZdwtVBeHXBp2hbZQXSFxoDOuaPfwnk+c53Hziht2t+M7jyTuvHm3l4ocSaSk2VTKJWISBacYSdHq5zZaBavgSDAo5gUk2n6UR4LkGUXRxzGGGITG1cQw+G4tXE1EVHdkWuWINjpgw1D0uhK+EMAT8VTG912r8Ybiumx6MbEXNRIH0BsDnC0e8VlIOt5UrSMwOgLUiWXHNMxNy8yOxnmKuPRy13PmiSgFLFXDl9fBRxVhYqALwxN1fba18RpJpMml7DRrp9pkwUKH2mDXo07peCZwnNVWBKbiCdmyZ3PUp2xQWivT0Wy3A+U0k/rbl5nC/U7Lwv0mq927TLp3BrMwmlk1QrUAk8l3t8wdw1XQoeg267I6ryDlQ59pHXbT9yVnWtvS8JHHnbfs8suv/7CWcHKZl/cDIsU0ZFwpabotSWcUs2I+2aC8lo+IEYxVRbw1enVMFozTKyRGiI1u9Vm1FftVTUzC1E6UmLXqdrp54m8SzMD6zxggQie5M1bEoZl7GRPVFRCM9FnJsGg0UYxchv1+BbAGHZO6RORSKOuknFFyHJN6Mv5I34c0kOYRdTFdoRq38ppcDLSMpP71weTOHihPKAZMq6M4+bUcG/71j8No9RcGta8uz1FGnoZxorXHGr0YJdPyydBGq7esdve+bNQxWdqAZlmZeLdrpLsa/IlPmmGVxlFfGqahoXSIuCTQet8n93npUTsWPAbizpt3Od94Pn6u4dYz06N50P7qVJYUCERDKuVESio+LTOKIhAjxqqVDaAqeq9yB3UsUJGpBEjWEJKjSXBOEk2lp8zEevWpsjVGkoo2H2I0pYyzdMlSxUBn9EM7NdmHXnQTz7DUVQ3r2uTynJz0c3O6ldr3+/+molnJVHSTT7+WDGglMKVjatqcaan0oXIBcZGUB6ePKorDQxcsrbEs81YglwLLfGSDdmrdsaK8ht4YnBlkCZ3kJRjiCUi/c/Kg0J2Epl9QAlwIaFLGidYfo3dxKBY00fZZ1sLXLLuKRVvRNBVh5aBVpbtpBbfIHFZDn125VcItM481ni/svJaGpWMYAqmA1yHiEYPWXuP59H7LHTccTQfssRRPv1mX2L774+ePDrQ2onQUicN4DJ1+kpNXwakAtF47iiKotYB6p0Nu3zv1l4+VZF7J0ZiE98qfTSpPFy0rqwR3ceccl4ljECug5aNlYnXfX9nBWAZ4VXOVeZb8oTS5VCwEfylFByK5qMxjD1jTsnmZRCfqFaHApsr4qe2obMC4Uh4egctDfuFSNJm4tlT5A69/h9O/t8+0hk5pCZ8bDRGhJjubSui7pWXkpohtx2UilGaHCm0rwsGNChle13EUqUWJIi0pavexz3vMavfBH0t6o8l+SUWrX5u2eGUNEocixUkxFh/wnG1doUzrvvt1t9wd129B65FGGXn6wKf3+SJuOtoHH/NbpaNYpBBhpJvOKnl80KzLBKQ1WCA2Ss4nMyzIsNkRIllDdI4UhZXTGbeUhOAMlQ14YwbyeU1zNRKTilFyHR2mtsS+G2lNUuvhfKBlrrDsUbSYvpwBRtuUw8BloYBVZdAyWcg5zaBVRnrKiI0xEW8vP9MaLGlyaTjyiG+Nza/HOucXudC2ppj8lZ+1RFaifFiTeTGbP7I10ObXaTwDWmYI4cLZ0LXFJCPQKmaMw3GMtkRnX6zeG6sMQWdvrLIouCjetUxkKA2LV1YXwMeBw8raLGLquawrxml98NMKWk+6fjtz+EjjplMTZpXtX8Mjj4cCLujHe4iZ1g6x128lb3X0JyQkGJIY3VotynmJF7oEaWJYJaGtA523NJXH2UjtvJYdmS8yXNhFK2DWGkttdHNx4cQ2SWQrnWYSEqhSWBvxMRL7zKkm9DxWAayJDKNlURp0U49nJ5eIO7ZlYnVwusvZ1uW97sW9VIghz24GqzyeifikjqX9MeXycByGRDShz0hjkjWw67PWEf+m15WIpZR9sR9xKjzfuAQ8mHcswl/93iqVFWB5yWo35Xw7YW81oW0qQmP7IWjTsl4SrhS43FJtv+0q5LLQqybLZwLeB5L3A6dVysMrJXnYZlqXHiLCHdfPrxxowcHAlWSQQpQOVrnSdQJWO42q3cozim2+2jeAyaLTWnQuzek4SJufMpQ1WEZn/lISJGuVUkrENeCSnn6rTOgV1pUEVskxyUZ0OsqiUcqe8YKHujg+SMxtfO2M6qjxmNwWqkQuHwsRrxY1xsQse7j88rC/xVGmFZUX0gMZfrxkWOXeiNaXw/cNURJdMkzSaOC5+JRJ2eQ9GqhemwAYdFdwMeua2FvXjBdhFMuZsk1n6bO0IRj1ee9ErWaCDtqbjt7myOTFwRJSnsIYJjF0MiOsARUhaHZVpjkOGY8YtO69f8HpqePMfDsofSlxxw1zPvjp/Sv7JJvARSCJfpST96qEF6MdRdCfDRaT3UxdTsuMN3l3YlLvc9GV8snYnFEITRS8tYTaqx2LiQQXEElaguVso2RaIoloBJ9Lgc7Y3rUgIpnDUbI55A+hoXhkDWEYabI2wsqI3E42Z2IlO1N7mNpohohNY4ODSwqJjJZaDNt4RBKdyeVw1tKtv00K7n1ZaBIQldtKsS/TTNJdiSamXEYrJ1hsqSHr1ChdVM8F0wE5xhlr6ciuYp0zrAln/ZxzfsrZbsZep2Z+i0bJ97TSIWjbjAz9Mo/lVgnXZNX7SqUNptEMS7ohw6LrNMvyviffU9Ku8WHjkkBrS8Jfejzp+jmvfPcnr/wTPVzGJX4Y9REhpYRZmT4LsiYDmANNE1Im58sWa6NjNUDKBH6wkej0eYyJ4OjJ5wJcjoiPyn/5DC7LUPXlz0Q8WNiPE/WdwlPnxOmgjlmXLFFC3tqjg79dCmvZFuRuZJZVFNmDLbKHy8y01laIRfrFuCEvbS1Z58WeZrOBUaLMAMZs8xNl2JCjw806WF2WrR6k3LhgxVj+/VWs6LCsYs35OGU/Trjf7/CAn3O2m/FgM2PR1eytJn23UFrBZPLd9lue0uDg0KRe8V46hdJkPVbWZQ2ApdTEuCxMV6o8vPf+Bc+67eiM7B5r8aQbd2gO6YV92VGAi0HD1XcURbTsizLY2Vgl542hJ6fVmjl7y+t2d2KTyXmjBH1KghcwI2LZJsEIGRgiJinARRFMtnPyWTjaBIfLpV6THDZqVjRsqEl5i/SmPinm0kY/xF2KgHYLbS4Vdff0EL1zaFauk5Xxl/c60/Na/cLWlIfOk+CKV/xoVOnhooz8wOD1XmYKlROLfdNCrZKH3w1F6DXqBhawKtMK4zX258Nsjcc6301YdHXeV2j7bqHJOqzegXTk927WBqJjXmunJWAvb8jlYcpOpT1gpaiAdchs6xGD1oceWPBln7HdJn2p8aSrzQUmLTkAyl7QXg+T0uB8WjqMMaqdDfTKeeMTEo2O/XSZnM8WuiHq7GKKynXFyhArNdqLSVTIaSPRBqwkUsm6SLQB4mjuzyfNwIqGa5V0U/RU2gu0RTr3aOnE0Ylu4Im0VBJZie6asSKElFglsjuFulIMXFLG9MuUaY3Lw3ILwWCMdln9aLSnjNKUUnGtUSEbkwW9ONWqPU9S7doqOc2sUiLieylEJ7oMhLSelRYHjjLMrtugda9lEys+1e2yDDWfand4sJmx1054cDGjbS3dfo2s1OPdLbIma1kI+OxAmsn3nnhfdpplrdrecia17YW6rEsALLgE0OpC2pLwlxFPOo7SepRx6WBqvoqL5FEKIblsICjKXUmnngvJGZIVrEnYWgtKW6MWv1aUD0oQrSElBcAw0iMlN7T2kxl9kEbKbyOWNpcGrdVTshpN/Mc8iqJzdJnDEeV5OgkEI71v/FQ8XdZt2ZT6TGwRB5/5YUvNUb2++neoMJecadFv5EkouF+sDOz/zqQvSJFEhCR00eoOysxt6Shi6mUQm8PVQYYu7Ngbq2RZSra7fhnvItQ82M3Z9zUPNjPO512FbTO4j0ozMvPrhnlC242yq7IsuGzVKdIG73OGlUHqMgELLlERfywfvEdJPP662dFu5Dls5IwrxVHXST9ZOcPKGVd2Q6XMKBpD2exTWSXn+xnFCCDESiULscpr4YOAS6RaiCFinZoTWjFEF/KqLM26osmllFVwO9dNmRivnuS2Ymo02ypiUhhGUUqpV0ddOLpjWqxE5tL0tsxAv6fvwTDnfJyqgV1wGVQ4cJvOI4mSaUlS2YO+BoZgEsFqJ1FMHHRZolXpmJgvZLyWfwnPwC+Ou41eFHCr5OjyDKd2XqsLZkDHbhvlVjqExWZmGSoebGcsfcW51WRk5FdBJ9h9m8l2zbBMC3aZSfeWXvFuVwGz6pAmQNNqSdh2pK7TkrDz6xzWJmAdUqMFlwha20zr0qOyhidcNzueJy+lopg+40rkWUTQjIsROR8jYgVbxH95tCda039Qk6iGK2tBiV6UB3NJv64iwUVCEIxJ+Giy5XGksrHvMrbR0hiHT5baeJrodOTGeBZZy1WyLVjnhmyWPVQjD/ZKAnXZz5g/uPf7XfbChLPdjH1f03oHfgCHSw0JGbTydqPktDxEEq3Rj1i0w2vrTFzXsOW/pTQmXM6Q1EBQtVvLUKlUQyIT6zEkJqbrX5cybF2i/M0xCU3U17WLOlbkk2HhaxqvNjN7TU0XLKtljW8yf7WwuRRUR1vbgluoeLRfTNFudAqbgLTdoMXquoF07+UNlwdYcAmgZY1w2xUaQXmsxLFnqqOOItGsLciALEQt4z6dgpoxMmyvbqUgm25oDmTXCO2gSRL9XjLaSQuaTZk8LhOtYIwhRgWvZAWbpN8k04rFRx3UbYylSzpL5/KH86AYE+xT0/Vq+FImdcmyFybsh0k/RxeCyWXdZSri4+gWcrblDVEgBJ0lLBunrdHX3koiiWrYoBj16eMV37AOizUqUi0A5kykiQ4jkYlx/aD5GMRLaemj6rvKei8fLaug84XLrtI1cd6yaipC9sTSZRQGu8rDz0XWkMd0TEcvbTDdqFPY6jYd8WFNPHrUgAWXAFp/4YueirNHPBb/GItjzVRHA9Zj1Xz5vmqycqnoTa+ctz4iCUxtkQShLVuDDaFSoaEuPxVCrVxWqFVtHitDqhPBJUJlEZsQGzFW5yStTZjscVWysNoFqixLmFidaxwb69H/FevjMGPl/ZhDikl0GYN3PLCasWhqmmWFNLox5nLCdNpdNY1gspgsWEvwCti+sxirjqbGDBlWWeh60cfNNjflZ4vBoM3c4EO9HiHPQIYk+GAJWfDqg9poe2+J3pB89sLqBLcy2aUhG/l1qnRX1wYtBW2XsrFfUHnDIpPuTYu0nQJW0yhgFfI9ZS5rk7+6BMCCSwCtv/Hlz7ykJ9rGENeMrc/FMi6AYDJwjbb7tB4D2H5PoCGZpNyWDFkGEV3yELKVc0CzLas8lzorGKKLiDGEvMHGSKKzCl6tz7ouG6hs1ZeRMGi/QMEosW7xUn5GYJ3jiQYfLPsr5W3SyqqdyuH2KVw0jEctfbx+4BGIrZCSySR85uyCvtbGDL5bBwlOx6E/M/ysoNmagtj6ccQR+R+y5CLmTmbRjkVvVJbRGcjjWabJK+sbFYwaD3aZFe6rgXR3K7WXsU3uEnYR81DjOUU4GtOFGqxLBCy4jG0827j0+HNf8JTjPoSHnlMs5LzN232iU3I+6XiG3idMZ5BoiZVmXNFBqMDWWgL2a+etarySHbKv5BKx0q5XtCqtCHkFFwJi1RJZDJhsj2yyOrN80AuRHqOow0IGsBLlMz025UtRBt1RXiJqLzPTsi2ArmMr247AEPPfmKqoUwS26MIUpNcOshz4CLTGTV/yqI/ke/27hp8rv5+yTowkEKQfLSplsHj92pZRHK/WyCbk8q8dwMp4JdzLnkK3Ckina+xV5T4i3ZtWu4RF3lBI9+JGehnl4GZsQeuxHBcBruH7Kbufpn7kJ0V1ZxIfkc4iMRGdwXRqbROdECrNqkKlkohoVaSaLIR8n6x2HZPRrKzfjJNBK+WZwGQymAlw0Ac9kX95xE2NifUxHuWft3l+TslldSm4nHCrVFY3IkkIXea18t8YnVVwdsPfdqA8Pv8tsvFnkv9EJGWH1OHHN39PYn7o0t1NQ2eTiA6/56aBCXpvWzSDagexqC32yG0cJA2NR0LErPzgidV2mqEfRLqXkvCIYwtaj/U4ALhKqQjovRGSB7FBl8Nmkt6gingT1AFUgkF0F0WWRWg30ViIIen3csmYbO60FdAqIGWz2l5rO8WjEZBt6hMkyqBID7JmE7P+g/lnyB9YnxeIZnL5csJ4Xfqqzq+5wDYM0pCgf5uE0d8m0idV/Z9U/o6D3qb+j4ALgLg8Rhq0YmW0iFGTgKRApeClNwkpZ1fZWsarut2uIiYU0FJ1u2mzvUyXV9jHOAKqA0j3/hiPLsvSv/WIHmgb29jGNq5GbNuA29jGNk5UbEFrG9vYxomKLWhtYxvbOFGxBa1tbGMbJyoecfdQRN4KrK7AsRxl3Ah86rgP4mFimlJ69nEfxDa2cdLiUiQPq5TSC478SI4wROSPT8IxHvcxbGMbJzG25eE2trGNExVb0NrGNrZxouJSQOtfH/lRHH1sj3Eb23iUxlYRv41tbONExbY83MY2tnGi4hGBloh8o4i8WUTeIiKvEpHPvlIHdqkhIl8uIu8SkXtE5PuO+3g2Q0SeKCKvEJG3i8jbROS7j/uYtrGNkxSPqDwUkc8H3pFSekBEvgL4gZTS51yxo3uEISIWeDfwUuBDwGuBr08pvf1YD2wUInIbcFtK6fUicgp4HfA/XEvHuI1tXMvxiDKtlNKrUkoP5P99NXD70R/SZcWLgHtSSu9LKbXAy4E/dczHtBYppY+mlF6fvz4PvAN4wvEe1Ta2cXLicjitbwf+41EdyBHFE4D7Rv//Ia5hQBCRJwPPBf7omA9lG9s4MXFJJoAi8sUoaH3h0R7OYydEZBf4D8BfSSmdu8SH2bZ+jyYuy2/5pearr/z7IKKr38hbk8Qg1oC1YAziHNh87xw4S6orcJY4qUgTS6wsYWbVXXZqCJVaYvtJ3qpUC9GqKWMsxoy23NLwPZf6f1tzk42og2reFVCcUSXoZh8JjL6n5ovGp37Dj/Hwhy//3od9Lx4WtETkLwHfkf/3T6JzfT8OfEVK6dOX8PJfyfgw8MTR/9+ev3dNhYhUKGD9bErpl477eLZxDUcxgh8DlrWICFTVAFzOIcbApCZVClpxWpMqQ5w5wkS9/Lu5IVaCn+rWpFgJYZKBqspglQEKg1pi2wQWkotgE1IVz/7Bfx+yZ39El2cEIXiD5OUZMvak96LusZ1ucbKNgqE9pIPsw4JWSumHgR/W10/uAH4J+KaU0rsfyWt/leK1wJ0i8hQUrL4O+IbjPaT1EBEB/g3a0Pinx30827iGYwRYJbvCiGZTZsiqxFmoK7CWNK1JlSXVjjB3RGfwc5tBSvCznF3Ny6o3CNOUffwTyWWAqtRsXqqIcbrirao91kYqG7BGV5mVbUchGrpgiNHQeru2pix6A23eMdnpPgHxulDEdDljK2u3DxGPtDz8u8ANwI/oZw9/LQ0mp5S8iPxPwO8AFviJlNLbjvmwNuMLgG8C3iIib8zf+1sppd86vkM6fLQ+8qtv/DC/9ZaPcmZW8ewnnOHrXnQHu5PtuoErEjnD6gHLWsRaXYpYAMu5vhRMtSNOHbGy+Jnrs6sw0YyqmytohRmEOml2NU15vVsElxCbsHVQoKoCzkacDcwqT2UDE+txJuLyRuuI4KOhi1Y3VXtHFwytd3SdJQaDd5bkTc7iDOS1bWV7UaxY20T0kC/JVhF/ouOqvnmfOLfi6/71q3nfp/a54/o5ISY+/OCS63dq/upL7+IbXnQHZnMZ38mIa4vTOiDDKmWg2CGrkrqGypEqR5pphhXmDj+1xNrQ7RhCLXRzCFMFLT/XjCrMErGOUCXM1CM2Udce5wKVDUwrT20DM9cxdy21CZyqVlQSmZgOl3dQxqT7FJtYsQwVTXQsfEUbHUtfZQCzrDpHCIaudfhOAUyWFmkEtxCqPS0T3/JPvufyOa1tbAPg7LLjm3/iNXzs3Iof/+YX8CV334yI8IZ7H+Af/vY7+du/8lZ+880f5e//6WfztJuukWW0JzEuUhL2gFW5gWyvHGlSkSq7ll2FqRBqQzfTfZN+roAVJ2SwSsRZgDpiay37nFOgmji97VQttfGcqhpmtqM2nl3bUElgmtcXGYl00dElSxM9E9PRxIrKBJrgcBKpTKCLFmsirbe6qNakfgM3yRBbbQDIIdFom2md7Lhqb953/fTr+M/v/Dg/8a0v5MV33rR+ECnx8tfexz/4zXew8oH/8XOfxDd97pN46skBr2sj03qoDKtyA+E+nfT8VekM+nml2dWuwU+UaPc7ylv5ncxb1YmwE6GKuJmnqj2VC+xOGyoT2a0bprZjaj3XVUsmpuO0WzG3DVPxzE2DlUglgZBLuVWqiUlYxAmrWNEkx56f0CXLvp/QREcbLXvdhC5all3Foq1ovWO5XxNXDrNnqc4Z7Are8YPbTGsbRxCveNcn+O23fYy//mXPuACwAESEr3/RHXzp3bfwj37nnfzUH36Q//v/+QBPu2mHu287zY27E3YmFitCZQ3TynLdvOKmUxNuf9ycO66fU7vH8BjsZocwc1ZrHcKq7gn3NNGyMM4q4sQRppZuV7uD7a7BT7Uc9HO0JNyJxEkiTSJ2x2NdYD5tmdYdU+c5lcHqTLViZltmtuN6t8/EdJwyK6amYyodlXhs5rFCMkQMdQy0yWIkaRaWLJUEumSZ2Y5lqPDJMrUdbXQsXI0zU1Y+4L2hTULshFiL7oU8RGxBaxsPGasu8AO/9jaedtMO3/Hipz7kz950asL/+Wc/m7/2J57BL7/hw7z2Aw/wpg89yIP7Hfut52LLhp0Rnn7zLi991i38mefdzlNu3LkCf8k1GrL+QRUjA2BZO+KyjJaEuURMlVXAmljCxOitVpAKUyFMwc9GRPskIpNAPemoXGBn0jKrOmau43S1YmY7rqsW7NqGuW04Y5dMpeW0XVGJpyJQiy7wDQgdjkAkZKCNeDBg8mLWLoNXATAngWWoMSR8NIgk9qua4C3ejnRfh4gtaG3jIeNnXv1BPvjpBT/75z/n0NnQzaenfOdLnsZ3vmT9+yklfEwsu8CD+x0fP7/ivvsX3POJPd5w74P88Cvu4f/3++/l+77imXz7Fz4FkRNJ6h8+LiZpsHYAqpJhTWpSXZHqijhXHqvbdYSpwU8N7a7yV92pTLjPEn5HO4Kyo6XgZOI5PVtR28CZesXctey4luuqBXPb8ji3zymzYsc0XGcXVOKZSoclYSQSk77/LRldkqMS3/85pWycSkfAEJMQMJnzqljEmj0/YWI957sJK+9ISfCNJVaJjIkPGycKtETkB4C9lNI/FpG/B7wypfSfLvGxfgL4SuAT2wUTB8ei9fzof3kvX/j0G/mCp9942Y8nIlRWS8TT04o7bpjzwidf3//7x8+t+Lu/+lb+/m++g/d+co9/8Kc/89ELXBcDrHGGNZY0VIV4t8Rab2GaM6yJECaiRPsk81e5HKSOuAxYs7pjXnVMrGe3athxSrKX7OqUWXHKLtkxjfJXRKYjUIpolhWTIaSSYRmsRAICCayAHQlOQxIihn2ZUGVUamJFFy0TG1iYiNiU1fWHe+lOFGiNI6X0dy/zIX4S+CHgpy7/aB6d8bOvvpdP7bV895feeVWe75bTU370f3w+//C338WP/pf38tm3X8fXveiOq/LcVzUeLsMaj+RMapKzpEmtgDXJotFK8FOjhPusSBnAz5OC1iwic4+rAjsz5a9265ZT1Yqp9ZyuVuzYhl3bcL3bZ24abnB7zKVhxyjxrhlWyhmTHnPAKHDl+xKWRF26isSe+wLlv+amYRWrnI0pOn3K7TCpKvZdJNpEOqRc5poHLRH5fuBbgE+gw9Cvy9//SeA3Ukq/KCIfAH4O+ApUtvYXgP8DeDrwj1JKP7r5uCmlV+aB5W0cEMs28K9eqVnWOBu60iEi/PUvewZv+8hZ/u6vvY3n3HEdz7z19FV7/isam2AFw0hOkTRkHZb0koZaRaOzSvmrqcXPjUoadvI4zix3CCcJvxOVv5oGpvOWygVOTRtmrmOnajhTr1TK4FbMTcspq9nV3DScMkum0jEVTyWDDqvFEJNhlSraZOmwPRFfohJPLQFDpJagpP2oub2falamAmARa7pkmVrVghkTCSYdVlt6bTuXisjz0VGc56Bzjy98iB+/N6X0HOAP0CzqzwKfC/xvV/QgH6Xxc6/RLOsvf8nVybLGYY3wz772OexOHP/rr76NR60sR0wPWJg8Q9gPPVvNsCpHrF1fEsZaASvU6Oxgf6+ke6oTMom4OjCptCScOb1Nrac2nonxTE3H3DZMcmdQu4NKnI/BppSDLbYHrC45uuT6jAnIUggFrrkpEomuv+1Iq89jOiZGn8uZgJGkWC6AXJkxnqsdLwZ+OaW0ABCRX3uIny3/9hZgN3tVnReRRkSuSyk9eGUP9dETjdcs63Oecj0vesrVy7LGccPuhL/60rv427/yVn77rR/jKz7ztmM5jiOJi+mvjFnvEFZVn2mlyl1Ausfa4GeGdieT7rtKuod5ottVDsvsdFQTz2zS9RnWqVpLwsJfTYxnblrmpu35q6l01AyA1SUt/1apoksKVKtUEZIS6yZnYlUuBXtQEs/ceCoSU4HCrVdEqhTokuNB0zI1HbVR9b0xKYPW4V7OazrTeoTR5Ps4+rr8/7UOztdU/MIff4iPn2v4n//bq59ljePrXvhEnnnrKX7wt95B4w/ZWrrW4mIK9w3AYqxyz4BVSHd1aMiEe5E1TBS4wjTp4PMki0brQF17zbJcx8R5ahOY5AzLGVWpl6yqkoCVgYMKKH+lQKW3VaqG0nCkS7AkzbAIudPomUhgLokdI0zFMBWhFqGWqBwZEUvEXIYu+loHrVcC/4OIzLI18X933Af0aI/GB37kFffw3Duu4wuefsOxHouzhu9/2d186IElP/2HHzzWY7mkuEiGJZls7yUNVaVZVgGsaTWQ7lPtEvpZVrlPRYWjM5U1hClKuk8DduaZTjrmdcdO3fZzgzOrHcOJyeWZeCoJ1OJ77qlkWDFnUgWoVqliFesMWI7ARkmIjvXsSMfceHZMZEcMc7FMxDEVy0QMlUCVwbHczCHLwc24pkEr2xL/PPAm1CX1tUf12CLyc8AfAs8QkQ+JyLcf1WOf5Hj5a+7jI2dX/NWX3nVNyA1efOdNvPjOG/mhV9zD2eUhDZeuhSimfWKGrmDlVOFeVzr0XNX69aTWknBak2Y1cVoRZhVh5ggzJd67uaGbi5Lvc+0WhnkiziNpFnDTjum0Y3fasFO37FRtHnT2TDKX5YyO4FRGwUqznmwtg9CyAVYZsAqvBWDRxyg82I5p2JGWufHMJXFKDHNTMZGKiTgqsdiNuq+XS2TmPSV0IO2QGHbNl00ppR8EfvCA73/r6Osnj77+SZSIv+DfNn7/64/sIB8lseoCP/yKe3jRk6/nC49Al3VU8Te+/Jl85b/8r/zI79/D3/yKu4/7cB4+RmB/gcK9EO/Wqmmf069TVrmXW6ytEu4TUYfROjuLZvI91jpLmOqImQTqOlA77cYNIBWYmICTgB2VZUXKsK6nMliSloeZzyr3Y8AyOUsqHcKpeKYSmEpiKkIlBv3voS94m8AFHNqa5poHrW1cvfi3r/oAnzjf8C++/rnXRJZV4tlPOMOfed4T+In/+n6++vm38/SbTx33IV08coZ1oGlfXYGxvWCU3B3sx3KmatoX5k4Bayq0O2aNdPez0SzhrsdOQp9hjecIp7ZjZjuqPOA8MV65LDOUhnakp4oY2kSvvyqSBuWtlE8sHcKKwI5Rx4e5eKYSmYswFUuVbwAhRUJKBBJdUnJfO4/KjflkSEmIUZAoh20eXtvl4TauXjy4aPnhV9zDFz/jJj73qcfLZR0Uf+tP3s28dvytX37rtSuBGPm4b5r2rbmMbhLuObOKlSWWWcKp4LNxX+jvIU6y2j3bytS1p3aeqRuVgRIzf6TjN0ZSP9BcSdDv5azpoCgEuyGDVL6VknCaJQtTCdQSqQQsQsmxFKwikUQk0qWYy09Dm6yO+GTjQB+NZlgJbZkdIragtQ0AfvgV97DXeL7vGi2/btyd8H1f8Uxe8/77+alrkZTPmakY0duIcKfKgFVV2aUh30qHcOqUdJ9l0j0DVk+6TyFM1QsrTBNpGpFZYJKdGuZ1x7xqey2WOotqWVht3CwpdwxT380D1u4L11WLDkkX7qpII3akZSeXhROBWgSzkZmPAatLKWdZdsi0osUnq2M+SdSK+dHCaW3jysd99y/4t6/6IF/1vNt5xq3Xbun1tS94Ir/79o/z93/z7Tz7Cad5/pOOR0N2QWyUhNoVrEbzg1nOUFckpy4NsXbgDGFiSVUWjM50Q47aI6tbQ7czHs+JpGnE7nRUVWA+6ZhVCljFXXSSuaxSFjoTezGplold3zUsYtAy9FwGo8dhR2T9AHqRqWgfsRbBQE+2R6LOKKZER2CVIqsE+1nntUoVi5B9toLFB/WRlyCHHpjeZlrb4P/8nXdhDHzvn3jGcR/KQ4Yxwv/1Nc/h8dfN+K6feT333b847kO6MLIGa0y4M1K4673RLTmV6QErFuCqB9Jdt+WMSfeE1FEtkUcuo7UN1EYV5gWw9OvY67BMBrG6lzuEvgQs2VeVea6aQE1YU8mX0Z5yM0B1AO0ZUiKkUZYFdL3ua+CzmqhC1RAFoiCRQ3cPt6D1GI833vcgv/6mj/AdL34qt56ZHvfhPGycmVf82De/gNZHvvHH/4iPnV0d9yENyyaK/ipnVVJVSF2rlGGS5QyzijCvCXOVNHQ7Dj+3dLuqdO92hG5XMyy/A35HLWbiTkDmnnresjNrODVt2K0bdlzLbnZrmNkui0i7Pruamq5Xv6+P7AweWT1XJT7rrTpOGS0BT0nHPItGp/lWkTKPNURAM6uOQJM8qxRYpcR+NCyi43yccS5OOR9m7IUJ+6Fm2amDacprxow/XPNnC1qP4Ugp8Q9+6x3cuFvznS952nEfzqHjrltO8W+/7UV8eq/hm/7NH/Hgoj3uQ1rflDMi3jclDbF2pNroDOHIVqZkWEX1rl8r6R7riNQBVwWqKlC7QG1DHjj2axlWKQk3uaySZRXJQgGsqnwvl32VRB25GZH5/Y2kpeAB2BJT6m8diVVKNAmaZNkvZWGcsIg1TXQ0weGjIQSjmVaCi/QFLnypj/SN28aJit9/1yd5zfvv57u/5M4TtwLsOU+8jh/7lhfwwU8v+I6f+mNW3TGO+RT9VV8SmjVJQ5q4vkOYakMopWAGLSXdycT7yBOrTsTeEytQ1Z5p5Zlnx9GpU2nDZMRljQFr0g8ne+rc/esdGHLJaEnUrINVLVGzqY3bJliUV1x9thSsxoC1nxz7qWI/TvrbItYsQ80quLwfMZeHgW33cBsPHTEm/uFvv5Mn3TA/sZ5Vn/+0G/mnX/vZ/PEHH+B/+/XjW2/ZC0fHXcIiZ6irfhdhmOp6r6Jy9zOhmxWFe/Z0n6rSXT2xEkwiZhqoJ55p3fUWyWU8R2/t2hD03LT9MooiURj4qWFsp4hFTc6ixtnUZozxJCT9/wi0KdGmRJcUrEpJeD5WnI81D8a53sKcs37OWT/jvJ+w8DVdsMRgEZ+7h1vQ2sZDxa+96SO882Pn+d4/8Qwqe3JPg6/8rMfznV/0NH7uNffxO2/72PEchMmODaPtOcmanHEZYqXke6wkE+5CqBiU7pUuK9XV9Hnbc15HLy5ibcSaSJW3O5eRHNcT47kELIPQWUBaiWeszSpC0iJvGM8cmtHX4yjgtPl1SOvf6/L3VklokmWVHPtxwirWrLLVcpMcbe4adlG3UMdQAOvw4tKTVRNs40jCh8g//8/v4Zm3nuIrT7LlS46/+tK7+K/3fJLv+w9v5kVPvp7H7dRX9wDGiyeKLmuUYcVJ9sHKkoYw0fX0oVIDv1ipJ1aYJaLTAehURWQSqSY+yxtapnkfYbGZ2XFNr3afmxabF6kOHb8WK+kCQ74SAcEgVOiW6DYZrCQOKrRDkn7sJ6CAN15UojY2li4Zzscpq1RxLk55MOywiDVn/Zxzfsq5bsrC16y8w3tD8gaT5Q7bTGsbF41ffsOHef+n9vmel951UjdCr0XtDP/kq5/D2WXHv/y9e676869vzNEsKzm9Raeke6wySNXDrTfwK7IGB6lKJJd0Pb2L/cbnysRe8T64NnjmVsvB0iUsncBafC734gWA1Y/qZEfSLt8ioj5aSfrvbf5b+fc2mf62SpZFdKySZT/VeosTzocZi1izCBP2wqTnspqgW6djNH1ZKOM07mFim2k9xqILkX/xe+/h2U84zZ941i3HfThHFs+49RRf84In8tOv/gDf8vlP4kk3XMU1ZGLA2DUH0lTlsZxa9xGOh5/7kZwqDz9XqQcuXIIqIlXEuoCzkcoFJs4ztb5XvM8yUE3Er7mBFh1WP9ycAWu8rxCJmDIgLXEYbk6Hy2HGljLFP35sZ3M+zPQ+TtkLUxahZhkqlqFiFSq6YOm8JQWBoPsO5RFwWlvQeozFL7/hw9x3/5If+JbPuKaGoo8i/upL7+JX3/gR/vH//938y69/7tV7YltU8K7vFsaJywZ+6jYaqjyWUw8dwljlLqFTwEqTCDZhJgHrQr9BZ+o8O67NK78aTjldoHrGLvsu4WYpuLlYordGFlTEme9LxnVQRnbRyD82ztYGK5uKB8OcVao46+d9hnW+m7IKjqWvWHbaOYydRbxBAhgP5pDOQ9vy8DEUPkR++BX38JlPOMN/+8ybj/twjjxuPj3lmz//Sfzmmz9yddXydpRlOUvKRHx0QnJCdNKT7es3XZ1VykFsAhcxJuFcxJlIbbU8rDOPVUSjE/G9pGEqbd8h7LVXF6m1invD+L6o1VssLbZ3Ly3/f9BNbZgdqzh4bxVZw3hUp9zaaGmjloU+KAlPUD/mInfYZlrbuCB+9Y0f4YOfXvBj3/yCR12WVeLPff5T+Dd/8H7+zX99Pz/w33/GVXlOGWmyYq27CcNM/bC6uaGbKX/lZ0VAOsqwJpFUZQ5rEjAuUk+6finF7mhl/Y5rOOOWnLHLvLJ+qYr2rMXazJQCMZeD9N5YACFZbB6NLvbHltRvh96MsnVnvDKsgF6bbJ9p7ccJTaw4G2asYsW+z8r3ULHXTWiCY9lVdN7iOwedYFrBNoJtwDaPjsUW2zii8CHyQ6+4h2fddpovvfvRl2WVuPXMlP/+OY/n3//xfXzPl97FmXl15Z+0bNJxViUOTrOsWA/ZVT9TuEm615phKYelPFbtdJPO1HmmtmPuVIs1N22fYRX9VcmsKsIFJSEofxVHXFX/tcQReEUCEXvAhWy8KqyUmP3C1rw9OmTQWsSaLmr2VTisZahYeV3O2gZL6y2+s0QvSGcwXhAPxieMv+DpD365L+1d2sZJi19/80d4/6f2+ctfcuejNssq8R0vfiqLNvDy1957dZ7Qmr5jGCuTV32Nu4QDYIV6IN01w1LS3YzGdKaZx1IRqUobdAu0ikZ3TFmo2o26hZsjO2HNTnkcCjau94Nv07AWrNxUX1X3PFUpA/djzaKUgfn7w3hOlRXvFU10PWCtgqPxjtY7Om8J3pA6g+lAOuWyTAum22Za28gRYuJf/t49PPPWU4+qjuHF4u7bTvOCJz2On3/tffyFL3rqFQfpNNJlhanLnlimX0ThsxbLzxLJoRYzdeavpgFrIy6DVe0Cu3kpxa5ruK5aMrMtZ9wyq90HX6u1sZyRrUxMhpC/bpOKSQth3uZtOuMN0SXbGm/aKQssSialvz8Q7+V31RvL0WR+q4mOfa981sLXLH1FExyLVoej29YRG4s0FrsS3FJwS3CrhFsdDrS2mdZjIH79TR/hfZ/c57u/5M5HhS7rMPG1L3wi7/vUPq/9wANX/smMIVkhOkNykr9Gb9Vwn4rS3Q0ZlrVaFlZuGISeWN/bJRfHhqJwLyVhuRXAKl5XNfFAFfw4xkS8fi153f3oliRnYYOdzLADcbCXaWLVm/op4T4m3vXWBKsupd4QvQFvkE4wneSuoZaGtt1mWttAdVn/1396N3ffdpov+4xbj/twrlq87LNu4+/9+tt5+WvuveILZ9PEqfp9MkgcysqvMMtK9xrCLGdYdcROAsaqrKF2Ogi9W7VMnOd0Jt13bcPjqgUT07FrV31ZOJemt5QxEvuB5xIGwSahI66tri9lYRhxVJaY5Q/D7yvB7ohJeg+sgNBFl/+9eMiLjuNg+pKwjY6Fr2hjkTdUNJ2jaSp8Z0kri1manGWBW4BbQrWI2OXhht63mdajPH7xdR/ig59e8Nf+xKND/X7YmNeO//45j+c33/JRzq2u7OqxZIuQdDRXuEa85wyrVuGo8leeug5Mq3XnhqLFmo0yLVW6t73baHFoGAPW2o1hCLqELquQtb2FJcbyiD4Lyz/bZ1bRrWVZJbvSjMvhk0oalkEBqwmuJ967kMn31iKtwTTaMTS5Y+iahGkjtjuc5mELWo/iWHWBf/Gf38Pz7rjuUanLerj4quffTuMj/+ntH7+iz5OqMrIjWf0+jOiESVLF+2QMWGULdLFLVsDarRp2bMuu1SzrlF0NoznFqYFMtBPWAWtkIWM3ACtmHqpkXTH7so+38WzGGLCKlisWB9K4Dl4+WpowAFWXb20h3ztL7Ay0BmkF29IDlm3AthHbRqQ9HGhty8NHcfzkqz7AR8+u+Kdf85xHfcfwoHjuE6/jCdfN+M03f5Q/87zbr9jzlKUUfqaaLD9T8j1M1WImzPPK+pnvCff5pKW2oV/5teNadqy6i552K+a2YW7atS5hz2Vl4JqMtuGU7CNCNuLLC1hzdzAwqOJNnkmsRvOJQCbZ8+OkTLiPAGucgfmo3FfJqpahYuFr2mjZ72raYFm2Fc2qIngDS4tpDG4huIVgl1DtQ7WMuEXELj1mdTjNwzbTepTGA/u6Euy/febNfN7Trr2VYFcjRIQ/+Zm38sr3fPKKbqdOToi2KN83VO95nlDqmCUNvtdgzbKJ33TkizWzJavyF0gYhnVeYVgRBnmxhN4MZbwmd/p6/qp0CuMaYBXnUp1TjBdV0gOj7uFwr1mXwedbF5V077xV4j0IqTWqycpCUtPmLKtLSsK3EekCckgjxy1oPUrjh19xD/uN5298+TOP+1CONV72WY+nC4nfvYIlohr7ZfK9mPrNEn6eSDPdnjOZdezOGk5PG66bLnncZMH1kwU3Tva5vl5wnVuo2t0tOGMXzM1IjzVybyj7Biti3upM3uwsWJG8Cac4MmQCPWlBVZatFu/4Ta2XyUD4cFHKS58sTR7PWYWKpa9YdBWLtmLVVnSNIy4csrS4fck3qPaSZll7kWov4PY7zLKD5nC22VvQehTGx86u+KlXf5A/c42vBLsa8dm3n+H2x834zTd/5Io9R6yL7QzESSbeJzoAbaaeulbX0Z26Zbdu2K0aTlWNEu6902i2mMmZ1OA26jOPlbfoZB5rKpEKFKzWxmtydy+DVekQ2rystTzuZtlpD3CEKHGgbKLPqlTp3ngVkK461WJ1rSOuHNIY7NKoHmsFbpmyLitiVwHTBKT10HZIty0PH7PxQ694DyklvvtL7jzuQzn2EBG+9O5beNV7P33FfOQH8l0G8n06eLsXDmsNsArh7poesPoB6DxLON6YMwassg2nANZ4UWqEtSyrlIelJNRZxSFz65+LAlzZ3bSUjL3b6bgTmT22kvSD0E1WvWun0BEaq4C1Mkq6r8Au9d6tIm6ZsKuAXXmk9QpY7eFK+C0R/yiL++5f8POvvY+vfeETeeL18+M+nGsiXnLXTfzkqz7Aa95/P190101H/vhlIYWf5y3Qs4TMAm7asTNrODNbsVO1XD/ZZ2Y7dq1KGioJzG3T81Y7phnd+7WScKfsHSSxYyTvHTRYhEDq/dpXydIWPVYm3ntveInsSIuRyFQuBIiDBqZN9t7ql1gkwecMq5DvS1+x39bsr2ra1hH2HNIa3L7B7eswdH1OO4X1XqTa0yzL7neYpkOWDTQtyW8zrcdk/NQffoCU4C998dOP+1Cumficp15PbQ2vfPcnr8jjR6flYVHBpyphqkhVBWb1+iIK3Us4kOzFp73qZwXjqCQ8OMMaA1aJkBJdImdAw5gNMGRYudycSked5xI3pRHjONAzPknvYlqI9y5YGn+AFmuV3RtW9C4OdqXEu2kz8d558IEUIoSt5OExF40P/OLrPsRLn3ULt52ZHffhXDMxrx0vfMrjeOV7rgxo+alyWWGaVPU+DUxnLfNJy5nJiuvqJaeqFafdam1FfTHwK46jhW8aL0+d5k7hXBK1CBVCJQZTNFfEXuLQIbQYWnQAupR7Fcpl1QTmpssbdxJd7gCOYyyN2IyiyyoZ1sLXnG8mLNuK5WKCXziksbjzqniv9sHtJ9wK6vNRy8L9gF10mMZjFivoPKlpoWlIYds9fMzF77zt4zyw6Pj6E7oS7ErGF915E+/++B4fPbs88sdOI6lDWV1fO1W5T0ebn/sV9Xm4ucgMLGVl/XjjswLWXAJzSUxFmIhhIo5KbG8j0+8bTAOXVexnLKnnxHTNve8Bq0SxmemSUz1XBrGSrRUniKJ8L+4NC1+z6GrtFDYVfqldQruftVhLBaxqUcj3iF1F7MpjVh2y6jTL6jrwXgFrC1qPvXj5a+7l9sfN+MKn33jch3LNReGy/uA9nzryx+6Hoh3gdAB6Uvk8+Kz34xVfMJReRtIgRejLN9+voC+ShqlYKiyVWAyjTCslQkq0ZenEyL3BZMX7sPNw2BpdYtB0Sa+aL49RxKVFCX8BYHUVq5VKG2RlsQuDXa7PFLplpFpE3DL0AlJpPNJ20HUk75XLCoGUti4Pj6n4+LkVr3rvp/maFzzxMTVjeNh45q2neNy84o8/cP+RP/ZYSCp11E3QzjNxClhOAkbSmlf7YAsTs/ZKS8JTZsVcPHMJnDLC3FjmUjGRiom4fglFJNIR8kZnXT/f5nEdUMAqXcfxgouQdAaxQ7foDN7ug3dWcW9YxJq9MMkLVqec7Wbc3+xw/3LOA4sZZ/dmdOcncLbCnTXUZ4X6LNRnE/W5xOR8pD4fqfY8bq/D7rfIokFWDawaLQsLeIW4zbQea/FfMsn80seAX9alhIjw3Dsex+vvffDIHztaSDY7kJqEMQkjWoaVUqxwRxElsoF+AcWYKK9RDdZUUP4KLQULWFkZu5Dm7c6jTGlMwOtzxA2yfSDSh/lBN7KhWR+MLs4Ne75mr5uogDSXhF2jOiyzMheQ7q5J2CZm0n1QvEsh3r2Wg6lkWCmS4taa5jEVr3z3J7np1IRnPsbFpA8Vz33idfzeOz/B2WXHmdnR2TAno15ZySasSZi8/RkUJHw21zMxdwlN6k37xlnWVDxz49kxSrgX/sog62BFpEuBVYprXFbLYOJXnCBg5O0+yvL6DdDZ273fAt27kdY00XG+m7IfFLAeWM1YNDV7+1Ml3VeW6mwm3c9DtZ+wK5icD5gmUS08Zum1U7hoEB+g7UhNozzWiMtKMa3Z4zxUbDOtR0GEmPiv93yKF99542NyMPqw8bwnPQ6AN9734NE+sMl4MHrpfeaCmuBUOZ5sP7NXwhJ7RfpUOiYSqEiDpCFnWGPACikSUiKgm6C7ESel/276W/FxLxlYi2GVXL6tb9FZxAmLoLbJi1hz3k8530052015YDXnwdWMs4sZ+6VLuLDYPTOM5uwn3CJRLSN2lXBZ7W7aMIhHO09qOwiRFKICVkyPCLDyy72Nkx5v/fBZHlx0vOQKCCcfTfFZt59BBF7/waN1M00CG1VZFmGqlqmJg51LCe0eprUh6OLYUCQNZuPjGVIkknKmpVlWmTPU59z4+VwyqgTC9tlYNwatzF+tklomL0LNvp+w52vO+wnn2yl7Xc1+UyvpvsqzhIs8mtN3B9UuWbuEeTynjOjkkpCugxh64p0CVgWwDknEb8vDR0G88t2fRIRt1/Bh4tS04hm3nOINR51pjSIlIQRDGyxGEs5kQ75Rx6509KYyHoRWwLIHPGZIpdQcAKtske9yp08dSYVu9Ahdsv1qsDKOU6QNZTnFIk76lV8PdnOWoWI/1DzYzFj5inOrCYtVrcPP5ytMY6jOD/Yy9Tk18av2s6ShidhFh3QRs2qRptPyr221JGy7nsta47EOCViwzbQeFfGq936aZ912mht2J8d9KNd8PPeOx/GGex8gHpL0PXQkvaUIMRpCvhVHhJDGCnXtJo4HlQ9Un6OZlX6tgNWXhin1JWHJsMalYRnjKfOHZctO2bwzlIdubW39eT9h4Wv22gn7bc2yyYC1cthsk2yXClhulTLhnjBtwvZKdyXeyWR7fx8CxAPKwkcAWLDNtE58xJh4y4fP8qef+4TjPpQTEc994nX83Gvu5X2f2ufpN+8eyWNK0u3IEoQUDMEbumCwxtAFS7RZN5WdFooCfryAYiz4jCkRJSpRJrF3uAopqcyhmPzl0rCY/XW59INCvltsSmsuDatY0SbL+TBT7ipMedBrhvVAM2ffq/7q7GJG21q6/VpFoyuhOmewDVTn6cvBek+7g24RMI3vMyx8UC1W22mnsO20U+i9AlbOtC4ltqB1wuP9n95nr/F85u1njvtQTkQ86/GnAXjnx84dGWgR0UwrQoqaWcU4jMiU8nBs6lfU8cNDCCGBEVW5k7RcLB0/Xaia6FIkoBY0pTQMmB6wCm8WUEuayMBtAX3H8HyYshcm7IUJ57opC19xtp1qdtVWLBe1zhHuq2jU5bEc22TSfZVwjS6jsF3UsZzRPKH4MEgbxor3TdL9EWZZsC0PT3y85UNnASWZt/Hwcectu1gjvOOj547sMftMK0oGMFmb6SuarbXVXge4hMbRTbuDObMiqJNDSms/o6XnYItc9hO2hXQfLWJtR0LSsXB030/Y6ybsd1oOLpqaZlURy1jOwvT8lVuUDmHK23MG0v1gwPJQSPdNPdYllIUltpnWCY83f+gss8ry9JuOKGt4lMfEWZ520w7v+Oj5o3vQVAArQRJSykAlicoGapNHeUygyjOI48WqEUOHySWipm0G6FJYW1WvPBZ0Cdpk+uHoAkZdXphaZBXFG2uVva+65DgbZixCzae7Hc53U/b8hPuXc5adY28xpVs50spiz1tsI7i9oUNYn0/YNpPuTVDh6KpT0WiTZQ0hDCXheEQnxCMBLNiC1omPt3z4QT7j8adxdps0Hzbuvu00r33/0Y3zHORQLJIQhtLQjaxoNqPorDqMlpnZJCaKApWFwc8q/3yX9Vfra+3XZRWRoVQsy1UXQXVYKmuYsJ+HnpvO4VvdS6jmfernrvOD6tTgMuFeVO4luxIfB6I9RAWsuE6+rwHWZcYWtE5whJh464fP8XUveuJxH8qJirtvO82vvvEjPLhouW5eX/4D5s6hJEASImDNkGXVJbvKKnhL1AxLhjX1Gn5tzKcajeAYBsBaJV0msZ90XnCRFe1lJGccIQ87L4Kuqv90t8O+n3B/M+dcM2XRVqpw7yxpz+nQ80qozgt2BfX5QYNV7Wt2ZRcdpg3gI9K0SIhZi+UVsLpWyfbCZV1Gp/Cg2ILWCY73fnKPZRe2fNYjjLtvUzL+HR89f6SbiorIVCRhTcTlWyV6v+m1HpKhE0tJvgp5XqEZWZdMvznakHrv9/GgcyHWV6nqM68SZWdhkxx7fsIy1jzQzln4mrPNlL3VRGcIFxW0BrdnVc6wgmpPTfuUdM8arIXH+Ji7hMpbaUkYe8eGonbvtViPcETnMLEFrRMcb84k/Gc+YQtajyTuvk3nM9/x0XNHu15NAJMQE7EZpNx4PdfYbQGDIarrgwAJbNZU9eAlgZAkzymmPAYkmb+yrGKdSXbX+2iNO4WlXNzzk550P9dOWflqAKzGQT/0rHYyhXS37QBYbhWwTc6uNgh3ipQhxPWSsGRYJY4gy4ItaJ3oeNfHzjFxhqfcuCXhH0nctDvhhp2ad37siDqIkoemDYhJWJuoSpZlQk/Kl1B3UMFiWFFhU6KT2Hu018V6OQ9UWyKkskzVcC5O6ZLjfJzm4eZJn1WVpapj074Hu1nfIbx/OVdJw36tHcLGUJ032EaHnt1icBrVTMv33UGz7IbsqvMqFG11NIcul4IpkTp/IYd1RIAFW9A60fGuj+/1LfxtHD5EhLtvO31kHUQtCxOY1JeHIoM9jeXg0igU4l0iIVnseJhahoypeLcXAem4U1iyrrX19Ul5rP0woQmOvW6incK2ZtFUtE11wXov044I95VmWGWRai9n2FC26/1YzrAhGr0CgAVb0DrR8e6Pnefzn/7Y3B59uXHXLaf4udfcS4zpSEwTez5r008rg9d4ryBkIEqRgFVfG+jnEwPSK9nbZLES+9GcLlnOxZmWfWF6AWD5OGRZZzudH7x/NWe/rVg2Nau9Cakx2D3bj+RU55W/qs8PQ8/VvqrbdY4wl4PZoaEn3McZVgGuTQ7riAELtqB1ouNj51Y845atf9alxF237LLsAh96YMkdN1zmqjUZbpIzLQWr2Ls5jH3hgV4IWgAJII4ysg6wSX8+Rv3Z0iEs5eAi1r17RMmwlqGmiZZVHstZdjr0vFrW+MbBnsuloOl3EdbndX6w3s+C0UZJd3zsXRrW+SvVYJHiUBJeJcCCLWid+LhrC1qXFHfm1+3dHz9/2aBVsqwkqRdtlSyrxAWdw0y2tyOJQtkGPfyMEJOaFSpguTzgPOmzqVIK6teGfV+zCi4PPdesyqacxsLK4vYNps1k+3JkK9Mk7DL2PljSetVfdX7oEHqv4BSzQ0PpEo4Bq39RrgxgwRa0TnzctXUqvaQoc4fv/sR5vvRyLaplVB6yzmlZWee0VAFfrGbsmikgoNYyafjZsil6DFRNdPhks7mg7h5cBbVF3i8LJ9pKDfuK/mo50l81UO2N9Ff90HOnHlhdWNdfZf5q3CHsS8JNpTtcUcCCLWid6NidOB5/Znrch3Ei48ys4tbTU97z8b2r8nzFSRSJMM6uRu6BvcVM/l4XB5V7k78uK+i7ZGijU6PBaFn6ijbafuC5aRzdUvVXRTBqs2mfaVPuEhZJQ8B0EWkCplXDPvFFthAHcCoANS4JrzDpflBsQesEx1237G7tlS8j7rxll3d//Ag6iAe8BWnko9UlS0VgFas173agH3YG1uQKhVRvYkWXDMtQ6Tr6ZGiD0wwr6P833rHsHF2wyl3lcRyzb7Gt4HJ25ZY5w2rSsJq+idhlnh9cdcP8YAatnnCPaT3DSumK6bAeLragdYLjGdvS8LLirltO8TOv/uDlP1Ba/zIlUXDJewIrico9GbvGc4094ws35aPN5d9Q9vl8H5LR7mDQpaqtt/hg8cHQNhXBG9LCIa3gVurfblpVt7uVLp2o9yK21U3PJmdWvf6qdAdj7H2v+hnC3gMr9YB1KVbJRxFb0DrBsSXhLy/uumWXxh/NeIn0zqVCjEKICjAKXLFfeLoJWrEXhAptdIQkrEJFGxxt8ZYPlm7khuqDilO7zhG8IQRDXFnwBrNQoahd6cIJ2w7+V7ZJuEXAjuYHpVNXhrXuYNFgZYK9B6xRhvVIVn4ddWxB6wTHl33Grcd9CCc67jwi0JdscCVeiJ2hs45zqwlNUNA5Z6Y6h5i3S5eRnAJYPoNRmwGqzdmTjwbvLTEKMai9TIpC8oYUBTpBvEE6oVoJ0qHeV81g1mfb7H21StgmYPe7dXfRom4vvFUpBVNcy67UJlkBPpWlqkc0AP1IYwtaJzgef93suA/hRMedR+VcmsAEQbyQOoO3lqZzfWZUWx3lsWbI6sacV/m51qvWyntLCNn91BuIQgqiXjURCIIEQTrBdILxYJd67xYqFLUNVIuE6RJuof5X0uZh5w0pw1opWLKoXCb2gFXAaZN4P4bYgtY2HrNxalrxT776sy/7cUync4e2FVKjWdASMDaxtAFrUy+DAAWslM0CYxRSNHo/yqBICkx40QmhAERBgn4tAUyrQGXavNm5UxmDbVHeahExXcQu/dAZbFotBcfZVRaKEgtftQFWm+XgMfBY49iC1jYe0/FVz7/9sh/DNnqfrE5OxyoRu4pgE8FUdJIG1Xz5nMfcciw2zUEBiKj3peQ0QfLPKFBJUpCUqGBluoTplGg3XkHLdFnVvtKsyiy7QdVeVniNjfouBlawDljHDFYltqC1jW1cZhifSEYwHWCk94tPNvWbp0eW8T1pT9INPgWMxOvvKXgV3/nRLWSw6hISFSyNz6DV6ICzXWVX0S5gVjqK0xPtxahvM7vqOaqN7AqOlXC/WEg6ZtTcxja2sY1HEltj8W1sYxsnKragtY1tbONExRa0trGNbZyo2ILWNraxjRMV2+7hCQ4ReSuwOu7jeJi4EfjUcR/Ew8Q0pfTs4z6IbRwutqB1smOVUnrBcR/EQ4WI/PFJOMbjPoZtHD625eE2trGNExVb0NrGNrZxomILWic7/vVxH8AhYnuM2zjS2Crit7GNbZyo2GZa29jGNk5UbEFrG9vYxomKLWidwBCRbxSRN4vIW0TkVSJy+aZQRxwi8uUi8i4RuUdEvu+4j2czROSJIvIKEXm7iLxNRL77uI9pG4eLLad1AkNEPh94R0rpARH5CuAHUkqfc9zHVUJELPBu4KXAh4DXAl+fUnr7sR7YKETkNuC2lNLrReQU8Drgf7iWjnEbB8c20zqBkVJ6VUrpgfy/rwYu38nuaONFwD0ppfellFrg5cCfOuZjWouU0kdTSq/PX58H3gE84XiPahuHiS1onfz4duA/HvdBbMQTgPtG//8hrmFAEJEnA88F/uiYD2Ubh4jtGM8JDhH5YhS0vvC4j+WkhojsAv8B+CsppXOX+DBbjuVo4lCbh7eZ1gkJEflLIvLGfHu8iHwW8OPAn0opffq4j28jPgw8cfT/t+fvXVMhIhUKWD+bUvql4z6ebRwutkT8CQwRuQP4PeCbU0qvOu7j2QwRcSgR/yUoWL0W+IaU0tuO9cBGISIC/Fvg/pTSX7nMhztRH6IYE/fev+AT5xvOzCruumUXfTmOPQ51EFvQOoEhIj8OfBVQdrr7a81JQUT+JPDPAAv8RErpB4/3iNZDRL4Q+APgLeg2QYC/lVL6rUt4uGv6QxRi4mPnVrzh3gd45bs/ySve9Uk+eb7p//3W01O+6yVP5Zs/78kYc6zgtQWtbWzjKsWxfojOLjped+/9vOOj57nnE3t86IEFn9prOb/yNF1g2QV83qhzaup4yV038YVPv5HHXzfjY2dX/PIbPswfvu/TfMHTb+BHvvH5nJlVx/WnbEFrG9u4SnHVP0QpJV75nk/xo7//Xl79/k/3qwhvOzPljuvn3HRqwqlpxbQyzGvL46+b8cxbT/PZt5/BWXPBY/3ca+7jf/21t/LsJ5zhp7/9c9idHEuPbgta29jGVYqr+iFadYG//otv5tff9BFuPT3la174RD7vqTfw7Cec5tT00rOk337rx/hL/+71vOjJ1/NT3/4iKnvV+3Rb0NrGNq5SXLUP0X7j+eafeA2v++ADfO9L7+IvvOSpTJw9ssf/xdd9iL/2C2/iO178FL7/Zc86ssc9ZBwKtLY6rW1s44RESom/8R/e/P+2d+/BUVV3HMC/v91NSFgSAuQFCSGBhBBEwfCQ8qqCrYBFGcQpYNXWR0eLTp2pb1sHnVqf7ThOHR8VRKc4WkVbtTx8AYoMEEQChCRgHiQgGxOSkISQ969/7GYaIyT7uLt3L34/Mxl2l3PP/U1m8p17zp57Lr6uqMPzK3Jx5UXDDT/H0smpyK+sxz++KENu2hAsuND4cwSK67SILGLtjnJ8uP8E7r4iOyiB1e1PvxiPiSPjcN/6/ThefyZo5/EXQ4v8IiKrRORuz+tHReRyP/vhbgteqKxtxhMbizB3XCJumzMmqOeKdNjw3LJJ6FLgrje/RkdnV/8HhRBDiwKmqg+r6id+Ht4B4A+qOh7AdAArRSTkkynh7rH/FsImgj8vnhCStVSjhjnx58UTkFdeh2c+Ohz08/mCoUVeE5GHROSwiGwHkN3j87UistTzulxEHvfcbrRHRHJFZLOIlIjIbb375G4L/dt+pAabCly4Y24mRsRFh+y8iy9OwXWXpOHFbSXYdPBEyM7bH4YWeUVEJgNYBmASgIUApvbRvEJVJ8G94nwtgKVwX0U90s850sHdFr5HVfH0R8VIiYvGzbMyQn7+hxe557fuemsf8sprQ37+s2FokbdmA3hPVZs9uyG830fb7v87AGCXqjaqajWAVhGJO9sBBu22cN7Zerga+ZX1uHNuJqIijFva4K0BDjtW3zgFI+KicdOredhXWR/yGnpjaFEwdN/Y1tXjdff7Hyyz4W4LZ6eqePaTI0gdEo0luebt8xg/aADW3XIJ4pwRWP7yTnxaWGVaLQBDi7z3OYDFIhLt2Z54kRGdenZbWA339tF/M6LP88UXR2qQX1mPlZdlItJh7p/q8MHRWH/7DGQmDsKtr+/Bm7srTKuFoUVe8UyWvwUgH+6dUvMM6nomgOsBzO2xX9hCg/q2tBe2liA5NgpLcsPje4nEmCi8+dvpmJ2VgPvfPYDnt3xjSh28jYcocIb/Ee2rrMfi57/EQwtzcOuc0UZ3H5D2zi7c83Y+/r3vW6xaNB6/nmnYFwS8jYfIql7cWoLYKAeWX5Jmdik/EGG34ZlrJ6K5rROrPjiExNgoLAzh7T4cHhKFmWJXIzYVuHDjjHSztojpl8Nuw3PLL8bFaXG45+18lNWcDtm5GVpEYea5z47AGWk3ZV2WL6Ii7Pj7ilxEOGxYuW4vWto7Q3JehhZRGDlS1YgNB07gxhnpiBsYaXY5/UqJi8Zfr52IQycaQjYxz9AiCiNPbirGwAg7bpkdXpPvfZmXk4QluSl4YWsJilzBXxfM0CIKE1uKvsMnhVW4c14WhjrD/yqrpz9eOR6x0RF44N0DCPaKBIYWURhoae/EIx8UYHSCEzcZt4QgZIY6I3H/gnH4uqIeH+4P7s3VDC2iMPDExiKUn2zGo1dNMH31u7+uyU3FuOQYPLW5CK0dwZuUt+Zvh+g88lGBC2t3lOOmmRmYlRVvdjl+s9sEDy7MQWXtGazbGbzbfBhaRCY6VteMe97ZjwkpsbhvQXb/B4S5OWMTMH30ULz0eUnQrrYYWkQmae3oxMp1e9GliudX5Br6VB0z3XFZFqoaWvHOV8eC0j9Di8gkT20qRv6xU3h66USMGuY0uxzDzMwchokj4/DitpKg7C/P0CIywe6yWqz5sgy/mp6G+ROSzS7HUCKClZeOQWXtGWw46DK8f4YWUYidaevEve/kI3VINB5YkGN2OUFxeU4SMuKdWLO9zPC+GVpEIfbCthKUn2zGk0sugjNMb4gOlM0m+M3MdOyrrMfeijpj+za0NyLqU2VtM17aVoJFE0dgRqZ1lzd445rcVMREObDa4KsthhZRCP1lg/v5hQ8sGGd2KUHnHODAsqkjsfmgC65TLYb1y9AiCpFdpSex8aALt186JqTPLzTT9dPT0amKNwzcU56hRRQCXV2KxzYUYvjgKNxqoR0cApU2bCDmZifijV0VaOswZvkDQ4soBP6Tfxz7j53CvfOzER15fiwi9dYNM9JR09SKjQY9pZqhRRRkLe2deHpTMS5MGYyrJ4bHk3VCaXZmPNKHDcS6XcYMERlaREG25ssyfHuqBQ8uzIHN5tUDZ84rNpvgl1PTsLusFiXVTYH3Z0BNRHQOtafb8MKWElyek4SfjBlmdjmmuWZyChw2wVt5lQH3xdAiCqLV20vR1NaB++ZbfweHQCTGRGFeTiLWf3Us4Al5hhZRkJw6047XdxzFggnJyEqKMbsc0y2bmoaTp9uw7XB1QP0wtIiC5J87j6KxtQO/uzTT7FLCwqyseMRGObC5ILCbqBlaREHQ0dmFV78sx6XZCZiQMtjscsJChN2GeTlJ+LSwKqAtaxhaREGwo+QkappasXxa+D3W3kxXXJCEuuZ27C6v9bsPhhZREHyQ/y1iBjjw07EJZpcSVuaMTcAAhw0fFVT53QdDi8hgrR2d2FTgws8vSEZUxI9r9Xt/BkY6MGdsAj4+VOX38xEZWkQG21ZcjcaWDlw1aYTZpYSl2VnxOF5/Bsfrz/h1PEOLyGCbC6oQNzACM37Ei0n7kps2BADw1VH/NgdkaBEZLK+8FtPShyLCzj+vsxmXHANnpJ2hRRQOqhpaUFHbjGkZQ80uJWw57DZMSovDnnKGFpHp8jxf5U9NZ2j1ZfKooShyNaCptcPnYxlaRAbKK6tFdIQd40fEml1KWJsyagi6FPjaj4deMLSIDJRXXofcUXGcz+rHpLQ4iMCvISJ/s0QGaWhpR6GrAVNGcWjYn9ioCIxJGISCbxt8PpahRWSQ/Mp6qAJT0oeYXYolZCfH4HBVo8/HMbSIDFJ4wn3VcMEI3iDtjeykGFTUNqO5zbfJeIYWkUGKXI1IiBmAoc5Is0uxhLGePcaOVPm2BTNDi8ggxa5GjEvmZn/eyvb8ropdvg0RGVpEBujo7MKR75qQzR1KvZY2dCCiImwo9nFei6FFZIDyk81o6+jCuOFcn+Utu02Qlej7ZDxDi8gA3UMcDg99MzYphsNDIjMUuxpgEyAzcZDZpVhKdvIgfNfYirrTbV4fw9AiMkCRqxHp8U5u+uej7GT3cNqXeS2GFpEBiqv4zaE/xiQ4AQBlNae9PoahRRSglvZOVNQ2IyuRoeWrEYOjEemwMbSIQunoyWaoAqM9Vw3kPZtNkDHMidJqhhZRyJRWu1d0j47nJLw/MuKdKKvxflU8Q4soQKWeoU0Gr7T8kh7vREVts9ftGVpEASqtPo2k2AEYNMBhdimWNDreifZO7x8nxtAiClBpTROHhgHw9QqVoUUUoNLq05yED0BGPEOLKKROnWn3+Q+P/m+YMxIxUd4PrRlaRAYYk8Dhob9EBKN9CH2GFpEBODwMjC9XqgwtogBF2m1IHTLQ7DIs7aLUOK/bMrSIAjQ1YwjsNjG7DEu7aVaG121F1fv1EUR0VvwjMoZXyc8rLSKyFIYWEVkK7zsgChwntEKIV1pEZCkMLSKyFIYWEVkKQ4uILIUT8UQBEpGDAFrMrqMf8QBqzC6iH1GqOqG/RgwtosC1qOoUs4voi4jssUKN3rTj8JCILIWhRUSWwtAiCtzLZhfghfOmRt4wTUSWwistIrIUhhZRAETkOhHZLyIHRGSHiEw0u6beRGS+iBSLyDcicr/Z9fQmIiNFZIuIHBKRAhH5fZ/tOTwk8p+IzABQqKp1IrIAwCpVvcTsurqJiB3AYQA/A3AMQB6A5ap6yNTCehCR4QCGq+peEYkB8BWAxeeqkVdaRAFQ1R2qWud5uxNAqpn1nMU0AN+oaqmqtgF4E8DVJtf0Pap6QlX3el43AigEkHKu9gwtIuPcDGCj2UX0kgKgssf7Y+gjEMwmIukALgaw61xtuCKeyAAichncoTXL7FqsSkQGAVgP4C5VbThXO15pEflIRFaKyD7PzwgRuQjAKwCuVtWTZtfXy3EAI3u8T/V8FlZEJALuwFqnqu/22ZYT8UT+E5E0AJ8BuEFVd5hdT28i4oB7In4e3GGVB2CFqhaYWlgPIiIAXgNQq6p39dueoUXkPxF5BcA1AI56PuoItxuTRWQhgGcB2AGsUdXHzK3o+0RkFoAvABwA0OX5+EFV3XDW9gwtIrISzmkRkaUwtIjIUhhaRGQpDC0ishSGFhFZCkOLyMJEZJWI3O15/aiIXO5nP1EisltE8j07LTxibKXG4W08ROcJVX04gMNbAcxV1SbP6vTtIrJRVXcaVJ5heKVFZDEi8pCIHBaR7QCye3y+VkSWel6Xi8jjnluN9ohIrohsFpESEbmtd5/q1uR5G+H5CctFnAwtIgsRkckAlgGYBGAhgKl9NK9Q1UlwrzZfC2ApgOkAzjr0ExG7iOwD8B2Aj1X1nDstmImhRWQtswG8p6rNnp0Q3u+jbff/HQCwS1UbVbUaQKuIxPVurKqdnpBLBTBNRPp9cKoZGFpE569Wz79dPV53vz/nfLaq1gPYAmB+0CoLAEOLyFo+B7BYRKI9WxMvMqJTEUnovvoSkWi4t2cuMqJvo/HbQyIL8eyj/haAfLjnnvIM6no4gNc8e8rbAPxLVT80qG9DcZcHIrIUDg+JyFIYWkRkKQwtIrIUhhYRWQpDi4gshaFFRJbC0CIiS2FoEZGl/A/6tLyGhCJesQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "posterior_samples = posterior.sample((5000,))\n", + "\n", + "fig, ax = pairplot(\n", + " samples=posterior_samples,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + " upper=['kde'],\n", + " diag=['kde'],\n", + " figsize=(5,5)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The 1D and 2D marginals of the posterior fill almost the entire parameter space! Also, the Pearson correlation coefficient matrix of the marginal shows rather weak interactions (low correlations):" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWYklEQVR4nO3df6xcZZ3H8fen97Y0u6wWKCmI2EJokBpWkAYwRGUBtRBDu4rabpTiQroY3F1FDSKJbHBJym4iiz+xgQooCygi1ohhkR+LRmEpbKH8WOQKKq2VSvmhhFK4vd/94zxThunMnXM7Z849c+7nZU46c84z85yJuV+eX+f5KiIwM+u3aZN9A2Y2NTjYmFkpHGzMrBQONmZWCgcbMyuFg42ZlcLBxqymJK2WtFnSgx2uS9KXJY1IekDS25quLZf0WDqWF3E/DjZm9XUFsGic6ycC89OxAvgGgKQ9gfOBo4AjgfMl7dHrzTjYmNVURNwJPDNOkcXAVZG5C5glaV/gvcAtEfFMRDwL3ML4QSuX4V6/wMyK85f7z4ztL43lKrvt6VceAl5qOrUqIlZNoLr9gCeb3m9I5zqd74mDjVmFjL00xgHvn52r7P+t2vRSRCzs8y0Vxt0osyoRTJumXEcBNgL7N71/YzrX6XxPHGzMKkbKdxRgDXBqmpU6Gng+IjYBNwPvkbRHGhh+TzrXE3ejzCpEwLSCmgCSrgGOBWZL2kA2wzQdICIuBW4CTgJGgBeBj6Vrz0j6InBP+qoLImK8geZcHGzMqkQwNFxMsyUilnW5HsBZHa6tBlYXciOJg41ZxaimgxsONmYVIsG0ggZkqsbBxqxi3LIxs1IUNUBcNQ42ZhUiuWVjZiUZGvKYjZn1meRulJmVQqiYRxEqx8HGrErcsjGzsniA2Mz6TvIAcVtp+8DrgHnAb4APpZ29WsttB9ant7+LiJN7qdeszurajer1Z30OuDUi5gO3pvftbI2Iw9LhQGPWgQBNU65j0PQabBYDV6bXVwJLevw+s6ktDRDnOQZNr2M2c9JmOwB/AOZ0KDdT0lpgFFgZETe2KyRpBdku72hYR8yYNXWGlA7+iwMn+xZKs23W1sm+hVI9+sDvno6IvfOWr+lzmN2DjaSfAvu0uXRe85uICEnR4WvmRsRGSQcCt0laHxG/bi2UNmteBTBz7xkxb0m+vVjr4OYjLp/sWyjNyOK2aYxq6x1vOPO3ectmm2fVM9p0DTYRcUKna5KekrRvRGxKKSA2d/iOjenfxyXdARwO7BRszKa8AjfPqppee35rgEa2vOXAD1sLpH1Md0uvZwPHAA/3WK9ZLQkxTfmOQdNrsFkJvFvSY8AJ6T2SFkq6LJU5BFgr6X7gdrIxGwcbs3YKzq4gaZGkR1OK3Z1miyVdLGldOn4l6bmma9ubrq3p9af1NAIbEVuA49ucXwuckV7/Aji0l3rMpooix2wkDQFfA95NlmjuHklrmv9jHxGfair/j2RDHA1bI+KwQm4Gp3Ixq5xpmpbryOFIYCQiHo+Il4FryZardLIMuKaAn9CWg41ZlShfFypn6yd3Gl1Jc4EDgNuaTs+UtFbSXZKW7OIv2mHqLGQxGwAChodytwFmp/VrDRPN9d1sKXB9RGxvOpdryUpeDjZmFZJtnpU72DzdJdf3RNLoLqUlh1TRS1bcjTKrlEK7UfcA8yUdIGkGWUDZaVZJ0puBPYBfNp0rfMmKWzZmVVJg3qiIGJX0CbI83UPA6oh4SNIFwNqIaASepcC1KUNmwyHANyWNkTVKel6y4mBjViFFP64QETeR5fRuPveFlvf/0uZzhS9ZcbAxqxKJoaGhyb6LvnCwMauQKf0gppmVy8HGzPpOIu/q4IHjYGNWKfkfshw0DjZmFTOI20fk4WBjViESDA97NsrM+qyxeVYdOdiYVYk8G2VmJXGwMbO+E576NrMyyFPfZlYCIYaHpk/2bfRFIe21HDu47ybpunT9bknziqjXrG4EDGko1zFoeg42TTu4nwgsAJZJWtBS7HTg2Yg4CLgYuKjXes1qSWLatKFcx6ApomWTZwf3xcCV6fX1wPFSTRcTmPVomoZyHYOmiDGbdju4H9WpTNo97HlgL+DpAuo3qw2hiexBPFAqNUAsaQWwAmB498GL3Ga9ksT0oRmTfRt9UUSwybODe6PMBknDwOuBLa1flNJQrAKYufeMaL1uVn+q7TqbIn5Vnh3c1wDL0+tTgNtaNlc2MxqpXIobIM4xU3yapD825fQ+o+nackmPpWN562cnqueWTc4d3C8Hvi1pBHiGLCCZ2U5U2LR2nlzfyXUR8YmWz+4JnA8sBAK4N3322V29n0LGbLrt4B4RLwEfLKIuszor+HGFHTPFAJIaM8V5UrK8F7glIp5Jn70FWEQPucDr2Tk0G1gTWmczO+XibhwrWr4sb67vD0h6QNL1khrjr7nzhOdVqdkos6lugrNR3dLv5vEj4JqI2CbpH8jWwx3X43e25ZaNWYU0ulF5jhy6zhRHxJaI2JbeXgYckfezE+VgY1YlxT6u0HWmWNK+TW9PBh5Jr28G3pNyfu8BvCed22XuRplVigp7FCHnTPE/SToZGCWbKT4tffYZSV8kC1gAFzQGi3eVg41ZhWQZMYvrcOSYKT4XOLfDZ1cDq4u6Fwcbs0opbp1N1TjYmFWIJIb9bJSZ9Zv3IDazckgMDeDGWHk42JhVSNaycbAxs76r7xYTDjZmFSLE8DQPEJtZv0nI3SgzK4PHbMys74SYhoONmZXALRsz6zsV+CBm1TjYmFWKGJJno8yszySvszGzktS1G1VICO0lN42ZNZNzfXfSS24aM3st4UV94+klN42ZtfA6m87a5Zc5qk25D0h6J/Ar4FMR8WRrgZT3ZgXAfnvO4Y4jvlfA7Q2GY++dOjn8PnLwoZN9C5UlFftslKRFwCVkexBfFhErW66fDZxBtgfxH4G/j4jfpmvbgfWp6O8i4uRe7qWsYe8fAfMi4q+BW8hy0+wkIlZFxMKIWLjX7rNKujWzKiluzKZpiONEYAGwTNKClmL/CyxMf5vXA//WdG1rRByWjp4CDRQTbHrJTWNmr5GN2eQ5ctgxxBERLwONIY4dIuL2iHgxvb2L7O+3L4oINr3kpjGzJiIbs8lzUFz63YbTgZ80vZ+ZvvcuSUt6/W09j9n0kpvGzFpNaFFfEel3s1qljwALgXc1nZ4bERslHQjcJml9RPx6V+soZFFfL7lpzOxVBQ8Q50qhK+kE4DzgXU3DHUTExvTv45LuAA4HdjnY1HNdtNkAE0O5jhzyDHEcDnwTODkiNjed30PSbun1bOAYelzO4scVzCqkyKe+cw5x/DuwO/A9SfDqFPchwDcljZE1Sla2Wag7IQ42ZpVS7BYTOYY4TujwuV8AhS6IcrAxqxjVdHTDwcascjTZN9AXDjZmFeI9iM2sRO5GmVkJ5G6UmfWfkLcFNbNyuGVjZn3nAWIzK427UWbWZ8IDxGZWCnkFsZmVxS0bMyuBWzZmVgLl3atm4DjYmFVINkDslo2ZlcCzUWbWfxL4cQUzK0NdWzaFhFBJqyVtlvRgh+uS9GVJI5IekPS2Iuo1q59snU2eI9e3SYskPZr+9j7X5vpukq5L1++WNK/p2rnp/KOS3tvrLyuqvXYFsGic6ycC89OxAvhGQfWa1U5R2RVypt89HXg2Ig4CLgYuSp9dQJaN4S1kf9tfV840nJ0UEmwi4k6y5HOdLAauisxdwKyWLJlmxquPK+T5Xw5d0++m91em19cDxytLs7AYuDYitkXEE8BI+r5dVtZIVK40oJJWNFKJbnnhuZJuzaxKNIGjkPS7O8pExCjwPLBXzs9OSKUGiCNiFbAK4K1z3xyTfDtm5Yt05FNY+t0ylNWyyZUG1MwCRb4jhzx/dzvKSBoGXg9syfnZCSkr2KwBTk2zUkcDz0fEppLqNhssYzmP7rqm303vl6fXpwC3RUSk80vTbNUBZJM7/9PDryqmGyXpGuBYsj7kBuB8YDpARFxKlpHvJLJBpheBjxVRr1kt5Wu15PiaXOl3Lwe+LWmEbJJnafrsQ5K+S5bfexQ4KyK293I/hQSbiFjW5XoAZxVRl1mtBajA0coc6XdfAj7Y4bMXAhcWdS+VGiA2MyYyQDxQHGzMqqagblTVONiYVU09Y42DjVmlBHmntQeOg41Z1dQz1jjYmFWOg42ZlcLdKDMrQ5HrbKrEwcasSib2IOZAcbAxq5SAsXpGGwcbswoR9e1G1XMbdzOrHLdszKrGs1Fm1nceIDazssgDxGZWinrGGgcbs0oJPPVtZmUIoqYDxJ76Nqua4jY870jSnpJukfRY+nePNmUOk/RLSQ+ltNkfbrp2haQnJK1Lx2Hd6nSwMauQCIixyHX06HPArRExH7g1vW/1InBqRDRS8P6HpFlN1z8bEYelY123Ct2NMquY2N5jsyWfxWQZUSBLv3sHcM5r7iPiV02vfy9pM7A38NyuVFhIy0bSakmbJT3Y4fqxkp5vanJ9oV05symvMUCc5+iefnc8c5pyt/0BmDNeYUlHAjOAXzedvjB1ry6WtFu3Cotq2VwBfBW4apwyP4uI9xVUn1lNTWiAeNz0u5J+CuzT5tJ5r6kxIqTOT2RJ2hf4NrA8IhrNrnPJgtQMspTZ5wAXjHezReWNulPSvCK+q2HbrK2MLG7bUKqljxx86GTfQmm+85/rJ/sWqq2gXlREnNDpmqSnJO0bEZtSMNncodzrgB8D50XEXU3f3WgVbZP0LeAz3e6nzAHit0u6X9JPJL2lXQFJKxpNwue2vFDirZlVR0TkOnrUnHZ3OfDD1gIpZe8PgKsi4vqWa/umfwUsAbq2DMoKNvcBcyPircBXgBvbFYqIVRGxMCIWztpr95JuzaxCJjZm04uVwLslPQackN4jaaGky1KZDwHvBE5rM8V9taT1wHpgNvCv3SosZTYqIv7U9PomSV+XNDsini6jfrNBUsZsVERsAY5vc34tcEZ6/R3gOx0+f9xE6ywl2EjaB3gqDUQdSdai2lJG3WaDJKKQNTSVVEiwkXQN2Zz9bEkbgPOB6QARcSlwCvBxSaPAVmBp1HVNtlmvSllmU76iZqOWdbn+VbKpcTProq7/HfYKYrMq8VPfZlaWkh5XKJ2DjVmVBB6zMbMyeDbKzMriAWIz67u0n00dOdiYVY2DjZn1W0R4NsrMyuFgY2b95zEbMyuHu1FmVoYAxhxszKzPIoKxV7ZP9m30hYONWcW4G2Vm/VfjbpQzYppVSr5smL3OWOVJv5vKbW/af3hN0/kDJN0taUTSdWlz9HE52JhVSWTdqDxHj/Kk3wXY2pRi9+Sm8xcBF0fEQcCzwOndKnSwMauQAGJsLNfRo8VkaXdJ/y7J+8GUvuU4oJHeJdfnPWZjViURRP7ZqNmS1ja9XxURq3J+Nm/63ZmpjlFgZUTcCOwFPBcRo6nMBmC/bhU62JhVSUxoNqqM9LtzI2KjpAOB21KuqOfz3mCznoONpP3JcnzPIWsFroqIS1rKCLgEOAl4ETgtIu7rtW6z+okiukjZNxWQfjciNqZ/H5d0B3A48H1glqTh1Lp5I7Cx2/0UMWYzCnw6IhYARwNnSVrQUuZEYH46VgDfKKBes/oJYHvkO3qTJ/3uHpJ2S69nA8cAD6c0TLeTpWjq+PlWPQebiNjUaKVExJ+BR9i5/7aYLF9wpOTksxq5gs3stUoaIM6TfvcQYK2k+8mCy8qIeDhdOwc4W9II2RjO5d0qLHTMRtI8smbW3S2X9gOebHrfGFDahJntUNZ+NjnT7/4COLTD5x8HjpxInYUFG0m7k/XlPtmc23uC37GCrJvFnP32LOrWzAZHMJHZqIFSyDobSdPJAs3VEXFDmyIbgf2b3rcdUIqIVRGxMCIWztpr9yJuzWzARFndqNL1HGzSTNPlwCMR8aUOxdYApypzNPB80xy/mTWUt4K4dEV0o44BPgqsl7Qunfs88CaAiLgUuIls2nuEbOr7YwXUa1ZDxU19V03PwSYifg6oS5kAzuq1LrPaa0x915BXEJtVSEQw9vJo94IDyMHGrEom9rjCQHGwMauSgBh1sDGzvnN2BTMrQbhlY2aliHCwMbMSBIxtq+fjCg42ZlVS0oOYk8HBxqxCPGZjZuXwmI2ZlcXdKDPrP3ejzKwMEcHYtno+G+UkdWZVksZs8hy9yJN+V9LfNKXeXSfpJUlL0rUrJD3RdO2wbnU62JhVSYXS70bE7Y3Uu2QZMF8E/qupyGebUvOu61ahg41ZlaQxm363bJh4+t1TgJ9ExIu7WqGDjVmllNONIn/63YalwDUt5y6U9ICkixv5pcbjAWKzComxCT2uMG6u74LS75JyvB0K3Nx0+lyyIDUDWEWWR+qC8W7WwcasUib0uMK4ub6LSL+bfAj4QUS80vTdjVbRNknfAj7T7WbdjTKrkvLGbLqm322yjJYuVCOjbcqusgR4sFuFbtmYVUl5jyusBL4r6XTgt2StFyQtBM6MiDPS+3lkOd/+u+XzV0vamyzZwTrgzG4V9hxsJO0PXEU2wBRk/cZLWsocSxY5n0inboiIcft3ZlNRlLQHcZ70u+n9b8hSZbeWO26idRbRshkFPh0R90n6K+BeSbc0JSBv+FlEvK+A+sxqzc9GdZAGijal13+W9AhZJGwNNmbWRUQwOubNs7pK/bvDgbvbXH67pPuB3wOfiYiH2nx+BbAivX3hHW8489Ei7y+n2cDTk1DvZJlKv3eyfuvciRTeHm7ZjEvS7sD3gU9GxJ9aLt8HzI2IFySdBNwIzG/9jrRGYFXr+TJJWjvedGLdTKXfOwi/NQjGahpsCpn6ljSdLNBcHRE3tF6PiD9FxAvp9U3AdEmzi6jbrG7GInIdg6aI2SgBlwOPRMSXOpTZB3gqrVQ8kizIbem1brM6qmvLpohu1DHAR4H1ktalc58H3gQQEZeSPcT1cUmjwFZgaURlQ/OkduMmwVT6vZX/rRH17Uapun/zZlPPQcNz4kuv+3Cusouf/cq9VR+DauYVxGaVEp6NMrP+CxjIwd88/CBmE0mLJD0qaUTSTjuX1Ymk1ZI2S+r6AN2gk7S/pNslPSzpIUn/PNn31FFkA8R5jkHjYJNIGgK+BpwILACWSVowuXfVV1cAiyb7JkrSeKRmAXA0cFZ1/7+N2gYbd6NedSQwEhGPA0i6lmzrxFo+dhERd6YV37U3SI/UBPhxhSlgP+DJpvcbgKMm6V6sT7o8UjPpwgPEZoOvyyM11RBe1DcVbCTbJKjhjemc1UC3R2qqos6zUQ42r7oHmC/pALIgsxT4u8m9JStCnkdqqqO+K4g9G5VExCjwCbId5B8BvttuG4y6kHQN8EvgYEkb0vaQddV4pOa4pgyOJ032TbWTtWw8G1V76Yn0myb7PsoQEcsm+x7KEhE/J9srt/IigldqOhvllo1ZxZTRspH0wbTAcSxtct6pXNuFrpIOkHR3On+dpBnd6nSwMauYkvazeRB4P3BnpwJdFrpeBFwcEQcBzwJdu+EONmYVEiWtII6IRyKi27a7Oxa6RsTLwLXA4jTgfhxwfSqXJ1e4x2zMqmQDz918dtyQdxfLmeOl3y1Ap4WuewHPpUmVxvmd0r20crAxq5CIKOx5tfFyfUfEeBkw+8LBxqymxsv1nVOnha5bgFmShlPrJtcCWI/ZmFknOxa6ptmmpcCatKXv7WTb/UL3XOGAg43ZlCTpbyVtAN4O/FjSzen8GyTdBF0Xup4DnC1phGwM5/KudXoPYjMrg1s2ZlYKBxszK4WDjZmVwsHGzErhYGNmpXCwMbNSONiYWSn+H4jRLO0e+4NOAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "corr_matrix_marginal = np.corrcoef(posterior_samples.T)\n", + "fig, ax = plt.subplots(1,1, figsize=(4, 4))\n", + "im = plt.imshow(corr_matrix_marginal, clim=[-1, 1], cmap='PiYG')\n", + "_ = fig.colorbar(im)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It might be tempting to conclude that the experimental data barely constrains our parameters and that almost all parameter combinations can reproduce the experimental data. As we will show below, this is not the case." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Because our toy posterior has only three parameters, we can plot posterior samples in a 3D plot:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "rc('animation', html='html5')\n", + "\n", + "# First set up the figure, the axis, and the plot element we want to animate\n", + "fig = plt.figure(figsize=(6,6))\n", + "ax = fig.add_subplot(111, projection='3d')\n", + "\n", + "ax.set_xlim((-2, 2))\n", + "ax.set_ylim((-2, 2))\n", + "\n", + "def init():\n", + " line, = ax.plot([], [], lw=2)\n", + " line.set_data([], [])\n", + " return (line,)\n", + "\n", + "def animate(angle):\n", + " num_samples_vis = 1000\n", + " line = ax.scatter(posterior_samples[:num_samples_vis, 0], posterior_samples[:num_samples_vis, 1], posterior_samples[:num_samples_vis, 2], zdir='z', s=15, c='#2171b5', depthshade=False)\n", + " ax.view_init(20, angle)\n", + " return (line,)\n", + "\n", + "anim = animation.FuncAnimation(fig, animate, init_func=init,\n", + " frames=range(0,360,5), interval=150, blit=True)\n", + "\n", + "plt.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "HTML(anim.to_html5_video())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Clearly, the range of admissible parameters is constrained to a narrow region in parameter space, which had not been evident from the marginals." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the posterior has more than three dimensions, inspecting all dimensions at once will not be possible anymore. One way to still reveal structures in high-dimensional posteriors is to inspect 2D-slices through the posterior. In `sbi`, this can be done with the `conditional_pairplot()` function, which computes the conditional distributions within the posterior. We can slice (i.e. condition) the posterior at any location, given by the `condition`. In the plot below, for all upper diagonal plots, we keep all but two parameters constant at values sampled from the posterior, and inspect what combinations of the remaining two parameters can reproduce experimental data. For the plots on the diagonal (the 1D conditionals), we keep all but one parameter constant." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAFJCAYAAADOhnuiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABtpElEQVR4nO39eZBl2X3fiX3OOXe/b8utMmvr7qruQi+AQIBLN0Q0uAIjkiJNOkzNaAkqFNbMWLYUI4Ul2wprhqYUZsyE7dDIMZ6xR5YYlMLykNJIsjUayRSapEg0IXSDBAkC6G50F6qX2nJ9+da733v8x3lZVWj0Uktmvsys84nI6Fxe3ncyq/P7zvnd7+/7E1prLBaL5bgg570Ai8ViuResaFkslmOFFS2LxXKssKJlsViOFVa0LBbLscKKlsViOVY49/E91iOxP4gHvcAfa/85rasKnef7sZ6Hms83/+S+/z0+J//E/P4mhAAhUQtdRBjSLHep2j5l2yFbUFShIO8JqgiqSFP1akRQE3dTumHGcjjl8dYWp7wxTwfXOesMWFUFChhrwSvFKm8Vy9wserwxXmE3j9gat0inHk3i4AwcVCZwx+CNNU4CYb/CSWrc3Qw5mKAnU+r+AJr6Q3+cu/l3uB/RshwR5EIPnabUVrQeWoRSCM9DtFvoOKTq+pQth6KlKFuCKhRUsRGsOm6QcYkXVCxEKcvhhLVwzPmgz5oz5DG3z5qqWZAh3ygq1usObxXLXM0W2cg7bCZtxplvBGvqIhOJMxU4mREsd6JxE407qlBJYQSrP6CeTO9KsO6WYy9a46ykaaAbufNeyqHT9NpIz0WVFU2SoKtq3kuyHCLCcZCtGNFq0fRa1JFL0XEpWpIyFhRtQe1D2dLUcY2IaqJWTivIORWNORMOOesPuOhtsuYMedTRJI3mzSrjrWqV9bLLW9kyN9IuO1nMIAnJMpdm4qImCicFdwIqA3ei8UcNzrTGGabISYYeDGnSbF8FC05ATesv/r9/n5/7pZfmvYy50EQuTRQg4gjhOOaoYHk4EMLssIIAHQXUkUsVOVShpAoEdWAEqw41TdCA3+D4FbFf0PFylvwpS+6UZWfMKTVmSeZEwiPTsFWHrJddtqs2/SJimIeMc58sc6lTB5lJVA4qE6gMnFTjphqV1qi0QiQ5pBnNNEVX5b7/6Md6p7U5yvjCG1toDVf7CecXo3kv6VApFgOc0MHBvPqIaUI9Gs17WZaDRiqk5yKXFtGdmKoTUCx4lJEk7wrK2NSwyo6mDhvolARRSTvKONMasuQnPBFtct7tc97d4Uk3RwrF5TLn7WqJ6+UCb6SrbBUtrk4W6E8j0tSjHnnIVOKOJO4UnAT8YYObavzdEmeUI8cZbPdpshxdFgfz4x/IVQ+Jf/W1m+y1Tv76qxvzXcwcqCJJGTk0LQ8dh4goRLie3XGdcGTgI+II3QppIo86cigjSRVIU8MKoQqNYGm/wQ0qoiCnG2Qs+QmL3pQVZ8yaM2RFpdRoxk3NRt1is2qzUXbplzGDImKU+aSZS5nt7bDM7kqloFKNk2lU2qDSCjnJEUmGzvIDLVUc653Wv/zDmzy52qZqGl54dZM/9+kL817SoVK0JI4jkLWHKCOklMgkpcmFvaN4UhEC2WlDFFJ1Q6rYpeg4FC1JHUAZQxVrqkij4xoV1HTijMUwYSWccC7YZdUd8pi7zaPOiFXlcbMu2Gl83iqXeTtfZqPocDPpMMwDxtOAcuohUoUzEeZO4YRbRXdvWOGkNWqYIoZjmvGEJs/hAIMYjq1o3Rik/O7bu/zVz32ESVHxSy++ySgr6QQPT0G+aAtqF0SjEFWAoyRO3kVOE+qy2vcCqGW+yCBAtGJ0t03T8im6HlWsTOG9LYxotbWpY8U1Xqsg8EuWoimnoxGn/DEX/E1OOWMuuiMk0G8K3qq6bFUd3sxXuJH12MjabE1jksynnHiIROHs3SlMZ0X3cYMzbXBHBWpaIHZHRrDS7EAFC47x8fD/9/V1AP74x0/zuadXKWvNb31za86rOlzqAOpAmMJrKKkj1xwTgwDpufaYeJIQAjE7FjYtnzp0qWdF9yqY/b/gQ+NrmrBBBDWBX9IKchb9hEV3yqo74pQzZkWN6UoFwLBR7NQttqoOO0WL3SJklAekuUeRO4hcoVKJyoy1QWXmSOikDU5aI5PSHAmTFF2Uh/JCeWx3Wi+8usGlUy0urrR4dEmzGHv8+qsb/NR3nZn30g6NogPKg73XHteXiCrCcRVSCNjapsmyua7Rsg9IhVpaRLRj6m5MsRBQxYqsq0zRPYaio6kDTd2pUXFJFOWstccsBAkX423OeX3Oezs86e4QS8G40VyrQq5XC7yanmWnjHlrush2EjNOfdJhALkpujsTs8PyBuYuoT+ozQ5rkiO3d9FJeqg3gI7lTmuYlrz8Zp/PPrMKgJKCH3nqFL/x2iZl3cx5dYdHHZijQBUye8UV1JFDHXvoVohoxcggmPcyLQ+AcD1kHCFaEU0roI5d6lDNiu57u+3ZLivQyLDC90vaQc5CkLDoJSy7Y9bcIStqjCug0Jrt2mWzbrNeddkpY7bzFoMsZJp75JkHuby1w1K5sTU4mcbJGpy0MkfCJEcnqalhHSLHUrR+6/Utqkbz2adXb33us0+vMsoqfvet3Tmu7HCpW41pzYigbAmKlqBoK8qOR90OEJ02otuxx8RjjIxDZKdN04moOgFlx6VoS4qWoJrtsqpYU7dqdFwRRgW9OGUpTFgLRpzzd3nM2+a80+cxx7TnJFrwTrXA28UK7+RL3Ei7bKRthmlAMvWppw7ORJka1nRWdJ+CN25wRzXOMEeOEhiMqCfTQ7/pcyyPhy+8ssFyy+MT53u3PveZS8t4juSFVzf4o48vzW9xh0mrpFYOaIFoBFoJVCHRSiAaH9FopOug8hyd5faoeIwQrmdqWAs9dBRQLoSm6N5WFG3TnlO27mjPaZn2nMU4udWe81iwzZoz5KK7TVtUgOBG7bFVt7mcr3E973Ez67I+7TDOfJKJTzN1UdPZkfBd7TnesLzdnjMc0+xze87dcux2WmXd8Jvf3ORHnjqFkrd3ELHv8OnHl3jh1Q0eltx7x68QYUUTNOao6EMVzI6KoaQOXZrIR4QhwrP+rWODEAjXQUQhOvRpIo8qVLfc7sbxbtzudaDRQY3rV4R+QcfPWPBSFl3jxVpxRizK6taxcKeO2azabJetmRfLHAmz3KWZud1lLmaOd/O2V3hXaYlMCvQ0RafpgZlHP4xjt9P68pt9xlnFj95xNNzjs8+s8jf++dd5Y3PCR1bbc1jd4bLYSZjmHlOg1C6NIxGNoHFBC3N3yPUkouwhfQ+pNc00sVaIo4wQqHbbNEB3W5SLEVXskPcUZSQoW4KyY8yjZaeGsMaPC5Y7Uzp+xsXWNmveiEf9bT7q3aArS9pScaPS3KjbfDM/w0bZ4UqyzFbaYjcJGY1DmtRBjRROInASgTfUOCkEgxp3XOOOC9TOGD1JqLe3D9zW8EEcu53W51/dwHckn7m0/B1f+9GnjJC98JC447t+RivIcYMKHTSzV16oQ2auaEkVKeOYjwJkHBkrxOx2t+WIIRXSn7nd45Am9qlihyoyETN1KGb/tmaHRbDndjc7rEU/YcmdsuoOWVEjurIkENCva7aaiPWqx0bZYatosZtFjLKAJPNoUgeRzYruqUClGD9WOrM2JCVymqMnCTpJ5ipYcMx2WlprXnh1g+efWCbyvnPpa92AP3K2ywuvbPC/+qEn5rDCw2UlnOCqmqJSDGtJLTVV7gICUQuKErSQqNRFC4FTN4i8QGiNzu1u66ghAx/h++hOi6YdULY9yraiCm73E1aRpoq1ac+JCuKwYCFKOR2OWPSmPOpvc97d4bwzYlEpSt3wRhVztVziWrHIzazLTh7TTyPGiU+RGPOoSgXOVMx6CjXetMFJZubRYYoYJ9SD4dyOhHdyrETr9Y0JV/vpBwrSZ59e5e/8+utsjXNW2v4hru7weSzaYctp0WhB00im0qMsJVpJ0HJWnAdVOmhHIrRG1TXK96g2t+0x8aggFcJ1kN0OOg6pFyLTntN1yNvydtF9rz2nVaHCim7rdnvOI2GfVXdoYmbUlFUl2agbBo3HG/ka14pFbuRdrk17jAqf4eR2e447fo/2nEGFk1So3cS054zGB5LYcD8cq+Ph3rHvR5869b6P+ewzp9AafvO1zcNa1txYdscsuVN6XkrsFwRBCX5DE5jjQ3WHY94U5h105EMYIFzHHhOPCEKZYyFhgI586tAxhfe9gvu3ebEaVFDj+yUtP6frpyx6U5bdsSm8y4S20CgEg8bcKezXMTtlzG4RMSp8prlHlTuQS2Qmbnuxsju9WDXqDrd7kx1sP+G9cKx2Wp9/ZYPvOtflVOf9DZPPnO5wphvwb17Z4N//vvOHuLrD5yn/JotqghSaBsGOG1NVitx1qfAAiXYEopY0jgbhgo5wPAdV1ejxmHownPeP8VAjXA/ZaSG6HeqlNlXski26lJGkbEHRnfUUdhqaqEa1KjrthE6Qc761y+lgxDlvl6f8G6ypCR9xAzbrhG+Wktfy02xXHV6frrGRtdlJI/qjmDJzYOTiTk0/oTcy5lF/qE3RfVLh9KeISUqztU1zSO05d8ux2WltjjP+4Org2wyl74UQgs8+s8qLl7fIyqPziz4I1tSINWfIqjti2Zuy4CfEQYEXVOiwNrutPcd8CGUoqCJFHbvGMR/PHPPWCnH4CHE7eTQybvcqdm8V3c2/mRGsKpj1E4Y1flDQCXJ6fsqKN2HVHbHqDlhTE7qyZrdJ2Woc1usOG1WXjbLDTh4xyELGmU+ZOejUQSXSFN1nIX5Oqk0NKzFRyWIyszVUFeij1WVybHZav/GqOe7tte58EJ99epV/+O/e5ncub7+nNeKkcM6p8MWQRPuM6wApGkZRgBCauhaUpQChqDKBKc6DKhUIkGmAAkRVIfqDI1FgfagQEuE4iG7b1LHaAWXLMRlpsaCKbhfe67BBRBVBVNCJMpbDCUv+lLP+gHPeDo84fc454OJxuWq4WvW4WixxPV9gO2+xnbYYpAFJ4qMTB5nKmbUB43ifGtFyJxVqVCAnKXo4oskPNhfrfjk2ovXCq5uc7YU8tfbh/qvnLi7S8h1eeHXzRIsWQFs2XHS3KbQiUjnTysdTNVoLdhtJrTRl6aLVTLgaaByFLH20I03qaV2bukWSzPvHeWhQ3Q4iCml6LaqWR9l1ybvGPFq0jbWhjDVVpwa/ptXO6IQZK+GUR6M+p7wxT/jrPOLscs6pGDeacaO5Up7iarHEtWKBa0mP3TwyyaOJRzN1cEYKlXOr6G7SR2tU2uDupsjhFD2emJkD9dE8qRyL42Fa1Lx4eYvPPbOKuIujjO8ofvAjK/z6qxs0zdEoHh4E2awwuigrltSEFWfMojel5yW0/BzfL5FBbRzzvnHMf1thPnBoIs845kN7TDwUhLjVorOX7V5HjmmAvjPbPdA0gQa/xvFrYr+g7eUszLxYC86UU2pMW5YEQjFuJP0mYLPqsF216Bcxw8Jku+eZS5MpRC6Rxe1sd1N8b24lj4okR6eZeavrI1N4fzfHYqf1O5e3ycrmQ+tZd/LZZ07xP37tJl+7PuS77uhRPEl8s+yyoqY84UguOkN6MmXa+LTU7QbWgRMyrAWV44BQCC1oXIGsFI0r0DLArzUi8JBFaaJy7VHxwJBhiFzo0Sx1aCLPZLvHirwtTNHdn2W7Rw06rok6GZFfcqY1ZCWYcD7Y5cngJmvOgCfdnEzDRl1xpVpmvezxRrrKzazLVtpiY9wiSz2qkWfGfSUCb2h6Cv1hgzdpcCY1Xj81wyi2d80O64in3h4L0Xrh1Q3avsOzFxbv+nt++EnTm/j5VzZOrGitVz1qLenJXQIBK6pgzRnQaMHIC5j4PloLksyjaAR1KagKCVpQRoCWyFLhpB4KkHGEFoK6Ko/sq+xxRkYRIo5NDSv2b2W7l3tu9+CObPegwQ1L4qC4le2+5o845Y5YcwYsyowSzVgLtuqQ6+Ui22WbjbxNP4/YzULSxKfKFTKTOKlxuzvpLMhvVnh3kgoxzRBJRpOm6PLo1bDezZEXrabRvPDqJj/wkRU85+5Ps73I43sfXeCFVzf4a3/syQNc4fy4ViySOS6xzHnGzViWPn1nFyUaksZnUhtz7TjymABFbYQLASoXgEbWEpW7aAFup4UARJoZI6EVrv1DKkS7Be2YphOa9pxYUcaScq/oHmtqX6OjGhVVhGHBYpjQ81POBANOuwMecXc47yREQpBoTb82QX43ix5bRZvtrEU/jZikPlXqIDKFSgQqEbiJcbu7qcad1DiTEjXKEePpzIt1PFJAjrxoffXagO1Jzufu4q7hu/ncM6v8H//HV0/seLEr6TJDN0SJBldcZ0WmPOpIIrlDrSWlVoSqJKtdXNWwCxSNmBXixaw4D6JWNI5AVjHKUSig3tm1x8R9QsaxqRkudmlaAUXXo+g5lKGk6MzGfcWasmUSG7x2QRzmLMUJ5+MBi96Ux/0Nzrq7POYMkUCiNVfKDu+Ui1wrlngrXWInj9mYtJgkAUXiIsd7I+tn7TnT2UDVpMHbzVHjDDGa0gyGxot1TDjyhfgXXt1AScEPPblyz9+7d+fwpI4X2y0iBlXEVtU2zufGwxWSSGhOqTHLzphld0LPS2n7OYFfIoIa7d+OsrldmBfUoSnME4WIwLeO+f1AKpOLFd4e91UH73K7+9D4oP07st39gq6X0nNN8ugpZ8ySTGhLQQOMG8Vm3Wa76hi3ex7dynYvcwdydcvt/mHZ7kfNPPphHPmd1guvbPJ9jy3Qi7x7/t4LyzGPr8QndrzY9UmXSelTa4FCM3ZDInGdtoQn3ZyaG7RVSta4hKpEotFA6vhUtY9WEi3FLM5GImuXxpV4QqCaBhEE1FsP17CQ/WTP7c5ijyYOTLZ7y0zPybsmLrnomMjsOm5wOgVBWLDWHrMUTDkbDrgUbsyK7jsEAiSCq5VJbHg9O82NvMd62ubGpMM080hHgZmeM5V4I3OX0Btq3KnGm9R4uwVqmiN3BjSTKc14PO9f0z1zpEXraj/hmxtj/tM//vR9X+Ozz6zy979wMseLjTOfRgsip+Cm00WKhjVnSMOUQDX0ZA7OLhtuj1IrprXHpPTQWjDJHGoNopZUufFwlalANBJZuog0QgqBnEboojiSJsOjjMl2Nzn9TRxQx2bcVxnermHtWRv2zKNhlBP7BQtBwrI/4ZQ34qzbZ0WNCQSUQL+WrFc91qsu63mX7Tymn8VMs1m2eyaRe9nud7jd3aTBSWrUOEcmGc1kis6O9l3C9+NIi9Zeg/T91LP2+NzTq/y3v3WF3/rm1omb1JMmPk0j2XEqIsfUn1Yc88q5KEcsyppIJKy7Jjc/bxwmpY8A8syl1FBXgqowxfkyE4BEVA4y80GCmMQghBWte0EII1hxjG5FVG2Ti1XGctZKZdp09nZYIqzxw9K05wQpa8GI096Q826fs2rIoiqJpGKjbrhedbg6u1O4mbfYyWKGaUCWmFwsNVW33O7G6Q7epDH9hOMCOUnQ0/RY7rD2OPKidelUi0eX4vu+xicfWWAx9njhBI4Xq0YedSnZAqTQZLVLS+VMPR8pGh5zJrSl4JK3SSzNq2rVKDadFnnlMFYNuYCycUxjdWWibEAiag/Xk7h51wQHak2TpvaO4odwZ7Z7E4dUCyFFzzRA552ZYMVmqGoTNsh2SRCaYRRnWkOW/SmPB1uccXd5zN1mVZW4QvBmKbleL/BWscKVdIXtvMWNSZdhGpgXr7GLSiXu+LZgeSONmzZ4gxJnbIZRNDu7x3aHtceRLcQP05KXrvTvqtfwg9gbL/abJ3C8mCgEOlMUucM49xnmoRkHVbbZqjpMG0mpNT1ZsagmrDgjem7CwswxH/olyn9vx3wdSqrQoYl8dOgjwgDhnKzj9b6zN1A1CNBRQBPNxn2F7+V2b9B+g+dXBF5p3O5eMst2H7GkJizKAiUEpdZszgaqbpdt+kXMqAzMuK/coc6MF0sWexEzt7Pd1SzbXSQ5JOmJMA8f2Z3We40Ju18++/Qq//3vXePLb/X5/se/M6b5uOKOJLUvKYEBkJcOvlMxrYw/S4mGM84uH3M1gSjwWKfQirZaIK1dfFUhhGaoI2rHoayNDUILU9tyXYEsAxxXoqRE1jXNlGP/P/2BIBUy8JG9LroV3sp2LzqKvGvuzhYdcySsWg10ZtNz2lMWgpSz0YAnok1WnSGXvA1WVcGqCnm9LNiqY76Zn+Fm0eVa1uPapMc490y2e+IgpwpvJJH5XtEdvGmDv1uiktJku4/G1MPRiTjmH1nReq8xYffLrfFir2yeKNFyEoFoNNpVlJ4DQtNPIySa2MnpOqYBek2t4wKrquTsrL617bdotCSvHbLCJQfqTCK0RNRiZj4Fp6UQjYsoA1SrhRSSemCNp9/GTLBEu4Vu3c52v2UcDe+oYYUaHdZ4s2z3hSBlJZiYmBlnyJoz5IwyLwobdcp63b2V7b6Rd9jOWowy/3a2e6pu9xLeynY3Xixneke2e5Yf2Qboe+VIitbemLAf++jat40Ju1/uHC/2n/3k03fVdH0ccFJAz6bvuA6lFowCHyk0gVPScYzDeU0NWVUpF9wWg8YUYLe9Do0WZLVDWrqMhSYJHSptImyqYnZHMZOIxjEzFLMIIQRiPD7SDbWHjXAdYx5tx7dysYxg3XGncDbyqwmN270VZbT9gpVgwil/zDmvzyNunxU15ZSK2GlSrlYe18sFNqqu6SfMW+xmIdPU5GKJROHMst2dhNt3CacNzqRETjLExIys12V1Yv69jqRo7Y0Je5C7hu/ms8+s8pv//Otc3pxw6YSMF/OGGjkTF4SkqgRTN6CqjCnUEQ1pbdp8pu4OMKQnoS2HTJt1fFkihaZqFJ6qqSpF4biUwpmlnQpkJWZ+LhdRxSjPQZUlzXhCM53O9ec/KqiVZXToUy/EFAs+VaRmPixB2Z55sQJN06lwgop2K2W1NWHRT3g82uKUO+Jxb4Mn3BGxkLxVJazXEW+VK7yenaZfxlyb9tjNQkbTgGLkI/K9oruZAu0NjXnU361wJyVqlMH2gCbL0EVxYgQLjmghfm9M2PPvMSbsftkbL/b5E+SOV7lGZXo2WNO4n5vMochdJrnHoAjpl7FxzFcd+o0x6AZCcEqZQZ4LTsKCn9DxMwK/xPFnjnmfWWrmzDEfSKpQUe855sMA4RzJ17xDR0cm270JHergXQNVfWh8Pct2r/AD43bveSbbfcGZcsoZsaKmBLMTwE7js1V32KraDMqI/izbPZm53cWHZLvLpDBu9ywzdwpPkGDBEdxp7Y0J+/T7jAm7X9a6AR8/d7LGiwWDmiqUmNeemWXBUdS1YESEEJq0Mm74pPapkSjvBsuq5KKbEckNFJpSK2JVkNcOfVUzFiGlnrnkK5MzjwBZOzSeRFZtpKOQQtLs7p6I4u6DUC/G1KFjpud0lIm27sxmT7Ya6naNDCs6rZRumHE6GvFYtMMpb8RT/g3OqDGPOg7bTUW/dngtP83NcoGr2SJvTRcZZCG744gi8WDq4IxNVLI7NtYGd6rxhzXOtMLZTUw/4XB0YgfzHjnR2hsT9r/8wf0Xlh996mSNF3OmNTTguoLGA4S5ra6FpFYOU9f8jJt+C4C2ylhSE2rGnHdKYlFx1tllx2uhRMN2ENNoQVUrqkLR4Bi3fGMEUeUSLcBJXJwmQFU1Ms9p0uyhvqNYxS51sFfDgnqv8B6YXCwZmR1WN8xYCqYs+xNOe0NW3QFn1JhYNkx0yXrts1W32ai6bBbGPLo7y3YvUhedzuYTJqbw7iZ69mZqWGpaIsYJemoK7ydRsOAIitatMWFPv/+YsPvls8+c4r984XV+47UN/oPve2Tfr3/YOJMCUbs0nqD2FKCp/Zm73ZEUnvFV7QTGnBurnK4ydxTPqB0iAWeclJ2mj6Jh02/TaEFZK/LCoWB2R7EB0QjK3Ax/dWOF0B6iapBphATqh1i0ypYy1pPImEf3iu71bBhFEBbEQcFSMOVUMGHNG3HG3WXNGbCmoEQwaDCtOVWP9bzLRt4xTdBpQJZ66NRku985AdqZCZaJmSkQ0ww9Hp/4F5EjJ1p7Y8JWP2BM2P2yN17shVc3T4Roqf4EEQV7p0PK0hTPRW3acUrpkleSLadFUZvivBSacRMQyZwVmXDGEVx0+rRlRtL4prFaaMpGMnF9JpWkVAot5ayxGpN66gi0FHhVg/A9ZFU9tD2KeUdRB1C0TeG9Ck22uwgrwlbOUithMUh4NOqz5g+56G3xlLdBV9ZMNfQbh6tVj8v5GhtlhzenS/SzmH4SMh0H6FThDPfuFII3Mu05waDGmdY4wxy5O0ZPE+rR5MhNz9lvjlQh/m7HhN0ve+PFvvDGCRkvlqTIJEMlFSptbhVkVWYK8zITkEvyzGWS+QyLkH5x2zE/1i6lbohlw4pMWHFGLDpTFr2EjpcTeiVqL8rmlmOe24X5QNFEnnHM+/5DW5jf+53ccrv7e9nuFZFf0vEzOl7Kkjs1A1WdEZGo8YQw2e51NHO735nt7pFlLnqW7a5ykPme0/2ObPekRCY5OjHZ7jQn34pypP4v25sK/aCtOx/ESRovVm1uo1oxCvAEyNKjdl1EBWBqW7ISlMpl0pg7U2pmg4hkwbTxaPQGT3sFq8oh0RsEssQVNWWjCJ2SqpZMZEApPOOYd5jZIEBLhax9HE/hVDVi4CCq6qHbbeVdU1MsO5qq3aADk+0eBwWn4gmPxX1WvPGtbPeLM9PvtNFcqZa4Xi7yTr7ElWSZQR6yNYlNVPLURY0UKhe3ewons6J72uDuZshJCoPxQ3VD5EiJ1udfufsxYffL7fFiG8detNCN8eCMp6jA2Bm8UAIK7UDjGQ9XHSgaAanyGHghAOteByUaYlnQlessqYJVpanZJWtc+r6pgw2LgLqRNI2kKSSVMLfamZlQnVyhBcgkRNUNsqmpd4cntgj8XtQh1L4ZjKvD2nixwpyun7ESTFjzh5xyR5x1dunKHCUEO7Vg2PhcLxe5WfS4mXXZyWLGuW8EK3UQezWsbGYevaOGpdIKOUkR4ynNdHpi3O53w5ERrb0xYX/y+x45UMf67fFimzSNRu6D435uaE1TlMjxGBn4KMCJXbQwdoXaMz9bnQq0UJSOy9gzdxS3/RZSaCJZsKQmwISLrkdDTuIM2XI7AOz4EXUjqRvJNFc0KKpIz0RLzOJsFE7LQzQNsq4Rkyk6f3j+iKrQHAl1YAQrjEzEzKKfcMofc9odsOYOWFXprSC/fhOwXvW4WfTYKDrs5DHDLGCaeTPBUqhkNpAi2xMsjTudteckxUywkoduXuWREa29MWEHcdfw3eyNF/vD68N96W2cK01NkzcwGCKKAtdVyMIHPLQQyApTnK+g0g6JCChL44DPaodGC1xRMXCHBGIdV8CT7ohC36CrpqS1S6AqlGxoGkHuupS1dzv1tJbUnkbULtoROFKi6sYUhXd35/3bORSqjklscFsF3XZKJ8g4Hw9Y9Udc8Le45K+zMotKHjeaK1XIa/kZNsou30qW2c5abCcxg1FEmTnIkWN8WBOBNwKVaoJhgzOtcUclqj9FTFOT418dn2z3/eLIiNbemLDnLiwd+HPtjRf79Vc3jr9oAWiNLgrTF5jkSEfipAonFGhpirhamjt/TaCopGaSe7gqpOXmbFdtXFHTdzx6smBRSpbUlBLFkjslb1ySymPoBUa4fIe6EsjSRNmIBmrfFOZl5CJDH6H1Q9OjqP0G4dczt3tOd+Z2X3ZN0X1JprRlQ6lhrB226g7bVZudMmZQRExmbvdqL9s9Nw3rptNh7+bKLHk0vZ3tfpJtDR/EkRCtvTFhP/jkvY0Ju1/2xot9/pUN/uq/dzLGi+k8py4KlOugZrPrtApnOy1TkDe2dkVdCUZuSFmr2U6rIfE9Yplz3t0hEglnVEFPbjMNfAJZIkVD0Sh23ZDtSlFJKKVC1LPhrzU0Spm007qF8lxknpvBCSf8+OK2c4KgZClOOBsPWfYnPB5sctbtc9Hps6I0rlB8vfBZr3pcKVa4ki6zk8esT9pMUp9s6sHYwUlNtruTzoruIz2bnlOYqOThhGZ759iM+zoIjoRoPciYsPvlRI4X0xo9TUzcr+vgeA40mipwQZhjoinOSyrfJQWGqmbLa9EgWHYnSNEQiIpVVRAJOOvs0mhJ1rgMAlPEn0YeCVA3GMe8EJSRgAZEo1CZMbU6qbmhIur6xDXt3kkryom8kqVgyqo/Ys0f8oi7w4oas6I0idZMa831aoEb5QLX8wW2sha7uZlPmKcuOnVwEpPtvmccdZLZfMJ0lu0+SdDj8UNzl/D9OBI+rVtjwj5y8PWsPU7qeLEmy9CTKWKSoKY5TlKZO06zPwInBZUKRCapU4dp6tNPI3bymJtFlxvlAut1hwZwhWBNJZxxdznj7bLqj1kOJnQCs7MQYU0dNlSBNnfQQpMdVbYUVezStAJEHCF9H8SR+F/tQOgEOYthYtzu/tDkYqkRK6qgKwPGjeJG3eZGucBm0WE7b9GfZbvnqUuTOmZs/d4E6FQbe0OicRIzVFVOU/RkSj2aPPSidSR2WntjwrrR4cX5XliOeeJUi8+/unHixos1eY7e2kZJicgjfCWRpQNamRNiKdCOibIpGsGO1OSVgycr0tqj0RKXmlNqwhOuxhUTPGoK7dBSOUXj4KqaHdkwaQSVqxCNMnUzJczwV1cg6ghHKZRSSK3RaXoi/+DOtQb03JSL4RZP+OusOUM+4goSLXi9LHijXOF6ucDlZJXtImZ92mFrHJNnnsl2TyTuROKOTCaWPzTWBm9Y4fYT5CSj2e6f6H7Ce2HuL3/v7JgxYZ97Zu3Qn/uzT6/y0pU+o+yE3YHRGl1V6CRFJBkqq3DS2hR0Z65qE2siEIWkyB2S3J055iO2yxZbdYedJqLUDS6wolKW1IRld8yil9D1MmK/wPEr8Bsa32RG7Q0gNQNgFU3k0kQBIggQ/vFvUn8vFr0pS96EZWfEKTVmSeY0NCRas1XHbFYddquYQRkyyI3bPc9d6txku6tc3HK734qaSRtUVpls9zQ7UcmjD8rcd1p7DdKfPQSrw7v57NOn+H/81rdO5HgxgGY8RuQ50ndxS1OP0spFlnK2E7rdo5hWkk3VUM56FH1ZMXYD2jLjjMp53G0x1Zu0ZUrSeDiiRoqGqlZMnIasEmip0NI45mvX2C2Q4EqBW5qpPs1eq8kJ4mK4xaKa8BFvgyfcjLb0uVxWrNddvlWs8q3sFNt5i+tTMz1nPAmpR94syE+aovsYM7I+1fiD0hwJhwnsDGjSh/dO4XtxJETrI6sPNibsfjnJ48UA88pcGBOiABxX4foKNNSB6bLWamZE1ZD6HlJoXFXTdjMaBFedJWAHT0zpSY1yRmy5AxotqbRiXAYIoSkLM/y10qbtRAsocoFoFDSg0gApBSrp0CQJOj/eY6zu5Iy7yyk15oyTUmrYqHOuVwusV12uFwvczDr085hBEjJNjXl0b6CqcbrfNo86SWPmE44zxGhCk87G1ltuMVfRGiYlL73Z53/xAxfn8vx748X+zTfWKesGV839tLy/7B0TxxMzLdp1cAIHcKhCgZbQuAKVznZcgctUaFynZsczLyKn3C6uqIjFDmeUJlINZ51dSq1IGo9+YO68TgOPtBY0jaDKZu1DmaCsQTSSKvVwANVpIeua+gSJ1llnl0WZsSw9NuqCYeOyXnW5WS6wVbSNYGUhSeZRZe7tYRSzwruTatxUm8SGpEZOc8Q0NZHWRXnidqYPylxF69++vknd6ANtkP4wTup4sTtp0hSaBlE3uFIic4/GCZGVBA0IQVUKtHIoK8FAC5RsmJQmqiZpPErtIL112lLzhDsiELcbq2NVUNaKvmpIVEBVm92bqE1hvlFylvsl8WqNkhLlOiemR/FJN6XUmu2m4ErVZavq8Hp2mo28w/Wky8akRZL5Jts9lTiTWb57Ohv5lTR44wa/nyOTwmS7J7P2nBNqE3kQ5rq1eOHVTTMm7FxvbmvYGy/2+VdOlvXh29jbcWUZIsmQWTkrzDe3BnuqHGRmCvNV7jDNPSaFPyvMt9mq2vSbgEQLAiFoy4IVNWLZnbDoTen4pjDv+SXab8wA2FuF+VnOfChpIhcdBYgwNJOrpZr3b+eBUQhqoF+77MyGqvbLmN0iZJQHpLlHMct2V9nsWJjddrs7qcZJK8Se2z3L0IUd0/Z+zG2nVVQN//abm/z4x9bm2rR853ixn//JZ07MeLF3Y46JY4TjIMsK11WI2tzN01IhKmF6FLWk0g5TZab6+E6PojE9ipHMmTpDev6ARVkTubtk2iWSOWltivNKaDYrSem4lJWLFhizayloHIUszeccJRBliUxSmvF4vr+cB2SsG/q1y+vlKS5na2yWbd6ZLjDIQvqTiHTsQ6ZM0X02PccdmSOhPzQ+LGeUIXdH6On02P8+Dpq5idaX3zJjwg4q8O9eMOPFtnhjc8JHTsh4sfejSRJkXSNDHwfQSlB7clacN7UoLSSV65Jpwe4sysaTFQvOEqV2OKUmdGVJWwrWnCFKNPQr46pvtCQpXKYCikJSNwqhMcV5qXFycyNAaHDSFlLK27fzj+lR8WoVsVO3uFYscSPvsV3E7KQRk8y43UXiIGfzCVVmhlF4U2NrcMclapIjRwl6PDF3Vy0fyNxE6/Ov7P+YsPvlR59a5W/w9dmdzJMtWjrPqcsKJ46QUqI8BzcwGVxVahqrtSNoMkkjFEngoaTGVy02PdNDeN3t4opdVoRgReYoNOvukFw75LVzq92nzB3q2phNqxBAUIYgaomsHVQcgNaIiQ9Zjj6morVe9dipW2yUHbaLmN3MCFaWejSJcysTy0TMzPLdU5PaoKazbPfJ9MRnu+8XcxEtrTW//toGz+/zmLD75SSOF/tAmpp6u48sSxytQbSQpUvjOMjKDLHQUlCVDrnyqStF3Qgc2TD2AwJZknkemd7mCVexqGoyfYNAlviiIm8c+m5E1UhTmFcaGueWN0wriVYCUYU4gYNTN+jBiGbcHEvH/Dez0+xWEW8ni9yYdBlnPtNRgE4d1FiZonsG/tC0UnmTBr9fopICuTMy7TnD0bHdaR42cynE740JO0rJoZ99epXfvzpga3xybsV/ELoq0bkZ6qmSEpVUd+TLc2sQqMgVVa5Ic49x4bNbhMYxX7XZqWOSpqbRmhWVzgrzYxa9KT0vJfYLXK9C+A1NcNsxf2uYaSipA8cU5qPQOOaPYU1xp4xNtnseMs09svzbs91vdyHc4XZPTZCf3vOsWcG6a+ayzZmnC/79+OzTq/ztz7/Ob762yb//fefnvZyDR2szzLMskY7CqSI8TyFqhdASrYxlQTuSqnHIgG0npqgVkVNSakWNJJAla2rKo46Hyy5KNCSNZ6wSlTGrDmRDMhskKyqJFgKkMFN93NmOSwhT38rzY7fbupouMMxDtqYx42lAnTioiboV5OeOTfOzP65xJjXesED1J5CkVP1de5fwHpmLaO2NCTt1AGPC7penT7c52wv5/KsbD4doAeiGpigRkymybnBDF/DQUpg0Um2ibDSSWjqknk/TCLZcM/zVFTU9lVBqRVsMcQU85gwZuDEKzSAMZ8V5QZG7VBLqXCA0gJg55yUydxGN+cOVaRedZjTT6dx+LffKZtJmWnhMEp966hov1nRmHp2CN5kV3Uc1TlIiRyl6NEYnqRWs++DQRWtvTNhf/dxHDvupPxAhBJ99+hS/+rtXycqawD3+/qEPRWvQ9a0/HjmNcJQJ9XMC04qjQtPqox1B7StyPIZ5gJINoSrZcLsoGgZqTFs2LErJmjOgRrDhdSgah6JWTEKfDKhCaXZzNVShAA1VZFIoRNXgtGITY3OMjJWjzCcrXON2zyRqNpDCyZh5sBozBTopkZPCuN0n04c6yO9BOHTR+o1XD35M2P3y2WdW+QcnZLzYvdCkKaIokI6DKlv4dYyWAlWaY6KsTBZ8Lh3qUtKXMUWlqBuJK2vGtSnOn3f6nFEF552EtiwotUMgSzxZUTaKkRvQbwSVZHb8nDVuNxItnZmLvkH6Hqqqjk2P4mgcUecKMXZwx2aH5c1iZrxxg79b4SQVameCmCTU/V0Timi5Lw5dtF54dePAx4TdL89dWDo548XuBa3RtdlxScdBei5O6oIEJ5NopU2PYi5AGMd86niM3Ip+EeGIms2qQyRy2rKkLTRtUbGiRoydgMT12PFjGi2Y+D55JWlqQR1oRGOK8rKUyBrq0DUN1nFkehTL6sgXqetMQSFntoZvz3Y3O60alczc7mmKLqtjs4s8ihyqaJkxYdv8B997/kg6zz1H8oMfWeGFVzf5xeM+Xuxe0do4sZsGCWaqT+lRexIxG/TaKDPVp3QdMg27GnxVk1UuoTLF+QbJx7wd2lLwuGsK866oSRoPT9YUtWJXaDKhqUoP7TCLyDFDOGTh4boSUbaRgNSaZjI50n/kYuKYgaoTgTuZBfmNZ9nugwJnkJoj4U7f3GQ44iJ81DlU0dobEzaPwL+75USNF7sPdJ7T7A5MX2DT4AUKoRVaKmqPWXFeUuNQCOh7IVUj6XgdzFTEhp5MWFQZK1KSqQm1K+n7Ma6omVQ+jTYimBZqJlQSoU3WvJNLtHKQRWDmKAKU5ZFOO3CS2wNV3eks231c4yYVzjhHjKbGPFqUoJt5L/fYc6iitTcm7NkLi4f5tPfEiRsvdo/o2Vh7kaRIIVCxT+MIHFfgBBJmE6a1lNSOIstcBNAPYnxZ48uKdbeLFA3nlGZRFtTOmHVnCJghsXnlUNWKPHRptEMdSEQFooYyFKAlVWjmN8q6QUzMSLKjOgBWpbMj4SxmxhwLa9S0vO12T5IjK7rHjUMTrcMeE3a/nMTxYvdD0x8g0wzHUcgqQtQeWrqowvQrykpQNopS+NSl4qZqU9aKvFEEsmTgRnhcZVXBk64i0zfpqYS8cfFkhe9UVLVk6niUjRFB7RjHfOOB0C5aCVxX4tQ1apJQbWweyWOiNzI1LG+s8YcN7nSW7T5O0bsDmqlNHt1PDk095jEm7H753DOrvLY+5mr/ZM/r+yB0XdPkuYmymU31MVE2GpVyy+ktckmTOaS5x6jwGexlzFdtNusWYy0odU1PFpxSY5adMUvulJ6XEPuFmeoT1MYx7/OdjvnQQUc+hAEyDBHO/Nu+3o0punN7h5VWyEl2O9v9IZwCfZAc2v8B/98/uIFzyGPC7pcffdrMRPw3r2zw558/WZN67pqmRheNqW/VNQ7geQpZKmr3tkG0cSR1LchcH60FWgsipyCtPdoyo0FS613OOJKezBg1m0ihkUKTVB6uiqkqZTxc0ly/me3kwEz4kUWIUgpVVjS7gyPnmPfG5kjoDSvcYY4cZ+jdoakPWi/WvnMoorU7LfjVL1/lpz9x9lDHhN0vF5ZjPnG+xy+9+CZ/5rlHHg6j6XuhtWmrmSqEUji+i6jcWb68RM8GwIoGSs8hB4bAhtumqBUdZ+nWpQKxQyDgMXdw63OjKsCRDVnloLUgFy5VLtHKDMcQDYBE5Y45khYtpNYg5ZG6o+hNmpnjvUAOE+PFShJjbbDsO4dyPPx/felt0rLmP55TFvz98L/9sSe5Pkj5h//urXkvZa7oqjJzFKcJcpqh0tIMX8jM5BhTgBbIVKAzRZE7jDKfYRGyU8ZsV202qzZj7VACixKW1JRTzogld8qyN6Xj5URBjuNXNGFD7ZujYhUKqtC46KtIUcceOgqQcYRQR+eFxJnWqKQyMTNJhp7aJuiD5J53Wn/tn3z1np/khVc3+OEnV3jyCBpK34/vf3yZH3pyhf/qNy7z+sZk36//f/kT37Xv1zwodJ5TFwVKSVQR4ymJLD1k7RjLQiRuNVjXtWAoNUXl4IiGonHIGtc0VjtDnnGnrKqCttyg0Iqus0CpJa6qcVXDdiOpHYeyceBWcV7iegLRBLhK4ngucjbXsUnmX3f0djNkWiL6Q5rBkOYYuPiPM/csWv/uWzv3/CS90OU/+dFL9/x98+Y//eNP85/8d39wXz/ziUNrdJqBkMgkQjkS5UmcUICEKhdoB9OjmJlUiFHg4zsxnqzYcHsoNEM1wRXQFpolNaHUDptuh6nnU9aKse+Ta0GTK+pKGyEMzFGxCiWycJCVhwpN0CBZbrxPczwqyqRApDk6mU3QPiLH1pPKPYvW7/z1HzmIdRxJnjjV5l/95c/MexlHhmY6ReQ5yvcQjTFJNp6PrOWsOC9ACxpXUdWCgRvd+t6Ok5Fph7ZMecwdsKxcHnOGxKJgGpiseikapqXHSDVMSpNVr4U0+fVKmOlBOGgJft5COgqZZnOPsxG7I3SWUY9Gc1vDw8TRu39sOdLoujZ3FLXGEcLcUawUtWdacQDjmK8FuesxNLcZiZwF8sahq1IAaj1kTUl8kTJqtm655Celj6tqytK0CtXCoSqNqbUojfEU4aCyAEdKZFnRDEfo6fzMm81giK6t0/2wsKJluTe0pkkzhOchfA+ZByglULlGudB4s5wsCVUhKQqHxPGYlD6erNitYjoypSMzHnVqfAE9lbDoTBg3AV0vo2gcAq+k9BV1Jc0EbB8aH+oc6lJQBwpRu4jQR6QuIhVz65CxtobDRWh7/rZYLMeIo9tPY7FYLO+BFS2LxXKssKJlsViOFVa0LBbLseKe7x4KIb4OHPXbJcvA9rwX8SEEWuuPzXsRFstx434sD5nW+nv3fSX7iBDid4/DGue9BovlOGKPhxaL5VhhRctisRwr7ke0/u6+r2L/sWu0WE4o1hFvsViOFfZ4aLFYjhX3JFpCiD8jhPhDIcTXhBBfFEIcuSQ7IcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rsliOE/d0PBRCfD/wqtZ6Vwjx48AvaK2fO7DV3SNCCAW8DnwOuAZ8GfhTWutX5rqwOxBCnAZOa62/IoRoA78H/MxRWqPFcpS5p52W1vqLWuvd2YdfAs7t/5IeiGeBy1rrK1rrAvgV4KfnvKZvQ2t9U2v9ldn7Y+BV4Ox8V2WxHB8epKb154F/vV8L2SfOAlfv+PgaR1gQhBCPAZ8EXprzUiyWY8N9hQAKIX4YI1rP7+9yHh6EEC3gnwJ/RWt9vzm99tbv/iAe5Js/J/+E/Xe4E6mQnotoxbDQRccB+XJI1VLkHUXRFlQRFD1NHWjqdo3TKvGDkld+5hc+9N/iQ0VLCPEXgf9o9uFPYPr6/h7w41rrozbx4Tpw/o6Pz80+d6QQQrgYwfpHWut/Nu/1WCz7iQx8RBTC0gL1QkQVu+SLLmUoKDqCsgVVpCk7DTpocNs5vXZK7BV3df0PFS2t9X8N/NcAQohHgH8G/JzW+vUH+cEOiC8Dl4QQFzBi9SeBPz3fJX07QggB/H3MDY2/Pe/1WCz7hXA9hOciFxfQrZBqIaLoelSxJO8KqlBQtKFsa+pQQ6fECyoW2glr8ZiOl97V89xrTevngSXgvxFC/MFRa/rVWlfAXwJ+DVPg/sda62/Md1XfwaeBnwN+ZPY7/AMhxE8c5BNe3hzzv/7VPyAr7fBQywEhBMJzEWGAbkc0rYCq5VLFkjKSlJE5EtaRpo4adFjjhSVxmNMLUlaCCav++K6e6p5qWlrr/xD4D+/nZzostNb/CvhX817H+6G1fpEHrKHcK3/zf3iFL7yxzU994gw//OSpw3xqy0OAcBxkK0b0ujStiHIloooUeXevfiXIe1AHmqpTI1slQViy2h2z4Cc8Eu/yeLDFonN3Q5GtI/6E88XL23zhDRMt9tKV/pxXYzlpCNdDhCGi3aZpRdRtnzJ2KGM1210JqhDq0OywRFgTRAWtMGcpmLISTDjtDTnj7nLW2f3wJ8SOEDvRaK35P/3aNzndDVhu+bz85lG7b2I57shWjIgjmsU2VS+gjB3ynjT1q72ie2h2WCKsiNo5i3HCYpBwId7hlDfikr/BJXeLRXl35QsrWieYt3cS/uDqgJ//yWfYmuT8P3/7CklREXn2n93yYMggQIQhLC/QxAHFYkjRdahCQd6VVCGUHU3Z0jRhg9vNCcKCldaU09GIVX/EU+FN1twBl9wdAnH3rhF7PDzBvPymOQ5+5tIyz11YpGo0v//OYL6Lshx/pDJHwjiiaYVULY9qdhy8VXAPoQqhmR0Jw7CgE5gj4ao/Ys0fctbdZU2NWJRGiLK71C37knuC+dKbOyzGHk+carHWDZACXrqyw6efWJ730izHFOF6qKUFdLdN3Z6ZRiNJ1pWUrW83jTadCq9VEAUFZ7tDFv0pF6IdHvc3WHOG/BFvFyUEidZcKTvs1C0ev4s1HKudlhDiF4QQf232/t8SQnz2Aa71S0KIzdmgjhPJy2/2efaxRYQQtAOXj57p8qU3bTHecn/IIEB2Wuh2TNMOqFoeZSxvF9xjZrssTRPVOGFlLA1Rymow5nQw4qy3y1l3l7OOaQKZNpr12ud6tcCNcuHu1nGQP+RBorX+ea31Cw9wiV8GfmyflnPkuD5Iubab8tzFxVufe+7CIn9wdUBeWb+W5R4RArnQg4Uu9VKLYjEgX3DJu5K8Kyk6xjRathuaToXTLum0E061JpyNh1yItnk82OQj3jpPukMedQSJho065LX8NJezVd5I786Oc+RFSwjxN4QQrwshXgSevOPzvyyE+NnZ+28JIf7zPcOrEOK7hRC/JoT4lhDiL7zXdbXWvw2c2G3H3p3CZy/cFq1nLyxSVA1fvTqc17Isxw0hkO02zuopmpUe9XKbfNEnW3DIepK8Jyi6UHQ1Zbeh6VZE3ZSF7pSznREX2ztcijd5OrjBR/1rPOMZA+lGXfFaucw38rO8lp7hjekprkzurmxxpEVLCPE9mFacT2D6Hr/vAx7+jtb6E8AXMLuonwU+BfzNA13kEeXlN/t0Aoen1jq3PrcnYNb6YLkrhEB4HjKO0O2YOvapYpcqkt/mwaoi05ajQ3MkbIc5C0HKij9h1Rtx2htw1tllRaW0hEuuYafxWS973Cx7bOQdtrOY3Sy8q2Ud9UL8Z4B/rrVOAIQQ/+IDHrv3ta8BrVlW1VgIkQshelrrwcEu9Wjx0pU+3/fYIkreNt/3Io+n1tq89GafvzTHtVmOPsJxEGGIXFqg6cbULZ9s2aMKTR9h2TKCVfQamlAjOgVxK6cV5Dza2WXZm/JEtMFFb5OzzoCPeYJMSy5XDVfKNdbLLq8kZ9jOW9yYdtmexBT53cnRkd5p3SP57L/NHe/vfXzUxXlf2RxnXNmefls9a4/nLizye2/vUtbNHFZmOfIIcastR7Zimk5E1fYp2y5lLCliQRmbu4RVrKnjBh1VBFFBJ8xYChPWghFn/AHn3T5nnQErqmDYFGzUDTeqLtfLBa4Vi6xnHbayFrtJSJp4lIl7V0s86qL128DPCCHCWTTxT817QceBPX/WsxeWvuNrz15YIilqvn7d1rUs34lQCuF5iIUeerFLtRBS9DzyriLvCMqOoJwlNVTtBtkuCds5S62E0/HoVh/hE8E6l7xNLjoVp1XIjcrhSrnIG/kaV9IV3kqWuDHpsjluMZkENGMXOb67vcWR3oHMctR/FfgqsImJntkXhBD/HfBDwLIQ4hrwf9Ba//39uv48efnNPpGn+OiZznd8ba+u9dKbfT75yN3dYrY8BAiB9H1Eu42IQ+rljsnBWnApWnLmvxLUARSdhiaukVFFt5PQCXLOtQY8EvY57Q15JrjGippyXjUMmoardck3irNslF2upCu8kywwyEK2RzFF5sLYxR1LZH53OQJHWrQAtNa/CPzie3z+z93x/mN3vP/LmEL8d3ztXd//p/ZtkUeMl670+Z5HF3DVd26kV9o+F1diXn6zz1/4wbux8llOPEKYHVbgI+KQphWZWJlQ3XK432p8DjRN1CCjCj8s6YYZC37Cqj/itDdk1R2wpia0ZY0rHMZacqPqcqPssVl02MxbDLKQYRpQpC46dVCJRCUCdXcZgEdftCz3xu604JsbY37qu06/72Oeu7DIv/zDm9SN/rZCveXhREYRIgxgZZGqHVC1PbJFxxTde4IyniWNdmdJo52cbiujF6ZcbG+z4k14IjBF9zU1ZVVJxg18pQi4Upxio+zy2nSNnTzm5rjDcBJQZS5i4OKkAnckcCfg3GUfz1GvaVnukZffMvWs5y5+Zz1rj+cuLDHOKl5bv99oesuJQCqE7yPbLUTHtOVUbY+yrYzTPcYIVqypI42OalRc0Y4zFsOElWDCGX/IOa/PeXeHJZkSCU2/rtmoPa6WS1wrFrme99hI2+ykEePEp0w8dKJwpsK8JeBONe707kTL7rROGC+/2cd3JB8/133fx9yqa13p89Ez7/84y8lGei4ijtALHZrIo+z5FG1FGUmKzsyD1dJUsaYJa7y26SNciU1Swyl/zAV/kzVnyCV3SCAEEsHlss161eXNfIW30yW2shYbkxZJ5pNPfMTYQaUCZyJwUvDGGn/UoO5yp2VF64Tx0ps7fPKRHr6j3vcxZ3oh5xdDXn6zz//8+QuHuDrLkUAqVCtGdNrodkS5FFPFDvmCQxELqti43KtQU3UbiCu8sGSlO6EXpDwW9zkf9Dnt7vJx/zptWRIIwXqtGDQB38jPsll2+NZ0hZtJh0EaMBjGNKmDHCvciUClAn9X46Tgjxq8YYVKq7tb/gH/eiyHyCgreeXG6D2tDu/m2ceWePmtPvcyYdxyAtgb7xVH6Di8I8v9jqTR6A6Xe1DjBLez3Bf9KSvemNOuaXzuyhJfQKY1gybgRrnAzaLH+szlPkgDksynSR1EJnFSI1hOCk4KbtLgTmucSYGa5h++fuxO60Txe2/t0mj41IXvNJW+m+cuLPJPv3KNy5sTLq22D2F1lqOAWugiwpBmuTszjTpkC8qE9/VmSQ2hpurViKCm1U1ZiFIWg4RL7U1OeSOe9G/ymNtnTdXUGgaN5I1ymSvFKW4WPd4Yr7CbR2xPYpKJT5M4OAMHlQncsTkOOgmE/QonqXF3M+TuGJ1kd/UzWNE6Qbz0Zh9XibvyX+255V96s29F6yFgb7yXaLfQUWAEqzPLcm8JquCOHVbUIGMz3qsbZiyHE1aCCWf9XdacIY84u8SiotCwXbts1i2ulktczxfYyNtspy3GmU+a+DRTF5lKnESgslnBfaJxU407qVDTEjlK0NMEnd6daNnj4QnipTd3+Pi5HqH3/vWsPR5ZjFjt+Lxk87UeCkTgI3tdml6LqhdSdF3yjiJvC4q2cblXLU3VrtGtiqiV02slrMUjzkUDLobbXPQ2uehtcsFtiATkGt6qlrhSnOJKusLVdIGbSZf+NGI8DajHLmqscCbG0rC3y9qrYTmDDDmYoneHNMMxzXR6Vz+L3WmdEJKi4mvXhvxHP3Dxrh4vhOC5C0u89OYOWmvMDFnLSUM4DnJhAbotmlZIvhxSh4psQVG8K2m07tSoVkkYFpzpjFgIEi5G2zzi73De2+EZd5tAwLCBG1XIet3l6+k5Nos2b02X2EpixmnAdBBCJnFHCmcscDJmRXeNP6zxBgVqWiA3d9FpSjOZouu7z3izO60Twu+/M6BqNM/dRT1rj2cvLLIxynl7JznAlVnmhXAchO8jWhE6Dm5nuceScpbjXu+53EONjCqT5R5mLAVTlr3prfFea2pEWwqUEAwah/XaND5vFm22Zy73cRqQpR4iVcblnhrBchKNm+hZ0b0ygjVO0dOEJs3QVQX3cEPI7rROCC9d2UEK+J5H776f8FMX9/K1+jy2HB/U0iyHjRAgJGplGR2HVMttypY7K7pLqmBmaWhpqlDT9CqcoKLXSViKpiwHU55qrXPKHfGUf4PHnAmryudapdmqQ14rTvN2vsxG0eFbo2WGeUB/FFFOjWC5AyNY3ghTv0o0wU6Jk1Q4uwn0hzSTKU2S3JNY7WF3WieEl97s87GzXdrB3cV7ADy+0mIp9viSDQU8OQiBDEPUQhfdadF0I8qOR9lxKFrSxMq0ZoIVaZq4xo0KoihnKZpyKhxzNhxwzuvziLvDY84ECfTrnKtVh3eqRa4Vi9zIu6ynbfppxCgJKBMPMVWoqcSZCtyZy92bNHjjGndU4AwzxGCMTlJ0Ud6XYIHdaZ0I8qrm968O+LOfevSevk8IwbMXFu3k6ROEcFxEFEKvQ70QUYcORUdRxEawypYZPFG29tpySrqtjE6QcTYacjowbTl7fYSPOC2uVRNu1B5vlctslF3eTpe4mXbYzcLbfYQThTMxPix3Ak6q8cYN3qjGmVaoYYqYptT9XXRZQXP/cwqsaJ0Avnp1SFE135YHf7c8e2GRf/31da4PUs727i7u1nL02KtfycUFdCem6gQUix5lJMl78lZwX9HV1GED3ZIgKmlHGefaA5b8hI/E6zzqbXPW2eVJNwXg1SLharXAjWqBbyRn2S5avDNZYHsSk6Ye5dBHZnLW9Gz6CP1Bg5tovEGFO8yQ4ww2t2myHF0U973D2sMeD08A7zXE4m55buaet7nxxxghTME9DNCtkDr2qGLjwapCM6L+VpZ7NEtqCCpae1nuwYRVf8SqM2TNGbKiUlwhaYCNusWNaoEbhfFgbWUtBmlAmrmUmYNMTf3KvIFKTQ3LSRqcSYEcZ4hpSpNmNA9wJLwTu9M6Abz0Zp8nV9v0Iu+ev/fJtTadwOGlK33+p588dwCrsxwkt3ZYp5bRUUCxElO2HMqWJOsJ6kBQ9GaCFTeIbkEQlJzqTFgOJ6wFY56M1ll1B3zUW2dFNXSlx6tFw2bd4ZX8LFezRTbyDm+NFhlnPpNxYEyjicQbSJwMvKE2TvdUE2wXqKRA7YzR/QHVZPpAx8F3Y0XrmFPWDb/39i4/+z33JzhKmrrWy9ZkeuwQroeMQ0Rsstzr2LslWEVshk/UvhGsqlVDWBPHObFfsBxOOBMOWfNGPOptsaQmdGVNpmFc5bxTnWK96nE1W+R61mMnixkkIVnm0kxd1MTssNwpqHR2l3Da4E5qnFGGmGbo4Zgmz/dVsMCK1rHnGzdGJEV965h3Pzx7YZEXXt1kc5RxqhPs4+osB4ZUyDBAtFrobouqF1BGDkVnVnCPTR9hHRqXu4gq/KhkIUrpBSnnogFn/V3Ou30ec/v0ZEVPOlytGm7Uba4Up9gsOlzPemwkbYZpQDr1aFIHNZG4E3lLrJwUc5dwWJkj4XCKnkypd3cP5Ee3onXMeemKqUV934X7z3u/Vdd6q89PfvzMvqzLcnDcShpd7FF3Qqq2T77oUoaCvCdvjacveg1N0OB0ClpxRjfMuNDZYdGd8mS0zmPuFmedEYuypga+XrgzS8MSr05Ps1uEXBv3GE5D8tSFgYuTzoruU2MaDXYb3KTB251ZGiYJzdaOKbgf1M9/YFe2HAovv9nn4krMqfb975A+eqZD7ClrfTjqzMZ7iTBARBFNK6Tec7nfGSsTmqSGJmgQYUUY5bSDnMVgyrI3Yc0fctbtc2p2JKyBcSO5Xi3MomW67OQRO1nMJPMpMgedOKhUorJ3udynDU5So6YFYpqiJ1OaNDUu9wPC7rSOMXWjZ7uj98+DvxscJfnuRxdsXeuII30f0YphZZE68siXQ6pYUbRMlnsVziwNkaaJarxuThQUnO6MWA3HnPaHPBXeYM0Z8oy3iycEIHilMEmj38xOcy1bYCtrcXXUJcl8spGPmCrcROIOZ0mjQ40/bnCmDf5OZgRrZ0AzGhuX+0H/Hg78GSwHxmvrI8ZZ9UD1rD0+dXGJb26M6U8PbltvuU+EQMYxotsxptFuSNXxKdtGsIqWuJXlXsXG5a5aJZ04YzFOOB2OOBsMeMTf4TFvmzVnjAIGDVytXN4ql3m7WOZatsDNtMNG0mI8DcinxuXuTOXtpIaJcbm74xpvVBrT6GhKM54c6JHwTqxoHWP2jnP34896N3vX+PJbdrd11BBKmR1WO77VllN0TVtO0b6jLSfW6LjCiUviWVvOajjmXLh7S7AecyYmvA/o1wHfKld4O182dwmTLttJzHAaUk49mDo4E4k7NjUsI1gab1TjjkrUMEMMx+jhiGY6PdAj4Z3Y4+Ex5uU3+5xfDDmzD072j5/r4juSl670+WMfXduH1Vn2A9WbJY2u9KhbPmXHM0mjgSBfuD18ouzWEDRE3ZROZMbTX2xtc8ob83Rwg/PuDmeUiTMeNHClXOKtYpmbZY9vTlbZzSLWx22miU+dOKhZ0qg3EjgT00cYDGqcaY23m6F2p+hpQt0f3FOszH5gd1rHmJff6vPsYw9+NATwHcUnH+nx8lvWGX8kEMKkjYYhuhVRt3yqlksZzxzue8NTZ1nuhDVOUNEOc7p+xkowYc0bccodcdbZpScLAiEYN5J+HXC9XGCj7LKed9jJYoZ5QJp61IljomX2stwTTME91caDNa1MW06Smsbnqtx3H9aHYXdax5j+tLgVm7wfPHthif/bb7zBKCvp3ENahGX/kVGEXOjRLHepY4980aOMTf0q3xtP39XUcY2ITJZ7K8g53x6wFow46w94Orhuiu5uTaKh38A3y1Osl11eS0+znnXYSltsjNpkmUs99ExKQyLwhqYtJxg0eOPG7LC2p4gkh80d6jRDl/Opf9qd1jHnXkL/PoxPXVik0fC7tq41P6RC9brIxQWaXpuq61N0XPKOnMUiz4ZPtDR1u0bEFUErZylOWImmnA93OR/0ueBv8ri7w5rKGTcVNyqHt8oebxXLvJMvcSPtspG06U8j0qlHPXFRE4U7mUXLjGf1q3GDO65wRzlyOIXhhCbPD/1I+G2/ork9s+WBWe34PLIY7dv1PvnIAq4SvPzmwTiZLR/C3nivbgfdiam7AUXbnU18Nm05t+8SNoioIogLunHKcjjhdDjkrD/gUW+bx9xtzjmwKB36jWK97vBWucLVbJFrWY/NmWBNprM+wqnCnQqcKbgTjGBNjGA5wxw5TEzBfTRCH0Brzr1gj4fHmOcuLO1rtnvoKT5+rsdLNvHhcBECoRRyaRERhdTLHaqWS9F1yLrK+K96wvQQxibLXYYVve6UXpixGo65GG+z6o74I8FVzjhjzimXjbqm33i8Vpy+NZ7+ymSZQRayPYopZuPp3aGpYXnDO5JG98Z77UwRw4nxYE0T0M28f1t2p3Wc2Q+rw3td82vXhiTF4dy+fugRwgT3hSEiCmnicFZwdyhDSRVBPYuWqQOowwYVVQRhQSfITZa7P+G0N+CMu8uqmtAWmoaGfuOxVbe5Ufa4WXTZzNvsZiGjzKdIXfQsy91JxGxw6qzgnjQmuG9SIMaJKbhns93VERjua3dax5j/ySf2v0/w2QuL/N//7bf4/XcGfPqJ5X2/vuXbueVyX+hSLbWoIpd80aGMTA2r6Jim56JrkkaduKTXSegEGRfbO6z6Ix7xd3jKv8GamvK4E7JZJ7xSKl7Jz7JRdnl9uspG1mYnjdgZxlSZC0MXNxE4U7PDclJNMDCmUXdS4WyNEUlGvbFl7hAeAbHaw4rWMeYg7vB976MLSGEasa1oHRzCccx4r14XfYdptIokeVtSRSYauWxrGl+jWxVOWNGKM1biCYt+wrlgl3Nen/PuDmfVhEjA9TrhRhXyVrnMm/kKO0WL60mXfhoxTgLKiYfIlEkZnRpLw14OljeqccYlziiD4Zhmz9JwhAQLrGhZ3kU7cPnomS5fsn2IB4ppfA7R3TZNy6fs+BRtM56+nM0jrOJZH2HQ4MYFcVjcastZ9KY86m9zxtnlvDNkWSkarblSBlwtl7haLnIj67GTR7cEK09cxNSYRp29ontiCu5O2hjBGqaIcUI9GO5LNPJBYEXL8h08d2GRf/ilt8nKmsD98GnVlrvnVtLo8iI6CiiXWya4r63Iu4IqFOQ9YxqtYpPl7vkVK90Ji2HC6XDIk9EGy86Ij/vX6cqSnpS8XQn6dcw38rNczxe4lvV4Z7zIOPcYjmLqiYNMFN5QInPwB9rcJZw2+P0SZ1qapNHhiHo4OrSWnPvBFuIt38GzFxYpqoY/vDac91JOFlLNhqfG6DikaQVUsUMVScpIzOJlZoIVanRY4wUlcZizGCas+BPW/BGrrsly78oSV0CmG7bqmOvVAjeLHht5h34eM8p8pqlpyxGZQmUClYGTmDc3bW5nuU9z9HiK3hueeoSxOy3Ld7B3V/KlKzsHcofyoUQIVKeFaLWMaXQppIwcskU1q1/dLrqXvdrkYMUFy+0pvSDlUmuTNX/Io942H/XW6cqaWEjWa7hadXklP8tm0eHydIWttMUgCRmNQpM0OnTMHcJkNp4+g2C3Nh6sSYHaGpo+wu3jYXWxomX5DnqRx1NrbV62zvh9Qfi+uUu40DOWhl5A0XFv3SGsw1nRvaVpwgbZKgnCgl6cshaPWPanPBbscMbd5TF3m0jUNMDbleJ63eVqscRb2TLbeYv1aYdBGpAkPs3ERaby1mgvZzorumcab1TiDHPEJEUPR8bScEywomV5T567sMg/+b1rlHWDq2wV4UGQYWCSGjqm8blsuxStWeNzPEtqiE0fIUFDFBW0Q5PUcCYcsuxOOO/tcMbZ5Ywy/X6ZhndmSaPv5EusZx12s4jdJDSNz7O2HJUbsXKnGndqstydpDYu91GCHk+pR5O5OtzvFStalvfk2QtL/IN/9zZfvz7kk4/cf/78w4zwfWS7BYs9msg3SaMtRd5WFF1BFUDRMykNdavG7RQEYcHp9pilYMr5aJePBOusuQOecbdxBdTAlapFv27x9fQc63mXG2mHa+Me08wjGYaIxAT3eUNTw/IHejaivsbrF6hpjtwa0EymxuV+jAQLbCHe8j7cqmtZ68N9IVxvNi0npokDqpZH1VKUkZxluEMdmhpWHTWIsCYIC9ozl/upYMyqO+Ksu8uKGhMIaIBho9iqOlwvF1jPu2zmLfpZzCT1yTMPkSpkasbTO6kxjd7Kcp/WqHGOnGToNDWWhmMmWGB3Wpb3YaXtc3El5uU3+/yFH3x83ss5XkiFWlowOVi9mGIxoIok6cLMh9WGsqOpA03du20aXW2PWfQTnmxtcNob8Ji7xTPeLrGQjDW3TKPfzE6zVbS5Mlmin0YMp6HJcs8UznCW0jA1Oywnm/UR7plGt3dpkpRmPJ73b+m+saJleV+eu7DEv/zqDepGo+T+NWafZGQcm/H0Cx2aVkDR9Sg6ijKUJlYmMgX3sqXRQY3XKojDnJV4yvl4wKI35YK/yVl3l8ecIQ0waBrerjpcrxZ4O1/mnXSRnTxmY9JimvrkUw85dpCZuBWN7ExNrIyTNLiDHDXOTJb7ZIouynn/mh4Iezy0vC/PXVhknFe8enM076UcD4RARBGi3aJuB5Rtj7LtUMSSMsbEykSaKjJZ7qpV0YoyFqKUlXDCmdnwiUfcPuedEaeVR61h2Li8Uy5yrVjkRt7jZtphO42ZJAFF4iESB5UI3OkdgjU1SaPuuLwtWLNpOfMK79sv7E7L8r7s1bVefrPPx85257yaI8xs2rPsdWkWOmZ46pI/6yMUFN1Z0mhPU4cNulURdTLioOCRzi6nggnn/F2eDG6y5gz4mJeTa7hWl7xRLrNe9ngtPc3NrMtG2mZj3CJLPcqhj0zMcdAbmRqWP2hM0X1U4e2kJml0q0+TpmZE/RFsy7lX7E7L8r6c6YWcXwxtvtaHsFdw162Iuu1TR7dd7lVkBKuKZoIV1rhhSTvMWQhSTgUTVrwxZ7xdzjq7rKiURmumjWarDrleLrJRdtnI2/TziGEWkCY+ZereUXCf+bBuDVCtUUmFmGZmgGqa0hRHr/H5frE7LcsH8tyFJX791Q201vsaOHiSkKsr6NCnXAgpeiapIVsQt4vubSNYolcQhCULrYTT8Yglf8pHonXOuLs87m7xpNvgi4BXy5Ib1QJvFcu8ka6yVbR4a7TEMA3MtJyRh0wl3kCaO4QTk+XuZJpgp0CNCuQ4mXuW+0Fhd1qWD+TZC4vsJiVvbE7mvZQjS70QUy6ElF2XoiPJO4KiMxOslqbq1NCuiNsZC62E1WjMo1Gfi+E2l/x1Hne3eNQpGTQVb1YZV8plvlWc4s18hbeTRa5PeyYaeRJQj12ckTIF9wm4I4031nijBm9Q4Qwy1GACu0OaNJtrlvtBYUXL8oF8aja92vq13p+qvedyn2W5x3dEy8yy3L2ooDdzua+FY876A855fc6qIauqoCsDho3iRtXmarHE9XyBm1mH7bRlXO6JRzM1SQ2mj1Dg7Lncpw3upMIdF8blPhrTDEdmh3UMfVgfhj0eWj6Q84sha52Al67s8HOfenTeyzmSpCsutS/IO5Kyw6zo3tCENapd0mmldIKcC50dTvljLvhbPOXfZEVNedQRDBr4w6LmteIcG2WXV6en2cpabKUxO6OYInNh4OIkEneWNKoyTbDb4E4b3FFpstwnKfXmFrqsTqRY7WFFy/KBCCF49sIiX7qyY+ta70MZSWofs7sKoQ40TVgjwpowLOiGGQt+wil/fCvLfVEltGVNogXDRnG96nGj7LFZdNjKWvSziFEazLLcHZzEFN3VzOXupMbl7k5NUoMYJ7dd7iek4P5+WNGyfCjPXVzkX3z1Bm/tJFxYjue9nCNH3hM0PhRtTdWeJY12c+KwYLk15Vw8YNmb8HR4gzPOLo+7uyxKCUi+VkbcKBd4u1jmcnKKnTzi+rhr2nKmHowcnNT0ETqpmZYTDEzSqN/PUeMcMZyYLPcTVnB/P6xoWT6U5275tXasaL0HZdvsrqqWpmlXqLCi105ZCFLOxEMeC3c47Q245K3TkzltKVivYaxd3sjXuFn2uJotcHXaY5gHDMchZeoipgp3JFGZwBsbS4M30abgnlSo/hQxnpr6VXW8Xe73gi3EWz6Ux1daLMUeL12xxfj3oopNUkMT1bhRSRSZpNGlYMppf2gK7m6fMyphUdZIoN+YLPdrxSLreZeNtEM/jRglAWXiIhJ1a0S9ycIySQ3upJ6N98qNYE2mNEly4o+Ed2J3WpYPxda1PphyoQa/JmgVLLanLAQpT7S3OOWOuehv8pS3zqIqCYRgpxZcrbq8UayxWXb45mSVnSxmexIzGoc0mUINHFRq2nK8gcZJIRjUZrzXuEBtj9BJSrW1cySGpx42dqdluSuev7TMjWHGt7am817KkUOEFW5Y0pq53FeCCae9IWe8Xc64u7RliQQGDWw1ETdmWe7reYedLGaQhiSZR5M4iNRMe97bYbkJuEljdlhJaWJlkhSdpEdmeOphY3dalrviBy6tAPBbr2/xxKnWnFdztGh1U2K/YDWa8Ejc55Q35sngJmecXS46BaU2SaOvFSu3kkavJMvsZhHro/btpNGRGT7hDWfzCCeaYLdGpbXpI5ykMBjRDIZHfvjEQWJFy3JXnF+MeHwl5rde3+LPP39h3ss5Uqy2J7TdjEfiXR4Ptlh1BzzpbhKIhkzDjdpnUEdczte4WXS5mi5wbdxjnPlMxwE6cVATiTuSOBn4Qz0rujd4gxKVFMidkekhHE9OpMv9XrDHQ8td84MfOcVLV3bIyof7j+bdLAVT1sKxORK6e43PmlgKcg1bdZur5RLX8x43sy47mRnvlaYeOnGQiTRHwpRvb3ye1DjjHDnO0JOpeTshSQ0Pgt1pWe6aH3pyhV/6nTf5H756g+9+9OTkxj++8mDH3Y+1b7DgTLnkrXPJ3aUnJYNGs9X4vFUuczlbY73ocHm8wjAP2J1EpKMAMok7UDip6SP0hsY0GvQr3EmFGmbInYEZ7zUeP/RitYcVLctd8+yFRSJP8b/57/9w3kvZV976L/74A33/aXfAkjPhrDO6PTy18VmvutwoF7iR99guYnazkEnmk6fuLMt9VnBPb++unNT0EapJjpwYl/tJycHaL6xoWe6awFX8yn/8Kd7ctncQ7+Qp/wYrKuWCE3CzTtmuXV7LT3OzXODtbIl3pgsMspD+KKZMXZg4OGNzJPRGM9PoWOMPKjPeqz9FjBOa3QFNmp3oPsL7wYqW5Z74+LkeHz/Xm/cyjhRPuikNcK1KebvqsF51eT07zVbR4up0gY1JiyTzKUY+IpU4E4k7Njssb6hxkwZvbNpyZFLA9oAmy6xgvQ+2EG+xPCDLKsYXkmHjslm32ao69MuYnTxmlAekuUeRO4hcojLTluNkJqnByUwdy5lWyKQwSaPTKTpNrWC9D0Lbs7LFYjlG2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrrOXhGCOE+DqQzXsdH8IysD3vRXwIgdb6Y/NehOXusKJ1vMm01t8770V8EEKI3z0Oa5z3Gix3jz0eWiyWY4UVLYvFcqywonW8+bvzXsBdYNdo2VesI95isRwr7E7LYrEcK6xoWSyWY4UVrWOIEOLPCCH+UAjxNSHEF4UQ3zXvNb0bIcSPCSG+KYS4LIT46/Nez7sRQpwXQvymEOIVIcQ3hBB/ed5rstwdtqZ1DBFCfD/wqtZ6Vwjx48AvaK2fm/e69hBCKOB14HPANeDLwJ/SWr8y14XdgRDiNHBaa/0VIUQb+D3gZ47SGi3vjd1pHUO01l/UWu/OPvwScG6e63kPngUua62vaK0L4FeAn57zmr4NrfVNrfVXZu+PgVeBs/NdleVusKJ1/PnzwL+e9yLexVng6h0fX+MIC4IQ4jHgk8BLc16K5S6wbTzHGCHED2NE6/l5r+W4IoRoAf8U+Cta69F9XsbWWPYHcTcPsjutY4IQ4i8KIf5g9nZGCPFx4O8BP6213pn3+t7FdeD8HR+fm33uSCGEcDGC9Y+01v9s3uux3B22EH8MEUI8AvwG8Ge11l+c93rejRDCwRTifxQjVl8G/rTW+htzXdgdCCEE8A+Avtb6rzzg5fbtjygtajbHGY8uxft1yeOE3WmdYH4eWAL+m9nO60ilFGitK+AvAb+GKXD/46MkWDM+Dfwc8CN37GB/Yt6L+r/++ht87r/8bW4M0nkv5chid1oWy4Ozb39EP/Z3fpvX1sf82T/6KH/rpx+6iC+707JYjhOb44zX1se0A4dfefkq68Ojnu84H6xoWSxHhN+5bAJe/88/+3Earfl7X7gy5xUdTaxoWSxHhC+8sc1i7PHvPbPGD35khc+/ujHvJR1JrGhZLEcArTUvvrHN9z++hJSCz1xa5u2dhHd2knkv7chhRctiOQK8sTlhc5zzmUvLADx/aQWAL1zemueyjiRWtCz3hRDiF4QQf232/t8SQnz2Pq9j0xYwR0O4LVaPr8Sc7ga8+MZRH2R0+Ng2HssDo7X++Qf49gr4q3emLQghPv+wpS184Y0tLi7HnO2FAAgheP6JZf7NKxvUjUbJu3IDPBTYnZblrhFC/A0hxOtCiBeBJ+/4/C8LIX529v5bQoj/fM/0KoT4biHErwkhviWE+AvvvqZNW4C8qnnpSp/nZ0fDPZ6/tMwwLfna9eGcVnY0saJluSuEEN8D/EngE8BPAN/3AQ9/R2v9CeALwC8DPwt8CvibH/Icj/EQpi185e0BaVnz/BPvEq3Zxy++Yetad2JFy3K3fAb451rrZJaG8C8+4LF7X/sa8JLWeqy13gJyIUTvvb5hn9IWjiUvXt5CScGnHl/6ts8vtXyeOd25Ve+yGKxoWQ6CfPbf5o739z7+jjrqw5628OIb23zifI9O4H7H1z5zaZmvvLPLNK/msLKjiRUty93y28DPCCHCWcH8p/bjorO0hb+PiY/+2/txzePEICn4w+vD7zga7vH8pWXKWvPym/1DXtnRxYqW5a6YFct/FfgqJin1y/t06SOZtnBY/M7lHbSGH/jIe4vW9z22iOdIe0S8A2t5sNw1WutfBH7xPT7/5+54/7E73v9lTCH+O752x+de5C67+08iL17eou07fNe53nt+PXAVzz62yIvWZHoLu9OyWOaE1povvLHNpx5fwlHv/6f4/KVlXt+YsDGyqQ9gRctimRtv7yRc201vte68H7etD/aICFa0LJa58YVZFM37FeH3eOZ0h6XY48XLVrTAipbFMjdefGOLs72QC8sfnAcvpeDTTyzzhTe2sUnDVrQslrlQ1Q1f/NYOzz+xjHF9fDDPX1pme5Lz2vr4EFZ3tLGiZbHMga9eGzLOKj7zPlaHd7NX97J1LStaFstcePGNbYSATz9+d6J1uhvy+Ep8qw72MGNFy2KZAy9e3uJjZ7osxN5df89nLq3w8ps7ZGV9gCs7+ljRslgOmUle8fvvDL4jiubDeP6JZbKy4Stv7x7Qyo4HVrQslkPmS9/aoWo0n/kQq8O7+dTjSzhSPPRHRCtaFssh8+LlbQJX8j2PLdzT97V8h+9+ZIEvPOT5Wla0LJZD5gtvbPHshSV8R93z9z5/aZlv3BjRnxYHsLLjgRUti+UQuTlM+dbW9J6Phns8f2kZrW8Pdn0YsaJlsRwit6fu3J9offxsl3bgPNR+LStaFssh8uIb2yy3fJ5aa9/X9ztK8v2PL/Hi5Ye3pceKlsVySDSN5ncub/P8E0t31brzfjx/aYXrg5Q3t6f7uLrjgxUti+WQeHV9xM60uDWQ9X75gb2Wnoe0rmVFy2I5JPbqUB8WRfNhPLoUc34xfGgjmK1oWSyHxIuXt/nIaou1bvDA13r+iRVjUq2bfVjZ8cKKlsVyCGRlzUtv9nn+iQc7Gu7xmUvLjPOKr14b7Mv1jhNWtCyWQ+DLb/UpquZDo5Xvlu9/fAkheCiPiFa0LJZD4MU3tnGV4NkLi/tyvV7k8fGz3YfSr2VFy2I5BL7wxjbf/cgCsb9/U/uev7TM718dMM7KfbvmccCKlsVywGxPcl65Odq3o+Eezz+xQt1ovnTl4Zo+bUXLYjlg9voEH9Sf9W6++9EeoaseutQHK1oWywHz4hvbdEOXP3K2u6/X9R3FcxcXH7q6lhUti+UA2Zsi/eknllDy/lt33o/nn1jmyvaU64N03699VLGiZbEcIN/amrA+yvbNn/VuPjM7cr74EB0RrWhZLAfIno9qv4vwe3xktcWptv9Q+bWsaFksB8iLb2zz6FLE+cXoQK4vhOD5J5b54rd2aJqHI6rGipbFckCUdcOXruw8cIP0h/H8pWX604JXbo4O9HmOCla0LJYD4vffGTAt6gM7Gu6xl4L62w9JXcuKlsVyQLz4xhZSwB+9yynS98updsBTa+2HxvpgRctiOSC+cHmb7zrfoxu6B/5czz+xzO++tUtanPzp01a0LJYDYJiWfPXq4L6n7twrz19apqgbXn7r5Lf0WNGyWA6Af/etHRq9/60778dzF5bwlHwo/FpWtCyWA+DFy1vEnuKTj/QO5flCT/E9jy48FH4tK1oWywHw4hvbfOriEq46vD+xz3xkmdfWx2yOs0N7znlgRcti2Weu9hPe2knueyDr/fKZWavQFy/vHOrzHjZWtCyWfeagW3fej4+e6bAQuSf+iGhFy2LZZ168vMVaJ+DxldahPq+Ugu9/YpkXL2+d6OnTVrQsln2kbjS/c3mH5y8tP9AU6fvlM08sszHKubw5OfTnPiysaFks+8jXrw8ZpuWhHw332KujneQjohUti2Uf2RtV/+lDMpW+m3MLEReX4xMdwWxFy2LZR77wxhZPn+6w3PLntobnLy3z0ptmzuJJxIqWxbJPJEXF7729O7ej4R6ffmKZpKj5yju7c13HQWFFy2LZJ156s09Z6wPPz/ow/ujjJo/+pKY+WNGyWPaJL7y+jefIfZsifb90ApfvOtflC5etaFkslg/gxctbPPvYIoGr5r0Unr+0wteuDRgmJ2/6tBUti2Uf2BhlvL4xOfTWnffjM5eWaTR88Vsnb7dlRcti2Qf26kfzrmft8YnzPVq+cyKPiFa0LJZ94MXL2yzFHs+c7sx7KQC4SvKpi0snshhvRctieUC01rx4eZvvf2IZeQBTpO+Xz1xa5p1+wts703kvZV+xomWxPCDf3BizNc7n7s96Nye1pceKlsXygLw4pyiaD+PicsyZbnDijohWtCyWB+QLb2zz+ErM6W4476V8G0IInr+0zBe/tU1Zn5yWHitaFssD8vKbfT5zSAMs7pUf/9hpRlnFr7z8zryXsm9Y0bJYHpC0rI+M1eHd/NCTK3zq4iJ/+/OvM0xPhtHUmfcCLJbjjiMFn3p8ad7LeE+EEPxnP/kMP/lfvcjP/f2XON0N5r2k9+W//bnvvavHWdGyWB6QP/G952j5R/dP6aNnuvzvfuwp/j+/f523d5J5L+eBESc5S9piOSTsH9H+cFcmN1vTslgsxworWhaL5VhxdA/iFsvx4ej07jwE2J2WxWI5VljRslgsxworWhaL5VhhRctisRwrbCHeYnlAhBBfB7J5r+NDWAaOetxDoLX+2Ic9yIqWxfLgZFrru+tBmRNCiN89Dmu8m8fZ46HFYjlWWNGyWCzHCitaFsuD83fnvYC74MSs0TZMWyyWY4XdaVkslmOFFS2L5QEQQvwZIcQfCiG+JoT4ohDiu+a9pncjhPgxIcQ3hRCXhRB/fd7reTdCiPNCiN8UQrwihPiGEOIvf+Dj7fHQYrl/hBDfD7yqtd4VQvw48Ata6+fmva49hBAKeB34HHAN+DLwp7TWr8x1YXcghDgNnNZaf0UI0QZ+D/iZ91uj3WlZLA+A1vqLWuvd2YdfAs7Ncz3vwbPAZa31Fa11AfwK8NNzXtO3obW+qbX+yuz9MfAqcPb9Hm9Fy2LZP/488K/nvYh3cRa4esfH1/gAQZg3QojHgE8CL73fY6wj3mLZB4QQP4wRrefnvZbjihCiBfxT4K9orUfv9zi707JY7hEhxF8UQvzB7O2MEOLjwN8DflprvTPv9b2L68D5Oz4+N/vckUII4WIE6x9prf/ZBz7WFuItlvtHCPEI8BvAn9Vaf3He63k3QggHU4j/UYxYfRn401rrb8x1YXcghBDAPwD6Wuu/8qGPt6Jlsdw/Qoi/B/zPgLdnn6qOWmOyEOIngL8DKOCXtNa/ON8VfTtCiOeBLwBfA5rZp//3Wut/9Z6Pt6JlsViOE7amZbFYjhVWtCwWy7HCipbFYjlWWNGyWCzHCitaFovlWGFFy2I5xgghfkEI8ddm7/8tIcRn7/M6gRDiZSHEV2dJC39zf1e6f9g2HovlhKC1/vkH+PYc+BGt9WTmTn9RCPGvtdZf2qfl7Rt2p2WxHDOEEH9DCPG6EOJF4Mk7Pv/LQoifnb3/lhDiP5+1Gv2uEOK7hRC/JoT4lhDiL7z7mtowmX3ozt6OpInTipbFcowQQnwP8CeBTwA/AXzfBzz8Ha31JzBu818Gfhb4FPCeRz8hhBJC/AGwCXxea/2+SQvzxIqWxXK8+Azwz7XWySwJ4V98wGP3vvY14CWt9VhrvQXkQojeux+sta5nIncOeFYI8aGDU+eBFS2L5eSSz/7b3PH+3sfvW8/WWg+A3wR+7MBW9gBY0bJYjhe/DfyMECKcRRP/1H5cVAixsrf7EkKEmHjm1/bj2vuNvXtosRwjZjnqvwp8FVN7+vI+Xfo08A9mmfIS+Mda63+5T9feV2zKg8ViOVbY46HFYjlWWNGyWCzHCitaFovlWGFFy2KxHCusaFkslmOFFS2LxXKssKJlsViOFVa0LBbLseL/D9e1JFwR1+cdAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "condition = posterior.sample((1,))\n", + "\n", + "_ = conditional_pairplot(\n", + " density=posterior,\n", + " condition=condition,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + " figsize=(5,5)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This plot looks completely different from the marginals obtained with `pairplot()`. As it can be seen on the diagonal plots, if all parameters but one are kept constant, the remaining parameter has to be tuned to a narrow region in parameter space. In addition, the upper diagonal plots show strong correlations: deviations in one parameter can be compensated through changes in another parameter." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can summarize these correlations in a conditional correlation matrix, which computes the Pearson correlation coefficient of each of these pairwise plots. This matrix (below) shows strong correlations between many parameters, which can be interpreted as potential compensation mechansims:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARsAAADxCAYAAAD7hRNxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWV0lEQVR4nO3df6xcZZ3H8ffn3tvSdVGLFAEBKYZmlxp2QRvAkCgLiIUYWhG13awUF3JXg7urqEEkkRWXWHYTu/gTG6iAEsCtiDViWOTHglFYKlsoPxapoNJaqZRSJIXC7f3uH+eZMkzv3Dm3c+bcM+d+XuakM+c8M88ZoV+eX+f5KiIwM+u1gcm+ATObGhxszKwUDjZmVgoHGzMrhYONmZXCwcbMSuFgY1ZTklZI2iTpwTbXJekrktZJekDS25quLZH0WDqWFHE/DjZm9XUlMH+c6ycDc9IxDHwTQNIbgAuBo4GjgAsl7dXtzTjYmNVURNwJPDNOkQXA1ZG5G5gpaX/gPcAtEfFMRGwBbmH8oJXLULdfYGbF+fODZsSOF0dzld3+9MsPAS82nVoeEcsnUN0BwJNN79enc+3Od8XBxqxCRl8c5ZDTZuUq+3/LN74YEfN6fEuFcTfKrEoEAwPKdRRgA3BQ0/sD07l257viYGNWMVK+owCrgDPSrNQxwNaI2AjcDJwkaa80MHxSOtcVd6PMKkTAQEFNAEnXAscBsyStJ5thmgYQEZcBNwGnAOuAbcBH0rVnJH0RuDd91UURMd5Acy4ONmZVIhgcKqbZEhGLO1wP4Jw211YAKwq5kcTBxqxiVNPBDQcbswqRYKCgAZmqcbAxqxi3bMysFEUNEFeNg41ZhUhu2ZhZSQYHPWZjZj0muRtlZqUQKuZRhMpxsDGrErdszKwsHiA2s56TPEA8prR94PXAbOA3wAfTzl6t5XYAa9Pb30XEqd3Ua1Znde1GdfuzPgvcGhFzgFvT+7G8EBFHpMOBxqwNARpQrqPfdBtsFgBXpddXAQu7/D6zqS0NEOc5+k23Yzb7ps12AP4A7Num3AxJq4ERYGlE3DhWIUnDZLu8oyG9ffrMqTOkdOCWmZN9C6V56o3PTvYtlOr5jS8/HRH75C1f0+cwOwcbST8F9hvj0gXNbyIiJEWbrzk4IjZIegtwm6S1EfHr1kJps+blADP2mR6zF+bbi7UOvrTytMm+hdIsG1412bdQqru+8ORv85bNNs+qZ7TpGGwi4sR21yQ9JWn/iNiYUkBsavMdG9Kfj0u6AzgS2CXYmE15BW6eVTXd9vxWAY1seUuAH7YWSPuY7pFezwKOBR7usl6zWhJiQPmOftNtsFkKvFvSY8CJ6T2S5km6PJU5DFgt6X7gdrIxGwcbs7EUnF1B0nxJj6YUu7vMFktaJmlNOn4l6dmmazuarnXd9+1qBDYiNgMnjHF+NXB2ev1z4PBu6jGbKoocs5E0CHwdeDdZorl7Ja1q/o99RHyyqfw/kg1xNLwQEUcUcjM4lYtZ5QxoINeRw1HAuoh4PCJeAq4jW67SzmLg2gJ+wpgcbMyqRPm6UDlbP7nT6Eo6GDgEuK3p9AxJqyXdLWnhbv6inabOQhazPiBgaDB3G2BWWr/WMNFc380WASsjYkfTuVxLVvJysDGrkGzzrNzB5ukOub4nkkZ3ES05pIpesuJulFmlFNqNuheYI+kQSdPJAsous0qS/hLYC/hF07nCl6y4ZWNWJQXmjYqIEUkfJ8vTPQisiIiHJF0ErI6IRuBZBFyXMmQ2HAZ8S9IoWaOk6yUrDjZmFVL04woRcRNZTu/mc59vef8vY3yu8CUrDjZmVSIxODg42XfREw42ZhUypR/ENLNyOdiYWc9J5F0d3HccbMwqJf9Dlv3GwcasYvpx+4g8HGzMKkSCoSHPRplZjzU2z6ojBxuzKpFno8ysJA42ZtZzwlPfZlYGeerbzEogxNDgtMm+jZ4opL2WYwf3PSRdn67fI2l2EfWa1Y2AQQ3mOvpN18GmaQf3k4G5wGJJc1uKnQVsiYhDgWXAJd3Wa1ZLEgMDg7mOflNEyybPDu4LgKvS65XACVJNFxOYdWlAg7mOflPEmM1YO7gf3a5M2j1sK7A38HQB9ZvVhtBE9iDuK5UaIJY0DAwDDO3Zf5HbrFuSmDY4fbJvoyeKCDZ5dnBvlFkvaQh4PbC59YtSGorlADP2mR6t183qT7VdZ1PEr8qzg/sqYEl6fTpwW8vmymZGI5VLcQPEOWaKz5T0x6ac3mc3XVsi6bF0LGn97ER13bLJuYP7FcB3JK0DniELSGa2CxU2rZ0n13dyfUR8vOWzbwAuBOYBAfwyfXbL7t5PIWM2nXZwj4gXgQ8UUZdZnRX8uMLOmWIASY2Z4jwpWd4D3BIRz6TP3gLMp4tc4PXsHJr1rQmts5mVcnE3juGWL8ub6/v9kh6QtFJSY/w1d57wvCo1G2U21U1wNqpT+t08fgRcGxHbJf0D2Xq447v8zjG5ZWNWIY1uVJ4jh44zxRGxOSK2p7eXA2/P+9mJcrAxq5JiH1foOFMsaf+mt6cCj6TXNwMnpZzfewEnpXO7zd0os0pRYY8i5Jwp/idJpwIjZDPFZ6bPPiPpi2QBC+CixmDx7nKwMauQLCNmcR2OHDPF5wPnt/nsCmBFUffiYGNWKcWts6kaBxuzCpHEkJ+NMrNe8x7EZlYOicE+3BgrDwcbswrJWjYONmbWc/XdYsLBxqxChBga8ACxmfWahNyNMrMyeMzGzHpOiAEcbMysBG7ZmFnPqcAHMavGwcasUsSgPBtlZj0meZ2NmZWkrt2oQkJoN7lpzKyZnOu7nW5y05jZqwkv6htPN7lpzKyF19m0N1Z+maPHKPd+Se8EfgV8MiKebC2Q8t4MA+yjPfnSytMKuL3+cP7pN0z2LZTmNVvrOQBaBKnYZ6MkzQcuJduD+PKIWNpy/VzgbLI9iP8I/H1E/DZd2wGsTUV/FxGndnMvZf1T/xEwOyL+CriFLDfNLiJieUTMi4h5rxv4s5JuzaxKihuzaRriOBmYCyyWNLel2P8C89LfzZXAvzVdeyEijkhHV4EGigk23eSmMbNXycZs8hw57BziiIiXgMYQx04RcXtEbEtv7yb7+9sTRQSbbnLTmFkTkY3Z5DkoLv1uw1nAT5rez0jfe7ekhd3+tq7HbLrJTWNmrSa0qK+I9LtZrdLfAfOAdzWdPjgiNkh6C3CbpLUR8evdraOQRX3d5KYxs1cUPECcK4WupBOBC4B3NQ13EBEb0p+PS7oDOBLY7WDjaQGzihGDuY4c8gxxHAl8Czg1IjY1nd9L0h7p9SzgWLpczuLHFcwqpMinvnMOcfw7sCfwn5LglSnuw4BvSRola5QsHWOh7oQ42JhVSrFbTOQY4jixzed+Dhxe2I3gYGNWOarp6IaDjVnlaLJvoCccbMwqxHsQm1mJ3I0ysxLI3Sgz6z0hbwtqZuVwy8bMes4DxGZWGnejzKzHhAeIzawU8gpiMyuLWzZmVgK3bMysBMq7V03fcbAxq5BsgNgtGzMrgWejzKz3JPDjCmZWhrq2bAoJoZJWSNok6cE21yXpK5LWSXpA0tuKqNesfrJ1NnmOXN8mzZf0aPq799kxru8h6fp0/R5Js5uunZ/OPyrpPd3+sqLaa1cC88e5fjIwJx3DwDcLqtesdorKrpAz/e5ZwJaIOBRYBlySPjuXLBvDW8n+bn9DOdNwtlNIsImIO8mSz7WzALg6MncDM1uyZJoZrzyukOd/OXRMv5veX5VerwROUJZmYQFwXURsj4gngHXp+3ZbWSNRudKAShpupBJ9bvSFkm7NrEo0gaOQ9Ls7y0TECLAV2DvnZyekUgPEEbEcWA5w6NAbY5Jvx6x8kY58Cku/W4ayWja50oCaWaDId+SQ5+/dzjKShoDXA5tzfnZCygo2q4Az0qzUMcDWiNhYUt1m/WU059FZx/S76f2S9Pp04LaIiHR+UZqtOoRscud/uvhVxXSjJF0LHEfWh1wPXAhMA4iIy8gy8p1CNsi0DfhIEfWa1VK+VkuOr8mVfvcK4DuS1pFN8ixKn31I0vfI8nuPAOdExI5u7qeQYBMRiztcD+CcIuoyq7UAFThamSP97ovAB9p89mLg4qLupVIDxGbGRAaI+4qDjVnVFNSNqhoHG7OqqWescbAxq5Qg77R233GwMauaesYaBxuzynGwMbNSuBtlZmUocp1NlTjYmFXJxB7E7CsONmaVEjBaz2jjYGNWIaK+3ah6buNuZpXjlo1Z1Xg2ysx6zgPEZlYWeYDYzEpRz1jjYGNWKYGnvs2sDEHUdIDYU99mVVPchudtSXqDpFskPZb+3GuMMkdI+oWkh1La7A81XbtS0hOS1qTjiE51OtiYVUgExGjkOrr0WeDWiJgD3Jret9oGnBERjRS8/yFpZtP1z0TEEelY06lCd6PMKiZ2dNlsyWcBWUYUyNLv3gGc96r7iPhV0+vfS9oE7AM8uzsVFtKykbRC0iZJD7a5fpykrU1Nrs+PVc5symsMEOc5OqffHc++Tbnb/gDsO15hSUcB04FfN52+OHWvlknao1OFRbVsrgS+Blw9Tpm7IuK9BdVnVlMTGiAeN/2upJ8C+41x6YJX1RgRUvsnsiTtD3wHWBIRjWbX+WRBajpZyuzzgIvGu9mi8kbdKWl2Ed/V8NQbn2XZcGvyvvp6zdapM3y27flSugn9q6D/eyLixHbXJD0laf+I2JiCyaY25V4H/Bi4ICLubvruRqtou6RvA5/udD9l/hv+Dkn3S/qJpLeOVUDScKNJ+PI2/wtpU1NE5Dq61Jx2dwnww9YCKWXvD4CrI2Jly7X9058CFgJjDqE0KyvY3AccHBF/DXwVuHGsQhGxPCLmRcS8aa+ZOv+lN9tpYmM23VgKvFvSY8CJ6T2S5km6PJX5IPBO4MwxprivkbQWWAvMAv61U4WlzEZFxHNNr2+S9A1JsyLi6TLqN+snZcxGRcRm4IQxzq8Gzk6vvwt8t83nj59onaUEG0n7AU+lgaijyFpUm8uo26yfRBSyhqaSCgk2kq4lm7OfJWk9cCEwDSAiLgNOBz4maQR4AVgUdV2Tbdatmg5XFjUbtbjD9a+RTY2bWQd1/e+wVxCbVYmf+jazspT0uELpHGzMqiTwmI2ZlcGzUWZWFg8Qm1nPpf1s6sjBxqxqHGzMrNciwrNRZlYOBxsz6z2P2ZhZOdyNMrMyBDDqYGNmPRYRjL68Y7JvoyccbMwqxt0oM+u9GnejvNGvWaXky4bZ7YxVnvS7qdyOpv2HVzWdP0TSPZLWSbo+bY4+LgcbsyqJrBuV5+hSnvS7AC80pdg9ten8JcCyiDgU2AKc1alCBxuzCgkgRkdzHV1aQJZ2l/TnwrwfTOlbjgca6V1yfd5jNmZVEkHkn42aJWl10/vlEbE852fzpt+dkeoYAZZGxI3A3sCzETGSyqwHDuhUoYONWZXEhGajyki/e3BEbJD0FuC2lCtqa94bbNZ1sJF0EFmO733JWoHLI+LSljICLgVOAbYBZ0bEfd3WbVY/UUQXKfumAtLvRsSG9Ofjku4AjgS+D8yUNJRaNwcCGzrdTxFjNiPApyJiLnAMcI6kuS1lTgbmpGMY+GYB9ZrVTwA7It/RnTzpd/eStEd6PQs4Fng4pWG6nSxFU9vPt+o62ETExkYrJSL+BDzCrv23BWT5giMlJ5/ZyBVsZq9W0gBxnvS7hwGrJd1PFlyWRsTD6dp5wLmS1pGN4VzRqcJCx2wkzSZrZt3TcukA4Mmm940BpY2Y2U5l7WeTM/3uz4HD23z+ceCoidRZWLCRtCdZX+4Tzbm9J/gdw2TdLPZ4/WBRt2bWP4KJzEb1laLS704jCzTXRMQNYxTZABzU9H7MAaU0bbcc4LVvml7PTT3MxlXcAHHVdD1mk2aargAeiYgvtym2CjhDmWOArU1z/GbWUN4K4tIV0bI5FvgwsFbSmnTuc8CbASLiMuAmsmnvdWRT3x8poF6zGqpvy6brYBMRPwPUoUwA53Rbl1ntNaa+a8griM0qJCIYfWmkc8E+5GBjViUTe1yhrzjYmFVJQIw42JhZzzm7gpmVINyyMbNSRDjYmFkJAka3+3EFM+u1kh7EnAwONmYV4jEbMyuHx2zMrCzuRplZ77kbZWZliAhGt9fz2SgnqTOrkjRmk+foRp70u5L+pin17hpJL0pamK5dKemJpmtHdKrTwcasSiqUfjcibm+k3iXLgLkN+K+mIp9pSs27plOFDjZmVZLGbHrdsmHi6XdPB34SEdt2t0IHG7NKKacbRf70uw2LgGtbzl0s6QFJyxr5pcbjAWKzConRCT2uMG6u74LS75JyvB0O3Nx0+nyyIDWdLEnBecBF492sg41ZpUzocYVxc30XkX43+SDwg4h4uem7G62i7ZK+DXy60826G2VWJeWN2XRMv9tkMS1dqEZG25RdZSHwYKcK3bIxq5LyHldYCnxP0lnAb8laL0iaB3w0Is5O72eT5Xz775bPXyNpH7JkB2uAj3aqsOtgI+kg4GqyAaYg6zde2lLmOLLI+UQ6dUNEjNu/M5uKoqQ9iPOk303vf0OWKru13PETrbOIls0I8KmIuE/Sa4FfSrqlKQF5w10R8d4C6jOrNT8b1UYaKNqYXv9J0iNkkbA12JhZBxHByKg3z+oo9e+OBO4Z4/I7JN0P/B74dEQ8NMbnh4Hh9Pb5u77w5KNF3l9Os4CnJ6HeyTKVfu9k/daDJ1J4R7hlMy5JewLfBz4REc+1XL4PODginpd0CnAjMKf1O9IageWt58skafV404l1M5V+bz/81iAYrWmwKWTqW9I0skBzTUTc0Ho9Ip6LiOfT65uAaZJmFVG3Wd2MRuQ6+k0Rs1ECrgAeiYgvtymzH/BUWql4FFmQ29xt3WZ1VNeWTRHdqGOBDwNrJa1J5z4HvBkgIi4je4jrY5JGgBeARRGVDc2T2o2bBFPp91b+t0bUtxul6v6dN5t6Dh3aN778ug/lKrtgy1d/WfUxqGZeQWxWKeHZKDPrvYC+HPzNww9iNpE0X9KjktZJ2mXnsjqRtELSJkkdH6Drd5IOknS7pIclPSTpnyf7ntqKbIA4z9FvHGwSSYPA14GTgbnAYklzJ/eueupKYP5k30RJGo/UzAWOAc6p7j/bqG2wcTfqFUcB6yLicQBJ15FtnVjLxy4i4s604rv2+umRmgA/rjAFHAA82fR+PXD0JN2L9UiHR2omXXiA2Kz/dXikphrCi/qmgg1kmwQ1HJjOWQ10eqSmKuo8G+Vg84p7gTmSDiELMouAv53cW7Ii5Hmkpjrqu4LYs1FJRIwAHyfbQf4R4HtjbYNRF5KuBX4B/IWk9Wl7yLpqPFJzfFMGx1Mm+6bGkrVsPBtVe+mJ9Jsm+z7KEBGLJ/seyhIRPyPbK7fyIoKXazob5ZaNWcWU0bKR9IG0wHE0bXLertyYC10lHSLpnnT+eknTO9XpYGNWMSXtZ/MgcBpwZ7sCHRa6XgIsi4hDgS1Ax264g41ZhURJK4gj4pGI6LTt7s6FrhHxEnAdsCANuB8PrEzl8uQK95iNWZWs59mbz40b8u5iOWO89LsFaLfQdW/g2TSp0ji/S7qXVg42ZhUSEYU9rzZeru+IGC8DZk842JjV1Hi5vnNqt9B1MzBT0lBq3eRaAOsxGzNrZ+dC1zTbtAhYlbb0vZ1su1/onCsccLAxm5IkvU/SeuAdwI8l3ZzOv0nSTdBxoet5wLmS1pGN4VzRsU7vQWxmZXDLxsxK4WBjZqVwsDGzUjjYmFkpHGzMrBQONmZWCgcbMyvF/wPZqxE8c4CsDAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "cond_coeff_mat = conditional_corrcoeff(\n", + " density=posterior,\n", + " condition=condition,\n", + " limits=torch.tensor([[-2., 2.]]*3),\n", + ")\n", + "fig, ax = plt.subplots(1,1, figsize=(4,4))\n", + "im = plt.imshow(cond_coeff_mat, clim=[-1, 1], cmap='PiYG')\n", + "_ = fig.colorbar(im)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So far, we have investigated the conditional distribution only at a specific `condition` sampled from the posterior. In many applications, it makes sense to repeat the above analyses with a different `condition` (another sample from the posterior), which can be interpreted as slicing the posterior at a different location. Note that `conditional_corrcoeff()` can directly compute the matrix for several `conditions` and then outputs the average over them. This can be done by passing a batch of $N$ conditions as the `condition` argument." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sampling conditional distributions\n", + "\n", + "So far, we have demonstrated how one can plot 2D conditional distributions with `conditional_pairplot()` and how one can compute the pairwise conditional correlation coefficient with `conditional_corrcoeff()`. In some cases, it can be useful to keep a subset of parameters fixed and to vary **more than two** parameters. This can be done by sampling the conditonal posterior $p(\\theta_i | \\theta_{j \\neq i}, x_o)$. As of `sbi` `v0.18.0`, this functionality requires using the [sampler interface](https://www.mackelab.org/sbi/tutorial/11_sampler_interface/). In this tutorial, we demonstrate this functionality on a linear gaussian simulator with four parameters. We would like to fix the forth parameter to $\\theta_4=0.2$ and sample the first three parameters given that value, i.e. we want to sample $p(\\theta_1, \\theta_2, \\theta_3 | \\theta_4 = 0.2, x_o)$. For an application in neuroscience, see [Deistler, Gonçalves, Macke, 2021](https://www.biorxiv.org/content/10.1101/2021.07.30.454484v4.abstract)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, we will use SNPE, but the same also works for SNLE and SNRE. First, we define the prior and the simulator and train the deep neural density estimator:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4683b28ba0614e87b33854d207e30da2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Running 1000 simulations.: 0%| | 0/1000 [00:00" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "from sbi.analysis import pairplot\n", + "\n", + "_ = pairplot(cond_samples, limits=[[-2, 2], [-2, 2], [-2, 2], [-2, 2]], figsize=(4, 4))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From d14fec3e06ec283d17cc0cf3343d2fe05257a585 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Fri, 22 Apr 2022 10:46:15 -0400 Subject: [PATCH 10/15] Ran isort, made sure _round is updated correctly --- sbi/inference/snle/snle_base.py | 8 ++++---- sbi/inference/snpe/snpe_base.py | 7 +++++-- sbi/inference/snre/snre_base.py | 8 ++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/sbi/inference/snle/snle_base.py b/sbi/inference/snle/snle_base.py index 33aefa8c4..2560c4e51 100644 --- a/sbi/inference/snle/snle_base.py +++ b/sbi/inference/snle/snle_base.py @@ -21,8 +21,8 @@ from sbi.utils import ( check_estimator_arg, check_prior, - mask_sims_from_prior, handle_invalid_x, + mask_sims_from_prior, validate_theta_and_x, warn_if_zscoring_changes_data, warn_on_invalid_x, @@ -182,11 +182,11 @@ def train( Returns: Density estimator that has learned the distribution $p(x|\theta)$. """ - - # Starting index for the training set (1 = discard round-0 samples). - start_idx = int(discard_prior_samples and self._round > 0) # Load data from most recent round. self._round = max(self._data_round_index) + # Starting index for the training set (1 = discard round-0 samples). + start_idx = int(discard_prior_samples and self._round > 0) + train_loader, val_loader = self.get_dataloaders( start_idx, diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 6d7b6ccef..7ba845f4f 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -26,13 +26,13 @@ from sbi.utils import ( RestrictedPrior, check_estimator_arg, + handle_invalid_x, test_posterior_net_for_multi_d_x, validate_theta_and_x, - x_shape_from_simulation, - handle_invalid_x, warn_if_zscoring_changes_data, warn_on_invalid_x, warn_on_invalid_x_for_snpec_leakage, + x_shape_from_simulation, ) from sbi.utils.sbiutils import ImproperEmpirical, mask_sims_from_prior @@ -249,6 +249,9 @@ def train( Returns: Density estimator that approximates the distribution $p(\theta|x)$. """ + # Load data from most recent round. + self._round = max(self._data_round_index) + if self._round == 0 and self._neural_net is not None: assert force_first_round_loss, ( "You have already trained this neural network. After you had trained " diff --git a/sbi/inference/snre/snre_base.py b/sbi/inference/snre/snre_base.py index 698a57612..df1ef022f 100644 --- a/sbi/inference/snre/snre_base.py +++ b/sbi/inference/snre/snre_base.py @@ -19,9 +19,9 @@ clamp_and_warn, handle_invalid_x, validate_theta_and_x, - x_shape_from_simulation, warn_if_zscoring_changes_data, warn_on_invalid_x, + x_shape_from_simulation, ) from sbi.utils.sbiutils import mask_sims_from_prior @@ -186,11 +186,11 @@ def train( Returns: Classifier that approximates the ratio $p(\theta,x)/p(\theta)p(x)$. """ - + # Load data from most recent round. + self._round = max(self._data_round_index) # Starting index for the training set (1 = discard round-0 samples). start_idx = int(discard_prior_samples and self._round > 0) - # Load data from most recent round. - self._round = max(self._data_round_index) + train_loader, val_loader = self.get_dataloaders( start_idx, From 2c87aa2080fbbdcb6b8c8b0602561812aae9095a Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Fri, 22 Apr 2022 12:29:38 -0400 Subject: [PATCH 11/15] Fixed bug with indicies --- sbi/inference/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index daae107f1..844cf09ec 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -258,13 +258,13 @@ def get_dataloaders( train_loader_kwargs = { "batch_size": min(training_batch_size, num_training_examples), "drop_last": True, - "sampler": SubsetRandomSampler(torch.arange(len(self.train_indices)).tolist() ), + "sampler": SubsetRandomSampler(self.train_indices.tolist() ), } val_loader_kwargs = { "batch_size": min(training_batch_size, num_validation_examples), "shuffle": False, "drop_last": True, - "sampler": SubsetRandomSampler(torch.arange(len(self.val_indices)).tolist() ), + "sampler": SubsetRandomSampler(self.val_indices.tolist() ), } if dataloader_kwargs is not None: train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) From 7eb4551782b5650fa3cc0c7b5bfcaeb379458064 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Mon, 25 Apr 2022 15:00:46 -0400 Subject: [PATCH 12/15] Changes to tests to match new syntax --- tests/base_test.py | 7 ++----- tests/inference_with_NaN_simulator_test.py | 4 +--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index ca78fce97..e4cc4078c 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,6 +1,5 @@ import pytest import torch -from torch.utils.data import TensorDataset from sbi.inference import SNPE @@ -11,12 +10,10 @@ def test_get_dataloaders(training_batch_size): N = 1000 validation_fraction = 0.1 - dataset = TensorDataset(torch.ones(N), torch.zeros(N)) - inferer = SNPE() - + inferer.append_simulations(torch.ones(N), torch.zeros(N),warn_if_zscoring=False) _, val_loader = inferer.get_dataloaders( - dataset, + 0, training_batch_size=training_batch_size, validation_fraction=validation_fraction, ) diff --git a/tests/inference_with_NaN_simulator_test.py b/tests/inference_with_NaN_simulator_test.py index 368c2be1d..4380285d6 100644 --- a/tests/inference_with_NaN_simulator_test.py +++ b/tests/inference_with_NaN_simulator_test.py @@ -102,9 +102,7 @@ def linear_gaussian_nan( inference = method(prior=prior) theta, x = simulate_for_sbi(simulator, prior, num_simulations) - _ = inference.append_simulations(theta, x).train( - exclude_invalid_x=exclude_invalid_x - ) + _ = inference.append_simulations(theta, x, exclude_invalid_x=exclude_invalid_x).train() posterior = inference.build_posterior() samples = posterior.sample((num_samples,), x=x_o) From 980a95bea4de6732b19e4ed0c5d2a8b5268a2f9e Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Fri, 27 May 2022 13:27:03 -0400 Subject: [PATCH 13/15] Now compatible with black and pyright --- sbi/examples/minimal.py | 3 +- sbi/inference/base.py | 46 ++++++++++-------- sbi/inference/snle/snle_base.py | 42 +++++++++------- sbi/inference/snpe/snpe_base.py | 56 ++++++++++++---------- sbi/inference/snre/snre_base.py | 50 ++++++++++--------- sbi/utils/sbiutils.py | 12 +++-- tests/base_test.py | 2 +- tests/inference_with_NaN_simulator_test.py | 4 +- 8 files changed, 123 insertions(+), 92 deletions(-) diff --git a/sbi/examples/minimal.py b/sbi/examples/minimal.py index 408d6d4e6..31864be94 100644 --- a/sbi/examples/minimal.py +++ b/sbi/examples/minimal.py @@ -37,7 +37,8 @@ def flexible(): inference = SNPE(prior) theta, x = simulate_for_sbi(simulator, proposal=prior, num_simulations=500) - density_estimator = inference.append_simulations(theta, x).train() + inference.append_simulations(theta, x) + density_estimator = inference.train() posterior = inference.build_posterior(density_estimator) posterior.sample((100,), x=x_o) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 844cf09ec..5f4ffafd4 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -128,14 +128,13 @@ def __init__( # Initialize roundwise (theta, x, prior_masks) for storage of parameters, # simulations and masks indicating if simulations came from prior. - self._dataset = None + self._dataset = data.Dataset() self._num_sims_per_round = [] self._model_bank = [] # Initialize list that indicates the round from which simulations were drawn. self._data_round_index = [] - self._round = 0 self._val_log_prob = float("-Inf") @@ -177,19 +176,21 @@ def get_simulations( Returns: Parameters, simulation outputs, prior masks. """ - #This is a pretty clunky implementation but not sure this will be used much with - #new implementation of `get_dataloaders` - indicies = get_simulations_indcies(self._num_sims_per_round, self._data_round_index, starting_round) - theta,x,prior_masks = [],[],[] + # This is a pretty clunky implementation but not sure this will be used much with + # new implementation of `get_dataloaders` + indicies = get_simulations_indcies( + self._num_sims_per_round, self._data_round_index, starting_round + ) + theta, x, prior_masks = [], [], [] for ind in indicies: - theta_cur,x_cur,prior_mask_cur = self._dataset[ind] + theta_cur, x_cur, prior_mask_cur = self._dataset[ind] theta.append(theta_cur) x.append(x_cur) prior_masks.append(prior_mask_cur) - - theta = torch.stack(theta).squeeze() - x = torch.stack(x).squeeze() + + theta = torch.stack(theta) + x = torch.stack(x) prior_masks = torch.stack(prior_masks).squeeze() return theta, x, prior_masks @@ -218,7 +219,7 @@ def get_dataloaders( validation_fraction: float = 0.1, resume_training: bool = False, dataloader_kwargs: Optional[dict] = None, - ) -> Tuple[data.DataLoader, data.DataLoader]: + ) -> Tuple[data.DataLoader, data.DataLoader]: """Return dataloaders for training and validation. Args: @@ -233,9 +234,11 @@ def get_dataloaders( Tuple of dataloaders for training and validation. """ - - #Generate indicies to use based on starting_round - indices = get_simulations_indcies(self._num_sims_per_round, self._data_round_index, starting_round) + + # Generate indicies to use based on starting_round + indices = get_simulations_indcies( + self._num_sims_per_round, self._data_round_index, starting_round + ) # Get total number of training examples. num_examples = len(indices) @@ -258,22 +261,22 @@ def get_dataloaders( train_loader_kwargs = { "batch_size": min(training_batch_size, num_training_examples), "drop_last": True, - "sampler": SubsetRandomSampler(self.train_indices.tolist() ), + "sampler": SubsetRandomSampler(self.train_indices.tolist()), } val_loader_kwargs = { "batch_size": min(training_batch_size, num_validation_examples), "shuffle": False, "drop_last": True, - "sampler": SubsetRandomSampler(self.val_indices.tolist() ), + "sampler": SubsetRandomSampler(self.val_indices.tolist()), } if dataloader_kwargs is not None: train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) train_loader = data.DataLoader(self._dataset, **train_loader_kwargs) - val_loader = data.DataLoader(self._dataset, **val_loader_kwargs) + val_loader = data.DataLoader(self._dataset, **val_loader_kwargs) - return train_loader,val_loader + return train_loader, val_loader def _converged(self, epoch: int, stop_after_epochs: int) -> bool: """Return whether the training converged yet and save best model state so far. @@ -356,8 +359,11 @@ def _report_convergence_at_end( ) def _summarize( - self, round_: int, x_o: Union[Tensor, None], theta_bank: Union[Tensor,None] - , x_bank: Union[Tensor,None] + self, + round_: int, + x_o: Union[Tensor, None], + theta_bank: Union[Tensor, None], + x_bank: Union[Tensor, None], ) -> None: """Update the summary_writer with statistics for a given round. diff --git a/sbi/inference/snle/snle_base.py b/sbi/inference/snle/snle_base.py index 2560c4e51..436125aac 100644 --- a/sbi/inference/snle/snle_base.py +++ b/sbi/inference/snle/snle_base.py @@ -90,8 +90,8 @@ def append_simulations( warn_on_invalid: bool = True, warn_if_zscoring: bool = True, return_self: bool = True, - data_device: str = None, - ) -> "LikelihoodEstimator": + data_device: Optional[str] = None, + ) -> Union["LikelihoodEstimator", None]: r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -131,20 +131,30 @@ def append_simulations( x = x[is_valid_x] theta = theta[is_valid_x] - if data_device is None: data_device = self._device + if data_device is None: + data_device = self._device theta, x = validate_theta_and_x(theta, x, training_device=data_device) prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - if self._dataset is None: - #If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + if len(self._num_sims_per_round) == 0: + # If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) else: - #Otherwise append to Dataset - self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) + # Otherwise append to Dataset + self._dataset = data.ConcatDataset( + self._dataset.datasets + + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) self._num_sims_per_round.append(theta.size(0)) - self._data_round_index.append(int(from_round) ) - + self._data_round_index.append(int(from_round)) + if return_self: return self @@ -186,7 +196,6 @@ def train( self._round = max(self._data_round_index) # Starting index for the training set (1 = discard round-0 samples). start_idx = int(discard_prior_samples and self._round > 0) - train_loader, val_loader = self.get_dataloaders( start_idx, @@ -202,15 +211,14 @@ def train( # This is passed into NeuralPosterior, to create a neural posterior which # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - - #Get theta,x from dataset to initialize NN - test_theta = self._dataset.datasets[0].tensors[0][:100] - test_x = self._dataset.datasets[0].tensors[1][:100] + # Get theta,x from dataset to initialize NN + theta, x, _ = self.get_simulations() self._neural_net = self._build_neural_net( - test_theta, test_x + theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") ) - self._x_shape = x_shape_from_simulation(test_x) + self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + del theta, x assert ( len(self._x_shape) < 3 ), "SNLE cannot handle multi-dimensional simulator output." diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index 7ba845f4f..44ffb19e7 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -96,8 +96,8 @@ def append_simulations( warn_on_invalid: bool = True, warn_if_zscoring: bool = True, return_self: bool = True, - data_device: str = None, - ) -> "PosteriorEstimator": + data_device: Optional[str] = None, + ) -> Union["PosteriorEstimator", None]: r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -127,9 +127,6 @@ def append_simulations( """ # Add ability to specify device data is saved on - if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) @@ -145,6 +142,9 @@ def append_simulations( x = x[is_valid_x] theta = theta[is_valid_x] + if data_device is None: + data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=data_device) self._check_proposal(proposal) @@ -169,13 +169,21 @@ def append_simulations( self._data_round_index.append(max(self._data_round_index) + 1) prior_masks = mask_sims_from_prior(1, theta.size(0)) - - if self._dataset is None: - #If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + if len(self._num_sims_per_round) == 0: + # If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) else: - #Otherwise append to Dataset - self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) + # Otherwise append to Dataset + self._dataset = data.ConcatDataset( + self._dataset.datasets + + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) self._num_sims_per_round.append(theta.size(0)) self._proposal_roundwise.append(proposal) @@ -194,7 +202,7 @@ def append_simulations( theta_prior = self.get_simulations()[0] self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) - #Add ability to not return self + # Add ability to not return self if return_self: return self @@ -301,22 +309,20 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - #Get theta,x from dataset to initialize NN - test_theta = self._dataset.datasets[0].tensors[0][:100] - test_x = self._dataset.datasets[0].tensors[1][:100] - + # Get theta,x from dataset to initialize NN + theta, x, _ = self.get_simulations() self._neural_net = self._build_neural_net( - test_theta, test_x + theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") + ) + self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + + test_posterior_net_for_multi_d_x( + self._neural_net, + theta[:training_batch_size].to("cpu"), + x[:training_batch_size].to("cpu"), ) - # If data on training device already move net as well. - if ( - not self._device == "cpu" - and f"{test_x.device.type}:{test_x.device.index}" == self._device - ): - self._neural_net.to(self._device) - test_posterior_net_for_multi_d_x(self._neural_net, test_theta, test_x) - self._x_shape = x_shape_from_simulation(test_x) + del theta, x # Move entire net to device for training. self._neural_net.to(self._device) diff --git a/sbi/inference/snre/snre_base.py b/sbi/inference/snre/snre_base.py index df1ef022f..df5204045 100644 --- a/sbi/inference/snre/snre_base.py +++ b/sbi/inference/snre/snre_base.py @@ -89,8 +89,8 @@ def append_simulations( warn_on_invalid: bool = True, warn_if_zscoring: bool = True, return_self: bool = True, - data_device: str = None, - ) -> "RatioEstimator": + data_device: Optional[str] = None, + ) -> Union["RatioEstimator", None]: r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -119,8 +119,6 @@ def append_simulations( NeuralInference object (returned so that this function is chainable). """ - theta, x = validate_theta_and_x(theta, x, training_device=self._device) - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) # Check for problematic z-scoring @@ -132,19 +130,30 @@ def append_simulations( x = x[is_valid_x] theta = theta[is_valid_x] - if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) + if data_device is None: + data_device = self._device + theta, x = validate_theta_and_x(theta, x, training_device=self._device) + prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - if self._dataset is None: - #If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( [data.TensorDataset(theta,x,prior_masks),] ) + if len(self._num_sims_per_round) == 0: + # If first round, set up ConcatDataset + self._dataset = data.ConcatDataset( + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) else: - #Otherwise append to Dataset - self._dataset = data.ConcatDataset( self._dataset.datasets + [data.TensorDataset(theta,x,prior_masks),] ) - + # Otherwise append to Dataset + self._dataset = data.ConcatDataset( + self._dataset.datasets + + [ + data.TensorDataset(theta, x, prior_masks), + ] + ) + self._num_sims_per_round.append(theta.size(0)) - self._data_round_index.append(int(from_round) ) + self._data_round_index.append(int(from_round)) if return_self: return self @@ -158,7 +167,6 @@ def train( stop_after_epochs: int = 20, max_num_epochs: int = 2**31 - 1, clip_max_norm: Optional[float] = 5.0, - exclude_invalid_x: bool = True, resume_training: bool = False, discard_prior_samples: bool = False, retrain_from_scratch: bool = False, @@ -187,10 +195,9 @@ def train( Classifier that approximates the ratio $p(\theta,x)/p(\theta)p(x)$. """ # Load data from most recent round. - self._round = max(self._data_round_index) + self._round = max(self._data_round_index) # Starting index for the training set (1 = discard round-0 samples). start_idx = int(discard_prior_samples and self._round > 0) - train_loader, val_loader = self.get_dataloaders( start_idx, @@ -215,14 +222,13 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - #Get theta,x from dataset to initialize NN - test_theta = self._dataset.datasets[0].tensors[0][:100] - test_x = self._dataset.datasets[0].tensors[1][:100] - + # Get theta,x from dataset to initialize NN + theta, x, _ = self.get_simulations() self._neural_net = self._build_neural_net( - test_theta, test_x + theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") ) - self._x_shape = x_shape_from_simulation(test_x) + self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + del x, theta self._neural_net.to(self._device) if not resume_training: diff --git a/sbi/utils/sbiutils.py b/sbi/utils/sbiutils.py index 4f26cf848..acfc3fdf4 100644 --- a/sbi/utils/sbiutils.py +++ b/sbi/utils/sbiutils.py @@ -348,6 +348,7 @@ def get_simulations_since_round( [t for t, r in zip(data, data_round_indices) if r >= starting_round_index] ) + def get_simulations_indcies( num_sims_per_round: List, data_round_indices: List, starting_round_index: int ) -> Tensor: @@ -363,15 +364,16 @@ def get_simulations_indcies( counting from 0. """ inds = [] - for j, (n,r) in enumerate(zip(num_sims_per_round,data_round_indices)): - - #Where to start counting + for j, (n, r) in enumerate(zip(num_sims_per_round, data_round_indices)): + + # Where to start counting s_ind = sum(num_sims_per_round[:j]) - + if r >= starting_round_index: - inds.append(torch.arange(s_ind,s_ind + n) ) + inds.append(torch.arange(s_ind, s_ind + n)) return torch.cat(inds) + def mask_sims_from_prior(round_: int, num_simulations: int) -> Tensor: """Returns Tensor True where simulated from prior parameters. diff --git a/tests/base_test.py b/tests/base_test.py index e4cc4078c..b41c47728 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -11,7 +11,7 @@ def test_get_dataloaders(training_batch_size): validation_fraction = 0.1 inferer = SNPE() - inferer.append_simulations(torch.ones(N), torch.zeros(N),warn_if_zscoring=False) + inferer.append_simulations(torch.ones(N), torch.zeros(N), warn_if_zscoring=False) _, val_loader = inferer.get_dataloaders( 0, training_batch_size=training_batch_size, diff --git a/tests/inference_with_NaN_simulator_test.py b/tests/inference_with_NaN_simulator_test.py index 4380285d6..a9d5378cc 100644 --- a/tests/inference_with_NaN_simulator_test.py +++ b/tests/inference_with_NaN_simulator_test.py @@ -102,7 +102,9 @@ def linear_gaussian_nan( inference = method(prior=prior) theta, x = simulate_for_sbi(simulator, prior, num_simulations) - _ = inference.append_simulations(theta, x, exclude_invalid_x=exclude_invalid_x).train() + _ = inference.append_simulations( + theta, x, exclude_invalid_x=exclude_invalid_x + ).train() posterior = inference.build_posterior() samples = posterior.sample((num_samples,), x=x_o) From 4198a82ad3ca23b509feca23b877a29fc672191d Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Thu, 23 Jun 2022 11:02:46 -0400 Subject: [PATCH 14/15] Reverted to storing data as tensors in list --- examples/HH_helper_functions.py | 2 +- sbi/examples/minimal.py | 3 +- sbi/inference/base.py | 55 ++++++++++--------------- sbi/inference/snle/snle_base.py | 51 +++++++---------------- sbi/inference/snpe/snpe_base.py | 72 ++++++++++----------------------- sbi/inference/snre/snre_base.py | 51 +++++++---------------- sbi/utils/sbiutils.py | 25 ------------ sbi/utils/user_input_checks.py | 22 +++++----- tests/base_test.py | 2 +- 9 files changed, 85 insertions(+), 198 deletions(-) diff --git a/examples/HH_helper_functions.py b/examples/HH_helper_functions.py index 38cb91b06..ecced908b 100644 --- a/examples/HH_helper_functions.py +++ b/examples/HH_helper_functions.py @@ -145,7 +145,7 @@ def tau_p(x): + g_leak * E_leak + gbar_M * p[i - 1] * E_K + I[i - 1] - + nois_fact * rng.randn() / (tstep ** 0.5) + + nois_fact * rng.randn() / (tstep**0.5) ) / (tau_V_inv * C) V[i] = V_inf + (V[i - 1] - V_inf) * np.exp(-tstep * tau_V_inv) n[i] = n_inf(V[i]) + (n[i - 1] - n_inf(V[i])) * np.exp(-tstep / tau_n(V[i])) diff --git a/sbi/examples/minimal.py b/sbi/examples/minimal.py index 31864be94..408d6d4e6 100644 --- a/sbi/examples/minimal.py +++ b/sbi/examples/minimal.py @@ -37,8 +37,7 @@ def flexible(): inference = SNPE(prior) theta, x = simulate_for_sbi(simulator, proposal=prior, num_simulations=500) - inference.append_simulations(theta, x) - density_estimator = inference.train() + density_estimator = inference.append_simulations(theta, x).train() posterior = inference.build_posterior(density_estimator) posterior.sample((100,), x=x_o) diff --git a/sbi/inference/base.py b/sbi/inference/base.py index 5f4ffafd4..880b12ef1 100644 --- a/sbi/inference/base.py +++ b/sbi/inference/base.py @@ -18,15 +18,8 @@ import sbi.inference from sbi.inference.posteriors.base_posterior import NeuralPosterior from sbi.simulators.simutils import simulate_in_batches -from sbi.utils import ( - check_prior, - get_log_root, - handle_invalid_x, - warn_if_zscoring_changes_data, - warn_on_invalid_x, - warn_on_invalid_x_for_snpec_leakage, -) -from sbi.utils.sbiutils import get_simulations_indcies +from sbi.utils import check_prior, get_log_root +from sbi.utils.sbiutils import get_simulations_since_round from sbi.utils.torchutils import check_if_prior_on_device, process_device from sbi.utils.user_input_checks import prepare_for_sbi @@ -128,8 +121,9 @@ def __init__( # Initialize roundwise (theta, x, prior_masks) for storage of parameters, # simulations and masks indicating if simulations came from prior. - self._dataset = data.Dataset() - self._num_sims_per_round = [] + self._theta_roundwise = [] + self._x_roundwise = [] + self._prior_masks = [] self._model_bank = [] # Initialize list that indicates the round from which simulations were drawn. @@ -176,22 +170,15 @@ def get_simulations( Returns: Parameters, simulation outputs, prior masks. """ - # This is a pretty clunky implementation but not sure this will be used much with - # new implementation of `get_dataloaders` - indicies = get_simulations_indcies( - self._num_sims_per_round, self._data_round_index, starting_round + theta = get_simulations_since_round( + self._theta_roundwise, self._data_round_index, starting_round + ) + x = get_simulations_since_round( + self._x_roundwise, self._data_round_index, starting_round + ) + prior_masks = get_simulations_since_round( + self._prior_masks, self._data_round_index, starting_round ) - theta, x, prior_masks = [], [], [] - - for ind in indicies: - theta_cur, x_cur, prior_mask_cur = self._dataset[ind] - theta.append(theta_cur) - x.append(x_cur) - prior_masks.append(prior_mask_cur) - - theta = torch.stack(theta) - x = torch.stack(x) - prior_masks = torch.stack(prior_masks).squeeze() return theta, x, prior_masks @@ -235,20 +222,20 @@ def get_dataloaders( """ - # Generate indicies to use based on starting_round - indices = get_simulations_indcies( - self._num_sims_per_round, self._data_round_index, starting_round - ) + # + theta, x, prior_masks = self.get_simulations(starting_round) + + dataset = data.TensorDataset(theta, x, prior_masks) # Get total number of training examples. - num_examples = len(indices) + num_examples = theta.size(0) # Select random train and validation splits from (theta, x) pairs. num_training_examples = int((1 - validation_fraction) * num_examples) num_validation_examples = num_examples - num_training_examples if not resume_training: # Seperate indicies for training and validation - permuted_indices = indices[torch.randperm(num_examples)] + permuted_indices = torch.randperm(num_examples) self.train_indices, self.val_indices = ( permuted_indices[:num_training_examples], permuted_indices[num_training_examples:], @@ -273,8 +260,8 @@ def get_dataloaders( train_loader_kwargs = dict(train_loader_kwargs, **dataloader_kwargs) val_loader_kwargs = dict(val_loader_kwargs, **dataloader_kwargs) - train_loader = data.DataLoader(self._dataset, **train_loader_kwargs) - val_loader = data.DataLoader(self._dataset, **val_loader_kwargs) + train_loader = data.DataLoader(dataset, **train_loader_kwargs) + val_loader = data.DataLoader(dataset, **val_loader_kwargs) return train_loader, val_loader diff --git a/sbi/inference/snle/snle_base.py b/sbi/inference/snle/snle_base.py index 436125aac..626690b51 100644 --- a/sbi/inference/snle/snle_base.py +++ b/sbi/inference/snle/snle_base.py @@ -86,12 +86,8 @@ def append_simulations( theta: Tensor, x: Tensor, from_round: int = 0, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: bool = True, - return_self: bool = True, data_device: Optional[str] = None, - ) -> Union["LikelihoodEstimator", None]: + ) -> "LikelihoodEstimator": r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -107,12 +103,6 @@ def append_simulations( With default settings, this is not used at all for `SNLE`. Only when the user later on requests `.train(discard_prior_samples=True)`, we use these indices to find which training data stemmed from the prior. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - warn_on_invalid: Whether to warn if data is invalid - warn_if_zscoring: Whether to test if z-scoring causes duplicates - return_self: Whether to return a instance of the class, allows chaining - with `.train()`. Setting `False` decreases memory overhead. data_device: Where to store the data, default is on the same device where the training is happening. If training a large dataset on a GPU with not much VRAM can set to 'cpu' to store data on system memory instead. @@ -120,43 +110,30 @@ def append_simulations( NeuralInference object (returned so that this function is chainable). """ - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + is_valid_x, num_nans, num_infs = handle_invalid_x(x, True) # Hardcode to True x = x[is_valid_x] theta = theta[is_valid_x] + # Check for problematic z-scoring + warn_if_zscoring_changes_data(x) + warn_on_invalid_x(num_nans, num_infs, True) + if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) + theta, x = validate_theta_and_x( + theta, x, data_device=data_device, training_device=self._device + ) + prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - if len(self._num_sims_per_round) == 0: - # If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( - [ - data.TensorDataset(theta, x, prior_masks), - ] - ) - else: - # Otherwise append to Dataset - self._dataset = data.ConcatDataset( - self._dataset.datasets - + [ - data.TensorDataset(theta, x, prior_masks), - ] - ) + self._theta_roundwise.append(theta) + self._x_roundwise.append(x) + self._prior_masks.append(prior_masks) - self._num_sims_per_round.append(theta.size(0)) self._data_round_index.append(int(from_round)) - if return_self: - return self + return self def train( self, diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index b0c90730a..e2c45c0dc 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -92,12 +92,8 @@ def append_simulations( theta: Tensor, x: Tensor, proposal: Optional[DirectPosterior] = None, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: bool = True, - return_self: bool = True, data_device: Optional[str] = None, - ) -> Union["PosteriorEstimator", None]: + ) -> "PosteriorEstimator": r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -112,12 +108,6 @@ def append_simulations( proposal: The distribution that the parameters $\theta$ were sampled from. Pass `None` if the parameters were sampled from the prior. If not `None`, it will trigger a different loss-function. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - warn_on_invalid: Whether to warn if data is invalid - warn_if_zscoring: Whether to test if z-scoring causes duplicates - return_self: Whether to return a instance of the class, allows chaining - with `.train()`. Setting `False` decreases memory overhead. data_device: Where to store the data, default is on the same device where the training is happening. If training a large dataset on a GPU with not much VRAM can set to 'cpu' to store data on system memory instead. @@ -126,26 +116,24 @@ def append_simulations( NeuralInference object (returned so that this function is chainable). """ - # Add ability to specify device data is saved on - - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) - warn_on_invalid_x_for_snpec_leakage( - num_nans, num_infs, exclude_invalid_x, type(self).__name__, self._round - ) + is_valid_x, num_nans, num_infs = handle_invalid_x(x, True) # Hardcode to True x = x[is_valid_x] theta = theta[is_valid_x] + # Check for problematic z-scoring + warn_if_zscoring_changes_data(x) + warn_on_invalid_x(num_nans, num_infs, True) + warn_on_invalid_x_for_snpec_leakage( + num_nans, num_infs, True, type(self).__name__, self._round + ) + if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=data_device) + theta, x = validate_theta_and_x( + theta, x, data_device=data_device, training_device=self._device + ) self._check_proposal(proposal) if ( @@ -169,23 +157,10 @@ def append_simulations( self._data_round_index.append(max(self._data_round_index) + 1) prior_masks = mask_sims_from_prior(1, theta.size(0)) - if len(self._num_sims_per_round) == 0: - # If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( - [ - data.TensorDataset(theta, x, prior_masks), - ] - ) - else: - # Otherwise append to Dataset - self._dataset = data.ConcatDataset( - self._dataset.datasets - + [ - data.TensorDataset(theta, x, prior_masks), - ] - ) + self._theta_roundwise.append(theta) + self._x_roundwise.append(x) + self._prior_masks.append(prior_masks) - self._num_sims_per_round.append(theta.size(0)) self._proposal_roundwise.append(proposal) if self._prior is None or isinstance(self._prior, ImproperEmpirical): @@ -202,9 +177,7 @@ def append_simulations( theta_prior = self.get_simulations()[0] self._prior = ImproperEmpirical(theta_prior, ones(theta_prior.shape[0])) - # Add ability to not return self - if return_self: - return self + return self def train( self, @@ -310,16 +283,15 @@ def train( if self._neural_net is None or retrain_from_scratch: # Get theta,x from dataset to initialize NN - theta, x, _ = self.get_simulations() - self._neural_net = self._build_neural_net( - theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") - ) - self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + x = self._x_roundwise[0][:training_batch_size] + theta = self._theta_roundwise[0][:training_batch_size] + self._neural_net = self._build_neural_net(theta.to("cpu"), x.to("cpu")) + self._x_shape = x_shape_from_simulation(x.to("cpu")) test_posterior_net_for_multi_d_x( self._neural_net, - theta[:training_batch_size].to("cpu"), - x[:training_batch_size].to("cpu"), + theta.to("cpu"), + x.to("cpu"), ) del theta, x diff --git a/sbi/inference/snre/snre_base.py b/sbi/inference/snre/snre_base.py index df5204045..b50a1278d 100644 --- a/sbi/inference/snre/snre_base.py +++ b/sbi/inference/snre/snre_base.py @@ -85,12 +85,8 @@ def append_simulations( theta: Tensor, x: Tensor, from_round: int = 0, - exclude_invalid_x: bool = True, - warn_on_invalid: bool = True, - warn_if_zscoring: bool = True, - return_self: bool = True, data_device: Optional[str] = None, - ) -> Union["RatioEstimator", None]: + ) -> "RatioEstimator": r"""Store parameters and simulation outputs to use them for later training. Data are stored as entries in lists for each type of variable (parameter/data). @@ -106,12 +102,6 @@ def append_simulations( With default settings, this is not used at all for `SNRE`. Only when the user later on requests `.train(discard_prior_samples=True)`, we use these indices to find which training data stemmed from the prior. - exclude_invalid_x: Whether to exclude simulation outputs `x=NaN` or `x=±∞` - during training. Expect errors, silent or explicit, when `False`. - warn_on_invalid: Whether to warn if data is invalid - warn_if_zscoring: Whether to test if z-scoring causes duplicates - return_self: Whether to return a instance of the class, allows chaining - with `.train()`. Setting `False` decreases memory overhead. data_device: Where to store the data, default is on the same device where the training is happening. If training a large dataset on a GPU with not much VRAM can set to 'cpu' to store data on system memory instead. @@ -119,44 +109,31 @@ def append_simulations( NeuralInference object (returned so that this function is chainable). """ - is_valid_x, num_nans, num_infs = handle_invalid_x(x, exclude_invalid_x) - - # Check for problematic z-scoring - if warn_if_zscoring: - warn_if_zscoring_changes_data(x[is_valid_x]) - if warn_on_invalid: - warn_on_invalid_x(num_nans, num_infs, exclude_invalid_x) + is_valid_x, num_nans, num_infs = handle_invalid_x(x, True) # Hardcode to True x = x[is_valid_x] theta = theta[is_valid_x] + # Check for problematic z-scoring + warn_if_zscoring_changes_data(x) + warn_on_invalid_x(num_nans, num_infs, True) + if data_device is None: data_device = self._device - theta, x = validate_theta_and_x(theta, x, training_device=self._device) + + theta, x = validate_theta_and_x( + theta, x, data_device=data_device, training_device=self._device + ) prior_masks = mask_sims_from_prior(int(from_round), theta.size(0)) - if len(self._num_sims_per_round) == 0: - # If first round, set up ConcatDataset - self._dataset = data.ConcatDataset( - [ - data.TensorDataset(theta, x, prior_masks), - ] - ) - else: - # Otherwise append to Dataset - self._dataset = data.ConcatDataset( - self._dataset.datasets - + [ - data.TensorDataset(theta, x, prior_masks), - ] - ) + self._theta_roundwise.append(theta) + self._x_roundwise.append(x) + self._prior_masks.append(prior_masks) - self._num_sims_per_round.append(theta.size(0)) self._data_round_index.append(int(from_round)) - if return_self: - return self + return self def train( self, diff --git a/sbi/utils/sbiutils.py b/sbi/utils/sbiutils.py index acfc3fdf4..bde9d7375 100644 --- a/sbi/utils/sbiutils.py +++ b/sbi/utils/sbiutils.py @@ -349,31 +349,6 @@ def get_simulations_since_round( ) -def get_simulations_indcies( - num_sims_per_round: List, data_round_indices: List, starting_round_index: int -) -> Tensor: - """ - Returns indicies for for all simulations round >= `starting_round`. Used in - `get_dataloaders` and `get_simulations` - - Args: - num_sims_per_round: Number of simulations per round - data_round_indices: List with same length as data, each entry is an integer that - indicates which round the data is from. - starting_round_index: From which round onwards to return the data. We start - counting from 0. - """ - inds = [] - for j, (n, r) in enumerate(zip(num_sims_per_round, data_round_indices)): - - # Where to start counting - s_ind = sum(num_sims_per_round[:j]) - - if r >= starting_round_index: - inds.append(torch.arange(s_ind, s_ind + n)) - return torch.cat(inds) - - def mask_sims_from_prior(round_: int, num_simulations: int) -> Tensor: """Returns Tensor True where simulated from prior parameters. diff --git a/sbi/utils/user_input_checks.py b/sbi/utils/user_input_checks.py index 66f5a5ec7..443808c78 100644 --- a/sbi/utils/user_input_checks.py +++ b/sbi/utils/user_input_checks.py @@ -647,7 +647,7 @@ def check_estimator_arg(estimator: Union[str, Callable]) -> None: def validate_theta_and_x( - theta: Any, x: Any, training_device: str = "cpu" + theta: Any, x: Any, data_device: str = "cpu", training_device: str = "cpu" ) -> Tuple[Tensor, Tensor]: r""" Checks if the passed $(\theta, x)$ are valid. @@ -679,21 +679,21 @@ def validate_theta_and_x( assert theta.dtype == float32, "Type of parameters must be float32." assert x.dtype == float32, "Type of simulator outputs must be float32." - if str(x.device) != training_device: + if str(x.device) != data_device: warnings.warn( - f"Data x has device '{x.device}' " - f"different from the training_device '{training_device}', " - f"moving x to the training_device '{training_device}'." + f"Data x has device '{x.device}'." + f"Moving x to the data_device '{data_device}'." + f"Training will proceed on device '{training_device}'." ) - x = x.to(training_device) + x = x.to(data_device) - if str(theta.device) != training_device: + if str(theta.device) != data_device: warnings.warn( - f"Parameters theta has device '{theta.device}' " - f"different from the training_device '{training_device}', " - f"moving theta to the training_device '{training_device}'." + f"Parameters theta has device '{x.device}'. " + f"Moving theta to the data_device '{data_device}'." + f"Training will proceed on device '{training_device}'." ) - theta = theta.to(training_device) + theta = theta.to(data_device) return theta, x diff --git a/tests/base_test.py b/tests/base_test.py index b41c47728..c1cef1131 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -11,7 +11,7 @@ def test_get_dataloaders(training_batch_size): validation_fraction = 0.1 inferer = SNPE() - inferer.append_simulations(torch.ones(N), torch.zeros(N), warn_if_zscoring=False) + inferer.append_simulations(torch.ones(N), torch.zeros(N)) _, val_loader = inferer.get_dataloaders( 0, training_batch_size=training_batch_size, From 0cbc4fada376193fd3c0a89a540d3db9013e4360 Mon Sep 17 00:00:00 2001 From: tbmiller-astro Date: Mon, 27 Jun 2022 12:19:34 -0400 Subject: [PATCH 15/15] Use all data to initialize NN --- sbi/inference/snle/snle_base.py | 10 ++++------ sbi/inference/snpe/snpe_base.py | 5 ++--- sbi/inference/snre/snre_base.py | 10 ++++------ sbi/utils/user_input_checks.py | 5 +++++ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sbi/inference/snle/snle_base.py b/sbi/inference/snle/snle_base.py index 626690b51..1c04584b8 100644 --- a/sbi/inference/snle/snle_base.py +++ b/sbi/inference/snle/snle_base.py @@ -189,12 +189,10 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - # Get theta,x from dataset to initialize NN - theta, x, _ = self.get_simulations() - self._neural_net = self._build_neural_net( - theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") - ) - self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + # Get theta,x to initialize NN + theta, x, _ = self.get_simulations(starting_round=start_idx) + self._neural_net = self._build_neural_net(theta.to("cpu"), x.to("cpu")) + self._x_shape = x_shape_from_simulation(x.to("cpu")) del theta, x assert ( len(self._x_shape) < 3 diff --git a/sbi/inference/snpe/snpe_base.py b/sbi/inference/snpe/snpe_base.py index e2c45c0dc..481b2de95 100644 --- a/sbi/inference/snpe/snpe_base.py +++ b/sbi/inference/snpe/snpe_base.py @@ -282,9 +282,8 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - # Get theta,x from dataset to initialize NN - x = self._x_roundwise[0][:training_batch_size] - theta = self._theta_roundwise[0][:training_batch_size] + # Get theta,x to initialize NN + theta, x, _ = self.get_simulations(starting_round=start_idx) self._neural_net = self._build_neural_net(theta.to("cpu"), x.to("cpu")) self._x_shape = x_shape_from_simulation(x.to("cpu")) diff --git a/sbi/inference/snre/snre_base.py b/sbi/inference/snre/snre_base.py index b50a1278d..f448eedd4 100644 --- a/sbi/inference/snre/snre_base.py +++ b/sbi/inference/snre/snre_base.py @@ -199,12 +199,10 @@ def train( # can `sample()` and `log_prob()`. The network is accessible via `.net`. if self._neural_net is None or retrain_from_scratch: - # Get theta,x from dataset to initialize NN - theta, x, _ = self.get_simulations() - self._neural_net = self._build_neural_net( - theta[:training_batch_size].to("cpu"), x[:training_batch_size].to("cpu") - ) - self._x_shape = x_shape_from_simulation(x[:training_batch_size].to("cpu")) + # Get theta,x to initialize NN + theta, x, _ = self.get_simulations(starting_round=start_idx) + self._neural_net = self._build_neural_net(theta.to("cpu"), x.to("cpu")) + self._x_shape = x_shape_from_simulation(x.to("cpu")) del x, theta self._neural_net.to(self._device) diff --git a/sbi/utils/user_input_checks.py b/sbi/utils/user_input_checks.py index 443808c78..8bd46c4e2 100644 --- a/sbi/utils/user_input_checks.py +++ b/sbi/utils/user_input_checks.py @@ -657,6 +657,10 @@ def validate_theta_and_x( 2) If they have the same batchsize. 3) If they are of `dtype=float32`. + Additionally, We move the data to the specified `data_device`. This is where the + data is stored and can be separate from `training_device`, where the + computations for training are performed. + Raises: AssertionError: If theta or x are not torch.Tensor-like, do not yield the same batchsize and do not have dtype==float32. @@ -664,6 +668,7 @@ def validate_theta_and_x( Args: theta: Parameters. x: Simulation outputs. + data_device: Device where data is stored. training_device: Training device for net. """ assert isinstance(theta, Tensor), "Parameters theta must be a `torch.Tensor`."