From 7e54a2d669b084795d0ea32c4731b12a3756c410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Francisco=20Barros=20dos=20Santos?= <36741643+jfsantos-ds@users.noreply.github.com> Date: Sun, 25 Jul 2021 20:47:38 +0000 Subject: [PATCH] feat(drift): added drift mvp Features: - reference covariate drift - reference label drift - sample covariate drift - sample label drift - sample concept drift --- requirements.txt | 1 + src/ydata_quality/core/engine.py | 44 ++- src/ydata_quality/drift/__init__.py | 9 + src/ydata_quality/drift/engine.py | 356 +++++++++++++++++ src/ydata_quality/labelling/engine.py | 41 +- tutorials/drift.ipynb | 531 ++++++++++++++++++++++++++ 6 files changed, 941 insertions(+), 41 deletions(-) create mode 100644 src/ydata_quality/drift/__init__.py create mode 100644 src/ydata_quality/drift/engine.py create mode 100644 tutorials/drift.ipynb diff --git a/requirements.txt b/requirements.txt index 54228512..17545d54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ pandas==1.2.* pydantic==1.8.2 scikit-learn==0.24.2 +matplotlib==3.4.2 diff --git a/src/ydata_quality/core/engine.py b/src/ydata_quality/core/engine.py index adefa214..a3337410 100644 --- a/src/ydata_quality/core/engine.py +++ b/src/ydata_quality/core/engine.py @@ -7,15 +7,17 @@ import pandas as pd from ydata_quality.core import QualityWarning from ydata_quality.core.warnings import Priority - +from ydata_quality.utils.modelling import infer_dtypes class QualityEngine(ABC): "Main class for running and storing data quality analysis." - def __init__(self, df: pd.DataFrame): + def __init__(self, df: pd.DataFrame, label: str = None, dtypes: dict = None): self._df = df self._warnings = set() self._tests = [] + self._label = label + self._dtypes = dtypes @property def df(self): @@ -27,6 +29,44 @@ def warnings(self): "Storage of all detected data quality warnings." return self._warnings + + @property + def label(self): + "Property that returns the label under inspection." + return self._label + + @label.setter + def label(self, label: str): + if not isinstance(label, str): + raise ValueError("Property 'label' should be a string.") + assert label in self.df.columns, "Given label should exist as a DataFrame column." + self._label = label + + @property + def dtypes(self): + "Infered dtypes for the dataset." + if self._dtypes is None: + self._dtypes = infer_dtypes(self.df) + return self._dtypes + + @dtypes.setter + def dtypes(self, dtypes: dict): + if not isinstance(dtypes, dict): + raise ValueError("Property 'dtypes' should be a dictionary.") + assert all(col in self.df.columns for col in dtypes), "All dtypes keys \ + must be columns in the dataset." + supported_dtypes = ['numerical', 'categorical'] + assert all(dtype in supported_dtypes for dtype in dtypes.values()), "Assigned dtypes\ + must be in the supported broad dtype list: {}.".format(supported_dtypes) + df_col_set = set(self.df.columns) + dtypes_col_set = set(dtypes.keys()) + missing_cols = df_col_set.difference(dtypes_col_set) + if missing_cols: + _dtypes = infer_dtypes(self.df, skip=df_col_set.difference(missing_cols)) + for col, dtype in _dtypes.items(): + dtypes[col] = dtype + self._dtypes = dtypes + def store_warning(self, warning: QualityWarning): "Adds a new warning to the internal 'warnings' storage." self._warnings.add(warning) diff --git a/src/ydata_quality/drift/__init__.py b/src/ydata_quality/drift/__init__.py new file mode 100644 index 00000000..7ea6b047 --- /dev/null +++ b/src/ydata_quality/drift/__init__.py @@ -0,0 +1,9 @@ +""" +Tools to check dataset for data drifting. +""" +from ydata_quality.drift.engine import DriftAnalyser, ModelWrapper + +__all__ = [ + "DriftAnalyser", + "ModelWrapper" +] diff --git a/src/ydata_quality/drift/engine.py b/src/ydata_quality/drift/engine.py new file mode 100644 index 00000000..ddac29de --- /dev/null +++ b/src/ydata_quality/drift/engine.py @@ -0,0 +1,356 @@ +""" +Implementation of DriftAnalyser engine to run data drift analysis. +""" +from typing import Callable, Optional, Union + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from scipy.stats import ks_2samp +from scipy.stats._continuous_distns import chi2_gen +from ydata_quality.core import QualityEngine, QualityWarning +from ydata_quality.utils.modelling import infer_dtypes + + +class ModelWrapper: + """Base class for model wrapper. + Defines a Model instance to enable concept drift analysis with the Sampling engine. + This class is meant to cover all functionality needed to interact with the engine. + Can be instantiated directly or inherited from a custom class. + In normal use only the preprocess and postprocess methods should need override.""" + + def __init__(self, model: Callable): + self._model = model + + @property + def model(self): + """Passes the provided callable as the property model.""" + return self._model + + @staticmethod + def _preprocess(x: pd.DataFrame): + """Performs any preprocessing of the model input. + By default returns input without any transformation. + Override to define custom preprocessing steps.""" + return x + + @staticmethod + def _postprocess(y: pd.Series): + """Performs any postprocessing of the models label predictions. + By default returns input without any transformation. + Override to define custom model predictions postprocessing steps.""" + return y + + def _predict(self, x: pd.DataFrame): + """Runs the provided callable model on pretransformed input.""" + if hasattr(self.model, "predict"): # Sklearn and tensorflow model standards + return self.model.predict(x) + else: # Pytorch and other __call__ prediction standards + return self.model(x) + + def __call__(self, x: pd.DataFrame) -> pd.Series: + """Returns a sample of labels predicted by the model from the covariate sample x. + The returned Series is expected to have the same number of rows as x.""" + transformed_x = self._preprocess(x) + raw_y = self._predict(transformed_x) + return self._postprocess(raw_y) + + +class DriftAnalyser(QualityEngine): + """Main class to run drift test analysis. + + Methods: + ref_covariate_drift: controls covariate drift in reference subsamples. + ref_label_drift: controls label drift in the reference subsamples. + sample_covariate_drift: detects covariate drift in the test sample, measured against the full reference sample. + sample_label_drift: detects label drift in the test sample, measured against the full reference sample. + sample_concept_drift: detects concept drift in the test sample based on a wrapped model provided by the user. + """ + + def __init__(self, ref: pd.DataFrame, sample: Optional[pd.DataFrame] = None, + label: Optional[str] = None, model: Callable = None, holdout_size: float = 0.2): + """ + Initializes the engine properties and lists tests for automated evaluation. + Args: + ref (pd.DataFrame): reference sample used to run sampling analysis, ideally the users dataset or a train dataset. + sample (Optional, pd.DataFrame): sample to test drift against the reference sample, can be new data, a slice of the train dataset or a test sample. + label (Optional, str): defines a feature in the provided samples as label. + model (Optional, ModelWrapper): a custom model wrapped by the ModelWrapper class. The model is expected to perform label prediction over the set of features (covariates) of the provided samples. + holdout_size (float): Fraction to be kept as holdout for drift test. + """ + super().__init__(df=ref, label=label) + self.sample = sample + self._model = model + self.has_model = None + self._holdout, self._leftover = self._random_split(ref, holdout_size) + self._tests = ['ref_covariate_drift', 'ref_label_drift', 'sample_covariate_drift', + 'sample_label_drift', 'sample_concept_drift'] + + @property + def sample(self): + "Returns the user provided test sample." + return self._sample + + @sample.setter + def sample(self, sample: pd.DataFrame): + if sample is not None: + assert sorted(list(sample.columns)) == sorted(list(self.df.columns)), "The reference and independent samples must share schema." + self._sample = sample + + @property + def has_model(self): + return self._has_model + + @has_model.setter + def has_model(self, _): + try: + self._has_model = self.__test_model() + except AssertionError: + print("The provided model failed to produce output in the expected format during test and will not be used by the engine.") + self._has_model = False + except: + self._has_model = False + + def __test_model(self): + """Tests the provided model wrapper. + Creates an example input from the provided samples. + A valid test output is a label series with the same number of rows as x. + Raises AssertionError if the model test fails. + Raises a general exception if the conditions for test were not met. + Please remove eventual label column from test_x before passing it to the test.""" + if self.label and self._model is not None: + test_x = self.df.head().copy() + test_x.drop(self.label, axis=1, inplace=True) + output = self._model(test_x) + assert isinstance(output, (pd.Series, np.ndarray)), "The provided model failed to produce the expected output." + assert len(output) == test_x.shape[0], "The provided model failed to produce output with the expected dimensionality." + return True + raise Exception + + @staticmethod + def _random_split(sample: Union[pd.DataFrame, pd.Series], split_size: float, shuffle=True): + """Shuffles sample and splits it into 2 partitions according to split_size. + Returns a tuple with the split first (partition corresponding to split_size, and remaining second). + Args: + sample (pd.DataFrame): A sample to be split + split_size (float): Fraction of the sample to be taken split + shuffle (bool): If True shuffles sample rows before splitting""" + assert 0<= split_size <=1, 'split_size must be a fraction, i.e. a float in the [0,1] interval.' + if shuffle: + sample = sample.sample(frac=1) # Shuffle dataset rows + split_len = int(sample.shape[0]*split_size) + split = sample.iloc[:split_len] + remainder = sample.iloc[split_len:] + return split, remainder + + @staticmethod + def _chisq_2samp(reference_data: pd.Series, test_data: pd.Series): + """Asserts validity of performing chisquared test on two samples. + Tests the hypothesis that the test_sample follows ref_sample's distribution. + Will raise an AssertionError in case the test is not valid. + Args: + reference_data (pd.Series): Reference data, used to compute degrees of freedom and expectation + test_data (pd.Series): Test data, compared to the reference data + Returns: + chi_stat (float): The chi squared statistic of this test + p_val (float): The p-value of the tested hypothesis + """ + ref_unique_freqs = reference_data.value_counts(normalize=True) + test_unique_counts = test_data.value_counts() + assert set(test_unique_counts.index).issubset(set(ref_unique_freqs.index)),"test_sample contains categories unknown to the ref_sample" + test_expected_counts = ref_unique_freqs*len(test_data) + assert sum(test_expected_counts<5)==0, "The test sample has categories with expected count below 5 (this sample is too small for chi-squared test)" + chi_stat = sum(((test_unique_counts-test_expected_counts)**2)/test_expected_counts) + p_val = 1-chi2_gen().cdf(x=chi_stat, df=len(ref_unique_freqs-1)) + return chi_stat, p_val + + def _2sample_feat_goof(self, ref_sample: pd.Series, test_sample: pd.Series): + """Performs a goodness of fit test between 2 samples. + The column dtype of the samples allows for an appropriate statistic test selection. + Returns tuple (statistic_value, p_value, test_name). + If the statistic test raises an exception, (-1, None, test_name) is returned instead. + Args: + ref_sample (pd.Series): Reference sample (Relevant distinction for chi-squared test) + test_sample (pd.Series): Test sample""" + statistics = {'categorical': ('Chi-Squared', self._chisq_2samp), + 'numerical': ('Kolmogorov-Smirnov', ks_2samp)} + feat_dtype = self.dtypes[ref_sample.name] + test_name, test = statistics[feat_dtype] + try: + statistic_value, p_value = test(ref_sample, test_sample) + except: + statistic_value, p_value = -1, None + return statistic_value, p_value, test_name + + def ref_covariate_drift(self, p_thresh: float= 0.05): + """Controls covariate drift in reference subsamples. + The controlled metric is the number of features with no drift detection. + This % is plotted against the size of the reference subsample. + A monotonic increase of the value is expected as the subsample size is increased. + The dtypes are used to decide the test to be applied per column (chi squared or KS). + The p-value threshold is adjusted for the multivariate case via Bonferroni correction. + Args: + p_thresh (float): The p_threshold used for the test. + """ + covariates = self._leftover.copy() + holdout = self._holdout.copy() + if self.label: + covariates.drop(self.label, axis=1, inplace=True) + holdout.drop(self.label, axis=1, inplace=True) + leftover_fractions = np.arange(0.2, 1.2, 0.2) + perc_index = ["{0:.0%}".format(fraction) for fraction in leftover_fractions] + control_metric = pd.Series(index=perc_index) + bonferroni_p = p_thresh/len(covariates.columns) # Bonferroni correction + all_p_vals = pd.DataFrame(index=perc_index, columns=covariates.columns) + for i, fraction in enumerate(leftover_fractions): + downsample, _ = self._random_split(covariates, fraction) + p_vals = [] + for column in covariates.columns: + _, p_val, _ = self._2sample_feat_goof(ref_sample = downsample[column], + test_sample = holdout[column]) + p_vals.append(p_val) + all_p_vals.iloc[i] = p_vals + control_metric.iloc[i] = 100*len([p for p in p_vals if p > bonferroni_p])/len(p_vals) + all_p_vals['Corrected p-value threshold'] = bonferroni_p + control_metric.plot(title='Reference sample covariate features no drift(%)', + xlabel='Percentage of remaining sample used', + ylabel='Percentage of no drift features', + ylim = (0, 104), style='.-') + plt.show() + return all_p_vals + + def ref_label_drift(self, p_thresh: float= 0.05): + """Controls label drift in the reference sample (df). + The p-value of the test is plotted against the size of the reference subsample. + A monotonic increase of this metric is expected as we increase the subsample size. + The dtype is used to decide the test to be applied to the label (chi squared or KS). + Args: + p_thresh (float): The p_threshold used for the test.""" + if self.label is None: + return "[REFERENCE LABEL DRIFT] No label was provided. Test skipped." + labels = self._leftover[self.label].copy() + holdout = self._holdout[self.label] + leftover_fractions = np.arange(0.2, 1.2, 0.2) + p_values = pd.DataFrame(index=["{0:.0%}".format(fraction) for fraction in leftover_fractions], + columns=['Label p-value', 'p-value threshold']) + for i, fraction in enumerate(leftover_fractions): + downsample, _ = self._random_split(labels, fraction) + _, p_val, test_name = self._2sample_feat_goof(ref_sample = downsample, + test_sample = holdout) + p_values['Label p-value'].iloc[i] = p_val + p_values['p-value threshold'] = p_thresh + p_values.plot(title='Reference sample label p-values', + xlabel='Percentage of remaining sample used', + ylabel=f'{test_name} test p-value', style='.-') + plt.show() + + def sample_covariate_drift(self, p_thresh: float= 0.05): + """Detects covariate drift in the test sample (measured against the full reference sample). + The p-value threshold is adjusted for the multivariate case via Bonferroni correction. + Any p-value below the adjusted threshold indicates test sample drift, raising a warning. + The dtypes are used to decide the test to be applied per column (chi squared or KS). + Args: + p_thresh (float): The p_threshold used for the test. + """ + if self.sample is None: + return "[SAMPLE LABEL DRIFT] To run sample covariate drift, a test sample must be provided. Test skipped." + covariates = self.df.copy() + test_sample = self.sample.copy() + if self.label: + covariates.drop(self.label, axis=1, inplace=True) + test_sample.drop(self.label, axis=1, inplace=True) + bonferroni_p = p_thresh/len(covariates.columns) # Bonferroni correction + test_summary = pd.DataFrame(index=covariates.columns, + columns=['Statistic', 'Statistic Value', 'p-value', 'Verdict']) + for column in covariates.columns: + stat_val, p_val, test_name = self._2sample_feat_goof(ref_sample = covariates[column], + test_sample = test_sample[column]) + test_summary.loc[column] = [test_name, stat_val, p_val, None] + test_summary['Verdict'] = test_summary['p-value'].apply( + lambda x: 'OK' if x > bonferroni_p else ('Drift' if x>= 0 else 'Invalid test')) + n_drifted_feats = sum(test_summary['Verdict']=='Drift') + n_invalid_tests = sum(test_summary['Verdict']=='Invalid test') + if n_drifted_feats>0: + self._warnings.add( + QualityWarning( + test='Sample covariate drift', category='Sampling', priority=2, data=test_summary, + description=f"""{n_drifted_feats} features accused drift in the sample test. The covariates of the test sample do not appear to be representative of the reference sample.""" + )) + elif n_invalid_tests>0: + self._warnings.add( + QualityWarning( + test='Sample covariate drift', category='Sampling', priority=3, data=test_summary, + description=f"""There were {n_invalid_tests} invalid tests found. This is likely due to a small test sample size. The data summary should be analyzed before considering the test conclusive.""" + )) + else: + print("[SAMPLE COVARIATE DRIFT] Covariate drift was not detected in the test sample.") + return test_summary + + def sample_label_drift(self, p_thresh: float= 0.05): + """Detects label drift in the test sample (measured against the full reference sample). + A p-value below the adjusted threshold indicates test sample drift, raising a warning. + The label dtype is used to decide the test to be applied (chi squared or KS). + Args: + p_thresh (float): The p_threshold used for the test. + """ + if self.sample is None or self.label is None or self.label not in self.sample.columns: + return "[SAMPLE LABEL DRIFT] To run sample label drift, a test sample must be provided with the defined label column. Test skipped." + labels = self.df[self.label].copy() + test_sample = self.sample[self.label].copy() + stat_val, p_val, test_name = self._2sample_feat_goof(ref_sample = labels, + test_sample = test_sample) + test_summary = pd.Series(data=[test_name, stat_val, p_val, None], + index=['Statistic', 'Statistic Value', 'p-value', 'Verdict']) + test_summary['Verdict'] = 'OK' if p_val > p_thresh else ('Drift' if p_val>= 0 else 'Invalid test') + if test_summary['Verdict']=='Drift': + self._warnings.add( + QualityWarning( + test='Sample label drift', category='Sampling', priority=2, data=test_summary, + description=f"""The label accused drift in the sample test with a p-test of {p_val}, which is under the threshold {p_thresh}. The label of the test sample does not appear to be representative of the reference sample.""" + )) + elif test_summary['Verdict']=='Invalid test': + self._warnings.add( + QualityWarning( + test='Sample label drift', category='Sampling', priority=3, data=test_summary, + description=f"""The test was invalid. This is likely due to a small test sample size.""" + )) + else: + print("[SAMPLE LABEL DRIFT] Label drift was not detected in the test sample.") + return test_summary + + def sample_concept_drift(self, p_thresh: float= 0.05): + """Detects concept drift in the test sample resorting to a user provided model wrapper. + Results may not be conclusive without first testing if the test sample has label or covariate drift. + A p-value below the adjusted threshold indicates test sample concept drift, raising a warning. + The label dtype is used to decide the test to be applied (chi squared or KS). + Args: + p_thresh (float): The p_threshold used for the test. + """ + if not self.has_model or self.sample is None: + return "[CONCEPT DRIFT] To run concept drift, a valid model, a test sample and label column must be provided. Test skipped." + ref_sample = self.df.copy() + test_sample = self.sample.copy() + ref_sample.drop(self.label, axis=1, inplace=True) + test_sample.drop(self.label, axis=1, inplace=True) + ref_preds = pd.Series(self._model(ref_sample), name=self.label) + test_preds = pd.Series(self._model(test_sample), name=self.label) + stat_val, p_val, test_name = self._2sample_feat_goof(ref_sample = ref_preds, + test_sample = test_preds) + test_summary = pd.Series(data=[test_name, stat_val, p_val, None], + index=['Statistic', 'Statistic Value', 'p-value', 'Verdict']) + test_summary['Verdict'] = 'OK' if p_val > p_thresh else ('Drift' if p_val>= 0 else 'Invalid test') + if test_summary['Verdict']=='Drift': + self._warnings.add( + QualityWarning( + test='Concept drift', category='Sampling', priority=2, data=test_summary, + description=f"""There was concept drift detected with a p-test of {p_val}, which is under the threshold {p_thresh}. The model's predicted labels for the test sample do not appear to be representative of the distribution of labels predicted for the reference sample.""" + )) + elif test_summary['Verdict']=='Invalid test': + self._warnings.add( + QualityWarning( + test='Concept drift', category='Sampling', priority=3, data=test_summary, + description=f"""The test was invalid. This is likely due to a small test sample size.""" + )) + else: + print("[CONCEPT DRIFT] Concept drift was not detected between the reference and the test samples.") + return test_summary diff --git a/src/ydata_quality/labelling/engine.py b/src/ydata_quality/labelling/engine.py index cc6487d9..a1a44507 100644 --- a/src/ydata_quality/labelling/engine.py +++ b/src/ydata_quality/labelling/engine.py @@ -26,46 +26,9 @@ class SharedLabelInspector(QualityEngine): """Shared structure for Numerical/Categorical Label Inspector""" def __init__(self, df: pd.DataFrame, label: str): - super().__init__(df) # Runs init from the Quality Engine - self._label = label - self._dtypes = infer_dtypes(self.df) + super().__init__(df=df, label=label) self._tdf = None - @property - def label(self): - "Property that returns the label under inspection." - return self._label - - @label.setter - def label(self, label: str): - if not isinstance(label, str): - raise ValueError("Property 'label' should be a string.") - assert label in self.df.columns, "Given label should exist as a DataFrame column." - self._label = label - - @property - def dtypes(self): - "Property that returns infered dtypes for the dataset." - return self._dtypes - - @dtypes.setter - def dtypes(self, dtypes: dict): - if not isinstance(dtypes, dict): - raise ValueError("Property 'dtypes' should be a dictionary.") - assert all(col in self.df.columns for col in dtypes), "All dtypes keys \ - must be columns in the dataset." - supported_dtypes = ['numerical', 'categorical'] - assert all(dtype in supported_dtypes for dtype in dtypes.values()), "Assigned dtypes\ - must be in the supported broad dtype list: {}.".format(supported_dtypes) - df_col_set = set(self.df.columns) - dtypes_col_set = set(dtypes.keys()) - missing_cols = df_col_set.difference(dtypes_col_set) - if missing_cols: - _dtypes = infer_dtypes(self.df, skip=df_col_set.difference(missing_cols)) - for col, dtype in _dtypes.items(): - dtypes[col] = dtype - self._dtypes = dtypes - @property def tdf(self): "Property that returns the transformed dataset centroids for all (not nan) classes." @@ -113,7 +76,7 @@ class CategoricalLabelInspector(SharedLabelInspector): Ordinal labels can be handled if passed as categorical.""" def __init__(self, df: pd.DataFrame, label: str): - super().__init__(df, label) + super().__init__(df=df, label=label) self._centroids = None self._tests = ["missing_labels", "few_labels", "unbalanced_classes", "one_vs_rest_performance", "outlier_detection"] diff --git a/tutorials/drift.ipynb b/tutorials/drift.ipynb new file mode 100644 index 00000000..3a8b3e6b --- /dev/null +++ b/tutorials/drift.ipynb @@ -0,0 +1,531 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# YData Quality - Data Drift Tutorial\n", + "Time-to-Value: 10 minutes\n", + "\n", + "This notebook provides a tutorial for the ydata_quality package funcionality for drift analysis.\n", + "\n", + "**Structure:**\n", + "\n", + "0. A data drifting introduction\n", + "1. Load dataset\n", + "2. Train and wrap example model\n", + "3. Distort dataset\n", + "4. Instantiate the Data Quality engine\n", + "5. Run the quality checks\n", + "6. Assess the warnings\n", + "7. (Extra) Detailed overview" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A data drifting introduction\n", + "### What is data drift?\n", + "Data drift is a broad term used for differences in the data observed by a model during training and prediction time.\n", + "\n", + "These divergences are part of most real world settings and can hinder the task of learning a mapping from input (features) space to output space (labels).\n", + "Sometimes these divergences are intended, such as in __[Domain Adaptation](https://en.wikipedia.org/wiki/Domain_adaptation)__ applications where the success is defined by the model's ability to learn from a rich, source dataset and generalize this capacity to a target dataset where labelled data can be scarce or unavailable.\n", + "On other cases, the differences are unintended and reflect the dynamic nature of the underlying use case (e.g. stock prices) or are just a reflex of the difficulty of creating a representative dataset of the target population.\n", + "\n", + "For the three scenarios presented below, drift is an important concept and a pivotal aspect of Data Quality.\n", + "***\n", + "### Types of data drift\n", + "We identify and diagnose the three main types of data drift:\n", + "* **Covariate, input drift (X)**\n", + " * Drift detected in the independent features between a reference sample (e.g. train data) and a target sample (p.e. test data). In an ideal scenario we expect $X_{ref}$ = $X_{target}$, which means that, statistically, input data from the target appears to be generated from the same distribution as the reference data.\n", + "* **Label drift (Y)**\n", + " * Drift detected in the dependent feature between a reference sample and a target sample. In an ideal scenario we expect $Y_{ref}$ = $Y_{target}$, which means that, statistically, labels from the reference sample appear to be generated from the same distribution.\n", + "* **Concept drift (Y|X)**\n", + " * Drift detected in the input/output mappings defined by a model between a reference and a target sample. This gains more relevance if we can first establish that there is no covariate or label drift. In an ideal scenario we expect $Y_{ref}|X_{ref}$ = $Y_{target}|X_{target}$, which means that, statistically, the model appears to follow the same mapping process in predicting the labels from the reference samples.\n", + "***\n", + "### How do we detect data drift\n", + "There are many strategies to detect data drift. Some strategies rely on fitting a classifier to distinguish which observations come from a reference sample and define the existence of drift based on the performance of this classifier (a bad performance suggests a low data drift). Other strategies rely on empirical distance metrics like the two sample __[Maximum Mean Discrepancy](https://jmlr.csail.mit.edu/papers/volume13/gretton12a/gretton12a.pdf)__ or on classical statistical tests and hypothesis testing.\n", + "\n", + "To depend on the least assumptions, we choose the latter alternative. For all numerical features we apply a __[Kolmogorov-Smirnov test](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test)__ and for categorical data we use the __[Chi-Squared test](https://en.wikipedia.org/wiki/Chi-squared_test)__. The standard p-value thresholds are used to determine the outcome of the performed tests. For concept drift we abstain from trying to fit a model and evaluating drift with that model, because that is no guarantee of similar behaviour by a user's model. For this reason we accomodate user provided models and analyse the model behaviour for the intended task which should provide the most useful insights.\n", + "\n", + "Since input data (X) is often multi-dimensional, we agglomerate the results of multiple statistical tests by applying a __[Bonferroni correction](https://en.wikipedia.org/wiki/Bonferroni_correction)__ to the p-value. This allows for a global assessment of drift from many individual statistical tests. So, when we say that there is covariate drift, we mean that at least one of the features has failed a statistical test with higher likelihood than our Bonferroni corrected threshold.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Drift engine demo and tutorial\n", + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn import datasets\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "from ydata_quality.drift import DriftAnalyser, ModelWrapper\n", + "import random" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the example dataset\n", + "We will use the wine dataset available from the sklearn package." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "df = datasets.load_wine(as_frame=True)['frame']\n", + "df['target'] = df['target'].apply(str) # Forcing categorical type inferral on target feature" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train and wrap example model\n", + "We will train a simple classifier on the full dataset and wrap it with the ModelWrapper class just to demonstrate the engine's concept drift test." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# First we train a classifier\n", + "x = df.loc[:, df.columns != 'target']\n", + "y = df['target']\n", + "clf = DecisionTreeClassifier(random_state=0)\n", + "clf.fit(x, y)\n", + "\n", + "# Now we wrap it by directly instantiating the ModelWrapper class.\n", + "# The wrapper will handle using __call__ or predict method depending on the passed model\n", + "wrapped_model = ModelWrapper(clf)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Distort the original dataset\n", + "Apply transformations to highlight the data quality functionalities." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def create_sample(df, frac=0.2):\n", + " # Forge a sample\n", + " sample = df.sample(frac=frac)\n", + " # Distort sample values\n", + " sample['alcohol'] = sample['alcohol'] + 0.8\n", + " sample['target'] = random.choices(['0', '1', '2'], weights=[0.7, 0.2, 0.1], k=len(sample))\n", + " return sample\n", + "\n", + "sample = create_sample(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the engine\n", + "Each engine contains the checks and tests for each suite. To create a DriftAnalyser, you provide:\n", + "- ref: reference DataFrame, DataFrame that we will assume as the reference for the modelled population\n", + "- sample (optional): A test sample which we will compare against the reference. It should have the same schema as the reference dataframe, although the label column can always be optional for this sample (even when provided for the reference)\n", + "- label (optional): A string defining the label feature (will be searched for both in the reference and test samples)\n", + "- model (optional): A callable that inherits or directly instantiates ModelWrapper. This is supposed to wrap a label prediction model that consumes the provided sample covariates as input\n", + "- holdout_size (optional): A fraction defining the percentage of rows from the reference sample that are held-out for the reference tests. A 20% random subsample is taken by default." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "da = DriftAnalyser(ref=df, sample=sample, label='target', model=wrapped_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Full Evaluation\n", + "The easiest way to assess the data quality analysis is to run `.evaluate()` which returns a list of warnings for each quality check. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAApWklEQVR4nO3debxVZb3H8c9XQEQFASUvgoiz2eAQOaR1Sc3SMjKHNDMxzWzQbHLodh2uVjZpapaa5nQVp5zScshEcw6UnND0KigKigYK5gDyu388z9ksdnufs86w98bD9/167ddZ8/Nb66y9fns9a61nKSIwMzMDWK7VAZiZ2dLDScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBQaQNLqkm6XNE/SL1odzzuVpPGS7mh1HF0laR9JNzWpLEk6V9IcSfc1o8x3OknnSTqhnfEflvR4oX9DSVPy9/rQdua7U9JmXYhndUlTJfXv7Lw9yUmhDknTJL0uab6kWXkHWrnk7AcBLwGDIuI7DQzTlmIRcVFE7FhmWknHSvrfbhS3LfAxYGREbNGN5bzjk3FPiYi/RsSGhUGHA7dGxMCIODUfI3YoziNpF2BeRDyQ+7eX9HQ+huxVmG6wpPslDSyU9wJwK+n40TJOCu3bJSJWBjYFNgOOKjnfWsCj0YUnAyX17ew8tvRpwf9xLWBaRLzW5HL/TW/Yh+usw1rAIx3MejBwYaH/l8AuwMeBX0vqk4f/GDgxIuZVzX8R8JVOB9yTIsKfGh9gGrBDof+nwPWF/q2Au4C5wN+BsXn4ecAC4C1gPrADKfkeCfwf8DJwGTA0Tz8aCOAA4Bng9jz8S8BUYA5wI7BWoewg7XxP5PJPB1QY/+U87zzgUWDzPHwN4PfAbOBp4NB21n/nPO884Dngu3n4EOC6vIw5uXtkYb6JwAl528wH/gCsStrZXwX+BoyuWpdDgadIZ1c/A5bL48YDdxSm3Qi4Gfgn8DiwZzvxDwXOBZ7PcV5dtX2ezMu5FlgjD/8N8POq5VwDfDt3t/0P27brroXpxgN3Aifn//EJNeI/BXg2b4fJwIfz8E/k/WVB3mZ/z8NXAc4BZub/wQlAnxrregDwBvB2nv+4PPxTwJS8j9wFvL8wT811Ad5dtay5hf/rgVXrW1y3AL5O2iefLlH+EXmd5uX/5fZ1/o/nkfbv6/O09wLrFsZ/iLRPvZL/fqidfWIz4P68nEuBS4AT8rixwIwc1yzSgX0sMCOP/0veJm/k7TIBWAS8nvsPB5bP/cXvw1OF7lnAu4AtgBvqxNgX+BeF73vTj32tKnhp/1BICsBI4CHglNw/gvTF35l0wP9Y7h9W2JFPKCzrm8A9eTn9gTOBCXnc6PyFugBYCRgAjCMdtN6dd5IfAHcVlhekg/FgYBTpAP2JPG6P/GX7ICBgPdIvnOVIB6Kj8867DulA/PE66z+TxQetISxOLKsCuwErAgOBy1nygDsxx74u6aD2KPAPUnLsm9fz3Kp1uZV0EB+Vpz0wjxtPPvDkbfMssH9ezmakJLJxnfivJ33xhwD9gP/Mw7fL822e/xensTgRfySXocJ6v87ipLEHKbEuB3wOeA0YXoh1IXBIjm8A/37g/ELefn2B75AOEivkcccC/1u1DleR9pWVSAeT+4Cv1Fnf6rI2A14EtgT6APuR9un+JdfljqrlT6TjpHBz/j8OaK98YMO8ndu262gKB/qqcs8jfbe2yNvtIuCSPG4oKeHvm8ftnftXrbGc5YHpwLfy/rA7KQkXk8JC4Cc5xgEUkkKdbTCNJX84vgd4rarce4BN8uf5XPbdwAbtHHseBD7dsmNfqwpe2j/5Hz6f9KsigFuAwXncEcCFVdPfCOxX2JGLSWEqhV9CwPC8Q/ZlcVJYpzD+T8ABhf7lKPx6yNNvWxh/GXBkIY5v1lifLYFnqoYdReEAXTXuGdJp7KAOttOmwJxC/0Tgvwr9vwD+VOjfBZhS6A9yQsv9XwNuyd3jWZwUPgf8tarsM4FjasQ0nPQrbkiNcecAPy30r5z/F6NJSfQZ4CN53JeBv7Sz7lOAcYVYq7dvJf46888BNsndx1JICsDqwJvAgMKwvUl12rWWtURZpLOe46umeZycHEusS1eSwnZlyif9UHmR9EOhXwf713nA2YX+nYHHcve+wH1V098NjK+xnI+QDsrFM+q7WDIpvEVO0oVhnUkK2wCzanw/JpLOcLYnnRUfD7yf9F29tfp/Qjrj/GJ726WRH19TaN9nImIgaefYCFgtD18L2EPS3LYP6ULf8DrLWQu4qjDtVNKp6OqFaZ6tmv6UwvT/JB2wRhSmmVXo/hfp4AawJqlaoFYMa1TF/P2qGIp2I30Bp0u6TdLWAJJWlHSmpOmSXgVuBwYX6koBXih0v16jv/qCfXHdp5N+wdaKf8uq+PcB/qPGtGsC/4yIOTXGrZHLACAi5pN+iY6I9I28hHTwBfg86ZcpAJK+mO8+aSv/vSzeJ6rX499I+m6+u+SVPP8qVfNXr28/YGahvDNJZwxlrAV8p2p7rUnetiXWpSuq9+Ga5UfEk8BhpET4oqRLJNX6n7ept68v8b/MprPk94TCtM/l/3Fx2qLZEfFGO3F0ZA7p7LkiIqZExNiI2JJ01vwl4EfA2cBxpDPfCyWpMNtAUpVbSzgplBARt5F+sfw8D3qWdKYwuPBZKSJOrLOIZ4GdqqZfISKeKxZTNf1XqqYfEBF3lQj3WVLVTa3hT1ctc2BE7Fxnnf8WEeNIB6GrSWcjkKo9NgS2jIhBpF9gkJJWV61Z6B5F+kVXK/7bquJfOSK+WmfaoZIG1xj3POmAlYKWViJV6bT9LyYAu0tai3R29fs83VrAb4FvkKonBgMPs+R6F/+HS5D0YVK9856kM5jBpHrwtvmr532WdKawWmF9B0XEe+qVUWP+H1ZtrxUjYkKJdam1Hq+Rqgzb1ErG1ftwzfIBIuLiiNiW9L8IUrVNZy3xv8xGsfh/WTQTGFF18B3VTvxlVE//JOnu4FpJCdL1ph9ExOvA+4BJETGNlPyHQeUC93qk65Qt4aRQ3i+Bj0naBPhfYBdJH5fUR9IKksZKGlln3jOAH+YvI5KGSRrXTllnAEdJek+efhVJe5SM82zgu5I+kO9dXy+Xex8wT9IRkgbkuN8r6YPVC5C0vNI99qtExALShdFFefRA0q/9uZKGAseUjKs935M0RNKapOsvl9aY5jpgA0n7SuqXPx+U9O7qCSNiJqkK7td5uf0ktSWvCcD+kjbN94P/CLg3fzmJdCvhS6TteGNEzM3zrUQ6CMwGkLQ/6dd1WQNJddazgb6SjgYGFca/AIyWtFxhHW4CfiFpkKTlJK0r6T9Llvdb4GBJW+b9YCVJn8y3QHa0Li8AIyUtXxg2BfhsPlNcj3Rxu0vlK93vv13e/m+Q9qdF7S+upj+S9onPS+or6XPAxqR9pdrdpO1/aN4fPku6TtEdL5CuzQEQEW8BfyZVkS1B0sdIVVNtsT0NbJe/4/1JZ6vkmKZFRPVZTNM4KZQUEbNJF0mPjohnSReDv0/6Yj0LfI/62/MU0l0uN0maR7r4tGU7ZV1F+uV0Sa6ieRjYqWSclwM/BC4mXQ+5mnSn09uku0E2Je2QbQe+Veosal9gWi7/YFJVDaTkOCDPfw9wQ5m4OnAN6SL4FNIF4nOqJ4h0696OwF6kX4izWHxRsF78C4DHSPXXh+Xl/Bn4b9IZwEzSWdVeVfNeTKrvvrhQ/qOk6yN3kw4G7yPV/ZZ1I2lb/YNUbfEGS1a3XJ7/vizp/tz9RdIF0kdJVRNXUL+KcgkRMYl0TeRXed4nSdcByqzLX0i3Xs6S9FIedjKpzv0F4HwK1WqdLZ/0PzuRtA+13ZFT9nbvYhkvk/bp75AOqocDn4qIl2pM+xbw2RzDP0nXqK7sbJlVfgz8IFePfTcPO5O071Xk5Pcz0g+eNoeQfvz9Gfha/n5C+p6d0c24uqXtLguzlpAUwPq5ntnsHU/SncA38llnZ+Z7F3AbsFk3r210i5OCtZSTgtnSxdVHZmZW4TMFMzOr8JmCmZlVvKMbrlpttdVi9OjRrQ7DzOwdZfLkyS9FxLBa497RSWH06NFMmjSp1WGYmb2jSKr7HISrj8zMrMJJwczMKpwUzMyswknBzMwqGpYUJP1O0ouSHi4MGyrpZklP5L9D8nBJOlXSk5IelLR5o+IyM7P6GnmmcB7pNYNFR5JeoLI+6aU1R+bhOwHr589BpBd0NMzk6XM4/dYnmTy9VnP7Vs3bq3O8vTrP26xzGrm9GnZLakTcLml01eBxpBfWQGppcSLpLWbjgAvyCzDukTRY0vDcfHCPmjx9DnueeTdvLwqWE2z0HwMZuEK/ni6m15j3xgIemzWPRYG3VwneXp3nbdY5bdsrAvr3W46LDtyKD6w1pMeW3+xrCqsXDvSzWPzWrxEs2YzwDGq/PQlJB0maJGnS7NmzOx3APU+9zKJFqWmPRQGvvrGw08tYlrz6xkLy5vL2KsHbq/O8zTqnbXsFsGDhIu556uUO5+mMlj28FhGRW8js7HxnAWcBjBkzptPzb7XOqvTvtxwLFi6iX9/lOGWvzXo0y/Y2k6fPYZ+z7/H2Ksnbq/O8zTqnentttc6qPbr8hjaIl6uProuI9+b+x4GxETFT0nBgYkRsKOnM3D2herr2lj9mzJjoyhPNk6fP4Z6nXmardVb1zleCt1fneHt1nrdZ53R3e0maHBFjao5rclL4GfByRJwo6UjSG8EOl/RJ0vtidya9kezUiOjwVXldTQpmZsuy9pJCw6qPJE0gXVReTdIM0rt8TwQuk3QA6ZWEe+bJ/0hKCE8C/wL2b1RcZmZWXyPvPtq7zqjta0wbwNcbFYuZmZXjJ5rNzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKyiU0lB0hBJ729UMGZm1lodJgVJEyUNkjQUuB/4raSTGh+amZk1W5kzhVUi4lXgs8AFEbElsENjwzIzs1YokxT6ShoO7Alc1+B4zMyshcokhf8BbgT+LyL+Jmkd4InGhmVmZq3Qt6MJIuJy4PJC/1PAbo0MyszMWqPMheYNJN0i6eHc/35JP2h8aGZm1mxlqo9+CxwFLACIiAeBvRoZlJmZtUaZpLBiRNxXNWxhI4IxM7PWKpMUXpK0LhAAknYHZjY0KjMza4kOLzQDXwfOAjaS9BzwNLBPQ6MyM7OWaDcpSOoDfC0idpC0ErBcRMxrTmhmZtZs7SaFiHhb0ra5+7XmhGRmZq1SpvroAUnXkp5VqCSGiLiyq4VK+hZwIOk6xUPA/sBw4BJgVWAysG9EvNXVMszMrPPKXGheAXgZ2A7YJX8+1dUCJY0ADgXGRMR7gT6kW1x/ApwcEesBc4ADulqGmZl1TZknmvdvULkDJC0AViTdzbQd8Pk8/nzgWOA3DSjbzMzq6DApSDqXfDtqUUR8qSsFRsRzkn4OPAO8DtxEqi6aGxFtzz/MAEZ0ZflmZtZ1Za4pFFtGXQHYFXi+qwVKGgKMA9YG5pKuVXyiE/MfBBwEMGrUqK6GYWZmNZSpPvp9sV/SBOCObpS5A/B0RMzOy7sS2AYYLKlvPlsYCTxXJ56zSM9NMGbMmH87gzEzs67ryjua1wfe1Y0ynwG2krSiJAHbA48CtwK752n2A67pRhlmZtYFZa4pzGPJawqzgCO6WmBE3CvpCtKrPRcCD5B++V8PXCLphDzsnK6WYWZmXVOm+mhgTxcaEccAx1QNfgrYoqfLMjOz8sq8T+GWMsPMzOydr+6ZgqQVSM8QrJbvGFIeNQjfLmpm1iu1V330FeAwYA3ScwRtSeFV4FeNDcvMzFqhblKIiFOAUyQdEhGnNTEmMzNrkTIXmk+T9F5gY9LDa23DL2hkYGZm1nxlbkk9BhhLSgp/BHYiPbzmpGBm1suUeXhtd9IDZrNy43ibAKs0NCozM2uJMknh9YhYBCyUNAh4EVizsWGZmVkrlGkQb5KkwcBvSXchzQfubmRQZmbWGmUuNH8td54h6QZgUEQ82NiwzMysFco80SxJX5B0dERMA+ZKcnMUZma9UJlrCr8Gtgb2zv3zgNMbFpGZmbVMmWsKW0bE5pIeAIiIOZKWb3BcZmbWAmXOFBZI6kNuPlvSMGBRQ6MyM7OWKJMUTgWuAt4l6YekB9d+1NCozMysJdprJXXtiHg6Ii6SNJn0AJuAz0TE1KZFaGZmTdPeNYUrgA9IuiUitgcea1JMZmbWIu0lheUkfR/YQNK3q0dGxEmNC8vMzFqhvWsKewFvkxLHwBofMzPrZdp7n8LjwE8kPRgRf2piTGZm1iId3n3khGBmtuwoc0uqmZktI8q0fdS/zDAzM3vnK3OmUKuZbDedbWbWC7X38Np/ACOAAZI2Iz24BjAIWLEJsZmZWZO195zCx4HxwEjgFyxOCq8C329sWGZm1grtJYWNI+KjkvaMiMuaFpGZmbVMe9cUdpYk4MhmBWNmZq3V3pnCDcAcYGVJrxaGC4iIGNTQyMzMrOnqnilExPciYjBwfUQMKnwGOiGYmfVOZZ5oHteMQMzMrPXqJgVJd+S/8yS9Wv23eSGamVmztNcg3rb5r1tENTNbRrRbfSSpj6Qef7mOpMGSrpD0mKSpkraWNFTSzZKeyH+H9HS5ZmbWvnaTQkS8DTwuaVQPl3sKcENEbARsAkwl3fp6S0SsD9yCb4U1M2u69m5JbTMEeETSfcBrbQMj4tNdKVDSKsBHSE9LExFvAW9JGgeMzZOdD0wEjuhKGWZm1jVlksJ/93CZawOzgXMlbQJMBr4JrB4RM/M0s4DVa80s6SDgIIBRo3r6BMbMbNnWYVKIiNsaUObmwCERca+kU6iqKoqIkBR14jkLOAtgzJgxNacxM7Ouaa+V1HlA3YNuNx5gmwHMiIh7c/8VpKTwgqThETFT0nDgxS4u38zMuqi9W1IHAkg6HpgJXEhq4mIfYHhXC4yIWZKelbRhfg/09sCj+bMfcGL+e01XyzAzs64pc03h0xGxSaH/N5L+DhzdjXIPAS6StDzwFLA/6U6oyyQdAEwH9uzG8s3MrAvKJIXXJO0DXEKqTtqbwl1IXRERU4AxNUZt353lmplZ95R5HefnSb/aX8ifPfIwMzPrZcrcfTQNcKN4ZmbLgDJnCmZmtoxwUjAzswonBTMzq+gwKUhaRdLJkiblzy9y+0VmZtbLlDlT+B3wKukOpD1z97mNDMrMzFqjzHMK60bEboX+4yRNaVA8ZmbWQmXOFF6XtG1bj6RtgNcbF5KZmbVKmTOFg4EL8nUEAf8kvwvBzMx6lzIPr/0d2ETSoNz/asOjMjOzlugwKUjqD+wGjAb6SgIgIv6noZGZmVnTlak+ugZ4hfSGtDcbG46ZmbVSmaQwMiI+0fBIzMys5crcfXSXpPc1PBIzM2u5MmcK2wLjJT1Nqj4S6TXK729oZGZm1nRlksJODY/CzMyWCmVuSZ3ejEDMzKz13EqqmZlV1E0K+fkEMzNbhrR3pnA3gKQLmxSLmZm1WHvXFJaX9HngQ5I+Wz0yIq5sXFhmZtYK7SWFg4F9gMHALlXjAnBSMDPrZeomhYi4A7hD0qSIOKeJMZmZWYuUeU7hQkmHAh/J/bcBZ0TEgsaFZWZmrVAmKfwa6Jf/AuwL/AY4sFFBmZlZa5RJCh+MiE0K/X+R9PdGBWRmZq1T5uG1tyWt29YjaR3g7caFZGZmrVLmTOF7wK2SniI1hrcWsH9DozIzs5Yo0/bRLZLWBzbMgx6PCL9sx8ysFypzpkBOAg82OBYzM2sxN4hnZmYVTgpmZlbRYVJQ8gVJR+f+UZK26G7BkvpIekDSdbl/bUn3SnpS0qWSlu9uGWZm1jllzhR+DWwN7J375wGn90DZ3wSmFvp/ApwcEesBc4ADeqAMMzPrhDJJYcuI+DrwBkBEzAG69Ste0kjgk8DZuV/AdsAVeZLzgc90pwwzM+u8MklhgaQ+pJZRkTQMWNTNcn8JHF5YzqrA3IhYmPtnACNqzSjpIEmTJE2aPXt2N8MwM7OiMknhVOAq4F2SfgjcAfyoqwVK+hTwYkRM7sr8EXFWRIyJiDHDhg3rahhmZlZDmYfXLpI0Gdie9ETzZyJiageztWcb4NOSdgZWAAYBpwCDJfXNZwsjgee6UYaZmXVBmbuPhgIvAhOAi4EXJPXraoERcVREjIyI0cBewF8iYh/gVmD3PNl+wDVdLcPMzLqmTPXR/cBs4B/AE7l7mqT7JX2gB2M5Avi2pCdJ1xj8Yh8zsyYr08zFzcAVEXEjgKQdgd2Ac0m3q27Z1cIjYiIwMXc/BXT7+QczM+u6MmcKW7UlBICIuAnYOiLuAfo3LDIzM2u6MmcKMyUdAVyS+z9Huq7Qh+7fmmpmZkuRMmcKnyfdDXR1/ozKw/oAezYqMDMza74yt6S+BBxSZ/STPRuOmZm1UodJIT/BfDjwHtJzBQBExHYNjMvMzFqgTPXRRcBjwNrAccA04G8NjMnMzFqkTFJYNSLOARZExG0R8SVS43VmZtbLlLn7aEH+O1PSJ4HngaGNC8nMzFqlTFI4QdIqwHeA00htFR3WyKDMzKw1yiSFORHxCvAK8FEASds0NCozM2uJMtcUTis5zMzM3uHqnilI2hr4EDBM0rcLowaRHlwzM7Nepr3qo+WBlfM0AwvDX2VxE9dmZtaL1E0KEXEbcJuk8yJiehNjMjOzFilzobm/pLOA0cXp/USzmVnvUyYpXA6cAZwNvN3YcMzMrJXKJIWFEfGbhkdiZmYtV+aW1D9I+pqk4ZKGtn0aHpmZmTVdmTOF/fLf7xWGBbBOz4djZmatVOZ9Cms3IxAzM2u9DquPJK0o6Qf5DiQkrS/pU40PzczMmq3MNYVzgbdITzcDPAec0LCIzMysZcokhXUj4qfkJrQj4l+AGhqVmZm1RJmk8JakAaSLy0haF3izoVGZmVlLlLn76BjgBmBNSRcB2wDjGxmUmZm1Rpm7j26WdD+wFana6JsR8VLDIzMzs6Yrc/fRrqSnmq+PiOuAhZI+0/DIzMys6cpcUzgmv3kNgIiYS6pSMjOzXqZMUqg1TZlrEWZm9g5TJilMknSSpHXz5yRgcqMDMzOz5iuTFA4hPbx2KXAJ8Abw9UYGZWZmrdFuNZCkPsB1EfHRJsVjZmYt1O6ZQkS8DSyStEqT4jEzsxYqc8F4PvCQpJuB19oGRsShXSlQ0prABcDqpKekz4qIU/I7Gi4lvfZzGrBnRMzpShlmZtY1ZZLClfnTUxYC34mI+yUNBCbnhDMeuCUiTpR0JHAkcEQPlmtmZh0o80Tz+bnto1ER8Xh3C4yImcDM3D1P0lRgBDAOGJsnOx+YiJOCmVlTlXmieRdgCqn9IyRtKunanihc0mhgM+BeYPWcMABmkaqXas1zkKRJkibNnj27J8IwM7OszC2pxwJbAHMBImIKPfAqTkkrA78HDouIV4vjIiLIrbJWi4izImJMRIwZNmxYd8MwM7OCMklhQbGZi2xRdwqV1I+UEC6KiLbrFS9IGp7HDwde7E4ZZmbWeWWSwiOSPg/0ya/iPA24q6sFShJwDjA1Ik4qjLoW2C937wdc09UyzMysa8o+0fwe0ot1LgZeAQ7rRpnbAPsC20makj87AycCH5P0BLBD7jczsyaqe/eRpBWAg4H1gIeArSNiYXcLjIg7qP86z+27u3wzM+u69s4UzgfGkBLCTsDPmxKRmZm1THvPKWwcEe8DkHQOcF9zQjIzs1Zp70xhQVtHT1QbmZnZ0q+9M4VNJLU9PyBgQO4X6VGCQQ2PzszMmqpuUoiIPs0MxMzMWq/MLalmZraMcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzs4qlKilI+oSkxyU9KenIVsdjZrasWWqSgqQ+wOnATsDGwN6SNm5tVGZmy5alJikAWwBPRsRTEfEWcAkwrsUxmZktU/q2OoCCEcCzhf4ZwJbVE0k6CDgo986X9HgXy1sNeKmL8zaS4+ocx9V5S2tsjqtzuhPXWvVGLE1JoZSIOAs4q7vLkTQpIsb0QEg9ynF1juPqvKU1NsfVOY2Ka2mqPnoOWLPQPzIPMzOzJlmaksLfgPUlrS1peWAv4NoWx2RmtkxZaqqPImKhpG8ANwJ9gN9FxCMNLLLbVVAN4rg6x3F13tIam+PqnIbEpYhoxHLNzOwdaGmqPjIzsxZzUjAzs4pelxQkrSnpVkmPSnpE0jfz8KGSbpb0RP47JA/fLU/3V0mr5mHrSrq0levRbJL6SHpA0nW5f21J9+YmRy7NF/+RdIikhyX9sTBsW0kntzL+ZpI0WNIVkh6TNFXS1t6/6pP0rbwNHpY0QdIK3r+WJOl3kl6U9HBhWL19SpJOzdvuQUmb5+EbSpqch22dh/WV9GdJK5YOJiJ61QcYDmyeuwcC/yA1m/FT4Mg8/EjgJ7l7IrAi8AXgkDxsArB+q9elydvt28DFwHW5/zJgr9x9BvDV3H0P6cfED4BdAJFuDhja6nVo4rY6Hzgwdy8PDPb+VXdbjQCeBgYU9qvx3r/+bTt9BNgceLgwrN4+tTPwp7xttgLuzcNPArYl3c7/+zzsEGB8Z2LpdWcKETEzIu7P3fOAqaQdcxzpy0z++5ncvQjoT/riLpD0YWBWRDzRzLhbSdJI4JPA2blfwHbAFXmS4vYS0I+8vUgHuz9FxD+bGHLLSFqF9AU+ByAi3oqIuXj/ak9fYICkvqTtMBPvX0uIiNuB6nWst0+NAy6I5B5gsKThpO21Iov3tcGkxHpBZ2JZam5JbQRJo4HNgHuB1SNiZh41C1g9d/8Y+DPwPGkHvJz0jMSy5JfA4aQzK4BVgbkRsTD3zyAlVoBfkX7NPQLcCVwDfLxpkbbe2sBs4FxJmwCTgW/i/aumiHhO0s+BZ4DXgZtI28z7V8fq7VO1mgQaQWpQ9ALSj5CvAP8N/CgiFnWm0F53ptBG0srA74HDIuLV4rhI51WRu2+OiA9ExC6kDPxHYINcZ/zbTtXFvQNJ+hTwYkRMLjN9RFwYEZtFxBeAbwGnAjvl7XWypF67T2V9Saf5v4mIzYDXSKf2Fd6/Fsv14ONIyXQNYCXgE/Wm9/5VW3GfameaZyJibERsDfyLVI00VdKF+brNBmXK6pUbWFI/UkK4KCKuzINfyKdY5L8vVs2zIqmu83TgOGA/4A5gnyaF3SrbAJ+WNI3UMu12wCmkU9K2M8l/a3JE0hrAFhFxNfAd4HPAXGD7pkTdOjOAGRFxb+6/gpQkvH/VtgPwdETMjogFwJWkfc77V8fq7VNlmgT6Iem6zKGkauHDgWPKFNrrkkKuDz8HmBoRJxVGXUv6IpL/XlM16/eAU/OOO4CUlReR6ud6rYg4KiJGRsRoUrXGXyJiH+BWYPc8Wa3tdTxwdO5elrbXLOBZSRvmQdsDj+L9q55ngK0krZi/m23by/tXx+rtU9cCX8x3IW0FvFKoZkLSfwLP5+tWK5K2W/lt1+qr7j39IV19D+BBYEr+7EyqJ78FeIJUxzu0MM8awPWF/j1YXKc5rNXr1MRtN5bFdx+tA9wHPEmqB+9fmG4z4JxC/2F5e91QnK63foBNgUl5H7saGOL9q93tdRzwGPAwcCGpztv715LbaALpAvwC0tnoAfX2KdLF+NOB/wMeAsYUliPg5sK07wbuz/vqNmVicTMXZmZW0euqj8zMrOucFMzMrMJJwczMKpwUzMyswknBzMwqnBSWAZLeljQltz55eSueopU0VtKHml1uofwP55Y6p0ga0ILyx0g6tcR0dzUjnp4maf5SEMN5knbveEprj5PCsuH1iNg0It4LvAUcXGamwhOnPWEs0LKkQHpy+Md5O7xea4L8MFBDvhMRMSkiDi0xXSu3kZmTwjLor8B6klbKbbjfp/QehXEAksZLulbSX4BbJK0s6VxJD+V22nfL0+0o6W5J9+ezj5Xz8GmSjsvDH5K0UW6Y8GDgW/mX+ocl7aLUnv4Dub331fP8w5Tajn9E0tmSpktaLY/7Qo53iqQzJfWpXjlJ2+dlPpTXr7+kA4E9geMlXVQ1/WhJj0u6gPRw1ZqSvifpb3l9jytM91j+NfoPSRdJ2kHSnUrt3W+Rp9sib5cHJN3V9uRzPlNqe1fFsTm2iZKeknRoIZ75heknavF7Gy7KTwQjaec8bLJSu/rX1dgO7ylsqwclrZ+HX53ne0TSQcVyJf0sD/9zXo+2+D5d2DeuycOfkFSz2YRa26/GNPML3btLOi9376F0Rvt3SbfnYX1ybG3L/EoeLkm/yv+/PwPvqlWWdVKrn+Tzp/EfYH7+25f0qPxXgR8BX8jDB5PeO7ESqX2eGSx+IvInwC8LyxoCrAbcDqyUhx0BHJ27p7H4vQFfA87O3ccC361aTtvDkwcCv8jdvwKOyt2fID2dvhrpycw/AP3yuF8DX6xazxVIrUdukPsvIDWICHAesHuNbTOa1ATAVrl/R9IL0UX60XQdqans0cBC4H15+GTgd3m6ccDVef5BQN/cvQOL27Ufy+KnxY8F7iI92bsa8HJhveYXpn+F1K7NcsDdpKf129Zx7TzdhLblVq3XacA+uXt5Fr/PoO3/OoCUBFfN/QHslLuvIrVm2g/YBJiSh48nPXW7amH+MVVx19x+9fbJ3L07cF7ufggY0bZf5r8HAT/I3f1JT5OvDXyW9PRuH9JT43Nr/Y/96dynVzedbRUDJE3J3X8ltQ11F6khvO/m4SsAo3L3zbG4/fodKDT1HBFzlFpW3Ri4M/94XZ500GrT1gjhZNIXt5aRwKVKDX0tT3oRC6QD3665rBskzcnDtwc+APwtlzmAqkbngA1Jja/9I/efD3yd1DR4e6ZHapce0kFtR+CB3L8ysD6pDZ+nI+IhAEmPALdEREh6iJQ0AFYBzs+/zIN0YK3l+oh4E3hT0oukZpFnVE1zX0TMyOVNyWXMB56KiLbtNYF00Kx2N/BfSu/KuDIWv7/hUEm75u4187q9TKpWvCEPfwh4MyIWVK0bpH3j5RzTlaT/16TC+Hrb7/Y626HancB5ki5j8X60I/B+Lb5esEpe5keACRHxNvC80tmtdZOTwrLh9YjYtDggV0XsFhGPVw3fktQcdHtEOjjsXWf8m/nv29Tfx04DToqIayWNJf167qjM8yPiqA6m64ri+op07eHMJQpPVWBvFgYtKvQvYvF6Hg/cGhG75nkm1imzuKx626nMNDVFxMWS7iW9POmPucplESnJbx0R/5I0kfRjAGBB5J/iFNYtIhZpyWtL1e3iVPfX3H61Qix0r1AZGHFw3gc/CUyW9IG8zEMi4sYlCpJ27qAM6wJfU1h23QgcUqin3qzOdDeTfm2TpxtCegnKNpLWy8NWUsdttc9j8Ut8IP3aa2vud7/C8DtJ9f9I2pFUzQSpYbDdJb0rjxsqaa2qMh4HRrfFBewL3NZBXNVuBL6kxddIRrSVWVJxvcZ3suwyHgfWyQkHUpPS/0bSOqQzilNJVYbvz7HNyQlhI9KrHDvrY3nbDyC9CezOqvFlt98Lkt6tdGG/7cwFSetGxL0RcTTpZUZr5mV+ValJfCRtIGkl0tnH5/I1h+HAR7uwPlbFSWHZdTypauPBXBVyfJ3pTgCGtF38Az4aEbNJB7wJkh4kVVVs1EF5fwB2zRc+P0w6M7hc0mTgpcJ0xwE7Kr3AfA/SG6fmRcSjpPbhb8pl3kx6H3dFRLwB7J+X+xDpF+8ZHW+KJZZxE+ld1XfnZVzBksmsIz8FfizpARpwJh7pzqmvATfkbTePdO2h2p7Aw7na6b2k6ys3AH0lTQVOJCX3zrqP9K6SB0nXS4pVR53ZfkeSrjfcRbpO0eZnSjcJPJzH/Z30PoBHgfvz8DNJ2/YqUguij+b1K1ZhWhe5lVRbqkjqD7wdEQslbU16w9mmLQ5rqSJp5YiYn8/yTgeeiIiTm1DueNKF5W80uixrHV9TsKXNKOCyXK3wFvDlFsezNPqypP1IF+gfIP1yNusRPlMwM7MKX1MwM7MKJwUzM6twUjAzswonBTMzq3BSMDOziv8HNZiXbg3m50IAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA3U0lEQVR4nO3deXwV9bnH8c83GyBbEBBIAgQVEUQgiKC44b6zVK1a6BWtWu1Va217q121ra2t1Vav9lpXtHXXsrhrFdzYBInsCEogCcgm+5btuX/MJBxilhPIySE5z/v1Oq/MPs/Mmcxz5veb+Y3MDOecc4krKd4BOOeciy9PBM45l+A8ETjnXILzROCccwnOE4FzziU4TwTOOZfgPBE4JHWS9IGkrZLuiXc8jZWksZI+2sd5p0i6Ospp8ySdsY/r2ed568v+7CcXGynxDsDVD0l5QCegFNgGvAncYGbbopj9WmA90Mb8wRLnEo5fETQtF5pZK2AAkAPcFuV83YGF+5IEJPmPCecaOU8ETZCZfQW8RZAQAJB0nKSpkjZJ+kzSsHD4OOAK4H8kbZN0hqQkSbdK+kLSBkkvSDo4nD5bkkn6nqSVwHvh8KskLZK0UdJbkrpHrNskXSdpabj+ByUpYvw14bxbJS2UNDAcniHpZUnrJC2XdFN12yzpvHDerZIKJf0kHN5O0qvhMjaG3VkR802R9Ptw32yT9Iqk9pKelrRF0ieSsitty02SvpS0XtLdkqr8P5J0pKR3JH0taYmkb0fz/Uk6TNJ74b5fH8aSXmmyY8Pt3SjpCUnNI+a/QFJuuK+nSuoX5XrHSXoojHmrpPcjv8dK0/6fpL9UGjZR0i1hd/nxU/6djqpmOeXHU0rEsL2Kyao7thT4q6S14Xc1T1LfaLbVVWJm/mkCHyAPOCPszgLmAfeF/ZnABuA8guR/ZtjfMRw/Dvh9xLJ+CEwPl9MM+AfwbDguGzDgKaAl0AIYASwDehMUN/4SmBqxPANeBdKBbsA64Jxw3CVAIXAsIOBwgiuUJGA28GsgDTgU+BI4u5rtXw2cFHa3AwaG3e2Bi4CDgNbAi8CEiPmmhLEfBrQFFgKfA2eE2/IU8ESlbZkMHBxuy+fA1eG4scBHYXdLIB+4MlxODkHxW59q4p8SsZzDw++oGdAR+AD4W6Xvej7QNYzj4/LvL1zPWmAIkEyQ5POAZpWPkypiGAdsBU4O131f+fZUMe3J4fYpYp/vBDIivteM8Hu8FNgOdKliP2WH+zSlmn1R7bEFnB0eI+kEx07v8nX4p47nj3gH4J96+iKDf/Bt4T+yAe8C6eG4nwH/rDT9W8AVYfc49k4Ei4DTI/q7AMXhP2L5P+6hEePfAL4X0Z8E7AC6h/0GnBgx/gXg1og4fljF9gwBVlYadhsRJ+VK41YC3yeo56hpPw0ANkb0TwF+EdF/D/BGRP+FQG5EvxEmsbD/B8C7YXfkCe5S4MNK6/4H8Jtq4qo4+VUxbiQwp9J3fV1E/3nAF2H3/wG/qzT/EuCUiHlrSgTPRfS3Iqhz6lrFtAr3+clh/zXAezXs91xgRBX7qfx4qi4RVHtsAacRJOLjgKRY/4815Y8XDTUtI82sNTAMOBLoEA7vDlwSFhVskrQJOJHgBF+V7sD4iGkXEZwQOkVMk19p+vsipv+a4ESRGTHNVxHdOwhOMhD8qv2imhgyKsX880oxRLqI4IS4IizSOB5A0kGS/iFphaQtBL+u0yUlR8y7JqJ7ZxX9rdhb5LavIPjlW1X8QyrFPxroXE38FRTcxfVcWMS1BfgXe77L2mLoDvy40nq7VhNjVSqWa8GNBl8TfA8/D4vOtkl6yIKz8nPA5eHk3wGejtiG/4oontoE9K1iG6JR7bFlZu8BDwAPAmslPSypzT6sI+F5ImiCzOx9gl935WW4+QRXBOkRn5Zmdlc1i8gHzq00fXMzK4xcTaXpv19p+hZmNjWKcPMJimWqGr680jJbm9l51WzzJ2Y2AjgEmEBw1QHwY6AXMMTM2hAUaUBwMtlXXSO6uwGrqon//UrxtzKz66NY/h8I9u/RYcxjqoi3uhjygTsrrfcgM3s2mg2LXK6kVgRFT6vM7A9h/K3M7LpwkmeBi8My+yHAy+F83YFHgBuA9maWTlCUVdU+3x7+PShiWGSyrPHYMrP7zewYoA9wBPDTKLfTRfBE0HT9DThTUn+CX5QXSjpbUrKk5pKGRVaaVvIQcGdEpVxHSSNqWNdDwG2SjgqnbyvpkijjfBT4iaRjwsq/w8P1zgS2SvqZpBZh3H0lHVt5AZLSJI2W1NbMioEtQFk4ujXBr/pNCiq8fxNlXDX5qYJK6K4E9SnPVzHNq8ARkr4rKTX8HCupdxTLb01QzLdZUiZVn9z+W1JWuE2/iIjhEeA6SUPC/dlS0vmSWke5bedJOlFSGvA7YLqZ5Vc1oZnNIaj3eBR4y8w2haNaEiSydQCSriS4IqhqGesI6ojGhN/xVez9w6DaYyvcn0MkpRIklF3s+d5dHXgiaKLCf7CngF+H/8gjCIpW1hH8yvop1X//9wGTgLclbSWoOB5Sw7rGA38CnguLMuYD50YZ54vAncAzBPUbE4CDzawUuICgTH85e044batZ1HeBvHD91xEUw0CQEFuE808neL5if00kqKTMBV4DHqs8gZltBc4CLiP4tf4VwT5qFsXy7wAGApvD5f+7immeAd4mqED/Avh9uN5ZBOX1DwAbCSpax0a5XeXL/Q1BEcwxBFcjtU1/RviXMIaFBHUt0wiK2Y4mqNCuzjUEx+MG4Cig4kqylmOrDUHi20hQPLYBuDuKbXSVlNf4O+eiIMmAnma2LN6x1DcFtxIXmNkv4x2La1h+ReCccwnOE4FzziU4LxpyzrkE51cEzjmX4Bpdg2EdOnSw7OzseIfhnHONyuzZs9ebWceqxjW6RJCdnc2sWbPiHYZzzjUqklZUN86LhpxzLsF5InDOuQTnicA55xKcJwLnnEtwngiccy7BxSwRSHo8fIXc/GrGS9L9kpZJmqvw9YTOOecaViyvCMYB59Qw/lygZ/i5luDNSu4AMXvFRh6cvIzZKzbGOxTnXIzF7DkCM/tAES/9rsII4KnwTUfTJaVL6mJmq2MVk6udmfH2wjXc8MynlJQaaSlJPHPNcRzTvV28Q3POxUg8HyjLZO/X7RWEw76RCCRdS3DVQLdu3RokuESxo6iEeQWbmZO/iTkrNzJn5SbWbt1dMX53SRk///c8fj+qL4O6t0Panxd7OecORI3iyWIzexh4GGDQoEHeSt4+MjO+XL+d3JWbmJMfnPQXf7WV0rJgl2a3P4gTDu9A+5ZpPDV9BSWlZUgif+MOLnloGkdltGHs0Gwu7J9B89TkWtbmnGss4pkICtn7vatZ4TBXTzbvKCa3YM8v/dz8TWzeWQxAq2YpDOiazg+GHUZOt3T6Z6XTvtWel2ede3QXpn+5geMObU/vLq2ZMGcV46Yu56cvzeWPbyzmO4O7Mea47nRu2zxem+dcQpm9YmPF/2R9F9XGtBnqsI7gVTP7xvtKJZ1P8HLr8wheg3i/mQ2ubZmDBg0yb2vom0pKy1iyZmvFCX/Oyo18sS54L7gEvTq1JqdbOjld2zGgWzqHdWxFclLdinnMjGlfbOCJqXn8Z9EakiTO7duZsUOzOcaLjVwTVlZmFJeVUVJqFJeWUVzxd093SalRFA4rqWJ8eXdJWRlFJWF3+fAyo7hk7+6SsnB5JWVs2F7EnJUbMYNmqUk8fXXd6+0kzTazQVWNi9kVgaRngWFAB0kFBO9BTQUws4eA1wmSwDJgB3BlrGJpitZu2RWW6wcn/bkFm9lZXApA+5Zp5HRrx7cGZpHTNZ1+XdNp1Wz/v2pJDD28A0MP70D+1zt4aloez32Sz6tzV9M3sw1jh/bggn5dvNjIRWXqsvV8/MV6+mel07tLG4oiTqBFtZ1Mw+6iyJNpNSfnbyyv4kQbzF/eXdX85csoKYttiXRachIpySI1OSn8BN0pySItOYmNO4ooD6G4pIzpX26o16uCRvdimkS8IthVXMqCVVuCIp78TeSu3EThpp0ApCaLPhltyemaTk63dAZ2a0dWuxYN9ut8R1EJ4+cUMu7jPJau3Ub7lml8Z0hQbNSpjRcbub1t3VXMm/O/4p/TVjC3cHO9Lz81WaQk7TmRpiYnkZoiUpOSKk6sqclJlU685SfdoLu6k3J5d0pSEqkpSaRFdKcmhdOUd6ckkRIOS4vorlhGGENqskhOUq3/r7NXbGT0o9MpLikjNaX+rwg8ERxgzIz8r3dWVObOyd/EwlWbKS4NvqfM9BZBEU+3dgzoms5RGW0OiF/gZsbULzbwxMd5vLt4DckS5x7dhbFDsxnYLd2LjRJYUUkZH3y+jvG5hfxn4Rp2l5TRtkUqW3YWY4CAc/p25py+nStOluW/hFPCk+peJ+ek8OQe0V1+8m/Kx9n+1hF4IjiAbdtdwtz8TXvdvrlhexEALVKT6ZfVlpxu7cLy/XQOaQS/slduCIqNnp+Vz9ZdJfTLasvYodmc368LzVLin7Rc7JkZn67cyPg5hbw2dzUbdxTT7qBULuyfwcicTKzMGP3YjP36hevqxhPBAaKszFi2bhtzVm4MK3Q3sWTNVsq/gsM6tow46bfjiE6tSEluvM1Bbd9dwr/nFDLu4+V8sW47HVql8Z0h3RkzpFujSGiu7pat3cbE3EIm5q5i5dc7aJ6axJl9OjMqJ4OTenYkNeJ4juVdMO6bPBHEydfbi8gtL+JZuYnP8jexdXcJAG1bpJLTLZ0BXcNinqx02h6UGueIY8PM+GjZesZ9nMd7S9aSLHF+v6DYKKebnwAau7Vbd/HKZ6uZmFvI3ILNJAlOOLwDIwdkcnbfzvVyo4Lbf54IGkBRSRmLv9pScRdPbv4m8jbsACA5SRzZec/tmznd0unRoWWTLs+sTt767Tw1bQUvzspn6+4S+ndN58qh2Zx3dBfSUhrv1U+i2b67hLcWfMWE3FV8tHQdZQZ9M9swckAmw/tn+BXfAcgTQQys3ryz4qQ/Z+Um5hVuZndJGQCHtG7GwLCIZ0DXdI7OastBaf6rKNK23SX8+9MCxk3N48t12+nYuhmjh3TjO0O6cUhrP4kciIpLy/ho6Xom5Bby9oI17CwuJatdC0YOyGRkTgaHH9I63iG6Gngi2E87i0qZV7i54qQ/J38ja7YE7fGkpSRxdGb57ZvByb9L2+YJ+Wt/X5SVGR8uW8+4j5czeck6UpPFBf0yGDs0m/5d0+MdXsIzM3LzNzExdxWvfLaKDduLaNsilQv6dWFUTqY/SNiIxOWBssbKzFi+fvueJ3TzN7Jo9Z72eLq3P4jjD21fcdI/snMbL9LYD0lJ4pQjOnLKER1Zvn47T07N46XZBYyfU0hOt3TGDs3m3L5ebNTQ8tZvZ0JuIRPmFJK3YQdpKUmc2bsTI3MyOeWIjv59NDEJf0WweWcxn5U/oZsflO1v2rF3ezwDwoe1BnTduz0eFxtbdxXz8uwCnpy2guXrt3NI62aMHtKd7wzpRsfWvv9jZcO23bw6dzXj5xSSm78JCY4/tD0jczI5p29n2jRvmjczJAovGiK4VW3qF+vJaNuCXSWlFb/4l63dBgTt8RxxSFihGz6wtS/t8bj6U1ZmvL90HeM+zuP9z9eRlpzEBf26MPaEbPplpcc7vCZhR1EJ7yxcw4Q5hXywdD2lZUbvLm0YlZPBhf0z6NK2RbxDdPUk4RPB7BUbufQf0/ZqLyRoj2fPE7r9strS2n/xHLC+WLeNf4Z3G20vKmVgt3TGntCDc8OnUV30SkrL+PiLDUycU8ibC75iR1EpGW2bMyInk5EDMunV2St9m6KEryOY/uUGysKElyS4+sRDue28I72SqxE5rGMrbh9+FD8+6wheml3Ak1PzuOnZOXRq04wxQ7pz+ZBudPBiu2qZGfMKNzN+TiGvfLaa9dt206Z5CiMGZDBiQCaDsw8mya9+E1ZCJILjDm1PWkpSxePsZ/ft7EmgkWrdPJUrT+jBFcdn8/7n63hiah73vPM5//veMi7sn8GVJ2TTN7NtvMM8YKzcsCOo9M0t5Mt120lLTuK0Iw9hZE4mpx7Z0Zv8cECCFA2BP87elC1bu42npgV3G+0oKmVQ93aMPSGbs49KzGKjr7cX8drcVUzIXcXsFRsBGNLjYEblZHJu3y5N9gl2V7OEryNwiWHLrmJenBUUG638eged2zTnu8d357Jjuzb5u712FZfyzsI1TMwtZMqSdZSUGUd0asWonCyGD8ggM90rfROdJwKXUErLjClL1jJuah4fLl1PWkoSI/pncMXQplVsVFoWvDFuQm4hb87/im27S+jcpnlFuX/vLq29CNRVSPjKYpdYkpPE6b07cXrvTixds5Unp+Xx8uxCXpxdwODsgxl7QjZn9enUKFt2NTMWrNpS0cLn2q27ad0shfOO7szIAZkMObS93/Ls6syvCFxC2LyzmBdn5fPktDzyv95Jl7blxUbdOLhlWrzDq1XBxh1MzF3FhDmFLF27jdRkMazXIYwckMnpvQ85IF5O5A5sXjTkXKi0zHhv8VrGTV3Ox8s20CwliZEDMrliaDZ9MtrEO7y9bNpRxOvzvmLCnEJm5n0NwLHZ7RgxIJPzj+5Cu0aQwNyBwxOBc1X4fM1Wxk3N49+fFrCruIzBPQ7myqHZnBnHYqNdxaVMXryW8XMKmbxkLcWlxmEdW/KtgVkM759B14MPiktcrvHzROBcDTbvKOaFsNioYONOMto257vHZ3PZsV0b5Fd3WZkxY/nXTJhTyOvzV7N1VwkdWzdjRPhax6My2nilr9tvngici0JpmfHuojWMm5rH1C+CYqNROUGxUe8u9V9stGj1FibkFjIpdxWrN++iZVoy5/TtwsicDIYe1sErfV298kTgXB0t/moLT05dwfg5QbHRcYcezNihPTizT6f9OkGv2rSTSZ8Flb6Lv9pKStgM98icTM7o3YkWaV7p62LDE4Fz+2jTjiKe/ySfp6atoHDTTjLTW/Bfx3fn0mO7kn5QdMVGm3cW8+b8oHnnGcu/xgwGdktnVE4m5/fLaBR3LbnGzxOBc/uppLSM/ywK7jaa/uXXNE9NYlROFmOHZlfZWufuklImL17HxNxC3l28lqKSMg7t0JKROZmMGJBB9/Yt47AVLpF5InCuHi1avYUnp+Yxfk4hu0vKGHpYe07q2YGSMiO9RSoLV2/ltbmr2LKrhA6t0riwfwYjB2TSL6utV/q6uPFE4FwMbNxexHOf5PPYh1+yfntRxfC05CTO79eFEQMyOPHwDo3yCWbX9HgTE87FQLuWaVw/7DBKysq49+3PMYL3Xfxg2GHcfOYR8Q7Puaj5TxXn9tPQwzrQLDWJZEFaShInHdEx3iE5Vyd+ReDcfjqmezuevvo4f9+Fa7Q8EThXD47p3s4TgGu0vGjIOecSnCcC55xLcFEnAkl1bvZQ0jmSlkhaJunWKsZ3kzRZ0hxJcyWdV9d1OOec2z+1JgJJQyUtBBaH/f0l/T2K+ZKBB4FzgT7A5ZL6VJrsl8ALZpYDXAbUulznnHP1K5orgr8CZwMbAMzsM+DkKOYbDCwzsy/NrAh4DhhRaRoDypt1bAusiiZo55xz9SeqoiEzy680qDSK2TKByPkKwmGRbgfGSCoAXgdurGpBkq6VNEvSrHXr1kUTsnPOuShFkwjyJQ0FTFKqpJ8Ai+pp/ZcD48wsCzgP+Kekb8RkZg+b2SAzG9Sxoz+s45xz9SmaRHAd8N8Ev+YLgQFhf20Kga4R/VnhsEjfA14AMLNpQHOgQxTLds45V09qfaDMzNYDo/dh2Z8APSX1IEgAlwHfqTTNSuB0YJyk3gSJwMt+nHOuAdWaCCQ9QVCpuxczu6qm+cysRNINwFtAMvC4mS2Q9FtglplNAn4MPCLpR+E6xlpjaw7VOecauWiamHg1ors5MIoo7+4xs9cJKoEjh/06onshcEI0y3LOORcb0RQNvRzZL+lZ4KOYReScc65B7UsTEz2BQ+o7EOecc/ERTR3BVoLye4V/vwJ+FuO4nHPONZBoioa++WZu55xzTUa1iUDSwJpmNLNP6z8c55xzDa2mK4J7ahhnwGn1HItzzrk4qDYRmNmpDRmIc865+IjqVZWS+hI0Jd28fJiZPRWroJxzzjWcaO4a+g0wjCARvE7wfoGPAE8EzjnXBETzHMHFBO0BfWVmVwL9Cd4d4JxzrgmIJhHsNLMyoERSG2Ate7cq6pxzrhGLpo5glqR04BFgNrANmBbLoJxzzjWcaB4o+0HY+ZCkN4E2ZjY3tmE555xrKNG8vH6SpO9IamlmeZ4EnHOuaYmmjuAe4ERgoaSXJF0sqXltMznnnGscoikaeh94X1IywdPE1wCPA21iHJtzzrkGEO0DZS2AC4FLgYHAk7EMyjnnXMOJ5oGyF4DBwJvAA8D74e2kzjnnmoBorggeAy43s9JYB+Occ67h1VpZbGZvlScBSd70tHPONTF1fVWlYhKFc865uKlrIngtJlE455yLm6gSgaTOkoYDMyR1jnFMzjnnGlA0TxZfDcwEvkXQEul0SVfFOjDnnHMNI5q7hn4K5JjZBgBJ7YGpBA+VOeeca+SiKRraAGyN6N8aDnPOOdcERHNFsIygbmAiwUvrRwBzJd0CYGb3xjA+55xzMRZNIvgi/JSbGP5tXf/hOOeca2jRNDp3R0ME4pxzLj7q+hyBc865JsYTgXPOJbhoniM4IZphzjnnGqdorgj+N8ph3yDpHElLJC2TdGs103xb0kJJCyQ9E81ynXPO1Z9qK4slHQ8MBTqW3yoaagMk17bg8I1mDwJnAgXAJ5ImmdnCiGl6ArcBJ5jZRkmH7NtmOOec21c1XRGkAa0IkkXriM8WgqYmajMYWGZmX5pZEfAcwTMIka4BHjSzjQBmtrZu4TvnnNtf1V4RRLyreJyZrQCQlAS0MrMtUSw7E8iP6C8AhlSa5ohwuR8TXGXcbmZvVl6QpGuBawG6desWxaqdc5UVFxdTUFDArl274h2Ki6HmzZuTlZVFampq1PNE80DZHyVdB5QCnwBtJN1nZnfvY5yV198TGAZkAR9IOtrMNkVOZGYPAw8DDBo0yOphvc4lnIKCAlq3bk12djaSv1qkKTIzNmzYQEFBAT169Ih6vmgqi/uEVwAjgTeAHsB3o5ivEOga0Z8VDotUAEwys2IzWw58TpAYnHP1bNeuXbRv396TQBMmifbt29f5qi+aRJAqKZUgEUwys2KCNodq8wnQU1IPSWnAZcCkStNMILgaQFIHgqKiL6OK3DlXZ54Emr59+Y6jSQT/APKAlgRFN90JKoxrZGYlwA3AW8Ai4AUzWyDpt+FLbgjHbZC0EJgM/LS8uWvnXNPTqlWrqKe9/fbb+ctf/hKz5e+vcePGccMNNzTY+mIpmraG7gfujxi0QtKp0SzczF4HXq807NcR3QbcEn6cc87FQTRPFneS9JikN8L+PsAVMY/MORd3s1ds5MHJy5i9YmPM1vHKK68wZMgQcnJyOOOMM1izZk3FuM8++4zjjz+enj178sgjj1QMv/vuuzn22GPp168fv/nNb2pcfl5eHkceeSSjR4+md+/eXHzxxezYsWOvacrKysjOzmbTpk0Vw3r27MmaNWtqjK/c2LFjeemllyr6I69M6hJrvERz19A44AngF2H/58DzwGMxisk5F2N3vLKAhatqLuHduquYxV9tpcwgSXBk59a0bl79LYl9MtrwmwuPqnMsJ554ItOnT0cSjz76KH/+85+55557AJg7dy7Tp09n+/bt5OTkcP755zN//nyWLl3KzJkzMTOGDx/OBx98wMknn1ztOpYsWcJjjz3GCSecwFVXXcXf//53fvKTn1SMT0pKYsSIEYwfP54rr7ySGTNm0L17dzp16lRjfLV5++236xxrPERTR9DBzF4AyqCi7L80plE55+Juy64SysLbQsos6I+FgoICzj77bI4++mjuvvtuFixYUDFuxIgRtGjRgg4dOnDqqacyc+ZM3n77bd5++21ycnIYOHAgixcvZunSpTWuo2vXrpxwQtBE2pgxY/joo4++Mc2ll17K888/D8Bzzz3HpZdeWmt8tdmXWOMhmiuC7eF7ig1A0nHA5phG5ZyLqWh+uc9esZHRj06nuKSM1JQk7rssh2O6t6v3WG688UZuueUWhg8fzpQpU7j99tsrxlW+A0YSZsZtt93G97///ajXUdVyZsyYUbGM3/72t1x44YUsW7aMdevWMWHCBH75y1/WGl+5lJQUysrKgKCYqaioCGCfYo2HaK4IbiG47fOw8Angp4CbYhqVcy7ujunejqevPo5bzurF01cfF5MkALB582YyMzMBePLJJ/caN3HiRHbt2sWGDRuYMmUKxx57LGeffTaPP/4427ZtA6CwsJC1a2tunWblypVMmzYNgGeeeYYTTzyRIUOGkJubS25uLsOHD0cSo0aN4pZbbqF37960b9++1vjKZWdnM3v2bAAmTZpEcXExwD7FGg/RXBEsAE4BegECluDvMXAuIRzTvV29JoAdO3aQlZVV0X/LLbdw++23c8kll9CuXTtOO+00li9fXjG+X79+nHrqqaxfv55f/epXZGRkkJGRwaJFizj++OOBoGL2X//6F4ccUn2blb169eLBBx/kqquuok+fPlx//fVVTnfppZdy7LHHMm7cuIphNcVX7pprrmHEiBH079+fc845h5YtWwJw1lln1TnWeFBwB2cNE0ifmtnA2oY1lEGDBtmsWbPisWrnGrVFixbRu3fveIfR4PLy8rjggguYP39+vENpMFV915Jmm9mgqqavqRnqzgQNx7WQlENwNQBBM9QH1U+4zjnn4q2moqGzgbEEbQTdw55EsAX4eWzDcs65+pGdnZ1QVwP7oqZmqJ8EnpR0kZm93IAxOeeca0C1Vvp6EnDOuabN7/5xzrkE54nAOecSXLWJQNK3avo0ZJDOOVed+mwO+g9/+ENFd15eHn379q2X5Uaqz+a1Kzd2t69quiK4MPx8j6CBudHh51Hgqv1es3POHWAiE0G0Skpi0wZTQ6o2EZjZlWZ2JZBK8LrKi8zsIuCocJhzrqnLnwkf3hP83U8HenPQt956Kzt37mTAgAGMHj0agNLSUq655hqOOuoozjrrLHbu3AnAsGHDuPnmmxk0aBD33Xcfs2fP5pRTTuGYY47h7LPPZvXq1QDcf//99OnTh379+nHZZZdVrGvhwoUMGzaMQw89lPvv3/O6l3vvvZe+ffvSt29f/va3v30jRjPjhhtuoFevXpxxxhn11lxFNE1MdDWz1RH9a4Bu9bJ251x8vHErfDWv5ml2b4E188HKQEnQqS80a1P99J2PhnPvqnGRB3Jz0HfddRcPPPAAubm5QJC4li5dyrPPPssjjzzCt7/9bV5++WXGjBkDQFFREbNmzaK4uJhTTjmFiRMn0rFjR55//nl+8Ytf8Pjjj3PXXXexfPlymjVrtldyW7x4MZMnT2br1q306tWL66+/nrlz5/LEE08wY8YMzIwhQ4ZwyimnkJOTUzHf+PHjWbJkCQsXLmTNmjX06dOHq67a/wKaaCqL35X0lqSxksYCrwH/2e81O+cObLs2B0kAgr+79r/R4cbWHHSPHj0YMGAAAMcccwx5eXl7xQlBcps/fz5nnnkmAwYM4Pe//z0FBQVA0FbS6NGj+de//kVKyp7f3eeffz7NmjWjQ4cOHHLIIaxZs4aPPvqIUaNG0bJlS1q1asW3vvUtPvzww73i+eCDD7j88stJTk4mIyOD0047Lep9UJNoXlV5g6RRQHnqfNjMxtfL2p1z8VHLL3cgKA56cjiUFkFyGlz0KHQdvF+rbWzNQTdr1qyiOzk5uaJoCKhoWM7MOOqooypaN4302muv8cEHH/DKK69w5513Mm/evCqXG+96hmhvH/0UeM3MfgS8Jal1DGNyzh0Iug6GKybBab8I/u5nEoADvzno1NTUinmi1atXL9atW1exXcXFxSxYsICysjLy8/M59dRT+dOf/sTmzZsr1l+Vk046iQkTJrBjxw62b9/O+PHjOemkk/aa5uSTT+b555+ntLSU1atXM3ny5DrFWp1arwgkXQNcCxwMHEbQEN1DwOn1EoFz7sDVdXC9JIByB3pz0Ndeey39+vVj4MCB3HnnnVFtU1paGi+99BI33XQTmzdvpqSkhJtvvpkjjjiCMWPGsHnzZsyMm266ifT09GqXM3DgQMaOHcvgwcH+vvrqq/eqHwAYNWoU7733Hn369KFbt24V27O/ommGOhcYDMwws5xw2DwzO7peIqgjb4bauX0T72aoE7E56HipazPU0RQN7TazooiFpRC+ttI551zjF00ieF/SzwneS3Am8CLwSmzDcs41Nd4c9IErmkTwM2AdMA/4PvA68MtYBuWcc67h1FhZLCkZWGBmRwKPNExIzrlYMbNv3MLpmpba6n2rUuMVgZmVAksk+ZPEzjVyzZs3Z8OGDft0onCNg5mxYcMGmjdvXqf5omlioh2wQNJMYHvECofXLUTnXDxlZWVRUFDAunXr4h2Ki6HmzZuTlZVVp3miSQS/2rdwnHMHktTUVHr06BHvMNwBKJomJt5viECcc87FR613DUk6TtInkrZJKpJUKmlLQwTnnHMu9qK5ffQB4HJgKdACuBp4MJZBOeecazhRNTpnZsuAZDMrNbMngHNiG5ZzzrmGEk0i2CEpDciV9GdJP4pyPiSdI2mJpGWSbq1huoskmaQq28FwzjkXO9Gc0L8LJAM3ENw+2hW4qLaZwofRHgTOBfoAl0vqU8V0rYEfAjOiD9s551x9ieauoRVh507gjjosezCwzMy+BJD0HDACWFhput8BfwJ+WodlO+ecqyfR3DW0XNKXlT9RLDsTyI/oLwiHRS57IME7kV+rJYZrJc2SNMsfhnHOufoVzQNlkeX2zYFLCF5Ss18kJQH3AmNrm9bMHgYehuB9BPu7buecc3vUekVgZhsiPoVm9jfg/CiWXUhQn1AuKxxWrjXQF5giKQ84DpjkFcbOOdewonlV5cCI3iSCK4RoriQ+AXpK6kGQAC4DvlM+0sw2Ax0i1jMF+ImZ+evHnHOuAUVzQr8norsEyAO+XdtMZlYi6QbgLYK7jh43swWSfgvMMrNJ+xCvc865ehbNXUOn7uvCzex1ghfZRA77dTXTDtvX9TjnnNt30RQN3VLTeDO7t/7Ccc4519CivWvoWKC8KOdCYCZB20POOecauWgSQRYw0My2Aki6HXjNzMbEMjDnnHMNI5omJjoBRRH9ReEw55xzTUA0VwRPATMljQdE0EzEuFgG5ZxzruFEc9fQnZLeAE4CDLjSzObEPDLnnHMNotqiIUkHSUoFMLNPgTcJngfwl54651wTUlMdwZtANoCkw4FpwKHAf0u6K/ahOeecawg1JYJ2ZlZ+i+gVwLNmdiPB+wWiaWvIOedcI1BTIohs5fM04B0AMysCymIZlHPOuYZTU2XxXEl/IWgw7nDgbQBJ6Q0Ql3POuQZS0xXBNcB6gnqCs8xsRzi8D/CXGMflnHOugVR7RWBmO4G9KoUlDTSzqcDUWAfmnHOuYUTzZHGkR2MShXPOubipayJQTKJwzjkXN3VNBHfEJArnnHNxU20dgaQjzWxxpVdVrizvD582ds4518jVdPvoLcC17P2qynJG8GyBc865Rq6mu4auDf/u86sqnXPOHfiiaYYaSUMJnieomN7MnopRTM455xpQNO8s/idwGJALlIaDjeA9Bc455xq5aN9Z3MfMrNYpnXPONTrR3D46H+gc60Ccc87FR023j75CUATUGlgoaSawu3y8mQ2PfXjOOediraaiIW9YzjnnEkBNiaAQ6GRmH0cOlHQisDqmUTnnnGswNdUR/A3YUsXwzeE455xzTUBNiaCTmc2rPDAclh2ziJxzzjWomhJBeg3jWtRzHM455+KkpkQwS9I1lQdKuhqYHbuQnHPONaSaKotvBsZLGs2eE/8gIA0YFeO4nHPONZCaGp1bAwyVdCrQNxz8mpm91yCROeecaxC1NjFhZpOByQ0Qi3POuTio6xvK6kTSOZKWSFom6dYqxt8iaaGkuZLeldQ9lvE455z7ppglAknJwIPAuUAf4HJJfSpNNgcYZGb9gJeAP8cqHuecc1WL5RXBYGCZmX1pZkXAc8CIyAnMbLKZ7Qh7pwNZMYzHOedcFWKZCDKB/Ij+gnBYdb4HvFHVCEnXSpolada6devqMUTnnHMxrSOIlqQxBLem3l3VeDN72MwGmdmgjh07NmxwzjnXxEX1qsp9VAh0jejPCoftRdIZwC+AU8xsd+XxzjnnYiuWVwSfAD0l9ZCUBlwGTIqcQFIO8A9guJmtjWEszjnnqhGzRGBmJcANwFvAIuAFM1sg6beSyl9qczfQCnhRUq6kSdUszjnnXIzEsmgIM3sdeL3SsF9HdJ8Ry/U755yr3QFRWeyccy5+PBE451yC80TgnHMJzhOBc84lOE8EzjmX4DwROOdcgvNE4JxzCc4TgXPOJThPBM45l+A8ETjnXILzROCccwnOE4FzziU4TwTOOZfgPBE451yC80TgnHMJzhOBc84lOE8EzjmX4DwROOdcgvNE4JxzCc4TgXPOJThPBM45l+A8ETjnXILzROCccwkucRJB/kz48J7gr6ud76+68f1Vd77P6iaG+yul3pd4IMqfCU+cC2UloCTo1BeatYl3VAeu3VtgzXywMt9f0fD9VXe+z+qmYn8ZpDSHKyZB18H1tvjEuCLI+xDKSoNuK4Ndm+Mbz4Fu1+ZgP4Hvr2j4/qo732d1U7G/DEqLgnNaPUqMK4Lsk4IsWloEyWlw0aP1mk2bnPyZ8ORw31/R8v1Vd77P6qby/so+qV4XLzOr1wXG2qBBg2zWrFl1nzF/ZpBFs0/yAy4avr/qxvdX3fk+q5v93F+SZpvZoCrHJUwicM65BFZTIkiMOgLnnHPV8kTgnHMJzhOBc84lOE8EzjmX4DwROOdcgvNE4JxzCa7R3T4qaR2wYh9n7wCsr8dw6ovHVTceV90dqLF5XHWzP3F1N7OOVY1odIlgf0iaVd19tPHkcdWNx1V3B2psHlfdxCouLxpyzrkE54nAOecSXKIlgofjHUA1PK668bjq7kCNzeOqm5jElVB1BM45574p0a4InHPOVeKJwDnnElyTSASSukqaLGmhpAWSfhgOP1jSO5KWhn/bhcMvCqf7UFL7cNhhkp6P53Y0NEnJkuZIejXs7yFphqRlkp6XlBYOv1HSfEmvRww7UdJf4xl/Q5KULuklSYslLZJ0vB9f1ZP0o3AfzJf0rKTmfnztTdLjktZKmh8xrLpjSpLuD/fdXEkDw+G9JM0Ohx0fDkuR9B9JB0UdjJk1+g/QBRgYdrcGPgf6AH8Gbg2H3wr8KeyeAhwEjAFuDIc9C/SM97Y08H67BXgGeDXsfwG4LOx+CLg+7J5O8KPhl8CFgIC3gIPjvQ0NuK+eBK4Ou9OAdD++qt1XmcByoEXEcTXWj69v7KeTgYHA/Ihh1R1T5wFvhPvmOGBGOPxe4EQgC3g5HHYjMLYusTSJKwIzW21mn4bdW4FFBAfjCIJ/YMK/I8PuMqAZwT9rsaSTgK/MbGlDxh1PkrKA84FHw34BpwEvhZNE7i8BqYT7i+AE94aZfd2AIceNpLYE/7SPAZhZkZltwo+vmqQALSSlEOyH1fjxtRcz+wCovI3VHVMjgKcsMB1Il9SFYH8dxJ5jLZ0gmT5Vl1ia3DuLJWUDOcAMoJOZrQ5HfQV0Crv/CPwHWEVw0L0IXNawkcbd34D/IbiCAmgPbDKzkrC/gCCZAjxA8KttAfAxMBE4u8Eijb8ewDrgCUn9gdnAD/Hjq0pmVijpL8BKYCfwNsE+8+OrdtUdU5lAfsR05fvvQYKTfjPg+8CvgD+YWVldVtokrgjKSWoFvAzcbGZbIsdZcM1kYfc7ZnaMmV1IkGlfB44Iy4AfqVPZWiMk6QJgrZnNjmZ6M/unmeWY2RjgR8D9wLnh/vqrpCZ1HFUhheAS/v/MLAfYTnDZXsGPrz3Ccu0RBAk0A2gJnFPd9H58VS3ymKphmpVmNszMjgd2EBQRLZL0z7Ae5oho1tVkdrCkVIIk8LSZ/TscvCa8fCL8u7bSPAcRlF0+CNwBXAF8BIxuoLDj5QRguKQ84DmCS/b7CC43y68Ss4DCyJkkZQCDzWwC8GPgUmATcHqDRB0/BUCBmc0I+18iSAx+fFXtDGC5ma0zs2Lg3wTHnB9ftavumCoEukZM9439B9xJUM9yE0GR7/8Av4lmpU0iEYTl248Bi8zs3ohRkwj++Qj/Tqw060+B+8ODtQVB9i0jKG9rsszsNjPLMrNsgiKL98xsNDAZuDicrKr99Tvg12F3Iu2vr4B8Sb3CQacDC/HjqzorgeMkHRT+b5bvLz++alfdMTUJ+K/w7qHjgM0RRUhIOgVYFdZDHUSw36Lfd/GuOa+PD0GtuQFzgdzwcx5Bufe7wFKCMtuDI+bJAF6L6L+EPWWUHeO9TQ2474ax566hQ4GZwDKCcu1mEdPlAI9F9N8c7q83I6drqh9gADArPMYmAO38+Kpxf90BLAbmA/8kKMP242vvffQsQSV6McFV5/eqO6YIKtQfBL4A5gGDIpYj4J2IaXsDn4bH6gnRxOJNTDjnXIJrEkVDzjnn9p0nAuecS3CeCJxzLsF5InDOuQTnicA55xKcJ4ImSlKppNywVccX4/E0q6RhkoY29Hoj1n9S2AJmrqQWcVj/IEn3RzHd1IaIp75J2nYAxDBO0sW1T+lq4omg6dppZgPMrC9QBFwXzUwRT37Wh2FA3BIBwRO8fwz3w86qJggf0InJ/4GZzTKzm6KYLp77yDlPBAniQ+BwSS3DNtBnKngPwQgASWMlTZL0HvCupFaSnpA0L2zn/KJwurMkTZP0aXiV0SocnifpjnD4PElHho3/XQf8KPxFfpKkCxW0Rz8nbC+9Uzh/RwVtry+Q9KikFZI6hOPGhPHmSvqHpOTKGyfp9HCZ88LtaybpauDbwO8kPV1p+mxJSyQ9RfDAU1dJP5X0Sbi9d0RMtzj81fm5pKclnSHpYwXtxQ8Opxsc7pc5kqaWP4EcXhGVv+vh9jC2KZK+lHRTRDzbIqafoj3vPXg6fDIXSeeFw2YraJf+1Sr2w1ER+2qupJ7h8AnhfAskXRu5Xkl3h8P/E25HeXzDI46NieHwpZKqbLKgqv1XxTTbIrovljQu7L5EwZXrZ5I+CIclh7GVL/P74XBJeiD8/v4DHFLVulwdxfvpOv/E7KnFbeHfFILH1K8H/gCMCYenE7y3oSVBezgF7Hky8U/A3yKW1Q7oAHwAtAyH/Qz4ddidx552938APBp23w78pNJyyh9ivBq4J+x+ALgt7D6H4CnxDgRPSL4CpIbj/g78V6XtbE7QKuMRYf9TBI0OAowDLq5i32QTPH5/XNh/FsFLwUXw4+hVgmans4ES4Ohw+Gzg8XC6EcCEcP42QErYfQZ72oUfxp6ntm8HphI8YdsB2BCxXdsipt9M0I5MEjCN4Kn58m3sEU73bPlyK23X/wKjw+409rwPoPx7bUGQ+NqH/QacG3aPJ2glNBXoD+SGw8cSPP3aPmL+QZXirnL/VXdMht0XA+PC7nlAZvlxGf69Fvhl2N2M4KnuHsC3CJ6iTSZ4entTVd+xf+r2aXLNULsKLSTlht0fErTFNJWgsbmfhMObA93C7ndsT/vvZxDRbLKZbVTQYmkf4OPwR2oawYmqXHlDf7MJ/lmrkgU8r6AxrTSCl5dAcLIbFa7rTUkbw+GnA8cAn4TrbEGlht2AXgQNnH0e9j8J/DdBM9s1WWFBu+4QnMjOAuaE/a2AngRt5iw3s3kAkhYA75qZSZpHkCgA2gJPhr/AjeBkWpXXzGw3sFvSWoImhgsqTTPTzArC9eWG69gGfGlm5fvrWYITZWXTgF8oeNfEv23P+w9ukjQq7O4abtsGgiLDN8Ph84DdZlZcadsgODY2hDH9m+D7mhUxvrr990E1+6Gyj4Fxkl5gz3F0FtBPe8r/24bLPBl41sxKgVUKrmLdfvJE0HTtNLMBkQPCYoaLzGxJpeFDCJpWrokITgiXVzN+d/i3lOqPq/8F7jWzSZKGEfxKrm2dT5rZbbVMty8it1cEdQn/2GvlQfHW7ohBZRH9ZezZzt8Bk81sVDjPlGrWGbms6vZTNNNUycyekTSD4IVDr4fFKWUEif14M9shaQrBDwCAYgt/chOxbWZWpr3riiq3Q1O5v8r9V1WIEd3NKwaaXRceg+cDsyUdEy7zRjN7a68VSefVsg63D7yOILG8BdwYUe6cU8107xD8qiacrh3Bi0NOkHR4OKylam/rfCt7XnwDwa+68qZzr4gY/jFBeT6SziIoQoKg8a2LJR0SjjtYUvdK61gCZJfHBXwXeL+WuCp7C7hKe+o8MsvXGaXI7Rpbx3VHYwlwaJhkIGie+RskHUpw5XA/QXFgvzC2jWESOJLgNYd1dWa471sQvDHr40rjo91/ayT1VlA5X36FgqTDzGyGmf2a4AVAXcNlXq+geXkkHSGpJcFVxqVhHUIX4NR92B5XiSeCxPI7gmKLuWExx++qme73QLvyCjzgVDNbR3CSe1bSXIJiiCNrWd8rwKiw8vIkgiuAFyXNBtZHTHcHcJaCl3hfQvBmpq1mtpCgffW3w3W+Q/B+6gpmtgu4MlzuPIJftg/Vviv2WsbbBO9unhYu4yX2TmC1+TPwR0lziMFVtgV3PP0AeDPcd1sJ6hIq+zYwPyxS6ktQX/ImkCJpEXAXQUKvq5kE7/qYS1D/EVksVJf9dytB/cFUgnqHcncrqOifH477jKA9/YXAp+HwfxDs2/EELXMuDLcvsnjS7SNvfdTFnaRmQKmZlUg6nuBNYAPiHNYBRVIrM9sWXs09CCw1s782wHrHElQO3xDrdbn48ToCdyDoBrwQFhkUAdfEOZ4D0TWSriCoZJ9D8AvZuXrhVwTOOZfgvI7AOecSnCcC55xLcJ4InHMuwXkicM65BOeJwDnnEtz/A2Tx0deA7XlPAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CONCEPT DRIFT] Concept drift was not detected between the reference and the test samples.\n" + ] + } + ], + "source": [ + "results = da.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check the status\n", + "After running the data quality checks, you can check the warnings for each individual test. The warnings are suited by priority and have additional details that can provide better insights for Data Scientists." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SAMPLE LABEL DRIFT] The label accused drift in the sample test with a p-test of 0.011552904433326017, which is under the threshold 0.05. The label of the test sample does not appear to be representative of the reference sample. (Priority 2: usage allowed, limited human intelligibility)\n", + "[SAMPLE COVARIATE DRIFT] 1 features accused drift in the sample test. The covariates of the test sample do not appear to be representative of the reference sample. (Priority 2: usage allowed, limited human intelligibility)\n" + ] + } + ], + "source": [ + "da.report()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Full Test Suite\n", + "In this section, you will find a detailed overview of the available tests in the Data Drift module of ydata_quality." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reference sample covariate drift\n", + "\n", + "In this test we look for evidence of the reference sample covariates being representative of the underlying population.\n", + "A holdout is taken (20% by default), and increasing size random slices of data are taken from the leftover 80% data.\n", + "The leftover slices are tested against the holdout in attempt to provide drift evidence.\n", + "\n", + "Due to the complexity of this strategy, we provide the tooling for Data Scientists to infer the healthiness of the reference sample and avoid drawing conclusions automatically based on heuristics. An healthy indicator of data quality would be a monotonic increase of the percentage of features with no drift evidence and increasing individual p-values for the least performant tests, as the leftover slices are increased." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAApWklEQVR4nO3debxVZb3H8c9XQEQFASUvgoiz2eAQOaR1Sc3SMjKHNDMxzWzQbHLodh2uVjZpapaa5nQVp5zScshEcw6UnND0KigKigYK5gDyu388z9ksdnufs86w98bD9/167ddZ8/Nb66y9fns9a61nKSIwMzMDWK7VAZiZ2dLDScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBQaQNLqkm6XNE/SL1odzzuVpPGS7mh1HF0laR9JNzWpLEk6V9IcSfc1o8x3OknnSTqhnfEflvR4oX9DSVPy9/rQdua7U9JmXYhndUlTJfXv7Lw9yUmhDknTJL0uab6kWXkHWrnk7AcBLwGDIuI7DQzTlmIRcVFE7FhmWknHSvrfbhS3LfAxYGREbNGN5bzjk3FPiYi/RsSGhUGHA7dGxMCIODUfI3YoziNpF2BeRDyQ+7eX9HQ+huxVmG6wpPslDSyU9wJwK+n40TJOCu3bJSJWBjYFNgOOKjnfWsCj0YUnAyX17ew8tvRpwf9xLWBaRLzW5HL/TW/Yh+usw1rAIx3MejBwYaH/l8AuwMeBX0vqk4f/GDgxIuZVzX8R8JVOB9yTIsKfGh9gGrBDof+nwPWF/q2Au4C5wN+BsXn4ecAC4C1gPrADKfkeCfwf8DJwGTA0Tz8aCOAA4Bng9jz8S8BUYA5wI7BWoewg7XxP5PJPB1QY/+U87zzgUWDzPHwN4PfAbOBp4NB21n/nPO884Dngu3n4EOC6vIw5uXtkYb6JwAl528wH/gCsStrZXwX+BoyuWpdDgadIZ1c/A5bL48YDdxSm3Qi4Gfgn8DiwZzvxDwXOBZ7PcV5dtX2ezMu5FlgjD/8N8POq5VwDfDt3t/0P27brroXpxgN3Aifn//EJNeI/BXg2b4fJwIfz8E/k/WVB3mZ/z8NXAc4BZub/wQlAnxrregDwBvB2nv+4PPxTwJS8j9wFvL8wT811Ad5dtay5hf/rgVXrW1y3AL5O2iefLlH+EXmd5uX/5fZ1/o/nkfbv6/O09wLrFsZ/iLRPvZL/fqidfWIz4P68nEuBS4AT8rixwIwc1yzSgX0sMCOP/0veJm/k7TIBWAS8nvsPB5bP/cXvw1OF7lnAu4AtgBvqxNgX+BeF73vTj32tKnhp/1BICsBI4CHglNw/gvTF35l0wP9Y7h9W2JFPKCzrm8A9eTn9gTOBCXnc6PyFugBYCRgAjCMdtN6dd5IfAHcVlhekg/FgYBTpAP2JPG6P/GX7ICBgPdIvnOVIB6Kj8867DulA/PE66z+TxQetISxOLKsCuwErAgOBy1nygDsxx74u6aD2KPAPUnLsm9fz3Kp1uZV0EB+Vpz0wjxtPPvDkbfMssH9ezmakJLJxnfivJ33xhwD9gP/Mw7fL822e/xensTgRfySXocJ6v87ipLEHKbEuB3wOeA0YXoh1IXBIjm8A/37g/ELefn2B75AOEivkcccC/1u1DleR9pWVSAeT+4Cv1Fnf6rI2A14EtgT6APuR9un+JdfljqrlT6TjpHBz/j8OaK98YMO8ndu262gKB/qqcs8jfbe2yNvtIuCSPG4oKeHvm8ftnftXrbGc5YHpwLfy/rA7KQkXk8JC4Cc5xgEUkkKdbTCNJX84vgd4rarce4BN8uf5XPbdwAbtHHseBD7dsmNfqwpe2j/5Hz6f9KsigFuAwXncEcCFVdPfCOxX2JGLSWEqhV9CwPC8Q/ZlcVJYpzD+T8ABhf7lKPx6yNNvWxh/GXBkIY5v1lifLYFnqoYdReEAXTXuGdJp7KAOttOmwJxC/0Tgvwr9vwD+VOjfBZhS6A9yQsv9XwNuyd3jWZwUPgf8tarsM4FjasQ0nPQrbkiNcecAPy30r5z/F6NJSfQZ4CN53JeBv7Sz7lOAcYVYq7dvJf46888BNsndx1JICsDqwJvAgMKwvUl12rWWtURZpLOe46umeZycHEusS1eSwnZlyif9UHmR9EOhXwf713nA2YX+nYHHcve+wH1V098NjK+xnI+QDsrFM+q7WDIpvEVO0oVhnUkK2wCzanw/JpLOcLYnnRUfD7yf9F29tfp/Qjrj/GJ726WRH19TaN9nImIgaefYCFgtD18L2EPS3LYP6ULf8DrLWQu4qjDtVNKp6OqFaZ6tmv6UwvT/JB2wRhSmmVXo/hfp4AawJqlaoFYMa1TF/P2qGIp2I30Bp0u6TdLWAJJWlHSmpOmSXgVuBwYX6koBXih0v16jv/qCfXHdp5N+wdaKf8uq+PcB/qPGtGsC/4yIOTXGrZHLACAi5pN+iY6I9I28hHTwBfg86ZcpAJK+mO8+aSv/vSzeJ6rX499I+m6+u+SVPP8qVfNXr28/YGahvDNJZwxlrAV8p2p7rUnetiXWpSuq9+Ga5UfEk8BhpET4oqRLJNX6n7ept68v8b/MprPk94TCtM/l/3Fx2qLZEfFGO3F0ZA7p7LkiIqZExNiI2JJ01vwl4EfA2cBxpDPfCyWpMNtAUpVbSzgplBARt5F+sfw8D3qWdKYwuPBZKSJOrLOIZ4GdqqZfISKeKxZTNf1XqqYfEBF3lQj3WVLVTa3hT1ctc2BE7Fxnnf8WEeNIB6GrSWcjkKo9NgS2jIhBpF9gkJJWV61Z6B5F+kVXK/7bquJfOSK+WmfaoZIG1xj3POmAlYKWViJV6bT9LyYAu0tai3R29fs83VrAb4FvkKonBgMPs+R6F/+HS5D0YVK9856kM5jBpHrwtvmr532WdKawWmF9B0XEe+qVUWP+H1ZtrxUjYkKJdam1Hq+Rqgzb1ErG1ftwzfIBIuLiiNiW9L8IUrVNZy3xv8xGsfh/WTQTGFF18B3VTvxlVE//JOnu4FpJCdL1ph9ExOvA+4BJETGNlPyHQeUC93qk65Qt4aRQ3i+Bj0naBPhfYBdJH5fUR9IKksZKGlln3jOAH+YvI5KGSRrXTllnAEdJek+efhVJe5SM82zgu5I+kO9dXy+Xex8wT9IRkgbkuN8r6YPVC5C0vNI99qtExALShdFFefRA0q/9uZKGAseUjKs935M0RNKapOsvl9aY5jpgA0n7SuqXPx+U9O7qCSNiJqkK7td5uf0ktSWvCcD+kjbN94P/CLg3fzmJdCvhS6TteGNEzM3zrUQ6CMwGkLQ/6dd1WQNJddazgb6SjgYGFca/AIyWtFxhHW4CfiFpkKTlJK0r6T9Llvdb4GBJW+b9YCVJn8y3QHa0Li8AIyUtXxg2BfhsPlNcj3Rxu0vlK93vv13e/m+Q9qdF7S+upj+S9onPS+or6XPAxqR9pdrdpO1/aN4fPku6TtEdL5CuzQEQEW8BfyZVkS1B0sdIVVNtsT0NbJe/4/1JZ6vkmKZFRPVZTNM4KZQUEbNJF0mPjohnSReDv0/6Yj0LfI/62/MU0l0uN0maR7r4tGU7ZV1F+uV0Sa6ieRjYqWSclwM/BC4mXQ+5mnSn09uku0E2Je2QbQe+Veosal9gWi7/YFJVDaTkOCDPfw9wQ5m4OnAN6SL4FNIF4nOqJ4h0696OwF6kX4izWHxRsF78C4DHSPXXh+Xl/Bn4b9IZwEzSWdVeVfNeTKrvvrhQ/qOk6yN3kw4G7yPV/ZZ1I2lb/YNUbfEGS1a3XJ7/vizp/tz9RdIF0kdJVRNXUL+KcgkRMYl0TeRXed4nSdcByqzLX0i3Xs6S9FIedjKpzv0F4HwK1WqdLZ/0PzuRtA+13ZFT9nbvYhkvk/bp75AOqocDn4qIl2pM+xbw2RzDP0nXqK7sbJlVfgz8IFePfTcPO5O071Xk5Pcz0g+eNoeQfvz9Gfha/n5C+p6d0c24uqXtLguzlpAUwPq5ntnsHU/SncA38llnZ+Z7F3AbsFk3r210i5OCtZSTgtnSxdVHZmZW4TMFMzOr8JmCmZlVvKMbrlpttdVi9OjRrQ7DzOwdZfLkyS9FxLBa497RSWH06NFMmjSp1WGYmb2jSKr7HISrj8zMrMJJwczMKpwUzMyswknBzMwqGpYUJP1O0ouSHi4MGyrpZklP5L9D8nBJOlXSk5IelLR5o+IyM7P6GnmmcB7pNYNFR5JeoLI+6aU1R+bhOwHr589BpBd0NMzk6XM4/dYnmTy9VnP7Vs3bq3O8vTrP26xzGrm9GnZLakTcLml01eBxpBfWQGppcSLpLWbjgAvyCzDukTRY0vDcfHCPmjx9DnueeTdvLwqWE2z0HwMZuEK/ni6m15j3xgIemzWPRYG3VwneXp3nbdY5bdsrAvr3W46LDtyKD6w1pMeW3+xrCqsXDvSzWPzWrxEs2YzwDGq/PQlJB0maJGnS7NmzOx3APU+9zKJFqWmPRQGvvrGw08tYlrz6xkLy5vL2KsHbq/O8zTqnbXsFsGDhIu556uUO5+mMlj28FhGRW8js7HxnAWcBjBkzptPzb7XOqvTvtxwLFi6iX9/lOGWvzXo0y/Y2k6fPYZ+z7/H2Ksnbq/O8zTqnentttc6qPbr8hjaIl6uProuI9+b+x4GxETFT0nBgYkRsKOnM3D2herr2lj9mzJjoyhPNk6fP4Z6nXmardVb1zleCt1fneHt1nrdZ53R3e0maHBFjao5rclL4GfByRJwo6UjSG8EOl/RJ0vtidya9kezUiOjwVXldTQpmZsuy9pJCw6qPJE0gXVReTdIM0rt8TwQuk3QA6ZWEe+bJ/0hKCE8C/wL2b1RcZmZWXyPvPtq7zqjta0wbwNcbFYuZmZXjJ5rNzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKzCScHMzCqcFMzMrMJJwczMKpwUzMyswknBzMwqnBTMzKyiU0lB0hBJ729UMGZm1lodJgVJEyUNkjQUuB/4raSTGh+amZk1W5kzhVUi4lXgs8AFEbElsENjwzIzs1YokxT6ShoO7Alc1+B4zMyshcokhf8BbgT+LyL+Jmkd4InGhmVmZq3Qt6MJIuJy4PJC/1PAbo0MyszMWqPMheYNJN0i6eHc/35JP2h8aGZm1mxlqo9+CxwFLACIiAeBvRoZlJmZtUaZpLBiRNxXNWxhI4IxM7PWKpMUXpK0LhAAknYHZjY0KjMza4kOLzQDXwfOAjaS9BzwNLBPQ6MyM7OWaDcpSOoDfC0idpC0ErBcRMxrTmhmZtZs7SaFiHhb0ra5+7XmhGRmZq1SpvroAUnXkp5VqCSGiLiyq4VK+hZwIOk6xUPA/sBw4BJgVWAysG9EvNXVMszMrPPKXGheAXgZ2A7YJX8+1dUCJY0ADgXGRMR7gT6kW1x/ApwcEesBc4ADulqGmZl1TZknmvdvULkDJC0AViTdzbQd8Pk8/nzgWOA3DSjbzMzq6DApSDqXfDtqUUR8qSsFRsRzkn4OPAO8DtxEqi6aGxFtzz/MAEZ0ZflmZtZ1Za4pFFtGXQHYFXi+qwVKGgKMA9YG5pKuVXyiE/MfBBwEMGrUqK6GYWZmNZSpPvp9sV/SBOCObpS5A/B0RMzOy7sS2AYYLKlvPlsYCTxXJ56zSM9NMGbMmH87gzEzs67ryjua1wfe1Y0ynwG2krSiJAHbA48CtwK752n2A67pRhlmZtYFZa4pzGPJawqzgCO6WmBE3CvpCtKrPRcCD5B++V8PXCLphDzsnK6WYWZmXVOm+mhgTxcaEccAx1QNfgrYoqfLMjOz8sq8T+GWMsPMzOydr+6ZgqQVSM8QrJbvGFIeNQjfLmpm1iu1V330FeAwYA3ScwRtSeFV4FeNDcvMzFqhblKIiFOAUyQdEhGnNTEmMzNrkTIXmk+T9F5gY9LDa23DL2hkYGZm1nxlbkk9BhhLSgp/BHYiPbzmpGBm1suUeXhtd9IDZrNy43ibAKs0NCozM2uJMknh9YhYBCyUNAh4EVizsWGZmVkrlGkQb5KkwcBvSXchzQfubmRQZmbWGmUuNH8td54h6QZgUEQ82NiwzMysFco80SxJX5B0dERMA+ZKcnMUZma9UJlrCr8Gtgb2zv3zgNMbFpGZmbVMmWsKW0bE5pIeAIiIOZKWb3BcZmbWAmXOFBZI6kNuPlvSMGBRQ6MyM7OWKJMUTgWuAt4l6YekB9d+1NCozMysJdprJXXtiHg6Ii6SNJn0AJuAz0TE1KZFaGZmTdPeNYUrgA9IuiUitgcea1JMZmbWIu0lheUkfR/YQNK3q0dGxEmNC8vMzFqhvWsKewFvkxLHwBofMzPrZdp7n8LjwE8kPRgRf2piTGZm1iId3n3khGBmtuwoc0uqmZktI8q0fdS/zDAzM3vnK3OmUKuZbDedbWbWC7X38Np/ACOAAZI2Iz24BjAIWLEJsZmZWZO195zCx4HxwEjgFyxOCq8C329sWGZm1grtJYWNI+KjkvaMiMuaFpGZmbVMe9cUdpYk4MhmBWNmZq3V3pnCDcAcYGVJrxaGC4iIGNTQyMzMrOnqnilExPciYjBwfUQMKnwGOiGYmfVOZZ5oHteMQMzMrPXqJgVJd+S/8yS9Wv23eSGamVmztNcg3rb5r1tENTNbRrRbfSSpj6Qef7mOpMGSrpD0mKSpkraWNFTSzZKeyH+H9HS5ZmbWvnaTQkS8DTwuaVQPl3sKcENEbARsAkwl3fp6S0SsD9yCb4U1M2u69m5JbTMEeETSfcBrbQMj4tNdKVDSKsBHSE9LExFvAW9JGgeMzZOdD0wEjuhKGWZm1jVlksJ/93CZawOzgXMlbQJMBr4JrB4RM/M0s4DVa80s6SDgIIBRo3r6BMbMbNnWYVKIiNsaUObmwCERca+kU6iqKoqIkBR14jkLOAtgzJgxNacxM7Ouaa+V1HlA3YNuNx5gmwHMiIh7c/8VpKTwgqThETFT0nDgxS4u38zMuqi9W1IHAkg6HpgJXEhq4mIfYHhXC4yIWZKelbRhfg/09sCj+bMfcGL+e01XyzAzs64pc03h0xGxSaH/N5L+DhzdjXIPAS6StDzwFLA/6U6oyyQdAEwH9uzG8s3MrAvKJIXXJO0DXEKqTtqbwl1IXRERU4AxNUZt353lmplZ95R5HefnSb/aX8ifPfIwMzPrZcrcfTQNcKN4ZmbLgDJnCmZmtoxwUjAzswonBTMzq+gwKUhaRdLJkiblzy9y+0VmZtbLlDlT+B3wKukOpD1z97mNDMrMzFqjzHMK60bEboX+4yRNaVA8ZmbWQmXOFF6XtG1bj6RtgNcbF5KZmbVKmTOFg4EL8nUEAf8kvwvBzMx6lzIPr/0d2ETSoNz/asOjMjOzlugwKUjqD+wGjAb6SgIgIv6noZGZmVnTlak+ugZ4hfSGtDcbG46ZmbVSmaQwMiI+0fBIzMys5crcfXSXpPc1PBIzM2u5MmcK2wLjJT1Nqj4S6TXK729oZGZm1nRlksJODY/CzMyWCmVuSZ3ejEDMzKz13EqqmZlV1E0K+fkEMzNbhrR3pnA3gKQLmxSLmZm1WHvXFJaX9HngQ5I+Wz0yIq5sXFhmZtYK7SWFg4F9gMHALlXjAnBSMDPrZeomhYi4A7hD0qSIOKeJMZmZWYuUeU7hQkmHAh/J/bcBZ0TEgsaFZWZmrVAmKfwa6Jf/AuwL/AY4sFFBmZlZa5RJCh+MiE0K/X+R9PdGBWRmZq1T5uG1tyWt29YjaR3g7caFZGZmrVLmTOF7wK2SniI1hrcWsH9DozIzs5Yo0/bRLZLWBzbMgx6PCL9sx8ysFypzpkBOAg82OBYzM2sxN4hnZmYVTgpmZlbRYVJQ8gVJR+f+UZK26G7BkvpIekDSdbl/bUn3SnpS0qWSlu9uGWZm1jllzhR+DWwN7J375wGn90DZ3wSmFvp/ApwcEesBc4ADeqAMMzPrhDJJYcuI+DrwBkBEzAG69Ste0kjgk8DZuV/AdsAVeZLzgc90pwwzM+u8MklhgaQ+pJZRkTQMWNTNcn8JHF5YzqrA3IhYmPtnACNqzSjpIEmTJE2aPXt2N8MwM7OiMknhVOAq4F2SfgjcAfyoqwVK+hTwYkRM7sr8EXFWRIyJiDHDhg3rahhmZlZDmYfXLpI0Gdie9ETzZyJiageztWcb4NOSdgZWAAYBpwCDJfXNZwsjgee6UYaZmXVBmbuPhgIvAhOAi4EXJPXraoERcVREjIyI0cBewF8iYh/gVmD3PNl+wDVdLcPMzLqmTPXR/cBs4B/AE7l7mqT7JX2gB2M5Avi2pCdJ1xj8Yh8zsyYr08zFzcAVEXEjgKQdgd2Ac0m3q27Z1cIjYiIwMXc/BXT7+QczM+u6MmcKW7UlBICIuAnYOiLuAfo3LDIzM2u6MmcKMyUdAVyS+z9Huq7Qh+7fmmpmZkuRMmcKnyfdDXR1/ozKw/oAezYqMDMza74yt6S+BBxSZ/STPRuOmZm1UodJIT/BfDjwHtJzBQBExHYNjMvMzFqgTPXRRcBjwNrAccA04G8NjMnMzFqkTFJYNSLOARZExG0R8SVS43VmZtbLlLn7aEH+O1PSJ4HngaGNC8nMzFqlTFI4QdIqwHeA00htFR3WyKDMzKw1yiSFORHxCvAK8FEASds0NCozM2uJMtcUTis5zMzM3uHqnilI2hr4EDBM0rcLowaRHlwzM7Nepr3qo+WBlfM0AwvDX2VxE9dmZtaL1E0KEXEbcJuk8yJiehNjMjOzFilzobm/pLOA0cXp/USzmVnvUyYpXA6cAZwNvN3YcMzMrJXKJIWFEfGbhkdiZmYtV+aW1D9I+pqk4ZKGtn0aHpmZmTVdmTOF/fLf7xWGBbBOz4djZmatVOZ9Cms3IxAzM2u9DquPJK0o6Qf5DiQkrS/pU40PzczMmq3MNYVzgbdITzcDPAec0LCIzMysZcokhXUj4qfkJrQj4l+AGhqVmZm1RJmk8JakAaSLy0haF3izoVGZmVlLlLn76BjgBmBNSRcB2wDjGxmUmZm1Rpm7j26WdD+wFana6JsR8VLDIzMzs6Yrc/fRrqSnmq+PiOuAhZI+0/DIzMys6cpcUzgmv3kNgIiYS6pSMjOzXqZMUqg1TZlrEWZm9g5TJilMknSSpHXz5yRgcqMDMzOz5iuTFA4hPbx2KXAJ8Abw9UYGZWZmrdFuNZCkPsB1EfHRJsVjZmYt1O6ZQkS8DSyStEqT4jEzsxYqc8F4PvCQpJuB19oGRsShXSlQ0prABcDqpKekz4qIU/I7Gi4lvfZzGrBnRMzpShlmZtY1ZZLClfnTUxYC34mI+yUNBCbnhDMeuCUiTpR0JHAkcEQPlmtmZh0o80Tz+bnto1ER8Xh3C4yImcDM3D1P0lRgBDAOGJsnOx+YiJOCmVlTlXmieRdgCqn9IyRtKunanihc0mhgM+BeYPWcMABmkaqXas1zkKRJkibNnj27J8IwM7OszC2pxwJbAHMBImIKPfAqTkkrA78HDouIV4vjIiLIrbJWi4izImJMRIwZNmxYd8MwM7OCMklhQbGZi2xRdwqV1I+UEC6KiLbrFS9IGp7HDwde7E4ZZmbWeWWSwiOSPg/0ya/iPA24q6sFShJwDjA1Ik4qjLoW2C937wdc09UyzMysa8o+0fwe0ot1LgZeAQ7rRpnbAPsC20makj87AycCH5P0BLBD7jczsyaqe/eRpBWAg4H1gIeArSNiYXcLjIg7qP86z+27u3wzM+u69s4UzgfGkBLCTsDPmxKRmZm1THvPKWwcEe8DkHQOcF9zQjIzs1Zp70xhQVtHT1QbmZnZ0q+9M4VNJLU9PyBgQO4X6VGCQQ2PzszMmqpuUoiIPs0MxMzMWq/MLalmZraMcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzswonBTMzq3BSMDOzCicFMzOrcFIwM7MKJwUzM6twUjAzs4qlKilI+oSkxyU9KenIVsdjZrasWWqSgqQ+wOnATsDGwN6SNm5tVGZmy5alJikAWwBPRsRTEfEWcAkwrsUxmZktU/q2OoCCEcCzhf4ZwJbVE0k6CDgo986X9HgXy1sNeKmL8zaS4+ocx9V5S2tsjqtzuhPXWvVGLE1JoZSIOAs4q7vLkTQpIsb0QEg9ynF1juPqvKU1NsfVOY2Ka2mqPnoOWLPQPzIPMzOzJlmaksLfgPUlrS1peWAv4NoWx2RmtkxZaqqPImKhpG8ANwJ9gN9FxCMNLLLbVVAN4rg6x3F13tIam+PqnIbEpYhoxHLNzOwdaGmqPjIzsxZzUjAzs4pelxQkrSnpVkmPSnpE0jfz8KGSbpb0RP47JA/fLU/3V0mr5mHrSrq0levRbJL6SHpA0nW5f21J9+YmRy7NF/+RdIikhyX9sTBsW0kntzL+ZpI0WNIVkh6TNFXS1t6/6pP0rbwNHpY0QdIK3r+WJOl3kl6U9HBhWL19SpJOzdvuQUmb5+EbSpqch22dh/WV9GdJK5YOJiJ61QcYDmyeuwcC/yA1m/FT4Mg8/EjgJ7l7IrAi8AXgkDxsArB+q9elydvt28DFwHW5/zJgr9x9BvDV3H0P6cfED4BdAJFuDhja6nVo4rY6Hzgwdy8PDPb+VXdbjQCeBgYU9qvx3r/+bTt9BNgceLgwrN4+tTPwp7xttgLuzcNPArYl3c7/+zzsEGB8Z2LpdWcKETEzIu7P3fOAqaQdcxzpy0z++5ncvQjoT/riLpD0YWBWRDzRzLhbSdJI4JPA2blfwHbAFXmS4vYS0I+8vUgHuz9FxD+bGHLLSFqF9AU+ByAi3oqIuXj/ak9fYICkvqTtMBPvX0uIiNuB6nWst0+NAy6I5B5gsKThpO21Iov3tcGkxHpBZ2JZam5JbQRJo4HNgHuB1SNiZh41C1g9d/8Y+DPwPGkHvJz0jMSy5JfA4aQzK4BVgbkRsTD3zyAlVoBfkX7NPQLcCVwDfLxpkbbe2sBs4FxJmwCTgW/i/aumiHhO0s+BZ4DXgZtI28z7V8fq7VO1mgQaQWpQ9ALSj5CvAP8N/CgiFnWm0F53ptBG0srA74HDIuLV4rhI51WRu2+OiA9ExC6kDPxHYINcZ/zbTtXFvQNJ+hTwYkRMLjN9RFwYEZtFxBeAbwGnAjvl7XWypF67T2V9Saf5v4mIzYDXSKf2Fd6/Fsv14ONIyXQNYCXgE/Wm9/5VW3GfameaZyJibERsDfyLVI00VdKF+brNBmXK6pUbWFI/UkK4KCKuzINfyKdY5L8vVs2zIqmu83TgOGA/4A5gnyaF3SrbAJ+WNI3UMu12wCmkU9K2M8l/a3JE0hrAFhFxNfAd4HPAXGD7pkTdOjOAGRFxb+6/gpQkvH/VtgPwdETMjogFwJWkfc77V8fq7VNlmgT6Iem6zKGkauHDgWPKFNrrkkKuDz8HmBoRJxVGXUv6IpL/XlM16/eAU/OOO4CUlReR6ud6rYg4KiJGRsRoUrXGXyJiH+BWYPc8Wa3tdTxwdO5elrbXLOBZSRvmQdsDj+L9q55ngK0krZi/m23by/tXx+rtU9cCX8x3IW0FvFKoZkLSfwLP5+tWK5K2W/lt1+qr7j39IV19D+BBYEr+7EyqJ78FeIJUxzu0MM8awPWF/j1YXKc5rNXr1MRtN5bFdx+tA9wHPEmqB+9fmG4z4JxC/2F5e91QnK63foBNgUl5H7saGOL9q93tdRzwGPAwcCGpztv715LbaALpAvwC0tnoAfX2KdLF+NOB/wMeAsYUliPg5sK07wbuz/vqNmVicTMXZmZW0euqj8zMrOucFMzMrMJJwczMKpwUzMyswknBzMwqnBSWAZLeljQltz55eSueopU0VtKHml1uofwP55Y6p0ga0ILyx0g6tcR0dzUjnp4maf5SEMN5knbveEprj5PCsuH1iNg0It4LvAUcXGamwhOnPWEs0LKkQHpy+Md5O7xea4L8MFBDvhMRMSkiDi0xXSu3kZmTwjLor8B6klbKbbjfp/QehXEAksZLulbSX4BbJK0s6VxJD+V22nfL0+0o6W5J9+ezj5Xz8GmSjsvDH5K0UW6Y8GDgW/mX+ocl7aLUnv4Dub331fP8w5Tajn9E0tmSpktaLY/7Qo53iqQzJfWpXjlJ2+dlPpTXr7+kA4E9geMlXVQ1/WhJj0u6gPRw1ZqSvifpb3l9jytM91j+NfoPSRdJ2kHSnUrt3W+Rp9sib5cHJN3V9uRzPlNqe1fFsTm2iZKeknRoIZ75heknavF7Gy7KTwQjaec8bLJSu/rX1dgO7ylsqwclrZ+HX53ne0TSQcVyJf0sD/9zXo+2+D5d2DeuycOfkFSz2YRa26/GNPML3btLOi9376F0Rvt3SbfnYX1ybG3L/EoeLkm/yv+/PwPvqlWWdVKrn+Tzp/EfYH7+25f0qPxXgR8BX8jDB5PeO7ESqX2eGSx+IvInwC8LyxoCrAbcDqyUhx0BHJ27p7H4vQFfA87O3ccC361aTtvDkwcCv8jdvwKOyt2fID2dvhrpycw/AP3yuF8DX6xazxVIrUdukPsvIDWICHAesHuNbTOa1ATAVrl/R9IL0UX60XQdqans0cBC4H15+GTgd3m6ccDVef5BQN/cvQOL27Ufy+KnxY8F7iI92bsa8HJhveYXpn+F1K7NcsDdpKf129Zx7TzdhLblVq3XacA+uXt5Fr/PoO3/OoCUBFfN/QHslLuvIrVm2g/YBJiSh48nPXW7amH+MVVx19x+9fbJ3L07cF7ufggY0bZf5r8HAT/I3f1JT5OvDXyW9PRuH9JT43Nr/Y/96dynVzedbRUDJE3J3X8ltQ11F6khvO/m4SsAo3L3zbG4/fodKDT1HBFzlFpW3Ri4M/94XZ500GrT1gjhZNIXt5aRwKVKDX0tT3oRC6QD3665rBskzcnDtwc+APwtlzmAqkbngA1Jja/9I/efD3yd1DR4e6ZHapce0kFtR+CB3L8ysD6pDZ+nI+IhAEmPALdEREh6iJQ0AFYBzs+/zIN0YK3l+oh4E3hT0oukZpFnVE1zX0TMyOVNyWXMB56KiLbtNYF00Kx2N/BfSu/KuDIWv7/hUEm75u4187q9TKpWvCEPfwh4MyIWVK0bpH3j5RzTlaT/16TC+Hrb7/Y626HancB5ki5j8X60I/B+Lb5esEpe5keACRHxNvC80tmtdZOTwrLh9YjYtDggV0XsFhGPVw3fktQcdHtEOjjsXWf8m/nv29Tfx04DToqIayWNJf167qjM8yPiqA6m64ri+op07eHMJQpPVWBvFgYtKvQvYvF6Hg/cGhG75nkm1imzuKx626nMNDVFxMWS7iW9POmPucplESnJbx0R/5I0kfRjAGBB5J/iFNYtIhZpyWtL1e3iVPfX3H61Qix0r1AZGHFw3gc/CUyW9IG8zEMi4sYlCpJ27qAM6wJfU1h23QgcUqin3qzOdDeTfm2TpxtCegnKNpLWy8NWUsdttc9j8Ut8IP3aa2vud7/C8DtJ9f9I2pFUzQSpYbDdJb0rjxsqaa2qMh4HRrfFBewL3NZBXNVuBL6kxddIRrSVWVJxvcZ3suwyHgfWyQkHUpPS/0bSOqQzilNJVYbvz7HNyQlhI9KrHDvrY3nbDyC9CezOqvFlt98Lkt6tdGG/7cwFSetGxL0RcTTpZUZr5mV+ValJfCRtIGkl0tnH5/I1h+HAR7uwPlbFSWHZdTypauPBXBVyfJ3pTgCGtF38Az4aEbNJB7wJkh4kVVVs1EF5fwB2zRc+P0w6M7hc0mTgpcJ0xwE7Kr3AfA/SG6fmRcSjpPbhb8pl3kx6H3dFRLwB7J+X+xDpF+8ZHW+KJZZxE+ld1XfnZVzBksmsIz8FfizpARpwJh7pzqmvATfkbTePdO2h2p7Aw7na6b2k6ys3AH0lTQVOJCX3zrqP9K6SB0nXS4pVR53ZfkeSrjfcRbpO0eZnSjcJPJzH/Z30PoBHgfvz8DNJ2/YqUguij+b1K1ZhWhe5lVRbqkjqD7wdEQslbU16w9mmLQ5rqSJp5YiYn8/yTgeeiIiTm1DueNKF5W80uixrHV9TsKXNKOCyXK3wFvDlFsezNPqypP1IF+gfIP1yNusRPlMwM7MKX1MwM7MKJwUzM6twUjAzswonBTMzq3BSMDOziv8HNZiXbg3m50IAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "ref_cov_drift_out = da.ref_covariate_drift()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Reference sample label drift\n", + "\n", + "In this test we look for evidence of the reference sample labels being representative of the underlying population.\n", + "The same holdout and undersampling strategy from the previous test are used and, since now we have a univariate test, only the p-values for the increasing leftover slices are shown." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA6FElEQVR4nO3deXwV9b3/8dc7KyHsq0JYLYqIyCZL3Zfi0iLFnYItWrXtrdrW2/6uXW61i7221t7WW3tbtYoCotZeFKtVaxVXAoJSVhcUQoLssoeQ5Xx+f8wkHGKWE8jJJDmf5+NxHpnle2Y+Z85kPme+35nvyMxwzjmXutKiDsA551y0PBE451yK80TgnHMpzhOBc86lOE8EzjmX4jwROOdcivNE4JDUU9KrkvZIuivqeFoqSdMlvX6Y750v6doEy66TdO5hruew39tYjmQ7ueTIiDoA1zgkrQN6AhXAXuA54AYz25vA268HtgEdzG8scS7l+BlB6zLRzNoBw4ERwPcTfF8/YNXhJAFJ/mPCuRbOE0ErZGabgOcJEgIAksZJelPSTkn/knRmOH0G8BXg/0naK+lcSWmSbpH0oaTtkh6X1CUs31+SSfqqpPXAS+H0ayStlrRD0vOS+sWt2yR9XdIH4frvkaS4+deF790jaZWkkeH0XpL+KmmrpLWSbqrtM0u6MHzvHkkbJH03nN5Z0t/CZewIh/Pi3jdf0s/DbbNX0tOSukqaLWm3pLck9a/2WW6S9JGkbZLulFTj/5GkwZL+IekTSe9JujyR70/SMZJeCrf9tjCWTtWKnRx+3h2SHpTUJu79X5C0NNzWb0oaluB6Z0j6YxjzHkmvxH+P1cr+r6RfV5v2lKSbw+HK/afyO51cy3Iq96eMuGmHVJPVtm8p8N+StoTf1XJJQxP5rK4aM/NXK3gB64Bzw+E8YDnwu3C8N7AduJAg+X8uHO8ezp8B/DxuWd8C8sPlZAN/AuaE8/oDBjwM5AI5wCRgDXA8QXXjj4A345ZnwN+ATkBfYCtwfjjvMmADcDIg4DMEZyhpwBLgx0AWMBD4CDivls+/ETgtHO4MjAyHuwKXAG2B9sBfgCfj3jc/jP0YoCOwCngfODf8LA8DD1b7LC8DXcLP8j5wbThvOvB6OJwLFAJXh8sZQVD9NqSW+OfHLecz4XeUDXQHXgV+W+27XgH0CeN4o/L7C9ezBRgLpBMk+XVAdvX9pIYYZgB7gNPDdf+u8vPUUPb08PMpbpvvB3rFfa+9wu/xCmAfcHQN26l/uE0zatkWte5bwHnhPtKJYN85vnId/mrg8SPqAPzVSF9k8A++N/xHNuCfQKdw3n8AM6uVfx74Sjg8g0MTwWrgnLjxo4Gy8B+x8h93YNz8vwNfjRtPA4qBfuG4AafGzX8cuCUujm/V8HnGAuurTfs+cQflavPWA18jaOeoazsNB3bEjc8Hfhg3fhfw97jxicDSuHEjTGLh+L8B/wyH4w9wVwCvVVv3n4Bba4mr6uBXw7wvAu9U+66/Hjd+IfBhOPy/wM+qvf894Iy499aVCB6NG29H0ObUp4ayCrf56eH4dcBLdWz3pcCkGrZT5f5UWyKodd8CziZIxOOAtGT/j7Xml1cNtS5fNLP2wJnAYKBbOL0fcFlYVbBT0k7gVIIDfE36AXPjyq4mOCD0jCtTWK387+LKf0JwoOgdV2ZT3HAxwUEGgl+1H9YSQ69qMf+gWgzxLiE4IBaEVRrjASS1lfQnSQWSdhP8uu4kKT3uvZvjhvfXMN6OQ8V/9gKCX741xT+2WvxTgaNqib+Kgqu4Hg2ruHYDszj4XdYXQz/g36utt08tMdakarkWXGjwCcH38IOw6myvpD9acFR+FJgSFv8SMDvuM3w5rnpqJzC0hs+QiFr3LTN7Cfg9cA+wRdK9kjocxjpSnieCVsjMXiH4dVdZh1tIcEbQKe6Va2Z31LKIQuCCauXbmNmG+NVUK/+1auVzzOzNBMItJKiWqWn62mrLbG9mF9bymd8ys0lAD+BJgrMOgH8HjgPGmlkHgioNCA4mh6tP3HBf4ONa4n+lWvztzOwbCSz/FwTb98Qw5mk1xFtbDIXA7dXW29bM5iTyweKXK6kdQdXTx2b2izD+dmb29bDIHODSsM5+LPDX8H39gPuAG4CuZtaJoCqrpm2+L/zbNm5afLKsc98ys7vNbBQwBDgW+F6Cn9PF8UTQev0W+Jykkwh+UU6UdJ6kdEltJJ0Z32hazR+B2+Ma5bpLmlTHuv4IfF/SCWH5jpIuSzDO+4HvShoVNv59JlzvImCPpP+QlBPGPVTSydUXIClL0lRJHc2sDNgNxMLZ7Ql+1e9U0OB9a4Jx1eV7Chqh+xC0pzxWQ5m/AcdKukpSZvg6WdLxCSy/PUE13y5Jvan54PZNSXnhZ/phXAz3AV+XNDbcnrmSPi+pfYKf7UJJp0rKAn4G5JtZYU0FzewdgnaP+4HnzWxnOCuXIJFtBZB0NcEZQU3L2ErQRjQt/I6v4dAfBrXuW+H2HCspkyChlHDwe3cN4ImglQr/wR4Gfhz+I08iqFrZSvAr63vU/v3/DpgHvCBpD0HD8dg61jUX+CXwaFiVsQK4IME4/wLcDjxC0L7xJNDFzCqALxDU6a/l4AGnYy2LugpYF67/6wTVMBAkxJzw/fkE91ccqacIGimXAs8Af65ewMz2ABOAKwl+rW8i2EbZCSz/J8BIYFe4/P+rocwjwAsEDegfAj8P17uYoL7+98AOgobW6Ql+rsrl3kpQBTOK4GykvvLnhn8JY1hF0NaygKCa7USCBu3aXEewP24HTgCqziTr2bc6ECS+HQTVY9uBOxP4jK6ayhZ/51wCJBkwyMzWRB1LY1NwKXGRmf0o6lhc0/IzAuecS3GeCJxzLsV51ZBzzqU4PyNwzrkU1+I6DOvWrZv1798/6jCcc65FWbJkyTYz617TvBaXCPr378/ixYujDsM551oUSQW1zfOqIeecS3GeCJxzLsV5InDOuRTnicA551KcJwLnnEtxSUsEkh4IHyG3opb5knS3pDWSlil8PKFzzrmmlcwzghnA+XXMvwAYFL6uJ3iykmsmlhTs4J6X17CkYEfUoTjnkixp9xGY2auKe+h3DSYBD4dPOsqX1EnS0Wa2MVkxucQsKdjBl+7Lp7Q8RnqamH5KfwYf1YG2WenkZKXTNjOd3OyMYDgrnbaZwXBWhtc0usQtKdhB/kfbGTewK6P6dY46nJQW5Q1lvTn0cXtF4bRPJQJJ1xOcNdC3b98mCS6V5X+0ndLyGAaUx4z7X1ub0Psy00VOZjptszKqkkZu1sGEUTlelVCy0snJyiA3brhtVjo5YaKJTzwZ6Z5kmquKmFFaHqO0PMaB8goOlMfCV0U4LVb1t3La+5v3cP9ra6mIGRnp4lvnDOL4ozuQnZFOdmYa2RlptMlMJzsjLZgWjmdlpJGediQPl3M1aRF3FpvZvcC9AKNHj/Ze8pJsZN9OVc+hbJORxh+mjWRQj/bsKy2nuLSC/aUVFJdWUByOB9MODhdXK7ezuJSPd4blyirYd6CcA+UNe5BUVkZaePZRmUSCRNE2HM6JG25bLfEcnJdOTmYGudkHl5GTmd5iDyyxmFFaUftBt+rAXBYLy9VwYC6r4EBF7GCZqr8VhyyjtFqZ+GWVx47sX7Kswvj1C+8nXD4zXWRnpNMm82CSyIpPHFUJpFoyyUyjTVyiqb6M7MxPJ59D39Ny95X6RJkINnDoc1fzwmkuYh/vLAHgitF5XH5y36SctlfEjP1lQdLYX1rBvgMV7C+LTywV7Avn1ZxgguFte0spLi0OppVVUHwgOGg1RHZGWlDVlZl+SHKJP3M5NMFkfKrc+k+KeXfjHk7o1Z4B3duFB82KQw+enzrAxh1o6/gVHf9LO35aQz9nXZ8/K/7gVzUeTGublUHnatOyDil3cDw7M42s9OBgHPxNI7vyb1y59zbt4cY571BeESMjPY07Lh7GMT1yw+QUo6Ssomp7HCiPGy+rYVqY0ErCv7v3lx1McnHJ8EAjbLOMNMUli7ik04Dkc0iZzFoSUlzyqSy/tHBn0qrSokwE84AbJD1K8BjEXd4+0DzMzC9gYPdc7rhkGFJyfgGlp4l22Rm0y278XbC8IkZx2cEzkn0HysOkc/DMZV/ccGXSiT+L2V9awabdJZ86+znSX7/xstJrP3hW/m3fJqPqgHJw3qHjhxyIqx2gazpYxx/QM9OVtO+4Lv265vLIdeOavI2gshorsQRTUS0pBdNKyg5NLvHT9h4oZ/veUkqqza/8QXCkBGRnpjH72nGNus2SlggkzQHOBLpJKiJ4DmomgJn9EXgWuJDgmarFwNXJisUlbnnRLpYW7uTWiUMiOUA0hoz0NDqkp9GhTWajL7u0PBaefYRnLwcqmL2wgMcWF2IGaYJLR+Vx6ag+hx50qx3ks9LTSGul1QyJGtWvc5M3EqeniZzwTK+pVVXllVVLKAkkmFff38Yba7ZhQFl5jPyPtreMRGBmU+qZb8A3k7V+d3hm5ReQk5nOxSPzog6lWcoKD+4dOZhkLqvow5NLN1BWHiMzI40rklSd5lq2tDTRJi2dNpnpQMN+pIzq14XFBZ9U7WPjBnZt1NhaRGOxaxq7ist46l8bmDwij445jf9rurUa1a8zs69t+moOlzqSvY95InBVnni7iJKyGNPG+SW6DRVFNYdLLcncx/zibAcE9Zez8gsY1a8zJ/TqGHU4zrkm5InAAfDGh9tYu20fV43rF3Uozrkm5onAATBzQQFdcrO44MSjog7FOdfEPBE4Pt65nxdXb+aKk/uQndH0l9U556LlicAxZ9F6DPjSGG8kdi4VeSJIcaXlMeYsKuScwT3o06Vt1OE45yLgiSDFPb9yE9v2HmCaNxI7l7I8EaS4mfkF9O3SltMHdY86FOdcRDwRpLB3N+1m0dpPmDaub8r3e+NcKvNEkMJm5ReQlZHGZaP61F/YOddqeSJIUXtKypj79gYmDutF59ysqMNxzkXIE0GKevKdDewrreDL472R2LlU54kgBZkZM/MLGJbXkZP6dIo6HOdcxDwRpKBFaz/h/c17/ZJR5xzgiSAlPZxfQMecTCYO6xV1KM65ZsATQYrZsruE51ds4rJReZE8rs851/x4Ikgxj75VSHnMmOrVQs65kCeCFFJeEeORhes5bVA3BnTLjToc51wz4Ykghby4egubdpfw5fH9ow7FOdeMeCJIIbPyC+jdKYezB/eIOhTnXDPiiSBFfLh1L6+v2caXxvYl3fsVcs7F8USQImblF5CZLi4f7f0KOecO5YkgBRSXlvPEkiIuGHo03dtnRx2Oc66Z8USQAuYt/Zg9JeVc5f0KOedq4ImglTMzHl5QwOCj2jO6X+eow3HONUOeCFq5dwp3smrjbq4a3w/JG4mdc5/miaCVm7WggHbZGXxxeO+oQ3HONVOeCFqx7XsP8LdlG7lkZG9yszOiDsc510x5ImjFHl9cRGlFzLubds7VyRNBK1URM2YvLGDcwC4M6tk+6nCcc81YUhOBpPMlvSdpjaRbapjfV9LLkt6RtEzShcmMJ5W88v4Winbs56px/aMOxTnXzCWcCCS1bciCJaUD9wAXAEOAKZKGVCv2I+BxMxsBXAn8oSHrcLWbuaCAHu2zmXBCz6hDcc41c/UmAkmflbQKeDccP0lSIgfsMcAaM/vIzEqBR4FJ1coY0CEc7gh8nHDkrlbrtxcz//2tTBnTl8x0r/1zztUtkaPEfwPnAdsBzOxfwOkJvK83UBg3XhROi3cbME1SEfAscGMCy3X1mL2ogDSJKWP6Rh2Kc64FSOjnopkVVptU0UjrnwLMMLM84EJgpqRPxSTpekmLJS3eunVrI626dSopq+DxtwqZMKQnR3VsE3U4zrkWIJFEUCjps4BJypT0XWB1Au/bAMR3dZkXTov3VeBxADNbALQBulVfkJnda2ajzWx09+7dE1h16npm2UZ2FJdxlV8y6pxLUCKJ4OvANwmqdTYAw8Px+rwFDJI0QFIWQWPwvGpl1gPnAEg6niAR+E/+IzAzv4CB3XMZf0zXqENxzrUQ9d5uambbgKkNXbCZlUu6AXgeSAceMLOVkn4KLDazecC/A/dJ+g5Bw/F0M7OGrssFlhftYmnhTm6dOMT7FXLOJazeRCDpQYKD9CHM7Jr63mtmzxI0AsdP+3Hc8CrglIQidfWalV9ATmY6l4zKizoU51wLkkgHNH+LG24DTMYv82x2dhWX8dS/NjB5RB4d2mRGHY5zrgVJpGror/HjkuYAryctIndYnni7iJKyGNPG+SWjzrmGOZy7jQYBPRo7EHf4YjFjVn4Bo/p15oReHaMOxznXwiTSRrCHoI1A4d9NwH8kOS7XAG98uI212/bxrXMGRR2Kc64FSqRqyLuubOZmLiigS24WF5x4VNShOOdaoFoTgaSRdb3RzN5u/HBcQ328cz8vrt7M1844huyM9KjDcc61QHWdEdxVxzwDzm7kWNxhmLNoPQZMHeuNxM65w1NrIjCzs5oyENdwpeUx5iwq5JzBPcjr3KBewp1zrkpCD7KVNJTgmQJVvZiZ2cPJCsol5rmVm9i294A/itI5d0QSuWroVuBMgkTwLMGDZl4HPBFEbNaCAvp2acvpg7wjPufc4UvkPoJLCTqG22RmVwMnETxExkXo3U27WbTuE6aN60tamvcr5Jw7fIkkgv1mFgPKJXUAtnBo99IuArPyC8jKSOOyUf5VOOeOTCJtBIsldQLuA5YAe4EFyQzK1W1PSRlz397AxGG96JybFXU4zrkWLpEbyv4tHPyjpOeADma2LLlhubo8+c4G9pVW8OXx3kjsnDtyiTy8fp6kL0nKNbN1ngSiZWbMzC9gWF5HTurTKepwnHOtQCJtBHcBpwKrJD0h6VJJ/jDciCxc+wnvb97rl4w65xpNIlVDrwCvSEonuJv4OuABoEOSY3M1mJlfQMecTCYO6xV1KM65ViLRG8pygInAFcBI4KFkBuVqtmV3Cc+v2MT0z/YnJ8v7FXLONY5Ebih7HBgDPAf8HnglvJzUNbFH3yqkPGZM9Woh51wjSuSM4M/AFDOrSHYwrnblFTEeWbie0wZ1Y0C33KjDcc61IvU2FpvZ85VJQJJ3PR2RF1dvYdPuEr48vn/UoTjnWpmGPqrS+zKIyMz8dfTulMPZg/0poc65xtXQRPBMUqJwdVqzZS9vrNnOl8b2Jd37FXLONbKEEoGkoyRdBCyU5M9DbGKzFxaQmS4uH+39CjnnGl8idxZfCywCLiboiTRf0jXJDswFikvLeWJJERcMPZru7bOjDsc51wolctXQ94ARZrYdQFJX4E2Cm8pcks1b+jF7Ssq5yvsVcs4lSSJVQ9uBPXHje8JpLsnMjIcXFDD4qPaM7tc56nCcc61UImcEawjaBp4ieGj9JGCZpJsBzOw3SYwvpb1TuJNVG3dz++ShSN5I7JxLjkQSwYfhq9JT4d/2jR+OizdzQQHtsjP44vDeUYfinGvFEul07idNEYg71Pa9B3hm2UamjOlDbnZCXUI559xhaeh9BK6JPL64iNKKmHc37ZxLOk8EzVBFzJi9sIBxA7swqKfXwDnnkiuR+whOSWSaazyvvL+Foh37uWpc/6hDcc6lgETOCP4nwWmfIul8Se9JWiPpllrKXC5plaSVkh5JZLmt3cwFBfRon82EE3pGHYpzLgXU2gopaTzwWaB75aWioQ5AvU9FCZ9odg/wOaAIeEvSPDNbFVdmEPB94BQz2yEp5XtUW7+9mPnvb+WmsweRme41d8655KvrSJMFtCNIFu3jXrsJupqozxhgjZl9ZGalwKME9yDEuw64x8x2AJjZloaF3/rMXlhAmsSUMX2jDsU5lyJqPSOIe1bxDDMrAJCUBrQzs90JLLs3UBg3XgSMrVbm2HC5bxCcZdxmZs9VX5Ck64HrAfr2bb0HyJKyCh5bXMiEIT05qmObqMNxrUxZWRlFRUWUlJREHYpLojZt2pCXl0dmZmbC70nkAvX/kvR1oAJ4C+gg6Xdmdudhxll9/YOAM4E84FVJJ5rZzvhCZnYvcC/A6NGjrRHW2yw9s2wjO4vLuMovGXVJUFRURPv27enfv7/fqd5KmRnbt2+nqKiIAQMGJPy+RCqhh4RnAF8E/g4MAK5K4H0bgPh+k/PCafGKgHlmVmZma4H3CRJDSpqZX8DA7rmMP6Zr1KG4VqikpISuXbt6EmjFJNG1a9cGn/UlkggyJWUSJIJ5ZlZG0OdQfd4CBkkaICkLuBKYV63MkwRnA0jqRlBV9FFCkbcyy4t2sbRwJ1eN6+f/qC5pfN9q/Q7nO04kEfwJWAfkElTd9CNoMK6TmZUDNwDPA6uBx81spaSfhg+5IZy3XdIq4GXge5XdXaeaWfkF5GSmc8movKhDcS5p2rVrl3DZ2267jV//+tdJW/6RmjFjBjfccEOTrS+ZEulr6G7g7rhJBZLOSmThZvYs8Gy1aT+OGzbg5vCVsnYVl/HUvzYweUQeHdok3sDjnHONIZE7i3tK+rOkv4fjQ4CvJD2yFPKXJYWUlMWYNq71XhHlWqYlBTu45+U1LCnYkbR1PP3004wdO5YRI0Zw7rnnsnnz5qp5//rXvxg/fjyDBg3ivvvuq5p+5513cvLJJzNs2DBuvfXWOpe/bt06Bg8ezNSpUzn++OO59NJLKS4uPqRMLBajf//+7Ny5s2raoEGD2Lx5c53xVZo+fTpPPPFE1Xj8mUlDYo1KIlcNzQAeBH4Yjr8PPAb8OUkxpZRYzJi9cD2j+nXmhF4dow7HpYifPL2SVR/XXcO7p6SMdzftIWaQJhh8VHva13HGOqRXB26deEKDYzn11FPJz89HEvfffz+/+tWvuOuuuwBYtmwZ+fn57Nu3jxEjRvD5z3+eFStW8MEHH7Bo0SLMjIsuuohXX32V008/vdZ1vPfee/z5z3/mlFNO4ZprruEPf/gD3/3ud6vmp6WlMWnSJObOncvVV1/NwoUL6devHz179qwzvvq88MILDY41Com0EXQzs8eBGFTV/VckNaoU8saH21i7bZ9fMuqand0l5cTCy0JiFownQ1FREeeddx4nnngid955JytXrqyaN2nSJHJycujWrRtnnXUWixYt4oUXXuCFF15gxIgRjBw5knfffZcPPvigznX06dOHU04JukibNm0ar7/++qfKXHHFFTz22GMAPProo1xxxRX1xlefw4k1ComcEewLn1NsAJLGAbuSGlUKmbmggC65WVxw4lFRh+JSSCK/3JcU7GDq/fmUlcfIzEjjd1eOYFQSHpl64403cvPNN3PRRRcxf/58brvttqp51a+AkYSZ8f3vf5+vfe1rCa+jpuUsXLiwahk//elPmThxImvWrGHr1q08+eST/OhHP6o3vkoZGRnEYjEgqGYqLS0FOKxYo5DIGcHNBJd9HhPeAfwwcFNSo0oRH+/cz4urN3PFyX3Izqi3+ybnmtSofp2Zfe04bp5wHLOvHZeUJACwa9cuevcOnsL30EMPHTLvqaeeoqSkhO3btzN//nxOPvlkzjvvPB544AH27t0LwIYNG9iype7eadavX8+CBQsAeOSRRzj11FMZO3YsS5cuZenSpVx00UVIYvLkydx8880cf/zxdO3atd74KvXv358lS5YAMG/ePMrKygAOK9YoJHJGsBI4AzgOEPAe/hyDRjFn0XoMmDrWG4ld8zSqX+dGTQDFxcXk5R28RPrmm2/mtttu47LLLqNz586cffbZrF27tmr+sGHDOOuss9i2bRv/+Z//Sa9evejVqxerV69m/PjxQNAwO2vWLHr0qL3PyuOOO4577rmHa665hiFDhvCNb3yjxnJXXHEFJ598MjNmzKiaVld8la677jomTZrESSedxPnnn09ubi4AEyZMaHCsUVBwBWcdBaS3zWxkfdOayujRo23x4sVRrLpRlZbH+OwdLzG8T0fu/8rJUYfjUsDq1as5/vjjow6jya1bt44vfOELrFixIupQmkxN37WkJWY2uqbydXVDfRRBx3E5kkYQnA1A0A1128YJN3U9t3IT2/Ye8EdROuciV1fV0HnAdII+gu7iYCLYDfwguWG1frMWFNC3S1tOH9Q96lCca9X69++fUmcDh6OubqgfAh6SdImZ/bUJY2r13t20m0XrPuEHFw4mLc37fnHORaveRl9PAo1vVn4B2RlpXDaqT/2FnXMuyfzqnya2p6SMuW9vYOJJveicmxV1OM4554mgqc19ZwP7Siv8TmLnXLNRayKQdHFdr6YMsrUwM2YuKGBYXkdO6tMp6nCcaxUaszvoX/ziF1XD69atY+jQoY2y3HiN2b129c7uDlddZwQTw9dXCTqYmxq+7geuOeI1p6CFaz/hgy17/ZJR55qp+ESQqPLy5PTB1JRqTQRmdrWZXQ1kEjyu8hIzuwQ4IZzmGmhmfgEdczKZOKxX1KE4l5jCRfDaXcHfI9Tcu4O+5ZZb2L9/P8OHD2fq1KkAVFRUcN1113HCCScwYcIE9u/fD8CZZ57Jt7/9bUaPHs3vfvc7lixZwhlnnMGoUaM477zz2LhxIwB33303Q4YMYdiwYVx55ZVV61q1ahVnnnkmAwcO5O67Dz7u5Te/+Q1Dhw5l6NCh/Pa3v/1UjGbGDTfcwHHHHce5557baN1VJNLFRB8z2xg3vhnwPhEaaMvuEp5fsYnpn+1PTpb3K+Qi9vdbYNPyussc2A2bV4DFQGnQcyhkd6i9/FEnwgV31LnI5twd9B133MHvf/97li5dCgSJ64MPPmDOnDncd999XH755fz1r39l2rRpAJSWlrJ48WLKyso444wzeOqpp+jevTuPPfYYP/zhD3nggQe44447WLt2LdnZ2Yckt3fffZeXX36ZPXv2cNxxx/GNb3yDZcuW8eCDD7Jw4ULMjLFjx3LGGWcwYsSIqvfNnTuX9957j1WrVrF582aGDBnCNdcceQVNIo3F/5T0vKTpkqYDzwAvHvGaU8yjbxVSHjOmerWQaylKdgVJAIK/JUfe6XBL6w56wIABDB8+HIBRo0axbt26Q+KEILmtWLGCz33ucwwfPpyf//znFBUVAUFfSVOnTmXWrFlkZBz83f35z3+e7OxsunXrRo8ePdi8eTOvv/46kydPJjc3l3bt2nHxxRfz2muvHRLPq6++ypQpU0hPT6dXr16cffbZCW+DuiTyqMobJE0GKlPnvWY2t1HWniLKK2I8snA9px/bnQHdcqMOx7l6f7kDQXXQQxdBRSmkZ8El90OfMUe02pbWHXR2dnbVcHp6elXVEFDVsZyZccIJJ1T1bhrvmWee4dVXX+Xpp5/m9ttvZ/ny5TUuN+p2hkQvH30beMbMvgM8L6l9EmNqdV5cvZlNu0v8klHXsvQZA1+ZB2f/MPh7hEkAmn930JmZmVXvSdRxxx3H1q1bqz5XWVkZK1euJBaLUVhYyFlnncUvf/lLdu3aVbX+mpx22mk8+eSTFBcXs2/fPubOnctpp512SJnTTz+dxx57jIqKCjZu3MjLL7/coFhrU+8ZgaTrgOuBLsAxBB3R/RE4p1EiSAEz8wvo3SmHswc3r65nnatXnzGNkgAqNffuoK+//nqGDRvGyJEjuf322xP6TFlZWTzxxBPcdNNN7Nq1i/Lycr797W9z7LHHMm3aNHbt2oWZcdNNN9GpU6dalzNy5EimT5/OmDHB9r722msPaR8AmDx5Mi+99BJDhgyhb9++VZ/nSCXSDfVSYAyw0MxGhNOWm9mJjRJBA7W0bqjXbNnLub95he+ddxzfPOszUYfjUljU3VCnYnfQUWloN9SJVA0dMLPSuIVlED620tVv9sICMtPF5aO9XyHnXPOUSCJ4RdIPCJ5L8DngL8DTyQ2rdSguLeeJJUVcMPRourfPrv8NzrVi3h1085VIIvgPYCuwHPga8Czwo2QG1VrMW/oxe0rKuWq8NxI755qvOhuLJaUDK81sMHBf04TUOpgZDy8oYPBR7RmdpId+O9dQZvapSzhd61Jfu29N6jwjMLMK4D1JfidxA729fierNu7mqvH9/B/PNQtt2rRh+/bth3WgcC2DmbF9+3batGnToPcl0sVEZ2ClpEXAvrgVXtSwEFPLrPwC2mVn8MXhvaMOxTkA8vLyKCoqYuvWrVGH4pKoTZs25OXlNeg9iSSC/zy8cFLX9r0HeGbZRqaM6UNudiKb2Lnky8zMZMCAAVGH4ZqhRLqYeKUpAmlNHl9cRGlFzLubds61CPVeNSRpnKS3JO2VVCqpQtLupgiuJaqIGbMXFjBuYBcG9fSeOJxzzV8il4/+HpgCfADkANcC9yQzqJbslfe3ULRjP1eN6x91KM45l5CEOp0zszVAuplVmNmDwPnJDavlenhBAT3aZzPhhJ5Rh+KccwlJJBEUS8oClkr6laTvJPg+JJ0v6T1JayTdUke5SySZpBr7wWgpCrbv45X3tzJlTF8y0xPt2NU556KVyNHqKiAduIHg8tE+wCX1vSm8Ge0e4AJgCDBF0pAayrUHvgUsTDzs5umRhetJk5gyxm+7cM61HIlcNVQQDu4HftKAZY8B1pjZRwCSHgUmAauqlfsZ8Evgew1YdrNTUlbBY4sLmTCkJ0d1bNjNHM45F6VErhpaK+mj6q8Elt0bKIwbLwqnxS97JMEzkZ+pJ4brJS2WtLi53gzzzLKN7Cwu84fPOOdanETudoqvt28DXEbwkJojIikN+A0wvb6yZnYvcC8EzyM40nUnw8z8Ao7pnsv4Y7pGHYpzzjVIvWcEZrY97rXBzH4LfD6BZW8gaE+olBdOq9QeGArMl7QOGAfMa4kNxsuLdrG0cCdXjfN+hZxzLU8ij6ocGTeaRnCGkMiZxFvAIEkDCBLAlcCXKmea2S6gW9x65gPfNbOW8/ix0Mz8deRkpnPxqIb17+Gcc81BIgf0u+KGy4F1wOX1vcnMyiXdADxPcNXRA2a2UtJPgcVmNu8w4m12dhWX8dTSj7l4ZB4d2mRGHY5zzjVYIlcNnXW4CzezZwkeZBM/7ce1lD3zcNcTpb8sKeRAeYxp4/ySUedcy5RI1dDNdc03s980XjgtSyxmzF64nlH9OnNCr45Rh+Occ4cl0auGTgYqq3ImAosI+h5KaW98uI212/bxrXMGRR2Kc84dtkQSQR4w0sz2AEi6DXjGzKYlM7CWYOaCArrmZnHBiUdFHYpzzh22RLqY6AmUxo2XhtNS2sc79/Pi6s1ccXIfsjPSow7HOecOWyJnBA8DiyTNBUTQTcSMZAbVEjyycD0GfGmsNxI751q2RK4aul3S34HTAAOuNrN3kh5ZM1ZaHuPRt9ZzzuAe5HVuG3U4zjl3RGqtGpLUVlImgJm9DTxHcD9Ayj/09LmVm9i2t9QfRemcaxXqaiN4DugPIOkzwAJgIPBNSXckP7Tma9aCAvp2acvpg7pHHYpzzh2xuhJBZzOrvET0K8AcM7uR4PkCifQ11Cq9u2k3i9Z9wrRxfUlL836FnHMtX12JIL6Xz7OBfwCYWSkQS2ZQzdms/AKyM9K4bFSf+gs751wLUFdj8TJJvyboMO4zwAsAkjo1QVzN0p6SMua+vYGJJ/Wic25W1OE451yjqOuM4DpgG0E7wQQzKw6nDwF+neS4mqW572xgX2mFP3zGOdeq1HpGYGb7gUMahSWNNLM3gTeTHVhzY2bMXFDAsLyOnNSnU9ThOOdco0nkzuJ49yclihZg4dpP+GDLXr9k1DnX6jQ0EaTsZTIz8wvomJPJxGG9og7FOecaVUMTwU+SEkUzt2V3Cc+v2MRlo/LIyfJ+hZxzrUutbQSSBpvZu9UeVbm+cjy82zglPPpWIeUxY6pXCznnWqG6Lh+9GbieQx9VWckI7i1o9corYjyycD2nH9udAd1yow7HOecaXV1XDV0f/j3sR1W2Bi+u3sym3SX87ItDow7FOeeSIpFuqJH0WYL7CarKm9nDSYqpWZmZX0DvTjmcPbhH1KE451xSJPLM4pnAMcBSoCKcbATPKWjV1mzZyxtrtvO9844j3fsVcs61Uok+s3iImVm9JVuZ2QsLyEwXl4/2foWcc61XIpePrgBS7qG8xaXlPLGkiAuGHk339tlRh+Occ0lT1+WjTxNUAbUHVklaBByonG9mFyU/vOg8tfRj9pSU8+Xxfsmoc651q6tqKCU7loOD/QoNPqo9o/p1jjoc55xLqroSwQagp5m9ET9R0qnAxqRGFbG31+9k1cbd3D55KJI3EjvnWre62gh+C+yuYfqucF6rNSu/gHbZGXxxeO+oQ3HOuaSrKxH0NLPl1SeG0/onLaKIbd97gGeWbeSSkb3JzU7oNgvnnGvR6koEneqYl9PIcTQbjy8uorQi5t1NO+dSRl2JYLGk66pPlHQtsCR5IUWnImbMXljAuIFdGNSzfdThOOdck6ir7uPbwFxJUzl44B8NZAGTkxxXJOa/t4WiHfv5wYXHRx2Kc841mbo6ndsMfFbSWUBlj2vPmNlLTRJZBGbmF9CjfTafG9Iz6lCcc67J1HtnsZm9bGb/E74alAQknS/pPUlrJN1Sw/ybJa2StEzSPyVFVjFfsH0fr7y/lSlj+pKZ3tDn9TjnXMuVtCOepHTgHuACYAgwRdKQasXeAUab2TDgCeBXyYqnPo8sXE+axJQxfaMKwTnnIpHMn75jgDVm9pGZlQKPApPiC4RnG8XhaD6Ql8R4alVSVsFjiwuZMKQnR3VsE0UIzjkXmWQmgt5AYdx4UTitNl8F/p7EeGr1zLKN7Cwu4yq/ZNQ5l4KaxR1TkqYRXJF0Ri3zryd4bCZ9+zZ+1c3M/AKO6Z7L+GO6NvqynXOuuUvmGcEGIL4j/7xw2iEknQv8ELjIzA5Unw9gZvea2WgzG929e/dGDXJ50S6WFu7kqnH9vF8h51xKSmYieAsYJGmApCzgSmBefAFJI4A/ESSBLUmMpVYz89eRk5nOxaMiaZ5wzrnIJS0RmFk5cAPwPLAaeNzMVkr6qaTKZxncCbQD/iJpqaR5tSwuKXYVl/HU0o/54ojedGiT2ZSrds65ZiOpbQRm9izwbLVpP44bPjeZ66/PX5YUcqA8xrRxfsmocy51peydU7GYMXvhekb168wJvTpGHY5zzkUmZRPBGx9uY+22fX7JqHMu5aVsInh4QQFdc7O44MSjog7FOecilZKJYMPO/fxz9WauOLkP2RnpUYfjnHORSslEMGfhegz40lhvJHbOuZRLBKXlMR59az3nDO5BXue2UYfjnHORS7lE8NzKTWzbW+qPonTOuVDKJYJZCwro26Utpw9q3K4qnHOupUqpRPDupt0sWvcJ08b1JS3N+xVyzjlIsUQwc0EB2RlpXDaqT/2FnXMuRaRMInjtg608vriQzx7Tlc65WVGH45xzzUZKJIIlBTu4+sG3KKsw3vhwO0sKdkQdknPONRspkQjyP9pOzAyAiooY+R9tjzgi55xrPlIiEYwb2JWsjDTSBZkZaYwb6E8ic865Ss3iUZXJNqpfZ2ZfO478j7YzbmBXRvXrHHVIzjnXbKREIoAgGXgCcM65T0uJqiHnnHO180TgnHMpzhOBc86lOE8EzjmX4jwROOdcivNE4JxzKc4TgXPOpThPBM45l+JSJxEULoLX7gr+uvr59moY314N59usYZK4vVLjzuLCRfDgBRArB6VBz6GQ3SHqqJqvA7th8wqwmG+vRPj2ajjfZg1Ttb0MMtrAV+ZBnzGNtvjUOCNY9xrEKoJhi0HJrmjjae5KdgXbCXx7JcK3V8P5NmuYqu1lUFEaHNMaUWqcEfQ/LciiFaWQngWX3N+o2bTVKVwED13k2ytRvr0azrdZw1TfXv1Pa9TFy8J++luK0aNH2+LFixv+xsJFQRbtf5rvcInw7dUwvr0azrdZwxzh9pK0xMxG1zgvZRKBc86lsLoSQWq0ETjnnKuVJwLnnEtxngiccy7FeSJwzrkU54nAOedSnCcC55xLcS3u8lFJW4GCw3x7N2BbI4bTWDyuhvG4Gq65xuZxNcyRxNXPzLrXNKPFJYIjIWlxbdfRRsnjahiPq+Gaa2weV8MkKy6vGnLOuRTnicA551JcqiWCe6MOoBYeV8N4XA3XXGPzuBomKXGlVBuBc865T0u1MwLnnHPVeCJwzrkU1yoSgaQ+kl6WtErSSknfCqd3kfQPSR+EfzuH0y8Jy70mqWs47RhJj0X5OZqapHRJ70j6Wzg+QNJCSWskPSYpK5x+o6QVkp6Nm3aqpP+OMv6mJKmTpCckvStptaTxvn/VTtJ3wm2wQtIcSW18/zqUpAckbZG0Im5abfuUJN0dbrtlkkaG04+TtCScNj6cliHpRUltEw7GzFr8CzgaGBkOtwfeB4YAvwJuCaffAvwyHJ4PtAWmATeG0+YAg6L+LE283W4GHgH+Fo4/DlwZDv8R+EY4nE/wo+FHwERAwPNAl6g/QxNuq4eAa8PhLKCT71+1bqvewFogJ26/mu7716e20+nASGBF3LTa9qkLgb+H22YcsDCc/hvgVCAP+Gs47UZgekNiaRVnBGa20czeDof3AKsJdsZJBP/AhH+/GA7HgGyCf9YySacBm8zsg6aMO0qS8oDPA/eH4wLOBp4Ii8RvLwGZhNuL4AD3dzP7pAlDjoykjgT/tH8GMLNSM9uJ7191yQByJGUQbIeN+P51CDN7Faj+GWvbpyYBD1sgH+gk6WiC7dWWg/taJ4Jk+nBDYml1zyyW1B8YASwEeprZxnDWJqBnOPxfwIvAxwQ73V+AK5s20sj9Fvh/BGdQAF2BnWZWHo4XESRTgN8T/GpbCbwBPAWc12SRRm8AsBV4UNJJwBLgW/j+VSMz2yDp18B6YD/wAsE28/2rfrXtU72BwrhyldvvHoKDfjbwNeA/gV+YWawhK20VZwSVJLUD/gp828x2x8+z4JzJwuF/mNkoM5tIkGmfBY4N64Dva1DdWgsk6QvAFjNbkkh5M5tpZiPMbBrwHeBu4IJwe/23pFa1H9Ugg+AU/n/NbASwj+C0vYrvXweF9dqTCBJoLyAXOL+28r5/1Sx+n6qjzHozO9PMxgPFBFVEqyXNDNthjk1kXa1mA0vKJEgCs83s/8LJm8PTJ8K/W6q9py1B3eU9wE+ArwCvA1ObKOyonAJcJGkd8CjBKfvvCE43K88S84AN8W+S1AsYY2ZPAv8OXAHsBM5pkqijUwQUmdnCcPwJgsTg+1fNzgXWmtlWMysD/o9gn/P9q3617VMbgD5x5T61/YDbCdpZbiKo8v1/wK2JrLRVJIKwfvvPwGoz+03crHkE/3yEf5+q9tbvAXeHO2sOQfaNEdS3tVpm9n0zyzOz/gRVFi+Z2VTgZeDSsFhN2+tnwI/D4VTaXpuAQknHhZPOAVbh+1dt1gPjJLUN/zcrt5fvX/WrbZ+aB3w5vHpoHLArrgoJSWcAH4ftUG0Jtlvi2y7qlvPGeBG0mhuwDFgavi4kqPf+J/ABQZ1tl7j39AKeiRu/jIN1lN2j/kxNuO3O5OBVQwOBRcAagnrt7LhyI4A/x41/O9xez8WXa60vYDiwONzHngQ6+/5V5/b6CfAusAKYSVCH7fvXodtoDkEjehnBWedXa9unCBrU7wE+BJYDo+OWI+AfcWWPB94O99VTEonFu5hwzrkU1yqqhpxzzh0+TwTOOZfiPBE451yK80TgnHMpzhOBc86lOE8ErZSkCklLw14d/xLF3aySzpT02aZeb9z6Twt7wFwqKSeC9Y+WdHcC5d5singam6S9zSCGGZIurb+kq4sngtZrv5kNN7OhQCnw9UTeFHfnZ2M4E4gsERDcwftf4XbYX1OB8AadpPwfmNliM7spgXJRbiPnPBGkiNeAz0jKDftAX6TgOQSTACRNlzRP0kvAPyW1k/SgpOVhP+eXhOUmSFog6e3wLKNdOH2dpJ+E05dLGhx2/vd14DvhL/LTJE1U0B/9O2F/6T3D93dX0Pf6Skn3SyqQ1C2cNy2Md6mkP0lKr/7hJJ0TLnN5+PmyJV0LXA78TNLsauX7S3pP0sMENzz1kfQ9SW+Fn/cnceXeDX91vi9ptqRzJb2hoL/4MWG5MeF2eUfSm5V3IIdnRJXPergtjG2+pI8k3RQXz9648vN18LkHs8M7c5F0YThtiYJ+6f9Ww3Y4IW5bLZM0KJz+ZPi+lZKuj1+vpDvD6S+Gn6Myvovi9o2nwukfSKqxy4Katl8NZfbGDV8qaUY4fJmCM9d/SXo1nJYexla5zK+F0yXp9+H39yLQo6Z1uQaK+u46fyXtrsW94d8MgtvUvwH8ApgWTu9E8NyGXIL+cIo4eGfiL4Hfxi2rM9ANeBXIDaf9B/DjcHgdB/vd/zfg/nD4NuC71ZZTeRPjtcBd4fDvge+Hw+cT3CXejeAOyaeBzHDeH4AvV/ucbQh6ZTw2HH+YoNNBgBnApTVsm/4Et9+PC8cnEDwUXAQ/jv5G0O10f6AcODGcvgR4ICw3CXgyfH8HICMcPpeD/cKfycG7tm8D3iS4w7YbsD3uc+2NK7+LoB+ZNGABwV3zlZ9xQFhuTuVyq32u/wGmhsNZHHweQOX3mkOQ+LqG4wZcEA7PJeglNBM4CVgaTp9OcPdr17j3j64Wd43br7Z9Mhy+FJgRDi8Helful+Hf64EfhcPZBHd1DwAuJriLNp3g7u2dNX3H/mrYq9V1Q+2q5EhaGg6/RtAX05sEnc19N5zeBugbDv/DDvb/fi5x3Sab2Q4FPZYOAd4If6RmERyoKlV29LeE4J+1JnnAYwo608oieHgJBAe7yeG6npO0I5x+DjAKeCtcZw7VOnYDjiPo4Oz9cPwh4JsE3WzXpcCCft0hOJBNAN4Jx9sBgwj6zFlrZssBJK0E/mlmJmk5QaIA6Ag8FP4CN4KDaU2eMbMDwAFJWwi6GC6qVmaRmRWF61sarmMv8JGZVW6vOQQHyuoWAD9U8KyJ/7ODzz+4SdLkcLhP+Nm2E1QZPhdOXw4cMLOyap8Ngn1jexjT/xF8X4vj5te2/V6tZTtU9wYwQ9LjHNyPJgDDdLD+v2O4zNOBOWZWAXys4CzWHSFPBK3XfjMbHj8hrGa4xMzeqzZ9LEHXynURwQFhSi3zD4R/K6h9v/of4DdmNk/SmQS/kutb50Nm9v16yh2O+M8rgraEPx2y8qB660DcpFjceIyDn/NnwMtmNjl8z/xa1hm/rNq2UyJlamRmj0haSPDAoWfD6pQYQWIfb2bFkuYT/AAAKLPwJzdxn83MYjq0rah6PzTVx2vcfjWFGDfcpmqi2dfDffDzwBJJo8Jl3mhmzx+yIunCetbhDoO3EaSW54Eb4+qdR9RS7h8Ev6oJy3UmeHDIKZI+E07LVf19ne/h4INvIPhVV9l17lfipr9BUJ+PpAkEVUgQdL51qaQe4bwukvpVW8d7QP/KuICrgFfqiau654FrdLDNo3flOhMU/7mmN3DdiXgPGBgmGQi6Z/4USQMJzhzuJqgOHBbGtiNMAoMJHnPYUJ8Lt30OwROz3qg2P9Htt1nS8Qoa5yvPUJB0jJktNLMfEzwAqE+4zG8o6F4eScdKyiU4y7gibEM4GjjrMD6Pq8YTQWr5GUG1xbKwmuNntZT7OdC5sgEPOMvMthIc5OZIWkZQDTG4nvU9DUwOGy9PIzgD+IukJcC2uHI/ASYoeIj3ZQRPZtpjZqsI+ld/IVznPwieT13FzEqAq8PlLif4ZfvH+jfFIct4geDZzQvCZTzBoQmsPr8C/kvSOyThLNuCK57+DXgu3HZ7CNoSqrscWBFWKQ0laC95DsiQtBq4gyChN9Qigmd9LCNo/4ivFmrI9ruFoP3gTYJ2h0p3KmjoXxHO+xdBf/qrgLfD6X8i2LZzCXrmXBV+vvjqSXeYvPdRFzlJ2UCFmZVLGk/wJLDhEYfVrEhqZ2Z7w7O5e4APzOy/m2C90wkah29I9rpcdLyNwDUHfYHHwyqDUuC6iONpjq6T9BWCRvZ3CH4hO9co/IzAOedSnLcROOdcivNE4JxzKc4TgXPOpThPBM45l+I8ETjnXIr7/3ijJGWhNA62AAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Here we notice the effects of changing all labels in the test sample to a fixed class\n", + "ref_label_drift_out = da.ref_label_drift()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test sample covariate drift\n", + "\n", + "In this test we look for evidence of the test sample covariates drifting from the reference sample.\n", + "The full reference sample is used in modelling the distribution against which we evaluate the test sample.\n", + "\n", + "In the multivariate case, Bonferroni correction for the p-value is used in order to mitigate eventual false positives in the drift diagnosis.\n", + "If any feature from the covariates accuses drift, we raise a warning for the full sample.\n", + "In the case that no drift was detected, but there were failing tests due to other data issues (e.g. different support on categoricals), we will raise your attention with a lower priority warning and with recommendations (usually this is due to too small test samples)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StatisticStatistic Valuep-valueVerdict
alcoholKolmogorov-Smirnov0.4875160.0Drift
malic_acidKolmogorov-Smirnov0.1345190.59822OK
ashKolmogorov-Smirnov0.1398250.549066OK
alcalinity_of_ashKolmogorov-Smirnov0.1254680.680859OK
magnesiumKolmogorov-Smirnov0.0873910.957606OK
\n", + "
" + ], + "text/plain": [ + " Statistic Statistic Value p-value Verdict\n", + "alcohol Kolmogorov-Smirnov 0.487516 0.0 Drift\n", + "malic_acid Kolmogorov-Smirnov 0.134519 0.59822 OK\n", + "ash Kolmogorov-Smirnov 0.139825 0.549066 OK\n", + "alcalinity_of_ash Kolmogorov-Smirnov 0.125468 0.680859 OK\n", + "magnesium Kolmogorov-Smirnov 0.087391 0.957606 OK" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# As expected the corrupted alcohol feature is detected after the corruption step, a small boost of 0.8 vol(%) triggered this alarm\n", + "sample_cov_drift_out = da.sample_covariate_drift()\n", + "sample_cov_drift_out.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test sample label drift\n", + "\n", + "In this test we look for evidence of the test sample label drifting from the reference sample.\n", + "The same strategy from the previous step applies.\n", + "If the label accuses drift, we raise a warning for the test sample.\n", + "In the case that no drift was detected, but there was a failing test due to other data issues (e.g. different support on categoricals), we will raise your attention with a lower priority warning and a recommendation (usually this is due to too small test samples)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Statistic Chi-Squared\n", + "Statistic Value 11.032223\n", + "p-value 0.011553\n", + "Verdict Drift\n", + "dtype: object" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_label_drift_out = da.sample_label_drift()\n", + "sample_label_drift_out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test concept drift\n", + "\n", + "In this test we look for evidence of concept drift.\n", + "If the provided model produces output for the test sample which appears not to be representative of the output generated from the reference sample, concept drift is detected." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[CONCEPT DRIFT] Concept drift was not detected between the reference and the test samples.\n" + ] + }, + { + "data": { + "text/plain": [ + "Statistic Chi-Squared\n", + "Statistic Value 0.538533\n", + "p-value 0.910348\n", + "Verdict OK\n", + "dtype: object" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sample_concept_drift_out = da.sample_concept_drift()\n", + "sample_concept_drift_out" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "cdc2bce73c2a9ac283f602628cabf735dbe06c4ee87a7849fc5f3d1177c8f304" + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit ('.venv': venv)", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "metadata": { + "interpreter": { + "hash": "cdc2bce73c2a9ac283f602628cabf735dbe06c4ee87a7849fc5f3d1177c8f304" + } + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file