diff --git a/src/ydata_quality/__init__.py b/src/ydata_quality/__init__.py index 886255b2..f5a28591 100644 --- a/src/ydata_quality/__init__.py +++ b/src/ydata_quality/__init__.py @@ -1,5 +1,11 @@ """ YData open-source lib for Data Quality. """ +from ydata_quality.core.data_quality import DataQuality from .version import __version__ + + +__all__ = [ + "DataQuality" +] diff --git a/src/ydata_quality/core/__init__.py b/src/ydata_quality/core/__init__.py index b09e9065..44065e11 100644 --- a/src/ydata_quality/core/__init__.py +++ b/src/ydata_quality/core/__init__.py @@ -2,10 +2,10 @@ Core functionality for Data Quality analysis. """ -from ydata_quality.core.warnings import QualityWarning from ydata_quality.core.engine import QualityEngine +from ydata_quality.core.warnings import QualityWarning __all__ = [ - "QualityWarning", - "QualityEngine" + "QualityEngine", + "QualityWarning" ] diff --git a/src/ydata_quality/core/data_quality.py b/src/ydata_quality/core/data_quality.py index 680c2a5c..5ec5fe11 100644 --- a/src/ydata_quality/core/data_quality.py +++ b/src/ydata_quality/core/data_quality.py @@ -1,20 +1,98 @@ """ Implementation of main class for Data Quality checks. """ +from typing import List, Union, Optional, Callable import pandas as pd +from ydata_quality.core.warnings import QualityWarning, Priority +from ydata_quality.duplicates import DuplicateChecker +from ydata_quality.labelling import LabelInspector +from ydata_quality.missings import MissingsProfiler +from ydata_quality.valued_missing_values import VMVIdentifier +from ydata_quality.drift import DriftAnalyser + class DataQuality: - "DataQuality gathers the multiple data quality engines." + "DataQuality contains the multiple data quality engines." + + def __init__(self, + df: pd.DataFrame, + label: str = None, + entities: List[Union[str, List[str]]] = [], + vmv_extensions: Optional[list]=[], + sample: Optional[pd.DataFrame] = None, + model: Callable = None + ): + """ + Engines: + - Duplicates + - Missing Values + - Labelling + - Valued Missing Values + - Drift Analysis - def __init__(self, df: pd.DataFrame): + Args: + df (pd.DataFrame): reference DataFrame used to run the DataQuality analysis. + label (str, optional): [MISSINGS, LABELLING, DRIFT ANALYSIS] target feature to be predicted. + If not specified, LABELLING is skipped. + entities: [DUPLICATES] entities relevant for duplicate analysis. + vmv_extensions: [VALUED MISSING VALUES] A list of user provided valued missing values to append to defaults. + sample: [DRIFT ANALYSIS] data against which drift is tested. + model: [DRIFT ANALYSIS] model wrapped by ModelWrapper used to test concept drift. + """ self.df = df + self._warnings = set() + self._engines = { # Default list of engines + 'duplicates': DuplicateChecker(df=df, entities=entities), + 'missings': MissingsProfiler(df=df, target=label), + 'valued-missing-values': VMVIdentifier(df=df, vmv_extensions=vmv_extensions), + 'drift-analysis': DriftAnalyser(ref=df, sample=sample, label=label, model=model) + } + + # Engines based on mandatory arguments + if label is not None: + self._engines['labelling'] = LabelInspector(df=df, label=label) + else: + print('Label is not defined. Skipping LABELLING engine.') + + + @property + def warnings(self): + "Set of warnings generated by individual QualityEngines." + return self._warnings + + def get_warnings(self, + category: Optional[str] = None, + test: Optional[str] = None, + priority: Optional[Priority] = None) -> List[QualityWarning]: + "Retrieves warnings filtered by their properties." + filtered = list(self.warnings) # convert original set + filtered = [w for w in filtered if w.category == category] if category else filtered + filtered = [w for w in filtered if w.test == test] if test else filtered + filtered = [w for w in filtered if w.priority == Priority(priority)] if priority else filtered + filtered.sort() # sort by priority + return filtered + + @property + def engines(self): + "Dictionary of instantiated engines to run data quality analysis." + return self._engines + + def __store_warnings(self): + "Appends all warnings from individiual engines into warnings of DataQuality main class." + for engine in self.engines.values(): + self._warnings = self._warnings.union(set(engine.get_warnings())) def evaluate(self): "Runs all the individual data quality checks and aggregates the results." - raise NotImplementedError - + results = {name: engine.evaluate() for name, engine in self.engines.items()} + self.__store_warnings() + return results def report(self): - "Returns a full list of warnings retrieved during the Data Quality checks." - raise NotImplementedError + "Prints a report containing all the warnings detected during the data quality analysis." + # TODO: Provide a count of warnings by priority + self.__store_warnings() # fetch all warnings from the engines + for warn in self.get_warnings(): + print(warn) + diff --git a/src/ydata_quality/core/engine.py b/src/ydata_quality/core/engine.py index a3337410..087aa722 100644 --- a/src/ydata_quality/core/engine.py +++ b/src/ydata_quality/core/engine.py @@ -5,8 +5,7 @@ from typing import Optional import pandas as pd -from ydata_quality.core import QualityWarning -from ydata_quality.core.warnings import Priority +from ydata_quality.core.warnings import QualityWarning, Priority from ydata_quality.utils.modelling import infer_dtypes class QualityEngine(ABC): @@ -76,11 +75,12 @@ def get_warnings(self, test: Optional[str] = None, priority: Optional[Priority] = None): "Retrieves warnings filtered by their properties." - filtered = self.warnings # original set + filtered = list(self.warnings) # convert original set filtered = [w for w in filtered if w.category == category] if category else filtered filtered = [w for w in filtered if w.test == test] if test else filtered filtered = [w for w in filtered if w.priority == Priority(priority)] if priority else filtered - return set(filtered) + filtered.sort() # sort by priority + return filtered @property def tests(self): @@ -97,4 +97,11 @@ def report(self): def evaluate(self): "Runs all the indidividual tests available within the same suite. Returns a dict of (name: results)." self._warnings = set() # reset the warnings to avoid duplicates - return {test: getattr(self, test)() for test in self.tests} + results = {} + for test in self.tests: + try: # if anything fails + results[test] = getattr(self, test)() + except Exception as exc: # print a Warning and log the message + print(f'WARNING: Skipping test {test} due to failure during computation.') + results[test] = "[ERROR] Test failed to compute. Original exception: "+f"{exc}" + return results diff --git a/src/ydata_quality/drift/engine.py b/src/ydata_quality/drift/engine.py index ddac29de..d0a31828 100644 --- a/src/ydata_quality/drift/engine.py +++ b/src/ydata_quality/drift/engine.py @@ -9,7 +9,6 @@ from scipy.stats import ks_2samp from scipy.stats._continuous_distns import chi2_gen from ydata_quality.core import QualityEngine, QualityWarning -from ydata_quality.utils.modelling import infer_dtypes class ModelWrapper: diff --git a/src/ydata_quality/valued_missing_values/engine.py b/src/ydata_quality/valued_missing_values/engine.py index 2bb66f84..1039a423 100644 --- a/src/ydata_quality/valued_missing_values/engine.py +++ b/src/ydata_quality/valued_missing_values/engine.py @@ -24,7 +24,7 @@ def __init__(self, df: pd.DataFrame, vmv_extensions: Optional[list]=[]): self._flatline_index = {} self.__default_index_name = '__index' self.vmvs = vmv_extensions - + @property def default_vmvs(self): """Returns the default list of Valued Missing Values. @@ -61,7 +61,7 @@ def __get_flatline_index(self, column_name: str, th: Optional[int] = 1): if column_name == self.__default_index_name: df[self.__default_index_name] = df.index # Index now in columns to be processed next column = df[column_name] - column.fillna('__filled') # So NaN values are considered + column.fillna('__filled') # So NaN values are considered sequence_indexes = column.ne(column.shift()).cumsum() # Everytime shifted value is different from previous a new sequence starts sequence_groups = column.index.to_series().groupby(sequence_indexes) # Group series indexes by sequence indexes data = {'length': sequence_groups.count().values, @@ -102,7 +102,6 @@ def predefined_valued_missing_values(self, skip: list=[], short: bool = True): Raises warning based on the existence of these values. VMVs of string type are case insensitive during search. Returns a DataFrame with count distribution for each predefined type over each column. - The result DataFrame will ommit any Arguments: skip: List of columns that will not be target of search for vmvs. Pass '__index' in skip to skip looking for flatlines at the index. @@ -122,7 +121,7 @@ def predefined_valued_missing_values(self, skip: list=[], short: bool = True): vmvs.drop(no_vmv_rows, inplace=True) if vmvs.empty: print("[PREDEFINED VALUED MISSING VALUES] No predefined vmvs from the set {} were found in the dataset.".format( - self.predefined_valued_missing_values + self.vmvs )) else: total_vmvs = vmvs.sum().sum() @@ -131,4 +130,4 @@ def predefined_valued_missing_values(self, skip: list=[], short: bool = True): test='Predefined Valued Missing Values', category='Valued Missing Values', priority=2, data=vmvs, description=f"Found {total_vmvs} vmvs in the dataset." )) - return vmvs \ No newline at end of file + return vmvs diff --git a/tutorials/data_quality.ipynb b/tutorials/data_quality.ipynb new file mode 100644 index 00000000..c129332f --- /dev/null +++ b/tutorials/data_quality.ipynb @@ -0,0 +1,231 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# YData Quality - DataQuality Tutorial\n", + "Time-to-Value: 4 minutes\n", + "\n", + "This notebook provides a tutorial to run the `ydata_quality.DataQuality` main class that aggregates all the individual data quality engines, each focused on a main topic of data quality (e.g. duplicates, missing values).\n", + "\n", + "**Structure:**\n", + "\n", + "1. Load dataset\n", + "2. Distort dataset\n", + "3. Instantiate the Data Quality engine\n", + "4. Run the quality checks\n", + "5. Assess the warnings\n", + "6. (Extra) Detailed overview" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import statsmodels.api as sm\n", + "import pandas as pd\n", + "import numpy as np\n", + "from ydata_quality import DataQuality" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the example dataset\n", + "We will use a dataset available from the statsmodels package." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "df_original = sm.datasets.get_rdataset('Guerry', 'HistData').data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Distort the original dataset\n", + "Apply transformations to highlight the data quality functionalities." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def apply_quality_transformations(df: pd.DataFrame):\n", + " \"Force data quality issues to highlight functionality.\"\n", + " # Copy to guarantee the original is kept intact\n", + " df = df.copy()\n", + "\n", + " # Duplicates\n", + " df = df.append(df[:20], ignore_index=True)\n", + " df[\"dept2\"] = df[\"dept\"]\n", + " return df\n", + "\n", + "df = apply_quality_transformations(df_original)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the main engine\n", + "The DataQuality class aggregates all the individual data quality engines, each focused on a main topic of data quality (e.g. duplicates, missing values). To create a DataQuality object, you provide:\n", + "- df: target DataFrame, for which we will run the test suite\n", + "- target (optional): target feature to be predicted in a supervised learning context\n", + "- entities (optional): list of feature names for which checking duplicates after grouping-by is applicable.\n", + "- vmv_extensions (optional): list of valued missing value to append to the defaults." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "VMV_EXTENSIONS = ['a_custom_VMV', 'another_VMV', 999999999, '!', '', 'UNKNOWN']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "dq = DataQuality(df=df, label='Pop1831', vmv_extensions=VMV_EXTENSIONS)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Full Evaluation\n", + "The easiest way to assess the data quality analysis is to run `.evaluate()` which returns a list of warnings for each quality check. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[ENTITY DUPLICATES] There are no entities defined to run the analysis. Skipping the test.\n", + "WARNING: Skipping test predict_missings due to failure during computation.\n", + "[PREDEFINED VALUED MISSING VALUES] No predefined vmvs from the set {'', '(blank)', 'na', 'unknown', 'unk', 'n/a', 'another_vmv', 'a_custom_vmv', '?', '!', 999999999} were found in the dataset.\n", + "WARNING: Skipping test ref_covariate_drift due to failure during computation.\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABFkUlEQVR4nO3dd3gVZfbA8e9Jh9Cr9KJ0pCUUK6iIWIB1BYEFlSKoq6Kirvpbe1m7qyi7YqGqgIoI9o7oKiWBSEeQGqSGFgiQdn5/zCRcQsoEcnOT3PN5nvvkzsw7M2fmTubMvDPzjqgqxhhjgldIoAMwxhgTWJYIjDEmyFkiMMaYIGeJwBhjgpwlAmOMCXKWCIwxJshZIjCISG0RmS8iySLyYqDjKa1EZJiI/HyK484TkRs9lt0kIj1PcT6nPG5ROZ31ZPwjLNABmKIhIpuA2kAGcAj4ErhNVQ95GH00sAeopPZgiTFBx84IypY+qloB6AB0BB7wOF4jYNWpJAERsYMJY0o5SwRlkKruAL7CSQgAiEg3EflFRPaLyG8i0sPtPxm4AfiHiBwSkZ4iEiIi94vIHyKSJCLvi0g1t3xjEVERGSkiW4Dv3f4jRGS1iOwTka9EpJHPvFVEbhaRde78x4uI+Awf5Y6bLCKrRKST27+uiMwSkd0islFExuS1zCJyhTtusohsE5F73P5VReRTdxr73O/1fcabJyJPuuvmkIh8IiLVReRdETkoIotFpHGOZRkjIhtEZI+IPC8iuf4fiUhLEflGRPaKyFoRudbL7yciZ4rI9+663+PGUiVHsc7u8u4TkUkiEuUz/lUikuCu619EpJ3H+U4WkdfdmJNF5Eff3zFH2f+KyAs5+s0RkbHu96ztJ+s3vTqP6WRtT2E+/U6oJstr2xLHv0Vkl/tbLReRtl6W1eSgqvYpAx9gE9DT/V4fWA684nbXA5KAK3CS/6Vud013+GTgSZ9p3QEscKcTCUwAprvDGgMKTAWigXJAP2A90AqnuvFB4Bef6SnwKVAFaAjsBnq7wwYA24DOgABn4ZyhhADxwMNABNAU2ABclsfybwcucL9XBTq536sD1wDlgYrAB8DHPuPNc2M/E6gMrAJ+B3q6yzIVmJRjWX4AqrnL8jtwoztsGPCz+z0a2AoMd6fTEaf6rXUe8c/zmc5Z7m8UCdQE5gMv5/itVwAN3Dj+l/X7ufPZBXQFQnGS/CYgMud2kksMk4Fk4EJ33q9kLU8uZS90l0981vkRoK7P71rX/R0HAoeBOrmsp8buOg3LY13kuW0Bl7nbSBWcbadV1jzsU8j9R6ADsE8R/ZDOP/gh9x9Zge+AKu6w+4BpOcp/Bdzgfp/MiYlgNXCJT3cdIM39R8z6x23qM/wLYKRPdwiQAjRyuxU432f4+8D9PnHckcvydAW25Oj3AD475RzDtgA34VznyG89dQD2+XTPA/7p0/0i8IVPdx8gwadbcZOY2/134Dv3u+8ObiDwU455TwAeySOu7J1fLsP+AizN8Vvf7NN9BfCH+/2/wBM5xl8LdPcZN79EMMOnuwLONacGuZQVd51f6HaPAr7PZ70nAP1yWU9Z21NeiSDPbQu4GCcRdwNC/P0/VpY/VjVUtvxFVSsCPYCWQA23fyNggFtVsF9E9gPn4+zgc9MImO1TdjXODqG2T5mtOcq/4lN+L86Oop5PmR0+31NwdjLgHNX+kUcMdXPE/H85YvB1Dc4OcbNbpXEOgIiUF5EJIrJZRA7iHF1XEZFQn3F3+nw/kkt3BU7ku+ybcY58c4u/a474hwBn5BF/NnHu4prhVnEdBN7h+G9ZUAyNgLtzzLdBHjHmJnu66txosBfnd/g/t+rskIi8rs5eeQYw2C3+N+Bdn2W43qd6aj/QNpdl8CLPbUtVvwdeA8YDu0TkDRGpdArzCHqWCMogVf0R5+guqw53K84ZQRWfT7SqPpPHJLYCl+coH6Wq23xnk6P8TTnKl1PVXzyEuxWnWia3/htzTLOiql6RxzIvVtV+QC3gY5yzDoC7gRZAV1WthFOlAc7O5FQ18PneEPgzj/h/zBF/BVW9xcP0/4Wzfs92Yx6aS7x5xbAVeCrHfMur6nQvC+Y7XRGpgFP19Keq/suNv4Kq3uwWmQ70d+vsuwKz3PEaAW8CtwHVVbUKTlVWbuv8sPu3vE8/32SZ77alquNUNQZoDTQH7vW4nMaHJYKy62XgUhFpj3NE2UdELhORUBGJEpEevhdNc3gdeMrnolxNEemXz7xeBx4QkTZu+coiMsBjnG8B94hIjHvx7yx3vouAZBG5T0TKuXG3FZHOOScgIhEiMkREKqtqGnAQyHQHV8Q5qt8vzgXvRzzGlZ97xbkI3QDnesrMXMp8CjQXketEJNz9dBaRVh6mXxGnmu+AiNQj953brSJS312mf/rE8CZws4h0dddntIhcKSIVPS7bFSJyvohEAE8AC1R1a24FVXUpznWPt4CvVHW/OygaJ5HtBhCR4ThnBLlNYzfONaKh7m88ghMPDPLcttz12VVEwnESylGO/+6mECwRlFHuP9hU4GH3H7kfTtXKbpyjrHvJ+/d/BZgLfC0iyTgXjrvmM6/ZwLPADLcqYwVwucc4PwCeAt7Dub7xMVBNVTOAq3Dq9DdyfIdTOY9JXQdscud/M041DDgJsZw7/gKc5ytO1xyci5QJwGfA2zkLqGoy0AsYhHO0vgNnHUV6mP5jQCfggDv9j3Ip8x7wNc4F9D+AJ935xuHU178G7MO50DrM43JlTfcRnCqYGJyzkYLK93T/4sawCuday6841Wxn41zQzssonO0xCWgDZJ9JFrBtVcJJfPtwqseSgOc9LKPJIeuKvzHGAxFRoJmqrg90LEVNnFuJE1X1wUDHYoqXnREYY0yQs0RgjDFBzqqGjDEmyNkZgTHGBLlS12BYjRo1tHHjxoEOwxhjSpX4+Pg9qlozt2GlLhE0btyYuLi4QIdhjDGliohszmuYVQ0ZY0yQs0RgjDFBzhKBMcYEuVJ3jSA3aWlpJCYmcvTo0UCHYvwoKiqK+vXrEx4eHuhQjClTykQiSExMpGLFijRu3BiR02lU0pRUqkpSUhKJiYk0adIk0OEYU6aUiaqho0ePUr16dUsCZZiIUL16dTvrM8YPykQiACwJBAH7jcuW+M37GP/DeuI37wt0KEGvTFQNGWNKl4Ubkhj69kIyMpWIsBDevbEbMY2qBjqsoFVmzggCrUKFnG8zzNujjz7KCy+8UHDBU5z+6Zo8eTK33XZbsc3PBJ8Xvl5LWoaSqZCWnsmCDUmBDimoWSIwxhSrn9btZvGmfYS4NX0KxNrZQEAFbSIojvrJTz75hK5du9KxY0d69uzJzp3H34n+22+/cc4559CsWTPefPPN7P7PP/88nTt3pl27djzySP5vVdy0aRMtW7ZkyJAhtGrViv79+5OSknJCmczMTBo3bsz+/fuz+zVr1oydO3fmG1+WYcOG8eGHH2Z3+56ZFCZWYwCSDh1j7Pu/0axWBd4Z2ZXL2tQmU+HdhVvIzLSWkAOlzF0jeOyTlaz682C+ZZKPprFmRzKZCiECLc+oSMWovO9Nb123Eo/0aVPoWM4//3wWLFiAiPDWW2/x3HPP8eKLLwKwbNkyFixYwOHDh+nYsSNXXnklK1asYN26dSxatAhVpW/fvsyfP58LL7wwz3msXbuWt99+m/POO48RI0bwn//8h3vuuSd7eEhICP369WP27NkMHz6chQsX0qhRI2rXrp1vfAX5+uuvCx2rCW6qyj8+XMaBI2lMHdGFVnUqce5ZNfjPvPU89+VaqkVH8Eif1nZTQACUuUTgxcGj6WQdfGSq051fIjhViYmJDBw4kO3bt5OamnrC/e/9+vWjXLlylCtXjosuuohFixbx888/8/XXX9OxY0cADh06xLp16/LduTZo0IDzzjsPgKFDhzJu3LgTEgHAwIEDefzxxxk+fDgzZsxg4MCBBcZXkK+//rrQsZrgNm3BZr5bs4tH+rSmVZ1K2f1v6X4mew+l8tbPG6keHcHtlzQLYJTBqcwlAi9H7vGb9zHkrQWkpWcSHhbCK4M6+uWOhdtvv52xY8fSt29f5s2bx6OPPpo9LOdRj4igqjzwwAPcdNNNnueR23QWLlyYPY3HH3+cPn36sH79enbv3s3HH3/Mgw8+WGB8WcLCwsjMzAScaqbU1FSAU4rVBK+1O5J58rPV9GhRk2HnNj5hmIjwf1e0Ym9KKi9+8ztVoyMY2q1RYAINUkF5jSCmUVXevbEbY3u18OttawcOHKBevXoATJky5YRhc+bM4ejRoyQlJTFv3jw6d+7MZZddxsSJEzl06BAA27ZtY9euXfnOY8uWLfz6668AvPfee5x//vl07dqVhIQEEhIS6Nu3LyLC1VdfzdixY2nVqhXVq1cvML4sjRs3Jj4+HoC5c+eSlpYGcEqxmuB0NC2DMdOXUikqnBcGtM+16ickRHj2mnZc0rIWD81ZwWfLtgcg0uBV5s4IvIppVLVIE0BKSgr169fP7h47diyPPvooAwYMoGrVqlx88cVs3Lgxe3i7du246KKL2LNnDw899BB169albt26rF69mnPOOQdwLsy+88471KpVK8/5tmjRgvHjxzNixAhat27NLbfckmu5gQMH0rlzZyZPnpzdL7/4sowaNYp+/frRvn17evfuTXR0NAC9evUqdKwmOD39+WrW7kxm8vDO1KgQmWe58NAQxg/pxHVvL+TOmUupXC6c85vVKMZIg1epe2dxbGys5nwxzerVq2nVqlWAIgqcTZs2cdVVV7FixYpAh1JsgvW3Lq2+W72TkVPiGHl+Ex66qrWncQ4cSWPghF/ZsjeF6aO60b5BFf8GGSREJF5VY3MbFpRVQ8YY/9t18Cj3friMVnUq8Y/eLTyPV7lcOFNHdKF6hQiGTVrE+l2H/BilAUsEpVrjxo2D6mzAlB6ZmcrdH/xGSmo6rw7uQGRYaKHGr1UpimkjuhIaEsL1by/kz/1H/BSpAUsExhg/ePvnjfy0bg8PX9WGs2pVPKVpNK4RzZQRnUk+ms71Exex73BqEUdpslgiMMYUqRXbDvDcV2u4rE1tBndpcFrTalO3Mm/eEMuWvSkMm7yYw8fSiyhK48sSgTGmyKSkpjNm+lKqR0fyzF/bFclTwt2aVue1wR1Znrifm9+JJzU9swgiNb78mghEpLeIrBWR9SJyfy7D/y0iCe7ndxHZ7894jDH+9fgnq9iYdJiXBrananREkU23V5szeOaadvy0bg93f/CbtUtUxPyWCEQkFBgPXA60BgaLyAn3j6nqXaraQVU7AK8CH/krntKoKJuD/te//pX9fdOmTbRt27ZIpuurKJvXztnYnSn5Pl++nRmLt3JL9zM598yiv///2tgGPHB5Sz757U8e/WQlpe3W95LMn2cEXYD1qrpBVVOBGUC/fMoPBqb7MZ6g5psIvEpPt/pY482f+49w/6xltK9fmbsube63+dzU/UxuurApU3/dzCvfrfPbfIKNPxNBPWCrT3ei2+8kItIIaAJ878d4TrR1Efz0ovP3NJX05qDvv/9+jhw5QocOHRgyZAgAGRkZjBo1ijZt2tCrVy+OHHFuz+vRowd33nknsbGxvPLKK8THx9O9e3diYmK47LLL2L7defR/3LhxtG7dmnbt2jFo0KDsea1atYoePXrQtGlTxo0bl93/pZdeom3btrRt25aXX375pBhVldtuu40WLVrQs2dPa66iFMnIVO6cmUBGpvLKoI6Eh/r30uP9l7dkQEx9Xv52HdN+3eTXeQWLktLExCDgQ1XNyG2giIwGRgM0bNgw/yl9cT/sWJ5/mWMHYecK0EyQEKjdFiIr5V3+jLPh8mfynWRJbg76mWee4bXXXiMhIQFwEte6deuYPn06b775Jtdeey2zZs1i6NChAKSmphIXF0daWhrdu3dnzpw51KxZk5kzZ/LPf/6TiRMn8swzz7Bx40YiIyNPSG5r1qzhhx9+IDk5mRYtWnDLLbewbNkyJk2axMKFC1FVunbtSvfu3bNbLgWYPXs2a9euZdWqVezcuZPWrVszYsQIT+vABNZ/561n0ca9vDigPY1rRPt9fiLC0389m30paTw8dyWVy0fQt31dv8+3LCswdYtIbRF5W0S+cLtbi8hID9PeBvjeO1bf7ZebQeRTLaSqb6hqrKrG1qxZ08OsC3D0gJMEwPl79MBpTzJnc9A///zzSWUGDhzIzJkzAU5qDvqyyy7j7LPP5vnnn2flypWe5+vbHHSnTp1Ys2YN69YVfMrcpEkTOnToAEBMTAybNm06IU5wktuKFSu49NJL6dChA08++SSJiYmA01bSkCFDeOeddwgLO348ceWVVxIZGUmNGjWoVasWO3fu5Oeff+bqq68mOjqaChUq8Ne//pWffvrphHjmz5/P4MGDCQ0NpW7dulx88cWe14EJnCVb9vHvb9fRt31d/top1xN+vwgLDeG1v3Wkc+Nq3P1+AvN/311s8y6LvJwRTAYmAf90u38HZgJvFzDeYqCZiDTBSQCDgL/lLCQiLYGqwK/eQi5AAUfugFMdNKUvZKRCaARc8xY06HJasy1tzUFHRh5v/Cs0NDS7agjIblhOVWnTpk1266a+PvvsM+bPn88nn3zCU089xfLly3Odrl1nKLuSj6Zxx4yl1KkcxZNXty32F8pEhYfy1g2xDJywgJumxfPeqK50bGivvDwVXirzaqjq+0AmgKqmA7lW4fhyy90GfAWsBt5X1ZUi8riI9PUpOgiYocV5C0CDLnDDXLj4n87f00wCUPKbgw4PD88ex6sWLVqwe/fu7OVKS0tj5cqVZGZmsnXrVi666CKeffZZDhw4kD3/3FxwwQV8/PHHpKSkcPjwYWbPns0FF1xwQpkLL7yQmTNnkpGRwfbt2/nhhx8KFaspfg/PWcm2fUd4ZVAHKvnhxU5eVIoKZ8qIztSqFMnwyYtZtzM5IHGUdl7OCA6LSHWcd0wjIt0AT3Upqvo58HmOfg/n6H7UU6RFrUGXIkkAWUp6c9CjR4+mXbt2dOrUiaeeesrTMkVERPDhhx8yZswYDhw4QHp6OnfeeSfNmzdn6NChHDhwAFVlzJgxVKlSJc/pdOrUiWHDhtGli7O+b7zxxhOuDwBcffXVfP/997Ru3ZqGDRtmL48pmWYvTWT20m3c1bM5MY2qBTSWWhWddomuef0Xrp+4iA9vOZd6VcoFNKbSpsBmqEWkE849/m2BFUBNoL+qLvN/eCcric1QB2Nz0IES6N/awJakFK4Y9xOt6lRk+qhuhPn5LiGvVm8/yLUTfqVmxUg+vPlcqhXhA21lwWk1Q62qS4DuwLnATUCbQCUBY0xgpWdkcsfMpYjAvwd2KDFJAKBVnUq8fUNntu07wvBJizhk7RJ55uWuoetxLvLGAJ1wnhC+3t+BlSbWHLQJFuO+W8fSLfv519VnU79q+UCHc5IuTarxnyGdWPHnQW6eFs+x9AIvZxq8XSzu7PO5AHgU6JvfCIFgj5uXffYbB9bCDUm89sN6+sfUp08Jvm//kla1ee6advy8fg9jZ/5GhrVLVKACLxar6u2+3SJSBae5iBIjKiqKpKQkqlevXuy3sJnioaokJSURFRUV6FCC0oGUNO6amUDDauV5tG+bQIdToGti6rP3cCpPfb6aKuXDefIvxX97a2lyKk8WH8ZpDqLEqF+/PomJiezebQ+VlGVRUVHUr18/0GEEHVXlgdnL2JV8jFm3nEuFyJLSIEH+Rl3YlKTDqbz+4x9Uj45gbC/vr8sMNgX+oiLyCe6tozhVSa2B9/0ZVGGFh4fTpEmJyk3GlBkfxCXy+fId3Ne7Zal7kfx9vVuw73Aq475fT9XoCIafZ/uJ3HhJ7b7tCqcDm1U10U/xGGNKkD92H+KRuSs598zq3HRh00CHU2giwlNXt2VfSiqPfbKKatER9OtQfE1hlBZerhH8WByBGGNKltT0TO6YsZTI8BBeurYDISGls449LDSEcYM7MmzSIu5+/zcqlQvnoha1Ch4xiOR515CIJIvIwVw+ySJysDiDNMYUvxe/XsuKbQd59pp2nFG5dF+kjwoP5c3rY2lxRkVueSee+M37Ah1SiZJnIlDViqpaKZdPRVXNp81mY0xp9/O6PUyYv4EhXRtyWZszAh1OkagYFc7k4V04o1IUIyYv5ndrlyib58cCRaSWiDTM+vgzKGNM4CQdOsbY9xM4q1YFHryydcEjlCI1K0YybWRXIsNCuP7tRSTuSyl4pCDg5cniviKyDtgI/AhsAr7wc1zGmABQVe6btYz9KWmMG9SRchGhgQ6pyDWoVp6pI7uQkprO9W8vIunQsUCHFHBezgieALoBv6tqE+ASYIFfozLGBMQ7Czbz7epd3H95S1rXLbs1wC3PqMTEYZ3588ARhk1aHPTtEnlJBGmqmgSEiEiIqv4A5NqCnTGm9Fq7I5knP1tNjxY1GX5e40CH43exjZ12iVZtP8joqXEcTQvedom8JIL9IlIBmA+8KyKv4DxdbIwpI46mZTBm+lIqRoXxfP/2QdMcw8Uta/PCgHb88kcSd85ICNp2ibwkgn5ACnAX8CXwB9DHn0EZY4rXM1+sYe3OZF4Y0J6aFSMLHqEMubpjfR66qjVfrtzBgx8vD8rGDb08WXwTMFNVtwG5v0fRGFNqfb9mJ5N/2cSI85rQI0gftBp5fhP2Hj7G+B/+oFp0BPde1jLQIRUrL4mgIvC1iOzFeWn9B6q6079hGWOKw66DR7nng2W0qlOJ+y4P7kbZ7unVgr2H09xkEMnI84OnXSIvbyh7TFXbALcCdYAfReRbLxMXkd4islZE1ovI/XmUuVZEVonIShF5r1DRG2NOWWamcvcHv5GSms64QR2IDCt7t4oWhojw5F/a0rvNGTzx6SpmLw2eJtUK8565XcAOIAko8PxRREKB8cDlOC2WDhaR1jnKNAMeAM5zk82dhYjHGHMaJv5vIz+t28NDV7WmWe2KgQ6nRAgNEV4e1IFzmlbn3g+W8cOaXYEOqVh4eaDs7yIyD/gOqA6MUtV2HqbdBVivqhtUNRXnZTb9cpQZBYxX1X0Aqhoca92YAFux7QDPfrmGXq1r87cu1lCAr6jwUN64PoaWdSpyy7vxxG3aG+iQ/M7LGUED4E5VbaOqj6rqKo/Trgds9elOdPv5ag40F5H/icgCEemd24REZLSIxIlInL18xpjTk5KazpgZS6kWHcGz17QLmltFCyOrXaK6lcsxYvJi1uwo2+1serlG8ICqJgCIyBtFPP8woBnQAxgMvOm+CjNnDG+oaqyqxtasWbOIQzAmuDzx6So27jnMv6/tQNXoiECHU2LVqBDJ1JFdKBcRyvVvL2Lr3rLbLlFhrhFA4Z4o3oZzNpGlvtvPVyIwV1XTVHUj8DtOYjDG+MEXy7czfdFWbu5+JueeVSPQ4ZR49auWZ9rIrhxLz+S6txeyO7lstktU2ERQmDr8xUAzEWkiIhHAIGBujjIf45wNICI1cKqKNhQyJmOMB3/uP8L9Hy2nff3KjL20eaDDKTWa167IxGGd2XnwGMMmLeLg0bRAh1TkCtMMdSVggNfyqpoO3AZ8BawG3lfVlSLyuIj0dYt9BSSJyCrgB+Bet10jY0wRyshU7pqZQFpGJq8M6kh4aGGPAYNbTKOq/HdoJ9buSGbUlLLXLpEU9Di1iHQGJuI8WAZwABihqvF+ji1XsbGxGhcXF4hZG1Nqjf9hPc9/tZbn+7djQGyDgkcwuZqTsI07ZiTQq3Vt/jOkE2GlKKGKSLyq5lq972Up3gb+rqqNVbUxzoNlk4owPmOMHy3dso+Xvvmdq9rVoX9M/UCHU6r161CPR/q05utVO/nn7BVlpl0iL01MZKjqT1kdqvqziAR3493GlBKHjqVzx4wEzqgUxVNXn223ihaB4ec1Ye/hVF79fj3VKkRwX+/S3y6Rl0Two4hMAKYDCgwE5olIJwBVXeLH+Iwxp+HhOStI3JfCzJvOoXK58ECHU2aMvbQ5SYdT+e+8P6geHcGNFzQNdEinxUsiaO/+fSRH/444ieHiIo3IGFMk5iRs46Ml27jjkmZ0blwt0OGUKSLCE/3asj8llSc/W03V8hFcU4qr3QpMBKp6UXEEYowpOlv3pvDg7BXENKrK7RefFehwyqTQEOHfAztw8Egc/5i1jMrlwunZunagwzolpeeStzHGk/SMTO6YsRSAlwd2KFV3tpQ2kWGhvH5dDG3rVuLW95awaGPpbJfIthBjyphx369nyZb9PPXXs2lQrXygwynzKkSGMWl4F+pVLcfIKYtZ9Wfpa5fIS+ujJ723Lrd+xpjAW7RxL699v45rOtWnb/u6gQ4naFSLjmDayK5UiAzjhkmL2JJUutol8nJG8KvHfsaYADqQksadM5bSoFp5HuvXJtDhBJ16VcoxdUQX0jIyGfr2QnYlHw10SJ7lmQhE5AwRiQHKiUhHEenkfnoAdr5pTAmiqvzfx8vZlXyMVwZ1pEKklxsCTVFr5rZLtDv5GDdMXFxq2iXK74zgMuAFnFZDX/T5jAX+z/+hGWO8+iA+kc+WbWdsr+Z0aFAl0OEEtU4Nq/L6dTGs35XMjaWkXaI8E4GqTnFvHR2mqher6kXup6+qflSMMRpj8rFh9yEenbuSc5pW56YLzwx0OAbo3rwmL17bgcWb9nLbe0tJz8gMdEj58nKNoL6IVBLHWyKyRER6+T0yY0yBUtMzuWNGAhFhIbw0sD2hIdaEREnRt31dHuvbhm9X7+SBj5aX6HaJvCSCEap6EOiF887i64Bn/BqVMcaTF79Zy/JtB3jmr+2oU7lcoMMxOVx/TmPuuKQZH8Qn8swXawIdTp68XFHKOsS4ApjqvlPADjuMCbCf1+1hwo8b+FvXhvRue0agwzF5uLNnM/alpDJh/gaqRUdwU/eSV33nJRHEi8jXQBPgARGpCJTsCi9jyri9h1MZ+34CZ9aM5qErWwc6HJMPEeHRPm3YeziVp79YQ9XoCK4tYe+E8JIIRgIdgA2qmiIi1YHhfo3KGJMnVeUfHy5jf0oak4Z3plxEaKBDMgUICRFeurYDB46kcf+sZVQpF06vNiXnLM7LNQIFWgNj3O5oIMpvERlj8vXOwi18u3on913ekjZ1Kwc6HONRRFgIrw+N4ez6Vbht+lIWbCg5b+X1kgj+A5wDDHa7k4HxfovIGJOn33cm8+Snq+jevCbDz20c6HBMIUVHhjFpWGcaVC3HqClxrNh2INAhAd4SQVdVvRU4CqCq+4AILxMXkd4islZE1ovI/bkMHyYiu0Ukwf3cWKjojQkiR9MyGDN9KRWjwnhhQHtC7FbRUimrXaKKUWEMm7SITXsOBzokT4kgTURCcaqIEJGaeLhY7I4zHrgcp2ppsIjkdlVrpqp2cD9veQ/dmODyzBdrWLMjmef7t6dmRWv3sTSrW6UcU0d2JSNTuW7iQnYdDGy7RF4SwThgNlBLRJ4Cfgae9jBeF2C9qm5Q1VRgBtDvlCM1Joh9v2Ynk3/ZxPDzGnNRy1qBDscUgbNqVWDS8C4kHUrl+omLOHAkcO0SFZgIVPVd4B84O//twF9U9X0P064HbPXpTnT75XSNiCwTkQ9FJNd7qkRktIjEiUjc7t27PczamLJjV/JR7v1gGS3PqFgmXpRujuvQoApvXBfLH7sPceOUxRxJDUy7RF7eRzBNVdeo6nhVfU1VV4vItCKa/ydAY1VtB3wDTMmtkKq+oaqxqhpbs2bNIpq1MSVfZqZy9/u/cehYOq8O7khUuN0qWtac36wGLw/sSNzmfdz23hLSAtAukZeqoRMaNnfr/mM8jLcN8D3Cr+/2y6aqSap6zO18y+N0jQkaE/+3kZ/W7eHBq1rTrHbFQIdj/OTKdnV4vF9bvluzi/tmLSMzs3jbJcrvfQQPiEgy0E5EDrqfZGAXMMfDtBcDzUSkiYhEAIOAuTnmUcensy+wutBLYEwZtfLPAzz35Vp6tqrN0K4NAx2O8bPrujXirp7N+WjJNv71+epibaQuzyeLVfVp4GkReVpVHyjshFU1XURuA74CQoGJbjtFjwNxqjoXGCMifYF0YC8w7FQWwpiy5kiqc6tolfLhPNe/Hda8V3AYc8lZ7D18jLd+3ki1ChH8vcdZxTLfApuYOJUk4DPu58DnOfo9nGPapzx9Y8qqJz5bxYY9h3lnZFeqRXt6bMeUASLCI33asC8ljee+XEu18hEM6uL/s0F7n50xJcyXK3bw3sIt3NS9KeedVSPQ4ZhiFhIivDCgPfuPpPF/s5dTpXyE31uX9XKx2BhTTLYfOML9Hy3j7HqVufvSFoEOxwSI0y5RJ9o3qMKYGUv59Q//tkvk5fbRcSJyrl+jMMaQkamMnfkbqemZvDKoAxFhdpwWzMpHOO0SNapWnlFT4/ggbivjf1hP/OZ9RT4vL1taPPCgiPwhIi+ISGyRR2GMYcL8P/h1QxKP9m1D05oVAh2OKQGqlI9g6sguRIWHcO+Hy3jx67UMeWtBkScDL08WT1HVK4DOwFrgWRFZV6RRGBPkErbu56Wvf+fKdnUYEFM/0OGYEqRO5XL06+A0ypCpkJaeWeRNWBfm3PMsoCXQCCi5L980ppQ5dCydO2YspXalKP71l7PtVlFzkivOrkNUWAihAuFhIXRrWr1Ip1/gXUMi8hxwNfAHMBN4QlX3F2kUxgSxR+asZOveFGaMPofK5cMDHY4pgWIaVeXdUd1YsCGJbk2rE9OoapFO38vto38A56jqniKdszGGOQnbmLUkkTGXNKNLk2qBDseUYDGNqhZ5Asji5YGyCSLSV0QudHv9qKqf+CUaY4LI1r0pPDh7BZ0aVmHMxcXzBKkxufFy++jTwB3AKvczRkT+5e/AjCnL0jMyuXNmAgCvDOpIWKjdKmoCx0vV0JVAB1XNBBCRKcBS4P/8GZgxZdmr3zv3g78yqAMNqpUPdDgmyHk9DKni872yH+IwJmgs3rSXV79fx1871cu+LdCYQPJyRvA0sFREfgAEuBA46UX0xpiCHTiSxp0zEqhftTyP92sb6HCMAbxdLJ4uIvNwHigDuE9Vd/g1KmPKIFXl/2YvZ+fBo3x4y7lUiLQ2H03J4LVqKATYA+wHmvvcQWSM8ejD+EQ+W7aduy5tTocGVQIdjjHZvDxQ9iwwEFgJZL1MU4H5fozLmDJl457DPDJ3Jd2aVuPm7mcGOhxjTuDl3PQvQAufdwsbYwohNT2TO2YsJTw0hH8P7EBoiDUhYUoWL1VDGwB77t2YU/TSN7+zLPEAz15zNnUqlwt0OMacxMsZQQqQICLfAdlnBao6xm9RGVNG/G/9HibM/4PBXRrSu22dQIdjTK68nBHMBZ4AfsF5N0HWp0Ai0ltE1orIehHJ85ZTEblGRNTedWDKkr2HUxn7fgJNa0Tz0FWtAh2OMXnK94xAREKBYap6UWEn7I47HrgUSAQWi8hcVV2Vo1xFnCYsFhZ2HsaUVKrKfbOWse9wGm/f0JnyEXarqCm58j0jUNUMIFNETuVp4i7AelXdoKqpwAygXy7lngCeBY6ewjyMKZHeXbiFb1bt5B+9W9C2nj2Mb0o2L4cph4DlIvINcDirp4drBPWArT7diUBX3wIi0glooKqfici9eU1IREYDowEaNmzoIWRjAmfdzmSe/GwVFzSrwYjzmgQ6HGMK5CURfOR+ipSIhAAvAcMKKquqbwBvAMTGxmpRx2JMUTmalsGYGQlER4Tx4rXtCbFbRU0p4KWJiSmnOO1tQAOf7vpuvywVgbbAPPfVfGcAc0Wkr6rGneI8jQmo575cy+rtB5k4LJZaFaMCHY4xnuSZCETkfVW9VkSW4zxJfAJVbVfAtBcDzUSkCU4CGAT8zWf8A0ANn/nNA+6xJGBKo/jN+3hv0WZmxW9j2LmNubhl7UCHZIxn+Z0R3OH+vepUJqyq6SJyG/AVEApMVNWVIvI4EKeqc09lusYEUmamsv9IGruTj7Hn0DF2Jx8jYes+pi3YQkamIsBlbSwJmNIlz0Sgqtvdv5sBRKRSfuXzmMbnwOc5+j2cR9kehZm2MUVFVTng7tx3uzv3PYdSs3f0vn+TDqWSnpn3ZSoRWLJlP+ecWSPPMsaUNF4anbsJeAzn9s6s/wAFmvoxLmNOi6py8Gj6STvyPT47et9+aRkn79zDQ4UaFSKpWTGS2pWiaFu3MjUqRlCzQiQ1KkZmD9u27wijp8WRlp5JeFgI3ZpWD8ASG3PqvBzh3wO0VdU9/g7GmPyoKoeOped7xJ69kz90jNT0zJOmERYiVK8QQU13R97ijIrZ352/EdRyuyuXC8e9kSFfZ9aswLs3dmPBhiS6Na1OTKOq/lh8Y/zGSyL4A6e9IWP84vCx9JN25rt9jth9/x7LZeceIlC9wvGd+Zm1KlCzYiQ1s3fux/9WKRful1s6YxpVtQRgSi0vieAB4BcRWYg1Omc8OpKawZ5Dx9iV6xH7idUzR9IyThpfBKpHR2TvxJvUiM4+Ys+5c69aPsKadjbmNHhJBBOA74HlHH8xjSnj4jfvO6mq42haRq517Ln9PZx68s4doFp0RPbOvGPDKtn17cfr3Z1h1cpHEBbq9QV6xpjT4SURhKvqWL9HYkqM+M37GPzmAlLTMwkROKNSFMnH0kk+mp5r+Srlw50j9AqRnF0/a+d+/Gg+q4qmWnQE4bZzN6bE8ZIIvnDb+vmEE6uG9votKhNQU37ZlH2hNVOhYrlwerU5I9eqmerRkUSE2c7dmNLMSyIY7P59wKef3T5aBmVmKi9/t465v/2JiNM0bXhYCP+6+my7EGpMGealrSFrPjEIHD6Wzt3v/8aXK3fQP6Y+/WPqEb95v90OaUwQyK+toc7AVlXd4XZfD1wDbAYetaqhsmPr3hRGTY3j953JPHhlK0ae3wQRoVtTezrWmGCQX+XuBCAVQEQuBJ4BpgIHcJuENqXfwg1J9Bv/P7btP8Kk4V248YKmnh6iMsaUHflVDYX6HPUPBN5Q1VnALBFJ8Htkxu+mL9rCQx+voGG18rx1QyxNa1YIdEjGmADINxGISJiqpgOX4L4hzMN4poRLy8jkyU9XMeXXzVzYvCavDu5I5XLhgQ7LGBMg+e3QpwM/isge4AjwE4CInIVTPWRKoX2HU7n1vSX88kcSoy5owv2Xt7Knco0Jcvk1Q/2UiHwH1AG+VtWs5hlDgNuLIzhTtH7fmcyoqXFs33+UFwa0p39M/UCHZIwpAfKt4lHVBb7dIjLafX+wKWW+XbWTO2cmEBUeyvTR3eyWUGNMtsI+EnqzX6IwfqOq/GfeekZNi6NJjWg+uf08SwLGmBMU9qKvVSaXIkfTMrhv1jLmJPxJn/Z1ee6adpSLCA10WMaYEqbAMwIR6eTT2acwExeR3iKyVkTWi8j9uQy/WUSWi0iCiPwsIq0LM32Ttx0HjnLthF+Zk/An917WgnGDOlgSMMbkyssZwYsicgbwITATSPQyYREJBcYDl7rjLBaRuaq6yqfYe6r6ulu+L/AS0LsQ8ZtcLN2yj9HT4kk5ls4b18XQq80ZgQ7JGFOCFXhGoKoXARcBu4EJ7hH8gx6m3QVYr6obVDUVmAH0yzHtgz6d0Rx/J7I5RR8tSWTgGwuICg/ho7+fZ0nAGFMgTxeLVXWHqo7DuVicADzsYbR6wFaf7kS33wlE5FYR+QN4DrC3np2ijEzl6c9XM/b93+jUsApzbj2fFmdUDHRYxphSwMs1glYi8qiIrABeBX4BiuwGdFUdr6pnAvcBuZ5piMhoEYkTkbjdu3cX1azLjINH0xg5ZTET5m/gum6NmDayK9WiIwIdljGmlPByjWAiTrVOL1X9sxDT3gY08Omu7/bLywzgv7kNcJ9deAMgNjbWqo98bNxzmBunLGZzUgpP/qUtQ7s1CnRIxphSxsv7CM4RkQiguYhUA9aqapqHaS8GmolIE5wEMAj4m28BEWmmquvcziuBdRjPflq3m1vfXUJoiDBtZFfOObN6oEMyxpRCBSYCEemO0/z0JpznCBqIyA2qOj+/8VQ1XURuA74CQoGJqrpSRB4H4lR1LnCbiPQE0oB9wA2ntTRBQlWZ/MsmnvxsNWfVrMBbN8TSoFr5QIdljCml5HgTQnkUEIkH/qaqa93u5sB0VY0phvhOEhsbq3FxcYGYdYlwLD2Dhz9eycy4rVzaujb/HtiBCpHWGKwxJn8iEq+qsbkN87IHCc9KAgCq+ruIWJvFAbA7+Ri3vBNP3OZ93H7xWdzVszkh1nKoMeY0eUkE8SLyFvCO2z0ECN5D8gBZ+ecBRk2JY29KKq8O7kif9nUDHZIxpozwkghuBm7l+D3+PwH/8VtE5iSfLdvOPR/8RpXy4Xx487m0rVc50CEZY8qQfBOB20zEb6raEqf5B1OMMjOVl79bx7jv1tGpYRVevy6GWhWjAh2WMaaMKeh9BBluo3ENVXVLcQVl4PCxdO5+/ze+XLmD/jH1eerqtkSGWaNxxpii56VqqCqwUkQWAYezeqpqX79FFeS27k1h1NQ4ft+ZzINXtmLk+U0QsYvCxhj/8JIIHvJ7FCbbwg1J3PLuEtIyMpk0vAvdm9cMdEjGmDLOy5PFP4pIbaCz22uRqu7yb1jBafqiLTz08QoaVivPWzfE0rRmhUCHZIwJAl4anbsWWAQMAK4FFopIf38HFkzSMjJ5ZM4KHvhoOeeeVYPZt55nScAYU2y8VA39E+icdRYgIjWBb3FeVGNO077Dqdz63hJ++SOJURc04f7LWxFqD4kZY4qRl0QQkqMqKInCv/Te5OL3ncmMmhrH9v1HeWFAe/rHFFnr3sYY45mXRPCliHwFTHe7BwKf+y+k4PDtqp3cOTOBqPBQpo/uRkyjqoEOyRgTpLxcLL5XRK4BznN7vaGqs/0bVtmlqvz3xz94/qu1tK1bmTeuj6FO5XKBDssYE8Q8NVupqrOAWX6Opcw7mpbBfbOWMSfhT/q0r8tz17SjXIQ9JGaMCSwv7yNI5uSXyh/AaXjublXd4I/AypodB44yelocyxIPcO9lLfh7jzPtITFjTIng5YzgZZwXz7+H82KaQcCZwBKc11j28FNsZcbSLfsYPS2elGPpvHFdDL3anBHokIwxJpuXu3/6quoEVU1W1YPu+4MvU9WZOM1PmHx8tCSRgW8sICo8hI/+fp4lAWNMiePljCDFfags67mB/sBR97u9SD4PGZnKc1+uYcL8DXRrWo3/DImhWnREoMMyxpiTeEkEQ4BXOP4Ogl+BoSJSDrjNX4GVZgePpjFm+lLmrd3Ndd0a8XCf1oSH2qMXxpiSycvtoxuAPnkM/jm/cUWkN04SCQXeUtVncgwfC9wIpAO7gRGqutlD3CXWxj2HuXHKYjYnpfDkX9oytFujQIdkjDH58tLWUH0RmS0iu9zPLBEp8BFY96U244HLgdbAYBFpnaPYUiBWVdvhVD09V/hFKDl+Wrebfq/9zN7DqUwb2dWSgDGmVPBSXzEJmAvUdT+fuP0K0gVYr6obVDUVmAH08y2gqj+oaorbuQAolW0sqCoTf97IDRMXUadyOebedj7nnFk90GEZY4wnXhJBTVWdpKrp7mcy4KWR/HrAVp/uRLdfXkYCX+Q2QERGi0iciMTt3r3bw6yLz7H0DO6ftZzHP13FJa1qM+vv59KgWvlAh2WMMZ55uVicJCJDOd7W0GCchueKjDv9WKB7bsPdW1bfAIiNjS0xdyrtTj7GLe/EE7d5H7dffBZ39WxOiLUcaowpZbwkghHAq8C/cW4X/QUY7mG8bUADn+76br8TiEhPnKauu6vqMQ/TLRFWbDvA6Klx7E1J5dXBHenTvm6gQzLGmFPi5a6hzcCpvJ94MdBMRJrgJIBBwN98C4hIR2AC0Ls0vfXss2XbueeD36hSPpwPbz6XtvUqBzokY4w5ZV7aGmoC3A409i1f0MvrVTVdRG4DvsK5fXSiqq4UkceBOFWdCzwPVAA+cNvd2VLQdAMpM1N5+bt1jPtuHZ0aVuH162KoVTEq0GEZY8xp8VI19DHwNs7dQpmFmbiqfk6Odxeo6sM+33sWZnqBdPhYOne//xtfrtxB/5j6PHV1WyLDrOVQY0zp5yURHFXVcX6PpATbujeFUVPj+H1nMg9e2YqR5zexlkONMWWGl0Twiog8AnwNZF/MVdUlfouqBFm4IYlb3l1CWkYmk4Z3oXtzL3fOGmNM6eElEZwNXAdczPGqIXW7y7Tpi7bw0McraFi9PG9dH0vTmhUCHZIxxhQ5L4lgANDUfTo4KKRlZPLkp6uY8utmujevybjBHalcLjzQYRljjF94SQQrgCpAqbm983TsO5zKre8t4Zc/khh1QRPuv7wVofaQmDGmDPOSCKoAa0RkMSdeIyixt3meqt93JjNqahzb9x/lhQHt6R9TKps+MsaYQvGSCB7xexQlwLerdnLnzASiwkOZcVM3OjW0l68ZY4KDlyeLfyyOQAJFVfnvj3/w/FdraVu3Mm9cH0OdyuUCHZYxxhSbPBOBiCST+6soBVBVreS3qIrJ0bQM7pu1jDkJf9KnfV2eu6Yd5SLsITFjTHDJMxGoasXiDKS47ThwlNHT4liWeIB7L2vB33ucaQ+JGWOCkpdrBIhIe+ACt3O+qi7zX0j+t3TLPkZPiyflWDpvXBdDrzZnBDokY4wJGC+vqrwDeBeo5X7eFZHb/R2Yv3y0JJGBbywgKjyEj/5+niUBY0zQ83JGMBLoqqqHAUTkWeBXnHcUlBqLN+3l+S/XsGjTPro1rcZ/hsRQLToi0GEZY0zAeUkEAmT4dGe4/UqN+M37GPzGAtIzldAQYeylzS0JGGOMy0simAQsFJHZbvdfcJqlLjUWbEgiU90boFRZvGkfXZrYy+WNMQbyuUYgIg0AVPUlnFdT7nU/w4H1xRJdEenWtDoRYSGECoSHhdCtqSUBY4zJkt8ZwTci0ltVN7lNTi8BEJEROO8Y/rQ4AiwKMY2q8u6N3ViwIYluTasT08ieGjbGmCz5JYKxwNcicqWqrgMQkfuBIUD34giuKMU0qmoJwBhjcpHfA2Wfi8gx4AsR+QtwI9AFuFBV9xVTfMYYY/ws3+cIVPU7nGsC84CmwMWFSQIi0ltE1orIevdsIufwC0VkiYiki0j/QsZujDGmCHhpa0iASOASYJc47TAU2NaQiIQC44FLgURgsYjMVdVVPsW2AMOAe05nIYwxxpw6f7Y11AVYr6obAERkBtAPyE4EqrrJHZaZ2wSMMcb4X4FNTJyGesBWn+5Et1+hichoEYkTkbjdu3cXSXDGGGMc/kwERUZV31DVWFWNrVmzZqDDMcaYMsWfiWAb0MCnu77bzxhjTAniz0SwGGgmIk1EJAIYBMz14/yMMcacAr8lAlVNB24DvgJWA++r6koReVxE+gKISGcRSQQGABNEZKW/4jHGGJM7Ty+mOVWq+jnweY5+D/t8X4xTZWSMMSZASsXFYmOMMf5jicAYY4KcJQJjjAlylgiMMSbIWSIwxpggZ4nAGGOCXPAkgq2L4KcXnb+mYLa+CsfWV+HZOiscP64vvz5HUGJsXQSTLofMdJAQqN0WIvNtRTu4HTsIO1eAZtr68sLWV+HZOiuc7PWlEBYFN8yFBl2KbPLBcUaw6SfIzHC+ayYcPRDYeEq6owec9QS2vryw9VV4ts4KJ3t9KWSkOvu0IhQcZwSNL3CyaEYqhEbANW8VaTYtc7Yugil9bX15Zeur8GydFU7O9dX4giKdvKhqkU7Q32JjYzUuLq7wI25d5GTRxhfYBueFra/CsfVVeLbOCuc015eIxKtqbK7DgiYRGGNMEMsvEQTHNQJjjDF5skRgjDFBzhKBMcYEOUsExhgT5CwRGGNMkLNEYIwxQa7U3T4qIruBzac4eg1gTxGGU1QsrsKxuAqvpMZmcRXO6cTVSFVr5jag1CWC0yEicXndRxtIFlfhWFyFV1Jjs7gKx19xWdWQMcYEOUsExhgT5IItEbwR6ADyYHEVjsVVeCU1NourcPwSV1BdIzDGGHOyYDsjMMYYk4MlAmOMCXJlIhGISAMR+UFEVonIShG5w+1fTUS+EZF17t+qbv9r3HI/iUh1t9+ZIjIzkMtR3EQkVESWisinbncTEVkoIutFZKaIRLj9bxeRFSLyuU+/80Xk34GMvziJSBUR+VBE1ojIahE5x7avvInIXe46WCEi00UkyravE4nIRBHZJSIrfPrltU2JiIxz190yEenk9m8hIvFuv3PcfmEi8q2IlPccjKqW+g9QB+jkfq8I/A60Bp4D7nf73w88636fB5QHhgK3u/2mA80CvSzFvN7GAu8Bn7rd7wOD3O+vA7e43xfgHDQ8CPQBBPgKqBboZSjGdTUFuNH9HgFUse0rz3VVD9gIlPPZrobZ9nXSeroQ6ASs8OmX1zZ1BfCFu266AQvd/i8B5wP1gVluv9uBYYWJpUycEajqdlVd4n5PBlbjbIz9cP6Bcf/+xf2eCUTi/LOmicgFwA5VXVeccQeSiNQHrgTecrsFuBj40C3iu74ECMddXzg7uC9UdW8xhhwwIlIZ55/2bQBVTVXV/dj2lZ8woJyIhOGsh+3Y9nUCVZ0P5FzGvLapfsBUdSwAqohIHZz1VZ7j21oVnGQ6tTCxlLl3FotIY6AjsBCorarb3UE7gNru96eBb4E/cTa6D4BBxRtpwL0M/APnDAqgOrBfVdPd7kScZArwGs5R20rgf8Ac4LJiizTwmgC7gUki0h6IB+7Atq9cqeo2EXkB2AIcAb7GWWe2fRUsr22qHrDVp1zW+huPs9OPBG4CHgL+paqZhZlpmTgjyCIiFYBZwJ2qetB3mDrnTOp+/0ZVY1S1D06m/Rxo7tYBv1mourVSSESuAnaparyX8qo6TVU7qupQ4C5gHHC5u77+LSJlajvKRRjOKfx/VbUjcBjntD2bbV/HufXa/XASaF0gGuidV3nbvnLnu03lU2aLqvZQ1XOAFJwqotUiMs29DtPcy7zKzAoWkXCcJPCuqn7k9t7pnj7h/t2VY5zyOHWX44HHgBuAn4EhxRR2oJwH9BWRTcAMnFP2V3BON7POEusD23xHEpG6QBdV/Ri4GxgI7AcuKZaoAycRSFTVhW73hziJwbav3PUENqrqblVNAz7C2eZs+ypYXtvUNqCBT7mT1h/wFM51ljE4Vb7/AB7xMtMykQjc+u23gdWq+pLPoLk4/3y4f+fkGPVeYJy7sZbDyb6ZOPVtZZaqPqCq9VW1MU6VxfeqOgT4AejvFsttfT0BPOx+D6b1tQPYKiIt3F6XAKuw7SsvW4BuIlLe/d/MWl+2fRUsr21qLnC9e/dQN+CATxUSItId+NO9DlUeZ715X3eBvnJeFB+cq+YKLAMS3M8VOPXe3wHrcOpsq/mMUxf4zKd7AMfrKGsGepmKcd314PhdQ02BRcB6nHrtSJ9yHYG3fbrvdNfXl77lyuoH6ADEudvYx0BV277yXV+PAWuAFcA0nDps275OXEfTcS6ip+GcdY7Ma5vCuaA+HvgDWA7E+kxHgG98yrYClrjb6nleYrEmJowxJsiViaohY4wxp84SgTHGBDlLBMYYE+QsERhjTJCzRGCMMUHOEkEZJSIZIpLgtur4QSCeZhWRHiJybnHP12f+F7gtYCaISLkAzD9WRMZ5KPdLccRT1ETkUAmIYbKI9C+4pMmPJYKy64iqdlDVtkAqcLOXkXye/CwKPYCAJQKcJ3ifdtfDkdwKuA/o+OX/QFXjVHWMh3KBXEfGWCIIEj8BZ4lItNsG+iJx3kPQD0BEhonIXBH5HvhORCqIyCQRWe62c36NW66XiPwqIkvcs4wKbv9NIvKY23+5iLR0G/+7GbjLPSK/QET6iNMe/VK3vfTa7vg1xWl7faWIvCUim0WkhjtsqBtvgohMEJHQnAsnIpe401zuLl+kiNwIXAs8ISLv5ijfWETWishUnAeeGojIvSKy2F3ex3zKrXGPOn8XkXdFpKeI/E+c9uK7uOW6uOtlqYj8kvUEsntGlPWuh0fd2OaJyAYRGeMTzyGf8vPk+HsP3nWfzEVErnD7xYvTLv2nuayHNj7rapmINHP7f+yOt1JERvvOV0Sed/t/6y5HVnx9fbaNOW7/dSKSa5MFua2/XMoc8vneX0Qmu98HiHPm+puIzHf7hbqxZU3zJre/iMhr7u/3LVArt3mZQgr003X28dtTi4fcv2E4j6nfAvwLGOr2r4Lz3oZonPZwEjn+ZOKzwMs+06oK1ADmA9Fuv/uAh93vmzje7v7fgbfc748C9+SYTtZDjDcCL7rfXwMecL/3xnlKvAbOE5KfAOHusP8A1+dYziicVhmbu91TcRodBJgM9M9l3TTGefy+m9vdC+el4IJzcPQpTrPTjYF04Gy3fzww0S3XD/jYHb8SEOZ+78nxduF7cPyp7UeBX3CesK0BJPks1yGf8gdw2pEJAX7FeWo+axmbuOWmZ003x3K9Cgxxv0dw/H0AWb9rOZzEV93tVuBy9/tsnFZCw4H2QILbfxjO06/VfcaPzRF3rusvr23S/d4fmOx+Xw7Uy9ou3b+jgQfd75E4T3U3Af6K8xRtKM7T2/tz+43tU7hPmWuG2mQrJyIJ7vefcNpi+gWnsbl73P5RQEP3+zd6vP33nvg0m6yq+8RpsbQ18D/3IDUCZ0eVJauhv3icf9bc1AdmitOYVgTOy0vA2dld7c7rSxHZ5/a/BIgBFrvzLEeOht2AFjgNnP3udk8BbsVpZjs/m9Vp1x2cHVkvYKnbXQFohtNmzkZVXQ4gIiuB71RVRWQ5TqIAqAxMcY/AFWdnmpvPVPUYcExEduE0MZyYo8wiVU1055fgzuMQsEFVs9bXdJwdZU6/Av8U510TH+nx9x+MEZGr3e8N3GVLwqky/NLtvxw4pqppOZYNnG0jyY3pI5zfK85neF7rb34e6yGn/wGTReR9jm9HvYB2crz+v7I7zQuB6aqaAfwpzlmsOU2WCMquI6rawbeHW81wjaquzdG/K07TyvkRnB3C4DyGH3P/ZpD3dvUq8JKqzhWRHjhHyQXNc4qqPlBAuVPhu7yCcy1hwgkzd6q3jvn0yvTpzuT4cj4B/KCqV7vjzMtjnr7Tyms9eSmTK1V9T0QW4rxw6HO3OiUTJ7Gfo6opIjIP5wAAIE3dQ258lk1VM+XEa0U526HJ2Z3r+sstRJ/vUdk9VW92t8ErgXgRiXGnebuqfnXCjESuKGAe5hTYNYLg8hVwu0+9c8c8yn2Dc1SNW64qzotDzhORs9x+0VJwW+fJHH/xDThHdVlN597g0/9/OPX5iEgvnCokcBrf6i8itdxh1USkUY55rAUaZ8UFXAf8WEBcOX0FjJDj1zzqZc3TI9/lGlbIeXuxFmjqJhlwmmc+iYg0xTlzGIdTHdjOjW2fmwRa4rzmsLAuddd9OZw3Zv0vx3Cv62+niLQS5+J81hkKInKmqi5U1YdxXgDUwJ3mLeI0L4+INBeRaJyzjIHuNYQ6wEWnsDwmB0sEweUJnGqLZW41xxN5lHsSqJp1AQ+4SFV34+zkpovIMpxqiJYFzO8T4Gr34uUFOGcAH4hIPLDHp9xjQC9xXuI9AOfNTMmqugqnffWv3Xl+g/N+6myqehQY7k53Oc6R7esFr4oTpvE1zrubf3Wn8SEnJrCCPAc8LSJL8cNZtjp3PP0d+NJdd8k41xJyuhZY4VYptcW5XvIlECYiq4FncBJ6YS3CedfHMpzrH77VQoVZf/fjXD/4Bee6Q5bnxbnQv8Id9htOe/qrgCVu/wk463Y2Tsucq9zl862eNKfIWh81AScikUCGqqaLyDk4bwLrEOCwShQRqaCqh9yzufHAOlX9dzHMdxjOxeHb/D0vEzh2jcCUBA2B990qg1RgVIDjKYlGicgNOBfZl+IcIRtTJOyMwBhjgpxdIzDGmCBnicAYY4KcJQJjjAlylgiMMSbIWSIwxpgg9/9xC68cBLpw6QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[MISSING LABELS] No missing labels were found.\n", + "[TEST NORMALITY] It was not possible to normalize the label values. See the warning message for additional context.\n" + ] + } + ], + "source": [ + "full_results = dq.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check the status\n", + "After running the data quality checks, you can check the warnings for each individual test. The warnings are suited by priority and have additional details that can provide better insights for Data Scientists." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[TEST NORMALITY] The label distribution failed to pass a normality test as-is and following a battery of transforms. \n", + "\tIt is possible that the data originates from an exotic distribution, there is heavy outlier presence or it is multimodal. \n", + "\tAddressing this issue might prove critical for regressor performance. (Priority 1: heavy impact expected)\n", + "[DUPLICATE COLUMNS] Found 1 columns with exactly the same feature values as other columns. (Priority 1: heavy impact expected)\n", + "[EXACT DUPLICATES] Found 20 instances with exact duplicate feature values. (Priority 2: usage allowed, limited human intelligibility)\n", + "[FLATLINES] Found 8 flatline events with a minimun length of 5 among the columns {'Region', 'MainCity'}. (Priority 2: usage allowed, limited human intelligibility)\n", + "[OUTLIER DETECTION] Found 2 potential outliers across the full dataset. \n", + "\tA distance bigger than 3.0 standard deviations of intra-cluster distances to the respective centroids was used to define the potential outliers. (Priority 2: usage allowed, limited human intelligibility)\n" + ] + } + ], + "source": [ + "# Print the overall status of Data Quality discovered during evaluation\n", + "dq.report()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# You can retrieve the full list of warnings or filtered by specific conditions\n", + "data_quality_warnings = dq.get_warnings()\n", + "duplicate_quality_warnings = dq.get_warnings(category='Duplicates')\n", + "priority_2_warnings = dq.get_warnings(priority=2)" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "fdb8905eeefe08da097059bda365f0d7e393b9cc818106eee5be3ebd28cc5e41" + }, + "kernelspec": { + "display_name": "Python 3.8.10 ('.venv': venv)", + "name": "pythonjvsc74a57bd0cdc2bce73c2a9ac283f602628cabf735dbe06c4ee87a7849fc5f3d1177c8f304" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "metadata": { + "interpreter": { + "hash": "cdc2bce73c2a9ac283f602628cabf735dbe06c4ee87a7849fc5f3d1177c8f304" + } + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file