EPA test date explore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sparqlwrapper in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (1.8.5)\n",
      "Requirement already satisfied: rdflib>=4.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from sparqlwrapper) (5.0.0)\n",
      "Requirement already satisfied: six in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from rdflib>=4.0->sparqlwrapper) (1.15.0)\n",
      "Requirement already satisfied: pyparsing in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from rdflib>=4.0->sparqlwrapper) (2.4.7)\n",
      "Requirement already satisfied: isodate in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from rdflib>=4.0->sparqlwrapper) (0.6.0)\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Warning: 'conda-forge' already in 'channels' list, moving to the top\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Requirement already satisfied: tensorflow-gpu in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (2.4.1)\n",
      "Requirement already satisfied: opt-einsum~=3.3.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (3.3.0)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (3.14.0)\n",
      "Requirement already satisfied: flatbuffers~=1.12.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.12)\n",
      "Requirement already satisfied: typing-extensions~=3.7.4 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (3.7.4.3)\n",
      "Requirement already satisfied: google-pasta~=0.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (0.2.0)\n",
      "Requirement already satisfied: numpy~=1.19.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.19.2)\n",
      "Requirement already satisfied: wheel~=0.35 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (0.36.2)\n",
      "Requirement already satisfied: wrapt~=1.12.1 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.12.1)\n",
      "Requirement already satisfied: tensorboard~=2.4 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (2.4.1)\n",
      "Requirement already satisfied: gast==0.3.3 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (0.3.3)\n",
      "Requirement already satisfied: tensorflow-estimator<2.5.0,>=2.4.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (2.4.0)\n",
      "Requirement already satisfied: grpcio~=1.32.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.32.0)\n",
      "Requirement already satisfied: termcolor~=1.1.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.1.0)\n",
      "Requirement already satisfied: h5py~=2.10.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (2.10.0)\n",
      "Requirement already satisfied: keras-preprocessing~=1.1.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.1.2)\n",
      "Requirement already satisfied: astunparse~=1.6.3 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.6.3)\n",
      "Requirement already satisfied: six~=1.15.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (1.15.0)\n",
      "Requirement already satisfied: absl-py~=0.10 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow-gpu) (0.11.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (0.4.2)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (2.25.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (1.0.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (51.3.3.post20210118)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (1.24.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (1.8.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorboard~=2.4->tensorflow-gpu) (3.3.3)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow-gpu) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow-gpu) (4.7)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow-gpu) (4.2.1)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow-gpu) (1.3.0)\n",
      "Requirement already satisfied: importlib-metadata in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow-gpu) (2.0.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow-gpu) (0.4.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow-gpu) (2020.12.5)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow-gpu) (1.26.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow-gpu) (2.10)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow-gpu) (4.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow-gpu) (3.1.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.4->tensorflow-gpu) (3.3.1)\n",
      "Requirement already satisfied: pydot in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (1.4.1)\n",
      "Requirement already satisfied: pyparsing>=2.1.4 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from pydot) (2.4.7)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow_probability in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (0.12.1)\n",
      "Requirement already satisfied: cloudpickle>=1.3 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (1.6.0)\n",
      "Requirement already satisfied: gast>=0.3.2 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (0.3.3)\n",
      "Requirement already satisfied: dm-tree in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (0.1.5)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (1.19.2)\n",
      "Requirement already satisfied: six>=1.10.0 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (1.15.0)\n",
      "Requirement already satisfied: decorator in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_probability) (4.4.2)\n",
      "Requirement already satisfied: tensorflow_addons in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (0.12.1)\n",
      "Requirement already satisfied: typeguard>=2.7 in /home/erik/anaconda3/envs/my-rdkit-env/lib/python3.7/site-packages (from tensorflow_addons) (2.10.0)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install sparqlwrapper\n",
    "!conda install -c conda-forge rdkit -y\n",
    "!conda install matplotlib -y\n",
    "!conda install numpy scipy scikit-learn -y\n",
    "!conda config --add channels conda-forge\n",
    "!conda install catboost -y\n",
    "!conda install tqdm -y\n",
    "!conda install xgboost -y \n",
    "!conda install hyperopt -y\n",
    "!pip3 install tensorflow-gpu\n",
    "!pip3 install pydot\n",
    "!pip3 install tensorflow_probability\n",
    "!pip3 install tensorflow_addons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import PandasTools\n",
    "\n",
    "DATASET = 'LC50'\n",
    "\n",
    "training_file = f'./test_data/DataSets/{DATASET}_training.sdf'\n",
    "testing_file = f'./test_data/DataSets/{DATASET}_prediction.sdf'\n",
    "\n",
    "\n",
    "train_frame = PandasTools.LoadSDF(training_file,\n",
    "                            smilesName='SMILES',\n",
    "                            molColName='Molecule')\n",
    "\n",
    "test_frame = PandasTools.LoadSDF(testing_file,\n",
    "                            smilesName='SMILES',\n",
    "                            molColName='Molecule')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install sparqlwrapper\n",
    "# https://rdflib.github.io/sparqlwrapper/\n",
    "from collections import defaultdict\n",
    "import sys\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "\n",
    "endpoint_url = \"https://query.wikidata.org/sparql\"\n",
    "\n",
    "query = \"\"\"select ?from ?to where{\n",
    "  ?chem wdt:P231 ?from ;\n",
    "        wdt:P592 ?to .\n",
    "}\"\"\"\n",
    "\n",
    "\n",
    "def get_results(endpoint_url, query):\n",
    "    user_agent = \"WDQS-example Python/%s.%s\" % (sys.version_info[0], sys.version_info[1])\n",
    "    # TODO adjust user agent; see https://w.wiki/CX6\n",
    "    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)\n",
    "    sparql.setQuery(query)\n",
    "    sparql.setReturnFormat(JSON)\n",
    "    return sparql.query().convert()\n",
    "\n",
    "\n",
    "results = get_results(endpoint_url, query)\n",
    "\n",
    "mapping = defaultdict(lambda : 'no mapping')\n",
    "\n",
    "for result in results[\"results\"][\"bindings\"]:\n",
    "    mapping[result['from']['value']] = result['to']['value']\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_frame['CHEMBL'] = [mapping[cas.upper()] for cas in train_frame['CAS']]\n",
    "test_frame['CHEMBL'] = [mapping[cas.upper()] for cas in test_frame['CAS']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_frame = train_frame.drop(train_frame[train_frame.CHEMBL=='no mapping'].index,axis=0).reset_index(drop=True)\n",
    "#test_frame = test_frame.drop(test_frame[test_frame.CHEMBL=='no mapping'].index,axis=0).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((659, 8), (164, 8))"
      ]
     },
     "execution_count": 175,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_frame.shape,test_frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "x = train_frame['Tox'].astype(float).values\n",
    "\n",
    "mean = np.mean(x)\n",
    "std = np.std(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f2f182d3090>"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfiUlEQVR4nO3deXSU5f338fdFQCKLuBARCDTwIK1CCIEAESEEoTECptQWCYLSUpr8UFxQwYUi0V9tezBVnpSKS1WwjUKr9KAVUPGBDiBKWEMAlyqggYgYioDs4Xr+mMk4WSYzSWYS7uTzOicnmXvu5XtPkk/uuXLNd4y1FhERcZ4m9V2AiIjUjAJcRMShFOAiIg6lABcRcSgFuIiIQzWty4O1bdvWxsTE1OUhRUQcb9OmTd9Ya6PKL6/TAI+JiWHjxo11eUgREcczxuytbLmGUEREHEoBLiLiUApwERGHqtMxcHGmM2fOUFhYyMmTJ+u7FJEGLTIykujoaJo1axbU+gpwCaiwsJDWrVsTExODMaa+yxFpkKy1FBcXU1hYSJcuXYLaRkMoEtDJkye57LLLFN4iYWSM4bLLLqvWM10FuARF4S0SftX9PVOAi4g4lMbApfoyM0O7v2efrfLuVq1acezYsdAes4784he/YNSoUfz85z+v0falL35r27YtOTk5zJ8/nz59+pCbmxviSsMjOzub6dOnc/DgQdq2bVvh/hUrVnD33XdTUlLC5MmTefDBB6u1fSBz584lIyODFi1aVLhvwYIFbNy4kXnz5gW1r9TUVD744AMGDRrEv/71ryrXfe211xgzZgx5eXkkJCQA8MADD/DWW28BMGvWLMaOHVvNs6lIV+DieHv3fv8Ryp2cPXu29sWF0NNPP82yZcscE95ffvkl7777Lp07d670/pKSEu644w6WL1/Ozp07efXVV9m5c2fQ2wdj7ty5HD9+vMbb+5o+fTp//etfA6539OhRcnJyGDBggHfZW2+9xebNm9m6dSsffvghTzzxBEeOHKl1TQpwcQxrLdOnT6dnz57ExsayePFiAM6dO8dvfnM7P/5xD0aNGsWIESN47bXXAPcV7OzZs+nTpw+xsbF89NFHABw6dIjRo0fTq1cvEhMTyd+1C4Csp54iIyODlJQUbrvtNrKyspg4cSIpKSnExMSwZMkSZsyYQWxsLKmpqZw5cwaAxx57jH79+tGzZ08yMjII9E5XycnJ3HPPPQwcOJCePXuyYcMGAIqLi0lJSSE+Pp7MzEzvfv7nf/6Hzz//nLS0NJ566qky+1qwYAGjR4/mxhtvpEuXLsybN48nn3yS+Ph4EhMTOXToEACfffYZqamp9O3bl8GDB3sfizfffJMBAwYQHx/P8OHDOXDggPuxyMpi0qRJJCcn07VrV3Jycqr1/Zo2bRpz5szxO667YcMGunXrRteuXbngggtIT09n6dKlQW/v67vvvmPkyJHExcXRs2dPFi9eTE5ODvv372fo0KEMHToUgJdeeonu3bszZMgQ1q1bV63zGTZsGK1btw643qxZs5gxYwaRkZHeZTt37mTIkCE0bdqUli1bEhcXx4oVK6p1/MoowMUxlixZwtatW9m2bRsrV65k+vTpFBUVsWLFEgoL9/D229v5y1/+wvr168ts17ZtWzZv3syUKVPIzs4GYPbs2cTHx5Ofn8/vfvc7brv3Xu/6mzZtYunSpbzyyiuAO/jeeustli5dyoQJExg6dCjbt2/nwgsv9D4lnjp1Knl5eRQUFHDixImAT7HBHTrvv/8+Tz/9NJMmTQLg0UcfZdCgQWzZsoW0tDS++OILAJ555hk6dOjAqlWrmDZtWoV9FRQU8Morr7BhwwZmzpxJixYt2LJlC9dccw0vv/wyABkZGfzpT39i06ZNZGdnc/vttwMwaNAgPvjgA7Zs2UJ6ejpz5szx7vejjz7i7bffZsOGDTz66KPeP1iDBw+md+/eFT5WrlwJwBtvvEHHjh2Ji4vze/779u2jU6dO3tvR0dHs27cv6O19rVixgg4dOrBt2zYKCgpITU3lrrvu8j5mq1atoqioiNmzZ7Nu3TrefffdMlf7ubm5lZ5PdYe+tmzZwpdffsmoUaPKLI+Li2P58uUcP36cb775hlWrVvHll19Wa9+V0Ri4OMbatWsZN24cERERtGvXjiFDhpCXl0de3lpGjBhDkyZNuOKKK7xXW6VuuukmAPr27cuSJUu8+3r99dcBuO666yg+fJhvPU9p09LSuPDCC73b33DDDTRr1ozY2FhKSkpITU0FIDY2lj179gCwatUq5syZw/Hjxzl06BA9evTgxhtvrPJ8xo0bB0BSUhJHjhzh8OHDuFwub40jR47kkksuCeqxGTp0KK1bt6Z169a0adPGe+zY2Fjy8/M5duwY77//PmPGjPFuc+rUKcA9z3/s2LEUFRVx+vTpMnOQR44cSfPmzWnevDmXX345Bw4cIDo6mjVr1vit5fjx4zz++OO88847VdZc2bMUY0zQ2/uKjY3l/vvv54EHHmDUqFEMHjy4wjoffvghycnJREW5m/qNHTuWTz75BIDx48czfvz4oI9XmXPnzjFt2jQWLFhQ4b6UlBTy8vIYOHAgUVFRXHPNNTRtWvv4VYCLY/gblgg0XNG8eXMAIiIivOPa/sIDoGXLlpVu36RJE5o1a+Zdr0mTJpw9e5aTJ09y++23s3HjRjp16kRWVlZQc3nLDw2U3q7JlM3SGkvr8q357NmznDt3josvvpitW7dW2PbOO+/k3nvvJS0tjdWrV5OVlVXpfn0fv8GDB3P06NEK+8rOzqZdu3bs3r3be/VcWFhInz592LBhA1dccYV33ejo6DJXoYWFhXTo0IHPPvssqO19de/enU2bNrFs2TIeeughUlJSeOSRRyqs5++xzc3N5YknnqiwvFu3bt7huECOHj1KQUEBycnJAHz11VekpaXxxhtvkJCQwMyZM5k5cyYAt9xyC1deeWVQ+61KwCEUY0wnY8wqY8wuY8wOY8zdnuVZxph9xpitno8Rta5GpApJSUksXryYkpISDh48iMvlon///vTrN4gVK17n3LlzHDhwgNWrVwe1r9J/Bq5evZq2l1zCRUGMb1amNKzbtm3LsWPHgv6FLx3DX7t2LW3atKFNmzZl6lq+fDn//e9/a1RTeRdddBFdunThH//4B+D+A7Zt2zYAvv32Wzp27AjAwoULg9rfmjVr2Lp1a4WP4cOHExsby9dff82ePXvYs2cP0dHRbN68uUL49uvXj08//ZTdu3dz+vRpFi1aRFpaWpXb79u3j2HDhlWoZ//+/bRo0YIJEyZw//33s3nzZgBat27t/UMzYMAAVq9eTXFxMWfOnPE+FuC+Aq/sfIL9XgK0adOGb775xlt3YmKiN7xLSkooLi4GID8/n/z8fFJSUoLetz/BXIGfBe6z1m42xrQGNhlj3vXc95S1NrvWVYizBJj2Fy4//elPWb9+PXFxcRhjmDNnDldccQU33PAz1q17j5SUnvTo0Z0BAwbQpk2bKveVlZXFL3/5S3r16kWLFi1Y+Mc/1riuiy++mF//+tfExsYSExNDv379gtrukksuYeDAgRw5coQXX3wRcI/Njxs3jj59+jBkyJBazcAoLzc3lylTpvDb3/6WM2fOkJ6eTlxcHFlZWYwZM4aOHTuSmJjI7t27Q3bM8vbv38/kyZNZtmwZTZs2Zd68eVx//fWUlJQwadIkevToUeX2RUVFlQ49bN++nenTp3ufJc2fPx9wj/vfcMMNtG/fnlWrVpGVlcU111xD+/bt6dOnDyUlJUHXXvqP32PHjhEdHc0LL7zA9ddfzyOPPEJCQgJpaWl+tz1z5ox3WOeiiy7ib3/7W0iGUEygp58VNjBmKTAPuBY4Vp0AT0hIsHpDB+fZtWsXV111VX2X4dfevfDdd8do2bIVrVoV079/f9atW+f36bbfnZT6wQ9CX2Q5ycnJZGdne+cIS3DmzZtH586dqwxLp6vs980Ys8laW+GHpVp/AowxMUA88CHuAJ9qjLkN2Ij7Kr3C8z1jTAaQAYT0akIaJ385O2nSKI4cOQycZtasWVxx6tT3K9dBIEvdmDp1an2XcF4JOsCNMa2A14F7rLVHjDHzgf8FrOfzH4FJ5bez1j4HPAfuK/BQFC1S3uLFqwGfrK7Vq3rCL5hxepFAgpoHboxphju8c621SwCstQestSXW2nPA80D/8JUpIiLlBTMLxQAvALustU/6LG/vs9pPgYLQlyciIv4EM4RyLXArsN0Ys9Wz7GFgnDGmN+4hlD1AiDsciYhIVQIGuLV2LVDZ7PdloS9HRESCpVdiSrXVcTdZv+1kX375ZebMmcPp0xZrLTffPInHH7+frKwsnn/mGaIuvRSA32VnM2KE+3Vmv//973nhhReIiIggJyeH66+/vsJ+k5OTKSoq8r6c/je/+Y3fnhjl25WGq/Wtv2mHycnJfP755+zdu9f7KsPRo0ezcuVKjh07xv79+7nrrruq9YIUcQ4FuDjSqlXLycmZyzvvvMOZMx04efIk//zn960+p/3qV9yfkeG+4ZmasnPnThYtWsSOHTvYv38/w4cP55NPPiEiIqLC/nNzc4Oaoz137lwmTJhQab/punLxxRezbt06Bg0axOHDhykqKvLe16FDB4V3A6ZuhHJeqG4/7/nzf092djYdOnQA3O/mPW7cr6vcZunSpaSnp9O8eXO6dOlCt27dvG1cA5kyZQoJCQn06NGD2bNnA1TarhRg5syZxMXFkZiY6G3NevDgQX72s5/Rr18/+vXr521lumHDBgYOHEh8fDwDBw7k448/BuDEiROkp6fTq1cvxo4dy4kTJ/zWlp6ezqJFiwB3x8bS5l0Ae/bsoWfPnoC77exNN91EamoqV155JTNmzPCu16pVq0rr3rt3L8OGDaNXr14MGzaML774gpKSErp27Yq1lsOHD9OkSRNcLhfgfrXif/7zn6AeU6k9Bbg40scfF9C3b1+/989buJBeqalMmj7d20+kqval5Y0fP97bUrS4uJjHH3+cjRs3kp+fz7///W/y8/MrtCsFd4vYxMREtm3bRlJSEs8//zwAd999N9OmTSMvL4/XX3+dyZMnA/CjH/0Il8vFli1beOyxx3j44YcBmD9/Pi1atCA/P5+ZM2eyadMmv+c6bNgwXC4XJSUlLFq0qMp3etm6dSuLFy9m+/btLF682NtMyl/dU6dO5bbbbiM/P5/x48dz1113ERERQffu3dm5cydr166lb9++rFmzhlOnTlFYWEi3bt38Hl9CSwEuDc6UKVP4zOVi67JltL/8cu677z6g6g6E5eXm5nobGl122WX8/e9/p0+fPsTHx7Njx44yvaR9XXDBBd5e0H379vW2m125ciVTp06ld+/epKWlceTIEY4ePcq3337LmDFj6NmzJ9OmTWPHjh0AuFwuJkyYAECvXr3o1auX3/ONiIhg0KBBLF68mBMnThATE+N33WHDhtGmTRsiIyO5+uqr2et52uOv7vXr13PLLbcAcOutt7J27VrAfaXtcrlwuVw89NBDrF27lry8vKD7wEhoKMDFkbp37+H3qrRdu3ZERETQpEkTfp2e7h0m8de+NJDdu3eTnZ3Ne++9R35+PiNHjvTbLta33axv+9Vz586xfv167x+Fffv20bp1a2bNmsXQoUMpKCjgzTffLLPf6rSVTU9P58477+Tmm2+ucj1/7WH91V1e6TqDBw9mzZo1bNiwgREjRnD48GFWr15NUlJS0DVL7SnAxZFuv/0hZsyYwVdffQW435zgpZfcb/nl+0+8f779tncMOC0tjUWLFnHq1Cl2797Np59+Sv/+gV9AfOTIEVq2bEmbNm04cOAAy5cv997n2660KikpKWXePLe0L7dvK1ffNwLwbStbUFBAfn5+lfsfPHgwDz30kPdNIkJl4MCB3vH13NxcBg0aBLhbs77//vs0adKEyMhIevfuzbPPPlvpGylI+GgWilRbXXeTPX78ONHR0QCUlMDkyfcyefK9WHuA4cOHc/q0xRjDzTe7W/HMmDGDrXl5GGOIiY7mWc9bivXo0YObb76Zq6++mqZNm/LnP/+50hko5cXFxREfH0+PHj3o2rUr1157rfe+8u1K/cnJyeGOO+6gV69enD17lqSkJJ555hlmzJjBxIkTefLJJ7nuuuu860+ZMsXb7rZ3794B/9AYY7j//vsDnkt15eTkMGnSJJ544gmioqJ46aWXAPeVfKdOnUhMTATcf0BeffVVYmNjQ16D+FftdrK1oXayzlQX7WSDbRxYWTfCSjsUVrc9bB23kxXxpzrtZDWEIiLiUBpCEWcovUI+einU8K3PRBoaXYFLUOpyqE2ksaru75kCXAKKjIykuLhYIS4SRtZaiouLiYyMDHobDaFIQNHR0RQWFnLw4MGwHcPzht0cP171CsWnDsHhyDLrlm5bZvtKFwZRQLDri4RBZGSkd8ZVMBTgElCzZs3o0qVLWI9R2uHQ7xRFzwqZrvGQFF9mXd/uiN7tK10YRAHBri9yHtAQioiIQynARUQcSgEuIuJQCnAREYdSgIuIOJQCXETEoRTgIiIOpQAXEXEoBbiIiEMpwEVEHEoBLiLiUOqFIqERhl4iZXYZkj1Wvv+wtj5RjxUJI12Bi4g4lAJcRMShFOAiIg4VMMCNMZ2MMauMMbuMMTuMMXd7ll9qjHnXGPOp5/Ml4S9XRERKBXMFfha4z1p7FZAI3GGMuRp4EHjPWnsl8J7ntoiI1JGAAW6tLbLWbvZ8fRTYBXQEfgIs9Ky2EBgdphpFRKQS1RoDN8bEAPHAh0A7a20RuEMeuDzk1YmIiF9BzwM3xrQCXgfusdYeMcYEu10GkAHQuXPnmtQoDVE150dnusaH/ril+8zMDc2+RepYUFfgxphmuMM711q7xLP4gDGmvef+9sDXlW1rrX3OWptgrU2IiooKRc0iIkJws1AM8AKwy1r7pM9dbwATPV9PBJaGvjwREfEnmCGUa4Fbge3GmK2eZQ8DfwD+boz5FfAFMCYsFYqISKUCBri1di3gb8B7WGjLERGRYOmVmCIiDqUAFxFxKLWTlfqXmfn9lD6Sar4flyukUwIzM937fDbJs0+1g5XzjK7ARUQcSgEuIuJQCnAREYdSgIuIOJQCXETEoRTgIiIOpQAXEXEozQOXuuPbyjWU+wxVq1k8bWvDUKZIOOgKXETEoRTgIiIOpQAXEXEoBbiIiEMpwEVEHEoBLiLiUApwERGH0jxwCYlMz1xsb+/sUHC5arV5aU2+87pr0tHbdz9lWoL7zmtXr3CpB7oCFxFxKAW4iIhDKcBFRBxKAS4i4lAKcBERh1KAi4g4lAJcRMShNA9c6kRmJuAaH3CeeDhahos0VLoCFxFxKAW4iIhDKcBFRBxKAS4i4lABA9wY86Ix5mtjTIHPsixjzD5jzFbPx4jwlikiIuUFcwW+AEitZPlT1treno9loS1LREQCCRjg1loXcKgOahERkWqozTzwqcaY24CNwH3W2v9WtpIxJgPIAOjcuXMtDif1xl/f6zKTtsfXWTllDu8K4ri+fcWTanFAlwsyg+h37nlc3H3EA899F6mpmv4Tcz7wf4DeQBHwR38rWmufs9YmWGsToqKiang4EREpr0YBbq09YK0tsdaeA54H+oe2LBERCaRGAW6Mae9z86dAgb91RUQkPAKOgRtjXgWSgbbGmEJgNpBsjOkNWGAPZd51UERE6kLAALfWjqtk8QthqEVERKpBr8QUEXEoBbiIiEOpH7g0epnBzCUXOQ/pClxExKEU4CIiDqUAFxFxKAW4iIhDKcBFRBxKAS4i4lAKcBERh9I88MbMX5/v2uzSNT78nXF8+3ufB3zPubKH0TvPPDNkD7MIoCtwERHHUoCLiDiUAlxExKEU4CIiDqUAFxFxKAW4iIhDKcBFRBxK88Al9M6zedp1ovScM3Prtw5pVHQFLiLiUApwERGHUoCLiDiUAlxExKEU4CIiDqUAFxFxKAW4iIhDaR54QxWGXt8V9luTzUt7YwPPJp2nc6ZrMY/d9/yqXjFM3x9pVHQFLiLiUApwERGHUoCLiDiUAlxExKECBrgx5kVjzNfGmAKfZZcaY941xnzq+XxJeMsUEZHygrkCXwCkllv2IPCetfZK4D3PbRERqUMBA9xa6wIOlVv8E2Ch5+uFwOjQliUiIoHUdB54O2ttEYC1tsgYc7m/FY0xGUAGQOfOnWt4OAml0inI1Zl9XDq/+bydu+0R9Dzshqgac8s1Db1hCPs/Ma21z1lrE6y1CVFRUeE+nIhIo1HTAD9gjGkP4Pn8dehKEhGRYNQ0wN8AJnq+nggsDU05IiISrGCmEb4KrAd+aIwpNMb8CvgD8GNjzKfAjz23RUSkDgX8J6a1dpyfu4aFuBYREakGvRJTRMShFOAiIg6lfuBSRmYm3n7YgeZ813bOdaXb16IXd42PWcPtwzEn3jtHvxpzs71zul3jv69JE70bBV2Bi4g4lAJcRMShFOAiIg6lABcRcSgFuIiIQynARUQcSgEuIuJQmgcufjXq3tpBCMvjUzoPPjPX//xtzfEWD12Bi4g4lAJcRMShFOAiIg6lABcRcSgFuIiIQynARUQcSgEuIuJQmgfekPjOD67uplcF14dbc8NDo0xf8TLfN5/HV/O9JQBdgYuIOJQCXETEoRTgIiIOpQAXEXEoBbiIiEMpwEVEHEoBLiLiUJoHXo/qZZpvZiaUzkFOqqNjipvLReZVtdtFZibgGs+zSbkhKcn/QTw0//y8pitwERGHUoCLiDiUAlxExKEU4CIiDlWrf2IaY/YAR4ES4Ky1NiEURYmISGChmIUy1Fr7TQj2IyIi1aAhFBERh6rtFbgF3jHGWOBZa+1z5VcwxmQAGQCdO3eu5eEECHqerrfndKam8zpNhb7rmX7uD6YFvOZ1N1i1vQK/1lrbB7gBuMMYU+GlIdba56y1CdbahKioqFoeTkREStUqwK21+z2fvwb+CfQPRVEiIhJYjQPcGNPSGNO69GsgBSgIVWEiIlK12oyBtwP+aYwp3c8r1toVIalKREQCqnGAW2s/B+JCWIuIiFSDphGKiDiUAlxExKHUD/x84NsnOinp+6m6mZVP8i0zR/gql8894yusW2H/PutUmGss5y+XK/A6Pny/t769w/38SHnvcG8XRL9xzS0/L+gKXETEoRTgIiIOpQAXEXEoBbiIiEMpwEVEHEoBLiLiUApwERGH0jzwOpKZiXcu77NJuZo7K2FR2dz+sst85pMnVej+XLNjXuXzcw362a5DugIXEXEoBbiIiEMpwEVEHEoBLiLiUApwERGHUoCLiDiUAlxExKE0D9wjHO2N/fVeznSNL9fH24fLBZm5369XWlOg/sziWPXWl93zugR3r/iyNXhryiRgf/pglfkd4/sbmbgP4O/3rrLDBvodbSztynUFLiLiUApwERGHUoCLiDiUAlxExKEU4CIiDqUAFxFxKAW4iIhDOWceeLgnavvMxc28irK9kl1B9FD2Wcc7Z7uG83sD93QW8XD5eT1BKPdfyesSquR3vrif39vSc8isok9+EL+Dfg9bRXYEipVaxU75gsIwIV1X4CIiDqUAFxFxKAW4iIhDKcBFRByqVgFujEk1xnxsjPmPMebBUBUlIiKB1TjAjTERwJ+BG4CrgXHGmKtDVZiIiFStNlfg/YH/WGs/t9aeBhYBPwlNWSIiEoix1tZsQ2N+DqRaayd7bt8KDLDWTi23XgaQ4bn5Q+DjmpcbNm2Bb+q7iDrQGM5T59hwNIbzDPYcf2CtjSq/sDYv5DGVLKvw18Ba+xzwXC2OE3bGmI3W2oT6riPcGsN56hwbjsZwnrU9x9oMoRQCnXxuRwP7a7E/ERGphtoEeB5wpTGmizHmAiAdeCM0ZYmISCA1HkKx1p41xkwF3gYigBettTtCVlndOq+HeEKoMZynzrHhaAznWatzrPE/MUVEpH7plZgiIg6lABcRcahGH+ANvR2AMaaTMWaVMWaXMWaHMebu+q4pXIwxEcaYLcaYf9V3LeFijLnYGPOaMeYjz/f0mvquKdSMMdM8P6sFxphXjTGR9V1TKBhjXjTGfG2MKfBZdqkx5l1jzKeez5dUZ5+NOsAbSTuAs8B91tqrgETgjgZ4jqXuBnbVdxFh9n+BFdbaHwFxNLDzNcZ0BO4CEqy1PXFPkEiv36pCZgGQWm7Zg8B71torgfc8t4PWqAOcRtAOwFpbZK3d7Pn6KO5f+I71W1XoGWOigZHAX+q7lnAxxlwEJAEvAFhrT1trD9drUeHRFLjQGNMUaEEDeX2JtdYFHCq3+CfAQs/XC4HR1dlnYw/wjsCXPrcLaYDhVsoYEwPEAx/WcynhMBeYAZyr5zrCqStwEHjJM1T0F2NMy/ouKpSstfuAbOALoAj41lr7Tv1WFVbtrLVF4L7YAi6vzsaNPcCDagfQEBhjWgGvA/dYa4/Udz2hZIwZBXxtrd1U37WEWVOgDzDfWhsPfEc1n3Kf7zxjwD8BugAdgJbGmAn1W9X5q7EHeKNoB2CMaYY7vHOttUvqu54wuBZIM8bswT0Mdp0x5m/1W1JYFAKF1trSZ1Cv4Q70hmQ4sNtae9BaewZYAgys55rC6YAxpj2A5/PX1dm4sQd4g28HYIwxuMdMd1lrn6zvesLBWvuQtTbaWhuD+3v4/6y1De6qzVr7FfClMeaHnkXDgJ31WFI4fAEkGmNaeH52h9HA/lFbzhvARM/XE4Gl1dm4Nt0IHa+BtQPw51rgVmC7MWarZ9nD1tpl9VeS1MKdQK7nguNz4Jf1XE9IWWs/NMa8BmzGPYNqCw3kJfXGmFeBZKCtMaYQmA38Afi7MeZXuP94janWPvVSehERZ2rsQygiIo6lABcRcSgFuIiIQynARUQcSgEuIuJQCnAREYdSgIuIONT/B37PeSEz4576AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(np.log(np.random.lognormal(mean=mean, sigma=std, size=x.shape)),color='r',bins=100, lw=5, alpha=0.6, label=f'lognormal pdf mean={round(mean,2)}, std={round(std,2)}')\n",
    "plt.hist(train_frame['Tox'].astype(float).values,color='b', bins=100,lw=5, alpha=0.6, label='LC50 Fathead Minnow')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_fingerprints(frame):\n",
    "    \n",
    "    ms = [Chem.MolFromSmiles(smiles) for smiles in frame.SMILES]\n",
    "    fps = [list(Chem.RDKFingerprint(x).GetOnBits()) for x in ms]\n",
    "    fp_max = max([max(fp) for fp in fps])\n",
    "    v = np.zeros((len(fps),fp_max+1))\n",
    "    for i,fp_idx in enumerate(fps):\n",
    "        v[i,fp_idx] = 1\n",
    "    return v\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((659, 2048), (164, 2048), (659,), (164,))"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = get_fingerprints(train_frame)\n",
    "X_test = get_fingerprints(test_frame)\n",
    "y_train = train_frame['Tox'].astype(float).values\n",
    "y_test = test_frame['Tox'].astype(float).values\n",
    "X_train.shape,X_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f2f18485d50>"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD4CAYAAAAHHSreAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc50lEQVR4nO3df5BV5Z3n8ffHRmBHnUIJWL20DLjTpsQdy5AecN3EaCwTIEk1UxtmMWYlxiokK8kmKSuSZK1iN/8w1mxSQ+KK7CwTyKRCrDizdCrMukrFJZkKI5AYBAmhJax06AAhY6uxVNDv/nEf9HLP7XtPd9/u++vzqrp1zz3neU6fc7jcz3me80sRgZmZWbEL6r0AZmbWeBwOZmaW4XAwM7MMh4OZmWU4HMzMLGNSvRegFt7xjnfEnDlz6r0YZmZNZe/evb+NiBnlprVEOMyZM4c9e/bUezHMzJqKpP833DR3K5mZWYbDwczMMhwOZmaW0RLHHMo5c+YMAwMDvPrqq/VelHE3depUurq6uPDCC+u9KGbWIlo2HAYGBrjkkkuYM2cOkuq9OOMmIjh9+jQDAwPMnTu33otjZi2iZbuVXn31VaZPn97SwQAgienTp7dFC8nMJk7LhgPQ8sFwTrusp5lNnFzhIGmRpEOS+iWtKTNdktan6fskza9WV9Jlkh6XdDi9X5rGXyhps6RnJB2U9MVarKiZmeVX9ZiDpA7gQeBWYADYLakvIp4tKrYY6E6vhcBDwMIqddcAOyJiXQqNNcB9wDJgSkT8iaQ/AJ6V9J2IODqmNb377jFVz3j44YqTT58+zS233ALAb37zGzo6Opgxo3Ah4lNPPcXkyZOHrbtnzx62bNnC+vXra7e8ZmYjkOeA9AKgPyKOAEjaCvQCxeHQC2yJwpODdkmaJqkTmFOhbi9wU6q/GXiSQjgEcJGkScC/AF4HXhz9KtbH9OnTefrppwFYu3YtF198Mffee+9b08+ePcukSeU3f09PDz09PROxmGY2AsX7mFX2D5tenm6lWcCxos8DaVyeMpXqXh4RgwDpfWYa/z3g98Ag8DzwlxHxu9KFkrRS0h5Je06dOpVjNervE5/4BJ///Oe5+eabue+++3jqqae44YYbeNe73sUNN9zAoUOHAHjyySf58Ic/DBSC5ZOf/CQ33XQTV155pVsTZjYh8rQcyh3tLH226HBl8tQttQB4A/iXwKXAjyQ9ca718dZMIjYCGwF6enqa5lmnv/zlL3niiSfo6OjgxRdfZOfOnUyaNIknnniCL33pSzz66KOZOr/4xS/44Q9/yEsvvcQ73/lOPvWpT/maBjMbV3nCYQC4ouhzF3A8Z5nJFeqekNQZEYOpC+pkGv8x4H9HxBngpKR/BHqA88KhWS1btoyOjg4AhoaGWLFiBYcPH0YSZ86cKVvnQx/6EFOmTGHKlCnMnDmTEydO0NXVNZGLbWZtJk+30m6gW9JcSZOB5UBfSZk+4I501tL1wFDqKqpUtw9YkYZXANvS8PPA+9O8LgKuB34xyvVrOBdddNFbw/fffz8333wz+/fv5/vf//6w1ypMmTLlreGOjg7Onj077stpZu2tasshIs5KWg08BnQAmyLigKRVafoGYDuwBOgHXgHurFQ3zXod8IikuygEwrI0/kHgb4D9FLql/iYi9tViZRvN0NAQs2YVDsF885vfrO/CmJkVyXX7jIjYTiEAisdtKBoO4J68ddP408AtZca/zNtBUTsNeGrBF77wBVasWMFXv/pV3v/+99d7ccysmp07iz7cWLfFmAgq/K43t56enih92M/Bgwe5+uqr67REE6/d1tesHu6++u1wePhg84eDpL0RUfa8+Za+fYaZmY2Ow8HMzDIcDmZmluFwMDOzDIeDmZllOBzMzCyjZR8TWmqC79g9plt2Q+Hme5MnT+aGG26oyfKajUQ73X3UymubcJho1W7ZXc2TTz7JxRdf7HCw+miji72sPHcrTaC9e/fyvve9j3e/+9188IMfZHBwEID169czb948rr32WpYvX87Ro0fZsGEDX/va17juuuv40Y9+VOclN7N245bDBIkIPv3pT7Nt2zZmzJjBd7/7Xb785S+zadMm1q1bx69+9SumTJnCCy+8wLRp01i1atWIWxtmZrXicJggr732Gvv37+fWW28F4I033qCzsxOAa6+9lttvv52lS5eydOnSOi6lmVmBw2GCRATXXHMNP/nJTzLTfvCDH7Bz5076+vr4yle+woEDB8rMwcyaXTPdm8nHHCbIlClTOHXq1FvhcObMGQ4cOMCbb77JsWPHuPnmm3nggQd44YUXePnll7nkkkt46aWX6rzUZtau2qblUO/T8S644AK+973v8ZnPfIahoSHOnj3LZz/7Wa666io+/vGPMzQ0RETwuc99jmnTpvGRj3yEj370o2zbto2vf/3rvPe9763vCrQSn6dpVlXbhEM9rV279q3hneedIljw4x//ODPuqquuYt++lnzGkZk1AXcrmZlZRq5wkLRI0iFJ/ZLWlJkuSevT9H2S5lerK+kySY9LOpzeL03jb5f0dNHrTUnX1WBdzcwsp6rhIKmDwnOdFwPzgNskzSspthjoTq+VwEM56q4BdkREN7AjfSYivh0R10XEdcB/AI5GxNOjWblWeMpdHu2ynmY2cfK0HBYA/RFxJCJeB7YCvSVleoEtUbALmCaps0rdXmBzGt4MLC3zt28DvjOSFTpn6tSpnD59uuV/OCOC06dPM3Xq1Hovipm1kDwHpGcBx4o+DwALc5SZVaXu5RExCBARg5Jmlvnb/55sEAEgaSWFVgqzZ8/OTO/q6mJgYIBTp06VX6sWMnXqVLq6uuq9GGbWQvKEg8qMK90dH65Mnrrl/6i0EHglIvaXmx4RG4GNAD09PZl5XnjhhcydOzfPnzIzsxJ5upUGgCuKPncBx3OWqVT3ROp6Ir2fLJnnckbZpWRmZmOTp+WwG+iWNBf4NYUf7Y+VlOkDVkvaSqHbaCh1FZ2qULcPWAGsS+/bzs1M0gXAMnyvYBsHd++8/a1hXwJnVl7VcIiIs5JWA48BHcCmiDggaVWavgHYDiwB+oFXgDsr1U2zXgc8Iuku4HkKYXDOjcBARBypwTqamdkI5bpCOiK2UwiA4nEbioYDuCdv3TT+NHDLMHWeBK7Ps2xmZlZ7vkLazMwyHA5mZpbhcDAzswyHg5mZZTgczMwsw89zsJbi5/iY1YZbDmZmluFwMDOzDHcrWWs57zGsb999pbi7ycyqc8vBzMwyHA5mZpbhbiVrD+d1N5lZNQ6HieZzLdub//2tSTgcJpifJWBmzcDHHMzMLMPhYGZmGe5WssbmPnqzusjVcpC0SNIhSf2S1pSZLknr0/R9kuZXqyvpMkmPSzqc3i8tmnatpJ9IOiDpGUlTx7qiZmaWX9VwkNQBPAgsBuYBt0maV1JsMdCdXiuBh3LUXQPsiIhuYEf6jKRJwN8CqyLiGuAm4MzoV9HMzEYqT8thAdAfEUci4nVgK9BbUqYX2BIFu4Bpkjqr1O0FNqfhzcDSNPwBYF9E/BwKz5qOiDdGt3pmZjYaeY45zAKOFX0eABbmKDOrSt3LI2IQICIGJc1M468CQtJjwAxga0Q8ULpQklZSaKUwe/bsHKthZjYK592Y6/Zhi7WaPOGgMuMiZ5k8dcst03uAPwVeAXZI2hsRO86bScRGYCNAT09PtXlOPB9INbMmliccBoArij53Acdzlplcoe4JSZ2p1dAJnCya1/+NiN8CSNoOzKdwXMKsPmoU9r4Isj68rzZyeY457Aa6Jc2VNBlYDvSVlOkD7khnLV0PDKUuo0p1+4AVaXgFsC0NPwZcK+kP0sHp9wHPjnL9zMxG7O673361q6oth4g4K2k1hR/tDmBTRByQtCpN3wBsB5YA/RS6gu6sVDfNeh3wiKS7gOeBZanOP0v6KoVgCWB7RPygVitszcV72lZvxd/BdpLrIriI2E4hAIrHbSgaDuCevHXT+NPALcPU+VsKp7Oa2XiaiP6WRujTGeYhUDY8XyFtlkOrtmBadb2aQiOEZgUOh3ZX2qnagF/SRtPg/6fNasLhYLXjX02z3Bq91eZwsJpp9C/7eHAeWqtyOLS50jMx/PuWQ/HBzRt9cLOaptxp8GNlHQ7jpSn/Q0ww73WbNS6HQ4PzD6iZ1YPDoREVJ8J53T7uwmg4Oc6fb+erbK15ORzMzMZRs+4cOBzMzMZTkx7cdjiYjbcm/XGw9uZwMGszzdrNYRPL4TAB/J9xGGO4GVqtzuLy2WBm5TkcrPm16WMczcaTw6GJeC93fLXN9vUxEMvB4dAC2uZHrdh5t7AYh/ln+gKbsEXSll+MxtAKXcm5wkHSIuCvKDzN7a8jYl3JdKXpSyg8Ce4TEfHTSnUlXQZ8F5gDHAX+PD0Fbg5wEDiUZr8rIlaNfhXNKmjh+yTV6hYulTLG+dO6qoaDpA7gQeBWYADYLakvIoqf67wY6E6vhcBDwMIqddcAOyJinaQ16fN9aX7PRcR1tVjBZjTixxL6KVdmjaUFuu7ytBwWAP0RcQRA0lagFygOh15gS3pc6C5J0yR1UmgVDFe3F7gp1d8MPMnb4WCtpB13L8f5IPndV7/94/PwQe8QWO3lCYdZwLGizwMUWgfVysyqUvfyiBgEiIhBSTOLys2V9DPgReA/R8SPShdK0kpgJcDs2bNzrMb4yPW71wJ7EdYkJvrMLbdaW1aecFCZcZGzTJ66pQaB2RFxWtK7gf8l6ZqIePG8mURsBDYC9PT0VJunjUaN9vjb8fblI+4abHPDHcBtl4ZmI8oTDgPAFUWfu4DjOctMrlD3hKTO1GroBE4CRMRrwGtpeK+k54CrgD251qiVeS+tsfl6C2shecJhN9AtaS7wa2A58LGSMn3A6nRMYSEwlH70T1Wo2wesANal920AkmYAv4uINyRdSeEg95ExrKONwHl92c6fpjMuLRbvlLSlquEQEWclrQYeo3A66qaIOCBpVZq+AdhO4TTWfgqnst5ZqW6a9TrgEUl3Ac8Dy9L4G4H/Kuks8AawKiJ+V5O1tRFpx+6gcxqyW6iVWyYtfErxSDXK+Ru5rnOIiO0UAqB43Iai4QDuyVs3jT8N3FJm/KPAo3mWy8xa0HAtlUb51WwTvkLazOqiFa4ibmUOB8unVmcu+QfBGlTx8ba6apBjPA6HsWqQf8im4Ws+zMZsInrYLhif2ZqZWTNzy8Fyaeczl/JqyDOcGtkIW5HnfQd9cHrcORxGo5VPKczD/zHNWp7DwZpezfbYJ/p4iEPWGpjDoQLf+dLG03mhliMn3G3VYGoZ7g3YG+FwMLOa8ynLI5Mr+M/bqOPf0nQ41JF7FewtPiXaGozDocRY9ngmdG+pjs3QpjlzqQGb6jYyDXNhWhlN8/9glBwOZs2uDUOw1X+Yyzm/62n8W5oOB0r+b+U4Y2U8DgxO5B6S+4Nbiw9Uj14jt0zqzeGQVxvuneXRyEHTrD+aPhZljcC3zzAzswy3HJrUmPaKa3mx1zDzcnO9zdXpBotuddWOw8HMWkdxKN397aIJzdnFWE+5wkHSIuCvKDzq868jYl3JdKXpSyg8JvQTEfHTSnUlXQZ8F5gDHAX+PCL+uWies4FngbUR8ZejX8UJ5NtRWy006TUPdWst+njguKh6zEFSB/AgsBiYB9wmaV5JscVAd3qtBB7KUXcNsCMiuoEd6XOxrwH/MIp1GrmdO99+mZlZrpbDAqA/Io4ASNoK9FLYqz+nF9iSniW9S9I0SZ0UWgXD1e0Fbkr1NwNPAvelckuBI8DvR79qdp4m2rtq5DOgmpW3qY1UnnCYBRwr+jwALMxRZlaVupdHxCBARAxKmgkg6SIKIXErcO9wCyVpJYVWCrNnz86xGmPTrKdFNo0Jvm+M1YdDqnnkCQeVGRc5y+SpW+q/AF+LiJcLhzLKi4iNwEaAnp6eavNsTOPRjeXTNWyU2uWHu13Wc6zyhMMAcEXR5y7geM4ykyvUPSGpM7UaOoGTafxC4KOSHgCmAW9KejUivpFjWdvecLcVcMvHWpW/2+MjTzjsBrolzQV+DSwHPlZSpg9YnY4pLASG0o/+qQp1+4AVwLr0vg0gIt57bqaS1gIvOxha33n/wZvnBJ32k9ntbsIfZp94kkvVcIiIs5JWA49ROB11U0QckLQqTd8AbKdwGms/hVNZ76xUN816HfCIpLuA54FlNV0zM6u5Me+lT+APs1sUY5PrOoeI2E4hAIrHbSgaDuCevHXT+NPALVX+7to8y2ctxnt2b6vVWWbepjZCvreSmZll+PYZZmaNqo7XJzkcrGn4FESzieNwMDNrUPU8qO5wMLMCH7QetVZs1ToczMzGqgWD1WcrmZlZhlsOLazlmrotuHdWjS/ksnpxy8HMzDLccmhlbbinbWa14ZaDmZllOBzMzCzD4WBmZhkOBzMzy3A4mJlZhsPBzMwyHA5mZpaRKxwkLZJ0SFK/pDVlpkvS+jR9n6T51epKukzS45IOp/dL0/gFkp5Or59L+rNarKiZmeVXNRwkdQAPAouBecBtkuaVFFsMdKfXSuChHHXXADsiohvYkT4D7Ad6IuI6YBHwsCRfrGdmNoHytBwWAP0RcSQiXge2Ar0lZXqBLVGwC5gmqbNK3V5gcxreDCwFiIhXIuJsGj8ViNGtmpmZjVaecJgFHCv6PJDG5SlTqe7lETEIkN5nniskaaGkA8AzwKqisKCozEpJeyTtOXXqVI7VMDOzvPKEg8qMK92bH65MnrrZAhH/FBHXAH8KfFHS1DJlNkZET0T0zJgxo9oszcxsBPKEwwBwRdHnLuB4zjKV6p5IXU+k95OlfzgiDgK/B/51juU0M7MayRMOu4FuSXMlTQaWA30lZfqAO9JZS9cDQ6mrqFLdPmBFGl4BbANIZSel4T8C3gkcHe0KmpnZyFU9CygizkpaDTwGdACbIuKApFVp+gZgO7AE6AdeAe6sVDfNeh3wiKS7gOeBZWn8e4A1ks4AbwL/MSJ+W5O1NTOzXHKdIhoR2ykEQPG4DUXDAdyTt24afxq4pcz4bwHfyrNcZmY2PnyFtJmZZTgczMwsw+FgZmYZDgczM8twOJiZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzjFzhIGmRpEOS+iWtKTNdktan6fskza9WV9Jlkh6XdDi9X5rG3yppr6Rn0vv7a7GiZmaWX9VwkNQBPAgsBuYBt0maV1JsMdCdXiuBh3LUXQPsiIhuYEf6DPBb4CMR8ScUni3tp8KZmU2wPC2HBUB/RByJiNeBrUBvSZleYEsU7AKmSeqsUrcX2JyGNwNLASLiZxFxPI0/AEyVNGV0q2dmZqORJxxmAceKPg+kcXnKVKp7eUQMAqT3mWX+9r8DfhYRr5VOkLRS0h5Je06dOpVjNczMLK884aAy4yJnmTx1y/9R6RrgL4C7y02PiI0R0RMRPTNmzMgzSzMzyylPOAwAVxR97gKO5yxTqe6J1PVEej95rpCkLuDvgTsi4rkcy2hmZjWUJxx2A92S5kqaDCwH+krK9AF3pLOWrgeGUldRpbp9FA44k963AUiaBvwA+GJE/OPoV83MzEZrUrUCEXFW0mrgMaAD2BQRByStStM3ANuBJUA/8ApwZ6W6adbrgEck3QU8DyxL41cDfwzcL+n+NO4DEfFWy8LMzMaXInIdAmhoPT09sWfPnlHXv/vqnTVcGjOzifPwwRtHXVfS3ojoKTfNV0ibmVmGw8HMzDIcDmZmluFwMDOzDIeDmZllOBzMzCzD4WBmZhkOBzMzy3A4mJlZhsPBzMwyHA5mZpbhcDAzswyHg5mZZTgczMwsw+FgZmYZDgczM8twOJiZWUaucJC0SNIhSf2S1pSZLknr0/R9kuZXqyvpMkmPSzqc3i9N46dL+qGklyV9oxYraWZmI1M1HCR1AA8Ci4F5wG2S5pUUWwx0p9dK4KEcddcAOyKiG9iRPgO8CtwP3Dv61TIzs7HI03JYAPRHxJGIeB3YCvSWlOkFtkTBLmCapM4qdXuBzWl4M7AUICJ+HxE/phASZmZWB3nCYRZwrOjzQBqXp0ylupdHxCBAep+Zf7FB0kpJeyTtOXXq1EiqmplZFXnCQWXGRc4yeeqOSkRsjIieiOiZMWNGLWZpZmZJnnAYAK4o+twFHM9ZplLdE6nrifR+Mv9im5nZeMoTDruBbklzJU0GlgN9JWX6gDvSWUvXA0Opq6hS3T5gRRpeAWwb47qYmVmNTKpWICLOSloNPAZ0AJsi4oCkVWn6BmA7sAToB14B7qxUN816HfCIpLuA54Fl5/6mpKPAHwKTJS0FPhARz459dc3MLI+q4QAQEdspBEDxuA1FwwHck7duGn8auGWYOnPyLJeZmY0PXyFtZmYZDgczM8twOJiZWYbDwczMMhwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HMzPLcDiYmVmGw8HMzDIcDmZmluFwMDOzDIeDmZllOBzMzCzD4WBmZhm5wkHSIkmHJPVLWlNmuiStT9P3SZpfra6kyyQ9Lulwer+0aNoXU/lDkj441pU0M7ORqRoOkjqAB4HFwDzgNknzSootBrrTayXwUI66a4AdEdEN7EifSdOXA9cAi4D/nuZjZmYTJE/LYQHQHxFHIuJ1YCvQW1KmF9gSBbuAaZI6q9TtBTan4c3A0qLxWyPitYj4FdCf5mNmZhNkUo4ys4BjRZ8HgIU5ysyqUvfyiBgEiIhBSTOL5rWrzLzOI2klhVYKwMuSDuVYl+G8A/jtGOq3C2+nfLyd8vO2ymfY7bRRY5rvHw03IU84lPvTkbNMnrqj+XtExEZgY5V55SJpT0T01GJerczbKR9vp/y8rfKpx3bK0600AFxR9LkLOJ6zTKW6J1LXE+n95Aj+npmZjaM84bAb6JY0V9JkCgeL+0rK9AF3pLOWrgeGUpdRpbp9wIo0vALYVjR+uaQpkuZSOMj91CjXz8zMRqFqt1JEnJW0GngM6AA2RcQBSavS9A3AdmAJhYPHrwB3VqqbZr0OeETSXcDzwLJU54CkR4BngbPAPRHxRq1WeBg16Z5qA95O+Xg75edtlc+EbydFVDsEYGZm7cZXSJuZWYbDwczMMto6HKrdFqTdSDoq6RlJT0vak8b5NieApE2STkraXzRuxNtG0rvTNu5Pt5wZ21nqDWaY7bRW0q/T9+ppSUuKprXrdrpC0g8lHZR0QNJ/SuMb5zsVEW35onCA/DngSmAy8HNgXr2Xq87b5CjwjpJxDwBr0vAa4C/S8Ly0zaYAc9O27Kj3OozjtrkRmA/sH8u2oXDm3b+hcD3PPwCL671uE7Cd1gL3linbztupE5ifhi8Bfpm2R8N8p9q55ZDntiDm25wAEBE7gd+VjB7RtknX8/xhRPwkCv+rtxTVaQnDbKfhtPN2GoyIn6bhl4CDFO4E0TDfqXYOh+Fu+dHOAvg/kvam25NAyW1OgOLbnLT79hvptpmVhkvHt4PV6Y7Nm4q6SrydAElzgHcB/0QDfafaORxGc2uPVvdvI2I+hbvo3iPpxgplvf2GV8vbybSCh4B/BVwHDAL/LY1v++0k6WLgUeCzEfFipaJlxo3rtmrncPBtOkpExPH0fhL4ewrdRL7NyfBGum0G0nDp+JYWESci4o2IeBP4H7zd/djW20nShRSC4dsR8XdpdMN8p9o5HPLcFqRtSLpI0iXnhoEPAPvxbU4qGdG2Sd0EL0m6Pp1RckdRnZZ17scu+TMK3yto4+2U1ut/Agcj4qtFkxrnO1Xvo/Z1PmNgCYWzBJ4Dvlzv5anztriSwtkQPwcOnNsewHQKD2M6nN4vK6rz5bTtDtFiZ5OU2T7fodAlcobC3tpdo9k2QA+FH8fngG+Q7lLQKq9httO3gGeAfelHrtPbifdQ6P7ZBzydXksa6Tvl22eYmVlGO3crmZnZMBwOZmaW4XAwM7MMh4OZmWU4HMzMLMPhYGZmGQ4HMzPL+P8FSlZOkA4aEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train_idx = np.where(X_train == 1)[1]\n",
    "X_test_idx = np.where(X_test == 1)[1]\n",
    "plt.hist(X_train_idx,bins=100,color='r',lw=5,alpha=0.6,label='Train',density=True)\n",
    "plt.hist(X_test_idx,bins=100,color='b',lw=5,alpha=0.6,label='Test',density=True)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import tensorflow as tf\n",
    "\n",
    "#encoder = tf.keras.models.Sequential([tf.keras.layers.Dense(512,activation='relu'),\n",
    " #                                    tf.keras.layers.Dense(128,activation='relu')])\n",
    "\n",
    "#decoder = tf.keras.models.Sequential([tf.keras.layers.Dense(512,activation='relu'),\n",
    "  #                                   tf.keras.layers.Dense(X_train.shape[-1],activation='sigmoid')])\n",
    "\n",
    "#autoencoder = tf.keras.models.Sequential([encoder,decoder])\n",
    "#autoencoder.compile(optimizer='adam',loss='binary_crossentropy')\n",
    "#autoencoder.fit(X_train,X_train,epochs=2000,batch_size=1024)\n",
    "\n",
    "#X_train = encoder.predict(X_train)\n",
    "#X_test = encoder.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "class RSquared(tf.keras.metrics.Metric):\n",
    "    def __init__(self,name='r_squared',**kwargs):\n",
    "        super(RSquared, self).__init__(name=name, **kwargs)\n",
    "        self.r2 = self.add_weight(name='r2', initializer='zeros')\n",
    "        \n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        SS_res = K.sum(K.square( y_true-y_pred ))\n",
    "        SS_tot = K.sum(K.square( y_true - K.mean(y_true) ) )\n",
    "        \n",
    "        self.r2.assign_add(1 - SS_res/(SS_tot + K.epsilon()))\n",
    "\n",
    "    def result(self):\n",
    "        return self.r2\n",
    "    \n",
    "def coeff_determination(y_true, y_pred):\n",
    "    SS_res =  K.sum(K.square( y_true-y_pred ))\n",
    "    SS_tot = K.sum(K.square( y_true - K.mean(y_true) ) )\n",
    "    return ( 1 - SS_res/(SS_tot + K.epsilon()) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "def set_all_seeds(seed=42):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "def robust_loss(alpha=1,c=1):\n",
    "    assert c > 0\n",
    "    #https://openaccess.thecvf.com/content_CVPR_2019/papers/Barron_A_General_and_Adaptive_Robust_Loss_Function_CVPR_2019_paper.pdf\n",
    "    def loss(y_true,y_pred):\n",
    "        e = y_true - y_pred\n",
    "        if alpha == 2:\n",
    "            l = 0.5 * (e/c)**2\n",
    "        elif alpha == 0:\n",
    "            l = tf.math.log(0.5*(e/c)**2 + 1)\n",
    "        else:\n",
    "            a = abs(alpha-2)/alpha\n",
    "            div = tf.math.pow(e/c,2) / abs(alpha-2)\n",
    "            div = tf.math.pow(div+1,alpha/2)\n",
    "            l = a * (div - 1)\n",
    "        return tf.reduce_mean(l)\n",
    "    \n",
    "    return loss\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Failed to import pydot. You must `pip install pydot` and install graphviz (https://graphviz.gitlab.io/download/), ', 'for `pydotprint` to work.')\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_probability as tfp\n",
    "tfd = tfp.distributions\n",
    "import tensorflow_addons as tfa\n",
    "\n",
    "def create_model(space):\n",
    "    model = tf.keras.models.Sequential()\n",
    "    \n",
    "    model.add(tf.keras.layers.Embedding(len(np.unique(X_train[0])),space['embedding_dim'],input_length=X_train.shape[-1],\n",
    "                                        embeddings_initializer=tf.keras.initializers.GlorotNormal()))\n",
    "    \n",
    "    #model.add(tf.keras.layers.Bidirectional(tf.keras.layers.GRU(64,kernel_initializer=tf.keras.initializers.HeNormal()),merge_mode='concat'))\n",
    "    #model.add(tf.keras.layers.Conv1D(32,3,activation='relu'))\n",
    "    #model.add(tf.keras.layers.AveragePooling1D(3))\n",
    "    \n",
    "    model.add(tf.keras.layers.Flatten())\n",
    "    \n",
    "    for i in range(space['num_layers']):\n",
    "        model.add(tf.keras.layers.Dense(space[f'units_{i}'],\n",
    "                                        kernel_initializer=tf.keras.initializers.GlorotNormal()))\n",
    "        model.add(tf.keras.layers.BatchNormalization())\n",
    "        model.add(tf.keras.layers.Activation(tf.keras.activations.relu))\n",
    "        model.add(tf.keras.layers.Dropout(space[f'dropout_{i}']))\n",
    "    model.add(tf.keras.layers.Dense(1,bias_initializer=tf.keras.initializers.Constant(y_train.mean()),\n",
    "                                   kernel_initializer=tf.keras.initializers.GlorotNormal()))\n",
    "    model.compile(optimizer=tf.keras.optimizers.SGD(space['learning_rate']),\n",
    "                  loss=tf.keras.losses.MeanSquaredError(),\n",
    "                 metrics = [coeff_determination])\n",
    "    return model\n",
    "\n",
    "model=create_model(defaultdict(lambda : 1))\n",
    "tf.keras.utils.plot_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold, train_test_split\n",
    "from sklearn.metrics import r2_score\n",
    "from hyperopt import Trials, STATUS_OK, tpe, hp, fmin\n",
    "import warnings\n",
    "\n",
    "FOLDS = len(y_train)//len(y_test)\n",
    "\n",
    "def objective(space):\n",
    "    \n",
    "    oof = np.zeros(y_train.shape)\n",
    "    for fold,(train_idx,test_idx) in enumerate(KFold(FOLDS).split(y_train)):\n",
    "        \n",
    "        regr = create_model(space)\n",
    "        hist = regr.fit(X_train[train_idx],y_train[train_idx],\n",
    "                validation_data=(X_train[test_idx],y_train[test_idx]),\n",
    "                epochs=1000,\n",
    "                batch_size=16,\n",
    "                verbose=0,\n",
    "                callbacks=[tf.keras.callbacks.EarlyStopping('val_loss',patience=20,\n",
    "                                                           restore_best_weights=True)])\n",
    "        \n",
    "        oof[test_idx] += regr.predict(X_train[test_idx]).ravel()\n",
    "        \n",
    "    try:\n",
    "        r2 = r2_score(y_train,oof)\n",
    "    except:\n",
    "        r2 = 0\n",
    "        \n",
    "    CrossValMean = 1 - r2\n",
    "    \n",
    "    return{'loss':CrossValMean, 'status': STATUS_OK}\n",
    "\n",
    "    \n",
    "max_layers = 4\n",
    "space = {'num_layers':hp.choice('num_layers',range(1,max_layers)),\n",
    "        'learning_rate':hp.quniform('learning_rate',1e-4,1e-2,1e-5),\n",
    "        'embedding_dim':hp.choice('embedding_dim',range(4,33))}\n",
    "for i in range(max_layers):\n",
    "    space[f'units_{i}'] = hp.choice(f'units_{i}',range(64,512))\n",
    "    space[f'dropout_{i}'] = hp.quniform(f'dropout_{i}',0.1,0.5,0.1)\n",
    "\n",
    "trials = Trials()\n",
    "#best = fmin(fn=objective,\n",
    "#            space=space,\n",
    "#            algo=tpe.suggest,\n",
    "#            max_evals=50,\n",
    "#            trials=trials)\n",
    "#print(\"Best: \", best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "31/31 - 1s - loss: 2.8109 - coeff_determination: -8.9537e-01 - val_loss: 2.0061 - val_coeff_determination: -1.1331e-02\n",
      "Epoch 2/200\n",
      "31/31 - 1s - loss: 2.0872 - coeff_determination: -1.9736e-01 - val_loss: 2.0664 - val_coeff_determination: 0.0101\n",
      "Epoch 3/200\n",
      "31/31 - 1s - loss: 1.5962 - coeff_determination: 0.1335 - val_loss: 1.7765 - val_coeff_determination: 0.1086\n",
      "Epoch 4/200\n",
      "31/31 - 1s - loss: 1.3918 - coeff_determination: 0.2327 - val_loss: 1.7118 - val_coeff_determination: 0.0416\n",
      "Epoch 5/200\n",
      "31/31 - 1s - loss: 1.0191 - coeff_determination: 0.4011 - val_loss: 1.5364 - val_coeff_determination: 0.1599\n",
      "Epoch 6/200\n",
      "31/31 - 1s - loss: 0.9579 - coeff_determination: 0.4796 - val_loss: 1.6384 - val_coeff_determination: 0.0776\n",
      "Epoch 7/200\n",
      "31/31 - 1s - loss: 0.8947 - coeff_determination: 0.4998 - val_loss: 1.3389 - val_coeff_determination: 0.3222\n",
      "Epoch 8/200\n",
      "31/31 - 1s - loss: 0.9028 - coeff_determination: 0.4685 - val_loss: 1.4998 - val_coeff_determination: 0.2998\n",
      "Epoch 9/200\n",
      "31/31 - 1s - loss: 0.8785 - coeff_determination: 0.4114 - val_loss: 1.8116 - val_coeff_determination: 0.1346\n",
      "Epoch 10/200\n",
      "31/31 - 1s - loss: 0.8352 - coeff_determination: 0.5222 - val_loss: 1.2290 - val_coeff_determination: 0.3081\n",
      "Epoch 11/200\n",
      "31/31 - 1s - loss: 0.7610 - coeff_determination: 0.5235 - val_loss: 1.6595 - val_coeff_determination: 0.1854\n",
      "Epoch 12/200\n",
      "31/31 - 1s - loss: 0.6923 - coeff_determination: 0.6091 - val_loss: 1.5501 - val_coeff_determination: -6.8298e-02\n",
      "Epoch 13/200\n",
      "31/31 - 1s - loss: 0.6384 - coeff_determination: 0.6523 - val_loss: 1.1145 - val_coeff_determination: 0.2817\n",
      "Epoch 14/200\n",
      "31/31 - 1s - loss: 0.6808 - coeff_determination: 0.6229 - val_loss: 1.1752 - val_coeff_determination: 0.2590\n",
      "Epoch 15/200\n",
      "31/31 - 1s - loss: 0.7096 - coeff_determination: 0.6206 - val_loss: 1.0866 - val_coeff_determination: 0.4419\n",
      "Epoch 16/200\n",
      "31/31 - 1s - loss: 0.7373 - coeff_determination: 0.6046 - val_loss: 1.2735 - val_coeff_determination: 0.3801\n",
      "Epoch 17/200\n",
      "31/31 - 1s - loss: 0.6727 - coeff_determination: 0.6391 - val_loss: 1.1557 - val_coeff_determination: 0.4128\n",
      "Epoch 18/200\n",
      "31/31 - 1s - loss: 0.6169 - coeff_determination: 0.6462 - val_loss: 1.1041 - val_coeff_determination: 0.4255\n",
      "Epoch 19/200\n",
      "31/31 - 1s - loss: 0.6188 - coeff_determination: 0.6336 - val_loss: 1.0534 - val_coeff_determination: 0.4418\n",
      "Epoch 20/200\n",
      "31/31 - 1s - loss: 0.5521 - coeff_determination: 0.7053 - val_loss: 1.0492 - val_coeff_determination: 0.4488\n",
      "Epoch 21/200\n",
      "31/31 - 1s - loss: 0.5612 - coeff_determination: 0.6876 - val_loss: 1.0444 - val_coeff_determination: 0.4515\n",
      "Epoch 22/200\n",
      "31/31 - 1s - loss: 0.6990 - coeff_determination: 0.5411 - val_loss: 1.0701 - val_coeff_determination: 0.4478\n",
      "Epoch 23/200\n",
      "31/31 - 1s - loss: 0.6042 - coeff_determination: 0.6821 - val_loss: 1.0721 - val_coeff_determination: 0.4466\n",
      "Epoch 24/200\n",
      "31/31 - 1s - loss: 0.5712 - coeff_determination: 0.6794 - val_loss: 1.0474 - val_coeff_determination: 0.4485\n",
      "Epoch 25/200\n",
      "31/31 - 1s - loss: 0.5579 - coeff_determination: 0.6931 - val_loss: 1.0466 - val_coeff_determination: 0.4488\n",
      "Epoch 26/200\n",
      "31/31 - 1s - loss: 0.5892 - coeff_determination: 0.6694 - val_loss: 1.0474 - val_coeff_determination: 0.4494\n",
      "Epoch 27/200\n",
      "31/31 - 1s - loss: 0.5794 - coeff_determination: 0.6732 - val_loss: 1.0484 - val_coeff_determination: 0.4492\n",
      "Epoch 28/200\n",
      "31/31 - 1s - loss: 0.5758 - coeff_determination: 0.6818 - val_loss: 1.0500 - val_coeff_determination: 0.4489\n",
      "Epoch 29/200\n",
      "31/31 - 1s - loss: 0.5606 - coeff_determination: 0.6484 - val_loss: 1.0505 - val_coeff_determination: 0.4488\n",
      "Epoch 30/200\n",
      "31/31 - 1s - loss: 0.6047 - coeff_determination: 0.6534 - val_loss: 1.0507 - val_coeff_determination: 0.4489\n",
      "Epoch 31/200\n",
      "31/31 - 1s - loss: 0.5780 - coeff_determination: 0.6631 - val_loss: 1.0510 - val_coeff_determination: 0.4486\n",
      "Epoch 32/200\n",
      "31/31 - 1s - loss: 0.5941 - coeff_determination: 0.6347 - val_loss: 1.0512 - val_coeff_determination: 0.4488\n",
      "Epoch 33/200\n",
      "31/31 - 1s - loss: 0.6046 - coeff_determination: 0.6531 - val_loss: 1.0508 - val_coeff_determination: 0.4489\n",
      "Epoch 34/200\n",
      "31/31 - 1s - loss: 0.5160 - coeff_determination: 0.7009 - val_loss: 1.0513 - val_coeff_determination: 0.4487\n",
      "Epoch 35/200\n",
      "31/31 - 1s - loss: 0.5723 - coeff_determination: 0.6923 - val_loss: 1.0520 - val_coeff_determination: 0.4487\n",
      "Epoch 36/200\n",
      "31/31 - 1s - loss: 0.5568 - coeff_determination: 0.6457 - val_loss: 1.0523 - val_coeff_determination: 0.4487\n",
      "Epoch 37/200\n",
      "31/31 - 1s - loss: 0.5696 - coeff_determination: 0.6760 - val_loss: 1.0512 - val_coeff_determination: 0.4486\n",
      "Epoch 38/200\n",
      "31/31 - 1s - loss: 0.6103 - coeff_determination: 0.6530 - val_loss: 1.0511 - val_coeff_determination: 0.4484\n",
      "Epoch 39/200\n",
      "31/31 - 1s - loss: 0.5530 - coeff_determination: 0.7092 - val_loss: 1.0515 - val_coeff_determination: 0.4485\n",
      "Epoch 40/200\n",
      "31/31 - 1s - loss: 0.5649 - coeff_determination: 0.7057 - val_loss: 1.0513 - val_coeff_determination: 0.4486\n",
      "Epoch 41/200\n",
      "31/31 - 1s - loss: 0.5110 - coeff_determination: 0.7225 - val_loss: 1.0509 - val_coeff_determination: 0.4487\n",
      "Epoch 1/200\n",
      "31/31 - 1s - loss: 2.9600 - coeff_determination: -6.1153e-01 - val_loss: 2.1525 - val_coeff_determination: -2.7873e-01\n",
      "Epoch 2/200\n",
      "31/31 - 1s - loss: 1.8741 - coeff_determination: -7.0114e-02 - val_loss: 1.5912 - val_coeff_determination: 0.0868\n",
      "Epoch 3/200\n",
      "31/31 - 1s - loss: 1.3444 - coeff_determination: 0.1937 - val_loss: 1.5046 - val_coeff_determination: 0.1312\n",
      "Epoch 4/200\n",
      "31/31 - 1s - loss: 1.2160 - coeff_determination: 0.3080 - val_loss: 1.7852 - val_coeff_determination: -7.3194e-02\n",
      "Epoch 5/200\n",
      "31/31 - 1s - loss: 1.0703 - coeff_determination: 0.3786 - val_loss: 1.3702 - val_coeff_determination: 0.2036\n",
      "Epoch 6/200\n",
      "31/31 - 1s - loss: 0.9880 - coeff_determination: 0.4658 - val_loss: 1.3615 - val_coeff_determination: 0.1902\n",
      "Epoch 7/200\n",
      "31/31 - 1s - loss: 1.1269 - coeff_determination: 0.3420 - val_loss: 1.7865 - val_coeff_determination: -1.0376e-01\n",
      "Epoch 8/200\n",
      "31/31 - 1s - loss: 0.9338 - coeff_determination: 0.4362 - val_loss: 1.4192 - val_coeff_determination: 0.1622\n",
      "Epoch 9/200\n",
      "31/31 - 1s - loss: 0.9020 - coeff_determination: 0.4845 - val_loss: 1.3225 - val_coeff_determination: 0.2349\n",
      "Epoch 10/200\n",
      "31/31 - 1s - loss: 0.8299 - coeff_determination: 0.5087 - val_loss: 1.3551 - val_coeff_determination: 0.2208\n",
      "Epoch 11/200\n",
      "31/31 - 1s - loss: 0.8847 - coeff_determination: 0.4920 - val_loss: 2.2694 - val_coeff_determination: -3.6833e-01\n",
      "Epoch 12/200\n",
      "31/31 - 1s - loss: 0.7317 - coeff_determination: 0.5872 - val_loss: 1.1188 - val_coeff_determination: 0.3369\n",
      "Epoch 13/200\n",
      "31/31 - 1s - loss: 0.7524 - coeff_determination: 0.5878 - val_loss: 2.5096 - val_coeff_determination: -6.2667e-01\n",
      "Epoch 14/200\n",
      "31/31 - 1s - loss: 0.7969 - coeff_determination: 0.5459 - val_loss: 1.1414 - val_coeff_determination: 0.3264\n",
      "Epoch 15/200\n",
      "31/31 - 1s - loss: 0.6766 - coeff_determination: 0.6259 - val_loss: 1.6560 - val_coeff_determination: -5.0686e-02\n",
      "Epoch 16/200\n",
      "31/31 - 1s - loss: 0.6853 - coeff_determination: 0.6224 - val_loss: 1.2596 - val_coeff_determination: 0.2297\n",
      "Epoch 17/200\n",
      "31/31 - 1s - loss: 0.5980 - coeff_determination: 0.6666 - val_loss: 1.1291 - val_coeff_determination: 0.3294\n",
      "Epoch 18/200\n",
      "31/31 - 1s - loss: 0.6196 - coeff_determination: 0.6354 - val_loss: 1.1568 - val_coeff_determination: 0.3088\n",
      "Epoch 19/200\n",
      "31/31 - 1s - loss: 0.5933 - coeff_determination: 0.6197 - val_loss: 1.1345 - val_coeff_determination: 0.3236\n",
      "Epoch 20/200\n",
      "31/31 - 1s - loss: 0.5911 - coeff_determination: 0.6898 - val_loss: 1.1211 - val_coeff_determination: 0.3332\n",
      "Epoch 21/200\n",
      "31/31 - 1s - loss: 0.6461 - coeff_determination: 0.6610 - val_loss: 1.1072 - val_coeff_determination: 0.3431\n",
      "Epoch 22/200\n",
      "31/31 - 1s - loss: 0.6369 - coeff_determination: 0.5812 - val_loss: 1.1002 - val_coeff_determination: 0.3477\n",
      "Epoch 23/200\n",
      "31/31 - 1s - loss: 0.5621 - coeff_determination: 0.6657 - val_loss: 1.0961 - val_coeff_determination: 0.3504\n",
      "Epoch 24/200\n",
      "31/31 - 1s - loss: 0.5703 - coeff_determination: 0.6951 - val_loss: 1.0881 - val_coeff_determination: 0.3568\n",
      "Epoch 25/200\n",
      "31/31 - 1s - loss: 0.6270 - coeff_determination: 0.6406 - val_loss: 1.0817 - val_coeff_determination: 0.3620\n",
      "Epoch 26/200\n",
      "31/31 - 1s - loss: 0.6584 - coeff_determination: 0.6284 - val_loss: 1.0798 - val_coeff_determination: 0.3634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/200\n",
      "31/31 - 1s - loss: 0.6824 - coeff_determination: 0.6112 - val_loss: 1.0778 - val_coeff_determination: 0.3651\n",
      "Epoch 28/200\n",
      "31/31 - 1s - loss: 0.6054 - coeff_determination: 0.6627 - val_loss: 1.0776 - val_coeff_determination: 0.3647\n",
      "Epoch 29/200\n",
      "31/31 - 1s - loss: 0.6598 - coeff_determination: 0.6238 - val_loss: 1.0783 - val_coeff_determination: 0.3643\n",
      "Epoch 30/200\n",
      "31/31 - 1s - loss: 0.5894 - coeff_determination: 0.6729 - val_loss: 1.0792 - val_coeff_determination: 0.3635\n",
      "Epoch 31/200\n",
      "31/31 - 1s - loss: 0.5312 - coeff_determination: 0.7016 - val_loss: 1.0783 - val_coeff_determination: 0.3639\n",
      "Epoch 32/200\n",
      "31/31 - 1s - loss: 0.6260 - coeff_determination: 0.6411 - val_loss: 1.0778 - val_coeff_determination: 0.3643\n",
      "Epoch 33/200\n",
      "31/31 - 1s - loss: 0.6023 - coeff_determination: 0.6733 - val_loss: 1.0772 - val_coeff_determination: 0.3651\n",
      "Epoch 34/200\n",
      "31/31 - 1s - loss: 0.5582 - coeff_determination: 0.6888 - val_loss: 1.0774 - val_coeff_determination: 0.3649\n",
      "Epoch 35/200\n",
      "31/31 - 1s - loss: 0.6826 - coeff_determination: 0.6224 - val_loss: 1.0769 - val_coeff_determination: 0.3655\n",
      "Epoch 36/200\n",
      "31/31 - 1s - loss: 0.6088 - coeff_determination: 0.6655 - val_loss: 1.0766 - val_coeff_determination: 0.3657\n",
      "Epoch 37/200\n",
      "31/31 - 1s - loss: 0.6245 - coeff_determination: 0.6610 - val_loss: 1.0769 - val_coeff_determination: 0.3653\n",
      "Epoch 38/200\n",
      "31/31 - 1s - loss: 0.6114 - coeff_determination: 0.6469 - val_loss: 1.0770 - val_coeff_determination: 0.3651\n",
      "Epoch 39/200\n",
      "31/31 - 1s - loss: 0.5631 - coeff_determination: 0.7085 - val_loss: 1.0772 - val_coeff_determination: 0.3651\n",
      "Epoch 40/200\n",
      "31/31 - 1s - loss: 0.6039 - coeff_determination: 0.6264 - val_loss: 1.0777 - val_coeff_determination: 0.3649\n",
      "Epoch 41/200\n",
      "31/31 - 1s - loss: 0.6034 - coeff_determination: 0.6872 - val_loss: 1.0772 - val_coeff_determination: 0.3654\n",
      "Epoch 42/200\n",
      "31/31 - 1s - loss: 0.6776 - coeff_determination: 0.6231 - val_loss: 1.0760 - val_coeff_determination: 0.3662\n",
      "Epoch 43/200\n",
      "31/31 - 1s - loss: 0.6512 - coeff_determination: 0.6311 - val_loss: 1.0754 - val_coeff_determination: 0.3667\n",
      "Epoch 44/200\n",
      "31/31 - 1s - loss: 0.6336 - coeff_determination: 0.6282 - val_loss: 1.0748 - val_coeff_determination: 0.3668\n",
      "Epoch 45/200\n",
      "31/31 - 1s - loss: 0.6169 - coeff_determination: 0.6773 - val_loss: 1.0754 - val_coeff_determination: 0.3663\n",
      "Epoch 46/200\n",
      "31/31 - 1s - loss: 0.6471 - coeff_determination: 0.6295 - val_loss: 1.0757 - val_coeff_determination: 0.3659\n",
      "Epoch 47/200\n",
      "31/31 - 1s - loss: 0.5954 - coeff_determination: 0.6651 - val_loss: 1.0753 - val_coeff_determination: 0.3663\n",
      "Epoch 48/200\n",
      "31/31 - 1s - loss: 0.6229 - coeff_determination: 0.6288 - val_loss: 1.0754 - val_coeff_determination: 0.3662\n",
      "Epoch 49/200\n",
      "31/31 - 1s - loss: 0.6455 - coeff_determination: 0.6594 - val_loss: 1.0760 - val_coeff_determination: 0.3658\n",
      "Epoch 50/200\n",
      "31/31 - 1s - loss: 0.6611 - coeff_determination: 0.6080 - val_loss: 1.0752 - val_coeff_determination: 0.3664\n",
      "Epoch 51/200\n",
      "31/31 - 1s - loss: 0.6090 - coeff_determination: 0.6480 - val_loss: 1.0758 - val_coeff_determination: 0.3659\n",
      "Epoch 52/200\n",
      "31/31 - 1s - loss: 0.5627 - coeff_determination: 0.6937 - val_loss: 1.0764 - val_coeff_determination: 0.3654\n",
      "Epoch 53/200\n",
      "31/31 - 1s - loss: 0.6524 - coeff_determination: 0.6289 - val_loss: 1.0762 - val_coeff_determination: 0.3656\n",
      "Epoch 54/200\n",
      "31/31 - 1s - loss: 0.5709 - coeff_determination: 0.6732 - val_loss: 1.0763 - val_coeff_determination: 0.3659\n",
      "Epoch 55/200\n",
      "31/31 - 1s - loss: 0.5954 - coeff_determination: 0.6617 - val_loss: 1.0769 - val_coeff_determination: 0.3652\n",
      "Epoch 56/200\n",
      "31/31 - 1s - loss: 0.6620 - coeff_determination: 0.6421 - val_loss: 1.0774 - val_coeff_determination: 0.3649\n",
      "Epoch 57/200\n",
      "31/31 - 1s - loss: 0.5610 - coeff_determination: 0.6864 - val_loss: 1.0766 - val_coeff_determination: 0.3655\n",
      "Epoch 58/200\n",
      "31/31 - 1s - loss: 0.6320 - coeff_determination: 0.6693 - val_loss: 1.0755 - val_coeff_determination: 0.3664\n",
      "Epoch 59/200\n",
      "31/31 - 1s - loss: 0.6616 - coeff_determination: 0.6325 - val_loss: 1.0758 - val_coeff_determination: 0.3665\n",
      "Epoch 60/200\n",
      "31/31 - 1s - loss: 0.5559 - coeff_determination: 0.6823 - val_loss: 1.0762 - val_coeff_determination: 0.3660\n",
      "Epoch 61/200\n",
      "31/31 - 1s - loss: 0.5966 - coeff_determination: 0.6695 - val_loss: 1.0764 - val_coeff_determination: 0.3661\n",
      "Epoch 62/200\n",
      "31/31 - 1s - loss: 0.6121 - coeff_determination: 0.6354 - val_loss: 1.0776 - val_coeff_determination: 0.3649\n",
      "Epoch 63/200\n",
      "31/31 - 1s - loss: 0.6125 - coeff_determination: 0.6794 - val_loss: 1.0790 - val_coeff_determination: 0.3636\n",
      "Epoch 64/200\n",
      "31/31 - 1s - loss: 0.5636 - coeff_determination: 0.6224 - val_loss: 1.0792 - val_coeff_determination: 0.3633\n",
      "Epoch 1/200\n",
      "31/31 - 1s - loss: 3.0367 - coeff_determination: -4.5121e-01 - val_loss: 1.3648 - val_coeff_determination: -1.3804e-01\n",
      "Epoch 2/200\n",
      "31/31 - 1s - loss: 2.2067 - coeff_determination: -1.0505e-01 - val_loss: 1.5156 - val_coeff_determination: -2.6729e-01\n",
      "Epoch 3/200\n",
      "31/31 - 1s - loss: 1.5462 - coeff_determination: 0.2102 - val_loss: 1.5329 - val_coeff_determination: -2.4598e-01\n",
      "Epoch 4/200\n",
      "31/31 - 1s - loss: 1.2188 - coeff_determination: 0.4548 - val_loss: 1.1434 - val_coeff_determination: 0.1003\n",
      "Epoch 5/200\n",
      "31/31 - 1s - loss: 1.3677 - coeff_determination: 0.3139 - val_loss: 1.0798 - val_coeff_determination: 0.1303\n",
      "Epoch 6/200\n",
      "31/31 - 1s - loss: 1.1399 - coeff_determination: 0.4751 - val_loss: 1.0741 - val_coeff_determination: 0.1079\n",
      "Epoch 7/200\n",
      "31/31 - 1s - loss: 1.0946 - coeff_determination: 0.4177 - val_loss: 0.9833 - val_coeff_determination: 0.1484\n",
      "Epoch 8/200\n",
      "31/31 - 1s - loss: 0.9808 - coeff_determination: 0.5183 - val_loss: 0.9915 - val_coeff_determination: 0.1405\n",
      "Epoch 9/200\n",
      "31/31 - 1s - loss: 0.9192 - coeff_determination: 0.5046 - val_loss: 1.0048 - val_coeff_determination: 0.0543\n",
      "Epoch 10/200\n",
      "31/31 - 1s - loss: 1.1143 - coeff_determination: 0.4240 - val_loss: 1.0032 - val_coeff_determination: 0.0861\n",
      "Epoch 11/200\n",
      "31/31 - 1s - loss: 0.9068 - coeff_determination: 0.5620 - val_loss: 1.0003 - val_coeff_determination: 0.0639\n",
      "Epoch 12/200\n",
      "31/31 - 1s - loss: 0.8645 - coeff_determination: 0.5772 - val_loss: 1.0286 - val_coeff_determination: 9.8244e-04\n",
      "Epoch 13/200\n",
      "31/31 - 1s - loss: 0.9659 - coeff_determination: 0.4875 - val_loss: 1.0137 - val_coeff_determination: 0.0222\n",
      "Epoch 14/200\n",
      "31/31 - 1s - loss: 0.8527 - coeff_determination: 0.6169 - val_loss: 1.0266 - val_coeff_determination: -6.8425e-03\n",
      "Epoch 15/200\n",
      "31/31 - 1s - loss: 0.8647 - coeff_determination: 0.5622 - val_loss: 1.0405 - val_coeff_determination: -3.3939e-02\n",
      "Epoch 16/200\n",
      "31/31 - 1s - loss: 0.9195 - coeff_determination: 0.5458 - val_loss: 1.0520 - val_coeff_determination: -5.7365e-02\n",
      "Epoch 17/200\n",
      "31/31 - 1s - loss: 0.8811 - coeff_determination: 0.5346 - val_loss: 1.0608 - val_coeff_determination: -7.3879e-02\n",
      "Epoch 18/200\n",
      "31/31 - 1s - loss: 0.7976 - coeff_determination: 0.5781 - val_loss: 1.0694 - val_coeff_determination: -8.9412e-02\n",
      "Epoch 19/200\n",
      "31/31 - 1s - loss: 0.8820 - coeff_determination: 0.5370 - val_loss: 1.0756 - val_coeff_determination: -9.9555e-02\n",
      "Epoch 20/200\n",
      "31/31 - 1s - loss: 0.8244 - coeff_determination: 0.5925 - val_loss: 1.0800 - val_coeff_determination: -1.0667e-01\n",
      "Epoch 21/200\n",
      "31/31 - 1s - loss: 0.8624 - coeff_determination: 0.5817 - val_loss: 1.0851 - val_coeff_determination: -1.1602e-01\n",
      "Epoch 22/200\n",
      "31/31 - 1s - loss: 0.9000 - coeff_determination: 0.5407 - val_loss: 1.0891 - val_coeff_determination: -1.2342e-01\n",
      "Epoch 23/200\n",
      "31/31 - 1s - loss: 0.8866 - coeff_determination: 0.5338 - val_loss: 1.0911 - val_coeff_determination: -1.2765e-01\n",
      "Epoch 24/200\n",
      "31/31 - 1s - loss: 0.9078 - coeff_determination: 0.5811 - val_loss: 1.0922 - val_coeff_determination: -1.2975e-01\n",
      "Epoch 25/200\n",
      "31/31 - 1s - loss: 0.7418 - coeff_determination: 0.6554 - val_loss: 1.0934 - val_coeff_determination: -1.3137e-01\n",
      "Epoch 26/200\n",
      "31/31 - 1s - loss: 0.9234 - coeff_determination: 0.5405 - val_loss: 1.0948 - val_coeff_determination: -1.3460e-01\n",
      "Epoch 27/200\n",
      "31/31 - 1s - loss: 0.8028 - coeff_determination: 0.6121 - val_loss: 1.0956 - val_coeff_determination: -1.3650e-01\n",
      "Epoch 1/200\n",
      "31/31 - 1s - loss: 2.7355 - coeff_determination: -6.2571e-01 - val_loss: 6.0883 - val_coeff_determination: -2.9477e+00\n",
      "Epoch 2/200\n",
      "31/31 - 1s - loss: 1.9019 - coeff_determination: -1.1322e-01 - val_loss: 3.6448 - val_coeff_determination: -1.1713e+00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/200\n",
      "31/31 - 1s - loss: 1.5027 - coeff_determination: 0.0896 - val_loss: 3.3912 - val_coeff_determination: -8.9188e-01\n",
      "Epoch 4/200\n",
      "31/31 - 1s - loss: 1.2081 - coeff_determination: 0.2823 - val_loss: 3.5849 - val_coeff_determination: -1.1641e+00\n",
      "Epoch 5/200\n",
      "31/31 - 1s - loss: 1.0622 - coeff_determination: 0.3598 - val_loss: 3.0045 - val_coeff_determination: -7.2897e-01\n",
      "Epoch 6/200\n",
      "31/31 - 1s - loss: 1.0609 - coeff_determination: 0.3546 - val_loss: 2.4628 - val_coeff_determination: -3.7668e-01\n",
      "Epoch 7/200\n",
      "31/31 - 1s - loss: 0.9981 - coeff_determination: 0.4409 - val_loss: 2.6589 - val_coeff_determination: -5.1069e-01\n",
      "Epoch 8/200\n",
      "31/31 - 1s - loss: 0.8923 - coeff_determination: 0.4721 - val_loss: 1.9321 - val_coeff_determination: -1.3531e-01\n",
      "Epoch 9/200\n",
      "31/31 - 1s - loss: 0.8429 - coeff_determination: 0.4994 - val_loss: 2.4450 - val_coeff_determination: -4.4722e-01\n",
      "Epoch 10/200\n",
      "31/31 - 1s - loss: 0.8654 - coeff_determination: 0.4680 - val_loss: 2.3795 - val_coeff_determination: -3.5723e-01\n",
      "Epoch 11/200\n",
      "31/31 - 1s - loss: 0.8060 - coeff_determination: 0.5200 - val_loss: 1.7891 - val_coeff_determination: -1.3919e-01\n",
      "Epoch 12/200\n",
      "31/31 - 1s - loss: 0.7700 - coeff_determination: 0.5533 - val_loss: 2.8401 - val_coeff_determination: -6.8069e-01\n",
      "Epoch 13/200\n",
      "31/31 - 1s - loss: 0.7281 - coeff_determination: 0.5787 - val_loss: 1.6878 - val_coeff_determination: 0.0212\n",
      "Epoch 14/200\n",
      "31/31 - 1s - loss: 0.7355 - coeff_determination: 0.5181 - val_loss: 1.7313 - val_coeff_determination: 0.0784\n",
      "Epoch 15/200\n",
      "31/31 - 1s - loss: 0.6145 - coeff_determination: 0.6567 - val_loss: 1.8639 - val_coeff_determination: -3.3029e-02\n",
      "Epoch 16/200\n",
      "31/31 - 1s - loss: 0.6505 - coeff_determination: 0.6232 - val_loss: 1.7653 - val_coeff_determination: -2.8255e-02\n",
      "Epoch 17/200\n",
      "31/31 - 1s - loss: 0.6963 - coeff_determination: 0.5711 - val_loss: 1.7105 - val_coeff_determination: -8.5449e-03\n",
      "Epoch 18/200\n",
      "31/31 - 1s - loss: 0.5871 - coeff_determination: 0.6672 - val_loss: 1.5506 - val_coeff_determination: 0.0673\n",
      "Epoch 19/200\n",
      "31/31 - 1s - loss: 0.6220 - coeff_determination: 0.6430 - val_loss: 1.5049 - val_coeff_determination: 0.0739\n",
      "Epoch 20/200\n",
      "31/31 - 1s - loss: 0.5346 - coeff_determination: 0.6813 - val_loss: 1.5078 - val_coeff_determination: 0.0697\n",
      "Epoch 21/200\n",
      "31/31 - 1s - loss: 0.5459 - coeff_determination: 0.6933 - val_loss: 1.5337 - val_coeff_determination: 0.0585\n",
      "Epoch 22/200\n",
      "31/31 - 1s - loss: 0.5997 - coeff_determination: 0.6204 - val_loss: 1.5180 - val_coeff_determination: 0.0656\n",
      "Epoch 23/200\n",
      "31/31 - 1s - loss: 0.6079 - coeff_determination: 0.6486 - val_loss: 1.5047 - val_coeff_determination: 0.0677\n",
      "Epoch 24/200\n",
      "31/31 - 1s - loss: 0.6412 - coeff_determination: 0.6359 - val_loss: 1.4985 - val_coeff_determination: 0.0680\n",
      "Epoch 25/200\n",
      "31/31 - 1s - loss: 0.6142 - coeff_determination: 0.6204 - val_loss: 1.4959 - val_coeff_determination: 0.0683\n",
      "Epoch 26/200\n",
      "31/31 - 1s - loss: 0.6605 - coeff_determination: 0.6017 - val_loss: 1.4922 - val_coeff_determination: 0.0683\n",
      "Epoch 27/200\n",
      "31/31 - 1s - loss: 0.6373 - coeff_determination: 0.6433 - val_loss: 1.4897 - val_coeff_determination: 0.0693\n",
      "Epoch 28/200\n",
      "31/31 - 1s - loss: 0.5343 - coeff_determination: 0.6877 - val_loss: 1.4870 - val_coeff_determination: 0.0689\n",
      "Epoch 29/200\n",
      "31/31 - 1s - loss: 0.5714 - coeff_determination: 0.6536 - val_loss: 1.4862 - val_coeff_determination: 0.0686\n",
      "Epoch 30/200\n",
      "31/31 - 1s - loss: 0.6136 - coeff_determination: 0.6470 - val_loss: 1.4809 - val_coeff_determination: 0.0702\n",
      "Epoch 31/200\n",
      "31/31 - 1s - loss: 0.6575 - coeff_determination: 0.6077 - val_loss: 1.4818 - val_coeff_determination: 0.0697\n",
      "Epoch 32/200\n",
      "31/31 - 1s - loss: 0.5694 - coeff_determination: 0.6438 - val_loss: 1.4792 - val_coeff_determination: 0.0724\n",
      "Epoch 33/200\n",
      "31/31 - 1s - loss: 0.6071 - coeff_determination: 0.6600 - val_loss: 1.4829 - val_coeff_determination: 0.0706\n",
      "Epoch 34/200\n",
      "31/31 - 1s - loss: 0.5535 - coeff_determination: 0.6468 - val_loss: 1.4825 - val_coeff_determination: 0.0715\n",
      "Epoch 35/200\n",
      "31/31 - 1s - loss: 0.6475 - coeff_determination: 0.6234 - val_loss: 1.4805 - val_coeff_determination: 0.0724\n",
      "Epoch 36/200\n",
      "31/31 - 1s - loss: 0.6173 - coeff_determination: 0.5945 - val_loss: 1.4799 - val_coeff_determination: 0.0722\n",
      "Epoch 37/200\n",
      "31/31 - 1s - loss: 0.5993 - coeff_determination: 0.6564 - val_loss: 1.4812 - val_coeff_determination: 0.0712\n",
      "Epoch 38/200\n",
      "31/31 - 1s - loss: 0.5691 - coeff_determination: 0.6773 - val_loss: 1.4817 - val_coeff_determination: 0.0713\n",
      "Epoch 39/200\n",
      "31/31 - 1s - loss: 0.6376 - coeff_determination: 0.5912 - val_loss: 1.4835 - val_coeff_determination: 0.0709\n",
      "Epoch 40/200\n",
      "31/31 - 1s - loss: 0.5852 - coeff_determination: 0.6664 - val_loss: 1.4825 - val_coeff_determination: 0.0704\n",
      "Epoch 41/200\n",
      "31/31 - 1s - loss: 0.5889 - coeff_determination: 0.6547 - val_loss: 1.4811 - val_coeff_determination: 0.0709\n",
      "Epoch 42/200\n",
      "31/31 - 1s - loss: 0.6095 - coeff_determination: 0.6089 - val_loss: 1.4827 - val_coeff_determination: 0.0700\n",
      "Epoch 43/200\n",
      "31/31 - 1s - loss: 0.5827 - coeff_determination: 0.6497 - val_loss: 1.4823 - val_coeff_determination: 0.0696\n",
      "Epoch 44/200\n",
      "31/31 - 1s - loss: 0.5759 - coeff_determination: 0.6761 - val_loss: 1.4830 - val_coeff_determination: 0.0699\n",
      "Epoch 45/200\n",
      "31/31 - 1s - loss: 0.5415 - coeff_determination: 0.6811 - val_loss: 1.4821 - val_coeff_determination: 0.0695\n",
      "Epoch 46/200\n",
      "31/31 - 1s - loss: 0.5565 - coeff_determination: 0.6922 - val_loss: 1.4819 - val_coeff_determination: 0.0702\n",
      "Epoch 47/200\n",
      "31/31 - 1s - loss: 0.6429 - coeff_determination: 0.6363 - val_loss: 1.4828 - val_coeff_determination: 0.0709\n",
      "Epoch 48/200\n",
      "31/31 - 1s - loss: 0.6054 - coeff_determination: 0.6680 - val_loss: 1.4833 - val_coeff_determination: 0.0715\n",
      "Epoch 49/200\n",
      "31/31 - 1s - loss: 0.6160 - coeff_determination: 0.6157 - val_loss: 1.4843 - val_coeff_determination: 0.0708\n",
      "Epoch 50/200\n",
      "31/31 - 1s - loss: 0.5404 - coeff_determination: 0.6908 - val_loss: 1.4831 - val_coeff_determination: 0.0710\n",
      "Epoch 51/200\n",
      "31/31 - 1s - loss: 0.6146 - coeff_determination: 0.6321 - val_loss: 1.4838 - val_coeff_determination: 0.0711\n",
      "Epoch 52/200\n",
      "31/31 - 1s - loss: 0.5424 - coeff_determination: 0.6768 - val_loss: 1.4843 - val_coeff_determination: 0.0704\n"
     ]
    }
   ],
   "source": [
    "best = {'num_layers':1,\n",
    "        'learning_rate':0.001,\n",
    "        'units_0':348,\n",
    "        'dropout_0':0.4,\n",
    "        'embedding_dim':14}\n",
    "\n",
    "set_all_seeds(42)\n",
    "\n",
    "oof = np.zeros(y_train.shape)\n",
    "models = []\n",
    "for fold, (train_idx,test_idx) in enumerate(KFold(FOLDS).split(y_train)):\n",
    "    regr = create_model(best)\n",
    "    hist = regr.fit(X_train[train_idx],y_train[train_idx],\n",
    "                validation_data = (X_train[test_idx],y_train[test_idx]),\n",
    "                epochs=200,\n",
    "                batch_size=16,\n",
    "                verbose=2,\n",
    "                callbacks=[tf.keras.callbacks.EarlyStopping('val_loss',\n",
    "                                                            patience=20,restore_best_weights=True),\n",
    "                          tf.keras.callbacks.ReduceLROnPlateau('val_loss',patience=3)])\n",
    "    \n",
    "    oof[test_idx] += regr.predict(X_train[test_idx]).ravel()\n",
    "    models.append(regr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OOF R2: 0.487\n"
     ]
    }
   ],
   "source": [
    "print('OOF R2:', np.around(r2_score(y_train,oof),3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = [model.predict(X_test) for model in models]\n",
    "std = np.std(y_pred,axis=0).ravel()\n",
    "preds = np.mean(y_pred,axis=0).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAArY0lEQVR4nO3deZgU1b3/8fdhYIQBNwaiMcqgiRsa0TjuV8kVY9zl+tNogjdoVBQ1blkVY1xCEhONuGIIbgkYF9Brohh3o0aCDJsgGFdAUMIoBkVRlvn+/jgz0DPTXV3dXdXVy+f1PPXMdE9V1+memU+fPnXqW87MEBGRytMl6QaIiEg8FPAiIhVKAS8iUqEU8CIiFUoBLyJSobom3YBUffr0sf79+yfdDBGRsjF9+vT3zaxvup+VVMD379+fpqampJshIlI2nHMLM/1MQzQiIhVKAS8iUqEU8CIiFUoBLyJSoRTwIiIVSgEvIlKhFPAiIhVKAS8ikqQ77oAnn4zloUvqRCcRkarx7rvwpS9tuB3DtTnUgxcRKbYLL2wf7kuXxrIbBbyISLG8/jo4B6NH+9vXXut77ltsEcvuNEQjIhI3MzjxRLj//g33rVgBm2wS627VgxcRidOMGdCly4Zw/+MffeDHHO4Qcw/eOXchcDpgwBzgVDP7LM59ioiUhJYWOOgg+Mc//O2+fWHRIujevWhNiK0H75z7EnAe0GhmuwI1wElx7U9EpGQ88wzU1GwI94cfhmXLihruEP8YfFegh3NuDVAHvBvz/kREkrNmDey4I7z9tr89cCBMn+7DPgGx9eDNbAlwDbAIeA9YYWaPd1zPOTfcOdfknGtqbm6OqzkiIvGaNAlqazeE+4svwqxZiYU7xDtEszlwLLAtsBXQ0zl3csf1zGysmTWaWWPfvmmvOiUiUro++QQ22giOP97fPuIIP/6+337Jtot4Z9EcArxtZs1mtgZ4ANg/xv2JiBTX738PvXrB6tX+9ty58Mgjfq57CYhzDH4RsK9zrg5YBQwGdMFVESl/y5dDff2G26edBuPGJdeeDOIcg58KTARm4KdIdgHGxrU/EZGiuOqq9uG+YEFJhjvEPIvGzH4O/DzOfYiIFMWSJbD11htuX3qpD/sSplIFIiLZnHMO3HLLhtvNzdCnT3LtCUmlCkREMnn1VX/AtC3cb7jBlxkog3AH9eBFRDozg+OOg//7vw33ffyxnzFTRtSDFxFJNW2aLw7WFu533+0Dv8zCHdSDFxHx2k5Oeuklf3urrfxZqbW1ybarAOrBi4g88YQvKdAW7n/7m581U8bhDurBi0g1W70attvOhznAXnvBP//ph2gqQGU8CxGRXN1zj68h0xru+zCV/steYsKfKycW1YMXkeqycqW/mpIZAH+tOZZj1j0IOFgIw4f71YYOTa6JUamctyoRkWxuugk23nh9uA/eaj7HrPs/YENxsE8/hZEjk2le1NSDF5HK9/77/pJ5bc46C8aM4ZkMXdxFi4rTrLipBy8ile2yy9qH+6JFMGYMAP36pd8k0/3lRgEvIpVp0SJfZqCtINgVV/ihmW22Wb/KqFFQV9d+s7o6f38lUMCLSOUZPhwaGjbcfv9935PvYOhQGDvWr+qc/zp2bGUcYAWNwYtIJZk3D3bZZcPtMWP8eHuAoUMrJ9A7Ug9eRMqfGRx11IZw79bNT4fMEu5JmzAB+vf351X17+9vRynOi27v6JyblbJ85Jy7IK79iUiVmjLFJ+Qjj/jb993nz1Dt2TPZdmUxYYIfSVq40L8/LWydgx9lyDtrnQ8aJ+dcDbAE2MfMFmZar7Gx0ZqadNlWEQlh3TpobIRZs/zt/v3htdd8770M9O/vQ72jhgZ/FcCwnHPTzawx3c+KNUQzGHgzKNxFRDrKOITx6KPQteuGcH/ySV/5McZwj3o4JdNc+yjn4Bcr4E8C/pzuB8654c65JudcU3Nzc5GaIyKlLt0QxrlnfM5nm34BjjjCr3TAAb4nP3hw0dtS6HBKMebgxx7wzrla4Bjg/nQ/N7OxZtZoZo19U09GEJGqNnKkLxvQ5jtM4MNV3en+UWtHcNo0eOGFolR+7NgWKLykQTHm4BejB384MMPM/l2EfYmUpbhnU5SjtqGKjfkIwzGBkwG4jxP8xTka0w47x9qWsPeHUYw5+MUI+G+TYXhGRIozm6Ic9esH5zOaj9h0/X078C9+3HCfT8QityWX+8MaOtQfUG1p8V+jno8fa8A75+qAbwAPxLkfkXIWx8f/srdsGQsWOkZzIQA38H0cxpK6HRIpI1CuJQ1iDXgz+9TM6s1sRZz7ESlnxZhNUVYuvhi22GL9zb2/tIQL3A2JlhEo15IGKlUgkrB+/dLPh66UioahLVgA22674faoUXDJJbyUWIPaK8eSBipVIJKwcv34H6lTTmkf7suXwyWXJNacSqGAF0lYuX78j8ScOf5J33WXvz12rD/SvPnmybarQmiIRqQElOPH/4KYwTe/CU884W/37AnLlnX+KCMFUQ9eRIqr7eSktnB/4AFf+VHhHjn14EWkONauhYEDfc12gB12gFde8TVlJBbqwYtI/F591RcCawv3Z5+Ff/1L4R4zBbyIxGftWvj1r2H33f3tQYP8aZuDBiXarGqhgBeReMyaBfvs409cOvJIeO8933MvcpmBaqaAF5FoffaZr7PQ2AhLlsDEiTBpEmy5ZdItqzoaABOR6PzjH3DaaX58/ZRT4NproXfvpFtVtdSDF5HCrVwJ550HBx7oe/CPPQZ33KFwT5gCXkQK8/jjsOuucNNNcO65MHcuHHpo0q0SFPAiZS+xi4UsXw6nnurPSO3eHZ5/Hm64AXr10gVMSoQCXqSMRX2xkNDBPGkSDBgAf/qTLwo2a5a/PmoMbZL8OTNLug3rNTY2WlNTU9LNECkb/funLzXc0OCr7+aiLZhTLz5SV9eh8NnSpX4YZtIk2GMPuP32DXPcY2iTZOecm25maa9fGPcVnTZzzk10zr3qnJvvnNsvzv2JVJsoLxYSeGUpM7jzTth5Z3j4YX/y0tSpncI96jZJYeIeorke+JuZ7QQMBObHvD+RqhLltUIzBbBbuMCPs596qj+YOns2/OQnvvRAzG2SwsQW8M65TYCDgNsAzGy1mf0nrv2JVKMoLxbSMYAdLZzLjcx1u8KUKXDzzfD3v8OOOxatTVKYOHvw2wHNwB3OuZnOuXHOuZ4dV3LODXfONTnnmpqbm2NsjkjlifJiIanBvBPzeZ4DuZHzWPHVA/3Ux7PP9kdfi9gmKUxsB1mdc43AP4EDzGyqc+564CMz+1mmbXSQVSQeEyb4sfRFi3xPfdSo9IF7911reGvEb/nRqitYSS8u7TWa/xpzMkNPVv2YUhV0kDXOUgWLgcVmNrX19kTgpzHuT0TS6Dg7pm3aInQI+RkzOOyy79F71Wzu5Vucxw0sW7kFfzwTcOqBl6PYhmjMbCnwjnOubcBuMDAvrv2JlKpMc8uLdTJQptkxw4a17nPVKvjpT2HvvVm75N8M4UFO4l6WscX6dUeOjKdtEq9Y58E753YHxgG1wFvAqWb2Yab1NUQjlSbT3PJhw/x1pgPnnEekSxc/yzGdQzZ6nkmbn84mS1+D006j922/5UM6X/DaOV/GXUpPYvPgzWyWmTWa2W5mNiQo3EVKUaG97Ey957FjA+acRyzd9MRefMxNnMMTnx/EiubV/vqo48axSUPncM/0GFL6VKpAJIMoTrnPNLd83brc1i9Ex2mLh/Eor7ALIxjDdVzAzuvmwiGHpF0XNMWxnCngpSzFMX7d8THPP7/wXnamnm9NTW7rF6Jt2mLfLh9wF9/lUY5gJb04gH9wEdfRp6Fnu3WHDdvQvpoaf1sHWMuUmZXMsueee5pINuPHm9XVmfl+tV/q6vz9UT5mpsW5wts6YkT0zyGjlhaz++6zTzf5gq2mq13Bz6yWz9LuM47XVuIFNFmGTE081FMXBbyE0dCQPngbGqJ/zCj2M36838Y5/7UtLDPdH6klS8yGDPEN33NPe/iXswP3GcdrK/EKCnhVk5Syk2lWSCEzPYJmmqSKa6ZL5Mx8pccf/AA+/xyuvBIuvBC6Bp/6EsdrK/FKbBaNVK4kL+gQRzGrTNvW15fhKfdvvQXf+AacfjoMHAgvvww/+lHWcAcVCqs0CnjJWdIXdIhjpkemx7z+el/DvKXFf20L93zf4IK2K/hNc906GD0avvpVeOklGDMGnnkGtt8+9ENoFk2FyTR2k8SiMfjyUArjtHGMX4d9zHwPRAZtl+0xs7btlVfM9t3Xb3jEEWaLFsX+OkhpQGPwEqVqH6fN94pFQdtB5p+NGhVwpaUTVsPVV8NVV8Emm/hron772/6XIVUhaAxeAS85q/ZLsuX7Bhe0HWT+Wb9+6V/vo7ecxl/6ngZz5sBJJ/lw79s33JOQiqGDrBKpah+nzfdAZNB2QT/reHZrDz7lan7Mg0v3hQ8+gIcegj//WeEunSjgJWfVdEGHdAc+832DC9ou6Gep4X8Qf2c2A/kxv+XeXqfBvHlwzDH5Pj2pdJkG55NYdJBVSkm2g6L5HIgM2i7ohKgteqywWzjLDOwNtrPDN3rKRozQwVDRQVaRvJTMsYZHHuGT755F9+Xvcj0XMHabqzj4qLqilRuW0pb3GLxzrnfQEk9zRYoj27zzTJUd46j4mFZzs0/ro46i5xc3peafL3KRXcuri+qYPLl45YalfGUbg58ONLV+bQZeA15v/X56vE0TiU7HMD/77OwnayV2VqcZ3HMPDBgA998Pl18OM2bAPvusXyXxNx8pC4EBb2bbmtl2wGPA0WbWx8zqgaOAB7I9uHNugXNujnNulnNOYy+SiHRn3t56a/YecCKzhZYsgWOP9XPZt9vOB/vPfw61te1W653h83Om+6U6hZ1Fs5eZTW67YWaPAoNCbvvfZrZ7pjEikbilu6pSpkNPqT3gos4WMoM//MH32p98Eq69Fl58EXbdNYadSbUIG/DvO+cudc71d841OOdGAh/E2TCRqOQybNFx+GXo0PS1aDoqqI7Mm2/C4MH+Y8aee/oTly66KPNVQYDly8Pfn2RhuFJqQ1XKNL0mdQF6A9cDM4EZwGigd4jt3m5dfzowPMM6w/Hj/E39+vWLczaRVKlMtXOcy72eTDp5XyRj7Vqza64x69HDbJNNzMaO9RfnKOA5dawHVAoX8CiFNlQyorrgB9Arx/W3av36BWA2cFDQ+poHL3EIuqpSvvPIU+es19RkDtuM897nzDHbay+/4tFHmy1enPHx07Ut3XNyzj+nVKVQGK4U2lDJCg54YH9gHrCo9fZA4JYw26Y8xuXAD4PWUcBLXKKskJjL5f06rrdpj89t9v/83KxbN7O+fc3uuadTrz3sCVY9e6bfX+pz6/gpJfXNoFhKoQ2VLIqAnwpsA8xMuW9ulm16AhunfP8icFjQNgr46hJ3Wdq4Sgpn6rF3XDqutxdTbQ67+BtDh5o1N6fdR6Yeb319uDeW1J5xKfSeS6ENlSySgG/9OjPlvtlZttmudVhmNvAKMDLbfhTw1SPucdmkL8ydul4PPrFruMjW0sUWsbUdwcOB+8nU4w27pPaMo3odCnmz1Bh8vKII+ImtwzQzgFrgh8A9YbbNZVHAV4+4e3VBF9Gur/dLUFilC7T6+uBgralpv35Njdl/85S9wXZmYLdwlm3MCqupyb/tYZZ0B1pT215fX/yA1kVE4hNFwPcBJgD/BpYB48PMosl1UcBXj6jGZTMFRy694I5hlS7QunXL7THsww9tLKebgb3GV+wgnm23frbnlC5Qs73BZAreQgNaQyylLYqAPyDMfYUuCvjqEUVoBAVXrr3gMOPWQT33dmH50ENmW21la+liV/Mj68EnwetneG4d37gyvfFk+zRS6Gutg6SlLYqAnxHmvkIXBXz1iOJjf1Bw5TJe3jGsch0DXz/k8e9/m514ohnYvG5ftT2ZlvGxCplzn+tQR6EBrR58acs74IH9gB8A7wAXpSyXZzvIms+igK8uhY7LZguuESPCh3UhPXhosVNrx9uqXvW2tmutXd7tKuvG5zntM06FBrQOkpa2oIDPVqqgFugFdAU2Tlk+Ao7Psq1IRhMm+Boxixb58gCjRuVe4yVTVccuXXztmDFjfBxl07GAWLoiY926dar3BcDWvMNfOZrbV5/MvNXb882+M7l8zaWsIc3KHcRR+THKK1C1qaYreFWcTMmfugANYdYrdFEPvjpENSujV69ce9obevj5zKJJHdt3rLMzGWMr2NhWUmfnMdpqWJvT8E7UPfg4rkAlpY8IxuCfADZLub058FiYbXNZFPDVIdOQQaaDjx3DacQIs9ra/MI9dQgn39AbtNVr9iwHmYE9wWDbljfXB3bcdW+CaKy8OkUR8DPD3FfoooCvDkG93DBTFgs9EagtzDu+SdTWZgndNWvMrr7a1nTrbh+yqZ3KbQYtnXrKUda9yeVNSLNdqlMUAT8d6JdyuwHNoqkqUX7Ez3YQs7ADntmX+vr0dVzafpb6fNs+WezGLJtdu6e/Y8gQm3TjkqwXz27bNvVNJRe5DmWpB1+dogj4w4BFwJ9al4XAN8Nsm8uigC9NYYImlzeAbFMYC5myGMWS2r5aPrMrudRW09WW8gUbWnufjf9T5+JgHc8UHTEi3mmg+f6epPIUHPD+MeiDv1Tf0UCfsNvlsijgS1O2oMknWLIV7aqp2TCskW9Qp5brzWf7fXnRXmFnM7A7+a715v1OATt+vFnXrrm1Kax8hlx0MLX65B3wwE6tX7+WbgnaNp9FAV+asgVNvkMDYU5Gqq3tPFbeNqYddOp+IZ8CevKxXcf5tg5nC9nGDmNyxscOUz4gbDh3lOSQi94oykchAf+H1q/PpFmeDto2n0UBX5qyBU0hB/fC9LCDTscPE4K59OAP4XF7i/5mYDdyjvXio8DHzvVTQVSlGOKk6ZblJZIhmmIsCvjSlC1ogqY9pptHni4c8g3GMLNWwvSyN2O5jeN7ZmCvsoP9F8+lXa9jwOYS7mGGrTLNvS9moOZSj15j/MkrpAd/XNAStG0+iwK+dAUFTdihlo4VGVPDIdtFNHIZd053gDPocYfwgL3LlraGGvslP7WNWJVx/Y5hFnaIJt3l9Do+h1IJz1yHtDRLJ1mFBPwdrcsjwIfApNZlOfBA0LYpj1GDv1j3w9nWVcCXr9SQDXvFo9RwGDEieL226YthhD3ZqH+PpXYvJ5iBzWB324PpOQfZ+PHZSwmHCcJSmuKY60FpzbNPVhTTJB8Gvphy+4s5BPxFwN0K+NIR98f+XHqAqeEweHDm9bp1C9/OoP03NJg5WuzC+rvss56b22fU2sWMsq6sDmxnUG86l6JmubY5ifDMtR69evDJiiLg53a43aXjfRm22xp4CjhYAV8aijEUkEsPMGiqZbpefJg3pqAe/IOjF5gddpi/Y//9bUfmh3oTyjS8km26Z8fHKZeTlMLWo9cYfPKiCPibgMeAU4BhwKPAjSG2mwjsCXw9U8ADw4EmoKlfv37FeUWqWDGCJGwt9jAHavPpVY8f37lH7FhnZ3OTfex6+dNYb7zRbN260GPomYZncqk5H/Q6l0t4ahZN6YnqRKf/Aa5rXf4nxPpHAbe0fp8x4FMX9eDjV6yhgI5nd3ZcOhYWy/eM1UyBmbrODrxqz/FfZmB/41Czt98O1cZsr08+b0o6SSlaes2iC/gG4JDW7+uAjbOs/ytgMbAAWAp8CowP2kYBH79iDQUE9W7T9UzzPds0U2A2NJh1ZbX9hF/ZKjayD9jcvsud1tCvJbB9XbqEf32C3pQy/Uzj1dEpl089cYtiiOYMYBrwZuvt7YGnwmzbur568CWiWP8U+ZQEzlQ1sqEh9wN8j/xihs10e5iB3c//sy14r93zzPR4ucz1DnqOUdSikWCldtwiKVEE/Cz81Z1mptw3J8y2poAvOcX4WBvUu820v1zn2qc7u3KHfqtszjGXmNXU2KebbmFn9pnY6fHGj8/ctrYDoWFen0xTO9sOyGZ7PtU+tFCoUpp5lKQoAn5q69eZrV+7Ai+H2TaXpZwDXv+w7QWNbUd5wenU4N+fF2w+O5qBvXHgKWYffJD2cYKGg3Lp/eXSg0xte3198ElfEo568F4UAf8b4BLgVeAbwIPAqDDb5rKUa8BrLLCzbAcvO/4T5vsG2dBg1ouP7AbOtXU4e5sG+waPBf6T5/PpIpfH6diDDDvbptqCqVD6v/OiCHjXOg5/f+vUxzMAF2bbXJZyDXj1JDrLNismNQQzBWB9ffZ/1m/yN1tAP1uHs9GcZz35OOvH9KBaK7kI+3sPewC52oYWoqBPzgUGfNiTmqJYyjXgK2ksMKp/mDChFqZee6Yqhvfd+oHZsGFmYPPYyfbnhdBvrlH1/MI+TtgpoNXcIVBQ5y+KHvwEUi7ZF9dSrgFfKT34KD/y5nKyU9jebdv3xzHRlrKFretSY3OOHWmb91iVc5ujCpQwjxPmza4ahxbaaKilMFEE/NPAx61lB/7StoTZNpelXAM+7j/QYvVuCn2jSlfVMdcLYgQtW/KuTeQ4M7Dp7GGHf3Fm2v2mzpYphV5hur+P2trMNe6rTaV0kJISRcAPSreE2TaXpVwD3iy+MInizSNs23I5aJg6I6QtxDtuH1SgKrelxU7hdlvOZraKjezH/NpqWJP1rNBS6hWWyptNKaqkIc4k5B3wQHfggtZaNGcCXYPWL3Qp54CPS9DJNPle4DrXE3dSe1L51F/JtgQV62rgbXuMb5iB/Z0DbQdeDdXDU6+wfOh3VZiggO9CsLuARmAOcDhwbZb1JWKLFqW/f906/2+wcCEMHw4TJqRfb+RI+PTT9vd9+qm/v6NRo6Curv19dXVwxBHQvz906QLDhnV+vELU1fn2d9xvF9bxfW5gLruyH1M4m5v5Os/yGjuu327UqMyPm+l1y3S/JCfT313Q71dCypT8/o1hw9mq+JObZgStX+iiHnxnYafYZert5PrxN904eiE99l69gksQpBsvP3irefbGlvubgU3mMNuGhRm3y/V1U6+wNGkIK38UMEQzI+h21IsCvrOwQyJBRbcKCbp8i4C1Ld26tb8+auA/7+rVZr/4hT8C2bu3/eOsP1pDv5a8/ulLbQxeJC6FBPw64KPW5WNgbcr3HwVtm8+igE8vtXeTabw6rjrj+ZbxzfnNpKnJbOBAv8G3vmW2dGm4BgZQr1CqQd4BX+xFAZ9dPoFdSNAV2oMP+nRhZmaffmr2k5/4d64ttzR78MHwjYvoOYqUMwV8hSlmmIUZImrr5QfOhknXzr//3Wz77f0Kp51mtnx5Tu2KunhXNb9JVPNzL3cVHfCV8odZys8j3YHXtp59LkM460N3xQqzs8/2d267rdmTT2bcV9ja8YUeUK3mMftqfu6VoGIDvlL+MIvxPHI927OQU/CDevLDvjDZbJtt/ANfcIHZypU5vw5xFO+q5lk31fzcK0HFBnyl/GHm8zxy6fFnCs5MVx0aPDj9Walhi2g51/ln9TTbHznZ3xgwwGzKlLxfh7CfGHL5O6jmsymr+blXgkQCHn8W7EvAbOAV4Ips2+Qa8JXyh5nPXPWgnm7H8M9ULiCopx0mMIMCecPPWuwE7rV/09dW09VGb/ozs88+K+h1CNPuXD8BVUpnIR/V/NwrQVIB74Berd93A6YC+wZtox58uOcRtH4cpQQyBW3QG8348WbbdV9iD3KsGdg09rS9u88ODN0oevD5HsOolOG+fFTzc68EiQ/RAHXADGCfoPU0Bh/ueQT1dKOY1hi2B9/W9k5DRS0tZuPG2ed1m9oq191+yG9tu35rQpXw7Tgbplu38GPwhb6xl/KB7rhV83Mvd4kFPFCDv2D3SuDqDOsMB5qApn79+uX85CrlDzOX5xEUcIWemBT05tHWMw9s55tvmh18sN9o0CCz11/P6TWorW2/39razvuolDd2kSiUQg9+M+AZYNeg9TQPPv9pgm0Blyn86+vD9e432qhzL9o5f0A2MFjXrjW77jqzujpb3X1ju7j3rdaFdTm96ebSM6+UN3aRQiUe8L4N/Bz4YdA61R7wufRMg6Y3Bj1GmJDv2jX9xSgybXvIF+ea7bOPGdji3Y+0r3R/J6/edaUcNBcppqQOsvYFNmv9vgfwPHBU0DbVHvBRjS0H9W4LOUmoYwB343P7GVfY53Tz7wgTJlhDv5a8n0OxDpqr9y+VJKmA3w2YCbwMzAUuy7ZNtQd8sXqwQUM5QftM3aaRl2w2XzUDe6juJLNlywp+DsU64Uvj91JJSmKIJsySVMCXSo+u2NM+g0I+07h3fY9P7Df80NbSxRazlR1f+1C716vQ55DrCVy5/t4qZWqtSBsFfIBS6tEVuy3ppiVC+pkrZmb2zDO2YouvmIH9nuG26zb/SWyGS7770Ti/VBoFfIBS69EV+9PE+PHtz3Str0+zz//8x+zMM/0KX/6y2dNPZ33MuJ9Dvr+3Uvt9ixQqKOCd/3lpaGxstKampqLus0sX/y/ekXPQ0lLUppSmhx+Gs86C996DCy+EK6/sfAHNBOT7e5swwV8DNvW6snV1MHYsDB0afTtF4uacm25mjel+lu2i2xWvX7/c7s9kwoQNF6bu3z/zRbCj2i52zc3wne/A0UfD5pvDlClwzTWRh3u+zz/f39vQoT7MGxr8m0FDg8JdKlimrn0SS7mOwef7GKU0/r9eS4vZ3Xeb9enjB+gvv9zs889j2VUhz78kXzuRBKAx+GCFjhnHNR5c9Nk977xjdtRRvhF77202Z06suyvmjBuRShUU8FU/Bh+FfMeDM20HUF8PH3zQ/r7YxopbWmDcOPjRj2DNGhg1Cs47D2pqIt5Rezr+IVI4jcHHLN/x4Ew/d65zuIM/MDhyZG5ty+qNN2DwYDjzTNhzT5gzxx9MjTncIbrjHyKSngI+AqNGdT72WFfn7w86iJhuO+cy9+oBFi2KqNFr18K118Juu8GMGfCHP8BTT8GXvxzRDrILet1EJAKZxm6SWMq5VEG68eAw5W87bhdUPiDd+HRe49Avv2y2117+AY8+2mzx4mhehDxoHF2kMOggazIyXSqvvj7zNkEh33GWSM4zST77zOyyy3y5yL59ze65x8+aEZGyFRTwGqKJUbpx9KD7If2wBUCvXp0PsI4c2f6EHQgYp5861Y+xX3klnHgizJvnvzqX9XnErWTPBRApcwr4EjN0KAwb1jl3080qyTQe3+7+Tz6Biy6C/faDFSv8manjx0OfPpG1uRBtZ5YuXOg/gyxc6G8r5EUKp4CPUX19bve3mTy584HWdD3zrLNQnn7aH0S97jpfbuCVV+DII7O2u5hy+hQiIjlRwMfo+uuhW7f293Xr5u8PEqpnTuZZKL+55D9wxhl++mOXLvDss3DLLbDJJrk0vyjCPlcRyZ0CPkZDh8Idd7Sve3LHHdlPVAo7PzxdXZXJZz7Ety4fALffDj/+Mbz8MgwaFM0TioHmwovEJ7aAd85t45x7xjk33zn3inPu/Lj2VcqGDoUFC/wY+oIF4c5CzWV++PrHX7qMBfuexKDrhvjx9alT4eqroUePwp9EjDQXXiQ+cfbg1wI/MLOdgX2Bc5xzA2LcX8XIqeKhmT9ouvPO8OCDcNVV0NQEjWnPXC45qu4oEp+i1aJxzj0E3GRmT2Rap1xr0STmnXf8wdPJk2HffeG222CA3kNFqknitWicc/2BPYCpaX423DnX5Jxram5uLkZzyl9LC4wZA7vs4g+gjh4NL7ygcBeRdmIPeOdcL2AScIGZfdTx52Y21swazayxb9++cTen/L32Gnz963D22bDPPjB3Lpx/flGKg4lIeYk14J1z3fDhPsHMHohzXxVv7Vr4zW9g4EA/M+a22+Dxx2HbbZNumYiUqK5xPbBzzgG3AfPN7Hdx7acqzJ4N3/uer/o4ZAjcfDNstVXSrRKREhdnD/4A4H+Bg51zs1qXI2LcX+X5/HP42c/8jJjFi+H+++GBBxTuIhJKbD14M3sBSL6SVbmaMgVOOw3mz4fvfhd+97vsNQ5ERFLoTNZSs3IlXHABHHCALxT26KNw110KdxHJWWw9eMnDE0/4UooLFsA558CvfgUbb5x0q0SkTKkHn0FRa5R/+KE/iHrooVBbC889BzfdpHAXkYKoB59GW43ytjK2bTXKIYZT6B980M9pb26Giy+Gyy6D7t0j3omIVCP14NMoSo3ypUvhhBPguONgyy3hpZfgl79UuItIZBTwacRao9zMHzQdMAD++lcf6i+9BF/7WgQPLiKygQI+jdhqlC9cCIcfDqec4qs/zprlh2U6XhVERCQCCvg0Iq9R3tLiD5rusosvCnbjjfD887DTTgW3VUQkEwV8GpHWKP/Xv+Cgg+D73/dz2+fOhXPP9dNzRERipFk0GQwdWuCMmTVr4Jpr4IorfPf/zjv9GalOJ/eKSHEo4OMwc6YvMzBzJhx/vB+S2XLLpFslIlVG4wRR+uwzuOQS2GsvePddmDTJFwhTuItIAtSDj8oLL/he+2uvwamnwrXXwuabJ90qEali6sEX6uOP/UHTAw/05X0fewxuv13hLiKJU8AX4rHHYNdd4ZZb4Lzz/AyZQw9NulUiIoACPj/Ll8OwYXDYYX6GzAsvwPXXQ69eSbdMRGS92ALeOXe7c26Zc25uXPtIxMSJ/izUu+/2xWlmzoT990+6VSIincTZg78TOCzGxy+u997zhcFOOAG23hqmTYNf/ELFwUSkZMUW8Gb2HLA8rscvGjO44w5fHGzyZPj1r2HqVNh996RbJiISKPFpks654cBwgH4FV/OK2Ntv+0LwTz7pZ8mMGwc77JB0q0REQkn8IKuZjTWzRjNr7Nu3b9LN8datgxtu8DNk/vlPuPlmePZZhbuIlJXEe/AlZ/58f8LSlCm+tO+tt0ZQJ1hEpPgS78GXjDVrfD3g3Xf3FSD/9Cd45BGFu4iUrTinSf4ZmALs6Jxb7Jw7La59FWz6dGhshEsvhSFDfC/+5JNV+VFEylpsQzRm9u24Hjsyq1bB5Zf7sr5bbOEvgD1kSNKtEhGJRPWOwT/3HJx+Orz+uh9zv+Ya2GyzpFslIhKZ6huD/+gjOPtsGDQI1q71UyDHjVO4i0jFqa6AnzzZT3289Va48EKYMwcGD066VSIisaiOIZr33/eBPn68PyP1xRdh332TbpWISKwquwdvBvfe60P9nnvgsstgxgyFu4hUhcrtwb/7LowYAX/5i58C+eSTsNtuSbdKRKRoKq8Hb+YPmg4YAI8/Dr/9rT8rVeEuIlWmsnrwb70FZ5wBTz/tZ8mMGwdf+UrSrRIRSURl9ODXrYPrrvMzZKZN87Nknn5a4S4iVa38e/AffuiLgk2dCkce6cN9662TbpWISOLKvwe/2Wbw5S/DhAnw178q3EVEWpV/D945H+4iItJO+ffgRUQkLQW8iEiFUsCLiFSoWAPeOXeYc+5fzrk3nHM/jXNfIiLSXpxXdKoBbgYOBwYA33bODYhrfyIi0l6cPfi9gTfM7C0zWw3cAxwb4/5ERCRFnAH/JeCdlNuLW+9rxzk33DnX5Jxram5ujrE5IiLVJc6AT3fFaut0h9lYM2s0s8a+ffvG2BwRkeoS54lOi4FtUm5vDbwbtMH06dPfd84tBPoA78fYtnKn1yc7vUbB9PpkVy6vUUOmHzizTp3qSDjnugKvAYOBJcA04Dtm9kqIbZvMrDGWhlUAvT7Z6TUKptcnu0p4jWLrwZvZWufcucBjQA1we5hwFxGRaMRai8bMJgOT49yHiIikV6pnso5NugElTq9PdnqNgun1ya7sX6PYxuBFRCRZpdqDFxGRAingRUQqVEkFvIqTBXPObeOce8Y5N98594pz7vyk21SKnHM1zrmZzrmHk25LKXLObeacm+ice7X1b2m/pNtUSpxzF7b+f811zv3ZOdc96Tblq2QCXsXJQlkL/MDMdgb2Bc7Ra5TW+cD8pBtRwq4H/mZmOwED0Wu1nnPuS8B5QKOZ7Yqf4n1Ssq3KX8kEPCpOlpWZvWdmM1q//xj/j9mpvk81c85tDRwJjEu6LaXIObcJcBBwG4CZrTaz/yTaqNLTFejRerJmHVnOwC9lpRTwoYqTieec6w/sAUxNuCmlZjTwY6Al4XaUqu2AZuCO1mGscc65nkk3qlSY2RLgGmAR8B6wwsweT7ZV+SulgA9VnEzAOdcLmARcYGYfJd2eUuGcOwpYZmbTk25LCesKfA0YY2Z7AJ8AOt7Vyjm3OX7kYFtgK6Cnc+7kZFuVv1IK+JyLk1Uj51w3fLhPMLMHkm5PiTkAOMY5twA/xHewc258sk0qOYuBxWbW9slvIj7wxTsEeNvMms1sDfAAsH/CbcpbKQX8NGB759y2zrla/IGNvyTcppLinHP4sdP5Zva7pNtTaszsYjPb2sz64/9+njazsu19xcHMlgLvOOd2bL1rMDAvwSaVmkXAvs65utb/t8GU8UHoWGvR5ELFyUI5APhfYI5zblbrfZe01vwRCev7wITWjtRbwKkJt6dkmNlU59xEYAZ+1tpMyrhkgUoViIhUqFIaohERkQgp4EVEKpQCXkSkQingRUQqlAJeRKRClcw0SZFic87VA0+13twSWIc/jR9g79aaSCJlS9MkRQDn3OXASjO7JuW+rma2NrlWiRRGPXiRFM65O4Hl+EJuM5xzH5MS/M65ucBRZragtUbJeUAtvujb2Wa2LpmWi3SmMXiRznYADjGzH2RawTm3M3AicICZ7Y4f3hlanOaJhKMevEhn94foiQ8G9gSm+ZIl9ACWxd0wkVwo4EU6+yTl+7W0/6Tbdvk2B9xlZhcXrVUiOdIQjUiwBbSW03XOfQ1fJxz87JvjnXNfaP1Zb+dcQyItFMlAAS8SbBLQu7V65wjgNQAzmwdcCjzunHsZeAL4YlKNFElH0yRFRCqUevAiIhVKAS8iUqEU8CIiFUoBLyJSoRTwIiIVSgEvIlKhFPAiIhXq/wNktYO403xz2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'R2=0.56, R2adj=-0.78, k=[0.95], RMSE=0.97'"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def r2_zero_score(y_true,y_pred):\n",
    "    #https://stats.stackexchange.com/questions/26176/removal-of-statistically-significant-intercept-term-increases-r2-in-linear-mo\n",
    "    return 1 - (sum(y_true-y_pred)**2) / sum(y_true**2)\n",
    "\n",
    "def rmse(y_true,y_pred):\n",
    "    return np.sqrt(np.mean((y_pred-y_true)**2))\n",
    "\n",
    "preds = preds.reshape((-1,))\n",
    "\n",
    "a, _, _, _ = np.linalg.lstsq(y_test[:,np.newaxis],preds,rcond=None)\n",
    "\n",
    "plt.plot(y_test, a*y_test, 'r-')\n",
    "plt.scatter(y_test, preds, color='b')\n",
    "plt.xlabel('True')\n",
    "plt.ylabel('Predicted')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "r2 = r2_score(y_test,preds)\n",
    "r2_adj = (r2-r2_zero_score(y_test,preds))/r2\n",
    "f'R2={np.around(r2,2)}, R2adj={np.around(r2_adj,2)}, k={np.around(a,2)}, RMSE={np.around(rmse(y_test, preds),2)}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4.113102, 4.090493902439023, 1.5448871, 0.361, 7.360203, 8.755)"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.mean(), y_test.mean(), preds.min(), y_test.min(), preds.max(),y_test.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}