Notebooks/ch6_linear_model_selection_and_regularisation_labs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Linear Model Selection and Regularisation – Labs\n",
    "\n",
    "Excercises from **Chapter 6** of [An Introduction to Statistical Learning](http://www-bcf.usc.edu/~gareth/ISL/) by Gareth James, Daniela Witten, Trevor Hastie and Robert Tibshirani."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import patsy\n",
    "import itertools\n",
    "from IPython.display import display, HTML\n",
    "from sklearn import linear_model\n",
    "from sklearn.model_selection import KFold\n",
    "from ipywidgets import interact\n",
    "import ipywidgets as widgets\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AtBat</th>\n",
       "      <th>Hits</th>\n",
       "      <th>HmRun</th>\n",
       "      <th>Runs</th>\n",
       "      <th>RBI</th>\n",
       "      <th>Walks</th>\n",
       "      <th>Years</th>\n",
       "      <th>CAtBat</th>\n",
       "      <th>CHits</th>\n",
       "      <th>CHmRun</th>\n",
       "      <th>...</th>\n",
       "      <th>PutOuts</th>\n",
       "      <th>Assists</th>\n",
       "      <th>Errors</th>\n",
       "      <th>Salary</th>\n",
       "      <th>League_A</th>\n",
       "      <th>League_N</th>\n",
       "      <th>Division_E</th>\n",
       "      <th>Division_W</th>\n",
       "      <th>NewLeague_A</th>\n",
       "      <th>NewLeague_N</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>315</td>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "      <td>24</td>\n",
       "      <td>38</td>\n",
       "      <td>39</td>\n",
       "      <td>14</td>\n",
       "      <td>3449</td>\n",
       "      <td>835</td>\n",
       "      <td>69</td>\n",
       "      <td>...</td>\n",
       "      <td>632</td>\n",
       "      <td>43</td>\n",
       "      <td>10</td>\n",
       "      <td>475.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>479</td>\n",
       "      <td>130</td>\n",
       "      <td>18</td>\n",
       "      <td>66</td>\n",
       "      <td>72</td>\n",
       "      <td>76</td>\n",
       "      <td>3</td>\n",
       "      <td>1624</td>\n",
       "      <td>457</td>\n",
       "      <td>63</td>\n",
       "      <td>...</td>\n",
       "      <td>880</td>\n",
       "      <td>82</td>\n",
       "      <td>14</td>\n",
       "      <td>480.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>496</td>\n",
       "      <td>141</td>\n",
       "      <td>20</td>\n",
       "      <td>65</td>\n",
       "      <td>78</td>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>5628</td>\n",
       "      <td>1575</td>\n",
       "      <td>225</td>\n",
       "      <td>...</td>\n",
       "      <td>200</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>500.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>321</td>\n",
       "      <td>87</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>42</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>396</td>\n",
       "      <td>101</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>805</td>\n",
       "      <td>40</td>\n",
       "      <td>4</td>\n",
       "      <td>91.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>594</td>\n",
       "      <td>169</td>\n",
       "      <td>4</td>\n",
       "      <td>74</td>\n",
       "      <td>51</td>\n",
       "      <td>35</td>\n",
       "      <td>11</td>\n",
       "      <td>4408</td>\n",
       "      <td>1133</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>282</td>\n",
       "      <td>421</td>\n",
       "      <td>25</td>\n",
       "      <td>750.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   AtBat  Hits  HmRun  Runs  RBI  Walks  Years  CAtBat  CHits  CHmRun  \\\n",
       "1    315    81      7    24   38     39     14    3449    835      69   \n",
       "2    479   130     18    66   72     76      3    1624    457      63   \n",
       "3    496   141     20    65   78     37     11    5628   1575     225   \n",
       "4    321    87     10    39   42     30      2     396    101      12   \n",
       "5    594   169      4    74   51     35     11    4408   1133      19   \n",
       "\n",
       "      ...       PutOuts  Assists  Errors  Salary  League_A  League_N  \\\n",
       "1     ...           632       43      10   475.0         0         1   \n",
       "2     ...           880       82      14   480.0         1         0   \n",
       "3     ...           200       11       3   500.0         0         1   \n",
       "4     ...           805       40       4    91.5         0         1   \n",
       "5     ...           282      421      25   750.0         1         0   \n",
       "\n",
       "   Division_E  Division_W  NewLeague_A  NewLeague_N  \n",
       "1           0           1            0            1  \n",
       "2           0           1            1            0  \n",
       "3           1           0            0            1  \n",
       "4           1           0            0            1  \n",
       "5           0           1            1            0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load data\n",
    "hitters_pd = pd.read_csv('./data/Hitters.csv')\n",
    "\n",
    "# Drop 59 rows missing data\n",
    "hitters_pd = hitters_pd.dropna()\n",
    "assert hitters_pd.isna().sum().sum() == 0\n",
    "\n",
    "# Create dummy variables for qualitative features\n",
    "qual = ['League', 'Division', 'NewLeague']\n",
    "hitters_pd = pd.get_dummies(hitters_pd, columns=qual)\n",
    "\n",
    "hitters_pd.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab 1: Subset Selection Methods\n",
    "\n",
    "### 6.5.1 Best subset selection\n",
    "\n",
    "Perform best subset selection by identifying the best model that contains a given number of predictors, where best is quantified using RSS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_models(k, X, y):\n",
    "    \"\"\"\n",
    "    Fit all possible models that contain exactly k predictors.\n",
    "    \"\"\"\n",
    "    # List all available predictors\n",
    "    X_combos = itertools.combinations(list(X.columns), k)\n",
    "    \n",
    "    # Fit all models accumulating Residual Sum of Squares (RSS)\n",
    "    models = []\n",
    "    for X_label in X_combos:\n",
    "        # Parse patsy formula\n",
    "        X_smf = ' + '.join(X_label)\n",
    "        f     = 'Salary ~ {}'.format(X_smf)\n",
    "        # Fit model\n",
    "        model = smf.ols(formula=f, data=pd.concat([X, y], axis=1)).fit()\n",
    "        # Return results\n",
    "        models += [(f, model)]\n",
    "    return models\n",
    "\n",
    "\n",
    "def min_rss(statsmodels):\n",
    "    \"\"\"Return model with lowest Residual Sum of Squares (RSS)\"\"\"\n",
    "    return sorted(statsmodels, key=lambda tup: tup[1].ssr)[0]\n",
    "\n",
    "\n",
    "def max_adjr2(statsmodels):\n",
    "    \"\"\"Return model with lowest R-squared\"\"\"\n",
    "    return sorted(statsmodels, reverse=True, key=lambda tup: tup[1].rsquared_adj)[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which model with 2 predictors yields lowest RSS score?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model yields the lowest RSS score for the subset of models with 2 predictors:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('Salary ~ Hits + CRBI',\n",
       " <statsmodels.regression.linear_model.RegressionResultsWrapper at 0x1c21bf36d8>)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This model yields the highest adjusted R-squared score for the subset of models with 2 predictors:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('Salary ~ Hits + CRBI',\n",
       " <statsmodels.regression.linear_model.RegressionResultsWrapper at 0x1c22224c50>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = hitters_pd.drop('Salary', axis=1)\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "# Set number ofr predictors in subset\n",
    "k = 2\n",
    "\n",
    "# Get all possible models in subset\n",
    "subset = get_models(2, X, y)\n",
    "\n",
    "# Display results\n",
    "print('This model yields the lowest RSS score for the subset of models with {} predictors:'.format(k))\n",
    "display(min_rss(subset))\n",
    "\n",
    "print('This model yields the highest adjusted R-squared score for the subset of models with {} predictors:'.format(k))\n",
    "max_adjr2(get_models(2, X, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years', 'CAtBat',\n",
       "       'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'PutOuts', 'Assists',\n",
       "       'Errors', 'League_A', 'League_N', 'Division_E', 'Division_W',\n",
       "       'NewLeague_A', 'NewLeague_N'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model with 2 predictors that yields the lowest RSS score is the one that uses the predictors 'Hits', and 'CRBI'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How do the the maximum $R^2$, minimum RSS, and BIC statistics vary for each subset with k predictors? \n",
    "\n",
    "First we fit all possible models in each subset of models with k predictors. This turns out to be a very computationally expensive process, running on my laptop for 30 minutes yielded results up to only k=6 and occupied 30GB of memeory. Not suprising as number of possible combinations without repetition is given by:\n",
    "\n",
    "$\\frac{p!}{k!(p-k)!}$\n",
    "\n",
    "Where *p* is number of predictors to choose from and we choose *k* of them.\n",
    "\n",
    "In this example p=22, there are 22 predictors.\n",
    "\n",
    "For k = 4 the number of possible combinations is given by:\n",
    "\n",
    "$\\frac{22!}{4!x 18!} = 74,613$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXJ3sCIWEJkJVNkH0NBLVq1YqoCIhakWix1evtvXprW/trta21rbW17e1mb/VWLS41SFVcsOJCUa/VlkBA2cEgkIUECFsSCFnn8/tjDjZClglk5szyeT4e5zEz3zkz8w6PhM+c8/2e71dUFWOMMcZXUW4HMMYYE1qscBhjjOkSKxzGGGO6xAqHMcaYLrHCYYwxpkuscBhjjOkSKxzGGGO6xAqHMcaYLrHCYYwxpkti3A7gD/369dPBgwe7HcMYY0LK2rVrD6hqWmf7hWXhGDx4MEVFRW7HMMaYkCIiJb7sZ6eqjDHGdInfCoeIJIjIahFZLyKbReRHTvuTIrJLRD5ytolOu4jIQyKyQ0Q2iMjkVu+1UESKnW2hvzIbY4zpnD9PVTUAF6vqURGJBd4Xkded5/6fqr5w0v6XA8OdLQ94BMgTkT7AfUAuoMBaEVmmqof9mN0YY0w7/HbEoV5HnYexztbRHO5zgKed160CUkUkHbgMWKGqh5xisQKY6a/cxhhjOubXPg4RiRaRj4D9eP/zL3SeesA5HfUbEYl32jKBslYvL3fa2ms/+bNuE5EiESmqqqrq9p/FGGOMl18Lh6q2qOpEIAuYJiJjgXuAkcBUoA/wHWd3aestOmg/+bMeVdVcVc1NS+t0NJkxxpjTFJBRVap6BHgXmKmqlc7pqAbgCWCas1s5kN3qZVlARQftxhhjXODPUVVpIpLq3E8EvgBsc/otEBEB5gKbnJcsA77kjK6aDlSraiXwJjBDRHqLSG9ghtNmjOu2VNTwxqZKt2MYE1D+HFWVDjwlItF4C9RzqvpXEXlbRNLwnoL6CPiqs/9y4ApgB1AHfBlAVQ+JyP3AGme/H6vqIT/mNsYnG8urWfDYKmobmvnR7DEsPHew25GMCQi/FQ5V3QBMaqP94nb2V+D2dp5bBCzq1oDGnIGP99XypUWF9EqMJXdwb+5btpmkuGiuy83u/MXGhLiwnHLEGH8qOXiMGx8vJCY6ioJb80hPTeDWp4r4ztIN9IiP4Ypx6W5HNMavbMoRY7qgsvo4Cx4rpLHFwzO35DG4Xw/iY6L5401TmJTTmzuXfMg72/a7HdMYv7LCYYyPDhxtIP/xQqqPN/H0V6Zx9sDkT59Lioth0c1TGTEgma8+s5ZVOw+6mNQY/7LCYYwPquuauOlPq6k4cpxFN09lfFbqKfukJMby9Femkd0niVueXMNHZUdcSGqM/1nhMKYTxxqaufnJ1ezYX8sfb8pl2pA+7e7bt2c8z9ySR5+ecSxctJpte2sCmNSYwLDCYUwH6pta+Leni9hQXs3vb5jEhSM6n5VgYEoCi2+dTkJsFDc+vpqdVUc7fY0xocQKhzHtaGrxcMfidfzjk4P88trxzBzr+2ip7D5JFNyah0eVGx8vZM+R435MakxgWeEwpg0tHuWbz63nb1v3c/+cMcybnNXl9zirfzJPf2UatQ3N5D+2iv219X5IakzgWeEw5iSqyvde2sir6yu4+/KR3HTO4NN+r7GZKTz55ansr23gpsdXc6SusfuCGuMSKxzGtKKq3P/XrSxZU8YdF53FVy8cdsbvOWVQHx77Ui67Dhxj4aLV1NY3dUNSY9xjhcOYVn7zt2IWfbCLm88dzF0zRnTb+553Vj/+kD+ZTRU13PJUEccbW7rtvY0JNCscxjgefe8THlpZzHVTsvjBrNF4J3DuPpeOHsCvvziBNbsP8R8Fa2ls9nTr+xsTKFY4jAEKCkv46fJtXDkunQevGU9UVPcWjRPmTMzkp1eP493tVdy55EOaW6x4mNBjhcNEvJc/3MP3X97ExSP785vrJxLtp6Jxwg3Tcvj+laN4fdNevrN0Ix7PKQtaGhPUbHZcE9He2ryXu55fT96QPjycP5m4mMB8l7r1/KEcbWjmt38rpmd8ND+cPabbT40Z4y9WOEzE+ntxFXcs/pBxmSk8vnAqCbHRAf38Oy8ZzrGGZh77+y56xMfw7ZkjA/r5xpwuKxwmIhXtPsRtT69laFoPnvzyVHrGB/5PQUT47hWjONrQwsPvfkKP+Bhuv+isgOcwpquscJiIs2lPNV9+Yg3pKQn8+ZY8UpPiXMsiIvxk7ljqGpv55Zvb6RkfY0vQmqBnhcNElOJ9tdz0J++Sr8/cmkdacrzbkYiOEv77ugnUNbZw37LN9IiP4dopXZ/ixJhA8VtPoIgkiMhqEVkvIptF5EdO+xARKRSRYhH5i4jEOe3xzuMdzvODW73XPU77dhG5zF+ZTXgrPVjHjX/615KvGamJbkf6VGx0FL+/YRKfO6sf335hPcs3VrodyZh2+XMISQNwsapOACYCM0VkOvBz4DeqOhw4DNzi7H8LcFhVzwJ+4+yHiIwG5gNjgJnAwyIS2F5ME/L2Vtez4PFVNDT/a8nXYJMQG82jX2q1BO12W4LWBCe/FQ71OrEQQayzKXAx8ILT/hQw17k/x3mM8/wl4h2fOAdYoqoNqroL2AFM81duE368S76u4kjdqUu+BpvPLEH7Z1uC1gQnvw5aF5FoEfkI2A+sAD4Bjqhqs7NLOZDp3M8EygCc56uBvq3b23iNMR1q8ShffmINezpY8jXYnLwErS0EZYKNXwuHqrao6kQgC+9Rwqi2dnNu27r6STto/wwRuU1EikSkqKqq6nQjmzDzzrb9bNxTzc/mjetwyddg07dnPH++ZRpNHuWJD3a7HceYzwjIZbKqegR4F5gOpIrIidFcWUCFc78cyAZwnk8BDrVub+M1rT/jUVXNVdXctLTOl/c0kaGgsIT+yfHMGp/hdpQuS09JZNa4dF76cA/HGpo7f4ExAeLPUVVpIpLq3E8EvgBsBd4BrnV2Wwi84txf5jzGef5tVVWnfb4z6moIMBxY7a/cJnyUHarj3Y+rmD81m9jo0JyWLX96Dkcbmlm2/pTvSsa4xp9/TenAOyKyAVgDrFDVvwLfAb4pIjvw9mH8ydn/T0Bfp/2bwN0AqroZeA7YArwB3K6qtpiB6dSSNaUIcP20HLejnLbJOb0ZOTCZZ1aV4P0eZYz7/HYBoKpuACa10b6TNkZFqWo9cF077/UA8EB3ZzThq6nFw1/WlHPxyP5kBtH1Gl0lIuTn5XDvK5vZUF7NhOzg79w34S80j9+N6cSKLfu8w3DzBrkd5YzNnZRJUlw0iwtL3Y5iDGCFw4SpgsISMlMTuWBE6A+USE6IZc7EDJatr6D6uK1XbtxnhcOEnZ1VR/lgx0EW5OX4fVGmQFkwbRDHm1p4+cM9bkcxxgqHCT/Pri4lJkq4Ljd8Jgocl5XChKwUCgqtk9y4zwqHCSv1TS08v7acy8YMpH9ygttxulV+3iA+3neUopLDbkcxEc4Khwkrr2+q5EhdE/l5oTsEtz2zJqSTnBBDwaoSt6OYCGeFw4SVglWlDO3Xg3OG9XU7SrdLiovhmslZLN+4l0PHGt2OYyKYFQ4TNrbtraGo5DAL8nLwTqwcfhbk5dDY4uGFtWWd72yMn1jhMGFjcWEpcTFRXDM5fDrFTzZiQDLTBvdhcWEpHo91kht3WOEwYeFYQzMvrtvDlePS6d3DvTXEA2FBXg67D9bxj09srQ7jDiscJiy8ur6Cow3NYdkpfrKZYwfSOymWgkLrJDfusMJhwkJBYSlnD0hmyqDebkfxu4TYaK7LzeatLfvYV1PvdhwTgaxwmJC3ofwIG/dUkz89fDvFT3bDtBxaPMpza6yT3ASeFQ4T8hYXlpIYG83cSZGzovCQfj343Fn9eHZ1KS3WSW4CzAqHCWk19U288lEFcyZm0Csh1u04AZWfl0NFdT3vbt/vdhQTYaxwmJD28od7ON7UEhbTp3fVF0YPIC05ngKbbt0EmBUOE7JUlYJVpYzPSmFcVorbcQIuNjqK+VOzeWf7fsoP17kdx0QQKxwmZK0tOcz2fbURMQS3PfOn5SDAX6yT3ASQFQ4TsgoKS0mOj+GqCRluR3FNZmoiF53dnyVrymhq8bgdx0QIKxwmJB061shrGyuZNzmTpLgYt+O4Kn96DlW1Dfxtyz63o5gI4bfCISLZIvKOiGwVkc0icqfT/kMR2SMiHznbFa1ec4+I7BCR7SJyWav2mU7bDhG521+ZTehYuracxmYPCyKwU/xkF47oT2ZqonWSm4Dx5xFHM3CXqo4CpgO3i8ho57nfqOpEZ1sO4Dw3HxgDzAQeFpFoEYkG/gBcDowGbmj1PiYCeTzK4tWlTB3cm7MHJrsdx3XRUcIN07J5f8cBdh045nYcEwH8VjhUtVJV1zn3a4GtQEdXaM0Blqhqg6ruAnYA05xth6ruVNVGYImzr4lQ/9x5kF0HjkXkENz2fDE3m5go4dnVdtRh/C8gfRwiMhiYBBQ6TXeIyAYRWSQiJyYXygRaDw0pd9raazcRqqCwhN5JscwcO9DtKEGjf68ELh09gOeLyqhvanE7jglzfi8cItITWAp8XVVrgEeAYcBEoBL41Yld23i5dtB+8ufcJiJFIlJUVVXVLdlN8NlfW89bm/dx7ZQsEmKj3Y4TVPLzBnG4rok3Nu11O4oJc34tHCISi7doFKjqiwCquk9VW1TVAzyG91QUeI8kslu9PAuo6KD9M1T1UVXNVdXctLS07v9hTFB4vqicZo9yw7TIvXajPecO68vgvkk23brxO3+OqhLgT8BWVf11q/b0VrtdDWxy7i8D5otIvIgMAYYDq4E1wHARGSIicXg70Jf5K7cJXi0eZXFhKeed1ZehaT3djhN0oqKEBXk5rNl9mO17a92OY8KYP484zgNuAi4+aejtL0Rko4hsAC4CvgGgqpuB54AtwBvA7c6RSTNwB/Am3g7255x9TYR57+Mq9hw5bp3iHbh2SjZx0VEstqMO40d+u3JKVd+n7f6J5R285gHggTbal3f0OhMZCgpLSEuO59LRA9yOErT69IjjinEDeXHdHr5z+ciIvzjS+IddOW5Cwp4jx3l7236uz80mNtp+bTuSP30QtQ3NvLr+lK5AY7qF/QWakPCX1aUoMH9adqf7RrrcQb0ZMaCnXUlu/MYKhwl6TS0elqwp46Kz+5PVO8ntOEFPRMjPG8SG8mo2lB9xO44JQ1Y4TNBbuXUf+2sbInr69K66enImibHRLLajDuMHVjhM0CsoLCUjJYHPn93f7Sgho1dCLLMnZPDKRxXU1De5HceEGSscJqjtPnCMvxcf4IZpOURHtTVIz7Qnf3oOx5taeOXDPW5HMWHGCocJas+uLiU6Srh+qnWKd9X4rFTGZaZQUFiK6imz9Bhz2qxwmKDV0NzCc0VlzBg9gP69EtyOE5Ly83LYtreWdaWH3Y5iwogVDhO03ti0l8N1TSywTvHTdtWEDHrGx1CwyjrJTffxqXCIyHki0sO5f6OI/FpEbN4H41cFhaUM6pvEecP6uR0lZPWIj+HqSZn8dWMlh481uh3HhAlfjzgeAepEZALwbaAEeNpvqUzEK95Xy+pdh1gwLYco6xQ/Iwvycmhs9rB0XbnbUUyY8LVwNKu3d20O8DtV/R1ga3YavykoLCUuOoprp2S5HSXkjUrvxZRBva2T3HQbXwtHrYjcA9wIvOasAx7rv1gmkh1vbGHpunIuHzeQvj3j3Y4TFvLzcth14Bj//OSg21FMGPC1cFwPNAC3qOpevEu3/tJvqUxEe3VDBbX1zTZ9eje6Ylw6qUmxNn+V6RY+FQ5V3auqv1bVvzuPS1XV+jiMXxQUljK8f0+mDu7d+c7GJwmx0Vw7OYs3N+9lf22923FMiPN1VNU8ESkWkWoRqRGRWhGp8Xc4E3k27almfdkR8vNy8C4iabrLDXk5NHuU54usk9ycGV9PVf0CmK2qKaraS1WTVbWXP4OZyFRQWEpCbBRXT7ZO8e42LK0n5w7ry+LCUlo81kluTp+vhWOfqm71axIT8Wrrm3jloz3MnpBBSqKNvfCH/LxB7DlynPc+rnI7iglhvq4rWSQifwFexttJDoCqvuiXVCYivfxRBXWNLdYp7keXjh5Av57xFBSWcNFIm23YnB5fC0cvoA6Y0apNASscpluoKgWrShib2YvxWSluxwlbcTFRXD81i0fe/YQ9R46TmZrodiQTgnwdVfXlNravdPQaEckWkXdEZKuIbBaRO532PiKywulsXyEivZ12EZGHRGSHiGwQkcmt3muhs3+xiCw8kx/YBKd1pUfYtreW/LxB1inuZ/On5qB4l+M15nT4OqoqS0ReEpH9IrJPRJaKSGe9l83AXao6CpgO3C4io4G7gZWqOhxY6TwGuBwY7my34Z3mBBHpA9wH5AHTgPtOFBsTPgoKS+gZH8PsCRluRwl72X2S+PyINJasKaOpxeN2HBOCfO0cfwJYBmTgvfjvVaetXapaqarrnPu1wFbntXOAp5zdngLmOvfnAE+r1yogVUTSgcuAFap6SFUPAyuAmT7mNiHgSF0jr22o5OpJmfSI9/XsqTkT+XmD2F/bwMqt+92OYkKQr4UjTVWfUNVmZ3sSSPP1Q0RkMDAJKAQGqGoleIsLcKKHLhMoa/WycqetvXYTJpau20NDs8emTw+gi0b2JyMlgYLCErejmBDka+E44EynHu1sNwI+TXojIj2BpcDXVbWjiwbbOrGtHbSf/Dm3iUiRiBRVVdlQw1ChqhQUljA5J5VR6XZpUKB4V1XM4e/FByg5eMztOCbE+Fo4vgJ8EdgLVALXOm0dEpFYvEWjoNXQ3X3OKSic2xPHyuVA6/VBs4CKDto/Q1UfVdVcVc1NS/P5YMi4bNXOQ+ysOmZDcF1w/dRsoqOExdZJbrrI11FVpao6W1XTVLW/qs5V1Q6PccU7NOZPwFZV/XWrp5YBJ0ZGLQReadX+JWd01XSg2jmV9SYwQ0R6O53iM5w2EwYKCktISYzlyvHpbkeJOANTEvjCqP48X1ROQ3OL23FMCOmwJ1JEvq2qvxCR39PG6SFV/VoHLz8PuAnYKCIfOW3fBR4EnhORW4BS4DrnueXAFcAOvNeMfNn5jEMicj+wxtnvx6p6yJcfzgS3qtoG3ty8ly+dM5iE2Gi340Sk/LxBvLl5H29s2sucidZ1aHzT2RCWE9OMFHX1jVX1fdrunwC4pI39Fbi9nfdaBCzqagYT3J5fW0ZTi1qnuIs+d1Y/cvokUVBYaoXD+KzDwqGqrzp361T1+dbPich1bbzEGJ94PMriwlLOGdqXYWk93Y4TsaKihAV5OTz4+jaK99UyfIAt7Gk652vn+D0+thnjk/eKqyg/fJz86Xa04bbrpmQRGy22yJPxWWd9HJfj7XfIFJGHWj3VC++V4cacloLCUvr1jGPG6IFuR4l4fXvGc/nYdJauK+c7M0eSGGf9TaZjnR1xVODt36gH1rbaluG9otuYLqusPs7Krfv4Ym42cTG+HvQaf8rPy6G2vplXN5wy0t2YU3TWx7EeWC8ii1W1KUCZTJhbsroMBW6YZqepgsW0IX04q39PCgpL+WJuducvMBHN1697g0XkBRHZIiI7T2x+TWbCUnOLhyVrSrlwRBrZfZLcjmMcIkJ+Xg7ry46waU+123FMkOvKJIeP4O3XuAh4Gvizv0KZ8PX2tv3sq2lggR1tBJ15k7JIiI2yK8lNp3wtHImquhIQVS1R1R8CF/svlglXBYWlDOyVwMW2+lzQSUmK5arxGbzy4R6ONtjYF9M+XwtHvYhEAcUicoeIXM2/ZrU1xielB+t4r7iK+dOyiYm2TvFgtCAvh2ONLbz84R63o5gg5utf79eBJOBrwBS8U4nYSnymS55dU0qUCPOn2mmqYDUxO5XR6b0oKCzFO5mDMafydZLDNap6VFXLnWVj5zmLLRnjk8ZmD8+tKeOSkf0ZmJLgdhzTDhEhf3oOWytr+LDsiNtxTJDydenYESLymIi8JSJvn9j8Hc6Ejzc37+XgsUbyp9v06cFuzsRMesRFU7DKOslN23xdp/N54H+BxwCbf9l0WUFhCdl9Ejn/rH5uRzGd6Bkfw9xJmbywtpx7Z40iNSnO7UgmyPjax9Gsqo+o6mpVXXti82syEzZ27D/Kqp2HWDBtEFFR7U2YbIJJft4gGpo9LF1nneTmVL4WjldF5D9FJF1E+pzY/JrMhI3FhaXERgvX5Wa5HcX4aHRGLyblpFJQWGKd5OYUvhaOhcD/A/7Bv+ar6vIaHSby1De18MLaMmaOTadfz3i345guyM8bxM6qY6zaaeummc/ydVTVkDa2of4OZ0LfXzdUUlPfTL4t1hRyZo1Pp1dCDAWFHa4SbSJQZ9OqX6yqb4vIvLaeV9UX/RPLhIuCwhKGpfUgb4id2Qw1CbHRXDslmz+v2k1VbQNpyXbEaLw6O+K40Lm9qo1tlh9zmTCwpaKGD0uPsCBvECLWKR6KFuTl0NSiPL+2zO0oJoh0Nq36fc7tlwMTx4STxatLiI+J4prJtpZ1qDqrf0+mD+3Ds6tL+eoFw2xUnAF8vwCwr4g8JCLrRGStiPxORPp28ppFIrJfRDa1avuhiOwRkY+c7YpWz90jIjtEZLuIXNaqfabTtkNE7j6dH9IE3tGGZl5at4dZ4zPsOoAQl583iLJDx/n7jgNuRzFBwtdRVUuAKuAa4Frn/l86ec2TwMw22n+jqhOdbTmAiIwG5gNjnNc8LCLRIhIN/AG4HBgN3ODsa4Lcso8qONbYYmuKh4HLxgykb484ClZZJ7nx8rVw9FHV+1V1l7P9BEjt6AWq+h7g6zi+OcASVW1Q1V3ADmCas+1Q1Z2q2oi3gM3x8T2NS1SVgsISRqX3YlJ2h78mJgTExUTxxanZrNy2n8rq427HMUHA18LxjojMF5EoZ/si8NppfuYdIrLBOZXV22nLBFr3vpU7be21myC2vryazRU15OflWKd4mLhhag4tHuUva6yT3HRSOESkVkRqgH8HFgONzrYE+MZpfN4jwDBgIlAJ/OrER7Wxr3bQ3lbW20SkSESKqqqqTiOa6S4Fq0roERfN3ElW48NFTt8kLhiRxpLVZTS3eNyOY1zWYeFQ1WRV7eXcRqlqjLNFqWqvrn6Yqu5T1RZV9eCdMHGa81Q5kN1q1yygooP2tt77UVXNVdXctLS0rkYz3aS6rolXN1QwZ1ImPeN9nUPThIL8vBz21tTz9rb9bkcxLvN5GTYRGS8is0Vk3omtqx8mIumtHl4NnBhxtQyYLyLxIjIEGA6sBtYAw0VkiIjE4e1AX9bVzzWB8+KH5dQ3eWxN8TB0ycj+DOgVT0GhTbce6Xz6Sigii4DxwGbgxHGqAu1eOS4izwKfB/qJSDlwH/B5EZnovHY33lNgqOpmEXkO2AI0A7eraovzPncAbwLRwCJV3dy1H9EEirdTvJSJ2amMzUxxO47pZjHRUcyfmsNDbxdTerCOnL5JbkcyLvH1XMJ0Ve3SMFhVvaGN5j91sP8DwANttC8Hlnfls407Vu86xI79R/nltePdjmL8ZP60bH7/djHPrinlOzNHuh3HuMTXU1X/tOsnTGcKCktJTohh1vgMt6MYP0lPSeSSUQN4bk0Zjc3WSR6pfC0cT+EtHtudobQbRWSDP4OZ0HLgaAOvb6rkmslZJMZFux3H+FF+Xg4HjzXy5ua9bkcxLvH1VNUi4CZgI//q4zDmUy+sLaepRW369AhwwfA0snonUlBYwlUT7OgyEvl6xFGqqsucq8ZLTmx+TWZChsejPLu6lGlD+jB8QLLbcYyfRUUJC/JyWLXT26dlIo+vhWObiCwWkRvOZDiuCU8ffHKAkoN1drQRQa6bkk1stPDsahuaG4l8LRyJQAMwA1uPw5ykYFUpfXrEMXPsQLejmABJS47nsjEDeWFtOfVNLW7HMQHmUx+Hrcdh2rOvpp4VW/dx6/lDiI+xTvFIkp83iL9uqOS1DZVcMyXL7TgmgHxdjyNLRF5y1tfYJyJLRcR+Uwx/WVNGi0ftSvEINH1oH4am9bA1ySOQr6eqnsA71UcG3tlpX3XaTARrbvHw7OpSzh/ej0F9e7gdxwSYiJCfN4h1pUfYUlHjdhwTQL4WjjRVfUJVm53tScBmEoxw726vorK6nvy8QW5HMS65ZnImcTFRLF5tRx2RxNfCcUBEbjyxKp+I3Agc9GcwE/wKCksY0CueS0b1dzuKcUlqUhyzxqfz0ro9HG1odjuOCRBfC8dXgC8Ce/Guo3EtYB3mEazsUB3vflzF9VNziI32eZJlE4by8wZxrLGFZR+1ueKBCUO+/sXfDyxU1TRV7Y+3kPzQb6lM0FuyphQB5k/N7nRfE94m56QycmAyBYUlqLa5zpoJM74WjvGqevjEA1U9BEzyTyQT7BqbPfxlTTkXj+xPRmqi23GMy0SE/OmD2FxRw/ryarfjmADwtXBEtVofHBHpg+/zXJkws2LLPg4cbbBOcfOpuRMzSIqLpmCVdZJHAl8Lx6+Af4jI/SLyY+AfwC/8F8sEs4LCEjJTE7lghA2sM17JCbHMmZjJqxsqqD7e5HYc42c+FQ5VfRq4BtgHVAHzVPXP/gxmgtPOqqP845ODLMjLITpK3I5jgkh+Xg71TR5eWlfudhTjZz6fblLVLXiXdjUR7NnVpcRECdfl2sQB5rPGZqYwITuVgsJSFp47GBH7YhGubByl8Vl9UwvPry3nsjED6Z+c4HYcE4Ty83Io3n+UNbsPd76zCVlWOIzPXt9UyZG6Jps+3bTrqvEZJCfE2PxVYc5vhUNEFjmTIm5q1dZHRFaISLFz29tpFxF5SER2OEvTTm71moXO/sUistBfeU3nClaVMrRfD84Z1tftKCZIJcZFc83kLF7fuJeDRxvcjmP8xJ9HHE8CM09quxtYqarDgZXOY4DLgeHOdhvwCHw67Pc+IA+YBtzXeliwCZxte2soKjnMgrwcO3dtOpSfl0Nji4cX1lonebjyW+FQ1feAQyc1zwGecu4/Bcxt1f60eq0CUkUkHbgMWKGqh5wLEFdwajEyAbC4sJR0fng+AAAS5ElEQVS4mCiumWyd4qZjwwckM21IHxavLsXjsSvJw1Gg+zgGqGolgHN7Yna8TKCs1X7lTlt77acQkdtEpEhEiqqqqro9eCTbWXWUJWvKmD0hg9494tyOY0LAl84ZRMnBOpasKet8ZxNygqVzvK1zH9pB+6mNqo+qaq6q5qal2YVp3UVV+e5LG4mPieLbl53tdhwTIq4cl845Q/vys9e3sr+m3u04ppsFunDsc05B4dzud9rLgdaz5WUBFR20mwB5rqiMVTsP8d0rRtG/lw3BNb4REX46bxwNzR7uW7bZ7TimmwW6cCwDToyMWgi80qr9S87oqulAtXMq601ghoj0djrFZzhtJgD219bzwGtbmTakD9fn2iy4pmuG9OvBnZcM5/VNe3lz816345hu5M/huM8C/wTOFpFyEbkFeBC4VESKgUudxwDLgZ3ADuAx4D/h01l47wfWONuPnTYTAD9atoX6Zg8/mzeOKJtexJyG2y4YysiByfzglU3U1NscVuFCwnH+/NzcXC0qKnI7Rkj725Z93Pp0Ed+aMYI7Lh7udhwTwtaXHeHqhz9gQV4OP5k7zu04pgMislZVczvbL1g6x00Qqa1v4t5XNnH2gGRuu2CY23FMiJuQncrN5w7hmVWlFO22EwbhwAqHOcUv39zO3pp6HrxmHHEx9itiztxdM0aQmZrId5ZuoKG5xe045gzZ/wrmM9aWHObPq0pYeM5gJuXYRfqme/SIj+EnV4/lk6pjPPzOJ27HMWfICof5VGOzh7uXbiC9VwLfsms2TDe76Oz+zJmYwcPv7qB4X63bccwZsMJhPvXIu59QvP8oP7l6LD3jbWVg0/3unTWaHvEx3P3iRpuOJIRZ4TAA7Nhfyx/e2cFVEzK4eOQAt+OYMNWvZzz3XjmatSWHber1EGaFw+DxKPe8uJHEuGh+MGu023FMmJs3OZPzh/fj529sp7L6uNtxzGmwwmFYvLqUNbsP8/0rR5GWHO92HBPmRIQH5o6j2ePh3pc3E47XkoU7KxwRbm91PT9/fRvnndWXa6fYlOkmMHL6JvHNS0fwt637eH2TTUcSaqxwRLj7lm2iscXDA3PH2QJNJqC+ct4Qxmb24r5lm6mus+lIQokVjgj2xqZK3ty8j29cOoLB/Xq4HcdEmJjoKB6cN55Dxxr52etb3Y5jusAKR4SqPt7ED17ZzOj0Xtz6uSFuxzERamxmCrd8bghL1pSxaudBt+MYH1nhiFA/f2MbB4428PNrxhMTbb8Gxj3f+MIIcvokcc+LG6lvsulIQoH9jxGBCnceZHFhKbd8bgjjslLcjmMiXGJcNA9cPZZdB47x+7eL3Y5jfGCFI8LUN7Vwz0sbyeqdyDcuHeF2HGMAOH94GvMmZ/LH/9vJ1soat+OYTljhiDAPv7ODnVXH+OnV40iKs2lFTPC498rRpCTGcveLG2mx6UiCmhWOCLJ9by0Pv/sJ8yZlcsGINLfjGPMZvXvE8YOrRrO+7AhP/WO323FMB6xwRIgWj3L3ixvolRjL921aEROkZk/I4MIRafz3W9spP1zndhzTDiscEeKZVSV8WHqEe2eNok+POLfjGNMmEeGBq8cCcO/Lm2w6kiBlhSMCVBw5zi/e2MYFI9KYOzHT7TjGdCirdxJ3zTibd7ZX8eqGSrfjmDa4UjhEZLeIbBSRj0SkyGnrIyIrRKTYue3ttIuIPCQiO0Rkg4hMdiNzqFJV7n15Ex6FB+aOtWlFTEi4+dzBTMhK4UfLNnP4WKPbccxJ3DziuEhVJ6pqrvP4bmClqg4HVjqPAS4HhjvbbcAjAU8awl7bWMnKbfu5a8YIsvskuR3HGJ9ERwk/mzeeI8ebeGC5TUcSbILpVNUc4Cnn/lPA3FbtT6vXKiBVRNLdCBhqjtQ18sNlmxmflcLN5w52O44xXTI6oxf/fsFQXlhbzvvFB9yOY1pxq3Ao8JaIrBWR25y2AapaCeDc9nfaM4GyVq8td9o+Q0RuE5EiESmqqqryY/TQ8dPlWzlc18SD82xaEROavnbJcAb3TeK7L23keKNNRxIs3Prf5DxVnYz3NNTtInJBB/u2dVL+lKEWqvqoquaqam5aml2j8I9PDvBcUTn/dv5QRmf0cjuOMaclITaan84bR+mhOn678mO34xiHK4VDVSuc2/3AS8A0YN+JU1DO7X5n93Igu9XLs4CKwKUNPfVNLXz3xY0M6pvE178w3O04xpyRc4f14/rcbB7/+y427al2O47BhcIhIj1EJPnEfWAGsAlYBix0dlsIvOLcXwZ8yRldNR2oPnFKy7TtdyuL2X2wjp9dPY6E2Gi34xhzxr57xSh6J8Vx94sbaG7xuB0n4rlxxDEAeF9E1gOrgddU9Q3gQeBSESkGLnUeAywHdgI7gMeA/wx85NCxpaKGR9/byXVTsjj3rH5uxzGmW6QkxfLD2aPZtKeGJz7Y7XaciBfwWe5UdScwoY32g8AlbbQrcHsAooW8E9OK9E6K5XtXjnI7jjHd6spx6bw8ag+/WrGdy8YMJKevDS93iw21CSNPfLCLDeXV3HfVGFKTbFoRE15EhB/PGUu0CN97eaNNR+IiKxxhonDnQX711sdcPLI/s8bbZS4mPGWkJvLtmSP5e/EB/vDODjw2/borrHCEuOONLfzo1c1c/+gq+veK5yc2rYgJczdOH8RlYwbw3299zA2PraL0oM2iG2hWOEJY0e5DXP6793jig93cfO5gXr/zfDJSE92OZYxfRUcJ/3vjFH5xzXi2VNQw83fv8edVJXb0EUC2BFwIqm9q4Vdvbefx93eRmZrIs/82nXOG9XU7ljEBIyJ8cWo25w3vx91LN3Dvy5t4Y1MlP79mPFm9rdPc3yQcO5hyc3O1qKjI7Rh+sa70MN96fj07q45x4/Qc7rl8FD3irf6byKWqPLu6jAde2wLA92eNZv7UbDtlexpEZG2riWfbZf/jhIj6phZ++7diHn3vE9JTEnnmljw+N9yu0zBGRFiQl8MFI/rx7Rc2cM+LG1m+0Xv0Yadu/cOOOELA+rIjfOv59RTvP8oN07L57hWjSE6IdTuWMUHH41EKVpfys+VbiRbh3qtGc92ULDv68JEdcYSBhuYWHlpZzP/+307Sesbz1FemceEIm8DRmPZERQk3TR/EhcPT+H8vrOfbL2zg9Y2V/GzeeAamJLgdL2zYEUeQ2rSnmrueW8/2fbVcNyWL788aTUqiHWUY4yuPR3n6n7t58I1txEVH8cPZY7h6UqYdfXTA1yMOG44bZBqbPfx6xcfM+cMHHK5rZNHNufzyuglWNIzpoqgo4ebzhvDGnRcwYkAy33xuPf/29Fr219a7HS3k2RFHENlSUcNdz69na2UN8yZlct9VY0hJsoJhzJlq8ShPfLCLX765ncS4aH40ewyzJ2TY0cdJ7IgjhDS1eHhoZTGz/+d9qmobePSmKfz6+olWNIzpJtFRwq3nD2X5neczpF8P7lzyEf/xzDoOHG1wO1pIsiMOl23fW8tdz3/Epj01zJ6QwY9mj6F3D5ug0Bh/afEoj/99J79a8TE942O4f85YrrT53QA74gh6zS0e/vDODq76/ftUHqnnkfzJPHTDJCsaxvhZdJTw7xcO47X/+hzZvRO5ffE6bl+8jkPHGt2OFjJsOK4Lduyv5a7n1rO+vJorxg3k/jlj6dsz3u1YxkSU4QOSWfof5/LH93by2799TOHOg/xk7lhmjrWjj87YEUcAtXiUP/7fJ1zx0PuUHqrjfxZM4uH8KVY0jHFJTHQUt190Fq/+1+cYmJLAV59Zx9ee/ZDDdvTRIevj8JPmFg87DxxjS0UNmyuq2VJZw+aKGo7UNTFj9AAeuHocaclWMIwJFk0tHh559xMeWllMdJQwcmAyozN6MTojhdHpvRiVnkxSXHifpPG1j8MKRzeoa2xm295aNlfUsKWihi0V1WzbW0tDsweAuJgo7y9hei8+f3Z/LhszwIYBGhOktlbWsHRt+adf9qqPNwEgAkP69WCMU0jGZPRidEYv+oXRGYOwKxwiMhP4HRANPK6qD7a3rz8Lx8GjDd4C4fxSbamoZteBY5xYCiAlMdb7C5Xu/aUak5HC0LQexEbbWUFjQo2qUlFdz+Y91a3+5mvYc+T4p/sM6BXvFJIU52++F9m9k4iKCr0vh2E1V5WIRAN/AC4FyoE1IrJMVbf46zNVldJDdc6pphOFopp9Nf8a952ZmsjojF5cNSHD+4uTmUJGSoIdTRgTJkSEzNREMlMTmTFm4KftR+oa2VJ54gyD9/+H94oP0OJ8g+wZH/Ppl8fRzhfJEQOSiYsJjy+QIVE4gGnADlXdCSAiS4A5QLcWjkPHGnloZTFbKmrYWllDbUMz4B2+d1ZaT84b1u8zvwipSTZ01phIlJoUx7nD+nHusH8tbVDf1ELxvqOf6dN8rqiMusYWAGKjhbP6JzNqYDKpSXHEx0YRHxNFQmw08TFRxMc4t7FRJMREO89Hf3afk9qiXTqqCZXCkQmUtXpcDuR194fEx0Txwtpyzh6YzNxJmZ+ewxwxIJmE2Oju/jhjTBhJiI1mXFYK47JSPm3zeJTdB4995jTXPz45yNGGZhqaW2hqObOugpgocQrKv4rJ2MwUfn/DpDP9cTr+XL++e/dpq6x+5l9cRG4DbgPIyck5rQ/pER/DhvtmhOS5SWNM8ImKEoam9WRoWk9mjc845fkWj9LQ3EJDk4eGZo/3frOH+ibvrbf95DbntnVbq/fI6u3/xatCpXCUA9mtHmcBFa13UNVHgUfB2zl+uh9kRcMYEyjRUUJSXAyhdtY7VHpq1gDDRWSIiMQB84FlLmcyxpiIFBJHHKraLCJ3AG/iHY67SFU3uxzLGGMiUkgUDgBVXQ4sdzuHMcZEulA5VWWMMSZIWOEwxhjTJVY4jDHGdIkVDmOMMV1ihcMYY0yXhMzsuF0hIlVAids52tEPOOB2iNNk2d0RqtlDNTdEbvZBqprW2U5hWTiCmYgU+TJtcTCy7O4I1eyhmhsse2fsVJUxxpguscJhjDGmS6xwBN6jbgc4A5bdHaGaPVRzg2XvkPVxGGOM6RI74jDGGNMlVjgCRESyReQdEdkqIptF5E63M3WFiESLyIci8le3s3SFiKSKyAsiss35tz/H7Uy+EpFvOL8rm0TkWRFJcDtTe0RkkYjsF5FNrdr6iMgKESl2bnu7mbE97WT/pfM7s0FEXhKRVDcztqet7K2e+5aIqIj0a+u1Z8IKR+A0A3ep6ihgOnC7iIx2OVNX3AlsdTvEafgd8IaqjgQmECI/g4hkAl8DclV1LN7lBOa7m6pDTwIzT2q7G1ipqsOBlc7jYPQkp2ZfAYxV1fHAx8A9gQ7loyc5NTsikg1cCpT640OtcASIqlaq6jrnfi3e/8Ay3U3lGxHJAq4EHnc7S1eISC/gAuBPAKraqKpH3E3VJTFAoojEAEmctOplMFHV94BDJzXPAZ5y7j8FzA1oKB+1lV1V31LVZufhKryrjgaddv7dAX4DfJuTltjuLlY4XCAig4FJQKG7SXz2W7y/hB63g3TRUKAKeMI5zfa4iPRwO5QvVHUP8N94vzFWAtWq+pa7qbpsgKpWgveLE9Df5Tyn6yvA626H8JWIzAb2qOp6f32GFY4AE5GewFLg66pa43aezojILGC/qq51O8tpiAEmA4+o6iTgGMF7uuQznP6AOcAQIAPoISI3upsq8ojI9/CeZi5wO4svRCQJ+B7wA39+jhWOABKRWLxFo0BVX3Q7j4/OA2aLyG5gCXCxiDzjbiSflQPlqnriyO4FvIUkFHwB2KWqVaraBLwInOtypq7aJyLpAM7tfpfzdImILARmAfkaOtctDMP7ZWO98zebBawTkYHd+SFWOAJERATvufatqvprt/P4SlXvUdUsVR2Mt3P2bVUNiW++qroXKBORs52mS4AtLkbqilJguogkOb87lxAiHfutLAMWOvcXAq+4mKVLRGQm8B1gtqrWuZ3HV6q6UVX7q+pg52+2HJjs/C10GyscgXMecBPeb+wfOdsVboeKAP8FFIjIBmAi8FOX8/jEOUp6AVgHbMT7txq0VzOLyLPAP4GzRaRcRG4BHgQuFZFivCN8HnQzY3vayf4/QDKwwvlb/V9XQ7ajnez+/9zQOQIzxhgTDOyIwxhjTJdY4TDGGNMlVjiMMcZ0iRUOY4wxXWKFwxhjTJdY4TAmQERkcFuzmBoTaqxwGGOM6RIrHMa4QESGOhMvTnU7izFdZYXDmABzpkBZCnxZVde4nceYropxO4AxESYN75xN16jqZrfDGHM67IjDmMCqBsrwzl1mTEiyIw5jAqsR70p4b4rIUVVd7HYgY7rKCocxAaaqx5wFslaIyDFVDZnpxo0Bmx3XGGNMF1kfhzHGmC6xwmGMMaZLrHAYY4zpEiscxhhjusQKhzHGmC6xwmGMMaZLrHAYY4zpEiscxhhjuuT/AxGEqNIUrC6TAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# How does the number of possible combinations of the 22 features vary with k?\n",
    "# Lets do a quick plot to find out!\n",
    "\n",
    "from scipy.special import factorial\n",
    "\n",
    "n = 14\n",
    "k = np.arange(1, n+1)\n",
    "y = factorial(n) / (factorial(k)*factorial(n-k))\n",
    "\n",
    "ax = sns.lineplot(x=k, y=y)\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('combinations');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The plot above shows that there up to ~700,000 possible combinations of the predictors without repetition for some values of k when p = 22/\n",
    "\n",
    "The ISL authors provide comparison of the $R^2$ statistic for all combinations of 19 predictors. When p=19 the max number of combinations for any given subset is reduced by 10x, when p=10 the max combinations is reduced by ~2000x. \n",
    "\n",
    "My current machine maxed out its memory trying to compute all models for p=22. I'll try p=14 by picking 14 of the predictors for brevity. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progess: k = 1, done\n",
      "Progess: k = 2, done\n",
      "Progess: k = 3, done\n",
      "Progess: k = 4, done\n",
      "Progess: k = 5, done\n",
      "Progess: k = 6, done\n",
      "Progess: k = 7, done\n",
      "Progess: k = 8, done\n",
      "Progess: k = 9, done\n",
      "Progess: k = 10, done\n",
      "Progess: k = 11, done\n",
      "Progess: k = 12, done\n",
      "Progess: k = 13, done\n",
      "Progess: k = 14, done\n"
     ]
    }
   ],
   "source": [
    "# Pick arbitrary subset of predictors to save compute/memory\n",
    "X = hitters_pd.loc[:, 'AtBat':'PutOuts']\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "# get all model results\n",
    "model_subsets = []\n",
    "for k in range(len(X.columns)):\n",
    "    k=k+1\n",
    "    subset = get_models(k, X, y)\n",
    "    model_subsets += [subset]\n",
    "    print('Progess: k = {}, done'.format(k))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAERCAYAAACU1LsdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8XHWd//HXZ5I0SXNpJm2SlpYktNAbFUoyRaSu2xtSVwXW+wVEQPGCioqX3XWX/am/XV13VdytckdBERSkFHFRsZQ7FNIrvXIptKTQJm2TJr0kaZLP/jHTGELSJm1OTmbm/Xw85pHJzHdm3vBo+55zvud8j7k7IiIiAJGwA4iIyPChUhARkS4qBRER6aJSEBGRLioFERHpolIQEZEuSVkKZnaLmdWZ2bp+jP2xma1O3J43s8ahyCgikowsGc9TMLN3APuA29x9xgBe90XgDHe/NLBwIiJJLCm3FNz9UWBP98fMbJKZ/dHMVpjZY2Y2tZeXfhS4Y0hCiogkocywAwyiG4DPuvsLZvZW4GfAvMNPmlkFcBLwUEj5RESGvZQoBTPLB84G7jKzww9n9xj2EeBud+8YymwiIskkJUqB+G6wRnefeYQxHwGuGKI8IiJJKSnnFHpy9ybgZTP7IIDFnX74eTObAkSBp0KKKCKSFJKyFMzsDuL/wE8xs1ozuwz4OHCZma0B1gPnd3vJR4E7PRkPtRIRGUJJeUiqiIgEIym3FEREJBhJN9E8ZswYr6ysDDuGiEhSWbFixS53LznauKQrhcrKSmpqasKOISKSVMxsa3/GafeRiIh0USmIiEgXlYKIiHRRKYiISBeVgoiIdFEpiIhIF5WCiIh0SZtS2LyjmX/7wwYOtmnlbBGRvgRWCmaWY2bPmNkaM1tvZt/uY9yHzGxDYsyvg8pT23CAGx97mbW1ukSziEhfgjyjuRWY5+77zCwLeNzMHnD3pw8PMLNTgH8EZrt7g5mVBhWmqjwKQM3WBt46cXRQHyMiktQCK4XEMtX7Er9mJW49l2T9NPBTd29IvKYuqDzRvBFMKslj5daGoD5CRCTpBTqnYGYZZrYaqAMedPflPYZMBiab2RNm9rSZLQwyT6yimBXbGujs1HLhIiK9CbQU3L0jcYnMCcCZZjajx5BM4BRgDvEL4dxkZkU938fMLjezGjOrqa+vP+Y81RVRGg8cYsuufUcfLCKShobk6CN3bwQeBnpuCdQCS9z9kLu/DGwmXhI9X3+Du8fcPVZSctSVX/tUXRmfV1ihXUgiIr0K8uijksPf+s0sF1gAbOox7F5gbmLMGOK7k7YElWnimDyiI7OoeUWlICLSmyCPPhoH3GpmGcTL57fufr+ZfQeocff7gD8B7zSzDUAH8HV33x1UIDOjuiLKim0qBRGR3gR59NFa4IxeHr+6230Hvpq4DYmqiih/2VjHnv1tFOeNGKqPFRFJCmlzRvNhsYpiAB2aKiLSi7QrhdMmjCIrw6hRKYiIvEnalUJOVgannjBKWwoiIr1Iu1KA+PkKa2obaWvvDDuKiMiwkpalEKuI0treyfrX9oYdRURkWEnLUqiu0ElsIiK9SctSKC3M4cTiXJWCiEgPaVkKED80tWZrA/FTJUREBNK4FKoqotQ3t1LbcDDsKCIiw0balkKs4vBFd/aEnEREZPhI21KYXFZAQXam5hVERLpJ21LIiBgzy4u0YqqISDdpWwoQPzR1885mmlsOhR1FRGRYSOtSiFUU4w6rtjWGHUVEZFhI61KYWV5ExHQSm4jIYWldCvnZmUwdW6hSEBFJSOtSgPi8wqptDXR06iQ2EZG0L4VYZZT9bR1s2tEUdhQRkdClfSlUlcdPYtP1FUREVApMiOZSVpitK7GJiKBSwMyorojqJDYREVQKAFRXFLO98SA79raEHUVEJFQqBXTRHRGRw1QKwKknFJKTFVEpiEjaUykAWRkRTptQxAotoy0iaU6lkBCriLL+tSYOtnWEHUVEJDQqhYTqiijtnc6aWi2OJyLpS6WQoMlmEZEAS8HMcszsGTNbY2brzezbRxj7ATNzM4sFledoikaO4OTSfJWCiKS1ILcUWoF57n46MBNYaGZn9RxkZgXAl4DlAWbpl+ryKCu3NdCpxfFEJE0FVgoety/xa1bi1tu/tt8FfgCEfuZYdWWUxgOH2LJr39EHi4ikoEDnFMwsw8xWA3XAg+6+vMfzZwAnuvv9QeboL80riEi6C7QU3L3D3WcCE4AzzWzG4efMLAL8GLjqaO9jZpebWY2Z1dTX1weWd+KYPKIjs7QOkoikrSE5+sjdG4GHgYXdHi4AZgAPm9krwFnAfb1NNrv7De4ec/dYSUlJYDkPL463YptKQUTSU5BHH5WYWVHifi6wANh0+Hl33+vuY9y90t0rgaeB89y9JqhM/VFdUcyW+v3s2d8WZgwRkVAEuaUwDlhmZmuBZ4nPKdxvZt8xs/MC/NzjcnheQRfdEZF0lBnUG7v7WuCMXh6/uo/xc4LKMhCnTRhFVoZRs7WBBdPLwo4jIjKkdEZzDzlZGZx6wihtKYhIWlIp9CJWEWVNbSNt7Z1hRxERGVIqhV5UV0Rpbe9k/Wt7w44iIjKkVAq90ElsIpKuVAq9KC3M4cTiXJWCiKQdlUIfYhXF1GxtwF2L44lI+lAp9KGqIkp9cyu1DQfDjiIiMmRUCn2IJeYVanTdZhFJIyqFPkwuK6AgO1OL44lIWlEp9CEjYswsL9Jks4ikFZXCEVRXRNm8s5mmlkNhRxERGRIqhSOIVRTjDqu3NYYdRURkSKgUjmBmeRERgxrtQhKRNKFSOIL87Eymji3U4ngikjZUCkdRXRFl1bYG2ju0OJ6IpD6VwlHEKqPsb+tg887msKOIiAROpXAUWhxPRNKJSuEoxhflUlaYrVIQkbSgUjgKM4svjqczm0UkDagU+qGqIsr2xoPs2NsSdhQRkUCpFPohpnkFEUkTKoV+mH5CITlZEZWCiKQ8lUI/ZGVEOH1CESu0jLaIpDiVQj9VV0RZ/1oTB9s6wo4iIhIYlUI/xSqjtHc6a2q1OJ6IpC6VQj9VlWuyWURSn0qhn4pGjuDk0nyVgoikNJXCAFSXR1m5rYHOTg87iohIIAIrBTPLMbNnzGyNma03s2/3MuarZrbBzNaa2VIzqwgqz2CorozSeOAQW3btCzuKiEgggtxSaAXmufvpwExgoZmd1WPMKiDm7qcBdwM/CDDPcdPieCKS6gIrBY87/JU6K3HzHmOWufuBxK9PAxOCyjMYJo7JIzoyS+sgiUjKCnROwcwyzGw1UAc86O7LjzD8MuCBIPMcLzOjuiLKim0qBRFJTYGWgrt3uPtM4lsAZ5rZjN7GmdmFQAz4zz6ev9zMasyspr6+PrjA/VBdUcyW+v3s2d8Wag4RkSAMydFH7t4IPAws7PmcmS0AvgWc5+6tfbz+BnePuXuspKQk0KxHo3kFEUllQR59VGJmRYn7ucACYFOPMWcA1xMvhLqgsgym0yaMIivDVAoikpIyA3zvccCtZpZBvHx+6+73m9l3gBp3v4/47qJ84C4zA9jm7ucFmOm45WRlcOoJo7Q4noikpMBKwd3XAmf08vjV3e4vCOrzgxSriHLb01tpa+9kRKbO/xOR1KF/0Y5BdUWUtvZO1r22N+woIiKDSqVwDKor45PNKzWvICIpRqVwDEoLcigvHqmT2EQk5agUjtHhk9jctTieiKQOlcIxqq6IUt/cyqt7DoYdRURk0KgUjlHXSWzbdGiqiKQOlcIxmlxWQEF2puYVRCSlqBSOUUbEmFlepDObRSSlqBSOQ6yimM07m2lqORR2FBGRQaFSOA7VFVHcYfW2xrCjiIgMCpXCcZhZXkTEoEa7kEQkRQy4FMwsYmaFQYRJNvnZmUwdW6gzm0UkZfSrFMzs12ZWaGZ5wAZgs5l9PdhoySFWGWXVtgbaOzrDjiIictz6u6Uw3d2bgAuA/wXKgYsCS5VEqiui7G/rYPPO5rCjiIgct/6WQpaZZREvhSXufgjQ+g7oSmwiklr6WwrXA68AecCjZlYBNAUVKpmML8qlrDBbpSAiKaFfF9lx9/8G/rvbQ1vNbG4wkZKLmRGrKNaZzSKSEvo70XxlYqLZzOxmM1sJzAs4W9KoqoiyvfEgO/a2hB1FROS49Hf30aWJieZ3AiXAJcD3A0uVZGKaVxCRFNHfUrDEz78Dfu7ua7o9lvamn1BIblYGd614lY5Ozb+LSPI6aimYmQG7zOxPxEvhT2ZWAOjA/ISsjAjfWDiFhzfX8y9L1unCOyKStI460ezubmZFwKeALe5+wMxGE9+FJAmXzD6JnU2tXPfIS5QWZPPlBZPDjiQiMmD9OvoIeArIcPdGAHffDewOLFWS+ubCKdQ3t3LNX16gpCCbj7+1IuxIIiID0t9SmAt8xsy2AvuJzye4u58WWLIkZGZ8//1voeFAG/9y7zpG541g4YxxYccSEem3/k40vwuYRPww1PcC70n8lB6yMiL89GNVnH5iEV+6czVPb9EGlYgkj36Vgrtv7e0WdLhklTsig1sunsWJ0Vw+fVsNG1/Xyd8ikhx0PYWARPNGcNtlbyVvRCYX3/IMr+45EHYkEZGjUikEaHxRLrdddiYthzq4+JZn2L2vNexIIiJHFFgpmFmOmT1jZmvMbL2ZfbuXMdlm9hsze9HMlptZZVB5wjK5rIBbPjmL7Y0HufTWGva3tocdSUSkT0FuKbQC89z9dGAmsNDMzuox5jKgwd1PBn4M/EeAeUITqyxm0ceqeK62kc/dvpJDuiCPiAxTgZWCx+1L/JqVuPU81fd84NbE/buB+YkzqFPOOdPL+N773sKjz9fzjbvX0qnlMERkGAp0TsHMMsxsNVAHPOjuy3sMGQ+8CuDu7cBeYHSQmcL04VnlfO2dk1m8ajvfe2Bj2HFERN6kvyevHRN37wBmJpbJWGxmM9x9XbchvW0VvOkrtJldDlwOUF5eHkjWoXLF3JOpb27lxsdepqQgm8vfMSnsSCIiXYbk6KPE8hgPAwt7PFULnAhgZpnAKGBPL6+/wd1j7h4rKSkJOG2wzIyr33sq7z5tHP/+v5u4Z2Vt2JFERLoEefRRSWILATPLBRYAm3oMuw+4OHH/A8BDngZLjGZEjB996HTOnjSab9y9loc314UdSUQECHZLYRywzMzWAs8Sn1O438y+Y2bnJcbcDIw2sxeBrwL/EGCeYSU7M4PrL6pmytgCPverlazapgv0iEj4LNm+mMdiMa+pqQk7xqCpa27hA9c+RXPLIe7+3NlMKskPO5KIpCAzW+HusaON0xnNISstyOG2S88kI2J84uZn2Nmk6zyLSHhUCsNA5Zg8fnHJmTQeaOPiW55h78FDYUcSkTSlUhgmZowfxfUXxXipfh+fvrWGlkMdYUcSkTSkUhhG3n7KGH70oZk8u3UPX7pjFR0661lEhphKYZh57+kn8K/vmc6fN+zkn+9dR7IdCCAiyS3QM5rl2Hxy9knUNbfys4dforQgm6+cMznsSCKSJlQKw9TXz53Crn2t/GTpC4wpyOaisyrCjiQiaUClMEyZGf/+929h9742rl6yjjF5I3jXW8aFHUtEUpzmFIaxzIwIiz5WRVV5lC/esYrbnnpFcwwiEiiVwjCXOyKDn18yi7+dXMLVS9bzjbvX6nBVEQmMSiEJFOZkceMnYnxp/inctaKWD1//FK81Hgw7loikIJVCkohEjK+eM5nrL6rmpfr9nLfocZZv2R12LBFJMSqFJHPuqWO594qzKczN4uM3LefWJzXPICKDR6WQhE4uLeDeK2YzZ0oJ/3rfer52l+YZRGRwqBSSVGFOFjdcFOPLC07hdytr+eB1mmcQkeOnUkhikYjx5QWTufETMV7etZ/3/s/jPPWS5hlE5NipFFLAOdPLuPeK2RSNzOLCm5dzy+Mva55BRI6JSiFFnFyaz71XzGbe1FK+c/8GrvrtGs0ziMiAqRRSSEFOFtdfWM1XFkzmnlXb+cB1T1LbcCDsWCKSRFQKKSYSMa5ccAo3Xxxj664DnLfoCZ58aVfYsUQkSagUUtT8aWUs+cJsivNGcNHNz3Cz5hlEpB9UCilsYkk+iz9/NvOnlvLd+zfwld+s5mCb5hlEpG8qhRRXkJPFdRdWc9U5k1my5jXef+2TvLpH8wwi0juVQhqIRIwvzo/PM7zacIDzFj3OEy9qnkFE3kylkEbmTS3jvi+8nTH52Vx083JuemyL5hlE5A1UCmnmpDF5LL5iNu+cPpb//4eNXHmn5hlE5K9UCmkoPzuTay+s4uvnTuH3a1/jfZpnEJEElUKaMjOumHsyt3xyFrUNB/j7nz3Biq0NYccSkZCpFNLc3CmlLP78bPKyM/nojU+zZPX2sCOJSIgCKwUzO9HMlpnZRjNbb2ZX9jJmlJn93szWJMZcElQe6dvJpfks/vxsZk4o4so7V3PNX57XBLRImgpyS6EduMrdpwFnAVeY2fQeY64ANrj76cAc4IdmNiLATNKH4rwR/PJTZ/L+qglc85cXuPLO1VpQTyQNZQb1xu7+OvB64n6zmW0ExgMbug8DCszMgHxgD/EykRBkZ2bwXx88jUmlefzgj5t5teEAN1wUo6QgO+xoIjJEhmROwcwqgTOA5T2eWgRMA14DngOudPfOXl5/uZnVmFlNfX19wGnTm5nx+Tknc92FVWx8vYkLfvoEm3c0hx1LRIZI4KVgZvnA74Avu3tTj6fPBVYDJwAzgUVmVtjzPdz9BnePuXuspKQk6MgCLJwxjrs+czaHOjp5/7VPsmxTXdiRRGQIBFoKZpZFvBBud/d7ehlyCXCPx70IvAxMDTKT9N9bJoxiyRdmUzF6JJfd+iw/f0IrrYqkuiCPPjLgZmCju/+oj2HbgPmJ8WXAFGBLUJlk4MaNyuW3n3kb86eV8e3fb+DqJetp73jTHj4RSRGBTTQDs4GLgOfMbHXisX8CygHc/Trgu8AvzOw5wIBvurtWahtm8rIzue7Can7wx01c/+gWXtm9n59+vIrCnKywo4nIILNk2x0Qi8W8pqYm7Bhp6zfPbuNbi9dx0pg8br54FuWjR4YdSUT6wcxWuHvsaON0RrMMyIdnlXPbZWdS19zKBT97gppX9oQdSUQGkUpBBuzsSWNY/PmzGZWbxcduXM7iVbVhRxKRQaJSkGNy+FKfVRVFfOU3a/jhnzfT2ZlcuyJF5M1UCnLMikaO4LZL38qHYhP4n4de5It3rtLSGCJJLsijjyQNjMiM8B/vP42TS/P53gObqG04yI2fqKa0ICfsaCJyDLSlIMfNzLj8HZO47sJqnt/RzAWLnmDj6z1PXheRZKBSkEFz7qljueuzb6PT4QPXPslDm3aGHUlEBkilIINqxvj40hgTS/L51K013Py4lsYQSSYqBRl0ZYU5/OYzZ/HO6WP57v0b+MwvV3DfmtdoPNAWdjQROQpNNEsgRo7I5Gcfr+InS1/gtqde4c8bdhIxqK6IMmdKKXOnlDJtXAHxJbJEZLjQMhcSuI5OZ01tI8s21bFscx3rtscnoccW5jBnSglzp5Yy++Qx5GfrO4pIUPq7zIVKQYZcXVMLDz9fz7JNdTz2wi72tbaTlWGceVIxc6eUMndqKRPH5GkrQmQQqRQkKRzq6KTmlQYe3hzfinh+5z4AyotHMndKCXOmlvK2iaPJycoIOalIclMpSFKqbTjAss31PLypjide2kXLoU5ysiKcPWlMvCSmlHJisVZmFRkolYIkvZZDHSx/eU/XXMTW3QcAOKU0n7lTS5kzpYRYRTEjMnUQncjRqBQkpbg7L+/aH9+K2FzH8i17aOvopCAnkyvnn8LFZ1eSlaFyEOmLSkFS2v7Wdp58aTe/Xr6VZZvrmTq2gO9eMINZlcVhRxMZlnSRHUlpedmZnDO9jFs+OYsbLqqmuaWdD173FF+7aw2797WGHU8kaakUJKmZGe88dSwPfvUdfG7OJO5dtZ15P3yE25dvpUPXdxAZMJWCpISRIzL55sKpPHDl3zBtXAHfWryO9137JM/V7g07mkhSUSlISjmlrIA7Pn0WP/nITLY3HOT8nz7O1UvWsffgobCjiSQFlYKkHDPj/JnjWXrV3/KJt1Xyq6e3Mv+HD7N4Va1WbBU5CpWCpKxRuVn8v/NO5b4vvJ3x0ZF85Tdr+MgNT/PCzuawo4kMWyoFSXkzxo9i8efO5nvvewubdjTzrp88xvcf2MSBtvawo4kMOyoFSQuRiPHRM8t56Kq/5X1V47nukZdY8MNH+OO6HdqlJNKNSkHSyuj8bH7wgdO5+7NvozA3i8/+agWX/uJZtiWW0BBJdyoFSUuxymLu/+Lb+ed3T+OZl/dwzo8f4b+XvkDLoY6wo4mESqUgaSszI8Kn/mYiS6+awznTy/jRg8+z8JpHefT5+rCjiYQmsFIwsxPNbJmZbTSz9WZ2ZR/j5pjZ6sSYR4LKI9KXsaNyWPSxKn552ZmYGZ+45RmuuH0lO/a2hB1NZMgFtiCemY0Dxrn7SjMrAFYAF7j7hm5jioAngYXuvs3MSt297kjvqwXxJEit7R3c8MgWFi17kcyI8bG3lrNgWhnVFVEytQqrJLFht0qqmS0BFrn7g90e+zxwgrv/c3/fR6UgQ+HVPQf4tz9sZOmmnRzqcEblZjFnSgnzppYyZ3Ipo0ZmhR1RZECGVSmYWSXwKDDD3Zu6PX4NkAWcChQAP3H323p5/eXA5QDl5eXVW7duDTyzCEBzyyEef2EXSzfVsWxTHbv3t5ERMaoroiyYVsq8qWVMKtH1pGX4GzalYGb5wCPAv7n7PT2eWwTEgPlALvAU8G53f76v99OWgoSlo9NZU9vIQxvr+MvGnWzaET8zumL0SOZPLWP+tFJmVepKcDI89bcUMgMOkQX8Dri9ZyEk1AK73H0/sN/MHgVOB/osBZGwZESMqvIoVeVRvnbuFLY3HuShTXUs3biTXy3fyi1PvExBdibvmJzYzTSlhNH52WHHFhmQICeaDbgV2OPuX+5jzDRgEXAuMAJ4BviIu6/r6321pSDD0YG2dp54cTdLN+5k6aY66ptbMYOq8ijzppYyf1opU8oKtJtJQhP67iMzezvwGPAc0Jl4+J+AcgB3vy4x7uvAJYkxN7n7NUd6X5WCDHednc661/aydGMdD22q47nt8Ws6jC/KZf60UuZNLeWsiaPJycoIOamkk9BLISgqBUk2O/a2sGxzfDfT4y/uouVQJyNHZDCrspgJ0VzKCnMoK8ymtDCHsoIcSguzKR45gkhEWxUyeFQKIsNQy6EOnnppN0s37aTmlQbqmlvZs7/tTeMyI0ZpQaIoCrMTxZFDacFf75cVZjMqN0u7pKRfhsVEs4i8UU5WBnOnljJ3amnXY63tHdQ3t7KzqZW6phZ2NrWws7mVnU0t1DW1sqV+P0+9tJumljcv9T0iMxIvjYJEaSQKpLQgm9ysDCIRI8OMjMhfbxEzMjPiPzMiRmbkr/czIpARiZBhRiQCmZEIkQi9vocZGEbE4hc2OvxTkptKQSRk2ZkZTIiOZEJ05BHHHWzroK65hZ1NicJo7lYiTa1s3NHEI8+3sq81/OtEHC4IAyLx9ui6b5b4CWB0K5i/3o//Boc75nDVdO8c63PMG4up63l74+ve/H7dX2O9Ps5Axw+yD886kU/9zcQAP0GlIJI0ckdkUDE6j4rReUcct6+1nbqmFlrbO+nodDrd6ejsduv2e/w56OjsjP9077rfmRjb3unx+z1eD+DuuEOng+N0OuCOA52J5w7f5/D9zjc+zxvGJt4j/jYJ3uP3v973Hs95H8/3+NGV/Y2f0PNz+zeeN4wPdnf8mCE4xFmlIJJi8rMzyS/JDzuGJCmdeikiIl1UCiIi0kWlICIiXVQKIiLSRaUgIiJdVAoiItJFpSAiIl1UCiIi0iXpFsQzs3pguF6PcwywK+wQxyhZsydrblD2sKRr9gp3LznaoKQrheHMzGr6swrhcJSs2ZM1Nyh7WJT9yLT7SEREuqgURESki0phcN0QdoDjkKzZkzU3KHtYlP0INKcgIiJdtKUgIiJdVAoiItJFpXCczOxEM1tmZhvNbL2ZXRl2poEyswwzW2Vm94edZSDMrMjM7jazTYn//28LO1N/mdlXEn9e1pnZHWaWE3amvpjZLWZWZ2bruj1WbGYPmtkLiZ/RMDP2pY/s/5n4M7PWzBabWVGYGfvSW/Zuz33NzNzMxgz256oUjl87cJW7TwPOAq4ws+khZxqoK4GNYYc4Bj8B/ujuU4HTSZL/BjMbD3wJiLn7DCAD+Ei4qY7oF8DCHo/9A7DU3U8BliZ+H45+wZuzPwjMcPfTgOeBfxzqUP30C96cHTM7ETgH2BbEh6oUjpO7v+7uKxP3m4n/wzQ+3FT9Z2YTgHcDN4WdZSDMrBB4B3AzgLu3uXtjuKkGJBPINbNMYCTwWsh5+uTujwJ7ejx8PnBr4v6twAVDGqqfesvu7n929/bEr08DE4Y8WD/08f8d4MfANyCYC0KrFAaRmVUCZwDLw00yINcQ/wPWGXaQAZoI1AM/T+z6usnMjnxF+2HC3bcD/0X8m97rwF53/3O4qQaszN1fh/gXI6A05DzH6lLggbBD9JeZnQdsd/c1QX2GSmGQmFk+8Dvgy+7eFHae/jCz9wB17r4i7CzHIBOoAq519zOA/QzfXRhvkNj/fj5wEnACkGdmF4abKv2Y2beI7/69Pews/WFmI4FvAVcH+TkqhUFgZlnEC+F2d78n7DwDMBs4z8xeAe4E5pnZr8KN1G+1QK27H94qu5t4SSSDBcDL7l7v7oeAe4CzQ840UDvNbBxA4mddyHkGxMwuBt4DfNyT52StScS/SKxJ/J2dAKw0s7GD+SEqheNkZkZ8v/ZGd/9R2HkGwt3/0d0nuHsl8YnOh9w9Kb6xuvsO4FUzm5J4aD6wIcRIA7ENOMvMRib+/MwnSSbJu7kPuDhx/2JgSYhZBsTMFgLfBM5z9wNh5+kvd3/O3UvdvTLxd7YWqEr8XRg0KoXjNxu4iPi37NWJ29+FHSpNfBG43czWAjOBfw85T78ktm7uBlYCzxH/ezhsl14wszuAp4ApZlZrZpcB3wfOMbMXiB8J8/0/LJLHAAAA8ElEQVQwM/alj+yLgALgwcTf1+tCDdmHPrIH/7nJs+UkIiJB05aCiIh0USmIiEgXlYKIiHRRKYiISBeVgoiIdFEpiAwCM6vsbTVLkWSjUhARkS4qBZFBZmYTE4v0zQo7i8hAqRREBlFi2Y3fAZe4+7Nh5xEZqMywA4ikkBLiawC9393Xhx1G5FhoS0Fk8OwFXiW+HpZIUtKWgsjgaSN+BbI/mdk+d/912IFEBkqlIDKI3H1/4uJFD5rZfndPmiWlRUCrpIqISDeaUxARkS4qBRER6aJSEBGRLioFERHpolIQEZEuKgUREemiUhARkS7/B0TXOW2dlRiQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XXWd//HXJzdLmzRNl6R0SboArVAKtBAKLvTnKGiVdUZ0wA6LMFMRKo7L7yeo4AjMjKKjM2rVHyirLOooQwdwKs5PHJGtaZNS2lrpftNASZvcNkub9fP7456U25I0S+899+bm/Xw87uPe8z1LPgea+84553u+x9wdERGRZMhJdwEiIpI9FCoiIpI0ChUREUkahYqIiCSNQkVERJJGoSIiIkmjUBERkaRRqIiISNIoVEREJGly011AOpSWlvrMmTPTXYaIyLCxevXqPe5e1t9yIzJUZs6cSVVVVbrLEBEZNsxsx0CW0+kvERFJGoWKiIgkjUJFRESSRqEiIiJJo1AREZGkUaiIiEjSKFRERCRpRuR9KiLZprOrm44up72rm47g1d7Z8+6HtbUHy/a0tQXLdXQevo28SA7jC/MZX5jHuMJ8xhflMaEwn3GF+eTn6u9R6Z1CRWSY6Ojq5tVd+3h5WwMvb2ugJhqjua2Tjq5uuj3cWoryI4wrzGdCUT7jCvMOhc/4onzGF8bbJiR8Hl+YT2F+BDMLt1AJXaihYmaLgX8DIsCP3f3rR8y/BvgmsCto+r67/ziYdzXwlaD9Tnd/IGg/E7gfGA08DXzG3UP+FRNJvoMdXayNxuIhsr2B1TsaaW3vAuD4siLef/Ikxhflkx/JIT+SQ15uDnmRHPIjRl4k/jkvNz4vPzeh7dDyFn+P5JCf2zPvreU6urppbG2nsaWDWGs7Da3tNLZ2EGuJvze2tgevDnY2tNLQ0k7Twc4+9yc/NycePIXxsJkwJp/SonwmFBUwcUw+E4vymTimgAlF+ZSOyWfsqDxychRCw01ooWJmEWA5cD5QC6wysxXuvuGIRX/m7suOWHcC8FWgEnBgdbBuI/BDYCnwIvFQWQz8OqU7I5ICzW2drNnReNiRSHtXN2Zw0uSxfKyygoWzJnDWzAmUFRekvJ5IToQpJaOZUjJ6wOt0dnUTOxCEUEs8eGJB8DS2vBVCjS3tbKzbz57mNvb3EUS5Ocb4op6wyWdi0VuBkxg+PaFUXJCrI6EMEOaRykJgs7tvBTCzx4BLgCNDpTcfBJ5x94Zg3WeAxWb2LDDW3V8I2h8ELkWhIsPAvtYOVm1v4KVte3l5WwOv1u2nq9uJ5BjzppVwzbtnsnDmBCpnjmdcYX66yx2Q3EgOpWMKKB0z8NBr74wfEe1tbmdvSxsNLe3saW6noaUtaGtnb3MbrzTG2NvcTlNb7yGUH8lhQlH8lFxpcQGlPe9j8ikdU8DEMfHPZUEg5UZ0XSgVwgyVaUA0YboWOLuX5T5iZouAPwOfdfdoH+tOC161vbSLZJw3mw6yalsjL2/by0vbGti0uwn3+Gmh+RXjuOG9J7Bw1gTOmD6eooKRc7kzPzeH48aO4rixowa0fFtnFw0t7YcFzpFBtKelnS1vNrOnuY22zu5etzO+MO9QACaGz1vv8SOg0jEFjMqLJHOXs1qY/3J7Oy498trHfwKPunubmV0PPAC87yjrDmSb8R9utpT4aTKmT58+0JpFhmzfgQ7+e+PuQ6eztu5pAaAwP8KZM8Zz4WlTWDhrIqeVl+hLaxAKcgd+Ws7daW7rjAdNcxt7mtuob44H0Z7mNvY0xdvX1R79KKi4IJfS4gImFgWBU/xW8JQVB+9Be2H+yPmDoDdh7n0tUJEwXQ7UJS7g7nsTJu8BvpGw7nuPWPfZoL38aNtM2PbdwN0AlZWVupAvKdPV7fxsVZRv/WYTDS3tjB2Vy8JZE7h8YQULZ03klKljydOpl1CYGcWj8igelcfM0qJ+lz/Y0RWET0LwNLdT39TG3pZ29jS1saW+mZe2tdHY2tHrNgrzI4cf8RT3BE4BZQlHQaXFBRRlYY+4MENlFTDbzGYR7911OfDxxAXMbIq7vx5MXgxsDD6vBP7JzMYH0x8AbnH3BjNrMrNzgJeAq4DvpXg/RPq0ansD/7BiPevr9rNw5gS+eNVJLKgYp15Mw8SovAjl4wspH1/Y77IdXd2HjoDqm9vY0xQPoENHRE1tbN/bQtWORhpb2+mtT+qovLeuQfUc9ZSNyY+/Fye0FRcMmyOg0Kp0904zW0Y8ICLAve6+3sxuB6rcfQVwk5ldDHQCDcA1wboNZnYH8WACuL3noj3wKd7qUvxrdJFe0uD1fQf456f/xIq1dUwpGcX3rljAhadNybq/QuUteZEcJpeMYnJJ/9eCOru6aWhppz4Im0Ph0/TW0VBtYyvVOxtp6COACvMjQei8PXAO/5xPQW76TqfaSLylo7Ky0vXkR0mGgx1d/PgPW1n+uy10uXP9ouO5/r0nDJu/KiXzHBlAPSFU3/TWEVB9cDQU6+MU3NhRub0Gz/QJhVx0+tQh1WVmq929sr/l9C9fZAjcnd9s2M2dT20g2nCAxadM5ssXnEzFhP5Pm4gcTW4kh0ljRzFpAL3h2jq73joFdyiADg+i9XX72dPURlNbJydNLh5yqAy4/pRuXSQLvba7ia/95wae27yHOceN4eG/PZt3n1ia7rJkBCrIjTB13Gimjuu/J9yB9i6a++jdlkwKFZEB2negg3/97Z958IUdFOVH+IeL5vI358zQTXQyLIzOjzA6P/XXWhQqIv3o6nZ+XhXlmys30djazhULp/P58+cwcRB3jYuMFAoVkaOo2t7AV4MuwmfNHM9XL1rIvGkl6S5LJGMpVER68ca+g/zzrzfyRE0dk8eO4rtXLOAidREW6ZdCRSTBwY4ufvLcNpb/bjOd3c6n33cin1IXYZEB02+KCPEuws9s2M2dT21kZ0MrHzzlOL784blMn6guwiKDoVCREW/zm/Euwn94bQ+zJ43hp9edzXtmq4uwyFAoVGRE2r3/IKt3NPKH1+r5RVUto/Mj3HbhXK585wwN9ihyDBQqkvW6up0/vbGfNTsaqdrRyOodjdQ2HgCgIDeHj1aW84UPvENdhEWSQKEiWWf/wQ5qdsao2tHImh2NVO9spCV4tvuk4gIqZ47nmnfNpHLmBOZOGUt+ro5MRJJFoSLDmrsTbTjA6p0NVG2PH4X0PFExJ3i2+1+dUc6ZM8Zz5ozxlI8frW7BIimkUJFhpa2zi/V1wams7Y2s3tlIfVMbAGMKclkwfRyL502mcsYETq8ooXhUXporFhlZFCqS0Tq6uvn9pnpW7WhgzY5G1tbuoz145vj0CYW858TSQ0chc44rJqKHYYmklUJFMlastZ0bHl7D81v2khcx5k0r4apzZlA5czxnTB8/oKHBRSRcChXJSFvqm7nu/lXUxQ7y9b86lUsXTGNUXvqeZiciA6NQkYzzh9fqueHhNeRHcnh06dmcOWNCuksSkQEKtS+lmS02s01mttnMbj7KcpeZmZtZZTC9xMxqEl7dZjY/mPdssM2eeZPC2h9Jvgdf2M41961i2rjRPLHs3QoUkWEmtCMVM4sAy4HzgVpglZmtcPcNRyxXDNwEvNTT5u4PAw8H808FnnD3moTVlri7Hjo/jHV2dfO1/9zAQy/u4LyTJ/Gvly9gTIEOpEWGmzCPVBYCm919q7u3A48Bl/Sy3B3AXcDBPrZzBfBoakqUdNjX2sE1963ioRd38MlFx/N/r6xUoIgMU2GGyjQgmjBdG7QdYmYLgAp3f/Io2/lr3h4q9wWnvm61Pu5sM7OlZlZlZlX19fVDKF9SYdueFv7yB3/kpW17ueuy07jlwyerW7DIMBZmqPT2TeGHZprlAN8BPt/nBszOBlrd/dWE5iXufipwbvC6srd13f1ud69098qysrKh1C9J9vzmPVy6/I/EDnTw8N+ew8cqK9JdkogcozBDpRZI/NYoB+oSpouBecCzZrYdOAdY0XOxPnA5RxyluPuu4L0JeIT4aTbJcA+/tIOr7n2Z48YW8MSN72bhLF2QF8kGYZ64XgXMNrNZwC7iAfHxnpnuvg849BALM3sW+ELPBfjgSOajwKKEZXKBce6+x8zygAuB36Z+V2SoOru6ufOpjdz//Hb+4h1lfPeKBRpKRSSLhBYq7t5pZsuAlUAEuNfd15vZ7UCVu6/oZxOLgFp335rQVgCsDAIlQjxQ7klB+ZIE+w92sOyRav7nz/Vc955ZfEnXT0Syjrl7/0tlmcrKSq+qUg/kMO3Y28J1D1SxfU8Ld146j8sXTk93SSIyCGa22t0r+1tO/TYl5V7cupfrf7oagIeuO5t3njAxzRWJSKooVCSlfrZqJ19+/FVmTCzkJ1efxczSonSXJCIppFCRlOjqdv756Y38+LltLJpTxvc/voCxuiAvkvUUKpJ0TQc7uOnRan63qZ5r3jWTr1xwMrkRPbJXZCRQqEhSRRtaue6BVWytb+Ef/3IeS86eke6SRCREChVJmlXbG/jkQ6vp6nYevHYh7zqxtP+VRCSrKFQkKX5RFeVLj6+jYnwhP7nmLGbpgrzIiKRQkWPS2dXNXSs3cff/bOU9J5ay/ONnUFKoC/IiI5VCRYasvqmNmx6t5oWte7nqnTO49cK55OmCvMiIplCRIVm9o5EbHl7NvgMdfPtjp/NXZ5SnuyQRyQAKFRkUd+fBF3Zw51MbmDpuNPdds5C5U8emuywRyRAKFRmw1vZObvnVOp6oqeO8kyfxLx+bT8loXT8RkbcoVGRAttY386mfruG1N5v43x98B5/6XyeQoxGGReQIChXp13+9+gb/+xdrycvN4cFrz+Y9s3X/iYj0TqEifers6uZbv/kzP/r9Fk4vL+EHf3Mm08aNTndZIpLBFCrSqz3NbXz6kXh34SVnT+e2i+ZSkBtJd1kikuEUKvI2q3c0cuPDa2hsbedfPno6HzlT3YVFZGBCvVPNzBab2SYz22xmNx9lucvMzM2sMpieaWYHzKwmeP0oYdkzzWxdsM3vmpmuHg+Ru/PA89u5/O4XyM/N4Vc3vEuBIiKDEtqRiplFgOXA+UAtsMrMVrj7hiOWKwZuAl46YhNb3H1+L5v+IbAUeBF4GlgM/DrJ5We91vZOvvSrdfxHTR3vP2kS3/7YfA23IiKDFuaRykJgs7tvdfd24DHgkl6WuwO4CzjY3wbNbAow1t1fcHcHHgQuTWLNI8K2PS385fLneWJtHZ8/fw73XFWpQBGRIQkzVKYB0YTp2qDtEDNbAFS4+5O9rD/LzKrN7Pdmdm7CNmuPtk05upXr3+Di7z3Hm00HeeATC/n0+2fr/hMRGbIwL9T39k3lh2aa5QDfAa7pZbnXgenuvtfMzgT+w8xO6W+bh/1ws6XET5Mxffr0wVWehRK7C59WXsIPlpxB+fjCdJclIsNcmKFSC1QkTJcDdQnTxcA84NngWvtkYIWZXezuVUAbgLuvNrMtwJxgm+VH2eYh7n43cDdAZWVlr8EzUuxpjo8u/PyWvXz87Ol8Vd2FRSRJwgyVVcBsM5sF7AIuBz7eM9Pd9wGHbtU2s2eBL7h7lZmVAQ3u3mVmxwOzga3u3mBmTWZ2DvEL+1cB3wttj4ahNTsbueGn8e7C37zsND5aWdH/SiIiAxRaqLh7p5ktA1YCEeBed19vZrcDVe6+4iirLwJuN7NOoAu43t0bgnmfAu4HRhPv9aWeX71wdx56cQd3PLmBySWj+NUN7+KUqSXpLktEsozFO02NLJWVlV5VVZXuMkLT1tnFzb9cx+PVu3jfSZP4jroLi8ggmdlqd6/sbzndUZ/lurudz/18LU+98jqfO38Oy/7iRPXuEpGUUahkMXfn9ic38NQrr/OlD5/E0kUnpLskEclyeqB4FvvBs1u4//nt/N25sxQoIhIKhUqW+vmqKN9cuYlL50/llg+dnO5yRGSEUKhkod9u2M0tj69j0Zwy7rrsdF1DEZHQKFSyzOodDdz4yBrmTR3LD5ecQX6u/heLSHj0jZNFXtvdxLX3VzF13GjuveYsigrUD0NEwqVQyRJ1sQNcde/L5Ofm8OC1C5k4piDdJYnICKRQyQKx1nauvvdlmg928sAnFlIxQQNDikh66PzIMHegvYvrHqhix95WHrh2IXOnjk13SSIygilUhrHOrm4+/ega1uxs5AcfP4N3njAx3SWJyAin01/DlLvzpcfX8duNb3L7JfP40KlT0l2SiIhCZbj6l9/8mZ9X1XLT+2dz5Tkz0l2OiAigUBmW7v/jNr7/u81csXA6nz1vdrrLERE5RKEyzDz5Sh1fe3IDH5h7HHdccgrBUzJFRDKCQmUYeX7zHj73s7WcNWMC371iAbkR/e8Tkcyib6Vh4tVd+1j60GpmlRZxz1WVjMrTM+VFJPMoVIaBnXtbuea+VZSMzuOBaxfqqY0ikrFCDRUzW2xmm8xss5ndfJTlLjMzN7PKYPp8M1ttZuuC9/clLPtssM2a4DUpjH0JS31TG1fe+xKd3d08cO1CJpeMSndJIiJ9Cu3mRzOLAMuB84FaYJWZrXD3DUcsVwzcBLyU0LwHuMjd68xsHrASmJYwf4m7Z91D55vbOvnE/S+ze/9BHvm7czhx0ph0lyQiclRhHqksBDa7+1Z3bwceAy7pZbk7gLuAgz0N7l7t7nXB5HpglJll9YiJ7Z3dXP/Qaja+3sQPl5zJGdPHp7skEZF+hRkq04BownQthx9tYGYLgAp3f/Io2/kIUO3ubQlt9wWnvm61LOhj293tfOEXa3lu8x6+8ZHT+IuTsuqMnohksTBDpbcvez800ywH+A7w+T43YHYK8A3gkwnNS9z9VODc4HVlH+suNbMqM6uqr68fQvnhcHfueGoDK9bWcfOHTuKyM8vTXZKIyICFGSq1QEXCdDlQlzBdDMwDnjWz7cA5wIqEi/XlwOPAVe6+pWcld98VvDcBjxA/zfY27n63u1e6e2VZWVnSdirZfvT7rdz3x+1c++5ZfHLR8ekuR0RkUMIMlVXAbDObZWb5wOXAip6Z7r7P3Uvdfaa7zwReBC529yozGwc8Bdzi7n/sWcfMcs2sNPicB1wIvBreLiXXL6qifOO//sTFp0/lKxecrLvlRWTYCS1U3L0TWEa859ZG4Ofuvt7Mbjezi/tZfRlwInDrEV2HC4CVZvYKUAPsAu5J3V6kzh9eq+fmX63j3NmlfOujp5OTo0ARkeHH3L3/pbJMZWWlV1VlVg/ka+57mdd2N7Pys4sYo2fLi0iGMbPV7l7Z33K6oz4DuDtrozHec2KpAkVEhjWFSgbYsbeVxtYO5k8fl+5SRESOiUIlA9REYwDMr1CoiMjwplDJADXRGIX5EeYcV5zuUkREjolCJQNUR2OcOq2EiHp8icgw12+oBCME32Nm84Pppakva+Ro6+xiY91+XU8RkawwkK5GNwCfAL5iZhOA+aktaWTZULef9q5uFuh6iohkgYGc/qp395i7fwH4AHBWimsaUap39lyk1yjEIjL8DSRUnur54O43Aw+mrpyRpyYaY0rJKD18S0SyQr+h4u5PHDH9vdSVM/LURGPqSiwiWWNAvb/M7EozqzezWjO7Kmg7x8zuNLPVqS0xe+1tbmNnQ6tCRUSyxkC7FN8GfJj4RfrjzewZ4BdAPvD3Kaot662t1U2PIpJdBjrQVLO7rwIws68Bu4E57h5LWWUjQM3OGJEc49TyknSXIiKSFAMNlcnB/SmbgletAuXYVUdjzDmumMJ8DSIpItlhoN9mXwVOA5YApwLFZvZboJr48+IfSVF9Wau7Oz4y8QWnTU13KSIiSTOgUHH3uxOng0f7nkY8YD5E/DG+Mghb97Sw/2CnbnoUkawypPMu7l5L/JnzTye3nJHj0MjEGp5FRLKIBpRMk5poI8UFuZxYNibdpYiIJE2ooWJmi81sk5ltNrObj7LcZWbmZlaZ0HZLsN4mM/vgYLeZaWqiMU6rKNGz6EUkq4QWKmYWAZYTvwYzF7jCzOb2slwxcBPwUkLbXOBy4BRgMfADM4sMdJuZ5mBHF396vUn3p4hI1gnzSGUhsNndt7p7O/AYcEkvy90B3AUcTGi7BHjM3dvcfRuwOdjeQLeZUV7dtY/ObtcgkiKSdcIMlWlANGG6Nmg7xMwWABXu/uQA1+13mwnbXmpmVWZWVV9fP7Q9SBI9PlhEslWYodLbxQM/NNMsB/gO8PlBrHvUbR7W6H63u1e6e2VZWdkAyk2d6miMaeNGU1ZckNY6RESSLcxbuWuBioTpcqAuYboYmAc8a2YAk4EVZnZxP+sebZsZqWZnTF2JRSQrhXmksgqYbWazzCyf+IX3FT0z3X2fu5e6+0x3nwm8CFzs7lXBcpebWYGZzQJmAy/3t81M9GbTQXbFDuimRxHJSqEdqbh7p5ktA1YCEeBed19vZrcDVe7eZxgEy/0c2AB0Aje6exdAb9tM9b4ci5rgSY8LdKQiIlko1JEM3f1pjrgL391v62PZ9x4x/Y/APw5km5msJhojN8c4ZapGJhaR7KM76kNWE41x8pSxjMqLpLsUEZGkU6iEqKvbeaV2n7oSi0jWUqiEaEt9M81tnQoVEclaCpUQ9VykV3diEclWCpUQVUdjjB2Vy6yJRekuRUQkJRQqIaqJxji9YpxGJhaRrKVQCUlLWyeb3tivmx5FJKspVEKybtc+uh0WTNfIxCKSvRQqIekZmfh0HamISBZTqISkZmeMGRMLmVCUn+5SRERSRqESkppoTPeniEjWU6iE4I19B3lj/0GFiohkPYVKCGqijYCe9Cgi2U+hEoLqaIz8SA5zp45NdykiIimlUAlB9c4YJ08dS0GuRiYWkeymUEmxzq5u1tXu002PIjIiKFRS7M+7mznQ0aUnPYrIiBBqqJjZYjPbZGabzezmXuZfb2brzKzGzJ4zs7lB+5KgrefVbWbzg3nPBtvsmTcpzH3qT89Nj7pILyIjQWiPEzazCLAcOB+oBVaZ2Qp335Cw2CPu/qNg+YuBbwOL3f1h4OGg/VTgCXevSVhvibtXhbEfg1UTbWRCUT7TJxSmuxQRkZQL80hlIbDZ3be6ezvwGHBJ4gLuvj9hsgjwXrZzBfBoyqpMsppojNPLSzDTyMQikv3CDJVpQDRhujZoO4yZ3WhmW4C7gJt62c5f8/ZQuS849XWrZdC3d9PBDl57s5n5FRpEUkRGhjBDpbcv+7cdibj7cnc/Afgi8JXDNmB2NtDq7q8mNC9x91OBc4PXlb3+cLOlZlZlZlX19fVD3YdBWVe7D3c96VFERo4wQ6UWqEiYLgfqjrL8Y8ClR7RdzhFHKe6+K3hvAh4hfprtbdz9bnevdPfKsrKyQZY+NNU9F+nLFSoiMjKEGSqrgNlmNsvM8okHxIrEBcxsdsLkBcBrCfNygI8SD5uetlwzKw0+5wEXAolHMWlVvTPG8aVFlBTmpbsUEZFQhNb7y907zWwZsBKIAPe6+3ozux2ocvcVwDIzOw/oABqBqxM2sQiodfetCW0FwMogUCLAb4F7Qtidfrk7NdEYi+aUprsUEZHQhBYqAO7+NPD0EW23JXz+zFHWfRY454i2FuDM5FaZHLtiB9jT3KY76UVkRNEd9Sny1k2P6vklIiOHQiVFanbGKMjN4aQpxekuRUQkNAqVFKmJxpg3rYS8iP4Ti8jIoW+8FOjo6mbdrn0a70tERhyFSgpseqOJts5uhYqIjDgKlRSo3qnHB4vIyKRQSYHqaIzSMfmUjx+d7lJEREKlUEmBmmiM+RXjNTKxiIw4CpUk29fawdb6Fj3pUURGJIVKkq2t1ZMeRWTkUqgkWU00hhmcVl6S7lJEREKnUEmymmiME8vGUDxKIxOLyMijUEminpGJdepLREYqhUoSRRsO0NDSric9isiIpVBJouqobnoUkZFNoZJE1TtjjM6L8I7jNDKxiIxMCpUkqonGOLW8hFyNTCwiI5S+/ZKkrbOLDXX79aRHERnRQg0VM1tsZpvMbLOZ3dzL/OvNbJ2Z1ZjZc2Y2N2ifaWYHgvYaM/tRwjpnButsNrPvWprGRtn4ehPtXRqZWERGttBCxcwiwHLgQ8Bc4Iqe0EjwiLuf6u7zgbuAbyfM2+Lu84PX9QntPwSWArOD1+KU7cRR1PSMTKyeXyIygoV5pLIQ2OzuW929HXgMuCRxAXffnzBZBPjRNmhmU4Cx7v6CuzvwIHBpcssemJpojOPGFjClRCMTi8jIFWaoTAOiCdO1QdthzOxGM9tC/EjlpoRZs8ys2sx+b2bnJmyztr9thkE3PYqIhBsqvV3reNuRiLsvd/cTgC8CXwmaXwemu/sC4HPAI2Y2dqDbBDCzpWZWZWZV9fX1Q9qBvjS0tLN9byvzK8YndbsiIsNNmKFSC1QkTJcDdUdZ/jGCU1nu3ubue4PPq4EtwJxgm+UD2aa73+3ule5eWVZWNuSd6M3aqEYmFhGBcENlFTDbzGaZWT5wObAicQEzm50weQHwWtBeFlzox8yOJ35Bfqu7vw40mdk5Qa+vq4AnUr8rh6uOxsjRyMQiIuSG9YPcvdPMlgErgQhwr7uvN7PbgSp3XwEsM7PzgA6gEbg6WH0RcLuZdQJdwPXu3hDM+xRwPzAa+HXwClVNNMac44opKgjtP6eISEYK9VvQ3Z8Gnj6i7baEz5/pY71fAr/sY14VMC+JZQ6Ku7M2GuPDp05OVwkiIhlDd9Qfo217Wth3oEPXU0REUKgcs5pDF+nV80tERKFyjGqiMYryI5w4aUy6SxERSTuFyjGqicY4rXwckZy0DDkmIpJRFCrH4GBHfGRijfclIhKnUDkG6+v20dntukgvIhJQqByD6p3xi/R6hoqISJxC5RjURGNMGzeaSWNHpbsUEZGMoFA5BhqZWETkcAqVIdrT3EZt4wGFiohIAoXKENUE11PU80tE5C0KlSGqicaI5BjzpmpkYhGRHgqVIaqONnLS5GJG50fSXYqISMZQqAxBd7fzSnSfrqeIiBxBoTIEW+qbaWrrVKiIiBxBoTIE1cHIxAuma2RiEZFECpUhqInGKB6Vy/GlRekuRUQkoyhUhqBmZ/ymxxyNTCwicphQQ8XMFpvZJjMDQXI1AAAHBUlEQVTbbGY39zL/ejNbZ2Y1Zvacmc0N2s83s9XBvNVm9r6EdZ4NtlkTvCalch8OtHexaXeTrqeIiPQitGfUm1kEWA6cD9QCq8xshbtvSFjsEXf/UbD8xcC3gcXAHuAid68zs3nASmBawnpLgmfVp9y6Xfvo0sjEIiK9CvNIZSGw2d23uns78BhwSeIC7r4/YbII8KC92t3rgvb1wCgzKwih5repiTYCKFRERHoRZqhMA6IJ07UcfrQBgJndaGZbgLuAm3rZzkeAandvS2i7Lzj1dauZ9Xqhw8yWmlmVmVXV19cPeSeqd8aomDCaiWPSkmkiIhktzFDp7cve39bgvtzdTwC+CHzlsA2YnQJ8A/hkQvMSdz8VODd4XdnbD3f3u9290t0ry8rKhrgLPSMTqyuxiEhvwgyVWqAiYbocqOtjWYifHru0Z8LMyoHHgavcfUtPu7vvCt6bgEeIn2ZLid37D/L6voM69SUi0ocwQ2UVMNvMZplZPnA5sCJxATObnTB5AfBa0D4OeAq4xd3/mLB8rpmVBp/zgAuBV1O1A4ee9KiRiUVEehVa7y937zSzZcR7bkWAe919vZndDlS5+wpgmZmdB3QAjcDVwerLgBOBW83s1qDtA0ALsDIIlAjwW+CeVO1DTTRGXsSYO2Vsqn6EiMiwFlqoALj708DTR7TdlvD5M32sdydwZx+bPTNpBfajJtrI3CljGZWnkYlFRHqjO+oHqKvbWVerkYlFRI5GoTJAzW2dfOCUySyaM/SeYyIi2S7U01/DWcnoPL7z1/PTXYaISEbTkYqIiCSNQkVERJJGoSIiIkmjUBERkaRRqIiISNIoVEREJGkUKiIikjQKFRERSRpzf9sjTbKemdUDO9JdRy9KiT86eThS7ekxXGsfrnXDyK19hrv3O6TIiAyVTGVmVe5eme46hkK1p8dwrX241g2qvT86/SUiIkmjUBERkaRRqGSWu9NdwDFQ7ekxXGsfrnWDaj8qXVMREZGk0ZGKiIgkjUIlA5hZhZn9zsw2mtl6M+v1scqZyswiZlZtZk+mu5bBMLNxZvbvZvan4L/9O9Nd00CZ2WeDfyuvmtmjZjYq3TX1xczuNbM3zezVhLYJZvaMmb0WvI9PZ4196aP2bwb/Zl4xs8fNLCMfB9tb7QnzvmBmbmalyf65CpXM0Al83t1PBs4BbjSzuWmuaTA+A2xMdxFD8G/Af7n7ScDpDJN9MLNpwE1ApbvPAyLA5emt6qjuBxYf0XYz8N/uPhv472A6E93P22t/Bpjn7qcBfwZuCbuoAbqft9eOmVUA5wM7U/FDFSoZwN1fd/c1wecm4l9u09Jb1cCYWTlwAfDjdNcyGGY2FlgE/ATA3dvdPZbeqgYlFxhtZrlAIVCX5nr65O7/AzQc0XwJ8EDw+QHg0lCLGqDeanf337h7ZzD5IlAeemED0Md/d4DvAP8HSMkFdYVKhjGzmcAC4KX0VjJg/0r8H2h3ugsZpOOBeuC+4NTdj82sKN1FDYS77wK+RfwvzdeBfe7+m/RWNWjHufvrEP+jCpiU5nqG6lrg1+kuYqDM7GJgl7uvTdXPUKhkEDMbA/wS+Ht335/uevpjZhcCb7r76nTXMgS5wBnAD919AdBC5p6COUxw/eESYBYwFSgys79Jb1Ujj5l9mfip64fTXctAmFkh8GXgtlT+HIVKhjCzPOKB8rC7/yrd9QzQu4GLzWw78BjwPjP7aXpLGrBaoNbde44I/514yAwH5wHb3L3e3TuAXwHvSnNNg7XbzKYABO9vprmeQTGzq4ELgSU+fO7LOIH4HyJrg9/ZcmCNmU1O5g9RqGQAMzPi5/Y3uvu3013PQLn7Le5e7u4ziV8o/n/uPiz+Ynb3N4Comb0jaHo/sCGNJQ3GTuAcMysM/u28n2HSySDBCuDq4PPVwBNprGVQzGwx8EXgYndvTXc9A+Xu69x9krvPDH5na4Ezgt+FpFGoZIZ3A1cS/0u/Jnh9ON1FjQCfBh42s1eA+cA/pbmeAQmOrv4dWAOsI/57nLF3eZvZo8ALwDvMrNbMrgO+DpxvZq8R74n09XTW2Jc+av8+UAw8E/yu/iitRfahj9pT/3OHz5GbiIhkOh2piIhI0ihUREQkaRQqIiKSNAoVERFJGoWKiIgkjUJFJAOY2czeRpMVGW4UKiIikjQKFZEMY2bHB4NcnpXuWkQGS6EikkGCYWN+CXzC3Velux6RwcpNdwEickgZ8TGwPuLu69NdjMhQ6EhFJHPsA6LEx4ITGZZ0pCKSOdqJPwFxpZk1u/sj6S5IZLAUKiIZxN1bgoefPWNmLe4+bIaEFwGNUiwiIkmkayoiIpI0ChUREUkahYqIiCSNQkVERJJGoSIiIkmjUBERkaRRqIiISNIoVEREJGn+P3h7lG7asSagAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW5x/HPk40ACZskAQkIkkQFBMEIiIJaN0SrVVtrsRZUQMVWa23vrVe7aHvv7W1v7XLrUjfUulK3WpcqWq0iAgKyo6wikS0QAiGQ/bl/5EQjBpKBzJxM8n2/XvNy8suZM8/4Ar5zzu85v2PujoiISFMlhF2AiIjEFwWHiIhERMEhIiIRUXCIiEhEFBwiIhIRBYeIiEREwSEiIhFRcIiISEQUHCIiEpGksAuIhu7du3vfvn3DLkNEJK7Mnz9/m7tnNLZdqwyOvn37Mm/evLDLEBGJK2a2vinb6VSViIhERMEhIiIRiVpwmFmqmc01s0VmtszMbgvGTzezBWa20MxmmllOMP67YGyhma00s+J6+5pgZquCx4Ro1SwiIo2L5hxHOfAVd99tZsnATDN7BbgbuMDdV5jZVOBWYKK731j3QjP7HjA0eN4N+BmQDzgw38xecPcdUaxdRET2I2pHHF5rd/BjcvDw4NEpGO8MbGzg5d8Cngienw3McPeiICxmAGOjVbeIiBxYVLuqzCwRmA/kAHe6+xwzmwS8bGZ7gV3AyH1ecwTQD/hnMNQL2FBvk4JgTEREQhDVyXF3r3b344BsYLiZDQJuBMa5ezYwDbhjn5ddCjzt7tXBz9bQrvcdMLMpZjbPzOYVFhY234cQEZEviElXlbsXA28B5wBD3H1O8KungFH7bH4pn5+mgtojjN71fs6mgdNb7n6vu+e7e35GRqPXrzSoYMcefvvaR2wo2nNQrxcRaQui2VWVYWZdguftgTOAFUBnM8sLNjszGKt7zVFAV+C9ert6FTjLzLqaWVfgrGCs2ZWUVfF//1zNBxuKG99YRKSNiuYcR0/g4WCeIwGY7u4vmtlk4BkzqwF2AFfWe823gCfd/bNTUe5eZGa/AN4Phm5396JoFHxkRkcSDFZvKYnG7kVEWoWoBYe7LyZoqd1n/Dnguf285uf7GX8QeLA562tIu6RE+h7WkZVbdje+sYhIG6Urx/eRk5nGqq064hAR2R8Fxz7ystL5ePseKqpqwi5FRKRFUnDsIzcrjeoa5+PtpWGXIiLSIik49pGTmQbASk2Qi4g0SMGxj/4ZaSQYrNIEuYhIgxQc+0hNTqRPtw6s3qrgEBFpiIKjATmZ6TpVJSKyHwqOBuRmpbFuWymV1eqsEhHZl4KjAXlZaVTVOOvVWSUi8iUKjgbkZqYD6ApyEZEGKDga0D8jDVNnlYhIgxQcDWifkkjvrh209IiISAMUHPuRm5mmllwRkQYoOPYjJyuNtYWlVKmzSkTkCxQc+5GbmU5FdQ3rdTdAEZEvUHDsR15W7ZpVmiAXEfkiBcd+9M+oCw5NkIuI1Kfg2I+O7ZLo1aU9qzRBLiLyBQqOA8jLSlNwiIjsQ8FxALlZ6awp3E11jYddiohIi6HgOICczDQqqmr4RJ1VIiKfUXAcQG6mJshFRPal4DiA3KzaxQ41zyEi8jkFxwGktUvi8M6pOuIQEalHwdGInKx0HXGIiNSj4GhEXrDYoTqrRERqKTgakZuVRnlVDQU71FklIgIKjkblBHcD1JpVIiK1ohYcZpZqZnPNbJGZLTOz24Lx081sgZktNLOZZpZT7zWXmNnyYPvH641PMLNVwWNCtGpuSG7dYoea5xARASApivsuB77i7rvNLBmYaWavAHcDF7j7CjObCtwKTDSzXOBm4CR332FmmQBm1g34GZAPODDfzF5w9x1RrP0znVKT6dFJnVUiInWidsThteq+picHDw8enYLxzsDG4Plk4M66QHD3rcH42cAMdy8KfjcDGButuhuSqzWrREQ+E9U5DjNLNLOFwFZq//GfA0wCXjazAuBy4FfB5nlAnpm9a2azzawuHHoBG+rttiAYi5mcoLOqRp1VIiLRDQ53r3b344BsYLiZDQJuBMa5ezYwDbgj2DwJyAVOBb4F3G9mXQBraNf7DpjZFDObZ2bzCgsLm/Vz5GWls7eymk+L9zbrfkVE4lFMuqrcvRh4CzgHGBIceQA8BYwKnhcAf3P3SndfB3xEbZAUAL3r7S6bz09v1X+Pe909393zMzIymrX+z9as2qp5DhGRaHZVZQRHDJhZe+AMYAXQ2czygs3ODMYAngdOC7bvTu2pq7XAq8BZZtbVzLoCZwVjMZOrllwRkc9Es6uqJ/CwmSVSG1DT3f1FM5sMPGNmNcAO4Mpg+7qAWA5UAz9y9+0AZvYL4P1gu9vdvSiKdX9J5w7JZKa30wS5iAhRDA53XwwMbWD8OeC5BsYd+EHw2Pd3DwIPRqHMJsvNSlNLrogIunK8yXIzaxc7rM03EZG2S8HRRDmZaeypqGbjzrKwSxERCZWCo4nygps6rdTpKhFp4xQcTVTXkrtanVUi0sYpOJqoa8cUuqel6FoOEWnzFBwRqJsgFxFpyxQcEcjNSmP1FnVWiUjbpuCIQG5mGiXlVWzepc4qEWm7FBwRyM3S0iMiIgqOCNR1VqklV0TaMgVHBA5La0e3jims1gS5iLRhCo4I5WTqboAi0rYpOCKUl5XGyi0l6qwSkTZLwRGh3Mx0Ssqq2FpSHnYpIiKhUHBE6LO7AaqzSkTaKAVHhD5rydXSIyLSRik4ItQ9LYUuHZJZqSMOEWmjFBwRMjNyM9NYrSMOEWmjFBwHITcrnZVas0pE2igFx0HIzUxj595KCners0pE2h4Fx0HIzaydINdNnUSkLVJwHITcrKAlV1eQi0gbpOA4CJnp7eiUmqSWXBFpkxQcB8HMPpsgFxFpaxQcB6m2JVfBISJtj4LjIOVmpVNUWsF2dVaJSBuj4DhIn9/USUcdItK2KDgOUl1nla4gF5G2JmrBYWapZjbXzBaZ2TIzuy0YP93MFpjZQjObaWY5wfhEMysMxhea2aR6+5pgZquCx4Ro1RyJHp1SSW+XpJZcEWlzkqK473LgK+6+28ySgZlm9gpwN3CBu68ws6nArcDE4DVPuft36+/EzLoBPwPyAQfmm9kL7r4jirU3yszICW7qJCLSlkTtiMNr1X0dTw4eHjw6BeOdgY2N7OpsYIa7FwVhMQMYG4WSI6bOKhFpi6I6x2FmiWa2ENhK7T/+c4BJwMtmVgBcDvyq3ksuNrPFZva0mfUOxnoBG+ptUxCMhS43M51tuysoKq0IuxQREWpqPCaLr0Y1ONy92t2PA7KB4WY2CLgRGOfu2cA04I5g878Dfd19MPA68HAwbg3tet8BM5tiZvPMbF5hYWFzf5QGfT5BrqMOEQlXeVU1Nzy1kLveWhP194pJV5W7FwNvAecAQ4IjD4CngFHBNtvdve6iiPuA44PnBUDvz/dGNg2c3nL3e909393zMzIymv9DNKDuboCa5xCRMO3cW8l3HpjL3xdtJDGhoe/azSuaXVUZZtYleN4eOANYAXQ2s7xgszODMcysZ72Xn183DrwKnGVmXc2sK3BWMBa6wzun0jElUUccIhKajcV7+cY9s1jwyQ7+cOlxXHNK/6i/ZzS7qnoCD5tZIrUBNd3dXzSzycAzZlYD7ACuDLa/3szOB6qAIoJOK3cvMrNfAO8H293u7kVRrLvJajur0rXYoYiE4sPNu5j44PuUllfx8BXDGZXTPSbvG7XgcPfFwNAGxp8Dnmtg/Gbg5v3s60HgweausTnkZqbxr5WxmVMREakza/U2rv7LfDq2S2L6NSdyTM9Ojb+omejK8UOUm5lGYUk5xXvUWSUisfG3hZ8yYdpcenZJ5dmpo2IaGqDgOGTqrBKRWHF37vnXGm54ciHD+nTlr9eM4vAu7WNeh4LjENXdRlaLHYpINFXXOD9/YRm/euVDzhvck0euGk7n9smh1BLNyfE2oVeX9rRPTtQEuYhETVllNTc8+QGvLtvC5NH9uPmcY0iIQdvt/ig4DlFCgpGjpUdEJEp2lFYw6ZF5LPhkBz89bwBXntwv7JIUHM0hNyuNWau3h12GiLQyG4r2MOHBuRQU7+XO8cMYd2zPxl8UA5rjaAa5mels3lXGzr2VYZciIq3EkoKdXHjXLLaXVvDoVSNaTGiAgqNZ1N0NUKerRKQ5vPXRVr5573u0S0rgmWtPZHi/bmGX9AUKjmaQF6xZpbsBisihmj5vA1c9PI++h3XkuamjyAk6N1sSzXE0g15d25OanKCWXBE5aO7OH99Yze9eX8no3O7cddkw0lPDabdtjIKjGSQmGP0z0nQbWRE5KFXVNfzkb0t5Yu4GLhrWi/+5eDDJiS33hJCCo5nkZqYxd12LWHtRROLInooqvvv4B/zzw61897QcbjorD7PwrtFoipYbaXEmNyudjTvLKClTZ5WINM223eVceu9s3vpoK/954SB+ePZRLT40QMHRbNRZJSKRWLetlIvumsXKLSXce3k+l404IuySmkzB0Uzq7gaoeQ4RacwHn+zg4rtnsbu8iicmj+SMAVlhlxQRzXE0kz7dOpCSlKAjDhE5oBnLt/C9JxaQ1SmVh64YTr/uHcMuKWIHPOIws1Qz+9INvM0s08xSo1dW/KnrrNL9x0Vkfx6dvZ6r/zKPo7LSeebaUXEZGtD4qao/AqMbGD8T+F3zlxPfcjPTWKVrOURkH+7Or//xIbc+v5RTj8rkiSkj6Z7WLuyyDlpjwXGyuz+776C7PwaMiU5J8Ss3M41Pi/dSWl4Vdiki0kJUVNXwg+mLuOutNXxreB/uvfx4OqTE9yxBY8FxoL4wTazvI/ezpUd01CEisKuskisemstzH3zKj84+iv+6cBBJLfjCvqZq7BNsNbPh+w6a2QlAYXRKil91t5FVZ5WIbN5ZxiX3vMectUX89htDuO60nLi4RqMpGjte+hEw3cweAuYHY/nAd4BLo1hXXDqiWwdSEhN0N0CRNu6jzSVMnDaXkrIqpl1xAqNzv9RjFNcOGBzuPtfMRgBTgYnB8DJghLtvjXJtcScpMYEjMzpqglykDXtvzXam/GUe7ZMTeerqkQw8vHPYJTW7Rmdo3H0L8LMY1NIq5GSmsaigOOwyRCQEf1v4KT/662KOOKwDD105nF5d2oddUlQcMDjMbAngDf0KcHcfHJWq4lhuZjovLdnEnoqquO+cEJGmcXfufXst//3Khwzv1437Ls+nc4eWuSR6c2jsX7bzYlJFK5KXlYY7rC0sZVCv1neIKiJfVF3j3P73ZTz83nrOG9yT314yhHZJiWGXFVWNzXGs33fMzLoD2929oSORNq+us2rllhIFh0grV1ZZzQ1PfsCry7YweXQ/bj7nGBISWkfn1IE0tuTISDN7y8yeNbOhZrYUWApsMbOxsSkxvhxxWEeSEkwtuSKtXFFpBePvm81ry7fw0/MGcMu5A9pEaEDjp6r+BPwH0Bn4J3COu882s6OBJ4B/RLm+uJOcmEC/7uqsEmnNPtm+h4nT5lJQvJe7xg/jnGN7hl1STDV2AWCSu7/m7n8FNrv7bAB3/7CxHQcLJM41s0VmtszMbgvGTzezBWa20MxmmlnOPq/7upm5meXXG7vZzFab2UdmdnbkHzO28rLSdS2HSCu1uKCYi+5+l6I9FTw+aUSbCw1oPDhq6j3fu8/vGpvjKAe+4u5DgOOAsWY2ErgbuMzdjwMeB26te4GZpQPXA3PqjQ2g9mLDgcBY4C4za9EzTzmZaXxStIeyyuqwSxGRZvTmh1v55p9nk5qcyNPXjCK/b7ewSwpFY8ExxMx2mVkJMDh4XvfzsQd6odeqO1+THDw8eHQKxjsDG+u97BfAr4GyemMXAE+6e7m7rwNWA19aBqUlyQ06q9YU6nSVSGvxxNxPmPTIPPpnduTZqaPICe762RY11lV1SN/sgyOD+UAOcKe7zzGzScDLZrYX2AWMDLYdCvR29xfN7If1dtMLmF3v54JgbN/3mgJMAejTp8+hlH3I8uruBrhld6u8alSkLXF3fjdjJX/852pOycvgrsuG0bFd275GK6rLNLp7dXBKKhsYbmaDgBuBce6eDUwD7jCzBGrv73FTA7tpqE3hS6fJ3P1ed8939/yMjHDXhel7WEcSE0zzHCJxrrK6hh/+dTF//OdqLsnP5v4J+W0+NCBGt45192Izews4Bxji7nVzGE9R25mVDgwC3gpWj+wBvGBm51N7hNG73u6y+eLprRYnJSmBvod1UGeVSBzbXV7FtY/O551V2/j+GbnccHpuq1nd9lBF7YjDzDLMrEvwvD1wBrAC6GxmecFmZwIr3H2nu3d3977u3pfaU1Pnu/s84AXgUjNrZ2b9gFxgbrTqbi65mem6L4dInNq6q3ZJ9FlrtvPriwfz/TPyFBr1RPOIoyfwcDDPkQBMD+YvJgPPmFkNsAO48kA7cfdlZjYdWA5UAde5e4tvV8rLSuO15Zspq6wmNblFN4GJSD2rt5Yw4cH32bGnggcm5HPqUZlhl9TiRC043H0xMLSB8eeA5xp57an7/PyfwH82Z33RlpOVTo3Dum2lHNOzU+MvEJHQzV1XxKSH3yclKZHpV5+oZYP2I/7vYdhC5WbqboAi8eTFxRv59v1z6J7ejuemjlJoHIDaA6LkyIyOJBis2qLOKpGWzN15YOY6fvnSCvKP6Mr9E/Lp0iEl7LJaNAVHlLRLSqTvYVqzSqQlq65xfvnScqa9+zHjju3BHZccpznJJlBwRFFOZpqu5RBpocoqq/n+kwv5x7LNXHVyP24Z1zaWRG8OCo4oystK540Pt1JeVd3qb+wiEk92lFYw6ZF5LPhkB7eeewyTRh8ZdklxRcERRblZaVTXOB9v28NRPdLDLkdEgA1Fe5jwYO2S6HeOH8a4Nri67aFSV1UU5XzWWaXTVSItweKCYi686122l1bw2KQRCo2DpCOOKOqfkRZ0VmmCXCRsb364lamPLaBbxxSenDK8Ta9ue6gUHFGUmpxIn24ddMQhErIn5n7Crc8v5Zie6Tw48QQy01PDLimuKTiiLCczXUccIiHRkujRoTmOKMvNSmPdtlIqq2sa31hEmk1F1edLon8zv7eWRG9G+r8YZXlZaVTVOB9vKyU3S51VIrFQUlbJ1McW8M6qbdx4Rh7Xn56j1W2bkYIjynIzg7sBbt2t4BCJgS27ypg47X1WbSnh118fzCX5vRt/kUREwRFl/TPSsLrOqgPepV1EDtXKLSVMfHAuO/dW8sDEEzglL9y7gbZWCo4oa5+SSO+uHVipziqRqHpvzXam/GUeqcmJPKUl0aNKwREDuZlprFZnlUjUvLBoIz+cvog+h3XgoStOILtrh7BLatXUVRUDOVlprN22myp1Vok0K3fn3rfXcP0TH3Bcny48c80ohUYM6IgjBnIz06msdtYX7aF/hq5WFWkO1TXO7X9fxsPvrefcwT357TeGaEn0GFFwxEBeVrBm1ZYSBYdIMyirrOb6Jz7gteVbmDy6HzefoyXRY0mnqmKgLix0BbnIoSsqrWD8fbOZsWILP/vqAG45d4BCI8Z0xBEDHdsl0atLe91/XOQQfbytlInT5rJpZxl3jR/GOVrdNhQKjhjJy0pjpe4/LnLQ5q/fweRH5uHuPD55JMcf0TXsktosnaqKkdysdNZuK1VnlchB+MfSTYy/bzbpqUk8O/UkhUbIFBwxkpOZRkVVDRt27A27FJG48sDMdVz72AIGHN6JZ68dRb/uHcMuqc3TqaoYyQ1uGrNyS4n+4Is0QXWN88uXljPt3Y85e2AWf7h0qNptWwgdccRI3QKHqzVBLtKosspqrntsAdPe/ZgrT+rHXZcdr9BoQXTEESNp7ZI4vHMq89fvwN21xLPIfmzfXc6kR+axcEMxPz1vAFee3C/skmQfOuKIoYuPz+afH27l96+vCrsUkRZp3bZSLrp7Fss37uLuy4YpNFqoqAWHmaWa2VwzW2Rmy8zstmD8dDNbYGYLzWymmeUE49eY2ZJ64wPq7etmM1ttZh+Z2dnRqjnabjwjj28cn80f3ljF/e+sDbsckRZl/vodXHTXu5SUVfH45JGMHaRrNFqqaJ6qKge+4u67zSwZmGlmrwB3Axe4+wozmwrcCkwEHnf3ewDM7HzgDmBsECCXAgOBw4HXzSzP3aujWHtUJCQY/33Rsewur+KXL62gU2oyl5ygm8yIvLJkE99/aiE9O6fy0BXD6asGkhYtakccXqtuJjg5eHjw6BSMdwY2BtvvqvfyjsF2ABcAT7p7ubuvA1YDw6NVd7QlJSbw+0uPY0xeBj9+djEvLd4UdkkioXpg5jqmPr6AgYd34tmpJyk04kBUJ8fNLBGYD+QAd7r7HDObBLxsZnuBXcDIettfB/wASAG+Egz3AmbX221BMLbve00BpgD06dOn+T9MM2qXlMg93x7Gdx6Yy/ef+oCO7RI59ajMsMsSian67bZjB/bg95cep86pOBHVyXF3r3b344BsYLiZDQJuBMa5ezYwjdpTUnXb3+nu/YF/p/YUFkBD7Uf+pQH3e909393zMzJa/u0iO6Qk8cDEE8jNTOeaR+fz/sdFYZckEjN7K6qZ+th8pr37MVed3I87Lxum0IgjMemqcvdi4C3gHGCIu88JfvUUMKqBlzwJfC14XgDUnwjIJji9Fe86t0/mkauGc3iX9lw57X2Wfroz7JJEom777nK+dd9sXlteu7rtT84bQKJWt40r0eyqyjCzLsHz9sAZwAqgs5nlBZudGYxhZrn1Xn4uUNez+gJwqZm1M7N+QC4wN1p1x1r3tHY8etUIOrVP5jsPztUFgtKqrS3czUV3z2LFpl3cfdnxXHGS2m3jUTSPOHoCb5rZYuB9YIa7vwhMBp4xs0XA5cCPgu2/G7TtLqR2nmMCgLsvA6YDy4F/ANfFY0fVgRzepT2PThpBghnfvn8OG4r2hF2SSLObv76Ii++eRUlZFU9MGcnYQT3CLkkOkrl/abog7uXn5/u8efPCLiNiKzbt4pt/fo9uHVOYfs2JZKanhl2SSLN4ZckmbnhqIb26tGfaxBPUOdVCmdl8d89vbDtdOd6CHNOzEw9dOZytJeVcfv9civdUhF2SyCFxd+5/Zy1TH1/Asb0688y1oxQarYCCo4UZ1qcr916ez7ptpUyc9j6l5VVhlyRyUKprnNv+vpxfvrSCsQN78NikEXTrmBJ2WdIMFBwt0Mm53fm/8UNZ8ulOJj8yj7LKVjWlI23A3opqrn10Pg/N+pjJo/tx53i127YmCo4W6uyBPfjN1wcza812vvfEB1TqzoESJwpLyrn0vtnMWLGFn391ALecO4AEtdu2KgqOFuyiYdncfsFAZizfwr89vZiamtbXyCCty8otJXztzndZubmEP3/7eCaq3bZV0v04WrjvnNiXkrIqfvPqR6S1S+L2CwbqXh7SIs1ctY1rH51Pakoi068+kWOzO4ddkkSJgiMOTD21P7v2VvLnt9fSqX0SPzr76LBLEvmCJ+d+wq3PLyUnM40HJp5Ary7twy5JokjBEQfMjB+fczS7yqq48801pKcmc80p/cMuS4SaGuc3r33E3W+tYUxeBneOH0p6anLYZUmUKTjihJnxy68NYnd5Fb965UPSU5O4bMQRYZclbVhZZTU3TV/ES0s2MX5EH24/fyBJiZo2bQsUHHEkMcG445IhlJZXcevzS0lrl8QFx31phXmRqNu2u5zJwX3Bbxl3DJNG99PcWxuirwdxJjkxgbsuG8bwvt24afoi3lixJeySpI1ZvbWEC+9697OFCiePOVKh0cYoOOJQanIi90/IZ8DhnZj62ALeW7M97JKkjZi1ehsX3jWLvRU1PDXlRC1U2EYpOOJUemoyD10xnD7dOjDp4fdZtKE47JKklZs+bwPfeXAuPTql8tzUUQzp3SXskiQkCo441q1jCn+5agTd0lKYMG0uK7eUhF2StEI1Nc7/vvoR//b0YkYeeRhPXzuK3t06hF2WhEjBEed6dE7lsatGkpKYwPj75vCPpZtojUvlSzjKKqu54amF/OnN1XwzvzfTrjiBzu3VbtvWKThagT6HdeCxSSPonpbCNY8u4LL75/DRZh19yKHZvrucy+6fw98XbeTfxx7Nry4+lmS12woKjlYjNyudF793Mr+4YCDLNu5i3B/f4ecvLGPnnsqwS5M4tCa4xeuST3dy5/hhXHtqf3VOyWcUHK1IUmICl5/Yl7d+eCrjh/fhkfc+5tT/fZPH5qynWgskShPNXrudi+6axe6yKp6YPJJzB/cMuyRpYRQcrVDXjin84muDeOn60eRlpXPLc0v56v/NZO66orBLkxbu2QUFXP7AHLqnpfDc1JM4/oiuYZckLZCCoxU7pmcnnpwykjvHD2Pn3kou+fN7fPfxBWws3ht2adLCuDt3zFjJD6YvIv+Ibjx77Un0OUydU9IwLTnSypkZ5w7uyVeOzuSef63hnn+t4fUVW5h6ag5Txhypu7IJ5VXV/PvTi3l+4Ua+fnw2/3XhsaQk6Tul7J+1xtbN/Px8nzdvXthltEgFO/bw3y9/yEtLNpHdtT23jDuGsYN6aOKzjdpRWsHVf5nP3I+L+NHZRzFVk+BtmpnNd/f8xrbT14o2JrtrB+68bBhPTB5JWrskrn1sAePvm8OHm3eFXZrE2LptpVx09ywWFhTzx28N5brTchQa0iQKjjbqxP6Hfda+u2LzLsb94R1+9relFO+pCLs0iTJ35+n5BZz/p5kU76ng8UkjOH/I4WGXJXFEp6qEHaUV3DFjJY/NWU/n9sn84KyjGD+8D4kJ+vbZ2mwtKeM/nl3C6yu2ckLfrvz2G8dpElw+09RTVQoO+cyKTbu47e/LmL22iGN6duLnXx3AiCMPC7ssaSZ/X7SRn/xtKXsqqvm3s4/iipP66cuBfIGCQ8FxUNydl5ds5r9eXsGnxXs5d3BP/mPcMbqHdBwrKq3gJ88v5aUlmxiS3ZnfXjKEnMz0sMuSFqipwaF2XPmChtp331ixhWtPyeHqU9S+G29eXbaZW55bws69lfzo7KO4esyRur2rHLKo/Qkys1Qzm2tmi8yzAe19AAAMJElEQVRsmZndFoyfbmYLzGyhmc00s5xg/AdmttzMFpvZG2Z2RL19TTCzVcFjQrRqls+1T0nkxjPzeOOmUzj96Cx+9/pKzv/TTC2eGCd27qnkxqcWcvVf5pOZnsoL3z2Z607LUWhIs4jaqSqr7evr6O67zSwZmAncADwCXODuK8xsKjDc3Sea2WnAHHffY2bXAqe6+zfNrBswD8gHHJgPHO/uO/b33jpV1fz+tbKQm6YvoqSskp+cN4DLRvRR62YL9eZHW/nxM4vZtruC607L4bun5eiCPmmS0K/j8Fq7gx+Tg4cHj07BeGdgY7D9m+6+JxifDWQHz88GZrh7URAWM4Cx0apbGnZKXgav3DCaEUcexq3PL+XaRxeodbeFKSmr5MfPLOaKae/TKTWZ56aO4gdn5ik0pNlFdY7DzBKpPULIAe509zlmNgl42cz2AruAkQ289CrgleB5L2BDvd8VBGMSYxnp7Xho4gk8MHMdv371Q8b9oZjfXzqU4f26hV1amzdr9TZ+9PRiNu3cy9WnHMmNZ+RpPkqiJqpfRdy92t2Po/boYbiZDQJuBMa5ezYwDbij/mvM7NvUnpb6Td1QQ7ved8DMppjZPDObV1hY2JwfQ+pJSDAmjzmSZ64dRUpSApfe+x6/f30lVdU1YZfWJu2pqOKnf1vK+PvnkJKUwF+vGcXN5xyj0JCoiskxrLsXA28B5wBD3H1O8KungFF125nZGcAtwPnuXh4MFwC96+0um+D01j7vca+757t7fkZGRvN/CPmCwdldePH60XxtaC9+//oqxt83R6vuxti8j4s45w/v8Mh767nipL68fP1oLYMuMRHNrqoMM+sSPG8PnAGsADqbWV6w2ZnBGGY2FPgztaGxtd6uXgXOMrOuZtYVOCsYk5CltUvijkuO43ffHMKyjTs55w/v8I+lm8Muq9Urq6zmP19azjf+/B417jw5ZSQ/++pA2qfoKENiI5pzHD2Bh4N5jgRguru/aGaTgWfMrAbYAVwZbP8bIA34a9Ct84m7n+/uRWb2C+D9YLvb3V13JGpBLhyazdDeXbn+yQ+45tH5fHtkH249d4BOl0TBwg3F3DR9IWsKSxk/og//Me4Y0trpciyJLV05Ls2moqqG3772EX9+ey15WWn837eGcVQPXaHcHMqrqvnjG6u4519ryUxvx/9cPJgxeTolK81LS44oOELz9spCftACrvkoq6zmnVXbmLN2OwN7deKMY7JIT02OeR2HatnGndw0fREfbi7h68dn85PzBtC5ffx9Dmn5FBwKjlAVlpRz018X8fbKQsYO7MGvLj6WLh1Sov6+u8oqefPDrby6bDNvfVTInopqEhOM6honJTGBMXndGXdsT84YkEWnFh4ixXsqeOS99fzxjVV07ZjCf194LGcMyAq7LGnFFBwKjtDV1Phn13xkpLWL2jUfhSXlzFi+hVeXbWbWmm1UVjsZ6e04c0AWYwf2YMSR3Vj66S5eWryJV5ZuYtPOMlISExid+3mItIRv8FXVNSzcUMzbKwt5e9U2FhcUU+Nw/pDDue38gXTtGP3glbZNwaHgaDEWFxRz/RMf8EnRHq4/PZfvNsOaSRuK9vDqss28umwz89bvwB36dOvA2QOzGDuoB0N7dyWhgSXDa2qcDzYU8/KSTbyyZBMbd5aRnGiMyc0IJUQ2FO3hXysLeWdVIbNWb6ekvIoEgyG9uzAmN4NTj8pgaB+12EpsKDgUHC3K7vIqfvr8Up794FOG9+3G7y89jsMjWKrd3floSwmvLq09sli+qfZWt0f3SOfsgT0YO6gHR/dIj2gupabGWVhQzMuLN/HK0s18WryX5ERjdBAiZ0YhRHaXV/Hemu28s6qQd1ZtY922UgB6dWnPmLzujM7N4KT+3encIfwjIGl7FBwKjhbp2QUF/OT5pSQlJvA/Fw9m7KAe+9227ujgteDI4uPtezCDYX26MnZgD84amMURh3VslrrcnYXBkcjLSz4PkZNzak9nnTWgx0H9Y15T4yzbuIu3VxXyr5WFLFi/g6oap31yIiOP7MaYvAzG5GVwZPeOWjRSQqfgUHC0WB9vK+X6Jz9gccHOL13zUVldw+y123l12WZeW7aFrSXlJCUYJ/Y/jLGDenDmgCwy01OjWp+7s6hgJy8t3viFEDkpCJGzGwmRLbvKeHtl7RHFzNXbKCqtXQxy4OGdGJ2bwZjc7hzftyvtknSdi7QsCg4FR4tWUVXD/772EfcG13xMGdOfWau38fqKLewqq6J9ciKn5GUwdlAPTjs6M7TJ67oQeXnJJl5avIlPi/eSlFAbIuce25OzBmaRmpzI3HVFvLOqkLdXbuOjLbX3LOme1o4xud0Zndedk3MyyEhvF8pnEGkqBYeCIy7U3udjIdt2V9C5fTKnH5PJ2IE9GJ2b0eKW0HB3FteFyJJNFOyoDZHEBKO8qoaUxARO6Nc1OKrI4Oge6Q1O0Iu0VAoOBUfc2FFawdptpQzO7kxynNyhzt1Z8ulOXlm6mYqqGk7O7c6Ift3okKLlPyR+6Z7jEje6dkzh+Di7RsHMGJzdhcHZXcIuRSTm4uPrnYiItBgKDhERiYiCQ0REIqLgEBGRiCg4REQkIgoOERGJiIJDREQiouAQEZGItMorx82sEFgfdh370R3YFnYRB0m1x1681g2qPSyHUvsR7t7ozexbZXC0ZGY2rymX9LdEqj324rVuUO1hiUXtOlUlIiIRUXCIiEhEFByxd2/YBRwC1R578Vo3qPawRL12zXGIiEhEdMQhIiIRUXDEiJn1NrM3zWyFmS0zsxvCrikSZpZoZh+Y2Yth1xIJM+tiZk+b2YfB//sTw66pqczsxuDPylIze8LMonuz9UNgZg+a2VYzW1pvrJuZzTCzVcF/u4ZZ4/7sp/bfBH9mFpvZc2bWIm+80lDt9X73QzNzM+ve3O+r4IidKuAmdz8GGAlcZ2YDQq4pEjcAK8Iu4iD8AfiHux8NDCFOPoOZ9QKuB/LdfRCQCFwablUH9BAwdp+xHwNvuHsu8Ebwc0v0EF+ufQYwyN0HAyuBm2NdVBM9xJdrx8x6A2cCn0TjTRUcMeLum9x9QfC8hNp/wHqFW1XTmFk2cC5wf9i1RMLMOgFjgAcA3L3C3YvDrSoiSUB7M0sCOgAbQ65nv9z9baBon+ELgIeD5w8DX4tpUU3UUO3u/pq7VwU/zgayY15YE+zn/zvA74B/A6Iyia3gCIGZ9QWGAnPCraTJfk/tH8KasAuJ0JFAITAtOM12v5l1DLuopnD3T4H/pfYb4yZgp7u/Fm5VEcty901Q+8UJyAy5noN1JfBK2EU0lZmdD3zq7oui9R4KjhgzszTgGeD77r4r7HoaY2bnAVvdfX7YtRyEJGAYcLe7DwVKabmnS74gmA+4AOgHHA50NLNvh1tV22Nmt1B7mvmxsGtpCjPrANwC/DSa76PgiCEzS6Y2NB5z92fDrqeJTgLON7OPgSeBr5jZo+GW1GQFQIG71x3ZPU1tkMSDM4B17l7o7pXAs8CokGuK1BYz6wkQ/HdryPVExMwmAOcBl3n8XLfQn9ovG4uCv7PZwAIz69Gcb6LgiBEzM2rPta9w9zvCrqep3P1md892977UTs7+093j4puvu28GNpjZUcHQ6cDyEEuKxCfASDPrEPzZOZ04mdiv5wVgQvB8AvC3EGuJiJmNBf4dON/d94RdT1O5+xJ3z3T3vsHf2QJgWPB3odkoOGLnJOByar+xLwwe48Iuqg34HvCYmS0GjgP+K+R6miQ4SnoaWAAsofbvaou9mtnMngDeA44yswIzuwr4FXCmma2itsPnV2HWuD/7qf1PQDowI/i7ek+oRe7HfmqP/vvGzxGYiIi0BDriEBGRiCg4REQkIgoOERGJiIJDREQiouAQEZGIKDhEYsTM+ja0iqlIvFFwiIhIRBQcIiEwsyODhRdPCLsWkUgpOERiLFgC5RngCnd/P+x6RCKVFHYBIm1MBrVrNl3s7svCLkbkYOiIQyS2dgIbqF27TCQu6YhDJLYqqL0T3qtmttvdHw+7IJFIKThEYszdS4MbZM0ws1J3j5vlxkVAq+OKiEiENMchIiIRUXCIiEhEFBwiIhIRBYeIiEREwSEiIhFRcIiISEQUHCIiEhEFh4iIROT/ATvFhifrlw6VAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get best rss score for each subset\n",
    "rss   = [min_rss(m)[1].ssr for m in model_subsets]\n",
    "\n",
    "# Get best r-squared score for each subset\n",
    "adjr2 = [max_adjr2(m)[1].rsquared_adj for m in model_subsets]\n",
    "k     = np.arange(1, len(rss)+1)\n",
    "\n",
    "\n",
    "# Get best BIC\n",
    "def min_bic(statsmodels):\n",
    "    \"\"\"Return model with lowest R-squared\"\"\"\n",
    "    return sorted(statsmodels, reverse=False, key=lambda tup: tup[1].bic)[0]\n",
    "\n",
    "bic = [min_bic(m)[1].bic for m in model_subsets]\n",
    "\n",
    "\n",
    "ax = sns.lineplot(x=k, y=rss)\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('rss')\n",
    "plt.show()\n",
    "\n",
    "sns.lineplot(x=k, y=adjr2)\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('$R^2$')\n",
    "plt.show()\n",
    "\n",
    "sns.lineplot(x=k, y=bic)\n",
    "plt.xlabel('k')\n",
    "plt.ylabel('BIC')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The stats above seem to confer that a model with around 8 predictors is best. But has each method chosen the same or similar models?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AtBat', 'Hits', 'Walks', 'CRuns', 'CWalks', 'PutOuts']\n"
     ]
    }
   ],
   "source": [
    "best_subset_6 = list(min_rss(model_subsets[5])[1].params.index)[1:]\n",
    "print(best_subset_6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AtBat', 'Hits', 'Walks', 'CRuns', 'CWalks', 'PutOuts']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(max_adjr2(model_subsets[5])[1].params.index)[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AtBat', 'Hits', 'Walks', 'CRuns', 'CWalks', 'PutOuts']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(min_bic(model_subsets[5])[1].params.index)[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All three statisticis yield the same model for the subset with 6 predictors. This is expected for RSS and R-squared stats because they are related."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.5.2 Forward and Backward Stepwise Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scale(df):\n",
    "    \"\"\"Xi = Xi - mu(Xi) / σ(Xi)\"\"\"\n",
    "    return (df-df.mean())/(df.std())\n",
    "\n",
    "\n",
    "def forward_stepwise(X, y, results=[(0, [])]):\n",
    "    \"\"\"Perform forward stepwise variable selection as described in\n",
    "    An Introductions to Statistical Learning (p.207)\n",
    "    Returns pandas dataframe object  with results for each step\"\"\"\n",
    "    # List predictors that havent's been used so far\n",
    "    p_all    = list(X.columns)\n",
    "    p_used   = results[-1][1]\n",
    "    p_unused = [p for p in p_all if p not in p_used]\n",
    "    \n",
    "    # Job done, exit recursion\n",
    "    if not p_unused:\n",
    "        rss   = [r[0] for r in results]\n",
    "        preds = [r[1] for r in results]\n",
    "        return pd.DataFrame({'rss': rss, 'predictors': preds}).drop(0).reset_index()\n",
    "    \n",
    "    # Get rss score for each possible additional predictor\n",
    "    r = []\n",
    "    for p in p_unused:\n",
    "        f     = 'Salary ~ {}'.format('+'.join([p]+p_used))\n",
    "        # Fit model\n",
    "        model = smf.ols(formula=f, data=pd.concat([X, y], axis=1)).fit()\n",
    "        r     += [(model.ssr, [p]+p_used)]\n",
    "    \n",
    "    # Choose predictor which yields lowest rss\n",
    "    min_rss     = sorted(r, key=lambda tup: tup[0])[0]   \n",
    "    new_results = results + [min_rss]\n",
    "    # Recursive call to self\n",
    "    return forward_stepwise(X, y, new_results)\n",
    "\n",
    "\n",
    "def backward_stepwise(X, y, results=[]):\n",
    "    \"\"\"Perform backward stepwise variable selection as described in\n",
    "    An Introductions to Statistical Learning (p.209)\n",
    "    Returns pandas dataframe object  with results for each step\"\"\"\n",
    "    # List predictors that havent's been used so far\n",
    "    p_all    = list(X.columns)\n",
    "\n",
    "    # Check if we're starting out here\n",
    "    if not results:\n",
    "        # Fit model with all features\n",
    "        f     = 'Salary ~ {}'.format('+'.join(p_all))\n",
    "        model = smf.ols(formula=f, data=pd.concat([X, y], axis=1)).fit()\n",
    "        # Begin backward stepwise recursion\n",
    "        return backward_stepwise(X, y, [(model.ssr, p_all)])\n",
    "    else:\n",
    "        p_used = results[-1][1]\n",
    "    \n",
    "    # Job done, exit recursion\n",
    "    if len(p_used) == 1:\n",
    "        rss   = [r[0] for r in results]\n",
    "        preds = [r[1] for r in results]\n",
    "        return pd.DataFrame({'rss': rss, 'predictors': preds})    \n",
    "    \n",
    "    # Get rss score for each possible removed predictor\n",
    "    r = []\n",
    "    for p in p_used:\n",
    "        p_test = [i for i in p_used if i != p]\n",
    "        f     = 'Salary ~ {}'.format('+'.join(p_test))\n",
    "        # Fit model\n",
    "        model = smf.ols(formula=f, data=pd.concat([X, y], axis=1)).fit()\n",
    "        r     += [(model.ssr, p_test)]\n",
    "    \n",
    "    # Choose removal of predictor which yields lowest rss\n",
    "    min_rss     = sorted(r, key=lambda tup: tup[0])[0]   \n",
    "    new_results = results + [min_rss]\n",
    "    return backward_stepwise(X, y, new_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>Forward Stepwise Selection</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>rss</th>\n",
       "      <th>predictors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>3.617968e+07</td>\n",
       "      <td>[CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3.064656e+07</td>\n",
       "      <td>[Hits, CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2.924930e+07</td>\n",
       "      <td>[PutOuts, Hits, CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2.823936e+07</td>\n",
       "      <td>[AtBat, PutOuts, Hits, CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2.717042e+07</td>\n",
       "      <td>[Walks, AtBat, PutOuts, Hits, CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>2.699220e+07</td>\n",
       "      <td>[CWalks, Walks, AtBat, PutOuts, Hits, CRBI]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>2.614246e+07</td>\n",
       "      <td>[CRuns, CWalks, Walks, AtBat, PutOuts, Hits, C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>2.565678e+07</td>\n",
       "      <td>[CAtBat, CRuns, CWalks, Walks, AtBat, PutOuts,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>2.552857e+07</td>\n",
       "      <td>[Runs, CAtBat, CRuns, CWalks, Walks, AtBat, Pu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>2.551233e+07</td>\n",
       "      <td>[Years, Runs, CAtBat, CRuns, CWalks, Walks, At...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>2.550552e+07</td>\n",
       "      <td>[CHmRun, Years, Runs, CAtBat, CRuns, CWalks, W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>2.550319e+07</td>\n",
       "      <td>[CHits, CHmRun, Years, Runs, CAtBat, CRuns, CW...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>2.550170e+07</td>\n",
       "      <td>[HmRun, CHits, CHmRun, Years, Runs, CAtBat, CR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>2.550169e+07</td>\n",
       "      <td>[RBI, HmRun, CHits, CHmRun, Years, Runs, CAtBa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index           rss                                         predictors\n",
       "0       1  3.617968e+07                                             [CRBI]\n",
       "1       2  3.064656e+07                                       [Hits, CRBI]\n",
       "2       3  2.924930e+07                              [PutOuts, Hits, CRBI]\n",
       "3       4  2.823936e+07                       [AtBat, PutOuts, Hits, CRBI]\n",
       "4       5  2.717042e+07                [Walks, AtBat, PutOuts, Hits, CRBI]\n",
       "5       6  2.699220e+07        [CWalks, Walks, AtBat, PutOuts, Hits, CRBI]\n",
       "6       7  2.614246e+07  [CRuns, CWalks, Walks, AtBat, PutOuts, Hits, C...\n",
       "7       8  2.565678e+07  [CAtBat, CRuns, CWalks, Walks, AtBat, PutOuts,...\n",
       "8       9  2.552857e+07  [Runs, CAtBat, CRuns, CWalks, Walks, AtBat, Pu...\n",
       "9      10  2.551233e+07  [Years, Runs, CAtBat, CRuns, CWalks, Walks, At...\n",
       "10     11  2.550552e+07  [CHmRun, Years, Runs, CAtBat, CRuns, CWalks, W...\n",
       "11     12  2.550319e+07  [CHits, CHmRun, Years, Runs, CAtBat, CRuns, CW...\n",
       "12     13  2.550170e+07  [HmRun, CHits, CHmRun, Years, Runs, CAtBat, CR...\n",
       "13     14  2.550169e+07  [RBI, HmRun, CHits, CHmRun, Years, Runs, CAtBa..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<h4>Backward Stepwise Selection</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rss</th>\n",
       "      <th>predictors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.550169e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.550170e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, Walks, Years, CAtBa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.550319e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, Years, CAtBat, CHit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.550552e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, Years, CAtBat, CHmR...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.551233e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, Years, CAtBat, CRun...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.552857e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.565678e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.614246e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, CRBI, CWalks, PutO...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.680370e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, CWalks, PutOuts]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.750952e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, PutOuts]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2.845081e+07</td>\n",
       "      <td>[AtBat, Hits, CRuns, PutOuts]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.940730e+07</td>\n",
       "      <td>[Hits, CRuns, PutOuts]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3.120346e+07</td>\n",
       "      <td>[Hits, CRuns]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3.643795e+07</td>\n",
       "      <td>[CRuns]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             rss                                         predictors\n",
       "0   2.550169e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...\n",
       "1   2.550170e+07  [AtBat, Hits, HmRun, Runs, Walks, Years, CAtBa...\n",
       "2   2.550319e+07  [AtBat, Hits, Runs, Walks, Years, CAtBat, CHit...\n",
       "3   2.550552e+07  [AtBat, Hits, Runs, Walks, Years, CAtBat, CHmR...\n",
       "4   2.551233e+07  [AtBat, Hits, Runs, Walks, Years, CAtBat, CRun...\n",
       "5   2.552857e+07  [AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...\n",
       "6   2.565678e+07  [AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...\n",
       "7   2.614246e+07  [AtBat, Hits, Walks, CRuns, CRBI, CWalks, PutO...\n",
       "8   2.680370e+07       [AtBat, Hits, Walks, CRuns, CWalks, PutOuts]\n",
       "9   2.750952e+07               [AtBat, Hits, Walks, CRuns, PutOuts]\n",
       "10  2.845081e+07                      [AtBat, Hits, CRuns, PutOuts]\n",
       "11  2.940730e+07                             [Hits, CRuns, PutOuts]\n",
       "12  3.120346e+07                                      [Hits, CRuns]\n",
       "13  3.643795e+07                                            [CRuns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = scale(hitters_pd.loc[:, 'AtBat':'PutOuts'])\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "forward_stepwise_results = forward_stepwise(X, y)\n",
    "display(HTML('<h4>Forward Stepwise Selection</h4>'))\n",
    "display(forward_stepwise_results)\n",
    "\n",
    "backward_stepwise_results = backward_stepwise(X, y)\n",
    "display(HTML('<h4>Backward Stepwise Selection</h4>'))\n",
    "display(backward_stepwise_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>Comparison of methods for subset with 6 predictors</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Subset Selection       : ['AtBat', 'CRuns', 'CWalks', 'Hits', 'PutOuts', 'Walks']\n",
      "Forward Stepwise Selection  : ['AtBat', 'CRBI', 'CWalks', 'Hits', 'PutOuts', 'Walks']\n",
      "Backward Stepwise Selection : ['AtBat', 'CRuns', 'CWalks', 'Hits', 'PutOuts', 'Walks']\n"
     ]
    }
   ],
   "source": [
    "display(HTML('<h4>Comparison of methods for subset with 6 predictors</h4>'))\n",
    "print('Best Subset Selection       : {}'.format(sorted(best_subset_6)))\n",
    "print('Forward Stepwise Selection  : {}'.format(sorted(forward_stepwise_results.loc[5]['predictors'])))\n",
    "print('Backward Stepwise Selection : {}'.format(sorted(backward_stepwise_results.loc[8]['predictors'])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above results show that Backward Stepwise Selection chooses the same 6 predictors as Best Subset Selection. Forward Stepwise Selection chooses all but one of the same predictors.\n",
    "\n",
    "Backward Stepwise Selection therefore looks like the best approach as it makes the same optimal selection as best subset selection, but with significantly lower computational cost."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.5.3 Choosing Among Models Using the Validation Set Approach and Cross-Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def best_subset_rss(X, y):\n",
    "    \"\"\"Perform best subset selection using Residual Sum of Squares to\n",
    "    select best model in each subset.\n",
    "    Notes: highly computationally expensive for large number of features in  X\n",
    "    Maxes out my laptop for p > 14\"\"\"\n",
    "    # get all model results\n",
    "    model_subsets = []\n",
    "    for k in range(len(X.columns)):\n",
    "        k=k+1\n",
    "        subset = get_models(k, X, y)\n",
    "        model_subsets += [subset]\n",
    "        print('Best subset selected: k = {}/{}, done'.format(k, len(X.columns)))\n",
    "    # Get best rss score for each subset\n",
    "    return [min_rss(m) for m in model_subsets]\n",
    "\n",
    "def mse(y_hat, y):\n",
    "    \"\"\"Calculate Mean Squared Error\"\"\"\n",
    "    return np.sum(np.square(y_hat - y)) / y.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best subset selected: k = 1/14, done\n",
      "Best subset selected: k = 2/14, done\n",
      "Best subset selected: k = 3/14, done\n",
      "Best subset selected: k = 4/14, done\n",
      "Best subset selected: k = 5/14, done\n",
      "Best subset selected: k = 6/14, done\n",
      "Best subset selected: k = 7/14, done\n",
      "Best subset selected: k = 8/14, done\n",
      "Best subset selected: k = 9/14, done\n",
      "Best subset selected: k = 10/14, done\n",
      "Best subset selected: k = 11/14, done\n",
      "Best subset selected: k = 12/14, done\n",
      "Best subset selected: k = 13/14, done\n",
      "Best subset selected: k = 14/14, done\n"
     ]
    }
   ],
   "source": [
    "# Create index for 50% holdout set\n",
    "np.random.seed(1)\n",
    "train = np.random.rand(len(hitters_pd)) < 0.5\n",
    "\n",
    "response   = 'Salary'\n",
    "\n",
    "# Assign train and test sets\n",
    "X_train = hitters_pd[train].loc[:, 'AtBat':'PutOuts']\n",
    "X_test  = hitters_pd[~train].loc[:, 'AtBat':'PutOuts']\n",
    "y_train = hitters_pd[train][response]\n",
    "y_test  = hitters_pd[~train][response]\n",
    "\n",
    "# Perform best_subset selection\n",
    "best_subsets = best_subset_rss(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>178139.358427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>165215.400785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>170773.977180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>154020.720967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>147281.971965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>148387.589992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>149105.946391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>138481.274569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>133463.544833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>134906.763925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>135302.429278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>135852.565942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>136032.669566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>136072.224129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    predictors            MSE\n",
       "0            1  178139.358427\n",
       "1            2  165215.400785\n",
       "2            3  170773.977180\n",
       "3            4  154020.720967\n",
       "4            5  147281.971965\n",
       "5            6  148387.589992\n",
       "6            7  149105.946391\n",
       "7            8  138481.274569\n",
       "8            9  133463.544833\n",
       "9           10  134906.763925\n",
       "10          11  135302.429278\n",
       "11          12  135852.565942\n",
       "12          13  136032.669566\n",
       "13          14  136072.224129"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl4VeW59/HvvTMSIAlTEiABBAJIFBAjQ3ucFdBaUasWK8qxtGqrVmv7ntr2fe057Tm97GjVVlurVhQVh1q1VkWKWrVH0CCzoKQqEOYxDCGEJPf7x17RDSYkgeysneT3ua59Ze1nTfdG8JdnrWevx9wdERGReIqEXYCIiLR/ChsREYk7hY2IiMSdwkZEROJOYSMiInGnsBERkbhT2IiISNzFLWzM7AEz22xmy2LaRpnZPDNbZGYlZjYmaDczu9PMSs1siZmNjtlnmpmtCl7TYtpPNLOlwT53mpkF7d3NbE6w/Rwz6xavzygiIk0Tz57Ng8CkQ9p+DvyXu48Cbg3eA5wDFAavq4F7IBocwI+AscAY4Ecx4XFPsG3dfnXnugWY6+6FwNzgvYiIhCg5Xgd299fNbMChzUBmsJwFrA+WJwMPefRxBvPMLNvMegOnAXPcfTuAmc0BJpnZa0Cmu78VtD8EXAC8GBzrtOC4M4DXgO81Vm/Pnj19wIBDyxURkcNZsGDBVnfv1dh2cQubBtwEzDazXxLtVX0uaO8LrI3ZrixoO1x7WT3tALnuvgHA3TeYWU5TChswYAAlJSXN+zQiIh2cma1uynatPUDgG8C33b0A+DZwf9Bu9WzrR9DeLGZ2dXDvqGTLli3N3V1ERJqotcNmGvB0sPwk0fswEO2ZFMRsl0/0Etvh2vPraQfYFFyCI/i5uaFi3P1edy929+JevRrtBYqIyBFq7bBZD5waLJ8BrAqWnwOuDEaljQPKg0ths4EJZtYtGBgwAZgdrNttZuOCUWhXAs/GHKtu1Nq0mHYREQlJ3O7ZmNljRG/U9zSzMqKjyr4O3GFmyUAl0dFkAC8A5wKlQAVwFYC7bzeznwDvBNv9uG6wANFLcg8CnYgODHgxaL8NeMLMpgNrgEvi9BFFRKSJTPPZRBUXF7sGCIiINI+ZLXD34sa20xMEREQk7hQ2IiISdwqbo/Tmqq3c/Vpp2GWIiCQ0hc1RemPVFn718gds2lUZdikiIglLYXOUvjK2HzW1zqy31za+sYhIB6WwOUr9e3TmlCG9eOztNVTX1IZdjohIQlLYtICpY/uxcVclc1c2+LACEZEOTWHTAs4YlkPvrHRmzmvS8+hERDochU0LSE6KMOWkfryxaiurt+0NuxwRkYSjsGkhU8YUkBQxHp2/JuxSREQSjsKmheRmpjNheC5PlKyl8kBN2OWIiCQUhU0LmjquPzsqDvDisg1hlyIiklAUNi1o/MAeHNOzMzPn6VKaiEgshU0LikSMy8f2Y8HqHazYsCvsckREEobCpoVdfGI+ackRHpmvYdAiInUUNi0sOyOV80b04S/vrmPP/uqwyxERSQgKmziYOq4fe6tqeGbhurBLERFJCAqbOBhVkM3w3pnMnLcazYQqIqKwiQszY+q4/qzcuJt31+wIuxwRkdApbOJk8qg+dElL5hENgxYRUdjES+e0ZC4a3Zfnl25gx96qsMsREQmVwiaOLh/bn6rqWp5coInVRKRjU9jE0dC8rpw0oBuPzF9Dba0GCohIx6WwibOp4/qzelsFb5ZuDbsUEZHQKGzibNJxefTonKonCohIh6awibO05CQuKS7g7ys2s7G8MuxyRERCobBpBV8Z049adx57W8OgRaRjUti0gn49MjilsBez3lnDgZrasMsREWl1CptWMnVcfzbt2s/cFZvCLkVEpNUpbFrJGcNy6JOVziPzdSlNRDoehU0rSYoYl43pxxurtvLR1r1hlyMi0qoUNq3oyycVkBwxHtUwaBHpYBQ2rSgnM50JRbk8uaCMygM1YZcjItJqFDatbOrY/uysOMDflmwIuxQRkVajsGll4wf1YGCvzszUpTQR6UAUNq3MzLh8bH8WrtnJ8vXlodVRVV3Ld55YzC9mrwytBhHpOOIWNmb2gJltNrNlMW2Pm9mi4PWxmS2KWfd9Mys1s/fNbGJM+6SgrdTMbolpP8bM5pvZquC4qUF7WvC+NFg/IF6f8UhdPDqftORIaMOgq2tquenxhfz53TIeePNj3T8SkbiLZ8/mQWBSbIO7f9ndR7n7KODPwNMAZjYcmAIUBfvcbWZJZpYE/A44BxgOXBZsC/Az4HZ3LwR2ANOD9unADncfDNwebJdQsjJS+OLIPjyzcB27Kw+06rlra53/+PMSXli6kS8c35t9B2p4Y5WeSC0i8RW3sHH314Ht9a0zMwMuBR4LmiYDs9x9v7t/BJQCY4JXqbt/6O5VwCxgcrD/GcBTwf4zgAtijjUjWH4KODPYPqFMHdefiqoanlm4rtXO6e7c+twynn53HTefPYTfTBlF1/RkZi/f2Go1iEjHFNY9m5OBTe6+KnjfF4idzrIsaGuovQew092rD2k/6FjB+vJg+88ws6vNrMTMSrZs2XLUH6o5RuZncVzfTGbOW4N7/CdWc3due3ElM+et4ZpTB3LDGYNJSYpw5rAc5q7YRLWe2SYicRRW2FzGp70agPp6Hn4E7Yc71mcb3e9192J3L+7Vq9dhym15ZsbUsf15f9NuFqzeEffz3fVKKX94/UOuGNefWyYNo66zN7Eojx0VB3j743o7oSIiLaLVw8bMkoGLgMdjmsuAgpj3+cD6w7RvBbKDY8W2H3SsYH0WDVzOC9v5o/rQNS2ZmfPiOwz6vjc+5NdzPuCi0X35r/OLiL2qeOrQXqQlR3h5uR4QKiLxE0bP5ixgpbuXxbQ9B0wJRpIdAxQCbwPvAIXByLNUooMInvPodadXgYuD/acBz8Yca1qwfDHwirfGdaojkJGazEWj+/LC0o1s27M/Lud4dP4a/vtvKzj3+Dx+/qURRCIHd/wyUpM5ubAXLy/f2CqX80SkY4rn0OfHgLeAoWZWZmZ1o8WmcPAlNNx9OfAE8B7wEnCdu9cE91yuB2YDK4Angm0BvgfcbGalRO/J3B+03w/0CNpvBm4hgV0+rj9VNbU8uaCs8Y2b6ZmF6/jhM0s5fWgvfvPlE0hOqv8/98SiXNaXV7Js3a4Wr0FEBMD022xUcXGxl5SUhHLuS//wFhvLK3ntu6d9pudxpF5atpHrHn2XkwZ048GrxpCektTgtjv2VlH8P3/nG6cO4rsTh7bI+UWkYzCzBe5e3Nh2eoJAApg6rj9rtlfw+qqWGRH3jw+28K3HFjIiP4v7pp102KAB6NY5lTEDumsItIjEjcImAUwqyqNnl9QWeaLA/A+3cc3DJQzO6cKD/z6GLmnJje8ETCjKZdXmPXy4Zc9R1yAiciiFTQJITY5waXEBc1dsYv3OfUd8nEVrdzJ9Rgl9szvx0PQxZGWkNHnfCUV5AMzWqDQRiQOFTYK4bEw/HJj19pH1blZs2MW0B96mW+cUHvnaOHp2SWvW/n2zO3F83yxdShORuFDYJIiC7hmcNqQXs95Zy4Fmfpv/wy17uOL++XRKSeLRr40jLyv9iGqYWJTLorU72bSr8oj2FxFpiMImgUwd15/Nu/cz572mX8pau72Cy++bjzvM/NpYCrpnHPH5JwaX0l5uxvlFRJpCYZNAThuaQ9/sTjzSxInVNu2q5PL75rN3fzUPTx/L4JwuR3X+wTldGNizMy/rUpqItDCFTQJJihhfGduPf5Zua3RU2LY9+5l633y27dnPjK+OYXifzKM+v5kxoSiPt/61jfKK1p36QETaN4VNgrmkOJ/kiB12GHT5vgNc+cDbrNlewX3TTuKEft1a7PwTi3KprnVeeV+X0kSk5ShsEkxO13QmHpfHUwvK6p1Bc+/+aq7609t8sGk3v7/iRMYPqnf2hCM2Mj+bnK5pzF6msBGRlqOwSUBTx/anfN8B/rp4/UHtlQdq+PpDJSxau5M7p5zA6UNzWvzckYgxoSiXf3ywRdNFi0iLUdgkoHEDuzM4p8tBl9Kqqmv55iPv8r//2sYvLxnJOcf3jtv5JxblabpoEWlRCpsEZGZcPrYfi9buZNm6cmpqnW8/vohXVm7mvy84jotG58f1/OMG9iBT00WLSAtq2oOzpNVdNDqfn720kpnzVlNd6/xt6QZ+cO4wpo7rH/dzpyRFOPPY3E+mi25oagIRkabS/0USVFanFM4f2YdZ76zlqQVl3HhmIVefMqjVzj+xKFfTRYtIi1HYJLArxw8gJcn4+snHcNNZha167lOGaLpoEWk5CpsEdlzfLEr+79n88AvDMWuZSdWaStNFi0hLUtgkuKxOTZ8moKXVTRe9dF15aDWISPugsJEGnXVsLkkR06U0ETlqChtpkKaLFpGWorCRw5qo6aJFpAUobOSwNF20iLQEhY0cVp/sTozI13TRInJ0FDbSqAnDo9NFbyzXdNEicmQUNtKouumi56zQpTQROTIKG2mUposWkaOlsJFGabpoETlaChtpEk0XLSJHQ2EjTTIyP5vcTE0XLSJHRmEjTRKJGBOG52m6aBE5IgobabIJRbnsO1DD6x9sCbsUEWljFDbSZHXTRb/8ni6liUjzKGykyQ6dLlpEpKkUNtIsmi5aRI6EwkaaRdNFi8iRiFvYmNkDZrbZzJYd0n6Dmb1vZsvN7Ocx7d83s9Jg3cSY9klBW6mZ3RLTfoyZzTezVWb2uJmlBu1pwfvSYP2AeH3GjigjNZlThmi6aBFpnnj2bB4EJsU2mNnpwGRghLsXAb8M2ocDU4CiYJ+7zSzJzJKA3wHnAMOBy4JtAX4G3O7uhcAOYHrQPh3Y4e6DgduD7aQFTSzK03TRItIscQsbd38dOPTC/jeA29x9f7DN5qB9MjDL3fe7+0dAKTAmeJW6+4fuXgXMAiabmQFnAE8F+88ALog51oxg+SngzGB7aSFnDsshKWKadkBEmqy179kMAU4OLm/9w8xOCtr7AmtjtisL2hpq7wHsdPfqQ9oPOlawvjzYXlpI3XTRum8jIk3V2mGTDHQDxgH/B3gi6HXU1/PwI2inkXUHMbOrzazEzEq2bNEXFZtD00WLSHO0dtiUAU971NtALdAzaC+I2S4fWH+Y9q1AtpklH9JO7D7B+iw+ezkPAHe/192L3b24V69eLfDxOg5NFy0izdHaYfMM0XstmNkQIJVocDwHTAlGkh0DFAJvA+8AhcHIs1Sigwie8+gwqFeBi4PjTgOeDZafC94TrH/FNWyqxWm6aBFpjngOfX4MeAsYamZlZjYdeAAYGAyHngVMC3o5y4EngPeAl4Dr3L0muOdyPTAbWAE8EWwL8D3gZjMrJXpP5v6g/X6gR9B+M/DJcGlpWROL8jRdtIg0iemX/qji4mIvKSkJu4w2pXTzbs769ev8ZHIRV4wfEHY5IhICM1vg7sWNbacnCMgRG5zTlYG9Ouu+jYg0SmEjR2XC8DzmfajpokXk8BQ2clQ0XbSINIXCRo6KposWkaZQ2MhR0XTRItIUhw0bM5sas/z5Q9ZdH6+ipG2ZWJSn6aJF5LAa69ncHLN81yHrvtrCtUgbNXZgd7I6pWhUmog0qLGwsQaW63svHVRKUoQzh+Uwd6WmixaR+jUWNt7Acn3vpQObUJTLTk0XLSINSG5k/TAzW0K0FzMoWCZ4PzCulUmbEjtd9OcG9Qy7HBFJMI2FzbGtUoW0ebHTRf/oi8PRfHUiEuuwl9HcfXXsC9gDjAZ6Bu9FPqHpokWkIY0NfX7ezI4LlnsDy4iOQnvYzG5qhfqkDTnrWE0XLSL1a2yAwDHuvixYvgqY4+5fBMaioc9yiOyMVMYe011DoEXkMxoLm9inK54JvADg7ruJzrIpcpCJRXmUbt7DvzRdtIjEaCxs1prZDWZ2IdF7NS8BmFknICXexUnbc/bwXABeVu9GRGI0FjbTgSLg34Evu/vOoH0c8Kc41iVtlKaLFpH6HHbos7tvBq6tp/1V4NV4FSVt28SiPH4x+302lleSl5UedjkikgAOGzZm9tzh1rv7+S1bjrQHE4ty+cXs95nz3kZNFy0iQONf6hwPrAUeA+aj56FJE8ROF62wERFo/J5NHvAD4DjgDuBsYKu7/8Pd/xHv4qTtmlgUnS56Y3ll2KWISAJo7AkCNe7+krtPIzoooBR4zcxuaJXqpM26tLiAlKQI1z/6LlXVGiUv0tE1OlOnmaWZ2UXATOA64E7g6XgXJm3bMT078/OLR1Cyegc/fWFF2OWISMgaGyAwg+gltBeB/4p5moBIo744sg+L1u7k/jc/YmRBFheekB92SSISksYGCFwB7AWGAN+KeZKvAe7umXGsTdqBW84ZxtJ15Xz/6aUMzc1keB/9lRHpiBq7ZxNx967BKzPm1VVBI02RkhThd18ZTVanFK6duYDyigON7yQi7U6j92xEjlavrmncffmJbCjfx02PL6S2VpO8inQ0ChtpFSf278at5w3n1fe3cOcrq8IuR0RamcJGWs3Ucf25aHRf7pi7ildXbg67HBFpRQobaTVmxk8vPJ5j8zK5cdZCVm/bG3ZJItJKFDbSqtJTkvjDFSdiZlw78132VdWEXZKItAKFjbS6gu4Z3DFlFCs37uIHf1mKuwYMiLR3ChsJxWlDc/j2WUP4y8J1PPTW6rDLEZE4U9hIaK4/fTBnHZvDT55/j5KPt4ddjojEkcJGQhOJGL+6dBT53TrxzUfeZfNuPSFapL2KW9iY2QNmttnMlsW0/aeZrTOzRcHr3Jh13zezUjN738wmxrRPCtpKzeyWmPZjzGy+ma0ys8fNLDVoTwvelwbrB8TrM8rRy+qUwu+vOJHdldVc/8hCDtToCdEi7VE8ezYPApPqab/d3UcFrxcAzGw4MAUoCva528ySzCwJ+B1wDjAcuCzYFuBnwbEKgR3A9KB9OrDD3QcDtwfbSQIblpfJbV86nrc/3q4nRIu0U3ELG3d/HWjqhfjJwCx33+/uHxGdN2dM8Cp19w/dvQqYBUy26BNBzwCeCvafAVwQc6wZwfJTwJkW8wRRSUyTR/Xlqs8P4E///JhnF60LuxwRaWFh3LO53syWBJfZugVtfYlOP12nLGhrqL0HsNPdqw9pP+hYwfryYHtJcD8491jGDOjOLX9eysqNu8IuR0RaUGuHzT3AIGAUsAH4VdBeX8/Dj6D9cMf6DDO72sxKzKxky5Yth6tbWkFKUoTfXn4CXdOTuebhBZTv0xOiRdqLVg0bd98UTDVdC/yR6GUyiPZMCmI2zQfWH6Z9K5BtZsmHtB90rGB9Fg1cznP3e9292N2Le/XqdbQfT1pATtd07pk6mnU79nHz44v0hGiRdqJVw8bMese8vRCoG6n2HDAlGEl2DFAIvA28AxQGI89SiQ4ieM6jXzl/Fbg42H8a8GzMsaYFyxcDr7i+ot6mnNi/O//vvOHMXbmZ375aGnY5La5GASodUGMzdR4xM3sMOA3oaWZlwI+A08xsFNHLWh8D1wC4+3IzewJ4D6gGrnP3muA41wOzgSTgAXdfHpzie8AsM/tvYCFwf9B+P/CwmZUS7dFMiddnlPi5cnx/Fq3dye1//4Dj87M4fWhO2CU1mbuzbW8Vq7dVsGb7XtZs28fq7XtZs62C1dsr2LpnP7/58igmj+rb+MFE2gnTL/1RxcXFXlJSEnYZEmNfVQ0X3fO/rN+5j79e/2/065ERdkmfqK6pZd3OfazZXhGESgWrt+1l9bYK1m6vYO8hDxjtnZVOv+4Z9OuewYLVO6iudeZ+51RSkvS9amnbzGyBuxc3tl3cejYiR6tTahK/nzqaL971JtfOXMCfv/E5OqUmtdr59+yvZk3QO6kLlLpwWbdz30GXw1KTI/TrnkH/7hmMG9iD/j0y6N8jg37dO5PfrRPpKZ/WPee9TXz9oRKeWbiOS4oL6ju1SLujnk1APZvE9erKzXx1xjtceEJffnXJSOLxtamN5ZUsWruTJWU7WVJWzooNu9i2t+qgbbplpNCvR+dPQqVfj09/5nZNJxJpWl3uznl3vcme/dXMvflUktW7kTZMPRtpN04flsONZxbym7+v4oSCbK4YP+CojldecYAl63ayeO1OFpeVs3jtTjbv3g9AcsQY1rsrZx2by4CenYPeSTRQMtNTWuDTRCeRu/HMQq5+eAHPLFrPxSfmt8hxRRKZwkbahG+dUcjSsnJ+/Px7DO+TxYn9uzW+E1B5oIbl68tZvLacxUGv5aOtn84QOrBXZz4/uCcj87MYUZDN8N6ZB13yipezh+cyvHcmd72yigtG9VHvRto9XUYL6DJa4ivfd4Dzf/smlQdq+OsN/0ZO1/SD1lfX1PLBpj0sKfu0x/L+pt2f3FvJy0xnZEEWI/KzGVWQzXF9s8jq1DK9lSMxe/lGrnl4Ab+6ZCRfUu9G2qimXkZT2AQUNm3Dig27uPDufzIiP5vbLjqepevKWRIEy7L15VQeiD41OjM9mZEF2YzMz2ZEfhYjC7LJzUxv5Oity935wp1vUlFVzd9170baKIVNMyls2o5nF63jxlmLPnmflhzhuL5ZjMjPYlRBNiPysxnQIyMuAwla2kvLNnLtzAX8+tKRXDRavRtpezRAQNqtyaP6UuvOvqpaRhZkMSS3a5v9vsqE4bkc2zuTu14p5fyRuncj7Zf+ZkubdOEJ+XxlbD+K+mS12aCB6GylN545mI+27uWvS9Y3voNIG9V2/5WKtBMThucxLK8rd80t1XPTpN1S2IiELNq7KeTDrXv562L1bqR9UtiIJICJRdHezZ2vrFLvRtolhY1IAohEjG+dWciHW/byvO7dSDuksBFJEJOK8hia25U75qp3I+2PwkYkQah3I+2ZwkYkgZxzXB5Dcrtw1ysamSbti8JGJIHU9W5KN+/hb0s3hF2OSItR2IgkmHOP601hThfu0r0baUcUNiIJpq53s2rzHl5Q70baCYWNSAI69/ho7+bOuauoVe9G2gGFjUgCSooYN9T1bpapdyNtn8JGJEF94fjeDFbvRtoJhY1IgkqKGDecMZgPNu3hxWUbwy5H5KgobEQS2Hkj+jCoV2f1bqTNU9iIJLCkYGTa+5t289Jy9W6k7VLYiCS480b0YaB6N9LGKWxEElxSxPjWGYWs3Lib2erdSBulsBFpA744sg8De3bmDvVupI1S2Ii0AdHv3Qxm5cbdvPyeejfS9ihsRNqIL46o692UqncjbY7CRqSNSE6KcP0Zg1mxYRcvv7cp7HJEmkVhI9KGnD+yD8f0jI5Mc1fvRtoOhY1IG5KcFOH60wfznno30sYobETamMmj+jCgRwZ3/F29G2k7FDYibUz03k0h723YxRz1bqSNiFvYmNkDZrbZzJbVs+67ZuZm1jN4b2Z2p5mVmtkSMxsds+00M1sVvKbFtJ9oZkuDfe40Mwvau5vZnGD7OWbWLV6fUSQsF4zqQ/8eGdyhezfSRsSzZ/MgMOnQRjMrAM4G1sQ0nwMUBq+rgXuCbbsDPwLGAmOAH8WExz3BtnX71Z3rFmCuuxcCc4P3Iu1K3b2b5et38fcVm8MuR6RRcQsbd38d2F7PqtuB/wBifx2bDDzkUfOAbDPrDUwE5rj7dnffAcwBJgXrMt39LY/+WvcQcEHMsWYEyzNi2kXalQtP6Bv0bj5Q70YSXqveszGz84F17r74kFV9gbUx78uCtsO1l9XTDpDr7hsAgp85LfYBRBJIclKE604fzLJ1u5ir3o0kuFYLGzPLAH4I3Frf6nra/Ajam1vT1WZWYmYlW7Zsae7uIqG78IS+9OuuezeS+FqzZzMIOAZYbGYfA/nAu2aWR7RnUhCzbT6wvpH2/HraATYFl9kIfjb4K5+73+vuxe5e3KtXr6P4aCLhSAnu3SxdV84rK9W7kcTVamHj7kvdPcfdB7j7AKKBMdrdNwLPAVcGo9LGAeXBJbDZwAQz6xYMDJgAzA7W7TazccEotCuBZ4NTPQfUjVqbFtMu0i5dOLovBd07qXcjCS2eQ58fA94ChppZmZlNP8zmLwAfAqXAH4FvArj7duAnwDvB68dBG8A3gPuCff4FvBi03wacbWariI56u60lP5dIoklJinDD6YUsKSvn1ffVu5HEZPpNKKq4uNhLSkrCLkPkiByoqeWMX71G94xUnrnu8wRfOxOJOzNb4O7FjW2nJwiItAN1924Wl5Xz2vsa7CKJR2Ej0k5cNDqf/G6d+Mnz77F5V2XY5YgcRGEj0k6kJEX45SUj2birkkv/8BZlOyrCLknkEwobkXZk3MAezPzaWLbvreLS37/Fh1v2hF2SCKCwEWl3RvfrxmNXj2N/dS2X/mEeKzbsCrskEYWNSHtU1CeLx68ZT3LEmHLvPBat3Rl2SdLBKWxE2qnBOV148trxZHVK4fI/zmPeh9vCLkk6MIWNSDtW0D2DJ68dT5/sTkx74G196VNCo7ARaedyM9N5/JrxFOZ24eqHSnhh6YawS5IOSGEj0gF075zKo18fx8j8bK5/9F2eWlDW+E4iLUhhI9JBZKan8ND0MXxuUE++++RiHnrr47BLkg4kOewCRKT1ZKQmc9+0Ym54bCG3PrucPfur+eZpg8MuSwI1tU7lgRr2V9dSeaAmeNVSXVtLrUOtO+4eXa6N/vzkvXuwvm6ZT943ts3JhT3pndUprp9NYSPSwaSnJHH35aP57pOL+flL77Onspr/M3GoHt7ZDO7OjooDbNpVycZdlWzZtZ89+6uprI6Gw/7qGvYfqP1McHy6XEtlsM3+mH0O1ITzYOQHrzpJYSMiLS8lKcKvLx1FRmoSd7/2Lyqqarj1vOFEIgqcfVU1bNxVyaaY18by/Z8u76pk8679VNXUNniMtOQIackR0lOSSE9JilmOkJGaTPfOEdJi25Oj69KCn4fukxyJkBQxzCBiFrzAgp+RSOz7YJmY7SMcsk+wn0W36dklLe5/rgobkQ4qKWL89MLj6ZyazH1vfsSe/dX87EsjSGqngVNdU8vWPVUxgRH9ubF8P5t3V7KxPBomuyqrP7NvRmoSeZnp5GamU9y/G7lZ6eR2TScvK9qW0zWNzPQU0lIipCZFFNr1UNgeR47LAAAJGElEQVSIdGBmxg+/cCyd05K5Y+4q9lXVcPuXR5GanNhjh6qqa9m5r4odew+wo6KKnRVV7KioWz7Ajr1V7DikbWdFFbWHXKVKihg5XdPIyUxnYK/OfG5QD3Iy0z8JlrysNHIz0+mSlqzLjEdJYSPSwZkZ3z57CF3SkvmfF1ZQUVXNPVNPJD0lqdVqcHdWb6tgzfaKTwMj+Lk9CI7Ytj37P9v7qJOWHKFbRirZGSl0y0hlWF5XumWk0r1zKrl1IZKZTm5WGj06p7XbnlyiUdiICABfP2UgndOS+eEzS7nqT+/wx2nFdEmL3/8iyisO8L//2srrq7byxqotlO3Y95ltuqYn0y0jlW4ZKfToksrgnC6fhEi3jBSyM1Kjy53r2lLplNp6ISlNp7ARkU98ZWw/MlKT+M6Ti5l633xmXDWGrIyUFjn2gZpaFq3dyRsfbOH1VVtZUraTWoeuacmMH9SDa04dxNDcrnTvHA2R7E4pJCcl9uU8aTqFjYgc5IIT+tIpNYkbHl3Il+99i4enj6VX1+aPVnJ3Pt5WwRurtvD6B1uZ9+E29uyvJmIwqiCbG84o5JQhPRmZn61Q6QDMPZxx3YmmuLjYS0pKwi5DJGG8sWoLVz+0gN5Z6cz82lj6ZDf+PYyGLo0VdO/EyYW9OKWwJ+MH9SSrU8v0liR8ZrbA3Ysb3U5hE6WwEfmsdz7ezlf/9A6ZnVJ49Otj6d+j80HrG7s0dvKQaMAcup+0HwqbZlLYiNRvaVk5Vz4wn5SkCDO/NpaUpEiDl8ZOLuzFyYU9GVmQTYoujXUICptmUtiINOyDTbuZet98tu2toib4skp+t06cMkSXxjq6poaNBgiISKOG5HblyWvH86d/fsygnC6cPLgn/Xtk6IuO0mQKGxFpkv49OvOf5xeFXYa0UbqoKiIicaewERGRuFPYiIhI3ClsREQk7hQ2IiISdwobERGJO4WNiIjEncJGRETiTo+rCZjZFmB12HU0oCewNewijkBbrRtUe1hUeziOpvb+7t6rsY0UNm2AmZU05dlDiaat1g2qPSyqPRytUbsuo4mISNwpbEREJO4UNm3DvWEXcITaat2g2sOi2sMR99p1z0ZEROJOPRsREYk7hU2CMrMCM3vVzFaY2XIzuzHsmprLzJLMbKGZPR92Lc1hZtlm9pSZrQz+/MeHXVNTmdm3g78vy8zsMTNLD7umhpjZA2a22cyWxbR1N7M5ZrYq+NktzBrr00Ddvwj+viwxs7+YWXaYNTakvtpj1n3XzNzMesbj3AqbxFUNfMfdjwXGAdeZ2fCQa2quG4EVYRdxBO4AXnL3YcBI2shnMLO+wLeAYnc/DkgCpoRb1WE9CEw6pO0WYK67FwJzg/eJ5kE+W/cc4Dh3HwF8AHy/tYtqogf5bO2YWQFwNrAmXidW2CQod9/g7u8Gy7uJ/g+vb7hVNZ2Z5QNfAO4Lu5bmMLNM4BTgfgB3r3L3neFW1SzJQCczSwYygPUh19Mgd38d2H5I82RgRrA8A7igVYtqgvrqdveX3b06eDsPyG/1wpqggT9zgNuB/wDidhNfYdMGmNkA4ARgfriVNMtviP7lrQ27kGYaCGwB/hRcArzPzDqHXVRTuPs64JdEfzvdAJS7+8vhVtVsue6+AaK/cAE5IddzJL4KvBh2EU1lZucD69x9cTzPo7BJcGbWBfgzcJO77wq7nqYws/OAze6+IOxajkAyMBq4x91PAPaSmJdyPiO4vzEZOAboA3Q2s6nhVtWxmNkPiV4CfyTsWprCzDKAHwK3xvtcCpsEZmYpRIPmEXd/Oux6muHzwPlm9jEwCzjDzGaGW1KTlQFl7l7Xi3yKaPi0BWcBH7n7Fnc/ADwNfC7kmpprk5n1Bgh+bg65niYzs2nAecDl3na+UzKI6C8ni4N/r/nAu2aW19InUtgkKDMzovcNVrj7r8Oupznc/fvunu/uA4jeoH7F3dvEb9juvhFYa2ZDg6YzgfdCLKk51gDjzCwj+PtzJm1kcEOM54BpwfI04NkQa2kyM5sEfA84390rwq6nqdx9qbvnuPuA4N9rGTA6+HfQohQ2ievzwBVEewWLgte5YRfVQdwAPGJmS4BRwE9DrqdJgt7YU8C7wFKi/74T9lvtZvYY8BYw1MzKzGw6cBtwtpmtIjo66rYwa6xPA3X/FugKzAn+rf4+1CIb0EDtrXPuttPbExGRtko9GxERiTuFjYiIxJ3CRkRE4k5hIyIicaewERGRuFPYiCQYM9sT/OxjZk81su1NwbfARRKahj6LtAIzS3L3miZuu8fduzRx24+JPuV5azxqEWkp6tmIHCUzGxDMZTIjmM/kqeBb/B+b2a1m9iZwiZkNMrOXzGyBmb1hZsOC/Y8xs7fM7B0z+8khx10WLCeZ2S/NbGlwjhvM7FtEn4H2qpm9Gmx3WbDNMjP7Wcyx9pjZj81sPjDezG4zs/eCY/2yNf+8pGNKDrsAkXZiKDDd3f9pZg8A3wzaK9393wDMbC5wrbuvMrOxwN3AGUTnz7nH3R8ys+saOP7VRJ9hdYK7V5tZd3ffbmY3A6e7+1Yz6wP8DDgR2AG8bGYXuPszQGdgmbvfambdiT4KaZi7e6JO9CXti3o2Ii1jrbv/M1ieCfxbsPw4fPL07s8BT5rZIuAPQO9gm88DjwXLDzdw/LOA39fNmeLu9c1JchLwWvAgzronD58SrKsh+lBXgF1AJXCfmV0EtJlneUnbpZ6NSMs49OZn3fu9wc8IsNPdRzVx/0NZE7dpSGXdfZqgZzSG6IM6pwDXE+1hicSNejYiLaOfmY0Pli8D3oxdGcxF9JGZXQLRp3qb2chg9T/5dPrmyxs4/svAtcEMnASXwgB2E30AJEQn1zvVzHqaWVJQxz8OPVDQy8py9xeAm4g+bFQkrhQ2Ii1jBTAteFJ0d+Ceera5HJhuZouB5UQnOgO4EbjOzN4Bsho4/n1EpxBYEuz/laD9XuBFM3s1mNny+8CrwGLgXXev7xH9XYHng1r/AXy7eR9VpPk09FnkKAXTdj/v7seFXIpIwlLPRkRE4k49GxERiTv1bEREJO4UNiIiEncKGxERiTuFjYiIxJ3CRkRE4k5hIyIicff/ARY6inOWXm5uAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Using 50% hold out set\n",
    "# Measure mean squared error for each model selected by best subset selection\n",
    "MSEs = []\n",
    "for m in best_subsets:\n",
    "    model = m[1]\n",
    "    y_hat = model.predict(X_test)\n",
    "    r     = mse(y_hat, y_test)\n",
    "    MSEs += [r]\n",
    "\n",
    "results = pd.DataFrame({'predictors': np.arange(1, 15), 'MSE': MSEs})\n",
    "display(results)\n",
    "sns.lineplot(x='predictors', y='MSE', data=results);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above we performed best subset selection up to 14 predictors using the training set createad from a 50% hold-out set. We then use the test set to measure the MSE for the selected model in each subset of p predictors. \n",
    "\n",
    "We observe the lowest MSE for the model with 9 predictors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Salary ~ AtBat + Hits + HmRun + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Which 9 predictors did best subset selection choose based on the TRAINING set?\n",
    "best_subsets[8][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best subset selected: k = 1/14, done\n",
      "Best subset selected: k = 2/14, done\n",
      "Best subset selected: k = 3/14, done\n",
      "Best subset selected: k = 4/14, done\n",
      "Best subset selected: k = 5/14, done\n",
      "Best subset selected: k = 6/14, done\n",
      "Best subset selected: k = 7/14, done\n",
      "Best subset selected: k = 8/14, done\n",
      "Best subset selected: k = 9/14, done\n",
      "Best subset selected: k = 10/14, done\n",
      "Best subset selected: k = 11/14, done\n",
      "Best subset selected: k = 12/14, done\n",
      "Best subset selected: k = 13/14, done\n",
      "Best subset selected: k = 14/14, done\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Salary ~ AtBat + Hits + Runs + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Which 9 predictors does best subset selection choose based on the FULL set?\n",
    "best_subsets_full = best_subset_rss(hitters_pd.loc[:, 'AtBat':'PutOuts'], hitters_pd['Salary'])\n",
    "best_subsets_full[8][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time the same model is selected using the full dataset, as is selected using just the training set. This won't always be the case however, so the ISL authors reccomend taking the model that best subset selection chooses using the full dataset.\n",
    "\n",
    "**Revision note:** My understanind of what the authors seem to be suggesting is:\n",
    "\n",
    "- fit all possible models on training set\n",
    "- choose best model in each subset by RSS\n",
    "- use test MSE to choose best subset, e.g optimal number of predictors *p*\n",
    "- fit all possible models with optimal value of p predictors on *full* dataset\n",
    "- choose best model by RSS\n",
    "\n",
    "This seems odd to me because the model finaly chosen has not been verified on the test set. If the actual predictors are to be chosen on the full set, why not pass this model through validation? Note that this does not mean the model fitted on full dataset is then used in test:\n",
    "\n",
    "- fit all possible models on full dataset\n",
    "- choose best combination of predictors in each subset by RSS\n",
    "- use above choices to fit models on training set\n",
    "- use test MSE to choose best subset\n",
    "- model with lowest MSE is chosen\n",
    "\n",
    "This approach also seems to simplify use of best subset selection with cross-validation:\n",
    "\n",
    "- fit all possible models on full dataset\n",
    "- choose best combination of predictors in each subset by RSS\n",
    "- choose from above predictor subsets using cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_val(formula, X, y, k):\n",
    "    \"\"\"Perform k-fold cross validation to return mean MSE score\n",
    "    Expects formula as Patsy formula\"\"\"\n",
    "    # Split dataset into k-folds\n",
    "    # Note: np.array_split doesn't raise excpetion is folds are unequal in size\n",
    "    X_folds = np.array_split(X, k)\n",
    "    y_folds = np.array_split(y, k)\n",
    "    \n",
    "    MSEs = []\n",
    "    for f in np.arange(len(X_folds)):\n",
    "        # Create training and test sets\n",
    "        X_test  = X_folds[f]\n",
    "        y_test  = y_folds[f]\n",
    "        X_train = X.drop(X_folds[f].index)\n",
    "        y_train = y.drop(y_folds[f].index)\n",
    "        \n",
    "        # Fit model\n",
    "        model = smf.ols(formula=formula, data=pd.concat([X_train, y_train], axis=1)).fit()\n",
    "        \n",
    "        # Measure MSE\n",
    "        y_hat = model.predict(X_test)\n",
    "        MSEs += [mse(y_hat, y_test)]\n",
    "    return (MSEs, formula)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best subset selected: k = 1/14, done\n",
      "Best subset selected: k = 2/14, done\n",
      "Best subset selected: k = 3/14, done\n",
      "Best subset selected: k = 4/14, done\n",
      "Best subset selected: k = 5/14, done\n",
      "Best subset selected: k = 6/14, done\n",
      "Best subset selected: k = 7/14, done\n",
      "Best subset selected: k = 8/14, done\n",
      "Best subset selected: k = 9/14, done\n",
      "Best subset selected: k = 10/14, done\n",
      "Best subset selected: k = 11/14, done\n",
      "Best subset selected: k = 12/14, done\n",
      "Best subset selected: k = 13/14, done\n",
      "Best subset selected: k = 14/14, done\n"
     ]
    }
   ],
   "source": [
    "# Get best subsets of predictors for full dataset\n",
    "X = hitters_pd.loc[:, 'AtBat':'PutOuts']\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "best_subsets_full = best_subset_rss(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE_mean</th>\n",
       "      <th>MSE_folds</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>142840.888022</td>\n",
       "      <td>[52976.57302399576, 80338.281999855, 282759.54...</td>\n",
       "      <td>Salary ~ CRBI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>124466.240992</td>\n",
       "      <td>[60439.110444577454, 44215.5427373037, 210423....</td>\n",
       "      <td>Salary ~ Hits + CRBI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>119812.512799</td>\n",
       "      <td>[71609.19418407278, 42531.326374113065, 189132...</td>\n",
       "      <td>Salary ~ Hits + CRBI + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>116723.518363</td>\n",
       "      <td>[70957.22282838816, 45714.23594875619, 174585....</td>\n",
       "      <td>Salary ~ AtBat + Hits + CRBI + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>112340.754784</td>\n",
       "      <td>[67046.46201253521, 43301.142890066694, 178826...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRBI + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>113132.633435</td>\n",
       "      <td>[88211.72188654834, 36778.67067994044, 188866....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRuns + CWalks...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>110827.847248</td>\n",
       "      <td>[87533.41276121038, 31125.84402692224, 172726....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CHmRun + CRuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>109522.011671</td>\n",
       "      <td>[86417.0694422955, 30716.914843531533, 165276....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>110455.064295</td>\n",
       "      <td>[83928.29301348924, 30475.3767711306, 162469.7...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>111701.134543</td>\n",
       "      <td>[88528.56340836613, 31534.424862039876, 162165...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + Years +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>112521.007737</td>\n",
       "      <td>[88525.39130127529, 32233.417562426817, 162177...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + Years +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>114113.347167</td>\n",
       "      <td>[90156.22832070397, 32996.29665092049, 163856....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + Years +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>114788.299569</td>\n",
       "      <td>[90168.61111067011, 32965.618160201964, 164098...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + Walks +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>116527.299332</td>\n",
       "      <td>[90817.75925294032, 32980.994138704256, 164194...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    predictors       MSE_mean  \\\n",
       "0            1  142840.888022   \n",
       "1            2  124466.240992   \n",
       "2            3  119812.512799   \n",
       "3            4  116723.518363   \n",
       "4            5  112340.754784   \n",
       "5            6  113132.633435   \n",
       "6            7  110827.847248   \n",
       "7            8  109522.011671   \n",
       "8            9  110455.064295   \n",
       "9           10  111701.134543   \n",
       "10          11  112521.007737   \n",
       "11          12  114113.347167   \n",
       "12          13  114788.299569   \n",
       "13          14  116527.299332   \n",
       "\n",
       "                                            MSE_folds  \\\n",
       "0   [52976.57302399576, 80338.281999855, 282759.54...   \n",
       "1   [60439.110444577454, 44215.5427373037, 210423....   \n",
       "2   [71609.19418407278, 42531.326374113065, 189132...   \n",
       "3   [70957.22282838816, 45714.23594875619, 174585....   \n",
       "4   [67046.46201253521, 43301.142890066694, 178826...   \n",
       "5   [88211.72188654834, 36778.67067994044, 188866....   \n",
       "6   [87533.41276121038, 31125.84402692224, 172726....   \n",
       "7   [86417.0694422955, 30716.914843531533, 165276....   \n",
       "8   [83928.29301348924, 30475.3767711306, 162469.7...   \n",
       "9   [88528.56340836613, 31534.424862039876, 162165...   \n",
       "10  [88525.39130127529, 32233.417562426817, 162177...   \n",
       "11  [90156.22832070397, 32996.29665092049, 163856....   \n",
       "12  [90168.61111067011, 32965.618160201964, 164098...   \n",
       "13  [90817.75925294032, 32980.994138704256, 164194...   \n",
       "\n",
       "                                                Model  \n",
       "0                                       Salary ~ CRBI  \n",
       "1                                Salary ~ Hits + CRBI  \n",
       "2                      Salary ~ Hits + CRBI + PutOuts  \n",
       "3              Salary ~ AtBat + Hits + CRBI + PutOuts  \n",
       "4      Salary ~ AtBat + Hits + Walks + CRBI + PutOuts  \n",
       "5   Salary ~ AtBat + Hits + Walks + CRuns + CWalks...  \n",
       "6   Salary ~ AtBat + Hits + Walks + CHmRun + CRuns...  \n",
       "7   Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  \n",
       "8   Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...  \n",
       "9   Salary ~ AtBat + Hits + Runs + Walks + Years +...  \n",
       "10  Salary ~ AtBat + Hits + Runs + Walks + Years +...  \n",
       "11  Salary ~ AtBat + Hits + Runs + Walks + Years +...  \n",
       "12  Salary ~ AtBat + Hits + HmRun + Runs + Walks +...  \n",
       "13  Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEKCAYAAAAmfuNnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOXd9/HPLwk7IUMIBMgiiGFfAkTFpXZxQ6tia2ldWmm1N3Vr692nd9Xau3vv2qe79XGrULHFpfW2ai0q1LpUi7JDAGUREAIh7EnYAkl+zx9zgmMMJIHMnEnyfb9e85oz1zln5je8SL65zrnOdczdERERibeUsAsQEZH2QYEjIiIJocAREZGEUOCIiEhCKHBERCQhFDgiIpIQChwREUkIBY6IiCSEAkdERBIiLewCkklWVpYPGDAg7DJERFqVhQsX7nD33o1tp8CJMWDAABYsWBB2GSIirYqZvdeU7XRITUREEkKBIyIiCaHAERGRhFDgiIhIQihwREQkIRQ4IiKSEAocERFJCAVOC3ht9XZ+99KasMsQEUlqCpwWMG/9Ln7z0hoOHKoJuxQRkaSlwGkBhXkRamqd5VvKwy5FRCRpKXBaQGF+BIAlG/eEXImISPJS4LSArO6dyO3ZhSWbFDgiIkejwGkhhXkRBY6IyDEocFpIYV6EzXsOsK3yYNiliIgkJQVOCxmr8zgiIsekwGkhI/pnkJZiOqwmInIUCpwW0rlDKsP69VDgiIgchQKnBRXmRVhWUk5NrYddiohI0lHgtKDCvAh7q6p5d/vesEsREUk6CpwWpAtARUSOToHTggb26kaPzmks1nkcEZEPiWvgmNl0M9tmZssbWPdNM3Mzywpem5ndbWZrzWyZmY2L2XaKma0JHlNi2sebWXGwz91mZkF7ppnNCbafY2Y94/k966SkGGN0AaiISIPi3cN5GJhYv9HM8oDzgY0xzRcBBcFjKnBfsG0m8D3gdOA04HsxAXJfsG3dfnWfdTvwkrsXAC8FrxOiMC/Cqq0V7D9UnaiPFBFpFeIaOO7+GrCrgVW/Br4FxA7nmgQ84lFvAhEz6wdcCMxx913uvhuYA0wM1vVw97nu7sAjwOUx7zUjWJ4R0x53hXkRah2KSzRztIhIrISfwzGzy4DN7r603qocYFPM65Kg7VjtJQ20A2S7eylA8Nynxb5AIwrzgoEDOqwmIvIBaYn8MDPrCtwJXNDQ6gba/Djam1vTVKKH5cjPz2/u7h/Sq3sn8jI1c7SISH2J7uEMAgYCS81sA5ALLDKzvkR7KHkx2+YCWxppz22gHaAsOORG8LztaAW5+4PuXuTuRb179z6Br/a+wryeChwRkXoSGjjuXuzufdx9gLsPIBoa49x9K/AscG0wWm0CUB4cDnsRuMDMegaDBS4AXgzWVZrZhGB02rXAM8FHPQvUjWabEtOeEIV5EUrLD1JWoZmjRUTqxHtY9GPAXGCImZWY2fXH2HwWsA5YC/weuAnA3XcBPwLmB48fBm0ANwIPBfu8CzwftN8FnG9ma4iOhrurJb9XY+rO4yzWBaAiIkfE9RyOu1/VyPoBMcsO3HyU7aYD0xtoXwCMbKB9J3BuM8ttMSP696BDanTm6Ikj+4ZVhohIUtFMA3Hw/szRu8MuRUQkaShw4qQwL0KxZo4WETlCgRMnhXkR9h2qYc22yrBLERFJCgqcODlyAagGDoiIAAqcuBmY1Y2MLh10PY6ISECBEydmmjlaRCSWAieOCvMirC6rZF+VZo4WEVHgxNHYYOboZZo5WkREgRNPYzRztIjIEQqcOMrs1pGTenXVBaAiIihw4q5QAwdERAAFTtwV5kUoq6iitPxA2KWIiIRKgRNnugBURCRKgRNnw/v3oGNqig6riUi7p8CJs05pqQzr30OBIyLtngInAcbmRSjerJmjRaR9U+AkQGFehP2HalhdppmjRaT9UuAkQKEuABURUeAkwkm9utKzaweNVBORdk2BkwCaOVpERIGTMIV5EVZvq2SvZo4WkXYqroFjZtPNbJuZLY9p+5GZLTOzJWY228z6B+0fM7PyoH2JmX03Zp+JZrbKzNaa2e0x7QPN7C0zW2NmT5hZx6C9U/B6bbB+QDy/Z1MU5kVwh2Ul6uWISPsU7x7Ow8DEem0/d/fR7l4IPAd8N2bdv9y9MHj8EMDMUoH/B1wEDAeuMrPhwfY/A37t7gXAbuD6oP16YLe7nwL8OtguVBo4ICLtXVwDx91fA3bVa6uIedkNaOzilNOAte6+zt0PAY8Dk8zMgE8ATwbbzQAuD5YnBa8J1p8bbB+aSNeODMzqpoEDItJuhXIOx8x+YmabgGv4YA/nDDNbambPm9mIoC0H2BSzTUnQ1gvY4+7V9do/sE+wvjzYPlR1M0e76wJQEWl/Qgkcd7/T3fOAmcAtQfMi4CR3HwP8Dng6aG+oZ+LHaD/WPh9iZlPNbIGZLdi+fXtTv8JxKcyLsK2yitLyg3H9HBGRZBT2KLVHgSsgeqjN3fcGy7OADmaWRbTnkhezTy6wBdgBRMwsrV47sfsE6zOod2ivjrs/6O5F7l7Uu3fvlvxuH6LzOCLSniU8cMysIOblZcA7QXvfuvMsZnZaUNtOYD5QEIxI6whcCTzr0eNSLwOfCd5rCvBMsPxs8Jpg/T89CY5jDevXg45pmjlaRNqntMY3OX5m9hjwMSDLzEqA7wEXm9kQoBZ4D7gh2PwzwI1mVg0cAK4MQqLazG4BXgRSgenuviLY5zbgcTP7MbAYmBa0TwP+aGZrifZsrozn92yqjmkpjOjfQwMHRKRdimvguPtVDTRPa6ANd78HuOco62YBsxpoX0d0FFv99oPA5GYVmyCFeREen7eJ6ppa0lLDPqIpIpI4+o2XYIV5EQ4crmGVZo4WkXZGgZNgY/N6Aho4ICLtjwInwfIyu5DZraPO44hIu6PASTAzO3IBqIhIe6LACUFhXoS12/dSefBw2KWIiCSMAicE788cXR52KSIiCaPACcEYzTggIu2QAicEGV06cHLvbizWwAERaUcUOCHRzNEi0t4ocEIyNi/Cjr1VbN5zIOxSREQSQoETkkJdACoi7YwCJyRD+6XTKS1FF4CKSLuhwAlJh9QURuZkqIcjIu2GAidEhXkRijeXc7imNuxSRETiToETosK8CFXVtazaqpmjRaTtU+CEqO6W04t1WE1E2gEFTohye3ahl2aOFpF2QoETovdnjt4ddikiInGnwAlZYV6Ed7fvo/yAZo4WkbZNgROywvzoeZxlJTqsJiJtmwInZKNzg5mjdR5HRNq4uAaOmU03s21mtjym7UdmtszMlpjZbDPrH7Sbmd1tZmuD9eNi9pliZmuCx5SY9vFmVhzsc7eZWdCeaWZzgu3nmFnPeH7PE5HRpQODenfTBaAi0ubFu4fzMDCxXtvP3X20uxcCzwHfDdovAgqCx1TgPoiGB/A94HTgNOB7MQFyX7Bt3X51n3U78JK7FwAvBa+TVmFeT80cLSJtXpMDx8w6mdnVZvZtM/tu3eNY+7j7a8Cuem0VMS+7AXW/ZScBj3jUm0DEzPoBFwJz3H2Xu+8G5gATg3U93H2uR39TPwJcHvNeM4LlGTHtSakwP8LOfYco2a2Zo0Wk7UprxrbPAOXAQqDqRD7UzH4CXBu838eD5hxgU8xmJUHbsdpLGmgHyHb3UgB3LzWzPseoZSrRXhL5+fnH+Y1OzNiYC0DzMruGUoOISLw1J3By3b3+4bHj4u53Anea2R3ALUQPmVlDmx5He3NreRB4EKCoqCiUY1pD+r4/c/RlY/qHUYKISNw15xzOv81sVAt//qPAFcFyCZAXsy4X2NJIe24D7QBlwSE3gudtLVx3i+qQmsKonAxdACoibVpzAudsYKGZrQpGkRWb2bLmfqCZFcS8vAx4J1h+Frg2GK02ASgPDou9CFxgZj2DwQIXAC8G6yrNbEIwOu1aoof96t6rbjTblJj2pFWYF2H5lgoOVWvmaBFpm5pzSO2i5r65mT0GfAzIMrMSoofOLjazIUAt8B5wQ7D5LOBiYC2wH/gSgLvvMrMfAfOD7X7o7nUDEW4kOhKuC/B88AC4C/izmV0PbAQmN7f2RCvMj/DQ6+t5Z2vFkWtzRETakiYHjru/BxCcgO/cxH2uaqB52lG2deDmo6ybDkxvoH0BMLKB9p3AuU2pMVnUzRy9ZNMeBY6ItEnNGRZ9mZmtAdYDrwIbeL9HIScoJ9KFrO6dNOOAiLRZzTmH8yNgArDa3QcS7UG8EZeq2qH3Z45W4IhI29ScwDkcHKpKMbMUd38ZKIxTXe3S2PwI63bso3y/Zo4WkbanOYGzx8y6A/8CZprZb4Hq+JTVPh05j6OZo0WkDWpO4EwiOnrsVuAF4F3g0ngU1V6Nzs3ATDNHi0jb1JxRavvM7CSgwN1nmFlXIDV+pbU/6Z07cErv7roAVETapOaMUvsP4EnggaApB3g6HkW1Z3UDBzRztIi0Nc05pHYzcBZQAeDua4CjToopx6cwP8Lu/YfZuGt/2KWIiLSo5gROlbsfqnthZmkcx2SZcmyxF4CKiLQlzQmcV83s20AXMzsf+Avwt/iU1X4NyU6nS4dUFmvggIi0Mc0JnNuB7UAx8BWic599Jx5FtWdpR2aOVuCISNvSnFFqtcDvg4fEUWF+hIff2MCh6lo6psX7LuAiIonRnFFql5jZYjPbZWYVZlZpZhWN7ynNVZgX4VBNLW+X6p9XRNqO5vz5/Bui95bp5e493D3d3XvEqa52TQMHRKQtak7gbAKWuy4Qibt+GZ3pk95JgSMibUpzbsD2LWCWmb0KVNU1uvuvWryqdk4zR4tIW9ScHs5PiM6l1hlIj3lIHBTmR1i/Yx979h9qfGMRkVagOT2cTHe/IG6VyAfEnsf52BBN6CAirV9zejj/MDMFToKMzo1EZ47WYTURaSOaO5faC2Z2QMOi4697pzQG90lX4IhIm9HkwAmGQae4e5eGhkWb2Yj6+5jZdDPbZmbLY9p+bmbvmNkyM/urmUWC9gFBmC0JHvfH7DPezIrNbK2Z3W1mFrRnmtkcM1sTPPcM2i3Ybm3wOeOO758nXIV5EZZq5mgRaSNa8jL2PzbQ9jAwsV7bHGCku48GVgN3xKx7190Lg8cNMe33AVOBguBR9563Ay+5ewHwUvAa4KKYbacG+7c6dTNHv7dTM0eLSOvXkoFj9Rvc/TVgV7222e5ed2vqN4HcY76pWT+gh7vPDa4BegS4PFg9CZgRLM+o1/6IR70JRIL3aVV0AaiItCUtGTjHc9znOuD5mNcDg+lzXjWzjwRtOUBJzDYlQRtAtruXAgTPfWL22XSUfVqNwdnpdO2YqsARkTahOcOiW5SZ3QlUAzODplIg3913mtl44OngvNCHek40Hm5N3sfMphI97EZ+fn5TSk+Y1BRjVE4GixU4ItIGtGQPp8lXKJrZFOAS4Jq6qXLcvcrddwbLC4F3gcFEeyexh91ygS3BclndobLgeVvQXgLkHWWfD3D3B929yN2Levfu3dSvkDCF+RHe3lJBVXVN2KWIiJyQRgPHzD4fs3xWvXW31C27+4SmfKCZTQRuAy5z9/0x7b3NLDVYPpnoCf91waGySjObEIxOuxZ4JtjtWaITihI8x7ZfG4xWmwCU1x16a23GBjNHr9iiEegi0ro1pYfzjZjl39Vbd92xdjSzx4C5wBAzKzGz64F7iE6JM6fe8OdzgGVmthR4ErjB3esGHNwIPASsJdrzqTvvcxdwvpmtAc4PXkP05nDrgu1/D9zUhO+ZlIoGZNK5Qwo/fm4lBw+rlyMirZc1do2HmS1297H1lxt63doVFRX5ggULwi7jQ2YVl3LTzEVcNqY/v72ykOAyJBGRpGBmC929qLHtmtLD8aMsN/Ra4uDiUf34rwuH8OzSLfz2pTVhlyMiclyaMkptqJktIzrya1CwTPD65LhVJh9w08cGsW77Pn7zjzUMzOrGpMJWN8pbRNq5pgTOsLhXIY0yM3766VFs2r2f//rLMnJ7dmH8SZlhlyUi0mSNHlJz9/diH8BeYByQFbyWBOmYlsIDnx9P/0hnpj6ykE27NOWNiLQeTRkW/ZyZjQyW+wHLiY5O+6OZ3Rrn+qSent06Mu2Lp3K4ppbrHp5PxcHDYZckItIkTRk0MNDd62Z7/hIwx90vBU6nkWHREh+Denfn/i+MZ/2Ofdw8cxHVNbVhlyQi0qimBE7sn9DnEr3GBXevBPSbLiRnDsriJ58ayb/W7OD7f1uhWxiISNJryqCBTWb2VaLTxYwDXgAwsy5AhzjWJo343Kn5rNu+jwdeW8fJWd257uyBYZckInJUTenhXA+MAL4IfM7d62aSnAD8IU51SRPdNnEoFwzP5sd/X8k/3ykLuxwRkaNqdKaB9iRZZxpozP5D1Xz2gbms376PJ288k2H9ejS+k4hIC2nqTANNmdrm2WOtd/fLmllb0mqtgQNQVnGQSfe8QYrB0zefRZ8encMuSUTaiaYGTlPO4ZxB9GZmjwFv0fC9ZiRk2T0689CUIibfP5cvP7KAJ6aeQZeOqWGXJSJyRFPO4fQFvg2MBH5LdFbmHe7+qru/Gs/ipHlG5mRw91VjKd5czjf+vITaWh0uFZHk0ZSZBmrc/QV3n0J0oMBa4JVg5JokmfOHZ/Pti4bx/PKt/GL2qrDLERE5okm3mDazTsAngauAAcDdwFPxK0tOxJc/MpB1O/Zy7yvvMjCrG5OL8hrfSUQkzhoNHDObQfRw2vPAD2JmHZAkZWb8cNJINu7az7f/WkxeZlcmnNwr7LJEpJ1ryjmcLwCDga8D/zaziuBRaWa673GS6pCawr1Xjycvsys3/Gkh63fsC7skEWnnmnIOJ8Xd04NHj5hHurvrgo8kltG1A3/44qkYcP3D89mz/1DYJYlIO9aUHo60Yif16sYDXyhi0+793PinRRyq1vR3IhIOBU47cNrATH52xWjmrtvJfz+9XBN9ikgomjRKTVq/T4/LZf2Offzun2s5uXc3vvLRQWGXJCLtTFx7OGY23cy2mdnymLafm9k7ZrbMzP5qZpGYdXeY2VozW2VmF8a0Twza1prZ7THtA83sLTNbY2ZPmFnHoL1T8HptsH5APL9na/Gf5w3mk6P7cdcL7/DC8q1hlyMi7Uy8D6k9DEys1zYHGOnuo4HVwB0AZjYcuJLozNQTgXvNLNXMUoH/B1wEDAeuCrYF+Bnwa3cvAHYTndma4Hm3u58C/DrYrt1LSTF+OXkMY3Ij3PrEYopLysMuSUTakbgGjru/Buyq1zbb3auDl28CucHyJOBxd69y9/VEZzQ4LXisdfd17n4IeByYZGYGfAJ4Mth/BnB5zHvNCJafBM4Ntm/3OndI5ffXFtGrWyeunzGf0vIDYZckIu1E2IMGriN6QSlADtFJQuuUBG1Ha+8F7IkJr7r2D7xXsL482F6A3umdmPbFIvYfquH6hxewr6q68Z1ERE5QaIFjZncC1cDMuqYGNvPjaD/WezVUx1QzW2BmC7Zv337sotuQoX178Lurx/LO1gqm/nGBrtERkbgLJXDMbApwCXCNvz9GtwSInfQrF9hyjPYdQMTM0uq1f+C9gvUZ1Du0V8fdH3T3Incv6t2794l+tVbl40P68LMrRjNv/S4uved1lm/WOR0RiZ+EB46ZTQRuAy5z9/0xq54FrgxGmA0ECoB5wHygIBiR1pHowIJng6B6GfhMsP8U4JmY95oSLH8G+Kfr4pMGTS7K44mvnMHhaueK+/7NkwtLwi5JRNqoeA+LfgyYCwwxsxIzux64B0gH5pjZEjO7H8DdVwB/BlYCLwA3B7dGqAZuAV4E3gb+HGwL0eD6hpmtJXqOZlrQPg3oFbR/AzgylFo+bFx+T5772tmMzY/wzb8s5TtPF1NVXRN2WSLSxjR6i+n2pDXfYrolVNfU8vMXV/HAa+sozItw3+fH0S+jS9hliUiSa+otpsMepSZJJC01hTsuHsa914xjTVkll9z9Ov9+d0fYZYlIG6HAkQ+5eFQ/nrnlLCJdO/D5h97igVff1fxrInLCFDjSoFP6pPPMLWdz4Yi+/PT5d7hp5iL26nodETkBChw5qu6d0rj3mnF8++KhvLhiK5PueZ212yrDLktEWikFjhyTmTH1nEH86cuns2f/YSbd8wazikvDLktEWiEFjjTJmYOyeO5rZ1OQnc5NMxfx01lvU12jm7mJSNMpcKTJ+mV04YmvTODzE/J54LV1fH7aW+zYWxV2WSLSSihwpFk6paXy48tH8YvJY1i8cQ+X3P06izbuDrssEWkFFDhyXD4zPpenbjqTDmnG5x6Yyx/ffE9Dp0XkmBQ4ctxG9M/guVs+wtmnZPHfTy/nm39ZxsHDmhJHRBqmwJETktG1A9OmnMqt5xXw1OISPn3vv9m4c3/jO4pIu6PAkROWkmLcet5gpk85lZLd+7n0ntd5edW2sMsSkSSjwJEW8/GhfXjuqx+hf6QL1z08n9/8YzW1tTqvIyJRChxpUfm9uvLUjWfyqcIcfvOPNXz5kQXsP6QpcUREgSNx0KVjKr/87Bh+NGkEr6zaxs0zF3FYF4mKJKXKg4eZ9vp6Kg4ejvtnpTW+iUjzmRlfOGMAKSnGnX9dzu3/W8wvJo/GzMIuTUSATbv284c3NvDnBZvYW1VNVveOTCrMietnKnAkrq45/SS2VVTx25fWkN2jE9+aODTskkTaLXdn/obdTHt9HXNWlpFixiWj+3H92SczKjcj7p+vwJG4u/W8ArZVVnHvK+/SJ70TXzxrYNglibQrh6prmVVcyrTX11O8uZxI1w7c8NFBXHvGAPpmdE5YHQociTsz48eXj2Tn3ip+8NxKstI7ccno/mGXJdLm7d53iEfnbeSRuRsoq6hiUO9u/ORTI/n02Fy6dExNeD0KHEmI1BTj7qvG8oVpb/GNJ5aS2a0jZw7KCrsskTZp7ba9TH9jPU8tKuHg4Vo+UpDFXVeM5qMFvUlJCe88qmn+q/cVFRX5ggULwi6jTSvff5jJD/ybLXsO8sRXJjCif/yPG4u0B+7O62t3MO319byyajsd01L49NgcvnTWQIb0TY/rZ5vZQncvamy7uA6LNrPpZrbNzJbHtE02sxVmVmtmRTHtA8zsgJktCR73x6wbb2bFZrbWzO62YKiTmWWa2RwzWxM89wzaLdhurZktM7Nx8fye0nQZXTvw8JdOI71zGl/8w3w27dI0OCIn4uDhGh6ft5ELf/MaX5g2j+WbK/jG+YOZe/snuOuK0XEPm+aI93U4DwMT67UtBz4NvNbA9u+6e2HwuCGm/T5gKlAQPOre83bgJXcvAF4KXgNcFLPt1GB/SRL9I1145LrTOFRdy7XT57FT99QRabZtlQf51exVnHnXP7n9qWJSU1L4xeQxvHH7x/nauQX06t4p7BI/JK7ncNz9NTMbUK/tbaDJ12OYWT+gh7vPDV4/AlwOPA9MAj4WbDoDeAW4LWh/xKPHC980s4iZ9XN33Rs5SRRkpzNtShHXPPQW181YwGP/cTpdO+qUokhjVmwpZ/rrG/jb0i0crq3l3KHZXHf2AM44uVfSX+eWbD/hA81sMVABfMfd/wXkACUx25QEbQDZdSHi7qVm1idozwE2NbDPhwLHzKYS7QWRn5/fgl9FGlM0IJPfXTWWG/60kJtmLuL31xbRIVWTX4jUV11Ty8urtjPt9XW8uW4XXTumctVpeXzxrIEMzOoWdnlNlkyBUwrku/tOMxsPPG1mI4CGIruxkQ5N3sfdHwQehOiggWbUKy3gghF9+cmnRnHHU8WajUDavdpaZ/OeA6wuq2R12V5Wl1Wyamsla7fv5VB1Lf0zOnPHRUO58tR8Mrp2CLvcZkuawHH3KqAqWF5oZu8Cg4n2TnJjNs0FtgTLZXWHyoJDb3Vz4pcAeUfZR5LMVafls62iil//YzV9enTiNs1GIG2cu7O9sopVQaCsKdvLqrJK1pRVsu/Q+zcx7J/RmcF90zm7IItx+RHOG5ZNWis+CpA0gWNmvYFd7l5jZicTPeG/zt13mVmlmU0A3gKuBX4X7PYsMAW4K3h+Jqb9FjN7HDgdKNf5m+T2tXNPoazyIPcFsxF8SbMRSBuxe9+hoMdSyaqySlZv3cvqbZXs2f/+ZJlZ3TsyODudyUV5DM5OZ0jf7hRkp9Ojc+vrxRxLXAPHzB4jelI/y8xKgO8Bu4gGRm/g72a2xN0vBM4Bfmhm1UANcIO77wre6kaiI966EB0s8HzQfhfwZzO7HtgITA7aZwEXA2uB/cCX4vg1pQWYGT+aNJIdlVX88LmVZHXvxKVjNBuBtB77qqrfD5ate48EzPbK90dhpndOY0h2OheP6seQ7HQKsrszODudrCQcURYPuvAzhi78DN/BwzVcO20eSzbt4eEvncqZp2g2Akle7s6C93bz6Fsb+XtxKYeqo7fh6NwhhcHZ6dHeSnY6g/umMzi7O317dG6T5yibeuGnAieGAic5aDYCSXblBw7z10UlPDpvI6vL9pLeKY3Lx+ZwzuDeDMlOJ7dnl1CnkEk0Bc5xUOAkj9LyA1xx7785XOs8deOZ5GV2DbskaefcnSWb9jDzrY08t2wLBw/XMjo3g2tOz+fSMf3b9XVkCpzjoMBJLmvKKvnM/XPJ7NaRJ284IymvnJa2r/LgYZ5esoVH39rI26UVdO2YyqTCHK45PZ+ROep9gwLnuChwks/C93Zx9e/fYmjfdB79jwl069R+/4qUxCouKefRee/xzJIt7D9Uw/B+Pbj69HwmFfYnvY2NHjtRTQ0c/fRKUht/Uib3XD2Or/xxATfNXMRDU+I/G4G7s6X8IAcP1zCod/e4fpYkl31V1fxt6RZmvrWR4s3ldO6QwqWj+3PNhJMYk5vRJk/4J5ICR5Le+cOz+Z9PjeL2p4q57X+X8cvJY1rsB78uXIpLylm+uZxlm6PPu/YdAuDSMf2546Kh9I90aZHPk+S0cksFj857j6cXb2FvVTWDs7vzg8tGcPnYHDK6qDfTUhQ40ipceVo+2yqr+NWc1fRJ78ztFzV/NgJ3p7T8IMVBqCwLQmZnEC6pKcbg7HTOH5bNyNwMtlUc5MHX1vGPlWXc9LFB/Md3NEtvAAAO9ElEQVQ5J9O5Q+LvkijxceBQDc8t28Kj8zayeOMeOqalcMmoflx9ej7jT+qp3kwcKHCk1fjqJ05hW+VB7n81OhvBdWcffTYCd2drRbTnUry5/EjI7Nj7frgU9OnOucP6MCong5E5GQzr1+NDgfLZojz+Z9bb/HLOav68cBN3XjycC0dk65dRK7amrJKZb23kqUUlVBys5uTe3fjOJ4dxxbhcenbrGHZ5bZoGDcTQoIHkV1Pr3DxzES+s2MrdV43lsjH9cXfKKqqiwVKy50jA1A+XUTkZjMqNhsvwBsLlWP69dgff/9sKVpft5exTsvjepcMpyE6eG1vJ0ZXvP8yyzXtYVlLOK6u2MX/DbjqkGhNH9uOa0/M5fWCm/oA4QRqldhwUOK3DwcM1XDt9Hos37ubMQVms2FLBjuAmbikGBX3SGZWbcaTnMrxfD7p0PPFDYdU1tcx8ayO/nL2KfYdquPaMk7j1vME6xp9EDhyqYWVpOUs2lbOsJBoy63fsO7L+lD7dmTw+l8+Mz9Uw+xakwDkOCpzWo/zAYW6auZDtlVWMyokwKqcHo3IzGN4vo0XC5Vh27TvEL2ev4tF5G+nZtSP/deEQPluUR2o7urI8GRyuqWV1WSVLg3BZWlLO6rJKamqjv9P69ujM6NwMxuRFGJ2bweicSKuc0r81UOAcBwWONMeKLeX84NmVzNuwixH9e/CDy0ZQNCAz7LLapNpaZ/3OfdFgCQJmxZYKqoK5yzK6dIiGS27kSMhk9+gcctXthwLnOChwpLncnb8tK+Wns96mtPwgkwr7c8dFw+iboV92x6tuNGFdr6Xu0FjlwWoAunRIZWROD0YH4VKYFyE/s6vOw4RIgXMcFDhyvPYfqua+V97lgdfWkZZi3PzxU7j+7IEaRt1E5QcO8/I725i9civzN+w+MqV/WooxtF86o3MjjAl6Lqf07t6qb0LWFilwjoMCR07Upl37+fHfV/LiijLyM7vynU8O4/zhGkbdkLKKg8xeWcbsFVuZ++5OqmudPumdOPuUrCPnXRoaqi7JR4FzHBQ40lJeX7ODH/xtBWu27eUjBdFh1Kf00TDqddv3MntlGS+u2MrijXsAGJjVjQtGZHPhiL4U5kba1bT+bYUC5zgocKQlHa6p5U9vvsev5qzmwKEarj1jAF8/r6BdDaN2d5ZvruDFFVt5ccVW1mzbC8ConAwuGJ7NhSP7UtCnu3qArZwC5zgocCQedu6t4hezV/P4/I1kdu3ItyYOYfL4vDb7l3x1TS3zN+zmxRVbmbOyjM17DpBicNrATC4c0ZcLRvQlR3PTtSkKnOOgwJF4Wr65nO8/u4IF7+1mVE4GX/7IQM4dlk33NnDLhYOHa/jXmh28uGIrL71dxu79h+mYlsI5BVlcMKIv5w3LJlPTxrRZCpzjoMCReHN3nl26hf/7wio27zlAx7QUPjq4NxeP6su5w7Lp0Yrus1I3suzFFVt5dfV29h+qIb1zGucO7cOFI/pyzuDeun9RO5EU98Mxs+nAJcA2dx8ZtE0Gvg8MA05z9wUx298BXA/UAF9z9xeD9onAb4FU4CF3vytoHwg8DmQCi4AvuPshM+sEPAKMB3YCn3P3DfH8riJNYWZMKszh0tH9WbhxN39fVsoLy6OHnjqmpnDO4CwuGtmP84ZnJ925Hnfn3e37mPvuDmavLPvAyLJPjc3hwhF9mXByLzqmaciyNCyuPRwzOwfYCzwSEzjDgFrgAeCbdYFjZsOBx4DTgP7AP4DBwVutBs4HSoD5wFXuvtLM/gw85e6Pm9n9wFJ3v8/MbgJGu/sNZnYl8Cl3/1xj9aqHI2GorXUWb9rN35dt5fnlpZSWH6RDqvGRgt5cNLIvFwzvG8qULIdralmxpYL563cxf8MuFry3+8h9gjSyTGIlRQ/H3V8zswH12t4GGhqVMgl43N2rgPVmtpZo+ACsdfd1wX6PA5PM7G3gE8DVwTYziPac7gve6/tB+5PAPWZmruOHkoRSUozxJ2Uy/qRMvvPJYSwp2cOsZaU8v3wr/3xnG99OLeasU7K4eFQ/LhieTaRrfM6F7KuqZvHGPczbsIsFG3axeOMeDhyuAeCkXl35+JA+nDawJ6cOyGRgVjeNLJNmS6YDrDnAmzGvS4I2gE312k8HegF73L26ge1z6vZx92ozKw+23xGf0kVaRkqKMS6/J+Pye3LnJ4extKScWcWl/H1ZKd9atYxvpxhnnpLFJ0dFez4ncv+WHXurWLBhF/PW72bBe7tYsaWCmlonxWBYvx587tQ8Th2QSdGAnpqXTFpEMgVOQ38uOdDQAWE/xvbHeq8Pf6jZVGAqQH5+fuNViiSImVGYF6EwL8IdFw2leHM5fy8uZVZxKbf9bzHf/utyzhzUi4tH9ePCEX2POQrM3Xlv537mbwgOj23Yzbpg2v5OaSkU5kW48aODOHVgJuPyI6S3osEL0nokU+CUAHkxr3OBLcFyQ+07gIiZpQW9nNjt696rxMzSgAxgV0Mf6u4PAg9C9BxOy3wVkZZlZsFklRFunziUFVsqjoTPHU8V852nlzPh5Mwj4dOza0feLq04Ei7zNuw6Mj9ZRpcOnDqgJ58NejAjc3rQKU3Tx0j8JVPgPAs8ama/IjpooACYR7S3UhCMSNsMXAlc7e5uZi8DnyE6Um0K8EzMe00B5gbr/6nzN9JWmBkjg5vLfevCIawsrWBWcSmzirdy51+X899PL6dLh1T2HYqef8mJdOGsQb0oGpDJaQMzOaV3d53kl1DEe5TaY8DHgCygDPge0Z7G74DewB5gibtfGGx/J3AdUA3c6u7PB+0XA78hOix6urv/JGg/mfeHRS8GPu/uVWbWGfgjMDb4vCvrBh0ci0apSWvm7ryztZLni0vZc+Aw40/qSdGATF3VL3GnCz+PgwJHRKT5mho4ukJLREQSQoEjIiIJocAREZGEUOCIiEhCKHBERCQhFDgiIpIQChwREUkIBY6IiCSELvyMYWbbgffCruMosmids1231rpBtYdFtYfjRGo/yd17N7aRAqeVMLMFTbmSN9m01rpBtYdFtYcjEbXrkJqIiCSEAkdERBJCgdN6PBh2AceptdYNqj0sqj0cca9d53BERCQh1MMREZGEUOAkMTPLM7OXzextM1thZl8Pu6bmMrNUM1tsZs+FXUtzmFnEzJ40s3eCf/8zwq6pqczsP4P/L8vN7LHghoRJycymm9k2M1se05ZpZnPMbE3w3DPMGhtylLp/Hvx/WWZmfzWzSJg1Hk1Dtces+6aZuZllxeOzFTjJrRr4P+4+DJgA3Gxmw0Ouqbm+DrwddhHH4bfAC+4+FBhDK/kOZpYDfA0ocveRRO+Se2W4VR3Tw8DEem23Ay+5ewHwUvA62TzMh+ueA4x099HAauCORBfVRA/z4doxszzgfGBjvD5YgZPE3L3U3RcFy5VEf+nlhFtV05lZLvBJ4KGwa2kOM+sBnANMA3D3Q+6+J9yqmiUN6GJmaUBXYEvI9RyVu79G9DbwsSYBM4LlGcDlCS2qCRqq291nu3t18PJNIDfhhTXBUf7NAX4NfAuI24l9BU4rYWYDgLHAW+FW0iy/IfofuDbsQprpZGA78IfgcOBDZtYt7KKawt03A78g+ldqKVDu7rPDrarZst29FKJ/dAF9Qq7neFwHPB92EU1lZpcBm919aTw/R4HTCphZd+B/gVvdvSLseprCzC4Btrn7wrBrOQ5pwDjgPncfC+wjOQ/rfEhwvmMSMBDoD3Qzs8+HW1X7YmZ3Ej0cPjPsWprCzLoCdwLfjfdnKXCSnJl1IBo2M939qbDraYazgMvMbAPwOPAJM/tTuCU1WQlQ4u51vckniQZQa3AesN7dt7v7YeAp4MyQa2quMjPrBxA8bwu5niYzsynAJcA13nquORlE9A+UpcHPay6wyMz6tvQHKXCSmJkZ0fMIb7v7r8Kupznc/Q53z3X3AURPWv/T3VvFX9ruvhXYZGZDgqZzgZUhltQcG4EJZtY1+P9zLq1kwEOMZ4EpwfIU4JkQa2kyM5sI3AZc5u77w66nqdy92N37uPuA4Oe1BBgX/By0KAVOcjsL+ALR3sGS4HFx2EW1E18FZprZMqAQ+J+Q62mSoFf2JLAIKCb6M560V7+b2WPAXGCImZWY2fXAXcD5ZraG6Kipu8KssSFHqfseIB2YE/ys3h9qkUdxlNoT89mtp9cnIiKtmXo4IiKSEAocERFJCAWOiIgkhAJHREQSQoEjIiIJocARSUJmtjd47m9mTzay7a3B1eIiSU3DokUSxMxS3b2midvudffuTdx2A9HZoXfEoxaRlqIejkgLMLMBwb1QZgT3Q3kyuNp/g5l918xeByab2SAze8HMFprZv8xsaLD/QDOba2bzzexH9d53ebCcama/MLPi4DO+amZfIzpn2stm9nKw3VXBNsvN7Gcx77XXzH5oZm8BZ5jZXWa2MnivXyTy30vap7SwCxBpQ4YA17v7G2Y2HbgpaD/o7mcDmNlLwA3uvsbMTgfuBT5B9P4797n7I2Z281HefyrROa/Gunu1mWW6+y4z+wbwcXffYWb9gZ8B44HdwGwzu9zdnwa6Acvd/btmlkl02qSh7u7JerMwaVvUwxFpOZvc/Y1g+U/A2cHyE3Bk1u8zgb+Y2RLgAaBfsM1ZwGPB8h+P8v7nAffX3XPF3Ru6p8mpwCvB5J11MxafE6yrIToRLEAFcBB4yMw+DbSaub+k9VIPR6Tl1D8hWvd6X/CcAuxx98Im7l+fNXGbozlYd94m6CGdRnRyzyuBW4j2tETiRj0ckZaTb2ZnBMtXAa/HrgzuZbTezCZDdDZwMxsTrH6D928Ffc1R3n82cENwJ0+Cw2IAlUQnjYToDfo+amZZZpYa1PFq/TcKelsZ7j4LuJXoBKUicaXAEWk5bwNTghmmM4H7GtjmGuB6M1sKrCB6szSArwM3m9l8IOMo7/8Q0dsPLAv2vzpofxB43sxeDu6QeQfwMrAUWOTuDU3vnw48F9T6KvCfzfuqIs2nYdEiLSC4Bfhz7j4y5FJEkpZ6OCIikhDq4YiISEKohyMiIgmhwBERkYRQ4IiISEIocEREJCEUOCIikhAKHBERSYj/Dy28HLD2n2clAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform cross validation for each subset of predictors\n",
    "results = []\n",
    "for sub in best_subsets_full:\n",
    "    results += [cross_val(sub[0], X, y, 10)]\n",
    "\n",
    "results_a_df = pd.DataFrame({'predictors': list(np.arange(1, 15)),\n",
    "                           'MSE_mean': [np.mean(i[0]) for i in results],\n",
    "                           'MSE_folds': [i[0] for i in results],\n",
    "                           'Model': [i[1] for i in results]})\n",
    "display(results_a_df)\n",
    "sns.lineplot(x='predictors', y='MSE_mean', data=results_a_df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE_mean</th>\n",
       "      <th>MSE_folds</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>109522.011671</td>\n",
       "      <td>[86417.0694422955, 30716.914843531533, 165276....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   predictors       MSE_mean  \\\n",
       "7           8  109522.011671   \n",
       "\n",
       "                                           MSE_folds  \\\n",
       "7  [86417.0694422955, 30716.914843531533, 165276....   \n",
       "\n",
       "                                               Model  \n",
       "7  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Salary ~ AtBat + Hits + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_subset_choice = results_a_df[results_a_df['MSE_mean'] == results_a_df['MSE_mean'].min()]\n",
    "display(best_subset_choice)\n",
    "\n",
    "best_subset_choice['Model'].iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that 10-fold cross validation selects an 8 variable model. The estimated test MSE is 109,522.\n",
    "\n",
    "**Revision note:** The ISL authors (p.250) land on an 11 variable model with a higher average MSE of 125,154. Why is this?\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Can we get a better model by Backwards Stepwise on all features?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rss</th>\n",
       "      <th>predictors</th>\n",
       "      <th>predictor_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.420070e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.420070e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.420070e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.420070e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.420184e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2.420945e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, CAtBat,...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.421938e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, RBI, Walks, CAtBat,...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.423518e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, Walks, CAtBat, CHit...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2.424866e+07</td>\n",
       "      <td>[AtBat, Hits, HmRun, Runs, Walks, CAtBat, CRun...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.428915e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2.433323e+07</td>\n",
       "      <td>[AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.438735e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.450040e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.481405e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.515923e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, CRBI, CWalks, PutO...</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.593349e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, CWalks, PutOuts, D...</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2.667409e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, PutOuts, Division_W]</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2.750952e+07</td>\n",
       "      <td>[AtBat, Hits, Walks, CRuns, PutOuts]</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2.845081e+07</td>\n",
       "      <td>[AtBat, Hits, CRuns, PutOuts]</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.940730e+07</td>\n",
       "      <td>[Hits, CRuns, PutOuts]</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3.120346e+07</td>\n",
       "      <td>[Hits, CRuns]</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>3.643795e+07</td>\n",
       "      <td>[CRuns]</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             rss                                         predictors  \\\n",
       "0   2.420070e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...   \n",
       "1   2.420070e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...   \n",
       "2   2.420070e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...   \n",
       "3   2.420070e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...   \n",
       "4   2.420184e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, Years, ...   \n",
       "5   2.420945e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, CAtBat,...   \n",
       "6   2.421938e+07  [AtBat, Hits, HmRun, Runs, RBI, Walks, CAtBat,...   \n",
       "7   2.423518e+07  [AtBat, Hits, HmRun, Runs, Walks, CAtBat, CHit...   \n",
       "8   2.424866e+07  [AtBat, Hits, HmRun, Runs, Walks, CAtBat, CRun...   \n",
       "9   2.428915e+07  [AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...   \n",
       "10  2.433323e+07  [AtBat, Hits, Runs, Walks, CAtBat, CRuns, CRBI...   \n",
       "11  2.438735e+07  [AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...   \n",
       "12  2.450040e+07  [AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...   \n",
       "13  2.481405e+07  [AtBat, Hits, Walks, CAtBat, CRuns, CRBI, CWal...   \n",
       "14  2.515923e+07  [AtBat, Hits, Walks, CRuns, CRBI, CWalks, PutO...   \n",
       "15  2.593349e+07  [AtBat, Hits, Walks, CRuns, CWalks, PutOuts, D...   \n",
       "16  2.667409e+07   [AtBat, Hits, Walks, CRuns, PutOuts, Division_W]   \n",
       "17  2.750952e+07               [AtBat, Hits, Walks, CRuns, PutOuts]   \n",
       "18  2.845081e+07                      [AtBat, Hits, CRuns, PutOuts]   \n",
       "19  2.940730e+07                             [Hits, CRuns, PutOuts]   \n",
       "20  3.120346e+07                                      [Hits, CRuns]   \n",
       "21  3.643795e+07                                            [CRuns]   \n",
       "\n",
       "    predictor_count  \n",
       "0                 1  \n",
       "1                 2  \n",
       "2                 3  \n",
       "3                 4  \n",
       "4                 5  \n",
       "5                 6  \n",
       "6                 7  \n",
       "7                 8  \n",
       "8                 9  \n",
       "9                10  \n",
       "10               11  \n",
       "11               12  \n",
       "12               13  \n",
       "13               14  \n",
       "14               15  \n",
       "15               16  \n",
       "16               17  \n",
       "17               18  \n",
       "18               19  \n",
       "19               20  \n",
       "20               21  \n",
       "21               22  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAESCAYAAAASQMmzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VPW9//HXJytkEUEiIIugglZRQAOubV1o1d5W21ttxZZrXUq13lZ/be36q721j/662EX9tbZyte7VurSF0qrlWpfaKhA0hM0EUREEkiACIYEsM5/7x5xJY0xIUM6cM5P38/HIY86c+c6Zj2OYd77nfOf7NXdHREQEIC/qAkREJD4UCiIi0kmhICIinRQKIiLSSaEgIiKdFAoiItIpK0PBzH5jZg1mtqIfbX9uZtXBT52ZbctEjSIi2ciy8XsKZvY+YCdwl7tP3ovnfQGY5u6XhFaciEgWy8qegrs/DWztus/MDjWzR81sqZn93cyO6OGps4D7MlKkiEgWKoi6gH1oLnC5u68xs+OBm4HT0w+a2cHABOBvEdUnIhJ7OREKZlYGnAQ8aGbp3cXdml0APOTuiUzWJiKSTXIiFEidBtvm7lP30OYC4MoM1SMikpWy8ppCd+6+A3jFzM4HsJQp6cfN7HBgKPBsRCWKiGSFrAwFM7uP1Af84Wa2wcwuBT4FXGpmy4CVwLldnjILuN+zcaiViEgGZeWQVBERCUdW9hRERCQcWXehefjw4T5+/PioyxARySpLly7d4u4VfbXLulAYP348VVVVUZchIpJVzGxdf9rp9JGIiHRSKIiISCeFgoiIdFIoiIhIJ4WCiIh0UiiIiEgnhYKIiHRSKIiIZIF7nltH9frwVxNWKIiIxNzu9gTfmb+S/1lVH/prKRRERGLu5cZmEkln0sjy0F9LoSAiEnNrGpoAOHyEQkFEZMCr3dxEQZ4xYXhp6K8VWiiY2SAzW2xmy8xspZl9t5d2nzCzVUGb34ZVj4hItqqrb+KQilKKCsL/Oz7MWVJbgdPdfaeZFQLPmNkj7v5cuoGZTQS+AZzs7m+a2YEh1iMikpVq65uYMmb/jLxWaLHjKTuDu4XBT/dl3j4L/NLd3wye0xBWPSIi2ai5tYP1W3dl5HoChHxNwczyzawaaAAWuvuibk0mAZPM7B9m9pyZndXLceaYWZWZVTU2NoZZsohIrKxpSP1tnYmRRxByKLh7wt2nAmOAGWY2uVuTAmAicCowC7jVzN7WR3L3ue5e6e6VFRV9LhwkIpIz6janRh5NyoWeQpq7bwOeBLr3BDYA89y93d1fAWpJhYSIiJC6nlBckMe4YSUZeb0wRx9VpP/qN7PBwEzgxW7N/gicFrQZTup00sth1SQikm3q6puYOKKM/DzLyOuF2VMYBTxhZjXAElLXFBaY2XVmdk7Q5jHgDTNbBTwBXOPub4RYk4hIVqmrb8rYqSMIcUiqu9cA03rYf22XbQe+FPyIiEgX21raqN/RmrGRR6BvNIuIxFZdfWZHHoFCQUQktmrrMzfnUZpCQUQkpuo2N1FeXMCoIYMy9poKBRGRmKqtb2LSyHLMMjPyCBQKIiKx5O7ByKOyjL6uQkFEJIYam1rZ1tKe0eGooFAQEYml9MijTF5kBoWCiEgspUceZXI4KigURERiqW5zEweUFjG8rDijr6tQEBGJodoMT2+RplAQEYmZZNJZU9/E4Rk+dQQKBRGR2Hl92y6a2xLqKYiISGpmVCDj31EAhYKISOykRx5NVE9BRETW1O9k1JBBDBlcmPHXViiIiMRM7eZoRh6BQkFEJFY6EkleatwZycgjUCiIiMTKuq0ttHUk1VMQEZHUN5kh83MepSkURERipLa+CTM47MDMD0cFhYKISKzU1Tdx8LASBhflR/L6CgURkRip3dwUyfcT0hQKIiIx0dqR4NU3WiK7ngAhhoKZDTKzxWa2zMxWmtl399D2PDNzM6sMqx4Rkbh7ubGZRNIzvoZCVwUhHrsVON3dd5pZIfCMmT3i7s91bWRm5cAXgUUh1iIiEnvpOY9ysqfgKTuDu4XBj/fQ9HvAj4HdYdUiIpINajc3UZBnTBheGlkNoV5TMLN8M6sGGoCF7r6o2+PTgLHuviDMOkREskFdfROHVJRSVBDd5d5QX9ndE+4+FRgDzDCzyenHzCwP+Dnw5b6OY2ZzzKzKzKoaGxvDK1hEJEJRrbbWVUbiyN23AU8CZ3XZXQ5MBp40s1eBE4D5PV1sdve57l7p7pUVFRUZqFhEJLOaWztYv3VXpNcTINzRRxVmtn+wPRiYCbyYftzdt7v7cHcf7+7jgeeAc9y9KqyaRETiak1D6hJslCOPINyewijgCTOrAZaQuqawwMyuM7NzQnxdEZGsk57zKOrTR6ENSXX3GmBaD/uv7aX9qWHVIiISd3X1TRQX5DFuWEmkdegbzSIiMVBb38TEEWXk51mkdSgURERioC4GI49AoSAiErltLW3U72iNfOQRKBRERCJXVx+PkUegUBARiVxtDOY8SlMoiIhErG5zE+XFBYwaMijqUhQKIiJRS488Mot25BEoFEREIuXurKlv4vAYXE8AhYKISKQad7byZkt7LIajgkJBRCRSdZtTI4/icJEZFAoiIpFKjzyKw3BUUCiIiESqbnMTB5QWMbysOOpSAIWCiEik4rCwTlcKBRGRiCST8Rp5BAoFEZHIvL5tF81tCfUUREQkNTMqwKQRZRFX8i8KBRGRiKQnwpuonoKIiNTVNzFqyCCGDC6MupROCgURkYjUbo7XyCNQKIiIRKIjkeSlxp2xGnkECgURkUis29pCW0dSPQUREUl9kxniM+dRmkJBRCQCtfVNmMFhB8ZnOCooFEREIlFX38S4YSUMLsqPupS3CC0UzGyQmS02s2VmttLMvttDmy+Z2SozqzGzx83s4LDqERGJk7r6nbG7ngDh9hRagdPdfQowFTjLzE7o1uYFoNLdjwEeAn4cYj0iIrHQ2pHglS3NsbueACGGgqfsDO4WBj/erc0T7t4S3H0OGBNWPSIicfFyYzOJpMdmDYWuQr2mYGb5ZlYNNAAL3X3RHppfCjzSy3HmmFmVmVU1NjaGUaqISMak5zwaUD0FAHdPuPtUUj2AGWY2uad2ZvZpoBK4vpfjzHX3SnevrKioCK9gEZEMqN3cREGeMWF4adSlvE1GRh+5+zbgSeCs7o+Z2UzgW8A57t6aiXpERKJUV9/EIRWlFBXEbwBomKOPKsxs/2B7MDATeLFbm2nALaQCoSGsWkRE4iRuq611FWZMjQKeMLMaYAmpawoLzOw6MzsnaHM9UAY8aGbVZjY/xHpERCLX3NrB+q27Ynk9AaAgrAO7ew0wrYf913bZnhnW64uIxNGahvitodBV/E5oiYjksM6RRzEcjgoKBRGRjKrb3ERxQR7jhpVEXUqPFAoiIhlUW9/ExBFl5OdZ1KX0SKEgIpJBdTEeeQQKBRGRjNnW0kb9jtbYjjwChYKISMbU1adGHsVxzqM0hYKISIas3LgdiOecR2kKBRGRDEgknbufXccRI8sZNWRQ1OX0SqEgIpIBf1q2kZe3NHPVGRMxi+fII1AoiIiELpF0bnp8DUeMLOfMo0ZGXc4eKRRERELWtZeQF9PvJ6QpFEREQpRNvQRQKIiIhCqbegmgUBARCU229RJAoSAiEpps6yWAQkFEJBSJpHPT37KrlwDvIBTMLM/M9gujGBGRXLGgZiMvNzbzxSzqJUA/Q8HMfmtm+5lZKbAKqDWza8ItTUQkOyWSzo2Pr+HwEeWclUW9BOh/T+FId98BfBT4CzAOmB1aVSIiWSzdS7hqZnb1EqD/oVBoZoWkQmGeu7cDHl5ZIiLZKZt7CdD/ULgFeBUoBZ42s4OBHWEVJSKSrbK5lwBQ0J9G7n4TcFOXXevM7LRwShIRyU7Z3kuA/l9oviq40GxmdpuZPQ+cHnJtIiJZJdt7CdD/00eXBBeaPwhUABcDP9zTE8xskJktNrNlZrbSzL7bQ5tiM/udmb1kZovMbPxe1i8iEgu50EuA/odCOvI+BNzu7su67OtNK3C6u08BpgJnmdkJ3dpcCrzp7ocBPwd+1M96RERiJRd6CdCPULDUahBbzOwxUqHwmJmVA8k9Pc9TdgZ3C4Of7iOWzgXuDLYfAs6wOK8+ISLSg/QcR9neS4B+hIK7O7A/8A1guru3AEWkTiHtkZnlm1k10AAsdPdF3ZqMBtYHr9MBbAcO6OE4c8ysysyqGhsb+3pZEZGMWlCzkbVZ+O3lnvT39NGzQL67bwNw9zfcvaavJ7l7wt2nAmOAGWY2uVuTnt69t33/wd3nunulu1dWVFT0s2QRkfClewmTRpRx9uTs7iVA/0PhNOBZM1trZjVmttzM+gyFtCBMngTO6vbQBmAsgJkVAEOArf09rohI1NK9hKvOmJT1vQTo5/cUgLP39sBmVgG0u/s2MxsMzOTtF5LnAxeR6omcB/wtOF0lIhJ7udZLgP5/eW3dOzj2KOBOM8sn1SN5wN0XmNl1QJW7zwduA+42s5dI9RAueAevIyISiXQv4ZcXHpsTvQTof09hrwXXHKb1sP/aLtu7gfPDqkFEJCy52EsALbIjIvKO/Hn5ppy6lpCmUBAR2Uu52ksAhYKIyF778/JNvNSwM+d6CaBQEBHZKzt2t/PzhXU52UsAhYKISL/taktw2R1VrN/awnc+clTO9RIgxNFHIiK5pK0jyRX3LmXJuq3cdME0Tj5seNQlhUI9BRGRPiSSzv95oJonaxv5fx87mo9MOSjqkkKjUBAR2QN351t/WM6fazbxrQ+9h1kzxkVdUqgUCiIivXB3vv/n1dy/ZD1fOP0wPvu+Q6IuKXQKBRGRXvziby9x6zOvcNGJB/OlD0yKupyMUCiIiPTgjn+8wk8X1vHv00bznY8cxUBZ/0uhICLSzcNLN/Bff1rFB48cwY/POyYnh572RqEgItLFoys2c81Dyzj5sAO4adY0CvIH1sfkwPqvFRHZg2fWbOGL973AlLH7M3d2JYMK86MuKeMUCiIiwNJ1bzLn7ioOqSjljs/MoLR4YH63V6EgIgPeqo07uPj2xRxYXsxdl85gSElh1CVFRqEgIgPaK1ua+Y/fLKK0uIB7LjueA8sHRV1SpBQKIjJgbdy2i0/fuoikw92XHs+YoSVRlxQ5hYKIDEhbdrby6dsWsWNXO3ddMoPDDiyLuqRYGJhXUkRkQNve0s7s2xazcdsu7rrkeCaPHhJ1SbGhnoKIDCg7Wzu46PbFrG3YyS2zK5kxYVjUJcWKegoiMmDsaktw6R1LWP76dm7+1LG8f1JF1CXFjnoKIjIgtHYkuPyepSx+dSs/+8QUzjwq95bS3BdCCwUzG2tmT5jZajNbaWZX9dBmiJn9ycyWBW0uDqseERm4OhJJrrqvmqfqGvnBx47m3Kmjoy4ptsI8fdQBfNndnzezcmCpmS1091Vd2lwJrHL3j5hZBVBrZve6e1uIdYnIAJJMOtc8VMOjKzfz7Q8fyQU5vkjOuxVaT8HdN7n788F2E7Aa6B7PDpRbak7aMmArqTAREXnX3J3/O28Ff3jhdb7ywUlcesqEqEuKvYxcaDaz8cA0YFG3h34BzAc2AuXAJ909mYmaRCS3pVdN++2i17ji1EO58rTDoi4pK4R+odnMyoCHgavdfUe3h88EqoGDgKnAL8xsvx6OMcfMqsysqrGxMeySRSQH3PA/azpXTfvqmYcPmEVy3q1QQ8HMCkkFwr3u/vsemlwM/N5TXgJeAY7o3sjd57p7pbtXVlRoCJmI7NktT63lxsfXcP5xYwbUqmn7Qpijjwy4DVjt7j/rpdlrwBlB+xHA4cDLYdUkIrnv7mdf5QePvMiHjxnFDz8+sFZN2xfCvKZwMjAbWG5m1cG+bwLjANz918D3gDvMbDlgwNfcfUuINYlIDnto6Qa+PW8lM99zID//5FTyFQh7LbRQcPdnSH3Q76nNRuCDYdUgIgPHX5Zv4qvBMpq/uPBYCgfYMpr7it41Ecl6f3uxni/e9wLHjhvKf//HwFxGc19RKIhIVvvnS1u4/J7nec+o/fjNxdMpKdKUbu+GQkFEstZTdY1cemcVEw4o5a5LZrDfoIG7jOa+olAQkaz055pNXHbnEiYML+Wey45naGlR1CXlBPWzRCTr3L/4Nb75h+Ucd/BQbr1oOkMGq4ewrygURCSr3PLUWn7wyIu8f1IFv/70cQwu0kXlfUmhICJZwd358WO1/OrJtXz4mFH87BNTKSrQGfB9TaEgIrGXSDrfnreC3y56jQuPH8f3zp2sL6aFRKEgIrHW1pHkSw9Us6BmE1eceqgmtwuZQkFEYmtXW4Ir7l3Kk7WNfP3sI7j8/YdGXVLOUyiISCxt39XOZXcuoWrdm/zg349mllZMywiFgojETmNTKxf9ZjFrGpr4xaxj+bdjRkVd0oChUBCRWNnwZguzb1vM5u27ufWi6bx/ktZQySSFgojExksNO5l92yKaWzu457IZHHfwsKhLGnAUCiISC8s3bOei2xeTZ8bvPnci7xn1tpV5JQMUCiISuWfWbOHye5ayf0kh91x6POOHl0Zd0oClUBCRyLg7dz27jusWrOKwijLuvGQGI4cMirqsAU2hICKRaOtI8p35K7lv8WvMfM+B3HDBNMqK9ZEUNf0fEJGM29rcxhX3LGXRK1v5/KmH8pUPHk6epq2IBYWCiGTUi5t3cNmdVTQ0tXLDJ6fy0Wmjoy5JulAoiEjGLFxVz9X3v0BpcQEPfO5Epo7dP+qSpBuFgoiEzt25+cm1/OSvtRw9eghzZ1fqgnJMKRREJFS72xN87eEa5lVv5CNTDuL6845hUKEWxokrhYKIhKZ+x27m3FXFsg3buebMw/n8qYdq2uuYCy0UzGwscBcwEkgCc939xh7anQrcABQCW9z9/WHVJCKZs2z9NubcXUXT7g5umX0cZx41MuqSpB/C7Cl0AF929+fNrBxYamYL3X1VuoGZ7Q/cDJzl7q+Z2YEh1iMiGTKv+nW++lANw8uKefiKkzRlRRYJLRTcfROwKdhuMrPVwGhgVZdmFwK/d/fXgnYNYdUjIuFLJp2fLqzll0+sZcb4Yfzq08dyQFlx1GXJXsjINQUzGw9MAxZ1e2gSUGhmTwLlwI3uflcPz58DzAEYN04LbYjEUUtbB1ffX81fV9VzwfSxXHfuZIoK8qIuS/ZS6KFgZmXAw8DV7r6jh9c/DjgDGAw8a2bPuXtd10buPheYC1BZWelh1ywie6dhx24uvbOKlRu3c+2Hj+Tik8frgnKWCjUUzKyQVCDc6+6/76HJBlIXl5uBZjN7GpgC1PXQVkRiqHZzE5fcsYStzW3MnV3JzCNHRF2SvAuh9e0s9WfCbcBqd/9ZL83mAe81swIzKwGOB1aHVZOI7Ft/X9PIeb/6J+2JJA9efqICIQeE2VM4GZgNLDez6mDfN4FxAO7+a3dfbWaPAjWkhq3e6u4rQqxJRPaR3y15jW/9YQWHHVjGbZ+Zzuj9B0ddkuwDYY4+egbo86Siu18PXB9WHSKybyWTzk/+WsvNT67lvROHc/OnjqV8UGHUZck+om80i0i/7W5P8JUHl7GgZhOzZqRGGBXma4RRLlEoiEi/bG1uY85dVVSte5Ovn30En3vfIRphlIMUCiLSp1e2NHPx7YvZuH03v7zwWP7tmFFRlyQhUSiIyB4teXUrn72rijwz7vvsCRx38NCoS5IQKRREpFfzql/nmgdrGDNsMLd/ZjoHH1AadUkSMoWCiLxNelGc6x+rZcaEYcydfRz7lxRFXZZkgEJBRN5id3uCa+et4IGqDXxs2mh++PGjKS7QojgDhUJBRABY8fp2Hqxazx+rN7J9VztXnTGRq2dO1AijAUahIDKAbWtpY171Rh6oWs/KjTsoKsjjrKNGcuHx4zjhkAOiLk8ioFAQGWCSSecfa7fwQNUGHlu5mbaOJJNH78d15x7FuVNGM6RE304eyBQKIgPE+q0tPLR0Aw8t3cDr23YxZHAhF84Yx/mVYzjqoCFRlycxoVAQyWG72xM8tnIzD1Zt4B9rtwBwymHD+frZR/CBI0cwqFAXkOWtFAoiOSaZdJ5/7U3mVW9k/rLUReMxQwdz9RmT+PhxoxkztCTqEiXGFAoiOcDdWb2pifnLNvKnZRt5fdsuigvyOPOokXxy+lhOPOQA8vI0ikj6plAQyWKvvdHC/GWvM696I2sadpKfZ7x34nC+cuYkPnDkSMqK9U9c9o5+Y0SyTMOO3Syo2cT8ZRupXr8NgBnjh/G9j07mQ5NHckBZccQVSjZTKIhkge0t7Ty6MhUEz659g6TDUQftxzfOPoIPTzlIq57JPqNQEImRZNJpaGpl/ZstbHizhQ1bd1Hz+naeqm2kLZFk/AEl/OfpEzlnykEcdmBZ1OVKDlIoiGRQMuls2Zn+0N8V/LSwfmvqduO23bQlkm95zuj9BzP7xIM5d+pBHD16iKadkFApFETegY5EkqbdHezY3Z663dXOjm73m3Z30LS7nR2729mxq4P6HbvZsG0XbR1v/dAfXlbE6KElHDV6CGdOHsnYoSWMGTqYMcGtvksgmaRQkAGhPZGkpTVBS3sHza0JWto62NnaQUtrguYu2ztbO4LHUm2aW1Pt022aWzto2t1BS1uiz9csKy6gfFD6p5AjRpUz88gRjO3ygT966GBKivTPUOJDv43yjrg7iaSTcMcdEkmnI+F0JJN0JD31kwi20/sT/9qfSDrtSScR7E8Ez0nfJjvvJ9+yP9Fluz2RZFdbgubWDlraE7S0pj6sUz9v3W5PeL//2wYX5lNaXEBpcT6lRQWUFRcwrLSIscNKKCtKfcjvN7gwdTuosPNDf7/Bqfv7DSqkbFAB+fpegGShARMKz738Bjc9vibqMvrNu3yGOd7Lfnq84zhJh6QHt0nv3E5/mCeDD/P0/kTSU4+5k0jSZTv1AZ3ocqx0EEStMN86P8AHF6U+wAcX5TO8rIiSohJKivJTP8UFlBQGt8G+0qICSorzKSsuSAVAUSoESor0YS4DW2ihYGZjgbuAkUASmOvuN/bSdjrwHPBJd38ojHqSnvrLMpsY1vXO2zYNsC53rMsj+XmGGeSZkWcE91PbqX1GXt6/7qfb5nfZn5+XapefZ53be9pfkJ9HYX5qX2FeHvl5RkG+UZCXF9waBfl5qdvgsfy8vM7tguBYBXl55Af38/P+dZvfeT+PPEMXXEVCEGZPoQP4srs/b2blwFIzW+juq7o2MrN84EfAYyHWwkmHDuekQ4eH+RIiIlkvL6wDu/smd38+2G4CVgOje2j6BeBhoCGsWkREpH9CC4WuzGw8MA1Y1G3/aOBjwK/7eP4cM6sys6rGxsawyhQRGfBCDwUzKyPVE7ja3Xd0e/gG4Gvuvsfxfe4+190r3b2yoqIirFJFRAa8UEcfmVkhqUC4191/30OTSuD+4ILhcOBDZtbh7n8Msy4REelZmKOPDLgNWO3uP+upjbtP6NL+DmCBAkFEJDph9hROBmYDy82sOtj3TWAcgLvv8TqCiIhkXmih4O7P8JbR9X22/0xYtYiISP9kZPSRiIhkB/M4zFewF8ysEWgGtkRdS4wNR+9PX/Qe7Znen75l23t0sLv3OXwz60IBwMyq3L0y6jriSu9P3/Qe7Znen77l6nuk00ciItJJoSAiIp2yNRTmRl1AzOn96Zveoz3T+9O3nHyPsvKagoiIhCNbewoiIhIChYKIiHTKqlAws7PMrNbMXjKzr0ddTxyZ2atmttzMqs2sKup64sDMfmNmDWa2osu+YWa20MzWBLdDo6wxSr28P/9lZq8Hv0fVZvahKGuMkpmNNbMnzGy1ma00s6uC/Tn5O5Q1oRCs0PZL4GzgSGCWmR0ZbVWxdZq7T83FMdTv0B3AWd32fR143N0nAo8H9weqO3j7+wPw8+D3aKq7/yXDNcVJehXJ9wAnAFcGnz05+TuUNaEAzABecveX3b0NuB84N+KaJAu4+9PA1m67zwXuDLbvBD6a0aJipJf3RwJ7WEUyJ3+HsikURgPru9zfQM/Lew50DvzVzJaa2Zyoi4mxEe6+CVL/6IEDI64njv7TzGqC00s5cWrk3eq2imRO/g5lUyj0NOOqxtO+3cnufiyp02xXmtn7oi5IstKvgEOBqcAm4KfRlhO9PlaRzBnZFAobgLFd7o8BNkZUS2y5+8bgtgH4A6nTbvJ29WY2CiC4bYi4nlhx93p3T7h7EvhvBvjvUS+rSObk71A2hcISYKKZTTCzIuACYH7ENcWKmZWaWXl6G/ggsGLPzxqw5gMXBdsXAfMirCV20h92gY8xgH+P9rCKZE7+DmXVN5qDYXE3APnAb9z9+xGXFCtmdgip3gGkFlD6rd4jMLP7gFNJTXVcD3wH+CPwAKmVAF8Dznf3AXmxtZf351RSp44ceBX4XPr8+UBjZqcAfweWA8lg9zdJXVfIud+hrAoFEREJVzadPhIRkZApFEREpJNCQUREOikURESkk0JBREQ6KRRERKSTQkEGPDPbGdweZGYP9dH2ajMryUxl746ZfVQzCcveUihITgqmWt8r7r7R3c/ro9nVwF6FwjupZR/5KKlp5kX6TaEgWcfMxpvZi2Z2ZzCL50NmVhIsMHStmT0DnG9mh5rZo8GMsX83syOC508ws2fNbImZfa/bcVcE2/lm9pNgwaIaM/uCmX0ROAh4wsyeCNrNCtqsMLMfdTnWTjO7zswWASf28t8x3cz+aWbLzGyxmZWb2SAzuz045gtmdlrQ9jNm9osuz11gZqd2ea3vB8d5zsxGmNlJwDnA9cEiOYfu0/8JkrMUCpKtDgfmuvsxwA7g88H+3e5+irvfD8wFvuDuxwFfAW4O2twI/MrdpwObezn+HGACMC14jXvd/SZSkzCe5u6nmdlBwI+A00lNCTHdzNJz6pcCK9z9eHd/pvvBg/m7fgdc5e5TgJnALuBKAHc/GpgF3Glmg/p4L0qB54LjPA181t3/SWpunmuCRXLW9nEMEUChINlrvbv/I9i+Bzgl2P4ddE5zfBLwoJlVA7cA6UneTgbuC7bv7uX4M4Ffu3sHQC9z2kwHnnT3xqDdvUB6qvKH76WuAAABl0lEQVQEqVk1e3M4sMndlwTH3xEc45R0Te7+IrAOmLSH4wC0AQuC7aXA+D7ai/SqIOoCRN6h7pN2pe83B7d5wDZ3n9rP53dn/WzTm93unngHx+/tmB289Y+4rr2Hdv/XJGYJ9O9a3gX1FCRbjTOz9Ln6WcBbTtEEi6C8YmbnQ2r6YzObEjz8D1JTrwN8qpfj/xW43MwKgucPC/Y3AeXB9iLg/WY2PLiYPAt4qp/1vwgcZGbTg+OXB6/1dLomM5tEagbOWlIzlU41szwzG0v/1jfoWqtIvygUJFutBi4ysxpgGKmVwrr7FHCpmS0DVvKvNb2vIrUq3RJgSC/Hv5XUdMg1wfMvDPbPBR4xsyeCqaS/ATwBLAOed/d+zakfrDP+SeD/B8dfSOqv/5uBfDNbTupU2GfcvZVUkL1CavrmnwDP9+Nl7geuCS5Y60Kz9IumzpasE6yTu8DdJ0dcikjOUU9BREQ6qacgEjIz+wOp4a1dfc3dH4uiHpE9USiIiEgnnT4SEZFOCgUREemkUBARkU4KBRER6fS/LdnLM/W3dfkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = hitters_pd.drop('Salary', axis=1)\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "backward_stepwise_subsets = backward_stepwise(X, y)\n",
    "backward_stepwise_subsets['predictor_count'] = np.arange(1, 23)\n",
    "display(backward_stepwise_subsets)\n",
    "\n",
    "sns.lineplot(x='predictor_count', y='rss', data=backward_stepwise_subsets);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE_mean</th>\n",
       "      <th>MSE_folds</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>116599.013674</td>\n",
       "      <td>[109666.77700982487, 37953.39175638854, 156572...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>116599.013674</td>\n",
       "      <td>[109666.7770098258, 37953.39175638837, 156572....</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>116599.013674</td>\n",
       "      <td>[109666.77700982455, 37953.39175638841, 156572...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>116599.013674</td>\n",
       "      <td>[109666.77700982522, 37953.39175638851, 156572...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>114963.338465</td>\n",
       "      <td>[108403.40077544513, 36671.65478063551, 156237...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>113635.830694</td>\n",
       "      <td>[103548.9789642152, 34980.6193407436, 156343.0...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>112921.381689</td>\n",
       "      <td>[103109.24440272473, 35032.07078000676, 155352...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>111081.510651</td>\n",
       "      <td>[103134.03455700394, 35026.977780520996, 15560...</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + Walks +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>110333.364753</td>\n",
       "      <td>[98326.25400454745, 35152.30612149243, 155838....</td>\n",
       "      <td>Salary ~ AtBat + Hits + HmRun + Runs + Walks +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>109979.410931</td>\n",
       "      <td>[95894.57297758816, 35826.8320322134, 155992.7...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>109465.608056</td>\n",
       "      <td>[95474.56130642013, 36674.05253166958, 155314....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>108361.210922</td>\n",
       "      <td>[96478.69537945797, 37028.22102864172, 156381....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>108080.651150</td>\n",
       "      <td>[95668.93613092588, 38005.3532605013, 157372.4...</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>106618.881999</td>\n",
       "      <td>[94639.91163274275, 40255.25372375629, 160138....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>107458.440564</td>\n",
       "      <td>[102234.83312594146, 44434.3360053928, 167235....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRuns + CRBI +...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>110072.974721</td>\n",
       "      <td>[93818.13946902656, 44682.17006469622, 183278....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRuns + CWalks...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>111600.002770</td>\n",
       "      <td>[60065.57347320864, 51405.12237311822, 197485....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRuns + PutOut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>114825.616417</td>\n",
       "      <td>[59641.61586843947, 42937.47973254201, 202202....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CRuns + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>118903.869957</td>\n",
       "      <td>[62757.32659627064, 45045.26338550094, 200078....</td>\n",
       "      <td>Salary ~ AtBat + Hits + CRuns + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>121665.135093</td>\n",
       "      <td>[63353.793665842975, 40682.828845045886, 21475...</td>\n",
       "      <td>Salary ~ Hits + CRuns + PutOuts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>21</td>\n",
       "      <td>127363.032055</td>\n",
       "      <td>[54080.259651755165, 40412.1288381487, 241637....</td>\n",
       "      <td>Salary ~ Hits + CRuns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>22</td>\n",
       "      <td>144100.590668</td>\n",
       "      <td>[43851.510692018775, 75280.33764810773, 317903...</td>\n",
       "      <td>Salary ~ CRuns</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    predictors       MSE_mean  \\\n",
       "0            1  116599.013674   \n",
       "1            2  116599.013674   \n",
       "2            3  116599.013674   \n",
       "3            4  116599.013674   \n",
       "4            5  114963.338465   \n",
       "5            6  113635.830694   \n",
       "6            7  112921.381689   \n",
       "7            8  111081.510651   \n",
       "8            9  110333.364753   \n",
       "9           10  109979.410931   \n",
       "10          11  109465.608056   \n",
       "11          12  108361.210922   \n",
       "12          13  108080.651150   \n",
       "13          14  106618.881999   \n",
       "14          15  107458.440564   \n",
       "15          16  110072.974721   \n",
       "16          17  111600.002770   \n",
       "17          18  114825.616417   \n",
       "18          19  118903.869957   \n",
       "19          20  121665.135093   \n",
       "20          21  127363.032055   \n",
       "21          22  144100.590668   \n",
       "\n",
       "                                            MSE_folds  \\\n",
       "0   [109666.77700982487, 37953.39175638854, 156572...   \n",
       "1   [109666.7770098258, 37953.39175638837, 156572....   \n",
       "2   [109666.77700982455, 37953.39175638841, 156572...   \n",
       "3   [109666.77700982522, 37953.39175638851, 156572...   \n",
       "4   [108403.40077544513, 36671.65478063551, 156237...   \n",
       "5   [103548.9789642152, 34980.6193407436, 156343.0...   \n",
       "6   [103109.24440272473, 35032.07078000676, 155352...   \n",
       "7   [103134.03455700394, 35026.977780520996, 15560...   \n",
       "8   [98326.25400454745, 35152.30612149243, 155838....   \n",
       "9   [95894.57297758816, 35826.8320322134, 155992.7...   \n",
       "10  [95474.56130642013, 36674.05253166958, 155314....   \n",
       "11  [96478.69537945797, 37028.22102864172, 156381....   \n",
       "12  [95668.93613092588, 38005.3532605013, 157372.4...   \n",
       "13  [94639.91163274275, 40255.25372375629, 160138....   \n",
       "14  [102234.83312594146, 44434.3360053928, 167235....   \n",
       "15  [93818.13946902656, 44682.17006469622, 183278....   \n",
       "16  [60065.57347320864, 51405.12237311822, 197485....   \n",
       "17  [59641.61586843947, 42937.47973254201, 202202....   \n",
       "18  [62757.32659627064, 45045.26338550094, 200078....   \n",
       "19  [63353.793665842975, 40682.828845045886, 21475...   \n",
       "20  [54080.259651755165, 40412.1288381487, 241637....   \n",
       "21  [43851.510692018775, 75280.33764810773, 317903...   \n",
       "\n",
       "                                                Model  \n",
       "0   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "1   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "2   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "3   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "4   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "5   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "6   Salary ~ AtBat + Hits + HmRun + Runs + RBI + W...  \n",
       "7   Salary ~ AtBat + Hits + HmRun + Runs + Walks +...  \n",
       "8   Salary ~ AtBat + Hits + HmRun + Runs + Walks +...  \n",
       "9   Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...  \n",
       "10  Salary ~ AtBat + Hits + Runs + Walks + CAtBat ...  \n",
       "11  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  \n",
       "12  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  \n",
       "13  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  \n",
       "14  Salary ~ AtBat + Hits + Walks + CRuns + CRBI +...  \n",
       "15  Salary ~ AtBat + Hits + Walks + CRuns + CWalks...  \n",
       "16  Salary ~ AtBat + Hits + Walks + CRuns + PutOut...  \n",
       "17    Salary ~ AtBat + Hits + Walks + CRuns + PutOuts  \n",
       "18            Salary ~ AtBat + Hits + CRuns + PutOuts  \n",
       "19                    Salary ~ Hits + CRuns + PutOuts  \n",
       "20                              Salary ~ Hits + CRuns  \n",
       "21                                     Salary ~ CRuns  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEKCAYAAAAmfuNnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VdW5//HPkwTCPCbMMwQFcYKIAw44o23FOhWsldv6uxSrbf1Zb9V6b70/be/V9ra2DsVSpaJVHKtQrxNVFAdEAZlFE+YIJowhAQMkeX5/nBU9xkAGc4bkfN+v13mdfZ699j7rHHjxsPZ+zlrm7oiIiMRaWqI7ICIiqUEJR0RE4kIJR0RE4kIJR0RE4kIJR0RE4kIJR0RE4kIJR0RE4kIJR0RE4kIJR0RE4iIj0R1IJllZWT5gwIBEd0NEpElZtGjRNnfPrq2dEk6UAQMGsHDhwkR3Q0SkSTGzDXVpF9NLamY23cyKzGxFDftuMDM3s6zweqyZFZvZkvD4ZVTbcWb2kZnlm9lNUfGBZrbAzPLM7AkzaxnimeF1ftg/IJafU0REahfrezgPAeOqB82sL3A2sLHarjfd/ZjwuC20TQfuA84DhgMTzWx4aH8ncJe75wA7gatC/Cpgp7sPAe4K7UREJIFimnDcfR6wo4ZddwE/B+oyVfVoIN/d17r7fuBxYLyZGXAG8HRoNwO4MGyPD68J+88M7UVEJEHiXqVmZhcAn7j70hp2n2hmS83sRTM7IsR6A5ui2hSEWFdgl7uXV4t/6Ziwvzi0FxGRBIlr0YCZtQFuAc6pYfdioL+7l5rZ+cBzQA5Q08jEDxGnln3V+zQZmAzQr1+/Q/ZfREQaLt4jnMHAQGCpma0H+gCLzayHu+9291IAd38BaBEKCgqAvlHn6ANsBrYBncwso1qc6GPC/o7UfGkPd5/m7rnunpudXWtVn4iINFBcE467L3f3bu4+wN0HEEkMI939UzPrUXWfxcxGh75tB94HckJFWktgAjDbI0uVzgUuCaefBMwK27PDa8L+11xLm4qIJFSsy6JnAvOBw8yswMyuOkTzS4AVZrYUuBuY4BHlwLXAy8CHwJPuvjIccyNwvZnlE7lH82CIPwh0DfHrgZsQEZGvKN57gDtfWk1eYUnM3yum93DcfWIt+wdEbd8L3HuQdi8AL9QQX0ukiq16vAy4tJ7dFRFJOR8VljD19TUcP7ALOd3bx/S9NJeaiEgK+ziMbGKdbEAJR0QkpeUXldK2ZTq9OraK+Xsp4YiIpLC8ohKGdGtHPH4br4QjIpLC8gpLGdIt9pfTQAlHRCRlFe89QFHJPnK6t4vL+ynhiIikqPytkYKBoUo4IiISS3mFpQDk6JKaiIjEUl5RKa1apNG7U+u4vJ8SjohIisorKmVIt3akpcVn9RYlHBGRFJVfWBK3y2mghCMikpJKyg6wubiMId3iUzAASjgiIilpzdY9AOQo4YiISCzlxXEOtSpKOCIiKSi/qJSWGWn07RyfCjVQwhERSUl5RaUMympLRnr80oASjohICsorKonr5TRQwhERSTl795dTsPOzuBYMQOyXmJ5uZkVmtqKGfTeYmZtZVnhtZna3meWb2TIzGxnVdpKZ5YXHpKj4KDNbHo6528L82mbWxczmhPZzzKxzLD+niEhTsnbrHtzjW6EGsR/hPASMqx40s77A2cDGqPB5QE54TAamhrZdgFuB44ksJ31rVAKZGtpWHVf1XjcBr7p7DvBqeC0iIkQupwFxmyW6SkwTjrvPA3bUsOsu4OeAR8XGAw97xLtAJzPrCZwLzHH3He6+E5gDjAv7Orj7fHd34GHgwqhzzQjbM6LiIiIpL6+wlIw0o3/XtnF937jfwzGzC4BP3H1ptV29gU1RrwtC7FDxghriAN3dfQtAeO7WaB9ARKSJyysqZWBWW1rEsUINICOeb2ZmbYBbgHNq2l1DzBsQr2+fJhO5LEe/fv3qe7iISJOTX1TKsJ7xrVCD+I9wBgMDgaVmth7oAyw2sx5ERih9o9r2ATbXEu9TQxygMFxyIzwXHaxD7j7N3XPdPTc7O/trfDQRkeRXdqCCDdv3xG1Z6WhxTTjuvtzdu7n7AHcfQCRpjHT3T4HZwJWhWu0EoDhcDnsZOMfMOodigXOAl8O+EjM7IVSnXQnMCm81G6iqZpsUFRcRSWnrtu2hMgEVahD7suiZwHzgMDMrMLOrDtH8BWAtkA/8BfgRgLvvAG4H3g+P20IM4GrggXDMGuDFEL8DONvM8ohUw93RmJ9LRKSpyisKq3zGuUINYnwPx90n1rJ/QNS2A9ccpN10YHoN8YXAiBri24Ez69ldEZFmL7+whDSDgVnxrVADzTQgIpJS8opKGdC1LZkZ6XF/byUcEZEUUrWsdCIo4YiIpIj95ZWs37YnIfdvQAlHRCRlrN++h/JKJycBJdGghCMikjLyCiMVarqkJiIiMZVXVIKZEo6IiMRYXlEp/bq0oVWL+FeogRKOiEjKyC8sTcgMA1WUcEREUkB5RSVrt5UmZA61Kko4IiIpYMOOvRyocI1wREQktqoq1BL1GxxQwhERSQn5YVnpwdlKOCIiEkN5RaX07tSatplxXXfzS5RwRERSQF5haUIvp4ESjohIs1dR6azZmtiSaFDCERFp9gp27mVfeWXC5lCrooQjItLMfT6Hmi6piYhILFUtK52oOdSqxDThmNl0MysysxVRsdvNbJmZLTGzV8ysV4iPNbPiEF9iZr+MOmacmX1kZvlmdlNUfKCZLTCzPDN7wsxahnhmeJ0f9g+I5ecUEUlmeUUl9OjQig6tWiS0H7Ee4TwEjKsW+627H+XuxwDPA7+M2vemux8THrcBmFk6cB9wHjAcmGhmw0P7O4G73D0H2AlcFeJXATvdfQhwV2gnIpKS8osSX6EGMU447j4P2FEttjvqZVvAaznNaCDf3de6+37gcWC8mRlwBvB0aDcDuDBsjw+vCfvPDO1FRFJKZaWTn8BlpaMl5B6Omf3azDYB3+XLI5wTzWypmb1oZkeEWG9gU1SbghDrCuxy9/Jq8S8dE/YXh/Y19WWymS00s4Vbt25thE8nIpI8Nhd/xt79FQmvUIMEJRx3v8Xd+wKPAteG8GKgv7sfDdwDPBfiNY1M/BDxQx1TU1+muXuuu+dmZ2fX9SOIiDQJVQUDzf6SWh08BlwMkUtt7l4atl8AWphZFpGRS9+oY/oAm4FtQCczy6gWJ/qYsL8j1S7tiYikgvyqkugEzqFWJe4Jx8xyol5eAKwO8R5V91nMbHTo23bgfSAnVKS1BCYAs93dgbnAJeFck4BZYXt2eE3Y/1poLyKSUvKKSshql0nnti0T3RViOoubmc0ExgJZZlYA3Aqcb2aHAZXABmBKaH4JcLWZlQOfARNCkig3s2uBl4F0YLq7rwzH3Ag8bma/Aj4AHgzxB4FHzCyfyMhmQiw/p4hIssorSvyUNlVimnDcfWIN4QdriOHu9wL3HmTfC8ALNcTXEqliqx4vAy6tV2dFRJoZdye/sJRvj+xde+M4SPQ9HBERiZHC3fso2VeeNCMcJRwRkWYqLyy6NiQJSqJBCUdEpNlKhmWloynhiIg0U3lFpXRp25KsdpmJ7gqghCMi0mzlFZYkxZQ2VZRwRESaIXdPqpJoUMIREWmWtpbuo/izA0o4IiISW/mfFwwkR4UaKOGIiDRLn0/aqRGOiIjEUl5RCR1aZZDdPjkq1EAJR0SkWcorLCWne3uSae1JJRwRkWYoP8kq1EAJR0Sk2dleuo/te/Yn1W9wQAlHRKTZyS9Kvgo1UMIREWl2krFCDZRwRESanfyiUtq2TKdnx1aJ7sqXKOGIiDQzeUUlDEmyCjWIccIxs+lmVmRmK6Jit5vZMjNbYmavmFmvEDczu9vM8sP+kVHHTDKzvPCYFBUfZWbLwzF3W/h2zayLmc0J7eeYWedYfk4RkWSSV5h8FWoQ+xHOQ8C4arHfuvtR7n4M8DzwyxA/D8gJj8nAVIgkD+BW4Hgiy0nfGpVApoa2VcdVvddNwKvungO8Gl6LiDR7xXsPUFSyL/USjrvPA3ZUi+2OetkW8LA9HnjYI94FOplZT+BcYI6773D3ncAcYFzY18Hd57u7Aw8DF0ada0bYnhEVFxFp1vK3Rlb5TJZF16JlJOJNzezXwJVAMXB6CPcGNkU1KwixQ8ULaogDdHf3LQDuvsXMujX2ZxARSUafr/KZJMtKR6tzwjGzTOBiYED0ce5+W33f1N1vAW4xs5uBa4lcMqvp7pY3IF4vZjaZyGU5+vXrV9/DRUSSSl5RKa1apNG7U+tEd+Ur6nNJbRaRS1XlwJ6ox9fxGJEkBpERSt+ofX2AzbXE+9QQBygMl9wIz0UH64C7T3P3XHfPzc7O/hofRUQk8fKKShnSrR1paclVoQb1u6TWx92rFwDUm5nluHteeHkBsDpszwauNbPHiRQIFIfLYS8D/xVVKHAOcLO77zCzEjM7AVhA5BLdPVHnmgTcEZ5nfd1+i4g0BfmFJRw/qGuiu1Gj+iScd8zsSHdfXtcDzGwmMBbIMrMCIpfOzjezw4BKYAMwJTR/ATgfyAf2At8HCInlduD90O42d68qRLiaSCVca+DF8IBIonnSzK4CNgKX1uNziog0SSVlB9hcXJZ0c6hVqU/CORn4FzNbB+wjcg/F3f2ogx3g7hNrCD94kLYOXHOQfdOB6TXEFwIjaohvB848WL9ERJqjNVsjdzmSsSQa6pdwzotZL0RE5GvLK6wqiU6+CjWoR8Jx9w0AocQ4uSboERER8otKaZmRRt/OyVehBvWoUjOzC8wsD1gHvAGs54t7JiIikmB5RaUMympLRnpyTpNZn17dDpwAfOzuA4ncI3k7Jr0SEZF6yysqYWiSXk6D+iWcA+FmfJqZpbn7XOCYGPVLRETqYe/+cgp2fpa0BQNQv6KBXWbWDngTeNTMioj8CFRERBJs7dY9uCfnHGpV6jPCGU/k9zHXAS8Ba4BvxaJTIiJSP3lFkQq1IUk4h1qV+lSp7TGz/kCOu88wszZAeuy6JiIidfVxYSkt0o3+XdskuisHVZ8qtX8Fngb+HEK9gedi0SkREamfBWu3Mzi7HS2StEIN6ndJ7RpgDLAbIMyHpmn/RUQSbMHa7SzeuIsJx/WtvXEC1Sfh7HP3/VUvzCyDBiwHICIijeveuflktWvJhNHJvcRKfRLOG2b2C6C1mZ0NPAX8IzbdEhGRuli6aRdv5m3j/5wyiFYtkvu2en0Szk3AVmA58EMiszv/eyw6JSIidXPv3Hw6tm7BFSf0T3RXalWfKrVK4C/hISIiCbb6093MWVXIdWfl0C6zPj+rTIz6VKl908w+MLMdZrY7LH62O5adExGRg7tv7hratkznX04akOiu1El9UuIfgIuA5WHtGhERSZC1W0t5ftlmfnjqYDq1aZno7tRJfe7hbAJWKNmIiCTe1NfX0DI9jatOHpjortRZfRLOz4EXzOxmM7u+6nGoA8xsupkVmdmKqNhvzWy1mS0zs2fNrFOIDzCzz8xsSXjcH3XMKDNbbmb5Zna3mVmIdzGzOWaWF547h7iFdvnhfUbW50sREUlmBTv38uwHnzBxdD+y22cmujt1Vp+E82sic6m1AtpHPQ7lIWBctdgcYERYmvpj4OaofWvc/ZjwmBIVnwpMBnLCo+qcNwGvunsO8Gp4DZHVSavaTg7Hi4g0C39+Yy1m8MPTBiW6K/VSn3s4Xdz9nPqc3N3nmdmAarFXol6+C1xyqHOYWU+gg7vPD68fBi4ksvjbeGBsaDoDeB24McQfDpf/3jWzTmbW09231Kf/IiLJpmh3GU8s3MQlo/rQs2Nyrux5MPUZ4fzTzOqVcOrgB3x51dCBoRLuDTM7JcR6AwVRbQpCDKB7VRIJz92ijtl0kGNERJqsv7y5lvKKSqacNjjRXam3+oxwrgF+bmb7gAOAAe7uHRryxmZ2C5H1dB4NoS1AP3ffbmajgOfM7IjwPtXVVrhQ52PMbDKRy27065fc00KISGrbsWc/jy7YyPhjetO/a9tEd6fe6jzCcff27p7m7q3dvUN4/XmyCcmhTsxsEvBN4LtVVW/uvi+sKIq7LyKy3s5QIqOTPlGH9wE2h+3CcMmt6tJbUYgXAH0Pckz1zzXN3XPdPTc7O7uuH0FEJO7++vY69u6v4Edjm97oBup3Sa02j9SlkZmNI3Kf5QJ33xsVzzaz9LA9iMgN/7XhUlmJmZ0QqtOuBGaFw2YDk8L2pGrxK0O12glAse7fiEhTtrvsAA+9s57zRvQgp3vyLrJ2KI05F8JXLmOZ2UwiN/WzzKwAuJVIVVomMCdUN78bKtJOBW4zs3KgApji7jvCqa4mUvHWmsg9n6r7PncAT5rZVcBG4NIQfwE4H8gnUln3/Ub8nCIicffI/A2UlJVzzelDEt2VBmvMhPOVeyTuPrGGdg/WeLD7M8AzB9m3EBhRQ3w7cGYNcSdyz0lEpMnbu7+cB99ax+mHZTOid8dEd6fBkndpOBERAeCxBRvZsWc/157RdEc30LgJZ3/tTUREpD72lVfwlzfXcuKgrozq3yXR3flaak04ZnZF1PaYavuurdp29xMat2siIvL0ogIKd+9r8qMbqNsIJ3q+tHuq7ftBI/ZFRESiHKioZOrrazi2XydOGtw10d352uqScOwg2zW9FhGRRjJ7yWYKdn7GtacPIVT1Nml1STh+kO2aXouISCOoqHTuez2fYT07cMbh3Wo/oAmoS1n04Wa2jMhoZnDYJrxuWlOViog0ES+t+JS1W/dw3+Ujm8XoBuqWcIbFvBciIvI5d+feufkMzm7LuBE9Et2dRlNrwnH3DdGvzawrkVkBNoY5z0REpBG9trqID7fs5neXHk16WvMY3UDdyqKfN7MRYbsnsIJIddojZnZdjPsnIpJS3J17Xsunb5fWXHBMr0R3p1HVpWhgoLtXLRH9fWCOu38LOB6VRYuINKp31mxnyaZdTDltMC3Sm9dkMHX5NAeits8kMjEm7l4CVMaiUyIiqeqe1/Lo3iGTS0b1qb1xE1OXooFNZvZjImvMjAReAjCz1kCLGPZNRCSlzHhnPe+u3cF/fHM4mRnpie5Oo6vLCOcq4AjgX4DvuPuuED8B+GuM+iUiklIenr+eW2ev5Ozh3bnyxP6J7k5M1KVKrQiYUkN8LjA3Fp0SEUklj8xfzy9nRZLNfZePbHb3bqrUmnDMbPah9rv7BY3XHRGR1PLIuxv4j1krOWtYJNm0zGieyQbqdg/nRGATMBNYgOZPExFpFH97dwP/8dwKzhrWjT99t3knG6hbwukBnA1MBC4H/heY6e4rY9kxEZHm7NEFG/j351Zw5uHduC8Fkg3UoWjA3Svc/SV3n0SkUCAfeD1Urh2SmU03syIzWxEV+62ZrTazZWb2rJl1itp3s5nlm9lHZnZuVHxciOWb2U1R8YFmtsDM8szsCTNrGeKZ4XV+2D+gjt+HiEjMPbZgI7c8u4IzDu/Gn64Y2Swr0mpSp5Qa/gG/CPgbcA1wN/D3Ohz6EDCuWmwOMMLdjwI+Bm4O7zEcmECkIm4c8CczSzezdOA+4DxgODAxtAW4E7jL3XOAnUQq6gjPO919CHBXaCciknAz39vIL55dzumHZTM1hZIN1G1qmxnAO0R+g/P/3P04d7/d3T+p7Vh3nwfsqBZ7xd3Lw8t3gapfN40HHnf3fe6+jshIanR45Lv7WnffDzwOjLfI9KlnAE+H42cAF0ada0bYfho405rLdKsi0mQ9/t5Gbv77csYels3UK0alVLKBut3D+R6wBxgK/CTq320D3N07fI33/wHwRNjuTSQBVSkIMYgULUTHjwe6Aruikld0+95Vx7h7uZkVh/bbqnfAzCYDkwH69ev3NT6KiMjBPfH+Rm76+3JOG5rN/VeMolWL1Eo2ULff4cTkTpaZ3QKUA49WhWp6e2oehfkh2h/qXF8Nuk8DpgHk5uZqQTkRaXRPvr/p82Tz5++lZrKBuo1wGp2ZTQK+CZzp7lX/yBcAfaOa9QE2h+2a4tuATmaWEUY50e2rzlVgZhlAR6pd2hMRiYcnF27ixr8v45Sc1E42UMeigcZkZuOAG4EL3H1v1K7ZwIRQoDAQyAHeA94HckJFWksihQWzQ6KaC1wSjp8EzIo616SwfQnwWlRiExGJi6cWbuLGZ5Zx8pAspqV4soEYj3DMbCYwFsgyswLgViJVaZnAnHA/6F13n+LuK83sSWAVkUtt17h7RTjPtcDLQDowPeo3QDcCj5vZr4APgAdD/EEi6/XkExnZTIjl5xQRqe7pRQX8PCSbv1yZm/LJBsD0H/8v5Obm+sKFCxPdDRFp4mYv3cxPH/+AMYOzeGBS8082ZrbI3XNra9f8f9oqIhJHm3bs5aZnlnFc/y4a2VSjhCMi0kgqK51/e3opaWbcNeEYWrdUsommhCMi0kgenl+1gNowendqnejuJB0lHBGRRrB+2x7ueGk1Yw/L5rLcvrUfkIKUcEREvqaKSueGp5bSMj2NOy46Cs2kVbOE/PBTRKQ5+evb61i4YSe/v+xoenRslejuJC2NcEREvob8olJ+8/JHnDWsO98+tnftB6QwJRwRkQYqr6jkZ08tpU3LdP7rohG6lFYLXVITEWmgaW+uZemmXdw98Vi6tdeltNpohCMi0gAffVrCH+bkcf6RPfjWUT0T3Z0mQQlHRKSeDlRU8rOnltC+VQa3j9eltLrSJbVG8Hb+NpYW7Ep0N+os3YxxI3rQv2vbRHdFpEma+voaVnyym6nfHUnXdpmJ7k6ToYTTCOauLuKBt9Yluhv18rs5HzPl1EFcPXaIpt8QqYeVm4u5+9U8Lji6F+cdqUtp9aHZoqM0dLboAxWVVFQ2ne9x+579/Oal1cxaspnenVrzH98cxrlH9NBlAZFa7C+v5IJ732L7nv28ct2pdG7bMtFdSgp1nS1aI5xG0CI9jaY0IWzvTq3544RjuXx0P26dvZIpf1vMKTlZ/OcFRzA4u12iuyeStO55LY/Vn5bwwJW5SjYNoKKBFHb8oK48/+OTufVbw1mycRfj/jCPO15czZ595YnumkjSWVawiz+9voaLR/bhrOHdE92dJkkJJ8VlpKfx/TEDee2GsYw/pjf3v7GGM3/3BrOXbkaXW0Uiyg5U8LMnl5LdLpNffmt4orvTZMU04ZjZdDMrMrMVUbFLzWylmVWaWW5UfICZfWZmS8Lj/qh9o8xsuZnlm9ndFm42mFkXM5tjZnnhuXOIW2iXb2bLzGxkLD9nc5DdPpP/ufRonrn6JLLat+QnMz9g4l/e5aNPSxLdNZGE+8M/88grKuWOi4+kY+sWie5OkxXrEc5DwLhqsRXARcC8GtqvcfdjwmNKVHwqMBnICY+qc94EvOruOcCr4TXAeVFtJ4fjpQ5G9e/MrGtO5lcXjuDDLSWcf/eb3P78KnaXHUh010QSYvHGnUybt4YJx/Vl7GHdEt2dJi2mCcfd5wE7qsU+dPeP6noOM+sJdHD3+R65xvMwcGHYPR6YEbZnVIs/7BHvAp3CeaQO0tOMK07oz9wbxnJZbl+mv72OM/7nDZ5ZVEBlE6rGE/m6yg5UcMOTS+nZsTW3fGNYorvT5CXbPZyBZvaBmb1hZqeEWG+gIKpNQYgBdHf3LQDhuVvUMZsOcozUUZe2Lfnvi45k1jVj6NO5NT97aikXTX2HuR8V6f6OpITfvvwRa7ft4TeXHEX7VrqU9nUlU8LZAvRz92OB64HHzKwDUNOPQ2r7167Ox5jZZDNbaGYLt27dWq8Op4qj+nTi71efxG8uOYqi3WV8/6/v86173+KlFVs04pFm69UPC5n+9jq+d0J/xgzJSnR3moWkSTjuvs/dt4ftRcAaYCiR0UmfqKZ9gM1hu7DqUll4LgrxAqDvQY6p/r7T3D3X3XOzs7Mb6+M0O2lpxmW5fXn9307nNxcfRWlZOVP+tphz/zCP5z74hPKKykR3UaTRrNq8mx/P/IARvTpy8/mHJ7o7zUbSJBwzyzaz9LA9iMgN/7XhUlmJmZ0QqtOuBGaFw2YDk8L2pGrxK0O12glAcdWlN/l6Wmakcdlxffnn9afxxwnHYAbXPbGEM3//Bk+8v5H95Uo80rQV7i7jqhnv07F1Cx6YlEublvp9fGOJ6dQ2ZjYTGAtkAYXArUSKCO4BsoFdwBJ3P9fMLgZuA8qBCuBWd/9HOE8ukYq31sCLwI/d3c2sK/Ak0A/YCFzq7jtCYrqXSDXbXuD77l7rnDUNndomlVVWOq+sKuS+ufks/6SYXh1bMWXsYC7L7UurpjT9ggiwd385l/15Puu27uGpKScxvFeHRHepSajr1DaaSy2KEk7DuTtvfLyVe1/LZ+GGnWS1y2TyqQP57vH9aZup/yFK8quodKb8bRGvfljIA5NyOeNwzSZQV5pLTeLKzBh7WDdOG5rNgnU7uPe1fP7rhdX86fU1/GDMQCadNEA/mJOkdudLq5mzqpBbvzVcySZGlHCkUZkZJwzqygmDuvLBxp3cNzef38/5mGnz1nL64d0YM7grY4Zk0bdLm0R3VeRzjy3YyLR5a5l0Yn++P2ZgorvTbOmSWhRdUouNlZuLmf7WeublbWVryT4A+nRuzZjBWZw0pCsnDu6q9eAlYd7K28akv77HKTlZPHBlLhnpSVNL1WToHk4DKOHElruTX1TKO2u2886abcxfs53dZZGZqXO6tWPMkCxOGtyV4wd11eU3iYu8whIumvoOvTu15qkpJ+rHnQ2khNMASjjxVVHprNq8m7fXbOPt/G28v34HZQcqSTM4sndHThycxZghXTluQBdVvEmj21a6j2//6W0+21/JrGvH0LtT60R3qclSwmkAJZzE2ldewZKNuz4fAX2wcRfllU6Xti25ffwIvnGUpsOTxlF2oILL//Iuq7bs5onJJ3J0306J7lKTpoTTAEo4yWXPvnLeW7+D37/yMcs/KeYbR/Xk9vEj6KKVFuVrcHd+8vgS/rF0M1O/O5LzjtR/ZL6uuiYc3R2TpNU2M4PTD+vGsz86iX879zBeWfkp59z1Bi+t0KQR0nB3/TOPfyzdzI3jDleyiTMlHEl6GelpXHP6EP7x45P0PiUkAAAQ00lEQVTp0bEVU/62mB/P/ICde/YnumvSxDz7QQF3v5rHZbl9mHLaoER3J+Uo4UiTcXiPDjz7ozH87OyhvLRiC2ffNY+XV36a6G5JE/Heuh3c+PRyThzUlV9deCRh4WCJIyUcaVJapKfx4zNzmH3tyXRrn8kPH1nEdY9/wK69Gu3Iwa3ftofJjyykT5fW3H/FKFpm6J++RNC3Lk3SsJ4dmHXtGP7vWUN5fllktDNnVWGiuyVJaNfe/fzgofcxYPqk4+jYRr+1SRQlHGmyWqSn8dOzcph17Riy2mXyrw8v5PonllC890CiuyZJwN15cfkWxt/3NgU7P2PalbkMyGqb6G6lNCUcafKO6NWRWdeM4adn5jB76WbOvusNXv1Qo51UtnjjTi65fz5XP7qYVhnpzPjBaI4b0CXR3Up5+h1OFP0Op+lb8UkxNzy1lNWflnDxyD788LRB5HRrpxvEKWLj9r3c+fJq/nfZFrLbZ3LDOUO5ZFRf0tP05x9L+uFnAyjhNA/7yyu597U87nt9DRWVTvcOmZySk80pOVmcPCSLru0yE91FaWTFew9wz2t5zJi/noy0NCafOojJpw7SWkxxooTTAEo4zcuW4s+Y9/FW5uVF5mrbFe7tHNGrw+cJaFT/zpqnrQnbX17JI+9u4O5X89hddoDLRvXl+nOG0r2DZh+Pp6RIOGY2HfgmUOTuI0LsUuA/gWHA6Oiln83sZuAqIktM/8TdXw7xccAfgXTgAXe/I8QHAo8DXYDFwPfcfb+ZZQIPA6OA7cB33H19bf1Vwmm+KiqdlZuLeTNvG/M+3srijTs5UOG0apHG8QO7ckpOFqfkZDO0uy6/NQXuzosrPuXOl1azYfteTsnJ4hfnD2NYTy0JnQjJknBOBUqBh6MSzjCgEvgzcENVwjGz4cBMYDTQC/gnMDSc6mPgbKAAeB+Y6O6rzOxJ4O/u/riZ3Q8sdfepZvYj4Ch3n2JmE4Bvu/t3auuvEk7q2LOvnAXrtjPv4228mbeVNVv3ANCtfSYn52QxekAX+nVpQ98ubejZsZXWSEkiizbs5Nf/u4rFG3dxWPf2/OIbwzhtaHaiu5XSkmKJaXefZ2YDqsU+BGr6X+R44HF33wesM7N8IskHIN/d14bjHgfGm9mHwBnA5aHNDCIjp6nhXP8Z4k8D95qZua4fStA2M4MzDu/++VLCm3d9xlt525iXt5W5q4v4++JPPm+bnmb07NiKvp3b0LdL6/D8xXZ2+0yNiuKgekHAHRcdyaW5KghoSpLpjlpv4N2o1wUhBrCpWvx4oCuwy93La2jfu+oYdy83s+LQfltsui5NXa9OrbnsuL5cdlxfKiudT3Z9xqYde9m0cy+bdnwWnvcy96MvVi2tkpmRRp/OrSNJqHMbhvZoz/CeHTi8R3vdtG4kr6z8lGtnfkC6GT89M0cFAU1UMv2J1fTfFKfm3wr5Idof6lxffVOzycBkgH79+tXeS2n20tIsjGDa1Li/7EAFBTv3smnnZxTsiDxXJadFG3ZSElYxNYMBXdsyvGcHhvfq8PlzN42I6mXWkk+4/smlHNm7I/dfMYoeHVUQ0FQlU8IpAPpGve4DbA7bNcW3AZ3MLCOMcqLbV52rwMwygI7Ajpre1N2nAdMgcg+ncT6KNGetWqQzpFt7hnRr/5V97s6W4jJWbd7Nqi27WbV5N8s/KeZ/l3+xpEKXti2/koQGZbXVfaIazHxvI794djnHD+zCA5OOo51GNU1aMv3pzQYeM7PfEykayAHeIzJayQkVaZ8AE4DL3d3NbC5wCZFKtUnArKhzTQLmh/2v6f6NxIOZ0atTa3p1as1Zw7t/Hi8pO8DqT0siiSgko4feWc/+8koAWmakcWzfTkwY3ZfzRvRUqTbw4FvruP35VYw9LJv7rxil76QZiHWV2kxgLJAFFAK3Ehlp3ANkA7uAJe5+bmh/C/ADoBy4zt1fDPHzgT8QKYue7u6/DvFBfFEW/QFwhbvvM7NWwCPAseH9JlQVHRyKqtQkng5UVLJ26x5WbSlm1ebdzFlVyPrte+nUpgUXj+zDxNH9GNKtXaK7GXfuzr2v5fO7OR9z3oge/HHCsZrdOcklRVl0U6OEI4lUWenMX7udxxZs5OWVn1Je6Ywe2IXvHt+PcSN6kJnR/P+H7+7c8dJq/vzGWi4a2ZvfXHyULjU2AUlRFi0idZeWZowZksWYIVlsLdnH04sKmPneRn76+BI6t2nBJaMio55B2c1z1FNZ6dw6eyWPvLuBK07ox20XjCBNJc/NikY4UTTCkWRTWem8vWYbjy3YyJxVhZRXOicO6srlx/fj3CN6NJtLTeUVldz4zHKeWVzAD08dxE3nHa5KviZEl9QaQAlHkllRSRlPLYyMegp2fkbXti0/H/U05XVe9pdXct0TH/DC8k+5/uyh/PiMIUo2TYwSTgMo4UhTUFnpvJm/jccWbOCfHxZRUem0b5VB9w6t6N4hk+7tW9GtajvqObt9ZtLdByo7UMHVf1vE3I+28u/fGMb/OWVQorskDaB7OCLNVFqacdrQbE4bmk3h7jKeX7aFTTv2Uri7jMLdZSxYt4OikjIOVHz1P5Nd2rakW/svEtGRvTtyaW7fhJQcl+4r519nLOTdddv574uOZOJo/fC6udMIJ4pGONJcuDs79x74PAkV7d7Hp2G7cPc+ikrK+LS4jKKSffTu1JrrzsrhopF94jYvWfHeA0z663ss/6SY3192NOOP6V37QZK0dEmtAZRwJNW8k7+NO19azdKCYnK6teOGcw/jnOHdY3oPZVvpPr734HusKSrlnsuP5dwjesTsvSQ+6ppwmkeJi4g0yElDsnjumjHcf8VIKtz54SOL+Paf3mH+mu0xeb/12/bwnT/PZ922Uh6YlKtkk2I0womiEY6ksvKKSp5ZXMAf/pnHluIyTh2azc/PPYwRvTs2+Jzuzsowi8KcVYWs2rKbdpkZTP+X4xg9sEsj9l4SSZfUGkAJRyRSOfbI/A3c93o+u/Ye4FtH9+JnZw+tc+n1vvIK3l27gzmrPuXVD4vYUlyGGeT278xZw7rzjaN60qdzzTNxS9OkhNMASjgiX9hddoC/zFvLA2+u40BFJd85ri8/OTOH7h2+ujzAzj37mftREf/8sJA3PtrKnv0VtG6RzqlDszhrWHfOOLwbXdtlJuBTSDwo4TSAEo7IVxWVlHHva/k8tmAjGenG98cMZMppg9m5Zz///DByqWzhhp1UVDrd2mdy1vDunD2sOycO7qoZnlOEEk4DKOGIHNzG7Xv5/ZyPmLV0My3S0thfEVla4fAe7Tl7eHfOGtadI3t31PxnKUgJpwGUcERqt2rzbma+t5FB2W05a1j3g66MKqlDMw2ISEwM79WB2y8ckehuSBOk3+GIiEhcKOGIiEhcKOGIiEhcxDThmNl0MysysxVRsS5mNsfM8sJz5xAfa2bFZrYkPH4Zdcw4M/vIzPLN7Kao+EAzWxDO9YSZtQzxzPA6P+wfEMvPKSIitYv1COchYFy12E3Aq+6eA7waXld5092PCY/bAMwsHbgPOA8YDkw0s+Gh/Z3AXeFcO4GrQvwqYKe7DwHuCu1ERCSBYppw3H0esKNaeDwwI2zPAC6s5TSjgXx3X+vu+4HHgfEWmc72DODpGs4V/R5PA2ealhAUEUmoRNzD6e7uWwDCc7eofSea2VIze9HMjgix3sCmqDYFIdYV2OXu5dXiXzom7C8O7b/CzCab2UIzW7h169av/+lERKRGyVQ0sBjo7+5HA/cAz4V4TSMTP0T8UMd8Neg+zd1z3T03Ozu7nl0WEZG6SsQPPwvNrKe7bzGznkARgLvvrmrg7i+Y2Z/MLIvIyKVv1PF9gM3ANqCTmWWEUUxVnKhjCswsA+jIVy/tfcWiRYu2mdmecG6pWRb6fmqj7+jQ9P3Urql9R/3r0igRCWc2MAm4IzzPAjCzHkChu7uZjSYy+toO7AJyzGwg8AkwAbg8tJsLXELkvs7n54p6j/lh/2tehzl83D3bzBbWZYqGVKXvp3b6jg5N30/tmut3FNOEY2YzgbFAlpkVALcSSTRPmtlVwEbg0tD8EuBqMysHPgMmhCRRbmbXAi8D6cB0d18ZjrkReNzMfgV8ADwY4g8Cj5hZPpGRzYRYfk4REamdJu+sprn+z6Kx6Pupnb6jQ9P3U7vm+h0lU9FAspiW6A4kOX0/tdN3dGj6fmrXLL8jjXBERCQuNMIREZG4UMIJDjZfm3zBzNab2fIw113Kr1RXn7kCU9VBvqP/NLNPouZNPD+RfUwkM+trZnPN7EMzW2lmPw3xZvn3SAmHWudrky87Pcx11+xuaDbAQ9RvrsBU9BBf/Y4gMgdi1byJL8S5T8mkHPiZuw8DTgCuCf/2NMu/R0o4ETXO15bgPkmSa6S5Apu1g3xHErj7FndfHLZLgA+JTM3VLP8eKeFEHGy+NvkyB14xs0VmNjnRnUlSh5orUL5wrZktC5fcmsXloq8rLKNyLLCAZvr3SAknos5zr6W4Me4+ksilx2vM7NREd0iapKnAYOAYYAvwu8R2J/HMrB3wDHBd9DRfzY0STsTB5muTKO6+OTwXAc8SuRQpX1YY5ggkeq5A+YK7F7p7hbtXAn8hxf8emVkLIsnmUXf/ewg3y79HSjgR7xPmawurhk4gMh+bBGbW1szaV20D5wArDn1USqqaxw++PL+fBFX/kAbfJoX/HoV1uh4EPnT330ftapZ/j/TDzyCUZv6BL+Zr+3WCu5RUzGwQkVENRObgeyzVv6PouQKBQiJzBT4HPAn0I8wV6O4pe9P8IN/RWCKX0xxYD/yw6n5FqjGzk4E3geVAZQj/gsh9nGb390gJR0RE4kKX1EREJC6UcEREJC6UcEREJC6UcEREJC6UcEREJC6UcESSkJmVhudeZvZ0LW2vM7M28emZSMOpLFokTsws3d0r6ti21N3b1bHteiDX3bfFoi8ijUUjHJFGYGYDzGy1mc0Ik1I+bWZtwhpCvzSzt4BLzWywmb0UJkB908wOD8cPNLP5Zva+md1e7bwrwna6mf1PWJNomZn92Mx+AvQC5prZ3NBuYmizwszujDpXqZndZmYLgBPN7A4zWxXO9T/x/L4kNWUkugMizchhwFXu/raZTQd+FOJl7n4ygJm9Ckxx9zwzOx74E3AG8Edgqrs/bGbXHOT8k4GBwLHuXm5mXdx9h5ldT2Sdom1m1gu4ExgF7CQyu/eF7v4c0BZY4e6/NLMuRKZUOdzd3cw6xeILEYmmEY5I49nk7m+H7b8BJ4ftJ+DzGYFPAp4ysyXAn4GqecXGADPD9iMHOf9ZwP3uXg5wkKlOjgNed/etod2jQNWs3hVEJokE2A2UAQ+Y2UXA3vp8UJGG0AhHpPFUvyFa9XpPeE4Ddrn7MXU8vjqrY5uDKau6bxNGSKOBM4lMVnstkZGWSMxohCPSePqZ2YlheyLwVvTOsM7JOjO7FCIzBZvZ0WH320T+4Qf47kHO/wowxcwywvFdQrwEaB+2FwCnmVlWWDp9IvBG9ROF0VbHsLzzdUQm0xSJKSUckcbzITDJzJYBXYgsNFbdd4GrzGwpsJIvljL/KZFF7d4HOh7k/A8QmTl4WTj+8hCfBrxoZnPDrMs3A3OBpcBid69pavv2wPOhr28A/7d+H1Wk/lQWLdIIwvLAz7v7iAR3RSRpaYQjIiJxoRGOiIjEhUY4IiISF0o4IiISF0o4IiISF0o4IiISF0o4IiISF0o4IiISF/8f0RXv9sy17VgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Perform cross validation for each subset of predictors\n",
    "results = []\n",
    "for preds in backward_stepwise_subsets['predictors']:\n",
    "    f = 'Salary ~ {}'.format(' + '.join(preds))\n",
    "    results += [cross_val(f, X, y, 10)]\n",
    "\n",
    "results_b_df = pd.DataFrame({'predictors': list(np.arange(1, len(results)+1)),\n",
    "                           'MSE_mean': [np.mean(i[0]) for i in results],\n",
    "                           'MSE_folds': [i[0] for i in results],\n",
    "                           'Model': [i[1] for i in results]})\n",
    "display(results_b_df)\n",
    "sns.lineplot(x='predictors', y='MSE_mean', data=results_b_df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "</br><h4>Best Subset Selection from 14 predictors, with 10-fold Cross Validation</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE_mean</th>\n",
       "      <th>MSE_folds</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>109522.011671</td>\n",
       "      <td>[86417.0694422955, 30716.914843531533, 165276....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   predictors       MSE_mean  \\\n",
       "7           8  109522.011671   \n",
       "\n",
       "                                           MSE_folds  \\\n",
       "7  [86417.0694422955, 30716.914843531533, 165276....   \n",
       "\n",
       "                                               Model  \n",
       "7  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Salary ~ AtBat + Hits + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "</br><h4>Backward Stepwise Selection from all 22 predictors, with 10-fold Cross Validation</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predictors</th>\n",
       "      <th>MSE_mean</th>\n",
       "      <th>MSE_folds</th>\n",
       "      <th>Model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>106618.881999</td>\n",
       "      <td>[94639.91163274275, 40255.25372375629, 160138....</td>\n",
       "      <td>Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    predictors       MSE_mean  \\\n",
       "13          14  106618.881999   \n",
       "\n",
       "                                            MSE_folds  \\\n",
       "13  [94639.91163274275, 40255.25372375629, 160138....   \n",
       "\n",
       "                                                Model  \n",
       "13  Salary ~ AtBat + Hits + Walks + CAtBat + CRuns...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Salary ~ AtBat + Hits + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts + Division_W\n"
     ]
    }
   ],
   "source": [
    "display(HTML('</br><h4>Best Subset Selection from 14 predictors, with 10-fold Cross Validation</h4>'))\n",
    "best_subset_choice = results_a_df[results_a_df['MSE_mean'] == results_a_df['MSE_mean'].min()]\n",
    "display(best_subset_choice)\n",
    "print(best_subset_choice['Model'].iloc[0])\n",
    "\n",
    "display(HTML('</br><h4>Backward Stepwise Selection from all 22 predictors, with 10-fold Cross Validation</h4>'))\n",
    "backward_select_choice = results_b_df[results_b_df['MSE_mean'] == results_b_df['MSE_mean'].min()]\n",
    "display(backward_select_choice)\n",
    "print(backward_select_choice['Model'].iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Abov we used backward stepwise selection to make it computationally feasible (on my laptop) to select from all 22 features. \n",
    "\n",
    "We see that 10-fold cross validation selects a 14 variable model. The estimated test MSE is 106,618. This represents an improvement on the lowest esitmate MSe achived for best subset selection from 14 features which was 109,522.\n",
    "\n",
    "Note that Backward Stepwise Selection found the same model, but gained an improved MSE by adding one additional predictor, 'Division_W', which wasn't available to best subset selection due to the constraints of the available compute. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab 2: Ridge Regression and the Lasso\n",
    "\n",
    "### 6.6.1 Ridge Regression\n",
    "\n",
    "Let's perform ridge regression with 10-fold cross-validation to produce results comparable to those obtained for model selection approaches aboce."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ridge_cv(X, y, λ, k):\n",
    "    \"\"\"Perform ridge regresion with \n",
    "    k-fold cross validation to return mean MSE scores for each fold\"\"\"\n",
    "    # Split dataset into k-folds\n",
    "    # Note: np.array_split doesn't raise excpetion is folds are unequal in size\n",
    "    X_folds = np.array_split(X, k)\n",
    "    y_folds = np.array_split(y, k)\n",
    "    \n",
    "    MSEs = []\n",
    "    for f in np.arange(len(X_folds)):\n",
    "        # Create training and test sets\n",
    "        X_test  = X_folds[f]\n",
    "        y_test  = y_folds[f]\n",
    "        X_train = X.drop(X_folds[f].index)\n",
    "        y_train = y.drop(y_folds[f].index)\n",
    "        \n",
    "        # Fit model\n",
    "        model = linear_model.Ridge(alpha=λ, fit_intercept=False, solver='cholesky').fit(X_train, y_train)\n",
    "        \n",
    "        # Measure MSE\n",
    "        y_hat = model.predict(X_test)\n",
    "        MSEs += [mse(y_hat, y_test)]\n",
    "    return MSEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuY3VV97/H3ZzKZZBgCuQ0RcjkJNSDQQ4GMlhqrtFa5VA9ye0pv5FCOES/H0/pUi49V6+UPOK038ACNiiR6jFKoJUewHE5E04dyMcEYAhgJIGYgTUImJCEzZDKzv+eP39rDzs6eW2Z+s/ee+byeZz/7t9fvMmtlJ/nOWuv7Wz9FBGZmZnlqqHYFzMxs/HOwMTOz3DnYmJlZ7hxszMwsdw42ZmaWOwcbMzPLnYONmZnlzsHGzMxy52BjZma5a6x2BWrF7NmzY+HChdWuhplZXdmwYcNLEdE62HEONsnChQtZv359tathZlZXJD0/lOM8jGZmZrlzsDEzs9w52JiZWe4cbMzMLHcONmZmljtno41AoRDsPtBNd08vTY2TmNXSREODql0tM7Oa42BzlAqFYMuO/bx31Xra93Qxb0YzX7uqjVPnTHPAMTMrk9swmqTbJO2UtLmk7ApJT0gqSGorO/5MSQ+l/Y9Lmlq2f03ZtWZKul/S0+l9RiqXpBslbZW0SdI5ebRv94HuvkAD0L6ni/euWs/uA915/Dgzs7qW55zN7cAFZWWbgUuBdaWFkhqBbwPXRsQZwHnAoZL9lwKvlF3rOmBtRCwG1qbPABcCi9NrOXDLyJtypO6e3r5AU9S+p4vunt48fpyZWV3LLdhExDqgo6zsqYjYUuHwdwKbIuLn6bjdEdELIOlY4CPA58vOuRhYmbZXAu8pKV8VmYeB6ZJOHI02lWpqnMS8Gc2Hlc2b0UxT46TR/lFmZnWvVrLRTgFC0n2SHpP0sZJ9nwO+AHSWnTMnIrYDpPcTUvlcYFvJce2pbFTNamnia1e19QWc4pzNrJam0f5RZmZ1r1YSBBqBtwBvJAsqayVtAHYDr4+Iv5K0cIjXqjQ7HxUPlJaTDbWxYMGCYVW4oUGcOmca3//AUmejmZkNolaCTTvwk4h4CUDSvcA5ZPM0SyT9iqyuJ0j6cUScB+yQdGJEbE/DZDtLrjW/5NrzgBcr/dCIWAGsAGhra6sYkAbS0CBap00Z7mlmZhNOrQyj3QecKemYlCzwNuDJiLglIk6KiIVkPZ9fpkADsAZYlraXAXeXlF+VstLOBfYWh9vMzKw6cuvZSFpNllU2W1I78GmyhIGbgFbgHkkbI+L8iNgj6YvAT8mGvO6NiHsG+RHXA3dIugb4NXBFKr8XuAjYSjYkd/XotszMzIZLEcMePRqX2traws+zMTMbHkkbIqJtsONqZRjNzMzGMQcbMzPLnYONmZnlzsHGzMxy52BjZma5c7AxM7PcOdiYmVnuHGzMzCx3DjZmZpY7BxszM8tdraz6XLcKhWD3gW4/ZsDMbAAONiNQKARbduznvavW076nq+8BaqfOmeaAY2ZWwsNoI7D7QHdfoAFo39PFe1etZ/eB7irXzMystjjYjEB3T29foClq39NFd09vlWpkZlabHGxGoKlxEvNmNB9WNm9GM02Nk6pUIzOz2uRgMwKzWpr42lVtfQGnOGczq6WpyjUzM6stThAYgYYGceqcaXz/A0udjWZmNoDcejaSbpO0U9LmkrIrJD0hqSCprez4MyU9lPY/LmmqpGMk3SPpF6n8+pLjp0j6nqStkh6RtLBk38dT+RZJ5+fVRsgCTuu0KcydcQyt06Y40JiZVZDnMNrtwAVlZZuBS4F1pYWSGoFvA9dGxBnAecChtPsfIuINwNnAUkkXpvJrgD0R8XrgS8AN6VqnA1cCZ6Sff7MkT6KYmVVRbsEmItYBHWVlT0XElgqHvxPYFBE/T8ftjojeiOiMiAdSWTfwGDAvnXMxsDJt3wm8XZJS+Xcj4mBEPAdsBd40ys0zM7NhqJUEgVOAkHSfpMckfaz8AEnTgXcDa1PRXGAbQET0AHuBWaXlSXsqMzOzKqmVBIFG4C3AG4FOYK2kDRGxFvqG2VYDN0bEs+mcSpMjMUD5ESQtB5YDLFiwYEQNMDOz/tVKz6Yd+ElEvBQRncC9wDkl+1cAT0fEl8vOmQ99weh4smG7vvJkHvBipR8aESsioi0i2lpbW0etMWZmdrhaCTb3AWem7LNG4G3AkwCSPk8WSP6y7Jw1wLK0fTnwo4iIVH5lylZbBCwGHh2DNpiZWT9yG0aTtJosq2y2pHbg02Q9j5uAVuAeSRsj4vyI2CPpi8BPyYa87o2IeyTNAz4B/AJ4LJv/56sR8XXgG8C3JG1N170SICKekHQHWbDqAT4YEV4/xsysipR1BqytrS3Wr19f7WqYmdWVNL/eNthxtTKMZmZm41itZKPVNT9AzcxsYA42I+QHqJmZDc7DaCPkB6iZmQ3OwWaE/AA1M7PBOdiMkB+gZmY2OAebEfID1MzMBucEgRHyA9TMzAbnYDMKig9QMzOzyjyMZmZmuXOwMTOz3DnYmJlZ7hxszMwsdw42ZmaWOwcbMzPLnYONmZnlzsHGzMxyl1uwkXSbpJ2SNpeUXSHpCUkFSW1lx58p6aG0/3FJU1P5kvR5q6QblZ4NLWmmpPslPZ3eZ6RypeO2Stok6Zy82liqUAh27T/IC3s62bX/IIWCn4BqZlaUZ8/mduCCsrLNwKXAutJCSY3At4FrI+IM4DzgUNp9C7AcWJxexWteB6yNiMXA2vQZ4MKSY5en83NVfKbNJTc/yNIbHuCSmx9ky479DjhmZkluwSYi1gEdZWVPRcSWCoe/E9gUET9Px+2OiF5JJwLHRcRDERHAKuA96ZyLgZVpe2VZ+arIPAxMT9fJjZ9pY2Y2sFqZszkFCEn3SXpM0sdS+VygveS49lQGMCcitgOk9xNKztnWzzm58DNtzMwGVisLcTYCbwHeCHQCayVtAPZVOHawsalKyy1XPEfScrKhNhYsWDDkypYrPtOmNOD4mTZmZq+plZ5NO/CTiHgpIjqBe4FzUvm8kuPmAS+m7R3F4bH0vrPkWvP7OecwEbEiItoioq21tfWoK+9n2piZDaxWejb3AR+TdAzQDbwN+FJEbJe0X9K5wCPAVcBN6Zw1wDLg+vR+d0n5hyR9F/htYG9xuC0vfqaNmdnAcgs2klaTZZXNltQOfJosYeAmoBW4R9LGiDg/IvZI+iLwU7Ihr3sj4p50qfeTZbY1Az9ML8iCzB2SrgF+DVyRyu8FLgK2kg3JXZ1XG0v5mTZmZv1TluRlbW1tsX79+mpXw8ysrkjaEBFtgx1XK3M2ZmY2jjnYmJlZ7hxszMwsdw42ZmaWu1pJfR43CoVg94Fup0CbmZVwsBlFxQU5i+ukFW/uPHXONAccM5vQPIw2irwgp5lZZQ42o8gLcpqZVeZgM4qKC3KW8oKcZmYONqPKC3KamVXmBIFR5AU5zcwqc7AZZV6Q08zsSB5GMzOz3DnYmJlZ7hxszMwsdw42ZmaWOwcbMzPLXW7BRtJtknZK2lxSdoWkJyQVJLWVlC+U1CVpY3rdWrLvjyU9LmmTpH+VNDuVz5R0v6Sn0/uMVC5JN0rams45J6829qdQCHbtP8gLezrZtf8ghYKfhmpmE1uePZvbgQvKyjYDlwLrKhz/TESclV7XAkhqBL4C/F5EnAlsAj6Ujr8OWBsRi4G16TPAhcDi9FoO3DJqLRqC4mKcl9z8IEtveIBLbn6QLTv2O+CY2YSWW7CJiHVAR1nZUxGxZRiXUXq1SBJwHPBi2ncxsDJtrwTeU1K+KjIPA9MlnXiUzRg2L8ZpZnakWpqzWSTpZ5J+Iul3ASLiEPB+4HGyIHM68I10/JyI2J6O2w6ckMrnAttKrtueysaEF+M0MztSrQSb7cCCiDgb+AjwHUnHSZpMFmzOBk4iG0b7+CDXqrQ2TMUxLEnLJa2XtH7Xrl1HX/sSXozTzOxINRFsIuJgROxO2xuAZ4BTgLNS2TMREcAdwJvTaTuKw2PpfWcqbwfml1x+Hq8NvZX/3BUR0RYRba2traPSFi/GaWZ2pJpYG01SK9AREb2STiab3H8WmAqcLqk1InYB7wCeSqetAZYB16f3u0vKPyTpu8BvA3uLw21jwYtxmpkdKbdgI2k1cB4wW1I78GmyhIGbgFbgHkkbI+J84K3AZyX1AL3AtRHRka7zGWCdpEPA88B/TT/ieuAOSdcAvwauSOX3AhcBW4FO4Oq82tgfL8ZpZnY4ZaNT1tbWFuvXr692NczM6oqkDRHRNthxNTFnY2Zm41tNzNmMR4VCsPtAt+dtzMxwsMlFcRWB4s2dxYy0U+dMc8AxswnJw2g58CoCZmaHc7DJgVcRMDM73IDBRtKflWwvLdv3oSPPMPAqAmZm5Qbr2XykZPumsn1/Mcp1GTe8ioCZ2eEGSxBQP9uVPlviVQTMzA43WLCJfrYrfbYSXkXAzOw1gwWbN0jaRNaL+Y20Tfp8cq41MzOzcWOwYHPamNTCzMzGtQGDTUQ8X/pZ0iyyRTN/nR4FYAPwKgJmZpkBg42kHwDXRcTm9MyYx4D1ZENqKyLiy2NRyXrkVQTMzF4zWOrzoojYnLavBu6PiHeTPSfGqc8D8CoCZmavGSzYHCrZfjvZs2KIiP1AIa9KjQdeRcDM7DWDJQhsk/TfyR61fA7wrwCSmoHJOdetrhVXESgNOF5FwMwmqsF6NtcAZ5A9HfOPIuLlVH4u8M0c61X3vIqAmdlrcntSp6TbgHcBOyPiN1PZFcDfkaVUvyki1qfyhcBTwJZ0+sMRcW3a1wR8lewR0wXgExFxl6QpwCpgCbCbLBj+Kp3zcbJA2Qt8OCLuG6y+eTyp09loZjbeDfVJnYNlo60ZaH9E/JcBdt9OFiRWlZRtBi4F/rHC8c9ExFkVyj9BFrBOkdQAzEzl1wB7IuL1kq4EbgD+SNLpwJVkPbKTgP8n6ZSIGPPJEq8iYGaWGWzO5neAbcBq4BGGsR5aRKxLPZbSsqcApGH9dv8XwBvS+QXgpVR+MVkvCeBO4KvKLnwx8N2IOAg8J2kr8CbgoeH8UDOz8apQCF7u6uZQT4FDhaAQwdTJk5jdMiW30ZfBgs3rgHcAfwz8CXAPsDoinsihLosk/QzYB/xtRPybpOlp3+cknQc8A3woInYAc8kCIRHRI2kvMCuVP1xy3fZUVhUeSjOzaisNLg0NYm/XIQ4c7KGzu5eP3rlpTO4FHDBBICJ6I+JfI2IZWVLAVuDHKUNtNG0HFkTE2WSPNfiOpOPIguE84MGIOIesd/IP6ZxKfxoxQPkRJC2XtF7S+l27do20DUco3th5yc0PsvSGB7jk5gfZsmM/hYLXMDWz/BQKQceBg+zY28Wu/a/y7EuvsK2jk//Y9yr7X+1hW0cXHQcO9QUayP9ewMF6NqSJ+D8k690sBG4E/nk0K5GGvA6m7Q2SngFOATYAncD306H/RDZXA1mPZT7QLqkROB7oKCkvmge82M/PXQGsgCxBYBSbBPR/Y+f3P7DUczlmNiqKoyeFQgFJNAheTj0XgGlTJ7Ot47VbMA71Bsc0ZbdgjOW9gIMlCKwEfhP4IfCZktUERpWkVqAjInolnQwsBp6NiJD0f8gy0X5EdmPpk+m0NcAyst7O5cCP0vFryHpGXyRLEFgMPJpHvQfjGzvNbLSVD4nt3HeQr6z9JR/4vdcjBg4ukP0S3DQpG9Qay3sBB+vZ/DlwgKyX8eGSiX0BERHH9XeipNVkQWK2pHbg02Q9j5uAVuAeSRsj4nyyxT0/K6mHLF352ojoSJf6G+Bbkr4M7CJbNgfgG6l8a7rulWSVekLSHWRBqQf4YDUy0cA3dprZyPU33wJZYHnftzfwyXedzp4D2YIvAwUXgLs2bOsLTH9/+ZlHzNnkdS9gbvfZ1Ju87rPxYpxmNhyDBZfnd3f2HTttaiOX3/oQ31t+bl/Zy12HDgsuqx99vi+4AHR29/LNB5/j6qWLmD+zGdCIstFG5T4bGxk/HtrMhqIYYLL3oc23QNZrmTej+bAAU9pzAbh66SJufmBrX3CZ09DAp959BoUImhrzTXcu5Z5NkkfPxsyskv56L5V6Lr0l2auVei3L3ryIlf/+XF+AKe+5NDY00JPjvTTu2dQQ32tjZkPpvQxnvgWyXss3H3yOy5bMZ/KkBma1NCGoSs9lMO7ZJHn1bDxvYzYxHU3vZbjzLXn3WoZiqD0bB5skr2Cza/9BLrn5wSMy0nyvjdn4M1DvpRhgBhoaq4fgUs7DaDXC99qYjV8D9V76m9gfaGisVibz8+BgkzPfa2M2vox07mWwbLHxElzKeRgt8ZyNmfWnUoA52rmXehgaGw7P2QxTnqnPzkYzqy9DmdwfydzLSG+krCUONsOU9302Djhmtat0McvGSWJP5+CT+xOp9zIQJwjUEA+lmdWe0qGxHSWLWR43xMn9iTr3crTcs0ny7Nk4/dmsNvQ393LVbY/yyXedTtOkhiEPj02k3stA3LOpIU5/Nque/gJMaeZY+54upjdPBoaXmuzey9A52IwBpz+bja2hBJjSobHSxSw9PJYPD6MleWejec7GLF+DpSf3N7lfvphll4fHhsXZaMM0FtloL3d109XdS6//0pqNipEEmPK5l8uWzGfejGZmtTTRWwh6A6ZObvC/00F4zqYG7dh30L0bsxEa7hBZpeyx/obG/EtgfhoGP+ToSLpN0k5Jm0vKrpD0hKSCpLaS8oWSuiRtTK9bK1xvTdm1Zkq6X9LT6X1GKpekGyVtlbRJ0jl5tXE4dh/o7gs0kCUIvHfVenYf6K5yzcxqX6EQdBw4yEv7X+XZl15hW0cn+17tYVtHFx0HDtFxoJtjmiZxTNMkdh/oprO7l87uXu7asI0ZLZOZ2TK5L8B0dvf2BZi5ae70pOObWTCzhROmTXWgyUluwQa4HbigrGwzcCmwrsLxz0TEWel1bekOSZcCr5Qdfx2wNiIWA2vTZ4ALgcXptRy4ZSSNGC3OSDMbvkIheLnzoAPMOJBbsImIdUBHWdlTEbFlONeRdCzwEeDzZbsuBlam7ZXAe0rKV0XmYWC6pBOHW//RVsxIK+WMNLMjlfdiOg4ccoAZB/Ls2QzXIkk/k/QTSb9bUv454AtAZ9nxcyJiO0B6PyGVzwW2lRzXnsqqalZLE1+7qq0v4BTnbGa1NFW5ZmbVN/AwmQPMeFArCQLbgQURsVvSEuBfJJ0BnAy8PiL+StLCIV6r0t+miil3kpaTDbWxYMGCYVd6OBoaxKlzprHmQ0sPy0gzm6iGOtE/3El+3/9Sm2oi2ETEQeBg2t4g6RngFOCNwBJJvyKr6wmSfhwR5wE7JJ0YEdvTMNnOdLl2YH7J5ecBL/bzc1cAKyBLfR71hlXgjDSb6AqFYN+r3bz0SveQMskcYMaHmhhGk9QqaVLaPplscv/ZiLglIk6KiIXAW4BfpkADsAZYlraXAXeXlF+VstLOBfYWh9uqzRlpNlGNZB7GQ2TjQ249G0mrgfOA2ZLagU+TJQzcBLQC90jaGBHnA28FPiupB+gFro2IjspX7nM9cIeka4BfA1ek8nuBi4CtZPM8V49qw0bAGWk2kQw0TDZtauOw7oX5+EWn0SDRNGkSs491D6YeeQWBJO8VBMCrP9vEUGmYrPyO/qGupDyeHjI2Xnm5mmEai2BTXCPtS/dv4bIl85nV0sQJ06Zw0vHNNDbWxIim2bCVLsU0uVHs6+qhQRpwyRgHmPHDwWaYxiLYAPT0FNiycz/v+9YGJwlYXSvtwezaf5BvPvgc1114Gs/v7hx0TbLyxS4dYOrXUIONf50eY3u6DvUFGnCSgNWX/ib6P3rnJi5bMr9vsn+w+2EAPn7RaZw0vZmmSZ7onwgcbMaYkwSsHvW/bEwWXIoPHysGmaHccNkypZH5M47hhOMcYCYCB5sx5mVrrF4MJV25GFyKDx9zurL1x3M2yVjN2ThJwGpdf9lkA030F+dgig8hu3rpIl53/FQaJJonO115PHOCwDCNVbABJwlY7Sm/J6ZSNtlgE/2vO34qUxobABHprv5ZLU3+Oz3OOUGghjlJwGrFYHMxpUNlg030tzQ18rrjmjlpejNzZxxD6zT3Zuw1DjZV4CQBq7bSIDPQXMxQlo3xRL8NhYNNFThJwKqh/wn/4fdinK5sw+U5m2Qs52ycJGBjabAJf990aSPhBIFhGstgA04SsPyVBpmBJvz7WzrG2WQ2FE4QqHFOErA8HM1QmedibCw42FSJkwRsNA2WVTbQhD94LsbyVxNP6pyIikkCrcdO4drzfoPpzZPp7O6luclJAjZ05UNl/T1SebDnxHiozPLmOZtkrOdsCoXgV7sPsGPfq3z0zk2et7FhqTQfM5Q7/D3hb6PNczY1rqFBHDu1sS/QgOdtbGCDzcd4qMxqmYfRquhQT4H2PV2cPX9631Day12HKBQK1a6a1ZBKqculj1b2UJnVg9x6NpJuk7RT0uaSsiskPSGpIKmtpHyhpC5JG9Pr1lR+jKR7JP0inXd9yTlTJH1P0lZJj0haWLLv46l8i6Tz82rjSDU1TuKdp5/AX59/Kp/7wZP80YqH+dwPnuSlA9kaVTaxDfUuf2eVWT3IcxjtduCCsrLNwKXAugrHPxMRZ6XXtSXl/xARbwDOBpZKujCVXwPsiYjXA18CbgCQdDpwJXBG+vk3S6rJWfdZLU387R+ezt/cdfhQ2vu+tcFDaRNY5SAzeOoyeKjMalduwSYi1gEdZWVPRcSWYVyjMyIeSNvdwGPAvLT7YmBl2r4TeLskpfLvRsTBiHgO2Aq8aUSNyUlDg5jUIKdAG4VCsGv/yOdjWprci7HaVEtzNosk/QzYB/xtRPxb6U5J04F3A19JRXOBbQAR0SNpLzArlT9ccmp7KjuCpOXAcoAFCxaMXkuGwSnQE1txPuaFPa/ylbW/5LoLT/N8jI1LtRJstgMLImK3pCXAv0g6IyL2AUhqBFYDN0bEs+mcSv+qYoDyIwsjVgArIEt9HmEbjsqsliZW/cWbKqZAT2/2s0DGo0rPjnnftzfwyXedflhPpjTIlAaY4nyMU5etntRE6nMa8tqdtjcAzwCnlByyAng6Ir5cUtYOzIe+YHQ82bBdX3kyD3gxv9qPjFOgJ46B7vJv39PF9ObJno+xcasmejaSWoGOiOiVdDKwGHg27fs8WSD5b2WnrQGWAQ8BlwM/ioiQtAb4jqQvAielaz06Ni05Ok6BHt+Gcpf/vBnNvNx1qGJPxkNlNh7kFmwkrQbOA2ZLagc+TdbzuAloBe6RtDEizgfeCnxWUg/QC1wbER2S5gGfAH4BPJbN//PViPg68A3gW5K2puteCRART0i6A3gS6AE+GBE1PdteTIFe9uZFfZlp82Y0849/voRW/+ZatyoFmeJcDHDYUNkNl53Jyn9/jmVvdpCx8cnL1SRjvVxNqUIhaN/TyZ98/ZHDMtPmzWjm+x9YSuu0KVWplx2dgZaSGejZMZctmc+8Gc0c3zwZkf0SMqvF83ZW24a6XE1NDKNNdKUp0B5Kq18D9WScVWYTnYNNjfBQWv0aTpBxVplNVB5GS6o5jAYeSqtHQxkuq7Tqsp+AaeOJh9HqjIfS6sfR9mQ8VGYTWU3cZ2MZL8xZ2wZas2wo98d4KRmbyDyMllR7GA0OH0orX77mt+Yfz8wWD6VVg4fLzPrnYbQ6VBxKaz12Cn99/qmHJwr82RIvXzOGKi0p4+Eys6Pnnk1SCz0bgF37D7L5hb188u7N7t1UQaUHlbknY9Y/92zq1KyWJhbNbnHvZowNZUkZ92TMjp4TBGpMQ4M4ZsokPvz2xUc+VO3bfqjaaBuNB5V54t9scO7Z1KDZLVNYNLvFadA5Gkr6MuCejNkocbCpQcXejVcUGH1eUsasOpwgkNRKgkCR06BH19GmL8+f2QzIS8qY9WOoCQKes6lR5WnQd23Yxstdh5g2tZH9r/bQ0+PhtKEY6Y2YflCZ2ehwzyaptZ4NvJYGvfrR5ysOp532uuP8n18/fCOm2dgYas/GwSapxWBTKAS/7uhky479fO4HT3o4bQgcZMzGlofRxoFiosCsliYPpw2gUAg6Dhxkx94ur1tmVqNyCzaSbpO0U9LmkrIrJD0hqSCpraR8oaQuSRvT69aSfUskPS5pq6QblZ4NLWmmpPslPZ3eZ6RypeO2Stok6Zy82jgWZrdM4YRpU/jw2xf3PTb4cz94kstvfYg//fojbNm5f0Iu0lkMMC/tf5VnX3qFLf+xnye373eQMatRefZsbgcuKCvbDFwKrKtw/DMRcVZ6XVtSfguwHFicXsVrXgesjYjFwNr0GeDCkmOXp/PrVkODOOn4ZhbOPobLlsznb+7aROuxU/jHP1/CF674LXbuO8jLXRPnRs/SCf9tHZ3se7WHbR1dfPTOTb4R06yG5RZsImId0FFW9lREbBnqNSSdCBwXEQ9FNrm0CnhP2n0xsDJtrywrXxWZh4Hp6Tp1q7GxgZYpjRN6OK1yVtmhvl5M+54uXu465CBjVqNqac5mkaSfSfqJpN9NZXOB9pJj2lMZwJyI2A6Q3k8oOWdbP+ccRtJySeslrd+1a9dotSMXlYbT7tqwjd0Hutm1/yAv7O0alwFnqMvJzJvRzK0/fsZBxqxG1Uqw2Q4siIizgY8A35F0HFDpf4XBJiiGfE5ErIiItohoa21tHVaFx1r5cNp4Djjl8zH9TfiX9mL+/vIz2fXKQT6z5kkaGxr41LvOYO6MZlqmOMiY1YKaWK4mIg4CB9P2BknPAKeQ9UrmlRw6D3gxbe+QdGJEbE/DZDtTeTswv59z6lrpcFppwFn5789x2ZLXmjz3+GYaG2vl94ihq7S8/3CWk/nWNW9y+rJZjaqJ/5EktUqalLZPJpvcfzYNj+2XdG7KQrsKuDudtgZYlraXlZVflbLSzgX2FofbxoPicFp5wKnXHk7/vZhDg2aVdXb3Mn9mM596t3sxZrUuz9Tn1cBDwKmS2iVdI+kSSe3A7wD3SLovHf5WYJOknwN3AtdGRDG54P3A14GtwDPAD1P59cA7JD0NvCN9BrgXeDYd/zXgA3m48im1AAAHTklEQVS1sRqKw2mtdR5w+ssqG+7y/l5Oxqw+eAWBpBZXEBhIT0+BF/Z2sWv/QXYf6OauDdsOG1Kb1dJE67QpNTWkVulRy8/v7gQ44i5/wHf6m9UBryAwzjU2NjB3gB7Ood4ChQhe2NvFzn2vVu3Gz/JhsqH2YmY6q8xsXHHPJqm3nk1RpR5OtXsD5T2Y4mT/cHsxXt7frPZ5Ic5hqtdgA0cGnLFebLIYXA71FGhoEHsHCDDAEQtiCuj0UJlZXXKwGaZ6DjZweMAZygPBGhsa6CnEsHsNpYElgEklwQUYNMCAezFm44mDzTDVe7CB1wLOs7sO9LusfvG/785+gs+kBpikBiLiiDthywMLHB5css8DBxj3YszGFwebYRoPwQaygPOrjgP9PsulqL/gU1pWrjywZGWvBRcYWoBxL8Zs/HCwGabxEmxg4AeIFZUHn0/evZlPvuv0w8rKlV+j0nUcYMwmlqEGm5pYrsZGV0ODmH7MFI6b2sS+V7spRPNh97QAzGppOixwtO/pYnrz5AGvW0w+KFW6dAwcvnxM8e7+QgRNjQ4wZhOZg804VinoVJrIB5g3o/mIXkq58sACRwaXOQ0NDjBmdgQPoyXjaRitP/2lKMPQ5mwqJRUAR5XVZmbjg+dshmkiBJty5WnMg2WjgQOLmR3OczY2qIYGMbNlSrWrYWYTgNdGMzOz3DnYmJlZ7hxszMwsdw42ZmaWOwcbMzPLnVOfE0m7gOeP4tTZwEujXJ1qGU9tgfHVHrelNrkt8J8ionWwgxxsRkjS+qHkmNeD8dQWGF/tcVtqk9sydB5GMzOz3DnYmJlZ7hxsRm5FtSswisZTW2B8tcdtqU1uyxB5zsbMzHLnno2ZmeXOwWYEJF0gaYukrZKuq3Z9hkvSryQ9LmmjpPWpbKak+yU9nd5nVLuelUi6TdJOSZtLyirWXZkb0/e0SdI51av5kfppy99JeiF9NxslXVSy7+OpLVsknV+dWlcmab6kByQ9JekJSf8jldfddzNAW+ruu5E0VdKjkn6e2vKZVL5I0iPpe/mepKZUPiV93pr2LxxxJSLCr6N4AZOAZ4CTgSbg58Dp1a7XMNvwK2B2Wdn/BK5L29cBN1S7nv3U/a3AOcDmweoOXAT8EBBwLvBItes/hLb8HfDXFY49Pf1dmwIsSn8HJ1W7DSX1OxE4J21PA36Z6lx3380Abam77yb9+R6bticDj6Q/7zuAK1P5rcD70/YHgFvT9pXA90ZaB/dsjt6bgK0R8WxEdAPfBS6ucp1Gw8XAyrS9EnhPFevSr4hYB3SUFfdX94uBVZF5GJgu6cSxqeng+mlLfy4GvhsRByPiOWAr2d/FmhAR2yPisbS9H3gKmEsdfjcDtKU/NfvdpD/fV9LHyekVwO8Dd6by8u+l+H3dCbxd0ogeXuVgc/TmAttKPrcz8F/EWhTA/5W0QdLyVDYnIrZD9o8NOKFqtRu+/uper9/Vh9LQ0m0lw5l105Y09HI22W/Rdf3dlLUF6vC7kTRJ0kZgJ3A/Wc/r5YjoSYeU1revLWn/XmDWSH6+g83RqxTl6y21b2lEnANcCHxQ0lurXaGc1ON3dQvwG8BZwHbgC6m8Ltoi6VjgLuAvI2LfQIdWKKup9lRoS11+NxHRGxFnAfPIelynVTosvY96Wxxsjl47ML/k8zzgxSrV5ahExIvpfSfwfbK/gDuKwxjpfWf1ajhs/dW97r6riNiR/nMoAF/jteGYmm+LpMlk/zn/74j451Rcl99NpbbU83cDEBEvAz8mm7OZLqn4xObS+va1Je0/nqEP9VbkYHP0fgosTtkcTWSTaGuqXKchk9QiaVpxG3gnsJmsDcvSYcuAu6tTw6PSX93XAFelzKdzgb3FIZ1aVTZvcQnZdwNZW65M2UKLgMXAo2Ndv/6kcf1vAE9FxBdLdtXdd9NfW+rxu5HUKml62m4G/oBsDuoB4PJ0WPn3Uvy+Lgd+FClb4KhVO0uinl9kmTS/JBv7/ES16zPMup9Mljnzc+CJYv3JxmXXAk+n95nVrms/9V9NNoRxiOy3sGv6qzvZkMD/St/T40Bbtes/hLZ8K9V1U/qHf2LJ8Z9IbdkCXFjt+pe15S1kwy2bgI3pdVE9fjcDtKXuvhvgTOBnqc6bgU+l8pPJAuJW4J+AKal8avq8Ne0/eaR18AoCZmaWOw+jmZlZ7hxszMwsdw42ZmaWOwcbMzPLnYONmZnlzsHGrIZJ+pGke9PNhWZ1y8HGrIZFxO8DB4E/rHZdzEbCwcas9v0Q+NNqV8JsJHxTp1mNk/QjYAkwPwZe1NKsZrlnY1bDJP1nskUQvwNcVuXqmB0192zMapikb5Atlvgc8JmI+IMqV8nsqDjYmNUoSa3AQ8BpEXFI0tPA2yI9GsKsnngYzax2vQ/4ekQcSp9Xkz3KwqzuuGdjZma5c8/GzMxy52BjZma5c7AxM7PcOdiYmVnuHGzMzCx3DjZmZpY7BxszM8udg42ZmeXu/wMgR9i7VGuMwwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = hitters_pd.drop('Salary', axis=1)\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "#lambdas = np.arange(0, 0.08, 0.0005)\n",
    "lambdas = np.arange(1, 300, 1)\n",
    "MSEs    = [] \n",
    "for l in lambdas:\n",
    "    MSEs += [np.mean(ridge_cv(X, y, λ=l, k=10))]\n",
    "\n",
    "sns.scatterplot(x='λ', y='MSE', data=pd.DataFrame({'λ': lambdas, 'MSE': MSEs}));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(114903.19824046272, 53)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(zip(MSEs, lambdas))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we use 10-fold cross-validation to find the optimal value for lambda. The lowest MSE score achieved is 114,903 when λ=53.\n",
    "\n",
    "**Revision note:** This result is worst than our results for backward stepwise selection (106,618) and best subset (109,522), and not as good as that reported by ISL authors (96,016) on p.254.\n",
    "\n",
    "A very small improvement was observed by setting fit_intercept and normalize to True - in which case the optimal values of lambda became significantly smaller."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([-40., -30., -20., -10.,   0.,  10.,  20.,  30.,  40.,  50.,  60.]),\n",
       " <a list of 11 Text xticklabel objects>)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEVCAYAAACbo1D5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu85WP5//HXe8Y5wxRDRWOYlFPamGSIKJVKMghDaTp8J0VK36l8O0hKITlEJT8xSI6jHFKkRhiHDOaEVFODymHkOJow4/r9cd9r9mfvWWvvtWbvvT5r7/V+Ph77sdf6HK81w7rmvj/3fd2KCMzMzIa6YWUHYGZm1gxOeGZm1hac8MzMrC044ZmZWVtwwjMzs7bghGdmZm3BCc/MzNqCE56ZmbUFJzwzM2sLK5UdgHVad911Y8yYMWWHYWY2qNx1111PRMSo3o5zwmshY8aMYebMmWWHYX208Mc/q7lv1Kc/3MRIzNqDpAfrOc5dmgWSJkgKSZvl92MkHVTYv6ukZyTNkjRH0g2S1uvlmh2S3jfQsZuZWc+c8LqaCNwCHJjfjwEO6nbMzRHRERFbA3cCh/VyzQ7ACc/MrGROeJmkNYGdgE/QmfCOB3bOLbojux0vYATwVH6/vaRbJd2Tf79R0irAscAB+RoHNO0DmZlZF36G12lv4DcR8WdJT0raFjgKmBIRe0Lq0iQnQGAd4HngK/n8PwG7RMQSSbsD34mIfSUdDYyLiMOr3VTSZGAywOjRowfw45mZtTe38DpNBC7Ory/O76updGm+DjgXODFvXxu4TNI84BRgy3puGhFnRcS4iBg3alSvg4zMzGwFuYUHSFoHeAewlaQAhgMBXNvLqVcB0/LrbwHTI2KCpDHAjQMSrJmZrRC38JL9gPMjYqOIGJNbb38HXiY9p6vlbcD8/Hpt4J/59aTCMc/1cg0zM2sCt/CSiaQBKkXTSINXlkiaDUwF7qHzGZ6AZ4BP5uNPBM6T9AXg94XrTAeOyud8NyIuGbBPYS3Bc+3MWpMiouwYLBs3blx44rmZWWMk3RUR43o7zi08syZ67MfdOxIG1vqfPqqp9zNrZX6GVydJi7q9nyTpjPz6UEmHFLa/towYzcysNrfw+kFEnFl4OwmYB/yrnGjMzKwaJ7x+IOkYYBGwABgHXChpMTAe+AawF7AEuD4ippQUpplZW3PCq9/qeaRlxatI8/CWiYjLJR1Oqs4yU9KrgAnAZhERkkZ2v6grrZiZNYef4dVvca6w0hERHcDRdZzzLPBf4GxJ+wD/6X6AK62YmTWHE94AioglwPakOX17A78pNyIzs/blLs3+t6yySl6BYY2IuFbS7cBfS43MzKyNOeH1v6nAmXnQynuBKyWtRqrMcmRPJ9rQ53lxZuVxpZUW4korZmaNc6WVTNKiiFiz8H4SPaxPV+MaC0hdlUFa8PWQiHiwn0O1NjD/9A+u0HljP3tlP0di1n48aKV+u0XE1qRlf75WcixmZtagtk54kqZK+rGk6ZL+Juntks6RdL+kqTVOuw3YIJ8/Ji/4WrnelDwJHUk3SjpB0h8l/VnSzgP9eczMrLYh36VJ7xPGX0la/HUv4GpgJ9KSP3dK6oiI4rkAewC/rPPeK0XE9pLeR6q4svuKfAAzM+u7dkh4i/NEcaDzGV5h/9W5Cspc4LGImJuPuxcYA1QS3nRJ6wOPU3+X5hX59135WstxpRUzs+Zo6y7N7IX8++XC68r74j8IdgM2Au4Fjs3bltD1z3C1GtdeSo1/XLjSiplZczjhNSAiFgOfBw7JdTIfA9aTtI6kVYE9Sw3QzMxqcsJrUEQ8AlwEHBYRL5Fae3cA1wB/KjM2MzOrzRPPW4gnnpuZNa7eiedu4ZmZWVtoh1GaDZG0FJhL+rP5O/CRiHi63KhsMLj1rN4f4e44+ZomRGJm1biFt7zKundbAU8Ch5UdkJmZ9Z0TXs+KVVV2lbTsn+eSzshz+pC0QNI3Jd0taa6kzfL2t0ualX/ukTSijA9hZmZOeDVJGg68k65VWXryRERsC/wYmJK3TSGN5uwAdgYW93ugZmZWFye85VVKkf2bVIbst3WeV62qygzgZElHACPzCuhdSJosaaakmQsXLuxb5GZmVpMT3vIqpcg2Alah8xlew1VVIuJ4Ul3O1YHbK12dRa60YmbWHE54NUTEM8ARwBRJKwMPAltIWlXS2qTuzh5JGhsRcyPiBGAmsFzCMzOz5nDC60FE3APMBg6MiIeBS4E5wIXAPXVc4vOS5kmaTXp+9+sBC9bMzHrkSistxJVWzMwa50orZmZmBa600qBalVgkjQHuBx4ABDwPfCwiHpC0KzAlIryaQhu65pz3Lnu958fdq21WFrfwGtdTJZb5ed+bgfOAr5QSoZmZLccJr2+WVWKpYi3gqSbGYmZmPXCX5goqVGL5aWHz2DxpfQSwBvDWMmIzM7PluYXXuJ4qsVS6NMeSVkY/q7eLudKKmVlzOOE1rlYllu6uAnbp7WKutGJm1hxOeCuoSiWW7t4GzG9uVGZmVouf4fVBRNyTq6gcCNxM5zM8AS+S6miamVkLcKWVFuJKK2ZmjXOlFTMzs4K27tKUdArwYEScmt9fBzwcEZ/M778P/DMiTq5x/qKIWNOVVKxe501994Bd+6OTrh+wa5sNBe3ewrsV2BFA0jBgXWDLwv4dSYu4mpnZINfuCW8GOeGREt084DlJr5S0KrA5cL+k30m6W9JcSR/s6YKS3iLpHkmbSHq7pFn55x5JIwb245iZWS1t3aUZEf+StETSaFLiq5QKGw88Q1r77j/AhIh4VtK6pJXLr4oqo30k7QicDnwwIh6SdBpwWETMkLQm8N8mfTQzM+um3Vt40NnKqyS82wrvbyVNMfiOpDnADaSEuH6V62xOqqzygYh4qHDtkyUdAYyMiCXdT3KlFTOz5nDC63yO9yZSl+btpBZe5fndwcAoYLtcYeUxYLUq13mE1ILbprIhIo4nzcVbndQy3Kz7Sa60YmbWHE54KantCTwZEUsj4klgJCnp3QasDTweES9J2o1UUqyap4H3k1qDuwJIGhsRcyPiBGAmsFzCMzOz5nDCS4u5rktq2RW3PRMRTwAXAuMkzSS19v5U60IR8RjwAeCHkt4KfF7SvFyNZTHg1T/NzEriSistxJVWzMwa50orZmZmBW09LQFAkkiFn4+LiF/nbfsDH4+IPUoNzoacU37+nj5f48iDruuHSMzaT9snvIgISYcCl0maDgwHjgP6lOwkrVRtGoKZmZWj7RMeQETMk3Q18GXgFcD5ETFf0kdJC7yuQpq+cHhEvCzpLGBb0nSDSyLiWABJ/wB+QkqWp0raEPgf4CVgbkR8uNmfzczMEie8Tt8E7iatYzdO0lbABGDHiFiSk9yBwM+BoyLiSUkrAdMlXR4R9+XrPB8ROwFIegTYKCJelDSy6Z/IzMyWccLLIuJ5SZcAiyLiBUm7A28BZqbHfKwOPJwPnyjpE6Q/v9cCWwCVhHdJ4bL3Aj+TdCXwy2r3lTQZmAwwevTo/v1QZma2jEdpdvVy/oFUUuyciOjIP2+MiG9J2hT4HPCOiNga+A1dK688X3j9HuBMYHtS4hze/YautGJm1hxOeLXdAOyfC0YjaZ1cZHot4DngWUmvISW15eTktmFE/B74Iqk82RpNidzMzJbjLs0aImKupG8CN+S18l4CDiWVCLuPVHfzb9ReL28l4Od5SaBhwAkR8dzAR25mZtW40koLcaUVM7PGudKKmZlZQVslPEmvlnSxpPmS7pN0raQ35H1HSvqvpLULx3dIel/h/SRJC/MK5vdKulxSj8/lJO2aF4a1NveFaS7cY1amtkl4uYTYL4AbI2JsRGwBfIXOxVwnAneS5t5VdADvo6tL8qjNLUlz9g7o5da7ktbWMzOzErVNwgN2A16KiDMrGyJiVkTcLGkssCbwNVLiQ9IqwLHAAblF1yWx5UnnrwCeyu8/IOkOSfdIukHS+pLGkAa6HJmvsfPAf0wzM6umnRLeVsBdNfZNBC4iFZF+o6T1IuJF4Gg6W3SVCeUHSJoF/BN4FXB13n4LsENEbANcDHwpIhaQ5uGdkq9xc/cbS5osaaakmQsXLuyfT2pmZstpp4TXkwOBiyPiZeAK4EM9HHtJRHQAryYtFPvFvH1D4DpJlW1b1nNjTzw3M2uOdkp49wLbdd8oaWtgU+C3khaQkt/E3i4WaT7H1cAuedPpwBkR8SbgU3StvmJmZiVrp4T3e2BVSf9T2SDpLcBpwDERMSb/vBbYQNJGpIoqI3q45tuA+fn12qRuToCPFo7p7RpmZtYEbZPwcotsAvCuPC3hXuAY0ijKX3Q7/Beklt50YItug1Yqg1jmANsA38rbjyGtqXcz8EThWlcDEzxoxU7e9zdlh2DW1lxppYW40oqZWeNcacXMzKzACa+gViUWSfO6HXeMpCn59bF57Twkfb63yitmZlYOr5aQFSqxnBcRB+ZtHXRWYqkqIo4uvP088DPgPwMVp5mZrRgnvE5VK7Hkaik1SZoKXENa+fy1wHRJTwC7Az8FxgFBWkz2lAGJ3MzMeuWE16mnSixjc3WVilcDJxUPiIgfSPoCsFtEPCFpO2CDiNgKQNLIaheWNBmYDDB69Og+fgQzM6vFz/DqMz+XBuvIVVbO7PWMtDjsJpJOl7QH8Gy1g1xpxcysOZzwOlWtxLKiIuIp4M3AjcBhwNn9dW0zM2ucE16nWpVYNmrgGsuqqkhaFxgWEdOArwPb9mOsZmbWICe8rIdKLP9q4DJnAb+WNB3YALgxP/ubCvxf/0ZsZmaNcKWVFuJKK2ZmjXOlFbMW875fHF92CGZtbcgkvBWpktLAtZfm4s/zJF1da4qBmZm1riGR8ApVUm6MiLERsQXwFXqpktKAxXlKwlbAk6RRl2ZmNogMiYRHjSopwMM9nSTpRkmnSLpJ0v2S3iLpCkl/kfTtGqfdRhqQgqRdJV1TuN4Zkibl1wskfVPS3ZLmStqsj5/RzMz6oNeEJ2l9ST+V9Ov8fgtJnxj40BrSa5WUyg9waLf9L0bELqTJ5FeSWm9bAZMkrVM8UNJw4J3AVXXG9UREbAv8GKjajSppsqSZkmYuXLiwzsuamVmj6mnhTQWuI9WJBPgzqUjyYNFblZRK8poL3BsRj0TEC6RKKa/L+1bPyfLfwKuA39Z57yvy77uAMdUOcKUVM7PmqCfhrRsRlwIvA0TEEmDpgEbVuL5USXkh/3658LryvlJrdHFOlhsBq9D5DG8JXf8MV6tx7aW4bqmZWanqSXjP5669AJC0A/DMgEbVuP6oktKriHgGOAKYImll4EFgC0mrSlqb1N1pZmYtqJ6E9wVSt99YSTOA84HPDmhUDeqnKin13useYDZwYEQ8DFwKzAEuBO7p7/vZ0HHthKPKDsGsrfVYaUXSMGAH4I/AGwEBD0TES80Jr7240oqZWePqrbTS43OliHhZ0vcjYjzpOdmQIenVwKnAW0jP2haQBuPMBh4gPaubCXzCCd76y/un/aTq9l/t+6kmR2LWfurp0rxe0r55cveQ0MtE9fl5gMqbgA2B/cuL1MzM+ku9z/AuA16Q9Kyk5yRVXcx0EOl1onpELCV15VYmmU+SdEZlv6RrJO2aXy+SdJyk2ZJul7R+3v6hXI5stqSbmvLJzMysql4TXkSMiIhhEbFKRKyV36/VjOAGUE8T1QGQtBrwVuA3dVzvFcDtEfFm4CagMlr0aOA9efteKx6umZn1Va9zwyTtUm17RAzVFsvYPMl8U+DyiJhTxzkvApUSY3cB78qvZwBTJV1K5yT0LiRNBiYDjB49ui9xm5lZD+qZDP3FwuvVgO1JX+rvGJCImuNeYL8a++ZHRIek15AWcN0rIq6i50nmL0XncNdlk8wj4lBJbwXeD8yS1BER/y7eLCLOIi0cy7hx47w4oZnZAKmnS/MDhZ93kboDHxv40AZUrxPVI+IR4Cg6VypfAHRIGibpdaTE3yNJYyPijog4GniCzlJlZmbWZCuyWsI/SElv0GpgovovgTUk7Uzqnvw7qebmScDdddzqe3mlhHmkZ3uz++kjmJlZg3qceA4g6XRyWTFSguwAFkTEhwc4trbjiedmZo3rl4nnWfEbeAlwUUTMWOHIzMzMSlBPwhsZEacVN0j6XPdtQ02jlVjynLwrSd2ew4DHgYMi4vG8KOy4iDi8yR/DWsyel1+47PU1+x1cYiRm7aeeZ3gfrbJtUj/H0VL6UInl5rzu3tbAnXQuI2RmZiWr2cKTNBE4CNhYUnGF7xGkhVCHsqqVWCSNKbxfKmlZJZainDBHAH8d+FDNzKwePXVp3go8AqwLfL+w/TnScjhDWSOVWD5X2LxznrS+DvA8qVVoZmYtoGbCi4gHSQucjm9eOINCT5VYbo6IPQEkfRk4ETi0p4u50oqZWXP0+gxP0g6S7swFkl+UtHQIFI/uzb3AdjX2VZ7hvR7YQVKtGplXAVXLshVFxFkRMS4ixo0aNWrFojUzs17VM2jlDGAi8BdgdeCTwOkDGVQLWJFKLN29DZg/kEGamVn96qq0EhF/BYZHxNKIOJc0qGPIWsFKLJCf4UmaDXwE+N9mxWxmZj2rp9LKTcDuwNnAo6SBLJPykjfWj1xpxcyscfVWWqmnhfeRfNzhpJGHrwP27Vt4ZmZmzdVrpZWIeFDS6sBrIuKbTYip39WomvICcF5E/DIf8wBwQUR8O7+fBlwYEbXWsRsDXBMRW7mSivVkr8uvXPb6qv0+WGIkZu2tnlGaHwBmkVf+ltTRbSJ6S+uhasrdwI75mHWARXSdgjGeNBfRzMyGgHq6NI8hrf32NKSKI8CYgQup31WtmgL8jpzw8u9rgFFKNgYWR8SjksZIulnS3flnx+XuUCDp/ZJuk7SupA9Jmidpdn4WamZmJamnePSSiHgmNZQGpVpVU+4CtpK0Cinh/QHYBNgc2Ia0/h2kItDvioj/StoUuAio+nBU0gTgC8D7IuIpSUcD74mIf0oa2Z8fyszMGlNPwpsn6SBgeP7CP4Ih0NUXES/k6QbbAjuQqqJsQkp+29D5GVcGzpDUASwF3lDjkruREuG7I6IyMX8GMFXSpUCtZ4GutGJm1gQ1uzQlXZBfzge2JA3yuAh4lrRMzmDRU9WUW0nVUEZExFPA7aSEtyOdLbwjgceAN5MS2io1rvU3UsHoZQkxIg4FvkYa2TorPyvswpVWzMyao6dneNtJ2gg4gFQ8+j3Au/PrNZoQW3+pWjVF0ttJSe1TpDXuIBXF3gEYTUqUAGsDj0TEy6QpGsNr3OdBYB/gfElb5vuMjYg7IuJo4AlS4jMzsxL01KV5Jmlk5iZ0XfVcQOTtLS8iIj9bO1XSUcB/6VzM9X7S5/huPnaJpMeBh3OCA/gRME3Sh4DppLmIte71gKSDgcvy6Nbv5W5gkQbJzK51rpmZDax6Kq38OCI+3aR42porrZiZNa7fKq042ZmZ2VBQV/HowSgvYzQrz4O7TFKPzx0lTZL02sL7VSSdmotH/0XSlZI2rOO+Xa5jVjRh2o1lh2DWtoZswiNNHO+IiK2AF+llIVZgElBMVN8hj7qMiE1JKyNcod4nJHa/jpmZtYB65uENBTcDWxfrXwJImgKsCcwjTTm4UNJiYCfgY8DGEbEUICLOlfRx4B2S5td5nfHAN4C9gCXA9RExpTkf2czMioZyCw8ASSsB7wXm1jomIi4njUQ9OK9mPhZ4qDCBvGImaU5ivddZnbSu3pYRsTXw7b58FjMzW3FDOeGtLmkWKQE9BPy0gXMrUy/q3V7Ls6RpEGdL2gf4z3IXlCZLmilp5sKFCxu4tJmZNWIoJ7zKM7yOiPhsRLxI6lYsfubVapz7V2AjSSO6bd8WuK/e60TEElLh7WnA3uQVJ7od40orZmZNMJQTXjWPAetJWkfSqsCehX3PkQapEBHPA+cBJ0saDiDpEFKFmd/Xex1JawJrR8S1pInuHQP54czMrLZ2GbQCQES8JOlY4A7g78CfCrunAmcWBpv8H3AS8GdJL+djJ0SaqV/vdd4LXClpNVJ36JED+PHMzKwHvVZaseZxpRUzs8b1W6UVMzOzoaDtEp6kCZJC0mYrcO6xknbvYf/ekrboW4Q2lH1o2ryyQzBrW22X8ICJwC3AgY2eGBFHR8QNPRyyN+CEZ2bWgtoq4eVRkzsBnyAnPEmvkXRToe7mzpKGS5qa38+VdGQ+dqqk/fLr4yXdJ2mOpJMk7UiqqPK9fK2xko4oHHNxSR/bzMxos1Ga5LlwEfFnSU9K2hbYDbguIo7LUxDWIE0f2KBQOmxk8SKSXkWqoLJZXm9vZEQ8LekqUsmxy/NxR5HKk73Q/RpmZtZcbdXCI3VnVlpaF+f3dwIfk3QM8KaIeA74G7CJpNMl7UGqmFLUawWVbA6pruaHSZPVl+NKK2ZmzdE2CU/SOsA7SElqAfBF4ABSYeldgH8CF0g6JCKeAt4M3AgcBpxdvFY9FVSy9wM/BLYD7sp1PbtwpRUzs+Zopy7N/YDzI+JTlQ2S/kBKdjMi4v9JegWwraRrgRcjYlpeGWFq8UL5WeAaEXGtpNtJpciga5WVYcDrImK6pFuAg0grKjw9oJ/SzMyqaqeENxE4vtu2aaRk9rykl4BFwCHABsC5OWlBqrpSNILqFVQuBv6fpCNIg2J+KmntfMwpEeFkZ2ZWEldaaSGutGJm1jhXWjEzMytopy7N5UhaSteFYS+OiO7dnmZ1+9Yv/tXj/q9PeG2TIjGz7to64ZHXzOvpAEnDI2Jp4f1KeZRmj+o9zszMmqPdE15VedrCOcC7gTMkHQrcSqrScpWky/P+UcBC4GMR8ZCkqcCTwDbA3Xki+mn5sgHskuf5mZlZk7V7wltd0qzC++9GxCX59X8j4m0AOeGNjIi35/dXk6Y4nCfp48APSPPxAN4A7B4RS/Nxh0XEjDyV4b/N+FBmZra8dk94PXVpXtLD+/HAPvn1BcCJhX2XFbpAZ5BWTb8QuCIi/tH9JpImA5MBRo8e3WD4ZmZWL4/SrO35Xt4XFed2LDsuD4D5JLA6cHu1JYlcacXMrDmc8FbMrXQuL3Qwabmh5UgaGxFzI+IEYCbQ8Bp8ZmbWP9q9S7P7M7zfRMRRdZx3BHCOpC+SB63UOO7zknYDlgL3Ab/uU7RmZrbCXGmlhbjSiplZ4+qttNLuLTyzfnHptCfqOm7/fdcd4EjMrBY/wzMzs7YwKBOepEVlx1AvSVdKuq3sOMzM2t2gTHiDhaSRwLbASEkblx2PmVk7GzIJT9IoSdMk3Zl/dsrbt5d0q6R78u835u1rSLpU0hxJl0i6Q9K4vG9R4br75ZJhNe/Rg32Bq0nr5B1Y7QBJkyXNlDRz4cKFff5zMDOz6oZMwiPVrDwlIt5CSjRn5+1/ItWw3AY4GvhO3v4Z4KmI2Br4FrBdH+5Ry0TgovwzsdoBnnhuZtYcQ2mU5u7AFpIq79eSNAJYGzhP0qakiigr5/1vIxd2joh5kuas6D2qFYSWtD7weuCWiAhJSyRtFRHzVvDzmZlZHwylhDcMGB8Ri4sbJZ0OTI+ICZLGADdWdvVwreLkxNV6u0cNBwCvBP6eE+RapG7Nr9VxrpmZ9bOhlPCuBw4HvgcgqSMiZpFaeP/Mx0wqHH8LsD8wXdIWwJsK+x6TtDnwADABqLTgat2jmonAHhFxWz52Y+C3OOENSZ5fZ9b6BuszvDUk/aPw8wVSua9xeRDKfcCh+dgTge9KmgEML1zjR8Co3JX5ZWAO8EzedxRwDfB74JHCObXu0UVuSY4Gbq9si4i/A89KemsfPreZma2gti0tJmk4sHJE/FfSWOB3wBsi4sWyYnJpsaHvpgs8ErdV7fIRDxobrFxarHdrkLozVyY9z/t0mcnOzMwG1qBLeJIWRcSafb1OHlnZ678I6ojnY8Dnum3+N7Ab0BERc/Jx84A9I2JBX+9pZmaNG3QJr9VExLnAucVtkiYBmwJfJY3WNDOzkg3WQStdtGiVlWuALSv37CF2V1oxM2uCIZHwaM0qKy+TRoh+paeDXGnFzKw5hkqXZktVWSn4OfBVF442MyvfUEl4rVZlJV0oYomk75Pm+ZmZWYmGSsJrtSorRVOBLwEjGv5UNuR4rpdZeQbjM7yWrrLSXZ7b9wNgvRX5sGZm1j/astJKK1ZZAVdaaQd/+tFjZYdg1pI2+8z6K3xuvZVWBk0LT9JSSbMk3StptqQvSBqW942T9INezj9U0iH57RrALZJmA78APg3sIemofox3jKTFOebKzyG9n2lmZgNhMD3DWxwRHQCS1iONgFwb+EZEzAR6bBpFxJmF17WqrFzVSEA1qqzMiIjD8uv5lZjNzKxcg6aFVxQRjwOTgcOV7CrpGknDJC2QNLJyrKS/Slpf0jGSpuRtR0i6Lz+PuzhvmyTpjPx6I0m/y/t/J2l03j5V0g/yJPa/Ac9FREe3n8OWC9jMzEo3KBMeQET8jRT/eoVtLwNXkkZXkpfiWRAR3R+cHAVskyeeVxt8cgZwft5/IWnQScVrSPP49gSO7yXMsd26NHfufoArrZiZNcegTXhZtfl0l9BZv/LA/L67OcCFkj4MLKmyfzypyxTgAlKCq/hlRLwcEfcBvT1lnd+t9Xdz9wNcacXMrDkGbcKTtAmwFHi8267bgNdLGgXsDVxR5fT3Az8klRS7S1JvzzKLQ1lfKIbRUNBmZlaaQZnwcjI7Ezgjus2ryO9/AZwM3B8R/+527jDgdRExnTQhfCTQfbmhW0mtQ4CDSRPVzcxsEBtMozRXlzSLVA9zCamr8eQax14C3EnX6ioVw4GfSVqb1EI7JSKeLtTIhDTJ/BxJXwQWAh9bwZjH5pgrzomIHqdP2NDWl7lGZtY3bTnxvFV54rmZWePqnXg+mFp41oNHT/pb2SFYHV49ZZOyQzBrWy37DK+fK6tU279Xf1RWkfSmHOfTkv6eX98h6QFJXyscN03SPn29n5mZrZhWbuH1W2WVGvuvosHKKjWuMxfoyM/7RkXElyStQ1pdYXzh0PGAJ6WbmZWkZVt4Ra1SWUXSfj2EOQPYMb/ekbTiwqgc78akBP5of//ZmJlZfQZFwoNBUVnlLmArSauRk+N4AAANLUlEQVSQEt5tpDX1Ns/vZ1Q7yZVWzMyaY9AkvKxlK6tExAvAvcC2wA7AHaSkt2P+ubXGea60YmbWBIMm4Q2Syiq3ArsAIyLiKeB2OhNe1RaemZk1x6BIeIOossoM4FPA7Px+Dqm1N5rU+jMzs5K08ijNwVhZ5VZgE+C7ABGxRNLjwMP5eeOA8fwuM7OeudJKC3GlFTOzxrnSilkLeuzUP5YdwnLW//z2ZYdg1hQD+gxPUkj6fuH9FEnH9PGau0q6ps/Brfj9K5VVij931Dh2JUlPSPpus+M0M7OuBnrQygvAPpLWHeD7NE1EzO22qGtHRLy1xuHvJs3F21/dHhqamVlzDXTCWwKcBRzZfYekUbm+5J35Z6e8fa6kkblCyb8r9TAlXSBp91o3krSdpD9IukvSdZJek7f/T77+7Hy/NfL2sZJuz/uOlbQob+/SgpR0hqRJPd2jBxOB04CHSKM1zcysJM2YlvBD4OA8SrLoNNKIybcA+wJn5+0zgJ2ALYG/ATvn7TuQ5rUtR9LKwOnAfhGxHXAOcFzefUVEvCUi3gzcD3yicP/T8v3/1duH6OUe1Y5fHXgnqcTYRaTkV+04V1oxM2uCAR+0EhHPSjqfNPR/cWHX7sAWhZ6+tSSNAG4mTd5+EPgxMFnSBsCTEbGoRs/gG4GtgN/m/cOBR/K+rSR9m875d9fl7eNJE9UhVVk5qZeP0tM9qtkTmB4R/5E0Dfi6pCMjYmnxoIg4i9QKZty4cR4ya2Y2QJo1SvNU4G7g3MK2YcD4iCgmQSTdRFpVYDTwVVKdzP1IibAWAfdGxPgq+6YCe0fE7Nw1uWsvsS6ha8t3tTruUc1EYCdJC/L7dYDdgBvqPN/MzPpRUyqtRMSTwKV0didCWj7n8MobSR352IeBdYFNc8HoW4Ap9JzwHiCtTDA+X2tlSVvmfSOAR3KX5MGFc24ndaVCZ5UVSC3LLSStmrth31nHPbqQtBapHufoiBgTEWNISbxqt6aZmQ28Zs7D+z6FBEfq4vyhpDk5jpvoXMngDlKXIaRE9126lvt6p6R/FN5/iNQK/EFOUiuRWpX3Al/P13sQmEtKgACfJ1Vg+V/gV8AzkBKupEtJZcH+AtyTt7+Ylweqdo/u9gF+nwtKV1wJnChp1W7brY14zptZedq20koerbk4IkLSgcDEiPhgmTG50oqZWeNcaaV32wFn5PlxTwMfLzkeawOPn/GbskMYUOsdvkfZIZjVNGAJT1IAJ0fE/+b3U4A1I+KYPlxzV2BKROzZ1/gi4mbgzX29jqQfkqZRFC0CNgY2iYgX8sT7mflZnpmZlWAgB60MuSor1UTEYd0rrwB/Ja3d51ajmVmLGMiE1+5VVk4FjlTvi82amVkTDPS0hLasspI9RBpZ+pFeru1KK2ZmTTCgrY82rrJS8R3gKtK0h6pcacXMrDma0d3WjlVWAIiIvyqt2r5/I+eZmVn/G/BKK+1WZaWK4/JnMDOzEjVrQEU7VVnpIiLulXQ3sG1vx9rQ53lqZuVpy0orrVhlJce1kJSca1kXeKJJ4TSiFeNqxZjAcTWiFWMCx9WoZsS1UUSM6u2gdh0y35JVVnr7C5M0s57yOc3WinG1YkzguBrRijGB42pUK8XVlglvgKusnBYR51Y73szMytOWCa+/RMRhZcdgZmb1acp6eNZvzio7gBpaMa5WjAkcVyNaMSZwXI1qmbjactCKmZm1H7fwzMysLTjhmZlZW/CgFbMBJGkz4IPABkCQipVfFRH3lxqYWRtyC89sgEj6MnAxqRbrH4E78+uLJB1VZmxm7ciDVlqcpPUptA4i4rGSQyJP2N+erq2WP0aJ/zG1aEx/BraMiJe6bV+FVIx803Iiaz25ZN//kVYxqRRgeBy4Ejg+Ip4uKa6VSHWAJwCvpfO/rSuBn3b/u23XmHJcLfl3WOQuzRaVC2qfCawN/DNv3lDS08BnIuLukuJ6N/AjUq3RZXEBr5f0mYi43jEt8zLpC6l7ubjX5H2laNEvpkuB3wO7RsSjOc5XAx8FLgPeVUJMABeQqjEdA1Rq+G5IiutnwAGOaZlW/Ttcxi28FpWXFfpURNzRbfsOwE/yorZlxHU/8N6IWNBt+8bAtRGxuWNadv89gDNIifjhvHk08Hrg8Ij4TUlxXUf6YjqvyhfT7hHR9C8mSQ9ExBsb3TfQeonrzxHxBse07N4t+XdY5Gd4resV3ZMdQETcDryihHgqVqLzX5VF/wRWbnIsFa0YEzmhvQH4Jmnx4etJ/yp/Y1nJLhsTESdUkh1ARDwaESeQEnIZHpT0pdyFD6Tu/Pwc9OEezhtoT0n6kKRl35WShkk6AHjKMXXRqn+Hy7hLs3X9WtKvgPPp/I/ldcAhQJlflucAd0q6mK5xHQj81DF1FREvk9ZfbCUPSvoSqYX3GCx7VjyJ8r6YDgCOAv6QYwngMeAqyl1A+UDgBNJyZpWu3pHAdLqupVlGTD+S9BRpINTaJccErft3uIy7NFuYpPcBe5EGYojUirkqIq4tOa4tasR1X4kxbU7n8P+WiKlVSXol6Yvpg8B6eXPli+n4iCillZCncGwI3B4Riwrb9yizRSzpraQv7/nA5sAOwH1l/38IIGkd0n/vp0bEh8uOp0jSzqSBZHNLfI7ehROemS0j6WNlrPYh6QjgMOB+oAP4XERcmffdHRGlLKAs6RvAe0m9Yb8lfYH/AdgduC4ijishpquqbH4H6bksEbFXcyNKJP0xIrbPrz9J+vv8JfBu4OqIOL6MuIqc8FpUHkTwDdJovqOBzwL7AH8ifRk8UlJca5FG+G1IGhByUWHfjyLiM2XEVYukX0fEe8uOY7CQ9FBENP05nqS5wPiIWCRpDHA5cEFEnCbpnojYptkxFeLqAFYFHgU2jIhnJa0O3BERW5cQ093AfcDZpJangIvI3ZkR8Ydmx5TjWvb3JOlO4H0RsVDSK0it9jeVEVeRn+G1rqnAr0gDVKYDFwJ7krqhzsy/y3AuadThNODjkvYDDoqIF0hdPU0nqda//kX6srICSXNq7QLWr7FvoA2vdGNGxAJJuwKXS9oox1WWJRGxFPiPpPkR8WyOcbGksqaWjAM+B3wV+GJEzJK0uKxEVzAsd5cPIzWmFgJExPOSlpQbWuKE17rWj4jTAfJcshPy9tMlfaLEuMZGxL759S8lfRX4vaRSulGyO0ndTNW+GEc2OZbBYH3gPSw/ok/Arc0PB4BHJXVExCyA3NLbkzQgqcyWwYuS1oiI/wDbVTbmuYylJLw8EOoUSZfl34/RGt/lawN3kf47CkmvjohHJa1Juf9oWaYV/pCsuuKUkfN72Ndsq0oalv+nIyKOk/QP4CZgzZJiup80Z/Ev3XdIaonh0C3mGmDNSnIpknRj88MB0ujjLq2AiFgCHCLpJ+WEBMAuufeikmgqVibNWyxNRPwD+JCk9wPPlhlLjmdMjV0vk6rClM7P8FqUpGOBE4uj1fL215NG0u1XUlwnAtdHxA3dtu8BnF5GuazcrTo3Ih6osm/viPhls2Mys9bjhDdISPpARFxddhzdtWJcrRiTmZXPCW+QkDSnjBFhvWnFuFoxJjMrn0uLDR4t8dC3ilaMqxVjMrOSOeENHq3aFG/FuFoxJjMrmROemZm1BSc8MzNrC054g0fpK53X0IpxtWJMZlYyj9I0M7O24BaemZm1BSc8MzNrC054ZoakVSXdIGmWpAMk7Szp3vx+A0mX93L+2Xlh4BW5966SdlyxyM3q5+LRZgawDbByRHQASDoTOKmwGGyPtVsj4pN9uPeuwCLKW6nB2oRbeGZDgKRDJM2RNFvSBZI2kvS7vO13kkbn40ZJmibpzvyzk6T1gJ8BHblF9ylgf+BoSRdKGiNpXj5/uKSTJM3N1/5s3n6jpHH59bsl3SbpbkmX5eVhkLRA0jfz9rmSNsuLvR4KHJnvvXOz/+ysfbiFZzbISdqStBjoThHxhKRXAecB50fEeZI+DvwA2Bs4DTglIm7JSfC6iNhc0ieBKRGxZ77meOCaiLg8J6WKycDGwDYRsSTfqxjLusDXgN3zwp9fBr4AHJsPeSIitpX0mXy/T+bW5KKIOGkA/njMlnHCMxv83gFcHhFPAETEkzlh7ZP3XwCcmF/vDmwhLSs3upakEQ3ca3fgzLxWHRHxZLf9OwBbADPyPVYBbivsvyL/vqsQn1lTOOGZDX6i9/qhlf3DgPERsbjLBVR3ve3e7iXgtxExscb+F/Lvpfj7x5rMz/DMBr/fAftLWgcgdzPeChyY9x8M3JJfXw8cXjlRUkeD97oeOFTSSoV7Fd0O7JQXKkbSGpLe0Ms1nwMaaWWarRAnPLNBLiLuBY4D/iBpNnAycATwMUlzgI8An8uHHwGMywNO7iMNGGnE2cBDwJx8r4O6xbIQmARclO99O7BZL9e8GpjgQSs20FxazMzM2oJbeGZm1hac8MzMrC044ZmZWVtwwjMzs7bghGdmZm3BCc/MzNqCE56ZmbUFJzwzM2sL/x8eQVHz7XIjjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# What coefficients does ridge regression choose for the optimal lambda value?\n",
    "model = linear_model.Ridge(alpha=53, fit_intercept=False, solver='cholesky').fit(X, y)\n",
    "df = pd.DataFrame({'feature': X.columns, 'coefficient': model.coef_})\n",
    "ax = sns.barplot(x='coefficient', y='feature', data=df);\n",
    "plt.xticks(rotation=90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.6.2 The Lasso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lasso_cv(X, y, λ, k):\n",
    "    \"\"\"Perform the lasso with \n",
    "    k-fold cross validation to return mean MSE scores for each fold\"\"\"\n",
    "    # Split dataset into k-folds\n",
    "    # Note: np.array_split doesn't raise excpetion is folds are unequal in size\n",
    "    X_folds = np.array_split(X, k)\n",
    "    y_folds = np.array_split(y, k)\n",
    "    \n",
    "    MSEs = []\n",
    "    for f in np.arange(len(X_folds)):\n",
    "        # Create training and test sets\n",
    "        X_test  = X_folds[f]\n",
    "        y_test  = y_folds[f]\n",
    "        X_train = X.drop(X_folds[f].index)\n",
    "        y_train = y.drop(y_folds[f].index)\n",
    "        \n",
    "        # Fit model\n",
    "        model = linear_model.Lasso(alpha=λ, copy_X=True, fit_intercept=False, max_iter=10000,\n",
    "                                   normalize=False, positive=False, precompute=False, random_state=None,\n",
    "                                   selection='cyclic', tol=0.0001, warm_start=False).fit(X_train, y_train)\n",
    "        \n",
    "        # Measure MSE\n",
    "        y_hat = model.predict(X_test)\n",
    "        MSEs += [mse(y_hat, y_test)]\n",
    "    return MSEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAELCAYAAAAP/iu7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X10XNV57/HvM5L1Ylm2ZVt2jGWKCY6DaRMCKpC6SUOT8BZaQoAbmjTmEhpDnay2q6WB3JYmhGYV0pfckpQQSEgwaQgplMLFkIQSUlZdXiol4Bi7DoZALCCWbEu2LMmSRvPcP+Yc5Xg8o9c5mjOj32ctLc3sOTOzj46kZ/bez97b3B0REZE4pUpdARERqXwKNiIiEjsFGxERiZ2CjYiIxE7BRkREYqdgIyIisYst2JjZHWbWaWbbImWXmNnzZpYxs9ac499iZk8Gj//EzOqC8lOD+7vM7GYzs6B8kZk9amYvBN+bgnILjttlZlvN7JS4zlFERCYmzpbNN4Bzcsq2AR8AnogWmlk18E3gKnc/CXgXMBw8/GVgA7A6+Apf81rgMXdfDTwW3Ac4N3LshuD5IiJSQrEFG3d/AtifU7bD3XfmOfwsYKu7Pxcct8/dR8xsOTDf3Z/07OzTTcD7g+dcANwZ3L4zp3yTZz0FLAxeR0RESiQpYzZvAtzMvmdmPzKzTwblK4COyHEdQRnAMnd/HSD4vjTynN0FniMiIiVQXeoKBKqB3wR+HegHHjOzduBgnmPHW1/HJvocM9tAtquNhoaGU9/85jdPuMIiIgLt7e173b15vOOSEmw6gP9w970AZvYwcArZcZyWyHEtwGvB7T1mttzdXw+6yTojr7WywHOO4O63AbcBtLa2eltbW5FOR0RkdjCzVyZyXFK60b4HvMXM5gbJAr8FbA+6x3rN7IwgC2098EDwnAeBy4Lbl+WUrw+y0s4ADoTdbSIiUhqxtWzM7G6yWWVLzKwD+DTZhIEvAs3AZjN71t3PdvduM/sH4L/Jdnk97O6bg5f6Q7KZbfXAI8EXwI3Ad8zsCuDnwCVB+cPAecAusl1yl8d1jiIiMjGmLQay1I0mIjJ5Ztbu7q3jHZeUbjQREalgCjYiIhK7pGSjlaVMxtnXN8RQeoSa6ioWN9SQSuXLvBYRmd0UbKYok3F27unlY5va6OgeoKWpntvXt7JmWaMCjohIDnWjTdG+vqHRQAPQ0T3Axza1sa9vqMQ1ExFJHgWbKRpKj4wGmlBH9wBD6ZES1UhEJLkUbKaoprqKlqb6I8pamuqpqa4qUY1ERJJLwWaKFjfUcPv61tGAE47ZLG6oKXHNRESSRwkCU5RKGWuWNXL/xnXKRhMRGYeCzTSkUkZzY22pqyEiknjqRhMRkdgp2IiISOwUbEREJHYKNiIiEjsFGxERiZ2CjYiIxE7BRkREYqdgIyIisVOwERGR2CnYiIhI7BRsREQkdgo2IiISOwUbERGJnYKNiIjETsFGRERip2AjIiKxU7AREZHYKdiIiEjsFGxERCR2CjYiIhK72IKNmd1hZp1mti1SdomZPW9mGTNrjZQfZ2YDZvZs8HVr5LEfmtnOyGNLg/JaM7vHzHaZ2dNmdlzkOZ8Kynea2dlxnaOIiExMdYyv/Q3gS8CmSNk24APAV/Ic/6K7n1zgtT7s7m05ZVcA3e5+gpldCtwEfNDM1gKXAicBxwD/bmZvcveRqZ+KiIhMR2wtG3d/AtifU7bD3XcW6S0uAO4Mbt8LvNvMLCj/trsPuvvPgF3AaUV6TxERmYIkjdmsMrMfm9l/mNk7ch77etCFdl0QUABWALsB3D0NHAAWR8sDHUHZUcxsg5m1mVlbV1dXUU9GRER+KSnB5nXgWHd/G/CnwLfMbH7w2Ifd/deAdwRfHwnK7eiXwccoP7rQ/TZ3b3X31ubm5mmdgIiIFJaIYBN0ee0LbrcDLwJvCu6/GnzvBb7FL7vEOoCVAGZWDSwg2203Wh5oAV6L/yxERKSQRAQbM2s2s6rg9vHAauAlM6s2syVB+RzgfLJJBgAPApcFty8GfuDuHpRfGmSrrQpe65mZOxsREckVWzaamd0NvAtYYmYdwKfJtjy+CDQDm83sWXc/G3gn8FkzSwMjwFXuvt/MGoDvBYGmCvh34PbgLb4G3GVmu4LXvRTA3Z83s+8A24E08HFloomIlJZlGwPS2trqbW252dUiIjIWM2t399bxjotzno3EKJNx9vUNMZQeoaa6isUNNaRS+XIjRERKT8GmDGUyzs49vXxsUxsd3QO0NNVz+/pW1ixrVMARkURKRIKATM6+vqHRQAPQ0T3Axza1sa9vqMQ1ExHJTy2bMjSUHqGje4C3rVzIVe96Iwvr59AzMEwmkyl11URE8lKwKTOZjDOScc5au5TLfmMV19y3dbQr7SsfOZXmxjp1pYlI4qgbrczs6xvirzdv59pzTxwNNJDtSrvyrnZ1pYlIIinYlJmh9Ajf397JgYHh0UAT6ugeYCitKUUikjzqRiszc6pTtDTV09k7SEtTPc3zakfHbfqHRqivqSp1FUVEjqJgU0YyGefQ4TR/e/Fb+PqWn/GlD72NgaER/vzerUekQC+s15wbEUkWdaNNUybjdPUO8mp3P129g2Qy8a3IsK9viPV3PMPnv7uTi05dSdPcmtFAA0qBFpHkUstmGmZ6cmWY8hwmA9yz4QylQItIWVDLZhpmenJlTXUVLU31o/d7BoY5a+1Srj57DTc8tJ0P3vYUNzy0nb19Q7G2sESkMsxkz4yCzTSELY2oODPCFjfUcPv61tGAc1/7bv7ifWuVAi0ik5ZOZ9jxi4NceMsW1t30OBfesoWde3pjCzjqRpuGsKURDTgtTfXUVMeXEbZsfi33bDiDEYe6OSmG0hmlQIvIpGQyzmsHsh9Mc3tm7t+4jubG2qK/p4LNNIQtjdwxm8UNNUV/r0LjQ4vn1SgFWkQmZW/fIJ29gzP6QVXBZhpSKWPNskbu37gu9qX+C40PPfiJdWz66GnsOXhYKdAiMq50OkPfYJp9fUMz2jOjMZtpSqWM5sZaVjTNpbmxNrZ/7oXGhwaGRphXV60UaBEZVzqd4dUDA7y8t5/72ndz00VvGR0DDtdXjKNnBtSyKRtjjQ/NdKKCiJSfcJymq3eQmx97gavPXsOd//Uzrjt/LYsbarIfmhfUx/aBWcGmTIw1PhQ2hzVuIyKFhOM0+/qG6Do0yN99b+fo/4vew2mOb26gujq+zi5z13wMgNbWVm9rayt1NcZUaCvoTMZ5eV9f3nEb7d4pIul0ht09/fx0zyHua9+dd3uSE98wf0r/K8ys3d1bxztOLZsyEo4P5SufV1fN+juOHreJK41RRMpD2H0WjtNc9hurZrT7LKQEgQoxrPk2IpJH2H1282MvjAaai05deUSgibP7LKRgUyFyl7KB+CeYikiyRdOcw3Gai05dOTpO01hXPSOBBtSNVjYKjdeEwgSCLzy6c/RTy9LGWprq55Sw1iJSKvnSnK+5bytX3tU+Ok6zsD6eNOd8FGzKwERWl06ljNXN8/jj97xpdAkKJQmIzE5hoClVmnM+6kYrAxNdXbp7YDjvWkea3Ckye0Tn0ySh+yyklk0ZmOikzfA47W8jMntF59MkofsspGBTBia6unRNdRVnrV2aN4e+ubFOXWkiFS6aEFDKNOd81I1WBnL3sSm0uvTihhr+UvvbiMxKuQkBpUxzzie2dzWzO8ys08y2RcouMbPnzSxjZq2R8uPMbMDMng2+bo08dqqZ/cTMdpnZzWZmQfkiM3vUzF4IvjcF5RYct8vMtprZKXGd40yJri695ZozuX/juryD/qmUUZUyzbcRmWVy1z1LWqCBeLvRvgF8CdgUKdsGfAD4Sp7jX3T3k/OUfxnYADwFPAycAzwCXAs85u43mtm1wf1rgHOB1cHX6cHzTy/C+ZRUodUDcoVdblonTWR2yGScXxw8XNJ1zyYitnd39yeA/TllO9x950Rfw8yWA/Pd/UnPLuK2CXh/8PAFwJ3B7Ttzyjd51lPAwuB1ZoXFDTVs+uhpfPKcNdzw0HY+eNtTXPfANvYcjHd/cRGZeeHWzq/1DByRENB1aJAr72rnz/7lOZbOry1JQkCuJI3ZrDKzH5vZf5jZO4KyFUBH5JiOoAxgmbu/DhB8Xxp5zu4Cz6l44Tpp2t9GpLKFYzThmGxuQsC9V72df/6D01mzNBnz7JKSjfY6cKy77zOzU4F/M7OTgHw/ofE+nk/4OWa2gWwXHccee+wkqptsWidNpLJFx2g6uge49Ycvjk7cjK4gckyJx2miEhFs3H0QGAxut5vZi8CbyLZKWiKHtgCvBbf3mNlyd3896CbrDMo7gJUFnpP7vrcBt0F2i4EinU7JadxGpLJF59K0NNXz4909o+M0ixtqOGZhPW+Yn6zpDokIeWbWbGZVwe3jyQ7uvxR0j/Wa2RlBFtp64IHgaQ8ClwW3L8spXx9kpZ0BHAi728pVJuN09Q7yanc/Xb3jj71o3EakcuXOpQm3dv7x7h5ueGg79TVViQs0EGPLxszuBt4FLDGzDuDTZBMGvgg0A5vN7Fl3Pxt4J/BZM0sDI8BV7h4mF/wh2cy2erJZaI8E5TcC3zGzK4CfA5cE5Q8D5wG7gH7g8rjOcSZMZF20XNrfRqQyFZpLk4RJm+PRTp2BpO7U2dU7yIW3bDlq9YDxgsar3f2su+nxo5auObllAcsW1Bd8nogkUybjdHT309k7yOc27zhqjKZUc2m0U2eFmOi6aLm0dI1IZYmO0yR1Ls1YklszAaa+KZqWrhGpHPnGaZI4l2YsatkkXLguWu6YTe66aLmiS9doFWiR8lXO4zRRCjZFMN4umtMRXRdtsq+vrjSR8pa75llSxmmmIvk1TLgwW+zCW7aw7qbHufCWLezc01vUFONwXbQVTXNpbqydcJBQV5pIecs3TlPqTdCmqjxqmWAT3UWzFLQKtEj5qoRxmih1o03TVLPFZopWExApP5UyThOlYDNNE91Fs1TC1QT2HDw8ujhnmGSwsL54Y0siUhyVNE4TVV61TaCJ7qJZKloFWqS8VNI4TZRaNtM0nWyxmRKuAq0UaJFkyzdOc819W7nyrvbRTNJyGqeJUrApgonuojkVxUirVgq0SPJV4jhNVPm1xWaRYqVVKwVaJNlyx2nCQFPu4zRRatkkWKG06smu3KzVBESSrdzXPZuI8q59hStmWnXYlXb12b/c4+aGh7azt29Ie9yIlFClzacpRC2bBCtmWnXYlfahrz59VFea9rgRKY1KH6eJUssmwYqZVq3VBESSJQw0lTxOE6WWTYIVO61aqwmIJEM0IaCSx2miFGwSrphp1VpNQCQZogkBlTafphAFm1kkXE1g/R1baZ5Xy3Xnr2Vh/Rx+ceAwy+bXsqhB4zYicctNCKjkcZqoMdtoZvb7kdvrch77RFyVkvgMpzM0z6s9Iivtuge28XrPYWWlicSsUEJApY7TRI13Rn8auf3FnMc+WuS6yAyoqa7ij969+ugJnt/UBE+ROM22hIBc43WjWYHb+e5LGVjcUMOqJQ2a4Ckyg2ZjQkCu8c7MC9zOd1/KQCplzK3VBE+RmZQvIaASJ26OxdwL/3Mxs35gF9lWzBuD2wT3j3f3hthrOENaW1u9ra1tys8vxoKZM/Ga4et2dPcfMcETsvN4NMFTpLjS6Qy7e/r56Z5DFTlOY2bt7t463nHjdaOdWKT6VLRwwcxwHbMwnXjNssYpB4c4XjOktdJEZsZsTgjINeYZuvsr0S/gEHAKsCS4LxReMHM6A+5xvGaU1koTiddsTwjINV7q80Nm9qvB7eXANrJZaHeZ2Z/MQP3KQjEXzIzzNaO07YBIfKKBptJ23Jyq8brRVrn7tuD25cCj7r7ezBqBLcD/jbV2ZaKYC2bG+ZpR6koTiUdu5tlsWSFgPOMFm+HI7XcDtwO4e6+Z6T9SIFwwM3d8ZSoLZsb5mrm0g6dI8eVmns2WFQLGM1422v8Dvg90AHeQben0mFk90ObuJ81MNeM3m7LRoq+vrDSR4qn0zLN8ipWNdgXwWeA9wAfdvScoPwP4+jgVuAM4H+h093Dc5xLgM2Sz3E5z97ac5xwLbAc+4+5/F5S9DPQCI0A6PCkzWwTcAxwHvAz8L3fvNjMD/hE4D+gH/re7/2ic85y2Yi6YGedr5r6+utJEikOZZ2MbLxut092vcvcL3P37kfLHw2Awhm8A5+SUbQM+ADxR4DlfAB7JU36mu5+cEz2vBR5z99XAY8F9gHOB1cHXBuDL49RzVlNWmsj0KfNsfGO2bMzswbEed/ffHeOxJ8zsuJyyHcHr5nuv9wMvAX1jvWfEBcC7gtt3Aj8ErgnKN3m2f/ApM1toZsvd/fUJvu6soh08RaanUObZbFqKZiLG60Z7O7AbuBt4mpjWQzOzBrKB4r3A1TkPO/B9M3PgK+5+W1C+LAwg7v66mS0NylcEdQ51BGVHBRsz20C29cOxxx5bpLMpL7k7eEa704bSI2QyPusGMkUmKjfQKPOssPGCzRvIBoDfAz4EbAbudvfni1yP64EvuPuhPK2ede7+WhBMHjWz/3H3Qt1wkD8g5u0PCgLXbZBNEJhCvStCdAfPq89ec0RmWrFWLRCpNPkCjTLPChsz2Lj7CPBd4LtmVks26PzQzD7r7rlbDkzH6cDFZvZ5YCGQMbPD7v4ld38tqEunmd0PnEZ2zGdP2D0WTDjtDF6rA1gZee0W4LUi1nVGxJ2JFhWmWf/iwGGuuU8bq4mMZ6xAo3Ga/MbdqTMIMu8jG2iOA24G/rWYlXD3d0Te7zPAIXf/UtC9lgrm9TQAZ5HNjgN4ELgMuDH4/kCk/BNm9m2yQexAuY3XxLkuWj6plLFmWSMNtVV5Wzdf+f1TtW20SCDfpE0FmvGNt1zNncB/kV0P7Xp3/3V3v8HdXx3vhc3sbuBJYI2ZdZjZFWZ2oZl1kB0L2mxm3xvnZZYB/2lmzwHPAJvd/bvBYzcC7zWzF8h29d0YlD9MNtFgF9lJqBvHq2vSxL0uWj6plFE/p1obq4mMo9CkTQWasY3XsvkI2eywNwF/FBlPMcDdfX6hJ7r77xV46P6x3tDdPxO5/RLw1gLH7SO7qkFuuQMfH+s9ki7uddEK0cZqImNLpzP0DaYVaKZgvDEb/cQmqRhjLXGvi1ZIdGM1LWEjciRN2pyeccdsZOKKNdYyE+uiFbKkoXZ03k00UaDz4CDLFwwpUUBmpdxJm1efvUaBZpIUbIqo0FjLZCdHhgP2929cNyPZaLnvXZUyJQqIBDRpszj00ymiYo61hOuirWiaS3Nj7Yz+g6+prlKigAiFJ212HRrkyrva+bN/eY6l82s1aXMC1LIpolKNtRSbEgVENGmz2NSyKaJwrKWlqR5gRsdaiimaKKAFOmU20qTN4htzP5vZZLr72YRmcuZ/nKJ73TTPqx1t3fQPjfDWlQuUKCAVS4Fmcoq1n41MUjH2oElCwFKigMxGCjTxUbBJmJleqmYs0USBI9KgewdZvlBp0FI5MhmnZ2CI3sNpBZqY6CeWMKVYqqaQMFEgbN2EYzfXPbCN13sOa+xGKkI6nWHHLw7y3O4DCjQx0k8tYUq1VE0+YaKA0qClUoWLal55Vztza6oUaGKkbrQYTGfMJWnp00saapUGLRUrXFSzo3uAnoFhBZoYKdgU2XTHXEq5VE0+Wi9NKlV0Uc2Wpnpu/eGLRy1Ds7SxlmMUaIpCqc+BYqU+d/UOcuEtW45qmUxmyZokZKPl1kdp0FJJwqyzl7r6uPuZV0Y/SDXPq+WP3r2a45Y00FBbxZKGmV29oxwp9blEijHmUoz06WJSGrRUkkKLauauCqDWTHEp2BRZ0sZcikVp0FIJtKhm6egnWmSVsmRNLqVBS7nTopqlpTGbQLHGbGB6Yy5JG6+J6uw9zPOvHuS6B7Zp7EbKilYGiI/GbEpoqmMuSVo9IJ8wDVpjN1JOFGiSQT/ZBEnS6gH5aJKnlBsFmuRQyyYmU+kOS9LqAYVokqeUCwWaZFGwicFUu8PKIZNNkzylHCjQJI9+yjGYandYuWSyLWmo5S/ft3Y0DforHzmVv7/krXQeHKRnQF1pUloKNMmklk0MptodlkoZa5Y1cv/GdYnMRgtpkqcklQJNcinYxGA63WFJWz2gEE3ylKRRoEk2/cRjUC7dYdOhSZ6SJAo0yaeWTQym2h2W5AmduXLToNW6kVJRoCkPCjYxmWx3WNIndOaTO8kz+sfdezjN/No5+uOWWIWbnynQJJ9++gmR9Amd+URbN3f+18+47DdWccND27n41if58FefZmdnr7rTJFbh5mcKNMkX6xUwszvMrNPMtkXKLjGz580sY2ZHradjZsea2SEzuzpSdo6Z7TSzXWZ2baR8lZk9bWYvmNk9ZlYTlNcG93cFjx8X53kWksk4Xb2DvNrdT1fv4Jj/eMthQmc+YevmolNXKhVaZkwm4/T0D45ufqZAk3xxX4VvAOfklG0DPgA8UeA5XwAeCe+YWRXwT8C5wFrg98xsbfDwTcAX3H010A1cEZRfAXS7+wnB69007TOZpLBb7MJbtrDupse58JYt7NxT+JN+mMEWlbQJnfmErZvFDTVKFpAZkU5neGnvIfb3DfPy3n4FmjIR65Vw9yeA/TllO9x9Z77jzez9wEvA85Hi04Bd7v6Suw8B3wYuMDMDfhu4NzjuTuD9we0LgvsEj787OH7GTLZbrJwz2JY01LK0sfaIZIHR1k2vWjdSPGEywO79A+zvG+Lmx15QoCkTiUkQMLMG4BrgvcDVkYdWALsj9zuA04HFQI+7pyPlK3Kf4+5pMzsQHL83thPIMdlusXKZ0JlPKmUcs6CeEXdN9JTYRLPO5tZUafOzMpOkK3I92S6xQznl+f5D+RjlYz3nyBc222BmbWbW1tXVNanKjmcq3WJhBtuKprk0N5bX3ufV1SkaaqvVupFY5KY39w+NaPOzMpOkYHM68Hkzexn4E+D/mNknyLZYVkaOawFeI9tKWWhm1TnlRJ8TPL6AnO48AHe/zd1b3b21ubm5qCdTzt1iU5WbCn1f+256BoZprKum93CadFqrQsvk5ZtH09Qwh8vXZbvPrjt/Lfde9Xa+9Qens2ZpcqcKzHaJ6UZz93eEt83sM8Ahd/9SECxWm9kq4FXgUuBD7u5m9jhwMdlxnMuAB4KXeDC4/2Tw+A98hrcknWy3WDlN6CwkXyp07qrQJ75hftmdl5ROoQmbtzy+i8vXreJT551Iyoz6OVUsmVdevQGzTazBxszuBt4FLDGzDuDTZFsYXwSagc1m9qy7n13oNYIxl08A3wOqgDvcPUwguAb4tpn9NfBj4GtB+deAu8xsV/B+lxb95CZgohM7y3FCZyH5UqFHVxY4OMjyBVpZQCZmvJUB5lSlSJkpGaBM2Ax/4E+s1tZWb2trK/rrTqTF0tU7yIW3bDlq4c77N64ri0U5c3X2Hubn+/r53OYdeZMFTlyu1o0Ulsk4Bw8PcWAgrZUByoCZtbv7UXMmcyWmG60STbTFUq4TOgtZ0lDL4aERrZsmk5ZOZ3h5fx8pM/b3DSnQVBBdqRhNdK5NuU7oLCRMhT5uyVxN9JQJy51Do0BTWXS1YjTRFkslZq4pFVomI98cGgWayqJutBhNZhO1ZfNruWfDGYw41M1JsaSh/DNrtCq0TERuIkBNVUqBpgIpQSAQR4JAOGbzhUd3jv7BLG2s5ZjIH0wlZaLl09l7mOdfPcjdz7yiVGg5Sr6Ms41nnsDA0Ahf3/Kzgn83khwTTRBQsAnElY2WTmfY2dnLlXe15w0mlZaJliuTcX6+v5+de3q54aHtNM+rHV1apH9ohLeuXKBkgVlovIyzy9et4g0L6jSHpgxMNNjoY0LMugeGRwMNHJ0kUGmZaLm0KrTkiq7aPJE5NEvn1ynQVACN2cRsrGCSyTgjGZ/wuE65Uiq0hKIZZ4111UoEmEUUbGIWJgnkdh/VBxk3f715Ozdd9JajxjLKORMtV75VoZUsMPso42x2U7CJ2eKGGjZ99DT2HDzMn9+79Yhxm8a6ar6/vZOu3qHRT/o9A8MsKcN10cYTTYXWummzS77xGWWczT4KNjFLpYx5ddWsv2PrEeM2//aj3XzojONoaarnx7t7uPKuduCXyQGVSOumzT6FVgTYeOYJXL5ulTLOZhEFmxkwnM7Q0T3A21Yu5JPnrGHVkrkMpp3PzYIutKh8yQLaZK1yjTU+o1WbZx8FmxlQU13FWWuXsvHMEzCgfyjD/r6hWdOFFqVkgdlhIuMzWrV5dlGwmQGLG2r4y/et5cWuPgCGR7IrQc+mLrSQkgUqm8ZnpBBd5RmQShlVKWNuTdURn/JuuugtR6yHVsldaFH5kgVueGg7F9/6JB/+6tPs7OzV3Jsyk8k4+/sG886fie6qGR2fUaCZXdSymSE11VX0D2UnakY/5V13/tojPuVVchdalJIFKkPYknm1+zCdvYMAGp+RvPSxYoYsbqjhVxbPZZE+5QH5kwXua99Nz8AwjXXV9B5Ok05nSl1NGUN0JYArv9l+VMtdKwJIlFo2MySVMo5b3EDPwBDD6Qx/9TsnkXGnbk5VRazwPBXRZIGwOy385xSabUG4HETHZcJMs47uAXoGhjU+IwXpys+gVMpY1FDLsgX1tDTN5dhFDSxtnL2f8qKbrF106srRgHNf+2729Q3R1TvIqwcG1MJJkNx1zcKWTEtTPbf+8EWNz0hBuvpSUmGywOKGmiMCjhIGkiffTpr9QyOjyS5dhwa5/sHtVKdSXHf+SZy4vJFjF8+lpWmuAo2oG01KL+xOA5QwkEC56czRcZnoSgBhssvSxlqWz69TgJEj6LdBSi7sTmturFXCQIJkMk5P/9HpzGFrJsw0A/jUeSfy5jc0cuwitWQkP/1GSCJUV6dYEQScaMKAxm9mXm6QiXab5Zs3M6cqRZUyzWQcCjaSGGHAUcJA6UQTAMIgk5vOrNaMTIXGbCRRCiUMaDuCeOVLZw6DTL50Zq1rJpOlYCOJo4SBmRMGmb2Hhka3AcgNMtoOQIpBvymSOEoYmBn5usyiCQDh2Iy6zaQY9NsiiaSEgfiECQDROTPRcZkmgz7qAAALm0lEQVR8QeaYhfU01FSzsmmukgBkShRsJLGUMFA84arMe3sP510BoFA6s4KMFIuCjSRaoYQBBZyJiaYx797fz8HD6TG7zJTOLHGJLdiY2R1m1mlm2yJll5jZ82aWMbPWSPlpZvZs8PWcmV0YeexlM/tJ8FhbpHyRmT1qZi8E35uCcjOzm81sl5ltNbNT4jpHmRlLGmpZGozfKOBMTP65MsMT6jLTuIzEIc7fpG8A5+SUbQM+ADyRp7zV3U8OnvMVM4tmyp3p7ie7e2uk7FrgMXdfDTwW3Ac4F1gdfG0AvlyEc5ESyk0YUMAprNCEzOjy/+oyk1KILdi4+xPA/pyyHe6+M8+x/e6eDu7WARNZdfEC4M7g9p3A+yPlmzzrKWChmS2fyjlIckQTBhRwjjZWkAkDjLrMpJQS00Y2s9PN7HngJ8BVkeDjwPfNrN3MNkSesszdXwcIvi8NylcAuyPHdQRl+d5zg5m1mVlbV1dXMU9HYqCAc6R8g/75gkwYYBapy0xKKDG/Xe7+tLufBPw68CkzqwseWufup5DtHvu4mb1znJfK97Esb0vJ3W9z91Z3b21ubp5y3WXmjBVwbnhoO5/bvIOXuvrY3dNPZ+/hityaYKxB/3xBJgww/UMjrFxUz1/9zkmsaKqnoVZdZjJzEhNsQu6+A+gDfjW4/1rwvRO4HzgtOHRP2D0WfO8MyjuAlZGXbAFei7/mMlPyBZxwlYGrz17D3c+8wk/3HOLn+/rp6O6vmFbORAb9x5qQeczCemqqqjhmQf2s37hPZl4igo2ZrQoTAszsV4A1wMtm1mBmjUF5A3AW2WQCgAeBy4LblwEPRMrXB1lpZwAHwu42qRy5Aaeje4Cr3vXGI7rVhkcyjLjz6oEBOg+WZytnvK6yiQYZDfxLqcWZ+nw38CSwxsw6zOwKM7vQzDqAtwObzex7weG/CTxnZs+Sbb1sdPe9wDLgP83sOeAZYLO7fzd4zo3Ae83sBeC9wX2Ah4GXgF3A7cDGuM5RSisacFqa6llYP2e0W23jmScA8DcP7+B/ftHLz/eXVytnol1lCjJSLsy9/D7txaG1tdXb2trGP1ASJ53OsLOzl86DgzTWVY8uInn3M6+MrlR8+bpVvGFBHSkz6udUsWRebaL++WYyzr6+ITKZDNVVRnf/MCkzXtnXD0BjXTUjGadnYHj03DaeeQIG9A+N8PUtyT9HqUxm1p4zLSX/cQo2WQo25S2TcXoGhug9nN26OPzHHK5aPBD8Q07SqsVhnTMZZ8/BQf7xsZ+y8cwTmF83h1f29Y8GGOCoIDMQCTArF9UDRsadujlVLGlQkJGZM9Fgk4gxG5HpSqWMRQ21o91q/UMjo8kD3X3DfH1LcsZy8nWRXfnN9tG6TrarTIP+Ug60n41UlHAcZ3gkQ8qy/3RHMn7EWM7A0Ah/8/CO0VbO4eGR2Fs50VZMz0C2i2z3/gEAhkecju4BFtbPAThqLxmD0SBz+bpVfOq8E9VVJmVHLRupONXVKY5fMo9FDXNK2srJzSTLN9AftmBamurpGRjW/BipWBqzCWjMpjKl0xle3t83ugtlobGcYgyuh62X4XSGVMo4MDBM32CaxmAMBo4e6AeOSGQodp1E4qYEgUlSsKlc4dbHBwbSvNTVd1TGWsEB94xTVzP2gHtu91jfYHaVpXwBBo4e6I9mk1106kpamupZ3FDDSMYZcaibk9KAvySags0kKdhUvkKtnLFSiVcuqqc6dWRvczrjZNxpqK3iQH+a3sPDRwQXyB9gAGWTScVRsJkkBZvZIV8rJ1+3Vhh4ckUD0bGL5vLTPYeAI4ML5A8wmhMjlUjBZpIUbGaXaCun0JyWfKKBY17dHPb2DuZ9zlgtJbVipJIo2EySgs3sE7Zy9h4ayjtbP59CgSi3NaQAI7OFgs0kKdjMXtGgk5s9lisaiG585H/49O+uLTjOE47tKMBIJZtosNGkTpn1Uilj4dxa5tfVjGaWrVxUP5pZFhUNRF2HBrn+we188pw1rGyq57rzT8LdqalWcBHJpZZNQC0biYrOmYn+hVRF5s/0D43w5/dupaN7gJamem5f38qaZY0KMjKrqGUjMg3hWmv5LG6oHQ1E395whrrKRCZAwUZkksYKRCKSn9ZGExGR2CnYiIhI7BRsREQkdgo2IiISOwUbERGJnebZBMysC3hlCk9dAuwtcnVKpZLOBSrrfHQuyaRzgV9x9+bxDlKwmSYza5vIhKZyUEnnApV1PjqXZNK5TJy60UREJHYKNiIiEjsFm+m7rdQVKKJKOheorPPRuSSTzmWCNGYjIiKxU8tGRERip2AzDWZ2jpntNLNdZnZtqeszWWb2spn9xMyeNbO2oGyRmT1qZi8E35tKXc98zOwOM+s0s22Rsrx1t6ybg+u01cxOKV3Nj1bgXD5jZq8G1+ZZMzsv8tingnPZaWZnl6bW+ZnZSjN73Mx2mNnzZvbHQXnZXZsxzqXsro2Z1ZnZM2b2XHAu1wflq8zs6eC63GNmNUF5bXB/V/D4cdOuhLvrawpfQBXwInA8UAM8B6wtdb0meQ4vA0tyyj4PXBvcvha4qdT1LFD3dwKnANvGqztwHvAIYMAZwNOlrv8EzuUzwNV5jl0b/K7VAquC38GqUp9DpH7LgVOC243AT4M6l921GeNcyu7aBD/fecHtOcDTwc/7O8ClQfmtwB8GtzcCtwa3LwXumW4d1LKZutOAXe7+krsPAd8GLihxnYrhAuDO4PadwPtLWJeC3P0JYH9OcaG6XwBs8qyngIVmtnxmajq+AudSyAXAt9190N1/Buwi+7uYCO7+urv/KLjdC+wAVlCG12aMcykksdcm+PkeCu7OCb4c+G3g3qA897qE1+te4N1mNq3NmhRspm4FsDtyv4OxfxGTyIHvm1m7mW0Iypa5++uQ/WMDlpasdpNXqO7leq0+EXQt3RHpziybcwm6Xt5G9lN0WV+bnHOBMrw2ZlZlZs8CncCjZFtePe4e7n8ere/ouQSPHwAWT+f9FWymLl+UL7fUvnXufgpwLvBxM3tnqSsUk3K8Vl8G3gicDLwO/H1QXhbnYmbzgPuAP3H3g2MdmqcsUeeT51zK8tq4+4i7nwy0kG1xnZjvsOB70c9FwWbqOoCVkfstwGslqsuUuPtrwfdO4H6yv4B7wm6M4Htn6Wo4aYXqXnbXyt33BP8cMsDt/LI7JvHnYmZzyP5z/md3/9eguCyvTb5zKedrA+DuPcAPyY7ZLDSzcMfmaH1HzyV4fAET7+rNS8Fm6v4bWB1kc9SQHUR7sMR1mjAzazCzxvA2cBawjew5XBYcdhnwQGlqOCWF6v4gsD7IfDoDOBB26SRVzrjFhWSvDWTP5dIgW2gVsBp4ZqbrV0jQr/81YIe7/0PkobK7NoXOpRyvjZk1m9nC4HY98B6yY1CPAxcHh+Vel/B6XQz8wINsgSkrdZZEOX+RzaT5Kdm+z78odX0mWffjyWbOPAc8H9afbL/sY8ALwfdFpa5rgfrfTbYLY5jsp7ArCtWdbJfAPwXX6SdAa6nrP4FzuSuo69bgD3955Pi/CM5lJ3Buqeufcy6/Sba7ZSvwbPB1XjlemzHOpeyuDfAW4MdBnbcBfxWUH082IO4C/gWoDcrrgvu7gsePn24dtIKAiIjETt1oIiISOwUbERGJnYKNiIjETsFGRERip2AjIiKxU7ARSTAz+4GZPRxMLhQpWwo2Ignm7r8NDALvK3VdRKZDwUYk+R4BPlzqSohMhyZ1iiScmf0AOBVY6WMvaimSWGrZiCSYmf0a2UUQvwVcVOLqiEyZWjYiCWZmXyO7WOLPgOvd/T0lrpLIlCjYiCSUmTUDTwInuvuwmb0A/JYHW0OIlBN1o4kk15XAV919OLh/N9mtLETKjlo2IiISO7VsREQkdgo2IiISOwUbERGJnYKNiIjETsFGRERip2AjIiKxU7AREZHYKdiIiEjs/j9HmgFFIKRvSgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = hitters_pd.drop('Salary', axis=1)\n",
    "y = hitters_pd['Salary']\n",
    "\n",
    "lambdas = np.arange(1, 300, 1)\n",
    "MSEs    = [] \n",
    "for l in lambdas:\n",
    "    MSEs += [np.mean(lasso_cv(X, y, λ=l, k=10))]\n",
    "\n",
    "sns.scatterplot(x='λ', y='MSE', data=pd.DataFrame({'λ': lambdas, 'MSE': MSEs}));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(113299.42601088567, 126)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(zip(MSEs, lambdas))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we use 10-fold cross-validation to find the optimal lambda value for the lasso. The lowest MSE score achieved is 113,299 when λ=126.\n",
    "\n",
    "**Revision note:** This result is marginally better than the result for ridge regression (114,903) which is suprising because the ISL authors observed the opposite."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([-2., -1.,  0.,  1.,  2.,  3.,  4.,  5.,  6.]),\n",
       " <a list of 9 Text xticklabel objects>)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAEPCAYAAAA57nEEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYXFWd//H3h7APS0aIqGiIRJRNDNAii2BQ3EZEEBQiinFUxIFBcOLIz3EQmHEdBFEcGUYloCgooCyjoCIohGVIIBuLjAgKihAEgWAEEj6/P+4pUul0dVcvVdXd9Xk9Tz1ddzv33IYn3z7nnvM9sk1ERMR4t0anKxAREdEOCXgREdEVEvAiIqIrJOBFRERXSMCLiIiukIAXERFdIQEvIiK6QgJeRER0hQS8iIjoCgl4ERHRFdbsdAVipU033dRTpkzpdDUiIsaUefPmPWR70kDnJeDVkXQAcBGwje07JE0Bdrf9nXJ8OnAxcDdV6/hB4F22H+ynzGnAC2z/aKD7T5kyhblz5w6p7ku+9u0hXTfWTfrwuztdhYjoMEm/bea8dGmuagZwLXBI2Z4CvKvXOdfYnmZ7B+Am4MgBypwG/N1IVjIiIgYvAa+QtAGwB/B+Vga8zwF7Spov6dhe5wvYEHikbO8i6TpJt5SfL5O0NnAScHAp4+C2PVBERKwiXZor7Q9cbvtOSQ9L2gk4Dphle194tktzT0nzgU2AJ4BPlOvvAPayvVzSPsBnbB8o6Xigx/ZR7X6giIhYKS28lWYA55Xv55XtvtS6NF8EnAV8oezfGPi+pMXAqcB2zdxU0uGS5kqau2TJkqHXPiIi+pUWHiBpE+C1wPaSDEwADAw00OQS4MLy/d+Aq2wfUAa7XN3MvW2fCZwJ0NPTk9V4IyJaJC28ykHAOba3sD2ltN7uBp6hek/XyKuBu8r3jYHfl+8z6855fIAyIiKiDRLwKjOAH/TadyHV4JXlkhbUDVqpDWJZALwH+Key/wvAZyXNoWoh1lwFbJtBKxERnSU7vWijRU9Pj4c6Dy8ioltJmme7Z6Dz8g4vxrQHvva5TlchIkbAZh8+ruX3SJdmkyQt7bU9U9Lp5fsRkg6r2/+CTtQxIiIaSwtvBNg+o25zJrAY+ENnahMREX1JwBsBkk4AlgL3AD3AuZKWAbsBnwL2A5YDP7E9q0PVjIjoagl4zVuvZFipeQ7VPLxn2b5A0lFU2VnmSnoOcACwtW1Lmti7UEmHA4cDTJ48uXW1j4jocnmH17xlJcPKNNvTgOObuOYx4K/A1yW9HfhL7xNsn2m7x3bPpEkDrm4RERFDlIDXQraXA7tQzenbH7i8szWKiOhe6dIcec9mVikrMKxv+0eSbgB+3dGaRUR0sQS8kTcbOKMMWnkzcLGkdQEBx/Z3YQxeO+buRMT4kEwro0gyrUREDF6zmVbG/Tu8/iaMD6KMeyQtkrRQ0i8kbTGytYyIiFYb9wFvBO1teweqZX8+2eG6RETEIHV1wJM0W9LXJF0l6TeSXiPpm5JulzS7wWXXA5uX66eUBV9r5c0qk9CRdLWkz0v6X0l3Stqz1c8TERGNdcOglYEmjP8t1eKv+wGXAnsAHwBukjTNdv21AG8Cftjkvde0vYukv6PKuLLPUB4gIiKGrxsC3rIyURyo3uFRpf+qubRkQVkEPGB7UTnvVmAKUAt4V0naDHiQ5rs0Lyo/55WyVpNMKxER7dHVXZrFk+XnM3Xfa9v1fxDsDWwB3AqcVPYtZ9Xf4boNyl5Bgz8ukmklIqI9EvAGwfYy4BjgsJIn8wHguZI2kbQOsG9HKxgREQ0l4A2S7fuB7wJH2n6aqrV3I3AZcEcn6xYREY1l4vkokonnERGDl4nnERERdbphlOagSFoBLKL63dwNvMf2nztbq4iIGK608FZXW/due+Bh4MhOVygiIoYvAa9/9VlVpku6rHZA0ullTl8t1+aJkm4uOTe3LvtfI2l++dwiacNOPERERCTgNSRpAvA6Vs3K0p+HbO8EfA2YVfbNohrNOQ3YE1g24hWNiIimJOCtrpaK7E9Uach+2uR1fWVVmQOcIuloYGJZAX0Vkg6XNFfS3CVLlgyv5hER0VAC3upqqci2ANZm5Tu8QWdVsf05qryc6wE31Lo66yXTSkREeyTgNWD7UeBoYJaktYDfAttKWkfSxlTdnf2SNNX2ItufB+YCqwW8iIhojwS8fti+BVgAHGL7XuB7wELgXOCWJoo4RtJiSQuo3t/9uGWVjYiIfiXTyiiSTCsREYOXTCsRERF1EvAGSdKKMq9usaRLJU0s+6dIWlaOLZB0naSXlWOrzOGLiIj2S8AbvP4ysdxVjr0COBv4REdqGBERq0nAG55nM7H0YSPgkTbWJSIi+pHk0UNUl4nlG3W7p5ZJ6xsC6wOv6kTdIiJidWnhDV5/mVhqXZpTqVZGP3OgwpJpJSKiPRLwBq9RJpbeLgH2GqiwZFqJiGiPBLwh6iMTS2+vBu5qb60iIqKRvMMbBtu3lCwqhwDXsPIdnoCnqPJoRkTEKJCAN0i2N+i1/da6zfUaXHM1cHXrahUREQNJl2ZERHSFrm7hSToV+K3tL5XtK4B7bX+gbH8R+L3tUxpcv9T2BpKmA7Ns79umqkdx9uw3dLoKEePee2f+pNNVGBHd3sK7DtgdQNIawKbAdnXHd6daxDUiIsa4bg94cygBjyrQLQYel/S3ktYBtgFul3SlpJslLZL0tv4KlPRKSbdI2lLSa0puzfll34atfZyIiGikq7s0bf9B0nJJk6kCXy1V2G7Ao1Rr3/0FOMD2Y5I2pVq5/BL3sa6SpN2BrwBvs/07SacBR9qeI2kD4K9terSIiOil21t4sLKVVwt419dtX0c1xeAzkhYCP6MKiJv1Uc42VJlV3mr7d3VlnyLpaGCi7eW9L0qmlYiI9kjAW/ke7+VUXZo3ULXwau/vDgUmATuXDCsPAOv2Uc79VC24HWs7bH+Oai7eelQtw617X5RMKxER7ZGAVwW1fYGHba+w/TAwkSroXQ9sDDxo+2lJe1OlFOvLn4G3ULUGpwNImmp7ke3PA3OB1QJeRES0RwIeLKIanXlDr32P2n4IOBfokTSXqrV3R6OCbD8AvBX4qqRXAceUhWIXAMuAH7foGSIiYgDqY+xFdEhPT4/nzp3b6WpERIwpkubZ7hnovLTwIiKiK3R9wFPlWklvrtv3TkmXd7JeERExsrp6Hh6AbUs6Avi+pKuACcCngTcNp1xJa/Y1DSEiIjqj6wMegO3Fki4FPg78DXCO7bskvZdqgde1qaYvHGX7GUlnAjtRTTc43/ZJAJLuA/6LKlh+SdILgQ8CTwOLbL+73c8WERGVBLyVTgRuplrHrkfS9sABwO62l5cgdwjwHeA42w9LWhO4StIFtm8r5Txhew8ASfcDW9h+StLEtj9RREQ8KwGvsP2EpPOBpbaflLQP8EpgriSoWnP3ltNnSHo/1e/vBcC2QC3gnV9X7K3AtyVdDPywr/tKOhw4HGDy5Mkj+1AREfGsrh+00ssz5QNVSrFv2p5WPi+z/W+StgI+ArzW9g7A5ayaeeWJuu9vBM4AdqEKnBN63zCZViIi2iMBr7GfAe8sCaORtElJMr0R8DjwmKTnUwW11ZTg9kLbPwc+RpWebP221DwiIlaTLs0GbC+SdCLws7JW3tPAEVQpwm6jyrv5Gxqvl7cm8J2yJNAawOdtP976mkdERF+SaWUUSaaViIjBS6aViIiIOl3VpSnpecCXqEZfPgncAxxj+05JxwKfBTaz/Wg5fxrwAts/Ktszgf8Afg+sBdwOHGb7L/3cczrwlO3rWvRY49JHL2xu3v8pByYhTkQ0p2taeKrmFvwAuNr2VNvbAp9g5WKuM4CbqObe1UwD/q5XUeeXUZvbUc3ZO3iAW0+nWlsvIiI6qGsCHrA38LTtM2o7bM+3fY2kqcAGwCepAh+S1gZOAg6WNF/SKoGtTDr/G+CRsv1WSTdKukXSzyRtJmkK1UCXY0sZe7b+MSMioi/dFPC2B+Y1ODYD+C5wDfAySc+1/RRwPCtbdLUJ5QdLmk/Vrfkc4NKy/1pgV9s7AucB/2z7Hqp5eKeWMq5pxYNFRMTAuing9ecQ4DzbzwAXAe/o59zzbU8Dnke1UOzHyv4XAldIqu3brpkbSzpc0lxJc5csWTLkB4iIiP51U8C7Fdi5905JOwBbAT+VdA9V8JsxUGGu5nNcCuxVdn0FON32y4EPsWr2lf7KSaaViIg26KaA93NgHUkfrO2Q9ErgNOAE21PK5wXA5pK2oMqosmE/Zb4auKt835iqmxPgvXXnDFRGRES0QdcEvNIiOwB4vaS7JN0KnEA1ivIHvU7/AVVL7ypg216DVmqDWBYCOwL/VvafQLWm3jXAQ3VlXQockEErERGdlUwro0gyrUREDF4yrURERNTpqkwrA2mUiQW4yPb2deedQLVu3smSTgJ+aftnko4Bzuwv80qMrDdffMSwy/jx284Y+KSIGPMS8Iq6TCxn2z6k7JvGykwsfbJ9fN3mMcC3gQS8iIhRJgFvpT4zsZRsKQ1Jmg1cRrXy+QuAqyQ9BOwDfAPoAUy1mOypLal5REQMKAFvpf4ysUwt2VVqngecXH+C7S9L+iiwt+2HJO0MbF7rCpU0sRWVjoiI5mTQSnPuKqnBppUsK8289PkNsKWkr0h6E/BYXycl00pERHsk4K3UZyaWobL9CPAK4GrgSODrDc5LppWIiDZIwFupUSaWLQZRxrNZVSRtCqxh+0LgX4GdRrCuERExSAl4RT+ZWP4wiGLOBH4s6Spgc+Dq8u5vNvD/RrbGERExGMm0Mook00pExOAl00pERESdcRPwJD1P0nmlO/I2ST+S9FJJi3udd4KkWYMse0VJ/rxY0qWZYhARMfaMi4BXlyXlattTbW8LfIIBsqQMwrIyJWF74GGqUZcRETGGjIuAR4MsKcC9/V0k6WpJp0r6paTbJb1S0kWS/k/Svze47HqqASlImi7psrryTpc0s3y/R9KJkm6WtEjS1sN8xoiIGIYBA56kzSR9Q9KPy/a2kt7f+qoNyoBZUmofoHe24ads70U1mfxiqtbb9sBMSZvUnyhpAvA64JIm6/WQ7Z2ArwGD6kaNiIiR1UwLbzZwBVWeSIA7qZIkjxUDZUmpBa9FwK2277f9JFWmlBeVY+uVYPkn4DnAT5u890Xl5zxgSl8nJNNKRER7NBPwNrX9PeAZANvLgRUtrdXgDSdLypPl5zN132vbtVyjy0qw3AJYm5Xv8Jaz6u9w3QZlr6BB3tJkWomIaI9mAt4TpWvPAJJ2BR5taa0GbySypAzI9qPA0cAsSWsBvwW2lbSOpI2pujsjImIUaibgfZSq22+qpDnAOcA/trRWgzRCWVKavdctwALgENv3At8DFgLnAreM9P0iImJk9JtpRdIawK7A/wIvAwT8yvbT7aled0mmlYiIwWs200q/6+HZfkbSF23vRvWebNyQ9DzgS8Arqd613UM1GGcB8Cuqd3VzgfcnwI9eb7nwvzpdhZb4nwM/1OkqRIw7zXRp/kTSgWVy97gwwET1u8oAlZcDLwTe2bmaRkTESGn2Hd73gSclPSbpcUl9LmY6hgw4Ud32Cqqu3Nok85mSTq8dl3SZpOnl+1JJn5a0QNINkjYr+99R0pEtkPTLtjxZRET0acCAZ3tD22vYXtv2RmV7o3ZUroX6m6gOgKR1gVcBlzdR3t8AN9h+BfBLoDZa9HjgjWX/fkOvbkREDFe/7/AAJO3V137b47XFMrVMMt8KuMD2wiaueQqopRibB7y+fJ8DzJb0PVZOQl+FpMOBwwEmT548nHpHREQ/Bgx4wMfqvq8L7EL1j/prW1Kj9rgVOKjBsbtsT5P0fKoFXPezfQn9TzJ/2iuHuz47ydz2EZJeBbwFmC9pmu0/1d/M9plUC8fS09OTxQkjIlqkmS7Nt9Z9Xk/VHfhA66vWUgNOVLd9P3AcK1cqvweYJmkNSS+iCvz9kjTV9o22jwceYmWqsoiIaLOhrJZwH1XQG7MGMVH9h8D6kvak6p68myrn5snAzU3c6j/KSgmLqd7tLRihR4iIiEHqd+I5gKSvUNKKUQXIacA9tt/d4rp1nUw8j4gYvBGZeF7U/wu8HPiu7TlDrllEREQHNBPwJto+rX6HpI/03jfeDDYTS5mTdzFVt+cawIPAu2w/WBaF7bF9VJsfY9zb94JzO10FLjvo0E5XISKa0Mw7vPf2sW/mCNdjVBlGJpZryrp7OwA3sXIZoYiI6LCGLTxJM4B3AS+WVL/C94ZUC6GOZ31mYpE0pW57haRnM7HUKwFzQ+DXra9qREQ0o78uzeuA+4FNgS/W7X+cajmc8WwwmVg+Urd7zzJpfRPgCapWYUREjAINA57t31ItcLpb+6ozJvSXieUa2/sCSPo48AXgiP4KS6aViIj2GPAdnqRdJd1UEiQ/JWnFOEgePZBbgZ0bHKu9w3sJsKukRjkyLwH6TMtWz/aZtnts90yaNGlotY2IiAE1M2jldGAG8H/AesAHgK+0slKjwFAysfT2auCuVlYyIiKa11SmFdu/BibYXmH7LKpBHePWEDOxQHmHJ2kB8B7gn9pV54iI6F8zmVZ+CewDfB34I9VAlpllyZsYQcm0EhExeM1mWmmmhfeect5RVCMPXwQcOLzqRUREtNeAmVZs/1bSesDzbZ/YhjqNuAZZU54Ezrb9w3LOr4Bv2f73sn0hcK7tRuvYTQEus719Mqn0bb8LLm75PS456G0tv0dEjA/NjNJ8KzCfsvK3pGm9JqKPav1kTbkZ2L2cswmwlFWnYOxGNRcxIiLGgWa6NE+gWvvtz1BlHAGmtK5KI67PrCnAlZSAV35eBkxS5cXAMtt/lDRF0jWSbi6f3Ve7Qx1Jb5F0vaRNJb1D0mJJC8q70IiI6JBmkkcvt/1o1VAakxplTZkHbC9pbaqA9wtgS2AbYEeq9e+gSgL9ett/lbQV8F2gz5ejkg4APgr8ne1HJB0PvNH27yVNHMmHioiIwWkm4C2W9C5gQvkH/2jGQVef7SfLdIOdgF2psqJsSRX8dmTlM64FnC5pGrACeGmDIvemCoRvsF2bmD8HmC3pe0Cjd4HJtBIR0QYNuzQlfat8vQvYjmqQx3eBx6iWyRkr+suach1VNpQNbT8C3EAV8HZnZQvvWOAB4BVUAW3tBmX9hiph9LMB0fYRwCepRrbOL+8KV5FMKxER7dHfO7ydJW0BHEyVPPqNwBvK9/XbULeR0mfWFEmvoQpqH6Ja4w6qpNi7ApOpAiXAxsD9tp+hmqIxocF9fgu8HThH0nblPlNt32j7eOAhqsAXEREd0F+X5hlUIzO3ZNVVzwW47B/1bLu8W/uSpOOAv7JyMdfbqZ7js+Xc5ZIeBO4tAQ7gP4ELJb0DuIpqLmKje/1K0qHA98vo1v8o3cCiGiSzoNG1ERHRWs1kWvma7Q+3qT5dLZlWIiIGb8QyrSTYRUTEeNDMKM0xSdIKYBHVM94OvNf2X/o5fybwE9t/KNtrU43cfCvwDHAbcKTt+wa47yrlRGsdcOHVQ7ruBwdOH9F6RMTo19RqCWPUMtvTbG8PPMUAC7ECM4EX1G1/hjLq0vZWVCsjXKSBJyT2LiciIkaBcdvC6+UaYIf6/JcAkmYBGwCLqaYcnCtpGbAH8D7gxbZXANg+S9LfA6+VdFeT5ewGfArYD1hO1fKb1Z5HjoiIeuO5hQeApDWBN1N1b/bJ9gVUI1EPLauZTwV+VzeBvGYu1ZzEZstZj2pdve1s7wD8+3CeJSIihm48B7z1JM2nCkC/A74xiGtrUy+a3d/IY1TTIL4u6e3Aau8QJR0uaa6kuUuWLBlE0RERMRjjOeDV3uFNs/2Ptp+i6lasf+Z1G1z7a2ALSRv22r8T1eCVpsqxvZwq8faFwP6UFSd6nZNMKxERbTCeA15fHgCeK2kTSesA+9Yde5xqkAq2nwDOBk6RNAFA0mFUGWZ+3mw5kjYANrb9I6qJ7tNa+XAREdFYtwxaAcD205JOAm4E7gbuqDs8GzijbrDJ/wNOBu6U9Ew59wBXM/WbLefNwMWS1qXqDj22hY8XERH9GDDTSrRPMq1ERAzeiGVaiYiIGA+6qksTnl2k9SJgG9t3DHR+r2tPAn5p+2cNju8P3Gn7tuHXNJrxjgsXN33u9w/cvoU1iYjRrhtbeDOAa4FDBnuh7eMbBbtif2DboVYsIiJap6sCXhk1uQfwfkrAk/R8Sb+UNF/SYkl7SpogaXbZXiTp2HLubEkHle+fk3SbpIWSTpa0O1VGlf8oZU2VdHTdOed16LEjIoLu69LcH7jc9p2SHpa0E7A3cIXtT5cpCOtTTR/YvC512MT6QiQ9hyqDytZlvb2Jtv8s6RKqlGMXlPOOo0pP9mTvMiIior26qoVH1Z1Za2mdV7ZvAt4n6QTg5bYfB34DbCnpK5LeRJUxpd6AGVSKhVR5Nd9NNVl9Ncm0EhHRHl0T8CRtAryWKkjdA3wMOJgqsfRewO+Bb0k6zPYjwCuAq4Ejga/Xl9VMBpXiLcBXgZ2BeSWv5yqSaSUioj26qUvzIOAc2x+q7ZD0C6pgN8f2f0v6G2AnST8CnrJ9YVkZYXZ9QeVd4Pq2fyTpBqpUZLBqlpU1gBfZvkrStcC7qFZU+HNLnzIiIvrUTQFvBvC5XvsupApmT0h6GlgKHAZsDpxVghZUWVfqbUjfGVTOA/5b0tFUg2K+IWnjcs6pthPsIiI6JJlWRpFkWomIGLxkWomIiKjTTV2aq5G0glUXhj3Pdu9uz4iIGAe6OuBR1szr7wRJE2yvqNtes4zS7Fez50VERHt0e8DrU5m28E3gDcDpko4ArqPK0nKJpAvK8UnAEuB9tn8naTbwMLAjcHOZiH5aKdbAXmWeX0REtFm3B7z1JM2v2/6s7fPL97/afjVACXgTbb+mbF9KNcXhbEl/D3yZaj4ewEuBfWyvKOcdaXtOmcrw13Y8VERErK7bA15/XZrn97O9G/D28v1bwBfqjn2/rgt0DtWq6ecCF9m+r/dNJB0OHA4wefLkQVY/IiKalVGajT0xwHa9+rkdz55XBsB8AFgPuEHS1qtdmEwrERFtkYA3NNexcnmhQ6mWG1qNpKm2F9n+PDAXWC3gRUREe3R7l2bvd3iX2z6uieuOBr4p6WOUQSsNzjtG0t7ACuA24MfDqm1ERAxZVwc82xMa7J/Sa3t6r+17qBJR975uZq/tfxxmFSMiYoSkSzMiIrpCAl5ERHSFMRnwJC3tdB2aJeliSdd3uh4REd1uTAa8sULSRGAnYKKkF3e6PhER3WzcBDxJkyRdKOmm8tmj7N9F0nWSbik/X1b2ry/pe5IWSjpf0o2SesqxpXXlHlRShjW8Rz8OBC6lWifvkAHOjYiIFhpPozRPo1pk9VpJk4ErgG2AO6hyWC6XtA/wGapA9A/AI7Z3kLQ9ML9RwU3co5EZwInAA8AFwGd7n5BMKxER7TGeAt4+wLaSatsbSdoQ2Bg4W9JWVBlR1irHX01J7Gx7saSFQ71HXwmhJW0GvAS41rYlLZe0ve3F9efZPhM4E6oFYJt/3IiIGIzxFPDWAHazvax+p6SvAFfZPkDSFODq2qF+yqoPPOsOdI8GDgb+Fri7BMiNqLo1P9nEtRERMcLGzTs84CfAUbUNSbWk0BsDvy/fZ9adfy3wznLutsDL6449IGkbSWsABzRxj77MAN5ke0qZyL4zeY8XEdExYzXgrS/pvrrPR6nSffWUQSi3AUeUc78AfFbSHKA+s8p/ApNKV+bHgYXAo+XYccBlwM+B++uuaXSPVZSW5GTghto+23cDj0l61TCeOyIihkh2d742kjQBWMv2XyVNBa4EXmr7qU7Vqaenx3Pnzu3U7SMixiRJ82z3DHTeeHqHN1jrA1dJWovqfd6HOxnsIiKitcZcwJO01PYGwy2njKwc8C+CJurzPuAjvXb/CdgbmGZ7YTlvMbBvSTwdERFtNuYC3mhj+yzgrPp9kmYCWwH/QjVaMyIiOmysDlpZxSjNsnIZsF3tnhER0VnjpYU3GrOsPEM1QvQTwHsbnZRMKxER7TFeAt6oyrJS5zvAv/SXODqZViIi2mO8BLzRlmWlKqhqWX6Rap5fRER00Lh4h8foy7JSbzZV63BSk+dHREQLjMWAN6qzrPRW5vZ9GXjuUB42IiJGRldmWhmNWVYgmVYiIoai2UwrY6aFJ2mFpPmSbpW0QNJHS7cjknokfXmA64+QdFjZXB+4VtIC4AfAh4E3STpuBOs7RdKyUufa57CBr4yIiFYYS4NWltmeBiDpuVQjIDcGPmV7LtBv08j2GXXfG2VZuWQwFWqQZWWO7SPL97tqdY6IiM4aMy28erYfpJq7dpQq0yVdJmkNSfdImlg7V9KvJW0m6QRJs8q+oyXdVt7HnVf2zZR0evm+haQry/Ery7w7JM2W9OUyif03wOO2p/X6HLlahSMiouPGZMADsP0bqvo/t27fM8DFlNGVZSmee2w/0Ovy44Adbe9A34NPTgfOKcfPpRp0UvN8qnl8+wKfG6CaU3t1ae7Z+wRJh0uaK2nukiVLBiguIiKGaswGvKKv+XTnszJ/5SFlu7eFwLmS3g0s7+P4blRdpgDfogpwNT+0/Yzt24DNBqjfXb1af9f0PsH2mbZ7bPdMmpSZCxERrTJmA56kLYEVwIO9Dl0PvETSJGB/4KI+Ln8L8FWqVcjnSRroXWb9UNYn66sxqEpHRETHjMmAV4LZGcDp7jWvomz/ADgFuN32n3pduwbwIttXAf8MTAR6Lzd0HVXrEOBQqonqERExho2lUZrrSZpPlQ9zOVVX4ykNzj0fuIlVs6vUTAC+LWljqhbaqbb/XJcjE6pJ5t+U9DFgCfC+IdZ5aqlzzTdt9zt9IiIiWqMrJ56PVpl4HhExeONu4nlERMRwjNqAN8KZVfo6vt9IZFaR9PJSzz9Lurt8v1HSryR9su68CyW9fbj3i4iIoRm1XZqSltreoHyvZVaZY/tTna1Z38r7vkm2/1nSJlSrK/zR9lvK8T8AO9n+Y6My0qUZETF446pLc7RkVpF0UD/VnAPsXr7vTrXiwqRS3xcgc0n+AAAKUUlEQVRTpUZrGOwiIqK1xkTAgzGRWWUesL2ktakC3vXAr4Btyvacvi5KppWIiPYYMwGvGLWZVWw/CdwK7ATsCtxIFfR2L5/rGlyXTCsREW0wZgLeGMmsch2wF7Ch7UeAG1gZ8Pps4UVERHuMiYA3hjKrzAE+BCwo2wupWnuTqVp/ERHRIaM508pYzKxyHbAl8FkA28slPQjcW943RkREh4zaaQndKNMSIiIGb1xNS4iIiBiulgY8SZb0xbrtWZJOGGaZ0yVdNuzKDf3+tcwq9Z8bG5y7pqSHJH223fWMiIhVtbqF9yTwdkmbtvg+bWN7Ua9FXafZflWD099ANRfvner10jAiItqr1QFvOXAmcGzvA5ImlfySN5XPHmX/IkkTS4aSP9XyYUr6lqR9Gt1I0s6SfiFpnqQrJD2/7P9gKX9Bud/6Zf9USTeUYydJWlr2r9KClHS6pJn93aMfM4DTgN9RjdaMiIgOacc7vK8Ch5ZRkvVOoxox+UrgQODrZf8cYA9gO+A3wJ5l/65U89pWI2kt4CvAQbZ3Br4JfLocvsj2K22/ArgdeH/d/U8r9//DQA8xwD36On894HVUKca+SxX8+jovmVYiItqg5dMSbD8m6Ryqof/L6g7tA2xb19O3kaQNgWuoJm//FvgacLikzYGHbS9t0DP4MmB74Kfl+ATg/nJse0n/zsr5d1eU/btRTVSHKsvKyQM8Sn/36Mu+wFW2/yLpQuBfJR1re0X9SbbPpGoF09PTkyGzEREt0q55eF8CbgbOqtu3BrCb7fogiKRfAkdSTdb+F6o8mQdRBcJGBNxqe7c+js0G9re9oHRNTh+grstZteW7bhP36MsMYA9J95TtTYC9gZ81eX1ERIygtkxLsP0w8D1WdidCtXzOUbUNSdPKufcCmwJblYTR1wKz6D/g/YpqZYLdSllrSdquHNsQuL90SR5ad80NVF2psDLLClQty20lrVO6YV/XxD1WIWkjqnyck21PsT2FKoj32a0ZERGt1855eF+kCmQ1RwM9ZUme21h1FYMbgTvL92uAzVk13dfrJN1X+1DlyDwI+LykBcB8Vi7V86+lvJ8Cd9SVcQzwUUn/S7UiwqPwbMD9HiXhNHBL2f9UP/fo7e3Az0tC6ZqLgf0krdPgmoiIaKGuzbRSRmsus21JhwAzbL+tk3VKppWIiMFrNtPKaM6l2Wo7A6eX+XF/Bv6+w/WJiIgWalnAk2TgFNv/VLZnARvYPmEYZU4HZtned7j1s30N8IrhliPpq1TTKOotBV4MbGn7yTLxfm55lxcRER3Qynd44y7LSl9sH9k78wrwa6q1+9JqjIgYJVoZ8Lo9y8qXgGM18GKzERHRBq0epdmVWVaK31GNLH3PAGUn00pERBu0tPXRxVlWaj4DXAL8T6MTkmklIqI92tHd1o1ZVgCw/WtVq7a/czDXRUTEyGv5xPNuy7LSh0+XZ4iIiA5qV6aVbsqysgrbt1K1cCMiooO6MtPKaMyyUuq1hKqV2U6bAg+1+Z4jKfXvvLH+DKl/5w33GbawPWmgk7p1yPyozLLSzH+wkSZpbjMpeUar1L/zxvozpP6d165n6MqA1+IsK6fZPquv8yMionO6MuCNFNtHdroOERHRnHYuDxSj05mdrsAwpf6dN9afIfXvvLY8Q1cOWomIiO6TFl5ERHSFBLyIiOgKGbQSY4qkrYG3USUkMFXy70ts397RisWYIWkXwLZvkrQt8CbgDts/6nDVhkTSObYP63Q9xoK8w4sxQ9LHgRnAecB9ZfcLqdLDnWf7c52qW7cof3BsDtxoe2nd/jfZvrxzNWuOpE8Bb6b6Y/+nwKuAq6kS2l9he6BVUDpK0iW9dwF7Az8HsL1f2ys1TJJeDewCLLb9k5beKwEvxgpJdwLb2X661/61qZJ7b9WZmg2fpPeN9vmbko6mSu5+OzAN+Ijti8uxm23v1Mn6NUPSIqq6rwP8EXhhWdVlPaogvkNHKzgASTcDt1EtqWaqgPddSk5g27/oXO2aI+l/be9Svn+Q6v+pHwBvAC5t5R+ueYcXAEj6cafr0IRngBf0sf/55dhYdmKnK9CEDwI7296fauWRf5X0kXKsz7W7RqHltlfY/gtwl+3HAMrKLWPh/6EeYB7VajKP2r6aKk3iL8ZCsCvWqvt+OPB62ydSBbxD+75kZOQdXheR1OgvcFH91TvaHQNcKen/gHvLvsnAS6hbfWO0krSw0SFgs3bWZYgm1Loxbd8jaTpwgaQtGDsB7ylJ65eAt3NtZ1kdZdQHPNvPAKdK+n75+QBj79/xNST9LVWDS7aXANh+QtLyVt54rP2iYnhuAn5B3/84TWxzXQbN9uWSXkrV37851XPcB9xke0VHK9eczYA3Ao/02i/guvZXZ9D+KGma7fkAZVHmfYFvAi/vbNWatpftJ+HZ4FGzFvDezlRp8GzfB7xD0luAxzpdn0HamKqVKsCSnmf7j5I2oMV/OOUdXheRtBg4wPb/9XHsXtsv6kC1uoakbwBn2b62j2Pfsf2uDlSraZJeSNUl+Mc+ju1he04HqhXjRFnFZjPbd7fsHgl43UPSQcAi27/q49j+tn/YgWpFRLRFAl4Xk/RW25d2uh4REe2QgNfFJC0c7cOwIyJGSqYldLexMrIuImLYEvC6W5r3EdE1EvAiIqIrJOBFRERXSMDrbg90ugIREe2SUZoREdEV0sKLiIiukIAXERFdIQEvIpC0jqSfSZov6WBJe0q6tWxvLumCAa7/elk9fCj3ni5p96HVPKJ5WS0hIgB2BNayPQ1A0hnAyXWL0h7U38W2PzCMe08HljI2VoyIMSwtvIhxQNJhkhZKWiDpW5K2kHRl2XelpMnlvEmSLpR0U/nsIem5wLeBaaVF9yHgncDxks6VNKWstIGkCZJOlrSolP2PZf/VknrK9zdIul7SzZK+X5Z9QdI9kk4s+xdJ2lrSFOAI4Nhy7z3b/buL7pEWXsQYJ2k7qhWw97D9kKTnAGcD59g+W9LfA18G9gdOA061fW0JglfY3kbSB4BZtvctZe4GXGb7ghKUag4HXgzsaHt5uVd9XTYFPgnsUxb0/DjwUeCkcspDtneS9A/lfh8orcmltk9uwa8n4lkJeBFj32uBC2w/BGD74RKw3l6Ofwv4Qvm+D7Ct9Gwa1Y0kbTiIe+0DnGF7ee1evY7vCmwLzCn3WBu4vu74ReXnvLr6RbRFAl7E2CcGzotaO74GsJvtZasUoKbziA90LwE/tT2jwfEny88V5N+faLO8w4sY+64E3ilpE4DSzXgdcEg5fihQW2X9J8BRtQslTRvkvX4CHCFpzbp71bsB2EPSS8rx9SW9dIAyHwcG08qMGJIEvIgxzvatwKeBX0haAJwCHA28T9JC4D3AR8rpRwM9ZcDJbVQDRgbj68DvgIXlXu/qVZclwEzgu+XeNwBbD1DmpcABGbQSrZbUYhER0RXSwouIiK6QgBcREV0hAS8iIrpCAl5ERHSFBLyIiOgKCXgREdEVEvAiIqIrJOBFRERX+P/5xckPQC8SWwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# What coefficients does the lasso choose for the optimal lambda value?\n",
    "model = linear_model.Lasso(alpha=126, copy_X=True, fit_intercept=False, max_iter=10000,\n",
    "                           normalize=False, positive=False, precompute=False, random_state=None,\n",
    "                           selection='cyclic', tol=0.0001, warm_start=False).fit(X, y)\n",
    "df = pd.DataFrame({'feature': X.columns, 'coefficient': model.coef_})\n",
    "ax = sns.barplot(x='coefficient', y='feature', data=df);\n",
    "plt.xticks(rotation=90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso makes quite different feature choices to ridge regression, completeley ignoring the dummy variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<h4>Model chosen by lasso:</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Salary ~ AtBat + Hits + Walks + CAtBat + CHits + CRuns + CRBI + CWalks + PutOuts + Assists\n",
      "MSE: 113299\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<h4>Model chosen by backward stepwise selection:</h4>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Salary ~ AtBat + Hits + Walks + CAtBat + CRuns + CRBI + CWalks + PutOuts + Division_W\n",
      "MSE: 106619\n"
     ]
    }
   ],
   "source": [
    "display(HTML('<h4>Model chosen by lasso:</h4>'))\n",
    "print('Salary ~ {}'.format(' + '.join(X.columns[model.coef_ != 0])))\n",
    "print('MSE: {:01.0f}'.format(min(MSEs)))\n",
    "\n",
    "display(HTML('<h4>Model chosen by backward stepwise selection:</h4>'))\n",
    "print(backward_select_choice['Model'].iloc[0])\n",
    "print('MSE: {:01.0f}'.format(backward_select_choice['MSE_mean'].iloc[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The results here suggesat that backward stepwise finds as simpler model tha toutperforms lasso and ridge regression. \n",
    "\n",
    "**Revision note:** This is contrary to ISL authors results so needs checking."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab 3: PCR and PLS Regression\n",
    "\n",
    "### 6.7.1 Principle Components Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Deps for pca/pcr\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "X = hitters_pd.drop('Salary', axis=1)\n",
    "y = hitters_pd['Salary']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCA\n",
    "\n",
    "What percentage of total variance in our dataset is ecplained by succesive accumulation of principal components?\n",
    "\n",
    "Get all principal components of the predictors in X, using principal component analysis (PCA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XHW9//HXJ1vTdE2abnTfaCkFSomUTdmRRdkUFblalcfFBRV3ud6fKIpeuHKvy72Ct4pSvVygIghuSFuhCEroArS1C2nTnZKkSZo2TZttPr8/zkkbyiQZkkxOZub9fDzmMeecOWe+n5xO5zPn+z3f79fcHRERkWNlRR2AiIj0T0oQIiISlxKEiIjEpQQhIiJxKUGIiEhcShAiIhKXEoSIiMSlBCEiInEpQYiISFw5UQfQE8XFxT558uSowxARSSmrVq3a6+4ju9ovpRPE5MmTWblyZdRhiIikFDPbnsh+qmISEZG4lCBERCQuJQgREYlLCUJEROJKWoIws5+bWaWZrWu3rcjMlphZWfhcGG43M/uRmW02szVmNi9ZcYmISGKSeQVxP3DpMdtuBZa5+wxgWbgOcBkwI3zcBNybxLhERCQBSUsQ7v4sUHPM5quAReHyIuDqdtt/6YEXgOFmNjZZsYmISNf6uh/EaHffA+Due8xsVLh9HLCz3X67wm17+jg+EemCu9Pc6rTEYsFza/Dc3BqjJRasN7XGaAn3aWoJnlvCfdof29wae9Pxza0xYjFNhdyVC08YzSkThie1jP7SUc7ibIv7CTGzmwiqoZg4cWIyYxJJSbGYc6Cxhf2HmjlwuIX9h5vZf6iZQ82tNDS1cqiplUPNwXPDkeWWI68fbm6/vZXGlhjNLTGaY21f+n3z5W3xvhXkiFFD89MuQVSY2djw6mEsUBlu3wVMaLffeOC1eG/g7guBhQAlJSX6mSFpq6Gpher6JmoOBo/qg03sa2hi/6Fm9h/54j+aANqSQX1jC57A/4wsg4K8HPJzsynIy2ZgbjYDw+cxQ3PJz8umIDebAblZ5GYHj5wsC5eNnHA9LyeLnKwscrKN3GwL9zu6z9FtdvR9so288DknK+vocraRm5VFVpayQ3/Q1wniCWABcGf4/Hi77Z82s4eA+UBdW1WUSLpwd/bWN/HavkNUHWg88qVfc7AxfG46khCqDzZyuDkW933MYMiAHIbk5zJ0YC5D83OYUFTAkPwchrbbFiwHz0PycykY8MZEkJedhelnunQiaQnCzB4EzgOKzWwX8A2CxLDYzG4EdgDXhbv/Ebgc2Aw0AB9NVlwiyRKLOVX1jeyqbWBX7aEjj937DrGrtoHX9h2K+6Wfn5vFiEEDKBqUR9GgPGaMGkzRoDxGDB7AiHBb0eA8RgzKY3hBHkMG5OgXtvSJpCUId7++g5cujLOvAzcnKxaR3tLY0sq2vQ2UVR6gvOogu2sPsWtfA7trD/HavsM0tb4xARQNymN84UBmjh7ChbNGMW74QMYVFjBqyIAwCeRRkNdfmgJF3kifTJE4Dje3sqWqns2V9ZRV1FNWeYCyynq2VzfQ2q6RduSQAYwvHMicccO4dM5YxhUOZHzhQMYPH8i4woH68peUpk+vZLSGppZ2SaCezWEi2FHTcKShNzvLmDyigONHDeGKk8YyfdRgZowawtSRg8jPzY72DxBJIiUIyRjuzq7aQ6zaXnvksfH1/bRdEORmG1OLBzNn3DCuOXUcM0YNYcbowUweMYi8HA1bJplHCULS1uHmVv7xWh2rtteyevs+Vu2opepAIwCD8rI5dWIhnz5/OrOPG8aM0YOZWFRAbrYSgUgbJQhJG5X7D7N6x9Grg3W79x9pNJ5YVMA504uZN6mQ0yYWMnPMELJ1J5BIp5QgJCW5O9urGyjdWk1peQ0rttews+YQAHk5WZw8bhgfPXsy8yYVMm9iISOHDIg4YpHUowQhKcHd2VJ18EhCKN1aTcX+oLpoxKA83ja5iAVnBgnhxOOGMiBHjcciPaUEIf2Su1NWWU9peTUvbK2htLyGvfVBQhg5ZADzpxQxf+oIzphSxPRRg9UjWCQJlCCk39hcWc9fy6ooLa/hxW011BxsAmDssHzOmT6C+VNHMH9KEVOKBykhiPQBJQiJ3Itba7jnmc08s6kKgPGFAzl/5ijmTy3ijCkjmFA0UAlBJAJKEBIJd+fpTZXc8/QWVm6vZcSgPL78zplcNfc4xhcWRB2eiKAEIX2spTXGH9bu4d5ntrDx9QOMGz6Q2688kfeVTGBgnhqWRfoTJQjpE4ebW/nN6l38z/JydtQ0MH3UYP7julO4cu5x6pwm0k8pQUhSHTjczAOlO7jvua1UHWjklAnD+dcrTuDiE0ZryGqRfk4JQpKiur6RXzy/jV/+fRv7D7dwzvRifvj+uZw5bYQanEVShBKE9KrKA4e55+ktPLRiB40tMS49cQyfPG8aJ49P7ty5ItL7lCCkV7g7j67ezbd+v56DjS1cc+o4Pn7uNKaPGhx1aCLSTUoQ0mN76g7xtUfX8vSmKkomFXLXe09m2kglBpFUpwQh3ebuPLRiJ9/9wwZaYs433z2bD585WY3PImlCCUK6ZWdNA7c+uobnN1dz1rQR3HntyUwcoQ5uIulECULekljM+dUL27nryY1kmfHda07i+tMn6M4kkTQUSYIws1uAfwYM+Km7/8DMioCHgcnANuB97l4bRXwS39a9B/nKI6+wYlst5x4/kn+79iSOGz4w6rBEJEn6vAurmc0hSA6nA6cA7zKzGcCtwDJ3nwEsC9elH2iNOT99tpxLf/Asm14/wN3XncL9H32bkoNImoviCuIE4AV3bwAws+XANcBVwHnhPouAZ4CvRhCftFNWcYAvPbKGV3bu4+LZo/nO1XMYNTQ/6rBEpA9EkSDWAd8xsxHAIeByYCUw2t33ALj7HjMbFe9gM7sJuAlg4sSJfRNxBmpujfE/y7fwo2WbGTQgmx9dfyrvPnms2hpEMkifJwh332BmdwFLgHrgFaDlLRy/EFgIUFJS4kkJMsPtqG7g5v9bzdrddbzr5LF888oTKR6sOZ1FMk0kjdTufh9wH4CZfRfYBVSY2djw6mEsUBlFbJnu6Y2V3PLQSwD85J/mcemcsRFHJCJRieouplHuXmlmE4FrgTOBKcAC4M7w+fEoYstUrTHnh8vK+NGyMmaPHcpP/uk09WsQyXBR9YP4TdgG0Qzc7O61ZnYnsNjMbgR2ANdFFFvG2dfQxC0PvczyV6t4z7zxfOeaOeTnavIekUwXVRXT2+NsqwYujCCcjLZudx2f+N9VVO5v5DvXzOGDp09UQ7SIAOpJndEWr9zJ//vtOooH5fHrT5zJKRM0JLeIHKUEkYEON7dy++/+wYMv7uSc6cX86PpTKRqUF3VYItLPKEFkmF21DXzqgdWs2VXHzedP4wsXzyRbo6+KSBxKEBnkr2VVfPbBl2hpdRZ+6DQuOXFM1CGJSD+mBJEBYjHn3uVbuPupTRw/agg/+dBpTCkeFHVYItLPKUGkubpDzXxx8css3VDJVXOP49+uPYmCPP2zi0jX9E2Rxsqr6vno/SvYXXuI2688kQ+fOUm3sIpIwpQg0lRLa4zPPfwy+w818/DHz+C0SUVRhyQiKUYJIk3d/7dtrNlVx48/OE/JQUS6pc8nDJLk21HdwN1PbeKiE0Zz+Um6U0lEukcJIs24O//627XkZGXx7atPVJuDiHSbEkSaeXT1bv5atpevXjqTscM0JaiIdF+HbRBmthbocEIedz85KRFJt+2tb+Tbf1jPaZMKuWH+pKjDEZEU11kj9bvC55vD51+FzzcADUmLSLrtW79bT0NjK3deexJZGj5DRHqowwTh7tsBzOxsdz+73Uu3mtnzwLeSHZwk7umNlTzxymt8/qLjmTF6SNThiEgaSKQNYpCZndO2YmZnARqnoR+pb2zhXx9by/GjB/PJ86ZFHY6IpIlE+kHcCPzczIYRtEnUAR9LalTyltz9503s2X+YRz54Fnk5uu9ARHpHlwnC3VcBp5jZUMDcvS75YUmiVu+oZdHft7HgzMmcNqkw6nBEJI10+XPTzEab2X3Aw+5eZ2azw3mjJWJNLTFu/c0axg7N50vvnBl1OCKSZhKpj7gf+DNwXLj+KvC5ZAUkifvJ8i28WlHPHdfMYfAAjZoiIr0rkQRR7O6LgRiAu7cArUmNSrq0ufIA//2XzVx5ynFcMGt01OGISBpKJEEcNLMRhJ3mzOwMgobqbjOzz5vZP8xsnZk9aGb5ZjbFzErNrMzMHjYzTZLcgVjMufU3aykYkM1t754ddTgikqYSSRBfAJ4ApoX9H34JfKa7BZrZOOCzQIm7zwGygQ8AdwHfd/cZQC3B3VMSxwMv7mDl9lq+fsVsigcPiDocEUlTidzFtNrMzgVmAgZscvfmXih3oJk1AwXAHuAC4IPh64uAbwL39rCctLOn7hB3/Wkjb59RzLXzxkUdjoiksURbNk8HJof7zzMz3P2X3SnQ3Xeb2d3ADuAQ8BSwCtgXtm8A7ALifvuZ2U3ATQATJ07sTggpy935+m/X0RpzvnvNSRqpVUSSKpHbXH8F3A2cA7wtfJR0t0AzKwSuAqYQ3Bk1CLgszq5xBwp094XuXuLuJSNHjuxuGCnpj2tfZ+mGSr54yfFMKCqIOhwRSXOJXEGUALPdvcORXd+ii4Ct7l4FYGaPAmcBw80sJ7yKGA+81kvlpYV9DU1844l1nDx+GB85a3LU4YhIBkikkXod0JvTku0AzjCzAgvqSC4E1gNPA+8N91kAPN6LZaa87/5xA7UNzdx57cnkZGs4DRFJvkSuIIqB9Wb2ItDYttHdr+xOge5eamaPAKuBFuAlYCHwB+AhM7sj3HZfd94/HT2/eS+LV+7ik+dNY/ZxQ6MOR0QyRCIJ4pu9Xai7fwP4xjGbywkaw6WdxpZWvvbYWiaPKOCWC2dEHY6IZJBEbnNd3heBSHwPvLCD7dUNLPrY6eTnZkcdjohkkM6mHH3O3c8xswO88Y4iA9zdVdeRZPsPN/NffynjnOnFnHt8Zt2xJSLR62xGuXPCZ01PFpGFy8upbWjmq5fOijoUEclACQ8BamajgPy2dXffkZSIBICK/Yf52XPlXHnKcZw0fljU4YhIBkqko9yVZlYGbAWWA9uAPyU5roz3g6VltMacL12ieR5EJBqJ3FD/beAM4FV3n0LQb+H5pEaV4TZX1rN45U5umD+JiSPUY1pEopFIgmh292ogy8yy3P1pYG6S48po3/vzRgbmZvOZC6ZHHYqIZLBE2iD2mdlg4FngATOrJOjgJkmwansNf/5HBV+8+HhGaChvEYlQIlcQVxGMuvp54ElgC/DuZAaVqdydO/+0keLBA7jx7VOiDkdEMlwiHeUOtltdlMRYMt7SDZWs2FbLHVfPoSBPc0yLSLQ66ygXt4Mc6iiXFC2tMf79yY1MLR7E+982IepwREQ67SinDnJ96Derd1FWWc+9N8wjV6O1ikg/kFA9hpnNI5gwyIHn3P2lpEaVYQ41tfL9JWXMnTCcS+f05sjqIiLdl0hHudsI2h5GEAz9fb+Z/b9kB5ZJfvG3rby+/zD/ctksTSMqIv1GIlcQ1wOnuvthADO7k2AuhzuSGVimqD3YxL3PbOHCWaOYP3VE1OGIiByRSGX3NtqNwQQMILjVVXrBj5/ezMHGFr6iAflEpJ9J5AqiEfiHmS0haIO4GHjOzH4E4O6fTWJ8aW1XbQO//Pt23jNvPDPH6J4AEelfEkkQj4WPNs8kJ5TM859PvYoZfP7i46MORUTkTRJJEH9y98r2G8xsprtvSlJMGWH9a/t57OXd3PSOqRw3fGDU4YiIvEkibRB/NbP3ta2Y2Rd54xWFdMNdT25kaH4unzpXA/KJSP+USII4D/iQmf3azJ4FjgdO726BZjbTzF5u99hvZp8zsyIzW2JmZeFzYXfL6O/+tnkvy1+t4ubzpzGsIDfqcERE4uoyQbj7HoJB+s4EJgO/dPf67hbo7pvcfa67zwVOAxoIrkhuBZa5+wxgWbiedmIx584nN3LcsHw+fObkqMMREelQIh3llgDzgTnA5cD3zezuXir/QmCLu28nGDW2bTDARcDVvVRGv/LHdXtYs6uOL1wyk/zc7KjDERHpUCJVTD929w+7+z53XwecBdT1UvkfAB4Ml0eHVyttVy2jeqmMfqOpJcb3/ryJWWOGcM2p46IOR0SkU4lUMf3WzCaZ2UXhplzgBz0t2MzygCuBX7/F424ys5VmtrKqqqqnYfSph1bsYHt1A1+9dBbZWRpSQ0T6t0SqmP4ZeAT4n3DTeOC3vVD2ZcBqd68I1yvMbGxY5ligMt5B7r7Q3UvcvWTkyJG9EEbfqG9s4YdLy5g/pYjzZqZO3CKSuRKpYroZOBvYD+DuZfRO9c/1HK1eAngCWBAuLwAe74Uy+o0HS3dQfbCJf7n8BA3IJyIpIZEE0ejuTW0rZpbDGycSesvMrIBgyI5H222+E7jYzMrC1+7sSRn9zZ/W7WHOuKHMnTA86lBERBKSSE/q5Wb2NWCgmV0MfAr4XU8KdfcGguHD22+rJrirKe1UHWjkpZ37uOXCGVGHIiKSsESuIG4FqoC1wMeBPwKaD+IteHpjJe5w8ezRUYciIpKwLq8g3D0G/DR8SDc8tb6CccMHMnuspvEWkdShyY+T7FBTK89truKiE0apcVpEUooSRJI9v3kvh5tjXKTqJRFJMQknCDMblMxA0tWS9RUMGZDD/CmaTlREUksiHeXOMrP1wIZw/RQzuyfpkaWBWMxZtrGCc2eOJC9HF2sikloS+db6PvBOoBrA3V8B3pHMoNLFSzv3sbe+SXcviUhKSuhnrbvvPGZTaxJiSTtLN1SQnWWcd3zajTsoIhkgkY5yO83sLMDDAfY+S1jdJJ1bur6C+VOKNCmQiKSkRK4gPkEwHtM4YBcwN1yXTmzbe5CyynouOkHVSyKSmhLpKLcXuKEPYkkrSzcEg9Sq/UFEUlUidzEtMrPh7dYLzeznyQ0r9T21voJZY4Ywoagg6lBERLolkSqmk919X9uKu9cCpyYvpNRXe7CJldtqVL0kIiktkQSRZWaFbStmVkRijdsZ6+lNlcQ0OJ+IpLhEvuj/A/ibmT0Srl8HfCd5IaW+JesrGDVkACeNGxZ1KCIi3ZZII/UvzWwVcD5gwLXuvj7pkaWow82tLH+1iqtPHUeW5p0WkRSWaFXRRqC2bX8zm+juO5IWVQp7obyahqZWLlb7g4ikuC4ThJl9BvgGUEHQg9oIphw9ObmhpaYl6ysoyMvmzGkanE9EUlsiVxC3ADPDKUGlE+7O0g0VvGPGSPJzs6MOR0SkRxK5i2knUJfsQNLB2t11VOxv1NwPIpIWErmCKAeeMbM/AI1tG939P5MWVYpaur6CLIMLZmlwPhFJfYlcQewAlgB5wJB2j24zs+Fm9oiZbTSzDWZ2ppkVmdkSMysLnwu7fqf+ZcmGSkomFVE0KC/qUEREeiyR21xvT0K5PwSedPf3hiPEFgBfA5a5+51mditwK/DVJJSdFDtrGtiwZz9fu3xW1KGIiPSKRO5iGgl8BTgRyG/b7u4XdKdAMxtKMOHQR8L3aQKazOwq4Lxwt0XAM6RQglh2ZHC+MRFHIiLSOxKpYnqAoB/EFOB2YBuwogdlTgWqgF+Y2Utm9rNwvuvR7r4HIHxOqYr8JRsqmDZyEFOKNXW3iKSHRBLECHe/D2h29+Xu/jHgjB6UmQPMA+5191OBgwTVSQkxs5vMbKWZrayqqupBGL2n7lAzpeU1untJRNJKIgmiOXzeY2ZXmNmpwPgelLkL2OXupeH6IwQJo8LMxgKEz5XxDnb3he5e4u4lI0eO7EEYvWf5q1W0xJxLlCBEJI0kkiDuMLNhwBeBLwE/Az7f3QLd/XWCaUxnhpsuBNYDTwALwm0LgMe7W0ZfW7K+ghGD8pg7IeVuvBIR6VAidzH9PlysIxiwrzd8BnggvIOpHPgoQbJabGY3Etxae10vlZVUTS0xntlUyWVzxpCtwflEJI10mCDM7Cvu/u9m9l8EYy+9gbt/truFuvvLQEmcly7s7ntGZcW2Gg4cbtHkQCKSdjq7gtgQPq/si0BS1ZL1FQzIyeKcGcVRhyIi0qs6TBDu/jszywbmuPuX+zCmlOHuLFlfwdtnFFOQp0n2RCS9dNpI7e6twGl9FEvK2bDnALv3HVL1koikpUR+9r5kZk8AvyboswCAuz+atKhSxNINFZjBBSekVJ8+EZGEJJIgioBqoP3QGg4oQWyoYO6E4Ywakt/1ziIiKSaR21w/2heBpJrX6w6zZlcdX37nzK53FhFJQYkM1pcP3MibB+v7WBLj6veWhoPzqfe0iKSrRHpS/woYA7wTWE4wzMaBZAaVCpasr2DSiAKmjxocdSgiIkmRSIKY7u5fBw66+yLgCuCk5IbVv9U3tvD3LdVcdMJozNR7WkTS01sZrG+fmc0BhgGTkxZRCvjrq1U0tca4WNVLIpLGErmLaWE4/efXCQbUGxwuZ6wl6ysYNjCXkkkanE9E0lciCeIXYYe55QST/WS0ltYYf9lUyQWzRpGTncgFmIhIakrkG26rmS00swtNFe6s2l7LvoZmVS+JSNpLJEHMBJYCNwPbzOy/zeyc5IbVfy1ZX0FedhbvOL5/TFYkIpIsXSYIdz/k7ovd/VpgLjCUoLopIy3dUMGZ00YweIAG5xOR9JZQJbqZnWtm9wCrCTrLvS+pUfVTu2ob2FbdwLm6ehCRDJBIT+qtwMvAYuDL7n6wi0PSVml5DQBnTB0RcSQiIsmXSD3JKe6+P+mRpIDSrdUMG5jLrDFDog5FRCTpEmmDUHIIlW6t4W2Ti8jS3NMikgF0I3+CXq87zPbqBuZPKYo6FBGRPqEEkaDSrdUAzJ+qBCEimSHhBGFmZ5jZX8zseTO7uieFmtk2M1trZi+b2cpwW5GZLTGzsvC5X41jUbq1hsEDcpg9dmjUoYiI9IkOE4SZjTlm0xeAK4FLgW/3Qtnnu/tcdy8J128Flrn7DGBZuN5vlJZXUzK5UMNriEjG6Ozb7idm9vVwwiCAfcAHgfcDyWi4vgpYFC4vAnp0ldKbqg40sqXqIPOn6PZWEckcHSYId7+aoP/D783sQ8DngBhQQM+/vB14ysxWmdlN4bbR7r4nLHsPMCregWZ2k5mtNLOVVVVVPQwjMS9uDfo/qP1BRDJJp/Ul7v47gpnkhgOPApvc/Ufu3tNv5rPdfR5wGXCzmb0j0QPdfaG7l7h7yciRfdOjuXRrNQV52Zw0bliflCci0h901gZxpZk9B/wFWAd8ALjGzB40s2k9KdTdXwufK4HHgNOBCjMbG5Y9FqjsSRm9qbS8htMmFZKr9gcRySCdfePdQXD18B7gLnff5+5fAG4DvtPdAs1skJkNaVsGLiFIQE8AC8LdFgCPd7eM3lRzsIlNFQfU/0FEMk5nQ23UEVw1DKTdr3l3Lwu3d9do4LFwaokc4P/c/UkzWwEsNrMbgR3AdT0oo9ccbX9QA7WIZJbOEsQ1wPUEc1J/sLcKdPdy4JQ426uBC3urnN5SurWaATlZnDxe7Q8iklk6TBDuvhf4rz6MpV8qLa9h3sRCBuRkRx2KiEifUqtrJ+oamtnw+n7d3ioiGUkJohMrttXgjjrIiUhGUoLoROnWavKyszh14vCoQxER6XNKEJ0o3VrD3AnDyc9V+4OIZB4liA4cONzMut11an8QkYylBNGBVdtrian9QUQymBJEB0q31pCTZcybpPYHEclMShAdKC2v5qTxwyjI66wvoYhI+lKCiKOhqYU1u+pUvSQiGU0JIo7V2/fREnM1UItIRlOCiKN0azVZBiWT+tW02CIifUoJIo7S8hrmjBvGkPzcqEMREYmMEsQxDje38vLOfZr/QUQynhLEMV7asY+m1pgaqEUk4ylBHKN0azVm8DZdQYhIhlOCOEZpeQ0njBnKsIFqfxCRzKYE0U5jSyurd9Tq9lYREZQg3mDNrjoaW9T+ICICShBvUFpeDcDpan8QEVGCaK90aw0zRw+haFBe1KGIiEQusgRhZtlm9pKZ/T5cn2JmpWZWZmYPm1mffks3t8ZYtV3tDyIibaK8grgF2NBu/S7g++4+A6gFbuzLYNbtrqOhqVXtDyIioUgShJmNB64AfhauG3AB8Ei4yyLg6r6MqXRrDaD2BxGRNlFdQfwA+AoQC9dHAPvcvSVc3wWMi3egmd1kZivNbGVVVVWvBVRaXs20kYMYOWRAr72niEgq6/MEYWbvAirdfVX7zXF29XjHu/tCdy9x95KRI0f2SkytMWfltlrmT1X1kohImyimSzsbuNLMLgfygaEEVxTDzSwnvIoYD7zWVwGtf20/BxpbNECfiEg7fX4F4e7/4u7j3X0y8AHgL+5+A/A08N5wtwXA430VU+nWoP+DGqhFRI7qT/0gvgp8wcw2E7RJ3NdXBb9QXsOkEQWMGZbfV0WKiPR7UVQxHeHuzwDPhMvlwOl9HUMs5qzYVsM7Txzd10WLiPRr/ekKIhIbXz9A3aFmVS+JiBwj4xPEkfYH9aAWEXkDJYjyGsYNH8j4woKoQxER6VcyOkG4Oy9uq9HVg4hIHBmdIMoq66k52MQZan8QEXmTjE4QbfM/6ApCROTNMjpBvLC1hjFD85lYpPYHEZFjZWyCcHdKy4P2h2AwWRERaS9jE0T53oPsrW9U/wcRkQ5kbIJ4MZz/Qe0PIiLxZWyCKC2vpnjwAKYWD4o6FBGRfikjE4S7U7pV7Q8iIp3JyASxs+YQe+oOc4bmfxAR6VBGJogXjoy/pAZqEZGOZGSCKCzI4+LZo5kxanDUoYiI9FuRzgcRlYtnj+bi2Zr/QUSkMxl5BSEiIl1TghARkbiUIEREJC4lCBERiavPE4SZ5ZvZi2b2ipn9w8xuD7dPMbNSMyszs4fNLK+vYxMRkaOiuIJoBC5w91OAucClZnYGcBfwfXefAdQCN0YQm4iIhPo8QXigPlzNDR8OXAA8Em5fBFzd17GJiMhRkbRBmFm2mb0MVAJLgC3APndvCXfZBYyLIjYREQlE0lHO3VuBuWY2HHgMOCHebvGONbObgJvdKJ7bAAAH30lEQVTC1Xozqwb2JiXQ9FCMzk9XdI46p/PTtVQ7R5MS2SnSntTuvs/MngHOAIabWU54FTEeeK2DYxYCC9vWzWylu5f0RbypSOenazpHndP56Vq6nqMo7mIaGV45YGYDgYuADcDTwHvD3RYAj/d1bCIiclQUVxBjgUVmlk2QoBa7++/NbD3wkJndAbwE3BdBbCIiEurzBOHua4BT42wvB07vxlsu7HqXjKbz0zWdo87p/HQtLc+RucdtCxYRkQynoTZERCSulE0QZnapmW0ys81mdmvU8fRHZrbNzNaa2ctmtjLqePoDM/u5mVWa2bp224rMbEk4zMsSMyuMMsYodXB+vmlmu8PP0ctmdnmUMUbJzCaY2dNmtiEcKuiWcHtafoZSMkGEDdw/Bi4DZgPXm9nsaKPqt85397npeAteN90PXHrMtluBZeEwL8vC9Ux1P28+PxAMgzM3fPyxj2PqT1qAL7r7CQS3598cfvek5WcoJRMEQWP2Zncvd/cm4CHgqohjkhTg7s8CNcdsvopgeBfI8GFeOjg/EnL3Pe6+Olw+QHCL/jjS9DOUqgliHLCz3bqG5ojPgafMbFXYA13iG+3ueyD4AgBGRRxPf/RpM1sTVkGlRfVJT5nZZII7MktJ089QqiYIi7NNt2O92dnuPo+gKu5mM3tH1AFJSroXmEYw+vIe4D+iDSd6ZjYY+A3wOXffH3U8yZKqCWIXMKHdeodDc2Qyd38tfK4kGPOqO/1MMkGFmY0FCJ8rI46nX3H3CndvdfcY8FMy/HNkZrkEyeEBd3803JyWn6FUTRArgBnhJEN5wAeAJyKOqV8xs0FmNqRtGbgEWNf5URnrCYLhXUDDvLxJ2xdf6Boy+HNkZkYwysMGd//Pdi+l5WcoZTvKhbfa/QDIBn7u7t+JOKR+xcymElw1QNBj/v90jsDMHgTOIxh9swL4BvBbYDEwEdgBXOfuGdlQ28H5OY+gesmBbcDH2+rbM42ZnQP8FVgLxMLNXyNoh0i7z1DKJggREUmuVK1iEhGRJFOCEBGRuJQgREQkLiUIERGJSwlCRETiUoKQPmVm3zKzi7p57B/bpqvtxrH3m9l7u94ztZnZeWZ2VtRxtGdmX4s6BukeJQjpM2aW7e63ufvS7hzv7pe7+77ejivNnAf0qwRB0E9AUpAShPSYmU02s41mtigc0O0RMysIX9tmZreZ2XPAde1/yYev3W5mq8N5K2aF2web2S/CbWvM7D3t9i/uorzbzGyFma0zs4Vhz9fOYp9uZkvN7JUwjmkW+F74HmvN7P3hvueZ2XIzW2xmr5rZnWZ2g5m9GO43LdzvfjP7iZn9NdzvXeH2/HZ/10tmdn64/SNm9qiZPRnOJ/Dv7eK7xMz+Hsb263AMoLjnLhw87hPA5y2Yt+HtZnZd+He8YmbPdnAOvhK+xytmdme4ba6ZvRCe38csHKDPzJ4xs5JwudjMtnX2N4TvNzCM54G38LGS/sDd9dCjRw9gMkEv27PD9Z8DXwqXtwFfabfv/cB72732mXD5U8DPwuW7gB+0O6aw3f7FXZRX1O64XwHvPrbcY2IvBa4Jl/OBAuA9wBKCXvqjCXrGjiX4db4vXB4A7AZuD4+9pS3msKwnCX6AzSAYOywf+CLwi3CfWeH75gMfAcqBYeH6doKxxoqBZ4FB4TFfBW7r4tx9s+1chOtrgXHh8vA4f/9lwN+AgvbnD1gDnBsuf6vd3/YMUBIuFwPbwuW4f0P4Wn3Un1E9uvfQFYT0lp3u/ny4/L/AOe1ee7iT49oGO1tF8MUPcBHBhFAAuHvtWyjvfDMrNbO1wAXAiR0VbMFYVePc/bGwnMPu3hC+14MeDFBXASwH3hYetsKDOQEagS3AU+H2te3iB1js7jF3LyP44pwVvu+vwrI2EnyJHh/uv8zd69z9MLAemEQwIc1s4Hkze5lgjJ9J7cqId+6O9Txwv5n9M0HCO9ZFBEmrIYyrxsyGESST5eE+i4BERgKO9zdICsuJOgBJG8eO2dJ+/WAnxzWGz60c/TxanPfrsjwzywfuIfiFu9PMvknwa7YjHVU/dVYt1dhuOdZuPcYb/z/FOx+Jvm/buTBgibtf38Ux7c/dGwt1/4SZzQeuAF42s7nuXt1ul0TOdXstHK2aPvbcxvsbJIXpCkJ6y0QzOzNcvh54rgfv9RTw6bYViz9BTbzy2r6w9oZ19Z3eteTBOP67zOzqsJwBYVvGs8D7zSzbzEYS/Hp+8S3+DdeZWVbYLjEV2BS+7w1hWccTDOy2qZP3eAE428ymh8cUhMd15gAwpG3FzKa5e6m73wbs5Y3D5ENwrj/Wrg2nyN3rgFoze3u4z4cIrqIgqNo6LVxO9K6wZguGyJYUowQhvWUDsMDM1gBFBJPMdNcdQGFb4ypwfiLleXCH008Jqnt+SzAsfFc+BHw2fJ+/AWMIRsFdA7wC/IWgDeX1t/g3bCL4Uv0T8Imw2uUeIDus/noY+EhYVRWXu1cR1O0/GMb3AkFVVWd+B1zT1kgNfC9sgF5HkKBeOaaMJwmGql4ZVmN9KXxpQXjsGoKRXL8Vbr8b+KSZ/Y2gDSIRC4E1aqROPRrNVXosvHvm9+4+Jx3Le6vM7H6C+B6JOhaRntAVhIiIxKUrCBERiUtXECIiEpcShIiIxKUEISIicSlBiIhIXEoQIiISlxKEiIjE9f8By+Nl6ZnfF5AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get principle components\n",
    "pca = PCA()\n",
    "X_reduced = pca.fit_transform(scale(X))\n",
    "\n",
    "# What variance (%c umulative) is explained by principle components?\n",
    "variance_explained   = np.cumsum(np.around(pca.explained_variance_ratio_ *100, decimals=1))\n",
    "principal_components = np.arange(1, len(variance_explained)+1)\n",
    "\n",
    "sns.lineplot(x='principal components count', y='% variance explained', \n",
    "             data=pd.DataFrame({'principal components count': principal_components, \n",
    "                                '% variance explained': variance_explained}));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(263, 22)"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_reduced = pca.fit_transform(scale(X))\n",
    "X_reduced.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCR \n",
    "\n",
    "How does linear regression perform when these principle components are used as predictors?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59346fa9cefb4850aa082d90ee583ecb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=10, description='seed', max=50, min=1), IntSlider(value=10, description=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def pcr_cv(seed, cv_folds):\n",
    "    \"\"\"Perform Principle Component Regression evaluated with\n",
    "    k-fold cross validation\"\"\"\n",
    "    # Load data\n",
    "    X = hitters_pd.drop('Salary', axis=1)\n",
    "    y = hitters_pd['Salary']\n",
    "    \n",
    "    # Get all principle components\n",
    "    pca = PCA()\n",
    "    X_reduced = pca.fit_transform(scale(X))\n",
    "    \n",
    "    # Get cv MSE for cumulative components\n",
    "    M = X_reduced.shape[1]\n",
    "    MSEs = []\n",
    "    for m in range(M):\n",
    "        model = LinearRegression()\n",
    "        cv    = KFold(n_splits=cv_folds, random_state=seed, shuffle=True)\n",
    "        cv10  = cross_val_score(model, X_reduced[:, 0:m+1], y, cv=cv, scoring='neg_mean_squared_error')\n",
    "        MSEs += [np.mean(np.abs(cv10))]\n",
    "        \n",
    "    ax = sns.lineplot(x='principal components count', y='MSE', \n",
    "                 data=pd.DataFrame({'principal components count': principal_components, \n",
    "                                    'MSE': MSEs}));\n",
    "    ax.axes.set_ylim(100000, 140000)\n",
    "\n",
    "\n",
    "interact(pcr_cv, \n",
    "         seed=widgets.IntSlider(min=1, max=50, step=1, value=10), \n",
    "         cv_folds=widgets.IntSlider(min=2, max=100, step=1, value=10));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The ISL authors report a minimum MSE when the number of principle componenents M = 16.  \n",
    "\n",
    "Above I've implemented the same with interactive sliders to control the random seed used to shuffle observations before splitting into cross validation folds, and a seperate control for number of cross validation folds k. \n",
    "\n",
    "Interacting with the seed slider we see that there is some significant variability in the results. With 10 folds, most seed values would lead us to conclude the between 16-18 principal components are optimal, but there are some seed values – such as 8 – that suggest 6-7 components is in fact optimal. This is a significantly different conclusion!\n",
    "\n",
    "The results seem to have higher variance when cv_folds is lower, with variance decreasing as the number of folds increases.\n",
    "\n",
    "**Revision note:** The above point seems counter to what we learned in ch.5, e.g. we expect LOOCV to exhibit highest variance and lowest bias."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6d188a465184df789f38ac12b3c128e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=10, description='seed', max=50, min=1), Output()), _dom_classes=('widget…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def pcr_holdout(seed):\n",
    "    \"\"\"Perform Principle Component Regression evaluated with\n",
    "    k-fold hold-out set\"\"\"\n",
    "    # With 50% holdout set\n",
    "    np.random.seed(seed)\n",
    "    train = np.random.rand(len(hitters_pd)) < 0.5\n",
    "    \n",
    "    pca = PCA()\n",
    "    X_train = pca.fit_transform(scale(hitters_pd.drop('Salary', axis=1)[train]))\n",
    "    y_train = hitters_pd['Salary'][train]\n",
    "    X_test  = pca.fit_transform(scale(hitters_pd.drop('Salary', axis=1)[~train]))\n",
    "    y_test  = hitters_pd['Salary'][~train]\n",
    "    \n",
    "    MSEs = []\n",
    "    for m in range(M):\n",
    "        model = LinearRegression().fit(X_train[:, 0:m+1], y_train)\n",
    "        y_hat = model.predict(X_test[:, 0:m+1])\n",
    "        MSEs += [mse(y_hat, y_test)]\n",
    "    \n",
    "    # Plot results\n",
    "    ax = sns.lineplot(x='principal components count', y='MSE', \n",
    "                      data=pd.DataFrame({'principal components count': principal_components, \n",
    "                                         'MSE': MSEs}))\n",
    "    ax.axes.set_ylim(50000, 200000);\n",
    "\n",
    "interact(pcr_holdout, seed=widgets.IntSlider(min=1, max=50, step=1, value=10));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The ISL authors report an 50% hold-out MSE of 96556 when M=7. This seems low in comparison to the initial result we observe above, with a random seed of 10 we see min MSE of about 140000 with M=1. \n",
    "\n",
    "Interacting with the seed slider we can find results comparable to those reported by the authors – for example when the seed = 3 a minimum MSE of about 85000 is observed for M=7. \n",
    "\n",
    "This outlines the importance of understanding the variance in results due to training/test set sampling, before making any comparisons.\n",
    "\n",
    "For example previously I concluded that backward stepwise selection yielded the lowest MSE, whereas the ISL found ridge regression most effective. It seems that either of these observations could be correct, or both wrong, given the unquantified variance in results due to cross validation sampling.\n",
    "\n",
    "Possible mitigations:\n",
    "\n",
    "1. Ensure that identical observation samples are used in each fold when comparing techniques. This approach is limited because a certain approach might just happen to perform particularly well with the chosen samples.\n",
    "2. Measure the probabilistic distribution of results for each approach and compare. This allows you to quantify the confidence of any conclusions.\n",
    "\n",
    "**Revision notes:** discuss\n",
    "\n",
    "#### PLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b9f788b2405448459a828a4a2e82b3e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=10, description='seed', max=50, min=1), IntSlider(value=10, description=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.cross_decomposition import PLSRegression\n",
    "\n",
    "def pls_cv(seed, cv_folds):\n",
    "    \"\"\"Perform Partial Least Squares Regression evaluated with\n",
    "    k-fold cross validation\"\"\"\n",
    "    # Load data\n",
    "    X = hitters_pd.drop('Salary', axis=1)\n",
    "    y = hitters_pd['Salary']\n",
    "    \n",
    "    # Get cv MSE for cumulative components\n",
    "    M = X.shape[1]\n",
    "    MSEs = []\n",
    "    for m in range(M):\n",
    "        cv      = KFold(n_splits=cv_folds, random_state=seed, shuffle=True)\n",
    "        results = cross_val_score(PLSRegression(n_components=m+1, scale=True, max_iter=10), \n",
    "                                  X, y, cv=cv, scoring='neg_mean_squared_error')\n",
    "        MSEs   += [np.mean(np.abs(results))]\n",
    "        \n",
    "    ax = sns.lineplot(x='principal components count', y='MSE', \n",
    "                 data=pd.DataFrame({'principal components count': principal_components, \n",
    "                                    'MSE': MSEs}));\n",
    "    ax.axes.set_ylim(100000, 140000)\n",
    "\n",
    "\n",
    "interact(pls_cv, \n",
    "         seed=widgets.IntSlider(min=1, max=50, step=1, value=10), \n",
    "         cv_folds=widgets.IntSlider(min=2, max=100, step=1, value=10));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PLS seems to perform similarly to PCR with fewer predictors, although we cannot be conclusive due to variance in our cv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}