Excercise F.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# F Linear Regression BOSTON\n",
    "\n",
    "Train a linear regression model on the BOSTON dataset to predict median\n",
    "values of houses. Test it on the test set. Show the code how you did it! What\n",
    "dependent variables (columns) have the biggest influence?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "plt.rcParams['figure.figsize'] = [15, 8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>crim</th>\n",
       "      <th>zn</th>\n",
       "      <th>indus</th>\n",
       "      <th>chas</th>\n",
       "      <th>nox</th>\n",
       "      <th>rm</th>\n",
       "      <th>age</th>\n",
       "      <th>dis</th>\n",
       "      <th>rad</th>\n",
       "      <th>tax</th>\n",
       "      <th>ptratio</th>\n",
       "      <th>black</th>\n",
       "      <th>lstat</th>\n",
       "      <th>medv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.23247</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.14</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5380</td>\n",
       "      <td>6.142</td>\n",
       "      <td>91.7</td>\n",
       "      <td>3.9769</td>\n",
       "      <td>4.0</td>\n",
       "      <td>307.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>18.72</td>\n",
       "      <td>15.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.02177</td>\n",
       "      <td>82.5</td>\n",
       "      <td>2.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4150</td>\n",
       "      <td>7.610</td>\n",
       "      <td>15.7</td>\n",
       "      <td>6.2700</td>\n",
       "      <td>2.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>14.7</td>\n",
       "      <td>395.38</td>\n",
       "      <td>3.11</td>\n",
       "      <td>42.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.89822</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6310</td>\n",
       "      <td>4.970</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1.3325</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>375.52</td>\n",
       "      <td>3.26</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.03961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.19</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5150</td>\n",
       "      <td>6.037</td>\n",
       "      <td>34.5</td>\n",
       "      <td>5.9853</td>\n",
       "      <td>5.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>396.90</td>\n",
       "      <td>8.01</td>\n",
       "      <td>21.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.69311</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.7130</td>\n",
       "      <td>6.376</td>\n",
       "      <td>88.4</td>\n",
       "      <td>2.5671</td>\n",
       "      <td>24.0</td>\n",
       "      <td>666.0</td>\n",
       "      <td>20.2</td>\n",
       "      <td>391.43</td>\n",
       "      <td>14.65</td>\n",
       "      <td>17.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>399</td>\n",
       "      <td>0.21977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.91</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4480</td>\n",
       "      <td>5.602</td>\n",
       "      <td>62.0</td>\n",
       "      <td>6.0877</td>\n",
       "      <td>3.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>17.9</td>\n",
       "      <td>396.90</td>\n",
       "      <td>16.20</td>\n",
       "      <td>19.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.16211</td>\n",
       "      <td>20.0</td>\n",
       "      <td>6.96</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4640</td>\n",
       "      <td>6.240</td>\n",
       "      <td>16.3</td>\n",
       "      <td>4.4290</td>\n",
       "      <td>3.0</td>\n",
       "      <td>223.0</td>\n",
       "      <td>18.6</td>\n",
       "      <td>396.90</td>\n",
       "      <td>6.59</td>\n",
       "      <td>25.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>401</td>\n",
       "      <td>0.03466</td>\n",
       "      <td>35.0</td>\n",
       "      <td>6.06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4379</td>\n",
       "      <td>6.031</td>\n",
       "      <td>23.3</td>\n",
       "      <td>6.6407</td>\n",
       "      <td>1.0</td>\n",
       "      <td>304.0</td>\n",
       "      <td>16.9</td>\n",
       "      <td>362.25</td>\n",
       "      <td>7.83</td>\n",
       "      <td>19.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>402</td>\n",
       "      <td>2.14918</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.58</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.8710</td>\n",
       "      <td>5.709</td>\n",
       "      <td>98.5</td>\n",
       "      <td>1.6232</td>\n",
       "      <td>5.0</td>\n",
       "      <td>403.0</td>\n",
       "      <td>14.7</td>\n",
       "      <td>261.95</td>\n",
       "      <td>15.79</td>\n",
       "      <td>19.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>403</td>\n",
       "      <td>0.01439</td>\n",
       "      <td>60.0</td>\n",
       "      <td>2.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.4010</td>\n",
       "      <td>6.604</td>\n",
       "      <td>18.8</td>\n",
       "      <td>6.2196</td>\n",
       "      <td>1.0</td>\n",
       "      <td>265.0</td>\n",
       "      <td>15.6</td>\n",
       "      <td>376.70</td>\n",
       "      <td>4.38</td>\n",
       "      <td>29.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>404 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        crim    zn  indus  chas     nox     rm    age     dis   rad    tax  \\\n",
       "0    1.23247   0.0   8.14   0.0  0.5380  6.142   91.7  3.9769   4.0  307.0   \n",
       "1    0.02177  82.5   2.03   0.0  0.4150  7.610   15.7  6.2700   2.0  348.0   \n",
       "2    4.89822   0.0  18.10   0.0  0.6310  4.970  100.0  1.3325  24.0  666.0   \n",
       "3    0.03961   0.0   5.19   0.0  0.5150  6.037   34.5  5.9853   5.0  224.0   \n",
       "4    3.69311   0.0  18.10   0.0  0.7130  6.376   88.4  2.5671  24.0  666.0   \n",
       "..       ...   ...    ...   ...     ...    ...    ...     ...   ...    ...   \n",
       "399  0.21977   0.0   6.91   0.0  0.4480  5.602   62.0  6.0877   3.0  233.0   \n",
       "400  0.16211  20.0   6.96   0.0  0.4640  6.240   16.3  4.4290   3.0  223.0   \n",
       "401  0.03466  35.0   6.06   0.0  0.4379  6.031   23.3  6.6407   1.0  304.0   \n",
       "402  2.14918   0.0  19.58   0.0  0.8710  5.709   98.5  1.6232   5.0  403.0   \n",
       "403  0.01439  60.0   2.93   0.0  0.4010  6.604   18.8  6.2196   1.0  265.0   \n",
       "\n",
       "     ptratio   black  lstat  medv  \n",
       "0       21.0  396.90  18.72  15.2  \n",
       "1       14.7  395.38   3.11  42.3  \n",
       "2       20.2  375.52   3.26  50.0  \n",
       "3       20.2  396.90   8.01  21.1  \n",
       "4       20.2  391.43  14.65  17.7  \n",
       "..       ...     ...    ...   ...  \n",
       "399     17.9  396.90  16.20  19.4  \n",
       "400     18.6  396.90   6.59  25.2  \n",
       "401     16.9  362.25   7.83  19.4  \n",
       "402     14.7  261.95  15.79  19.4  \n",
       "403     15.6  376.70   4.38  29.1  \n",
       "\n",
       "[404 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston = tf.keras.datasets.boston_housing.load_data()\n",
    "X_train = boston[0][0]\n",
    "Y_train = boston[0][1]\n",
    "\n",
    "X_test = boston[0][0]\n",
    "Y_test = boston[0][1]\n",
    "\n",
    "c = [\"crim\", \"zn\", \"indus\", \"chas\", \"nox\", \"rm\", \"age\", \"dis\", \"rad\", \"tax\", \"ptratio\", \"black\", \"lstat\", 'medv']\n",
    "boston = pd.DataFrame({c: X_train[:, i] for i, c in enumerate(c[:-1])})\n",
    "boston_test = boston\n",
    "boston[c[-1]] = Y_train\n",
    "#boston.drop(boston.columns[[1,2,3,4,7,8,9,10,12,13]], axis=1, inplace=True)\n",
    "boston"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)\n"
     ]
    }
   ],
   "source": [
    "regr = LinearRegression()\n",
    "print (regr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = regr.predict(X_test)\n",
    "\n",
    "#print (pred)\n",
    "#print (Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coefficients: \n",
      " [ -0.11999751   0.05700033   0.0039838    4.12698187 -20.50029633\n",
      "   3.38024903   0.00756808  -1.71189793   0.33474754  -0.01177972\n",
      "  -0.90231804   0.00871913  -0.55584251]\n",
      "Mean squared error: 22.00\n",
      "Variance score: 0.74\n"
     ]
    }
   ],
   "source": [
    "print('Coefficients: \\n', regr.coef_)\n",
    "# The mean squared error of the linear regression\n",
    "print(\"Mean squared error: %.2f\"\n",
    "      % mean_squared_error(Y_test, pred))\n",
    "# Varaince Score: PI for accuracy of regression: a 1 would be a perfect variance score\n",
    "print('Variance score: %.2f' % r2_score(Y_test, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our first intention was to compare coefficients for analysing the influence of the different variables. However we found this:\n",
    "\n",
    "Regular regression coefficients describe the relationship between each predictor variable and the response. The coefficient value represents the mean change in the response given a one-unit increase in the predictor. Consequently, it’s easy to think that variables with larger coefficients are more important because they represent a larger change in the response.\n",
    "\n",
    "However, the units vary between the different types of variables, which makes it impossible to compare them directly. For example, the meaning of a one-unit change is very different if you’re talking about temperature, weight, or chemical concentration.\n",
    "\n",
    "This problem is further complicated by the fact that there are different units within each type of measurement. For example, weight can be measured in grams and kilograms. If you fit models for the same data set using grams in one model and kilograms in another, the coefficient for weight changes by a factor of a thousand even though the underlying fit of the model remains unchanged. The coefficient value changes greatly while the importance of the variable remains constant.\n",
    "\n",
    "Takeaway: Larger coefficients don’t necessarily identify more important predictor variables.\n",
    "(https://blog.minitab.com/blog/adventures-in-statistics-2/how-to-identify-the-most-important-predictor-variables-in-regression-models)\n",
    "\n",
    "### So we are going to need the MSE of each variable in order to determine the one with the biggest influence!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "crim :\n",
      "Coefficients : [-0.37725684]\n",
      "MSE: 72.49923060928539\n",
      "\n",
      "\n",
      "zn :\n",
      "Coefficients : [0.14737305]\n",
      "MSE: 72.3835733788123\n",
      "\n",
      "\n",
      "indus :\n",
      "Coefficients : [-0.6446655]\n",
      "MSE: 65.38896540382329\n",
      "\n",
      "\n",
      "chas :\n",
      "Coefficients : [6.43943008]\n",
      "MSE: 82.21505727944826\n",
      "\n",
      "\n",
      "nox :\n",
      "Coefficients : [-34.41973083]\n",
      "MSE: 68.36363721633272\n",
      "\n",
      "\n",
      "rm :\n",
      "Coefficients : [8.84314508]\n",
      "MSE: 45.3221194654976\n",
      "\n",
      "\n",
      "age :\n",
      "Coefficients : [-0.12004703]\n",
      "MSE: 73.39948383390582\n",
      "\n",
      "\n",
      "dis :\n",
      "Coefficients : [1.15186574]\n",
      "MSE: 79.16704438078895\n",
      "\n",
      "\n",
      "rad :\n",
      "Coefficients : [-0.39762194]\n",
      "MSE: 72.68954082972854\n",
      "\n",
      "\n",
      "tax :\n",
      "Coefficients : [-0.02484192]\n",
      "MSE: 67.58232478739882\n",
      "\n",
      "\n",
      "ptratio :\n",
      "Coefficients : [-2.06776036]\n",
      "MSE: 63.972246486587544\n",
      "\n",
      "\n",
      "black :\n",
      "Coefficients : [0.03366193]\n",
      "MSE: 74.61110841268861\n",
      "\n",
      "\n",
      "lstat :\n",
      "Coefficients : [-0.92782169]\n",
      "MSE: 39.42905636385886\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(0,13):\n",
    "    print(c[i],\":\")\n",
    "    reg = LinearRegression()\n",
    "    reg.fit(X_train[:, i].reshape(-1, 1), Y_train)\n",
    "    pred = reg.predict(X_test[:, i].reshape(-1, 1))\n",
    "    print('Coefficients : {}'.format(reg.coef_))\n",
    "    print(\"MSE: {}\".format(mean_squared_error(Y_test, pred)))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A lower MSE means a bigger influence of the variable, so looking at the data we can say that in this dataset \"rm\" has the biggest impact on property prices, followed by \"lstat\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}