PandasDataClean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>john</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>anne</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      0\n",
       "0  john\n",
       "1  anne\n",
       "2  mary"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "names = ['john','anne','mary']          # List\n",
    "df = pd.DataFrame(data=names)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Email</th>\n",
       "      <th>Course</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>john</td>\n",
       "      <td>john@mit.edu</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>anne</td>\n",
       "      <td>anne@mit.edu</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mary</td>\n",
       "      <td>mary@mit.edu</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Name         Email  Course\n",
       "1  john  john@mit.edu       1\n",
       "2  anne  anne@mit.edu       2\n",
       "3  mary  mary@mit.edu       3"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = ['john','anne','mary']          # List\n",
    "emails = ['john@mit.edu','anne@mit.edu','mary@mit.edu'] #dictionary\n",
    "courses = [1,2,3]\n",
    "contacts = {'Name': names,\n",
    "            'Email': emails,\n",
    "            'Course': courses}\n",
    "df = pd.DataFrame(data=contacts, index = [1, 2, 3])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Name              john\n",
       "Email     john@mit.edu\n",
       "Course               1\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# rows and columns are Series - we can operate on them \n",
    "# get rows or add a row\n",
    "df.loc[1]\n",
    "#df.iloc[1:4]\n",
    "#df.loc[4] = ['joe','joe@mit.edu',5]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get columns or add a column\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Email</th>\n",
       "      <th>Course</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>john</td>\n",
       "      <td>john@mit.edu</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>anne</td>\n",
       "      <td>anne@mit.edu</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mary</td>\n",
       "      <td>mary@mit.edu</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Name         Email  Course  Price\n",
       "1  john  john@mit.edu       1     10\n",
       "2  anne  anne@mit.edu       2     20\n",
       "3  mary  mary@mit.edu       3     30"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# operations on columns\n",
    "df['Price'] = df['Course']*10\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# operations on rows\n",
    "df.loc[5] = df.loc[1] + df.loc[2]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.417022</td>\n",
       "      <td>0.720324</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.302333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.146756</td>\n",
       "      <td>0.092339</td>\n",
       "      <td>0.186260</td>\n",
       "      <td>0.345561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.396767</td>\n",
       "      <td>0.538817</td>\n",
       "      <td>0.419195</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.204452</td>\n",
       "      <td>0.878117</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.670468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.417305</td>\n",
       "      <td>0.558690</td>\n",
       "      <td>0.140387</td>\n",
       "      <td>0.198101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B         C         D\n",
       "0  0.417022  0.720324  0.000114  0.302333\n",
       "1  0.146756  0.092339  0.186260  0.345561\n",
       "2  0.396767  0.538817  0.419195       NaN\n",
       "3  0.204452  0.878117       NaN  0.670468\n",
       "4  0.417305  0.558690  0.140387  0.198101"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "cols = list('ABCD')\n",
    "np.random.seed(seed=1)\n",
    "myArr = np.random.ranf(size=(5,4))\n",
    "myArr[2,3] = np.NaN\n",
    "myArr[3,2] = np.NaN\n",
    "df = pd.DataFrame(data=myArr, columns=cols)\n",
    "df.fillna(value = 0.0)\n",
    "# note df has not changed - the above statement is a \"view\"\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.417022</td>\n",
       "      <td>0.720324</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.302333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.146756</td>\n",
       "      <td>0.092339</td>\n",
       "      <td>0.186260</td>\n",
       "      <td>0.345561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.396767</td>\n",
       "      <td>0.538817</td>\n",
       "      <td>0.419195</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.204452</td>\n",
       "      <td>0.878117</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.670468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.417305</td>\n",
       "      <td>0.558690</td>\n",
       "      <td>0.140387</td>\n",
       "      <td>0.198101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B         C         D\n",
       "0  0.417022  0.720324  0.000114  0.302333\n",
       "1  0.146756  0.092339  0.186260  0.345561\n",
       "2  0.396767  0.538817  0.419195       NaN\n",
       "3  0.204452  0.878117       NaN  0.670468\n",
       "4  0.417305  0.558690  0.140387  0.198101"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Should we drop the whole Column or Row or Both - try inplace = True\n",
    "df.fillna(value = 0.0,inplace = False)\n",
    "#df.dropna(axis = 1,inplace = True)   # drops a row -  axis=1 drops a column\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          A         B         C         D\n",
      "0  0.417022  0.720324  0.000114  0.302333\n",
      "1  0.146756  0.092339  0.186260  0.345561\n",
      "2  0.396767  0.538817  0.419195       NaN\n",
      "3  0.204452  0.878117       NaN  0.670468\n",
      "4  0.417305  0.558690  0.140387  0.198101\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.417022</td>\n",
       "      <td>0.720324</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.302333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.146756</td>\n",
       "      <td>0.092339</td>\n",
       "      <td>0.186260</td>\n",
       "      <td>0.345561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.417305</td>\n",
       "      <td>0.558690</td>\n",
       "      <td>0.140387</td>\n",
       "      <td>0.198101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B         C         D\n",
       "0  0.417022  0.720324  0.000114  0.302333\n",
       "1  0.146756  0.092339  0.186260  0.345561\n",
       "4  0.417305  0.558690  0.140387  0.198101"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numeric = df.select_dtypes(include=[np.number])\n",
    "print(numeric)\n",
    "d = df.dropna(axis=0)\n",
    "d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley  \\\n",
       "0           0   1          60       RL         65.0     8450   Pave   NaN   \n",
       "1           1   2          20       RL         80.0     9600   Pave   NaN   \n",
       "2           2   3          60       RL         68.0    11250   Pave   NaN   \n",
       "3           3   4          70       RL         60.0     9550   Pave   NaN   \n",
       "4           4   5          60       RL         84.0    14260   Pave   NaN   \n",
       "\n",
       "  LotShape LandContour  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0      Reg         Lvl  ...        0    NaN   NaN         NaN       0      2   \n",
       "1      Reg         Lvl  ...        0    NaN   NaN         NaN       0      5   \n",
       "2      IR1         Lvl  ...        0    NaN   NaN         NaN       0      9   \n",
       "3      IR1         Lvl  ...        0    NaN   NaN         NaN       0      2   \n",
       "4      IR1         Lvl  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold SaleType  SaleCondition  SalePrice  \n",
       "0   2008       WD         Normal     208500  \n",
       "1   2007       WD         Normal     181500  \n",
       "2   2008       WD         Normal     223500  \n",
       "3   2006       WD        Abnorml     140000  \n",
       "4   2008       WD         Normal     250000  \n",
       "\n",
       "[5 rows x 82 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "data = pd.read_csv('houseSmallData.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In Class Exercise - list out the columns with the most NaN's (list top 5)\n",
    "data = pd.read_csv('testData.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0       0.169908\n",
       "Id               0.169908\n",
       "MSSubClass      -0.075538\n",
       "LotFrontage      0.348097\n",
       "LotArea          0.458530\n",
       "OverallQual      0.786003\n",
       "OverallCond     -0.048835\n",
       "YearBuilt        0.389623\n",
       "YearRemodAdd     0.459630\n",
       "MasVnrArea       0.374830\n",
       "BsmtFinSF1       0.504946\n",
       "BsmtFinSF2       0.012698\n",
       "BsmtUnfSF       -0.004520\n",
       "TotalBsmtSF      0.548819\n",
       "1stFlrSF         0.540889\n",
       "2ndFlrSF         0.401720\n",
       "LowQualFinSF     0.106511\n",
       "GrLivArea        0.744012\n",
       "BsmtFullBath     0.382605\n",
       "BsmtHalfBath    -0.018271\n",
       "FullBath         0.421878\n",
       "HalfBath         0.283732\n",
       "BedroomAbvGr     0.022901\n",
       "KitchenAbvGr    -0.148677\n",
       "TotRmsAbvGrd     0.600238\n",
       "Fireplaces       0.505403\n",
       "GarageYrBlt      0.395083\n",
       "GarageCars       0.662996\n",
       "GarageArea       0.694061\n",
       "WoodDeckSF       0.278904\n",
       "OpenPorchSF      0.426016\n",
       "EnclosedPorch   -0.098568\n",
       "3SsnPorch       -0.027394\n",
       "ScreenPorch      0.405428\n",
       "PoolArea         0.055527\n",
       "MiscVal         -0.114435\n",
       "MoSold           0.001486\n",
       "YrSold           0.106782\n",
       "SalePrice        1.000000\n",
       "Name: SalePrice, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# replace values of NaN with zeroes\n",
    "#df.fillna(value = 0.0)\n",
    "X=data.drop(['SalePrice'], axis = 1)\n",
    "numeric = data.select_dtypes(include=[np.number])\n",
    "numeric.head()\n",
    "data.corr()['SalePrice']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remember inplace !!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catagorical Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>apple</th>\n",
       "      <th>orange</th>\n",
       "      <th>pear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.417022</td>\n",
       "      <td>0.720324</td>\n",
       "      <td>0.000114</td>\n",
       "      <td>0.302333</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.146756</td>\n",
       "      <td>0.092339</td>\n",
       "      <td>0.186260</td>\n",
       "      <td>0.345561</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.396767</td>\n",
       "      <td>0.538817</td>\n",
       "      <td>0.419195</td>\n",
       "      <td>0.685220</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.204452</td>\n",
       "      <td>0.878117</td>\n",
       "      <td>0.027388</td>\n",
       "      <td>0.670468</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.417305</td>\n",
       "      <td>0.558690</td>\n",
       "      <td>0.140387</td>\n",
       "      <td>0.198101</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B         C         D  apple  orange  pear\n",
       "0  0.417022  0.720324  0.000114  0.302333      1       0     0\n",
       "1  0.146756  0.092339  0.186260  0.345561      0       0     1\n",
       "2  0.396767  0.538817  0.419195  0.685220      0       1     0\n",
       "3  0.204452  0.878117  0.027388  0.670468      0       0     1\n",
       "4  0.417305  0.558690  0.140387  0.198101      1       0     0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "cols = list('ABCD')\n",
    "np.random.seed(seed=1)\n",
    "myArr = np.random.ranf(size=(5,4))\n",
    "df = pd.DataFrame(data=myArr, columns=cols)\n",
    "df['Fruit'] = ['apple', 'pear', 'orange', 'pear','apple']\n",
    "one_hot = pd.get_dummies(df['Fruit'])\n",
    "df = df.join(one_hot)\n",
    "df.drop('Fruit',axis = 1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Correlation of Data in DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['A'].corr(df['B'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In Class Exercise  - Predict Price based on Features\n",
    "####  Get to know this data\n",
    "#### 1) Find what \"features\" are most correlated with price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 82)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "data = pd.read_csv('houseSmallData.csv')\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src = 'XY.png'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PoolQC          100\n",
       "Alley            94\n",
       "MiscFeature      91\n",
       "Fence            77\n",
       "FireplaceQu      54\n",
       "               ... \n",
       "BsmtHalfBath      0\n",
       "FullBath          0\n",
       "HalfBath          0\n",
       "BedroomAbvGr      0\n",
       "Unnamed: 0        0\n",
       "Length: 82, dtype: int64"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find NaN and clean\n",
    "nulls = data.isnull().sum().sort_values(ascending=False)\n",
    "nulls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop NaNs cols axis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check for NaNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>...</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.140812</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0.898305</td>\n",
       "      <td>0.175785</td>\n",
       "      <td>0.375532</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.236434</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.422539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.010101</td>\n",
       "      <td>0.010101</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.164438</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.440678</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.520213</td>\n",
       "      <td>...</td>\n",
       "      <td>0.347725</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.354832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.020202</td>\n",
       "      <td>0.020202</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.198336</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.919192</td>\n",
       "      <td>0.881356</td>\n",
       "      <td>0.145291</td>\n",
       "      <td>0.258511</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.162791</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.460153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.030303</td>\n",
       "      <td>0.030303</td>\n",
       "      <td>0.294118</td>\n",
       "      <td>0.163410</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.050505</td>\n",
       "      <td>0.338983</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.114894</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135659</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.250765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.040404</td>\n",
       "      <td>0.040404</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.260175</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>0.847458</td>\n",
       "      <td>0.313901</td>\n",
       "      <td>0.348404</td>\n",
       "      <td>...</td>\n",
       "      <td>0.224037</td>\n",
       "      <td>0.325581</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.526606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.959596</td>\n",
       "      <td>0.959596</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.167827</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.838384</td>\n",
       "      <td>0.728814</td>\n",
       "      <td>0.060987</td>\n",
       "      <td>0.164894</td>\n",
       "      <td>...</td>\n",
       "      <td>0.270712</td>\n",
       "      <td>0.244186</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.685714</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.363609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.178079</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.898990</td>\n",
       "      <td>0.830508</td>\n",
       "      <td>0.164126</td>\n",
       "      <td>0.618085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.184364</td>\n",
       "      <td>0.112403</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.436331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>0.979798</td>\n",
       "      <td>0.979798</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.191577</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.254237</td>\n",
       "      <td>0.043049</td>\n",
       "      <td>0.276596</td>\n",
       "      <td>...</td>\n",
       "      <td>0.140023</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.137294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.989899</td>\n",
       "      <td>0.989899</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>0.185496</td>\n",
       "      <td>0.285714</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.101010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.057447</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.283088</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.107829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.158685</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.494949</td>\n",
       "      <td>0.152542</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.302660</td>\n",
       "      <td>...</td>\n",
       "      <td>0.410735</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.223055</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0        Id  MSSubClass   LotArea  OverallQual  OverallCond  \\\n",
       "0     0.000000  0.000000    0.235294  0.140812     0.571429          0.5   \n",
       "1     0.010101  0.010101    0.000000  0.164438     0.428571          1.0   \n",
       "2     0.020202  0.020202    0.235294  0.198336     0.571429          0.5   \n",
       "3     0.030303  0.030303    0.294118  0.163410     0.571429          0.5   \n",
       "4     0.040404  0.040404    0.235294  0.260175     0.714286          0.5   \n",
       "..         ...       ...         ...       ...          ...          ...   \n",
       "95    0.959596  0.959596    0.235294  0.167827     0.428571          1.0   \n",
       "96    0.969697  0.969697    0.000000  0.178079     0.571429          0.5   \n",
       "97    0.979798  0.979798    0.000000  0.191577     0.142857          0.5   \n",
       "98    0.989899  0.989899    0.058824  0.185496     0.285714          0.5   \n",
       "99    1.000000  1.000000    0.000000  0.158685     0.142857          0.5   \n",
       "\n",
       "    YearBuilt  YearRemodAdd  MasVnrArea  BsmtFinSF1  ...  WoodDeckSF  \\\n",
       "0    0.939394      0.898305    0.175785    0.375532  ...    0.000000   \n",
       "1    0.666667      0.440678    0.000000    0.520213  ...    0.347725   \n",
       "2    0.919192      0.881356    0.145291    0.258511  ...    0.000000   \n",
       "3    0.050505      0.338983    0.000000    0.114894  ...    0.000000   \n",
       "4    0.909091      0.847458    0.313901    0.348404  ...    0.224037   \n",
       "..        ...           ...         ...         ...  ...         ...   \n",
       "95   0.838384      0.728814    0.060987    0.164894  ...    0.270712   \n",
       "96   0.898990      0.830508    0.164126    0.618085  ...    0.184364   \n",
       "97   0.555556      0.254237    0.043049    0.276596  ...    0.140023   \n",
       "98   0.101010      0.000000    0.000000    0.057447  ...    0.000000   \n",
       "99   0.494949      0.152542    0.000000    0.302660  ...    0.410735   \n",
       "\n",
       "    OpenPorchSF  EnclosedPorch  3SsnPorch  ScreenPorch  PoolArea   MiscVal  \\\n",
       "0      0.236434       0.000000        0.0          0.0       NaN  0.000000   \n",
       "1      0.000000       0.000000        0.0          0.0       NaN  0.000000   \n",
       "2      0.162791       0.000000        0.0          0.0       NaN  0.000000   \n",
       "3      0.135659       1.000000        0.0          0.0       NaN  0.000000   \n",
       "4      0.325581       0.000000        0.0          0.0       NaN  0.000000   \n",
       "..          ...            ...        ...          ...       ...       ...   \n",
       "95     0.244186       0.000000        0.0          0.0       NaN  0.685714   \n",
       "96     0.112403       0.000000        0.0          0.0       NaN  0.000000   \n",
       "97     0.000000       0.000000        0.0          0.0       NaN  0.000000   \n",
       "98     0.000000       0.283088        0.0          0.0       NaN  0.571429   \n",
       "99     0.000000       0.000000        0.0          0.0       NaN  0.571429   \n",
       "\n",
       "      MoSold  YrSold  SalePrice  \n",
       "0   0.090909    0.50   0.422539  \n",
       "1   0.363636    0.25   0.354832  \n",
       "2   0.727273    0.50   0.460153  \n",
       "3   0.090909    0.00   0.250765  \n",
       "4   1.000000    0.50   0.526606  \n",
       "..       ...     ...        ...  \n",
       "95  0.272727    0.75   0.363609  \n",
       "96  0.636364    0.00   0.436331  \n",
       "97  0.363636    0.25   0.137294  \n",
       "98  0.363636    1.00   0.107829  \n",
       "99  0.000000    1.00   0.223055  \n",
       "\n",
       "[100 rows x 37 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select only numeric cols\n",
    "numeric = data.select_dtypes(include=[np.number]).dropna(axis  = 1)\n",
    "numeric\n",
    "df = numeric\n",
    "normalized_df=(df-df.min())/(df.max()-df.min())\n",
    "normalized_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['SalePrice', 'OverallQual', 'GrLivArea', 'GarageArea', 'GarageCars',\n",
       "       'YearBuilt'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find cols most correlated with SalePrice\n",
    "corr = numeric.corr()\n",
    "cols = corr['SalePrice'].sort_values(ascending = False)[0:6].index\n",
    "cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>GarageArea</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>YearBuilt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>1710</td>\n",
       "      <td>548</td>\n",
       "      <td>2</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1262</td>\n",
       "      <td>460</td>\n",
       "      <td>2</td>\n",
       "      <td>1976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>1786</td>\n",
       "      <td>608</td>\n",
       "      <td>2</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1717</td>\n",
       "      <td>642</td>\n",
       "      <td>3</td>\n",
       "      <td>1915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>2198</td>\n",
       "      <td>836</td>\n",
       "      <td>3</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>6</td>\n",
       "      <td>1470</td>\n",
       "      <td>420</td>\n",
       "      <td>2</td>\n",
       "      <td>1993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>7</td>\n",
       "      <td>1588</td>\n",
       "      <td>472</td>\n",
       "      <td>2</td>\n",
       "      <td>1999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>4</td>\n",
       "      <td>960</td>\n",
       "      <td>432</td>\n",
       "      <td>1</td>\n",
       "      <td>1965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>5</td>\n",
       "      <td>835</td>\n",
       "      <td>366</td>\n",
       "      <td>1</td>\n",
       "      <td>1920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>4</td>\n",
       "      <td>1225</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1959</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    OverallQual  GrLivArea  GarageArea  GarageCars  YearBuilt\n",
       "0             7       1710         548           2       2003\n",
       "1             6       1262         460           2       1976\n",
       "2             7       1786         608           2       2001\n",
       "3             7       1717         642           3       1915\n",
       "4             8       2198         836           3       2000\n",
       "..          ...        ...         ...         ...        ...\n",
       "95            6       1470         420           2       1993\n",
       "96            7       1588         472           2       1999\n",
       "97            4        960         432           1       1965\n",
       "98            5        835         366           1       1920\n",
       "99            4       1225           0           0       1959\n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select X training data and Y dependent\n",
    "X = numeric[cols]\n",
    "Y = numeric['SalePrice']\n",
    "X = X.drop(['SalePrice'], axis = 1)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "R^2 is: 0.8309859964337734\n"
     ]
    }
   ],
   "source": [
    "from sklearn import linear_model\n",
    "lr = linear_model.LinearRegression()\n",
    "model = lr.fit(X, Y)\n",
    "predictions = model.predict(X)\n",
    "print(f\"R^2 is: {model.score(X,Y)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f99f7c6bcd0>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAD4CAYAAAAUymoqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAn20lEQVR4nO3dfZBc1Xnn8e+P0QADjhkBWkoewUqJVaLAWUswAaVIpWw5lgROIiUhDuwmqLyUlV1Dle04iqXNVsBvFXkphywVmwQHgnC8AQJYKDaOokVypZYqXkaRQAhQGIMdNJaRjDSQRDIeSc/+0afhqnW7+/ZM90x3z+9T1aXu577Tw336vNxzFBGYmZk14pSpPgEzM+s8Th5mZtYwJw8zM2uYk4eZmTXMycPMzBo2Y6pPoNnOPffcmDt37lSfhplZR9m+ffuPImJW0fW7LnnMnTuXoaGhqT4NM7OOIun7jaxfuNpKUo+kHZK+mT7fLellSTvTa2GKS9JtkoYlPSPpksw+Vkl6Mb1WZeKXStqVtrlNklL8bElb0vpbJM1s5OLMzKw1Gmnz+DjwfEVsTUQsTK+dKXYlMD+9VgO3QykRADcBlwOXATdlksHtwEcz2y1P8bXAoxExH3g0fTYzsylWKHlImgN8CPjLAquvAO6JkseBfkmzgWXAlog4GBGHgC3A8rTsnRHxeJQed78HWJnZ14b0fkMmbmZmU6hoyeNPgT8AjlfEv5Cqpm6VdFqKDQCvZNbZm2K14ntz4gDnRcS+9P6HwHl5JydptaQhSUMHDhwoeElmZjZedZOHpF8G9kfE9opF64ALgZ8DzgY+3fzTe1sqleQOxBURd0TEYEQMzppVuLOAmZmNU5HeVlcAvyrpKuB04J2S/joifjstf1PSXwG/nz6PAOdntp+TYiPA+yri30nxOTnrA7wqaXZE7EvVW/uLXpiZ2XSxcccIt2zeww9Gj/Cu/j7WLFvAykUD9TecgLolj4hYFxFzImIucA2wNSJ+O93MST2jVgLPpk02AdelXleLgddT1dNmYKmkmamhfCmwOS17Q9LitK/rgIcz+yr3ylqViZuZGaXEse6hXYyMHiGAkdEjrHtoFxt3jNTddiIm8oT51yXtAnYB5wKfT/FHgJeAYeCrwMcAIuIg8DngqfT6bIqR1vnLtM13gW+n+Hrgg5JeBH4pfTYzs+SWzXs4MnbshNiRsWPcsnlPS4/b0EOCEfEdSlVNRMSSKusEcEOVZXcBd+XEh4D35MRfAz7QyDmamU0nPxg90lC8WTy2lZlZB3tXf19D8WZx8jAz62Brli2gr7fnhFhfbw9rli1o6XG7bmwrM7PppNyrarJ7Wzl5mJl1uJWLBlqeLCq52srMzBrm5GFmZg1z8jAzs4Y5eZiZWcOcPMzMrGFOHmZm1jAnDzMza5iTh5mZNczJw8zMGubkYWZmDXPyMDOzhhVOHpJ6JO2Q9M30eZ6kJyQNS7pP0qkpflr6PJyWz83sY12K75G0LBNfnmLDktZm4rnHMDOzqdVIyePjwPOZz18Ebo2IdwOHgOtT/HrgUIrfmtZD0kWUprG9GFgOfCUlpB7gy8CVwEXAtWndWscwM7MpVCh5SJoDfIjSVLHlecuXAA+kVTZQmsccYEX6TFr+gbT+CuDeiHgzIl6mNOXsZek1HBEvRcRPgHuBFXWOYWZmU6hoyeNPgT8AjqfP5wCjEXE0fd4LlMcDHgBeAUjLX0/rvxWv2KZavNYxTiBptaQhSUMHDhwoeElmZjZedZOHpF8G9kfE9kk4n3GJiDsiYjAiBmfNmjXVp2Nm1vWKTAZ1BfCrkq4CTgfeCfxvoF/SjFQymAOMpPVHgPOBvZJmAGcBr2XiZdlt8uKv1TiGmZlNobolj4hYFxFzImIupQbvrRHxX4BtwNVptVXAw+n9pvSZtHxrRESKX5N6Y80D5gNPAk8B81PPqlPTMTalbaodw8zMptBEnvP4NPB7koYptU/cmeJ3Auek+O8BawEiYjdwP/Ac8PfADRFxLJUqbgQ2U+rNdX9at9YxzMxsCqn0A797DA4OxtDQ0FSfhpk10cYdI9yyeQ8/GD3Cu/r7WLNswaTP2d3tJG2PiMGi6xdp8zAzmzIbd4yw7qFdHBk7BsDI6BHWPbQLwAlkCnl4EjNra7ds3vNW4ig7MnaMWzbvmaIzMnDyMLM294PRIw3FbXI4eZhZW3tXf19DcZscTh5m1tbWLFtAX2/PCbG+3h7WLFswRWdk4AZzM2tz5UZx97ZqL04eZtb2Vi4acLJoM662MjOzhjl5mJlZw5w8zMysYU4eZmbWMCcPMzNrmJOHmZk1zMnDzMwa5uRhZmYNc/IwM7OG1U0ekk6X9KSkpyXtlvSZFL9b0suSdqbXwhSXpNskDUt6RtIlmX2tkvRieq3KxC+VtCttc5skpfjZkrak9bdImtn0/wJmZtawIiWPN4ElEfFeYCGwXNLitGxNRCxMr50pdiWl+cnnA6uB26GUCICbgMuBy4CbMsngduCjme2Wp/ha4NGImA88mj6bmdkUq5s8ouTf0sfe9Ko1d+0K4J603eNAv6TZwDJgS0QcjIhDwBZKiWg28M6IeDxKc+LeA6zM7GtDer8hEzczsylUqM1DUo+kncB+SgngibToC6lq6lZJp6XYAPBKZvO9KVYrvjcnDnBeROxL738InFfl/FZLGpI0dODAgSKXZGZ2ko07Rrhi/Vbmrf0WV6zfysYdI1N9Sm2rUPKIiGMRsRCYA1wm6T3AOuBC4OeAs4FPt+ok0zkEVUo8EXFHRAxGxOCsWbNaeRpm1qXKc6WPjB4heHuudCeQfA31toqIUWAbsDwi9qWqqTeBv6LUjgEwApyf2WxOitWKz8mJA7yaqrVI/+5v5HzNzIryXOmNKdLbapak/vS+D/gg8ELmpi5KbRHPpk02AdelXleLgddT1dNmYKmkmamhfCmwOS17Q9LitK/rgIcz+yr3ylqViZuZNZXnSm9MkcmgZgMbJPVQSjb3R8Q3JW2VNAsQsBP4b2n9R4CrgGHgMPARgIg4KOlzwFNpvc9GxMH0/mPA3UAf8O30AlgP3C/peuD7wIfHeZ1mZjW9q7+PkZxE4bnS86nUlNA9BgcHY2hoaKpPw8w6TLnNI1t11dfbwx//+s9Oi1kMJW2PiMGi63saWjMzPFd6o5w8zMwSz5VenMe2MjOzhrnkYWZdb+OOEVdHNZmTh5l1tcqG8PLDf4ATyAS42srMupof/msNlzzMrG01o7rJD/+1hkseZtaWmjXWVLWH/Pzw38Q4eZhZW2pWddOaZQvo6+05IdZ7ijj8k6MePXcCXG1lZm2pWdVNlQ//ndXXy7//5CiHDo8BbkAfL5c8zKwtNbO6aeWiAR5bu4SX13+IM0+bwdixE4dlcgN645w8zKwt5VU39fX2sGbZggnt1w3ozeFqKzNrS0XHmmq0R5ZHz20OJw8za5qNO0a4edNuRo+U2hNmntHLTb9y8bjbEuqNNTWeBwDXLFuQO3ruREs0042rrcysKTbuGGHN3z79VuIAOHR4jDUPPN2y3kzj6ZG1ctEAf/zrP8tAfx8CBvr7ps2w683kkoeZNcUtm/cwdvzk+YHGjgW3bN7TkpvzeNsvPHruxBWZhvZ0SU9KelrSbkmfSfF5kp6QNCzpPkmnpvhp6fNwWj43s691Kb5H0rJMfHmKDUtam4nnHsPM2k+tG3arGqP9AODUKVJt9SawJCLeCywElqe5yb8I3BoR7wYOAden9a8HDqX4rWk9JF0EXANcDCwHviKpJ01v+2XgSuAi4Nq0LjWOYWZtptYNu1U381b1yLL66iaPKPm39LE3vQJYAjyQ4huAlen9ivSZtPwDkpTi90bEmxHxMqU5zi9Lr+GIeCkifgLcC6xI21Q7hpm1mTXLFtB7ik6K9/aoZTdzt19MnUJtHql0sB14N6VSwneB0Yg4mlbZC5S/rQHgFYCIOCrpdeCcFH88s9vsNq9UxC9P21Q7RuX5rQZWA1xwwQVFLsnMmqx8w25mb6uix3WymHyFkkdEHAMWSuoHvgFc2MqTalRE3AHcATA4OHhyi52ZTQrfyKePhrrqRsQosA34eaBfUjn5zAHKffFGgPMB0vKzgNey8YptqsVfq3EMMzObQkV6W81KJQ4k9QEfBJ6nlESuTqutAh5O7zelz6TlWyMiUvya1BtrHjAfeBJ4CpifeladSqlRfVPaptoxzMxsChWptpoNbEjtHqcA90fENyU9B9wr6fPADuDOtP6dwNckDQMHKSUDImK3pPuB54CjwA2pOgxJNwKbgR7grojYnfb16SrHMDOzKaTSD/zuMTg4GENDQ1N9GmZmHUXS9ogYLLq+hycxM7OGeXgSsy7XjHnAzSo5eZh1sfGMOjsZ5+Rk1vlcbWXWxZo1D3izlJPZyOgRgreTmecQ7zxOHmZdrN1mzWu3ZGbj5+Rh1sXabdTZdktmNn5OHmZdrN1GnW23ZGbj5+Rh1sXabdTZdktmNn7ubWXW5dppsMLyebi3Vedz8jCzSdVOyczGz9VWZmbWMJc8zGxK+aHBzuTkYWZTph2fgLdinDzMrKVqlSxqPTTo5NHenDzMrGXqlSz80GDncoO5mbVMveFI/NBg5yoyDe35krZJek7SbkkfT/GbJY1I2pleV2W2WSdpWNIeScsy8eUpNixpbSY+T9ITKX5fmo6WNGXtfSn+hKS5Tb16M2uJjTtGuGL9VkbqlCzG89Bged/z1n6LK9Zv9aCKU6RIyeMo8KmIuAhYDNwg6aK07NaIWJhejwCkZdcAFwPLga9I6knT2H4ZuBK4CLg2s58vpn29GzgEXJ/i1wOHUvzWtJ6ZtbHsyLnVlEsWjT4B71F520fdNo+I2AfsS+//VdLzQK2WrBXAvRHxJvBymsv8srRsOCJeApB0L7Ai7W8J8J/TOhuAm4Hb075uTvEHgD+TpOi2uXPNOki9rrV5VVVZlSWLRh4adAN7+2iozSNVGy0CnkihGyU9I+kuSTNTbAB4JbPZ3hSrFj8HGI2IoxXxE/aVlr+e1q88r9WShiQNHThwoJFLMps2mlHdU+SXf63G7omOreUG9vZROHlIegfwIPCJiHiDUsngZ4CFlEomX2rFCRYREXdExGBEDM6aNWuqTsOsbTWruqfIfBzVGrsH+vt4bO2SCZUQ3MDePgolD0m9lBLH1yPiIYCIeDUijkXEceCrvF01NQKcn9l8TopVi78G9EuaURE/YV9p+VlpfTNrQLMmYSryy3/NsgX09uiE5b09asrIuR6Vt30U6W0l4E7g+Yj4k0x8dma1XwOeTe83AdeknlLzgPnAk8BTwPzUs+pUSo3qm1L7xTbg6rT9KuDhzL5WpfdXA1vd3mHWuGZV9xT+5V/5f2mT/q9ttyHmp7MiDwleAfwOsEvSzhT7H5R6Sy2k9GfxPeB3ASJit6T7geco9dS6ISKOAUi6EdgM9AB3RcTutL9PA/dK+jywg1KyIv37tdTofpBSwjGzBr2rvy+391Oj1T1rli044aE/OPmX/y2b9zB2/MRsMXY8mtao7VF520OR3lb/D1DOokdqbPMF4As58Ufytks9sC7Lif8Y+M1652hmtRW56RdRZD4ON2pPDx6exGwaaOYkTPV++TerlGPtzcnDbJpodnVPtec9mlXKsfbm5GFmDSsylLrn6OhuTh5mHWaikyc1Y/Klek96u1G7+3lUXbMOMtGH/fK2/+R9O/mfG3c1dB5uFDcnD7MOMtGH/fK2D+Drj/9LQ0+b+0lvc7WVWQeZ6C/+ausF1HwOo7Kq6/0XzuLB7SNuFJ/GXPIw6yAT/cVfa71qiSWvquvB7SP8xqUDftJ7GnPJw6yDTLQb7JplC/jkfTtzRwuplliqVZVte+EAj61dUvjcrbs4eZi1ucoqo9+4dIBtLxwYV2+plYsGGPr+Qb7++L+ckEBqJSA3jlseJw+zNpb3PMWD20cmVEX0+ZU/y+B/PLtwd10/MW55nDzM2thEZs6r9TxHI89h+Ilxy+PkYW2lGQ+wdZPxVhkVeQK8KD8xbnmcPKxtNPOG1y2qVRkFcMX6rVVv4s2e69tPjFsld9W1ttGs2e66Sd7MeWW1ni6vVjIZGT0y7vnLzbKKzCR4vqRtkp6TtFvSx1P8bElbJL2Y/p2Z4pJ0m6RhSc9IuiSzr1Vp/RclrcrEL5W0K21zW5q9sOoxrDu5V8/JsjPn5amWXGs1Zo93/nKzrCIlj6PApyLiImAxcIOki4C1wKMRMR94NH0GuJLS1LPzgdXA7VBKBMBNwOWUJn66KZMMbgc+mtlueYpXO4Z1IQ95kW/logEeW7skd0Y2KCXXjTtGuGL9Vuat/RZXrN/K+y+cVbXEAi7R2cTVTR4RsS8i/im9/1fgeWAAWAFsSKttAFam9yuAe6LkcaA/zXe+DNgSEQcj4hCwBVielr0zIh5P85PfU7GvvGNYF8qronGvnrdVS6L9Z/TWfAK8mulcorOJa6jNQ9JcYBHwBHBeROxLi34InJfeDwCvZDbbm2K14ntz4tQ4hnWhbBWNh7w4WbXkGkHNJ8BnntGbu7/+KnGzIgr3tpL0DuBB4BMR8UZqlgAgIkJS3ogHTVPrGJJWU6oi44ILLmjlaViLuVfPybLdl8/q6+X03lMYPTz2VpfZT963M3e7cskiqvyfWS1uVkShkoekXkqJ4+sR8VAKv5qqnEj/7k/xEeD8zOZzUqxWfE5OvNYxThARd0TEYEQMzpo1q8glmXWEykEJR4+M8eOx49z6Wwt5bO0SVi4aqNtW9PqRsdzl1eJmRRTpbSXgTuD5iPiTzKJNQLnH1Crg4Uz8utTrajHweqp62gwslTQzNZQvBTanZW9IWpyOdV3FvvKOYTYtFOm+XK+tyB0RrBWKlDyuAH4HWCJpZ3pdBawHPijpReCX0meAR4CXgGHgq8DHACLiIPA54Kn0+myKkdb5y7TNd4Fvp3i1Y5hNC0W6L9drK3JHBGsFRZdVfA4ODsbQ0NBUn4Z1uHYZJuWK9VtznzDvkfjSh99b+Jza5XqsfUnaHhGDRdf38CRmFdppmJS8QQkBjkU0dE7uiGDN5uRhXaXIL+x66zR7XKiJKB/vU/c/zbGKWoKpOiczcPKwLlKkxFBknXYbJmXlooG63XHLmpE8zYrwwIjWNYr0TCqyTjv2TipyTnlzjVeOYVVkHbMinDysaxQpMRRZ5/0X5j8rlBevHFOqVTfhIj2mmpU8zYpwtZV1jSLTpRZZZ9sLB3L3XxmfzIb1IhMyNSt5lrl6y2px8rCWmeybT5HpUousU/QGO9kN6/V6TDUreUJ79Tiz9uRqK2uJqahbLzKwYpF1irZ5jKdhvZXVXEWqtoo+MOjqLavHJQ9rianq7lrkeYZ66xQpnUDxX/Flrf41X6Rqq+h85O3W48zaj5OHtUSzbj5TUe9e6wabPZ/+M3rpPUWMHX/7+Ytaw35MRkJtRvKExhOjTT9OHh2mUxoxm3Hzmcp697wbbOX5HDo8Rm+P6O/r5fUjY3W/j076NV+09GXTl5NHB+mkRsxm3Hzq/VKf7ESadz5jx4IzT5vBzpuW1t2+k37NF63esunLyaODtNOwGfU04+ZT65d6XiL95H07+cR9OxlowY1u446R3Bt/rfOs1Gm/5j0eltXi5NFBOqnaAyZ+86n2S73/jN7csZ7Kn5pdIisnqlrnWYR/zVs3cfLoIO1S7TFZ1UV5v9R7e8S//fjoSYmjUjNLZHklvrJGSw7+NW/dwsmjg7RDtUfRwQebkVzyfqn/+5tHGS04fWojJbJa51xrP5XPiJhNF0Wmob1L0n5Jz2ZiN0saqZhZsLxsnaRhSXskLcvEl6fYsKS1mfg8SU+k+H2STk3x09Ln4bR8btOuukMVecCt1eo9PNbshwNXLhrgsbVLeHn9h3hs7ZKG5t0uWiKrd87V9jPQ3+fEYdNWkSfM7waW58RvjYiF6fUIgKSLgGuAi9M2X5HUI6kH+DJwJXARcG1aF+CLaV/vBg4B16f49cChFL81rTftVd5MJ/vmVa/d5TN/t7spTyZXexK7aEJopERWLyF6Glezk9VNHhHxj8DBeuslK4B7I+LNiHiZ0pzkl6XXcES8FBE/Ae4FVkgSsAR4IG2/AViZ2deG9P4B4ANpfZtCtYbu2LhjhEOH80sGjVYhVSsJVLuR//biC8ZdIquXENuhxGfWbibS5nGjpOuAIeBTEXEIGAAez6yzN8UAXqmIXw6cA4xGxNGc9QfK20TEUUmvp/V/VHkiklYDqwEuuOCCCVySVcq2BZzV18vYseMnrVP+FV6rdNFIo36tksBja5e8tU62fQKqj4ZbT5GOCG7oNjvReAdGvB34GWAhsA/4UrNOaDwi4o6IGIyIwVmz8udisMZVlgBGj4zx7z858abe39f71q/was9BAA1V8RQpCZSr7tYsW8DNm3bzift2jrudxdVSZo0bV8kjIl4tv5f0VeCb6eMIcH5m1TkpRpX4a0C/pBmp9JFdv7yvvZJmAGel9W2S1OqiWjZ6ZOytEkePlNuFVmrseYvxDhueVa2rbmVJSoLRw2Oc1dfL6b2nMHq4/jAjZjbOkoek2ZmPvwaUe2JtAq5JPaXmAfOBJ4GngPmpZ9WplBrVN0VEANuAq9P2q4CHM/tald5fDWxN69skKdpOUf6lX+3ZiwgKlwI27hjh8E+OnhQvOmx4Vt783pUlqUOHx956/+Ox49z6WwunpCOCWaepW/KQ9DfA+4BzJe0FbgLeJ2khpYd6vwf8LkBE7JZ0P/AccBS4ISKOpf3cCGwGeoC7ImJ3OsSngXslfR7YAdyZ4ncCX5M0TKnB/pqJXmynm+yxnKqVAPLUK6EUeeK7Wkmiv6+Xm3/14sLDhpdVllTqJZt2HerFrB3VTR4RcW1O+M6cWHn9LwBfyIk/AjySE3+JUm+syviPgd+sd37TxWQPilitBDBeRW7M1W7uZ542I3e7Wsktr6RSpCRVNFmaTXeeSbBDTObMbuVEVdnttq/3FGae0Tvu/dabYa/RgQfzGroBZp7Rm9uVtkiPL1G8is1sOvPwJB1iMgdFrFYCOPvM03hs7ZLS8xYPPM3YscaaoMo373L128jokbca2Ws9wFPtpt/oQIN5w7tUirS/qRjy3ayTOHl0iGpVNGf19bLos//wVimhWvtAIwolqjp5QxWrlKuRKqvfyo3s1XZXr8tsI89fVCabasesNuR7u86dYjYV1G0dmAYHB2NoaGiqT6Pp8hqTe08Rx4Fjx0/8DntPEbf85nvHfZO7Yv3WqlVI1brj5iknkOz8GrX2nedPf2thy27W1c5lIJV0qi0rP6ho1k0kbY+IwaLru82jQ+QNkfGO02eclDgAxo5HbltItfGiKlVrSwAKJw54O3Fku742Us3W6oEHaz0c2Glzp5hNNldbdZDKKpp5a79Vdd1qzzgUqYYpf7550+7Cw58XPY+i3X8n4wnvWm0m5TaZSu04ZazZVHDJo4PVupEVecahVm+tlYsGOPO0if+2qDyPWqWacqP5ZA48WG2UYg9ZYlabSx4dbM2yBbm9nnpP0VuN00Uah6uZaBVN3s02+2s/29uqFfOOT4SnjDWrzcmjg5VvZJ/5u90n9bYC6nZLhfqll0Yat3t7xJmnzuD1I7XHh+qUEWo75TzNpoKTR4erdoO7Yv3WuomjXjVMreciBvr7eP+Fs9j2woHCv8z93IRZ93DyaCPNvLnWqnISFNp/M6tu/NyEWXdx8mgTRW6utZJL5bKz+npze0oVeU6hFSWEWg32Th5mncfJo03Uu7nWSi7ASct6e0TvKWIs8xxIrWqq7JAh2afDm1VC8HMTZt3FyaNN1Lu51utqW7ls7Fgw84xezjh1Rt0SRGViquyZ1YwSQtEJnsysMzh5tIlq1Uxn9ZVGsR3PL/fRw2Ps+KOldY9dZMbAiZYQ8hrf/dyEWefyQ4JtQlWGlS3Hq/1Cf1d/X81lRRRJDBMtIeQNrzJZDwKaWfPVTR6S7pK0X9KzmdjZkrZIejH9OzPFJek2ScOSnpF0SWabVWn9FyWtysQvlbQrbXObVLpdVjtGtxo9nD8MSDle64nnvGUC3n/hrELHrpcYmlVCqPY0t5l1niIlj7uB5RWxtcCjETEfeDR9BriS0rzl84HVwO1QSgSUpq+9nNKsgTdlksHtwEcz2y2vc4yuVK/0UOuX+8pFA/zGpQMnzIkRwIPbRwpNbFQt+cD4SghFB2A0s85VZBraf5Q0tyK8gtK85gAbgO9Qmot8BXBPlMZ5f1xSv6TZad0tEXEQQNIWYLmk7wDvjIjHU/weYCXw7RrH6EpF2gRqPfG87YUD427o9vMcZtao8TaYnxcR+9L7HwLnpfcDwCuZ9famWK343px4rWOcRNJqSiUdLrjggkavpS1M9AbeSIN6tec4mnFz9/McZtPDhHtbRURIaumMUvWOERF3AHdAaTKoVp5LK03kBl60K2yrSwZ+nsNsehhv8nhV0uyI2Jeqpfan+Ahwfma9OSk2wttVUOX4d1J8Ts76tY5hOWpVe2VLGqfkzARYr2TQyBPnfp7DbHoYb1fdTUC5x9Qq4OFM/LrU62ox8HqqetoMLJU0MzWULwU2p2VvSFqcelldV7GvvGNYjmoN6lB6+nwkDctebSbAaiWDckmlvH25pNLILIR+nsOs+9QteUj6G0qlhnMl7aXUa2o9cL+k64HvAx9Oqz8CXAUMA4eBjwBExEFJnwOeSut9ttx4DnyMUo+uPkoN5d9O8WrHsCryqr2KjK4L1UsGjbZheB4Ms+mhSG+ra6ss+kDOugHcUGU/dwF35cSHgPfkxF/LO0a76JThxYu0NdQqGYynDcPzYJh1Pw9PMg5T0R11446RE+YUn3lGLzf9ysV1j1etDaJH4niE2zDMbFycPMahSFVOM0smG3eMsOZvnz5hhNxDh8dY88DTQO2EVa0hveiDfx6TyszyeGyrcahXldNoI3M9t2zec0LiKBs7Fm+NqlvNRMeU8phUZpbHJY9xqFeV0+wH5Wq1LxRp05hoG4TbMMyskkse41CvO2qzH5Sr1b7gtgczmwpOHuNQrypnokOkV1qzbAG9p5w8Zntvj9z2YGZTwtVW41SrKqfZjczl44ynt5WZWSs4edD8ZzZa8aCc2x3MrJ1M++TRqmc2fLM3s2427ds8avWMMjOzfNM+eXgIcTOzxk375NHsnlFmZtPBtE8eHkLczKxx077B3EOIm5k1btonD3DPKDOzRk37aiszM2vchJKHpO9J2iVpp6ShFDtb0hZJL6Z/Z6a4JN0maVjSM5IuyexnVVr/RUmrMvFL0/6H07Ynj9FhZmaTrhklj/dHxMKIGEyf1wKPRsR84NH0GeBKYH56rQZuh1KyoTS17eXAZcBN5YST1vloZrvlTThfMzOboFZUW60ANqT3G4CVmfg9UfI40C9pNrAM2BIRByPiELAFWJ6WvTMiHk/T296T2ZeZmU2hiSaPAP5B0nZJq1PsvIjYl97/EDgvvR8AXslsuzfFasX35sRPImm1pCFJQwcOHJjI9ZiZWQET7W31CxExIuk/AFskvZBdGBEh6eQp8JosIu4A7gAYHBxs+fHMzKa7CSWPiBhJ/+6X9A1KbRavSpodEftS1dP+tPoIcH5m8zkpNgK8ryL+nRSfk7N+V2v2CL9mZq0w7morSWdK+qnye2Ap8CywCSj3mFoFPJzebwKuS72uFgOvp+qtzcBSSTNTQ/lSYHNa9oakxamX1XWZfXWlZs99bmbWKhMpeZwHfCP1np0B/J+I+HtJTwH3S7oe+D7w4bT+I8BVwDBwGPgIQEQclPQ54Km03mcj4mB6/zHgbqAP+HZ6da1mz31uZtYq404eEfES8N6c+GvAB3LiAdxQZV93AXflxIeA94z3HDuNR/g1s07hJ8zbiEf4NbNO4eTRRjzCr5l1Cg+M2EY8wq+ZdQonjzbjEX7NrBO42srMzBrm5GFmZg1z8jAzs4Y5eZiZWcOcPMzMrGEqPfjdPSQdoDQsSlHnAj9q0elMtm66Fuiu6+mmawFfTzsb77X8x4iYVXTlrksejZI0lJkFsaN107VAd11PN10L+Hra2WRdi6utzMysYU4eZmbWMCePNANhl+ima4Huup5uuhbw9bSzSbmWad/mYWZmjXPJw8zMGubkYWZmDeua5CHpe5J2SdopaSjFzpa0RdKL6d+ZKS5Jt0kalvSMpEsy+1mV1n9R0qpM/NK0/+G0rZp8/ndJ2i/p2Uys5edf7RgtuJabJY2k72enpKsyy9al89ojaVkmvjzFhiWtzcTnSXoixe+TdGqKn5Y+D6flc5twLedL2ibpOUm7JX08xTv1u6l2PZ36/Zwu6UlJT6fr+cx4z6FZ19mCa7lb0suZ72Zhik/t31pEdMUL+B5wbkXsfwFr0/u1wBfT+6sozYcuYDHwRIqfDbyU/p2Z3s9My55M6ypte2WTz/8XgUuAZyfz/KsdowXXcjPw+znrXgQ8DZwGzAO+C/Sk13eBnwZOTetclLa5H7gmvf9z4L+n9x8D/jy9vwa4rwnXMhu4JL3/KeCf0zl36ndT7Xo69fsR8I70vhd4Iv23bOgcmnmdLbiWu4Grc9af0r+1pt38pvpFfvLYA8zO/E+zJ73/C+DayvWAa4G/yMT/IsVmAy9k4ies18RrmMuJN9yWn3+1Y7TgWm4m/+a0DliX+bwZ+Pn02ly5Xvqj/xEwI8XfWq+8bXo/I62nJn9HDwMf7OTvpsr1dPz3A5wB/BNweaPn0MzrbMG13E1+8pjSv7WuqbYCAvgHSdslrU6x8yJiX3r/Q+C89H4AeCWz7d4UqxXfmxNvtck4/2rHaIUbU/H6rkyxuNFrOQcYjYijFfET9pWWv57Wb4pUxbGI0i/Cjv9uKq4HOvT7kdQjaSewH9hCqaTQ6Dk08zqbdi0RUf5uvpC+m1slnVZ5LQXPual/a92UPH4hIi4BrgRukPSL2YVRSqkd2y95Ms6/xce4HfgZYCGwD/hSi47TEpLeATwIfCIi3sgu68TvJud6Ovb7iYhjEbEQmANcBlw4tWc0fpXXIuk9lEo6FwI/R6kq6tMtPodCf2tdkzwiYiT9ux/4BqU/olclzQZI/+5Pq48A52c2n5NiteJzcuKtNhnnX+0YTRURr6b/MY4DX6X0/VDnnPPirwH9kmZUxE/YV1p+Vlp/QiT1UrrRfj0iHkrhjv1u8q6nk7+fsogYBbZRqkJq9ByaeZ3NvJblEbEvSt4E/orxfzdN/VvriuQh6UxJP1V+DywFngU2AeWeBqso1e+S4tel3gqLgddTkW0zsFTSzFRsX0qpHnMf8Iakxal3wnWZfbXSZJx/tWM0VfkPM/k1St9P+fjXpF4w84D5lBr1ngLmpx4tp1Jq3NyUfhVtA67OOefstVwNbE3rT+S8BdwJPB8Rf5JZ1JHfTbXr6eDvZ5ak/vS+j1L7zfPjOIdmXmczr+WFzE1dwEpO/G6m7m+tWQ08U/mi1BPi6fTaDfxhip8DPAq8CPxf4OwUF/BlSnWju4DBzL7+KzCcXh/JxAfTl/Zd4M9ofkPs31CqLhijVBd5/WScf7VjtOBavpbO9Zn0hzo7s/4fpvPaQ6YXG6XeJP+clv1hxff9ZLrGvwVOS/HT0+fhtPynm3Atv0CpCP8MsDO9rurg76ba9XTq9/OfgB3pvJ8F/mi859Cs62zBtWxN382zwF/zdo+sKf1b8/AkZmbWsK6otjIzs8nl5GFmZg1z8jAzs4Y5eZiZWcOcPMzMrGFOHmZm1jAnDzMza9j/BxHzNrzGrjCMAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.scatter(predictions, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build Linear Regression Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='beta.png' width= 500>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find NaNs and clean\n",
    "nulls = data.isnull()\n",
    "data[nulls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.dropna(axis=1,inplace=True)\n",
    "sum(train.isnull().sum() != 0)\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = train.corr()\n",
    "cols = corr['SalePrice'].sort_values(ascending=False)[0:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train[cols.index]\n",
    "Y = X['SalePrice']\n",
    "X = X.drop(['SalePrice'], axis = 1)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import linear_model\n",
    "lr = linear_model.LinearRegression()\n",
    "model = lr.fit(X, Y)\n",
    "predictions = model.predict(X)\n",
    "print(f\"R^2 is: {model.score(X,Y)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.scatter(predictions, Y, color = 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Repeat this but normalize the numeric data before training\n",
    "df = numeric\n",
    "normalized_df=(df-df.min())/(df.max()-df.min())\n",
    "normalized_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}