Week2_Empirical Exercise_Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\textbf{Empirical Exercise-Week 2}$  Page 141, Problem 3.26\n",
    "Lets look at the example from your book, relating wages to experience in the work force."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tab1= pd.read_csv('cps5_small.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>black</th>\n",
       "      <th>educ</th>\n",
       "      <th>exper</th>\n",
       "      <th>faminc</th>\n",
       "      <th>female</th>\n",
       "      <th>metro</th>\n",
       "      <th>midwest</th>\n",
       "      <th>south</th>\n",
       "      <th>wage</th>\n",
       "      <th>west</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>44.44</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>25</td>\n",
       "      <td>45351</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>16.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>27</td>\n",
       "      <td>91946</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>15.38</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>42</td>\n",
       "      <td>48370</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13.54</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>41</td>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1195</th>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1196</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>60400</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21.63</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1197</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>36</td>\n",
       "      <td>69524</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18.75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198</th>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1199</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>52</td>\n",
       "      <td>61270</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>16.46</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1200 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      black  educ  exper  faminc  female  metro  midwest  south   wage  west\n",
       "0         0    13     45       0       1      1        0      0  44.44     1\n",
       "1         0    14     25   45351       1      1        1      0  16.00     0\n",
       "2         0    18     27   91946       1      1        0      0  15.38     0\n",
       "3         0    13     42   48370       0      1        1      0  13.54     0\n",
       "4         0    13     41   10000       1      1        0      0  25.00     1\n",
       "...     ...   ...    ...     ...     ...    ...      ...    ...    ...   ...\n",
       "1195      0    13     45       0       1      1        0      0  11.00     1\n",
       "1196      0    16     17   60400       0      1        1      0  21.63     0\n",
       "1197      1    16     36   69524       1      1        0      0  18.75     0\n",
       "1198      0    14     17       0       0      1        0      1  18.00     0\n",
       "1199      0    10     52   61270       0      1        1      0  16.46     0\n",
       "\n",
       "[1200 rows x 10 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Tab1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "wage = Tab1.iloc[:,8] \n",
    "exper = Tab1.iloc[:,2]  \n",
    "metro = Tab1.iloc[:,5] \n",
    "edu = Tab1.iloc[:,1]\n",
    "wage1 = np.array(wage)\n",
    "exper1 = np.array(exper)\n",
    "metro1 = np.array(metro)\n",
    "edu1 = np.array(edu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "yy = wage1 \n",
    "xx = sm.add_constant(exper1)   # create regression variabels;\n",
    "result1 = sm.OLS(yy,xx).fit()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([21.62307479,  0.08629043])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1.params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "T stat for the coefficient is 2.61. We have 1200 observations and 2 regressors, so we have 1198 degress of freedom. We are asked to do a one-tailed test of signifcance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.        , 0.99544161])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import t\n",
    "pval1 = t.cdf(result1.tvalues, 1198)\n",
    "pval1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The pval1 gives the area to the left of the distributon. We look at the area to the right, the probability that the null of zero is true even though we reject it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.004558388246773126"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pval = 1- pval1[1]  #Evaluate the p-value for the reported t-stat, 2.61, at 1198 degrees of freedom\n",
    "pval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So at a one-tailed 5 percent test, we can reject the null hypothesis that the coefficeint of experience is zero. Next question: extract the data for metro1==1, do a regression for the folks living in a metro area. It is easy, see below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Data = np.hstack([metro1.reshape(-1,1), yy.reshape(-1,1), xx])  # Combine data set with metro1 and the y and x variables\n",
    "WhichRow = Data[:,0]==1;   # Find indicator for metro1 = 1;\n",
    "Data1 = Data[WhichRow,1:];   # Extrat the data for metro1 == 1;\n",
    "yyy = Data1[:,0]; xxx = Data1[:,1:]\n",
    "result2 = sm.OLS(yyy,xxx).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[21.88860925, 21.4719512 ],\n",
       "       [ 0.11280127,  2.93595005]])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([result2.params, result2.tvalues]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that living in a metro area has a stronger effect that the total sample. How about the non-metro area?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "WhichRow = Data[:,0]==0;   # Find indicator for metro1 = 1;\n",
    "Data1 = Data[WhichRow,1:];   # Extrat the data for metro1 == 0;\n",
    "yyyy = Data1[:,0]; xxxx = Data1[:,1:];\n",
    "result3 = sm.OLS(yyyy,xxxx).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.94103373e+01, 1.30461832e+01],\n",
       "       [1.32143451e-02, 2.52242552e-01]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([result3.params, result3.tvalues]).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We see that living in a non-metro area means that experience has a much lower effect on wage and the effect is not statistically different from zero. So experience matters a lot more in metro areas than non-metro areas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}