Gradient boosting on whole dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Tatenda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sb\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn\n",
    "\n",
    "from pandas import Series, DataFrame\n",
    "from pylab import rcParams\n",
    "from sklearn import preprocessing\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn import metrics \n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata=pd.read_csv(\"movie_metadata_with_score_metacritic.csv\", index_col=\"Unnamed: 0\") #reading in metacritic dataset\n",
    "metadata=metadata.loc[~metadata[\"metacritic_metascore\"].isna()] #removing rows with no metacritic data\n",
    "metadata=metadata.drop([\"homepage\",\"keywords\",\"overview\",\"status\",\"tagline\",\"imdb_metascore\",\"budget\",\"revenue\"],1) #dropping unnecessary features\n",
    "metadata=metadata.drop_duplicates() #dropping duplicates\n",
    "metadata[\"release_year\"]=metadata.release_date.str[0:4].astype(int) #creating new column with year for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_data=pd.read_csv(\"the_numbers_budget.csv\") #reading in revenue data\n",
    "num_data=num_data.rename(columns={\"Movie\":\"title\"}) #renaming column to match metadata\n",
    "num_data[\"num_year\"]=num_data[\"Release Date\"].str[-4:].astype(int)\n",
    "num_data=num_data.drop_duplicates()#dropping duplicates\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "critic_revenue=metadata.merge(num_data,on=\"title\") #merging datasets\n",
    "critic_revenue=critic_revenue.drop_duplicates() #dropping duplicates\n",
    "critic_revenue=critic_revenue.loc[(critic_revenue[\"Worldwide Gross\"]!=0)] #removing rows with no revenue data\n",
    "critic_revenue=critic_revenue.loc[(np.abs(critic_revenue.release_year-critic_revenue.num_year)<5)] #removing rows where the years don't match, as this indicates different movies\n",
    "\n",
    "critic_revenue[\"log Worldwide Gross\"]=np.log(critic_revenue[\"Worldwide Gross\"])#adding column for log of revenue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Normalizing by the median for merged data set\n",
    "\n",
    "critic_revenue[\"Production Budget\"]=(critic_revenue[\"Production Budget\"]-critic_revenue[\"Production Budget\"].median())/critic_revenue[\"Production Budget\"].std()\n",
    "critic_revenue[\"Worldwide Gross\"]=(critic_revenue[\"Worldwide Gross\"]-critic_revenue[\"Worldwide Gross\"].median())/critic_revenue[\"Worldwide Gross\"].std()\n",
    "critic_revenue[\"log Worldwide Gross\"]=(critic_revenue[\"log Worldwide Gross\"]-critic_revenue[\"log Worldwide Gross\"].median())/critic_revenue[\"log Worldwide Gross\"].std()\n",
    "critic_revenue[\"Domestic Gross\"]=(critic_revenue[\"Domestic Gross\"]-critic_revenue[\"Domestic Gross\"].median())/critic_revenue[\"Domestic Gross\"].std()\n",
    "critic_revenue[\"popularity\"]=(critic_revenue[\"popularity\"]-critic_revenue[\"popularity\"].median())/critic_revenue[\"popularity\"].std()\n",
    "critic_revenue[\"vote_count\"]=(critic_revenue[\"vote_count\"]-critic_revenue[\"vote_count\"].median())/critic_revenue[\"vote_count\"].std()\n",
    "critic_revenue[\"metacritic_metascore\"]=(critic_revenue[\"metacritic_metascore\"]-critic_revenue[\"metacritic_metascore\"].median())/critic_revenue[\"metacritic_metascore\"].std()\n",
    "critic_revenue[\"runtime\"]=(critic_revenue[\"runtime\"]-critic_revenue[\"runtime\"].median())/critic_revenue[\"runtime\"].std()\n",
    "\n",
    "# Reindex since the dataset has been subset many times\n",
    "critic_revenue.index=range(len(critic_revenue))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3319, 21)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replace every nan values with 0\n",
    "critic_revenue.fillna(value=0,axis=1,inplace=True)\n",
    "critic_revenue.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Defining features and target for this dataset based on co-relation\n",
    "features = ['metacritic_metascore']\n",
    "target = ['Worldwide Gross']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>genres</th>\n",
       "      <th>id</th>\n",
       "      <th>original_language</th>\n",
       "      <th>original_title</th>\n",
       "      <th>popularity</th>\n",
       "      <th>production_companies</th>\n",
       "      <th>production_countries</th>\n",
       "      <th>release_date</th>\n",
       "      <th>runtime</th>\n",
       "      <th>spoken_languages</th>\n",
       "      <th>...</th>\n",
       "      <th>vote_average</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>metacritic_metascore</th>\n",
       "      <th>release_year</th>\n",
       "      <th>Release Date</th>\n",
       "      <th>Production Budget</th>\n",
       "      <th>Domestic Gross</th>\n",
       "      <th>Worldwide Gross</th>\n",
       "      <th>num_year</th>\n",
       "      <th>log Worldwide Gross</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2770</th>\n",
       "      <td>[{\"id\": 35, \"name\": \"Comedy\"}, {\"id\": 10749, \"...</td>\n",
       "      <td>378200</td>\n",
       "      <td>en</td>\n",
       "      <td>The Perfect Match</td>\n",
       "      <td>-0.400200</td>\n",
       "      <td>[{\"name\": \"Lionsgate\", \"id\": 1632}, {\"name\": \"...</td>\n",
       "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
       "      <td>2016-03-11</td>\n",
       "      <td>-0.439008</td>\n",
       "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
       "      <td>...</td>\n",
       "      <td>5.2</td>\n",
       "      <td>-0.234094</td>\n",
       "      <td>-0.765790</td>\n",
       "      <td>2016</td>\n",
       "      <td>3/11/2016</td>\n",
       "      <td>-0.379488</td>\n",
       "      <td>-0.259943</td>\n",
       "      <td>-0.185281</td>\n",
       "      <td>2016</td>\n",
       "      <td>-0.636367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>[{\"id\": 878, \"name\": \"Science Fiction\"}, {\"id\"...</td>\n",
       "      <td>80274</td>\n",
       "      <td>en</td>\n",
       "      <td>Ender's Game</td>\n",
       "      <td>0.879321</td>\n",
       "      <td>[{\"name\": \"Summit Entertainment\", \"id\": 491}, ...</td>\n",
       "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
       "      <td>2013-10-23</td>\n",
       "      <td>0.439008</td>\n",
       "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
       "      <td>...</td>\n",
       "      <td>6.6</td>\n",
       "      <td>1.467792</td>\n",
       "      <td>-0.218797</td>\n",
       "      <td>2013</td>\n",
       "      <td>11/1/2013</td>\n",
       "      <td>2.110903</td>\n",
       "      <td>0.507815</td>\n",
       "      <td>0.479793</td>\n",
       "      <td>2013</td>\n",
       "      <td>0.481812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 80, \"nam...</td>\n",
       "      <td>8838</td>\n",
       "      <td>en</td>\n",
       "      <td>Mercury Rising</td>\n",
       "      <td>0.057314</td>\n",
       "      <td>[{\"name\": \"Imagine Entertainment\", \"id\": 23}, ...</td>\n",
       "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
       "      <td>1998-04-03</td>\n",
       "      <td>0.292672</td>\n",
       "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.011291</td>\n",
       "      <td>-0.984588</td>\n",
       "      <td>1998</td>\n",
       "      <td>4/3/1998</td>\n",
       "      <td>0.925002</td>\n",
       "      <td>0.083828</td>\n",
       "      <td>-0.056551</td>\n",
       "      <td>1998</td>\n",
       "      <td>-0.117150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2251</th>\n",
       "      <td>[{\"id\": 18, \"name\": \"Drama\"}, {\"id\": 80, \"name...</td>\n",
       "      <td>10220</td>\n",
       "      <td>en</td>\n",
       "      <td>Rounders</td>\n",
       "      <td>0.043474</td>\n",
       "      <td>[{\"name\": \"Miramax Films\", \"id\": 14}, {\"name\":...</td>\n",
       "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
       "      <td>1998-09-11</td>\n",
       "      <td>0.780459</td>\n",
       "      <td>[{\"iso_639_1\": \"ru\", \"name\": \"P\\u0443\\u0441\\u0...</td>\n",
       "      <td>...</td>\n",
       "      <td>6.9</td>\n",
       "      <td>0.064733</td>\n",
       "      <td>-0.054699</td>\n",
       "      <td>1998</td>\n",
       "      <td>9/11/1998</td>\n",
       "      <td>-0.213462</td>\n",
       "      <td>-0.064531</td>\n",
       "      <td>-0.113356</td>\n",
       "      <td>1998</td>\n",
       "      <td>-0.277905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979</th>\n",
       "      <td>[{\"id\": 35, \"name\": \"Comedy\"}]</td>\n",
       "      <td>16538</td>\n",
       "      <td>en</td>\n",
       "      <td>I Love You, Man</td>\n",
       "      <td>0.255117</td>\n",
       "      <td>[{\"name\": \"The Montecito Picture Company\", \"id...</td>\n",
       "      <td>[{\"iso_3166_1\": \"US\", \"name\": \"United States o...</td>\n",
       "      <td>2009-03-20</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>[{\"iso_639_1\": \"en\", \"name\": \"English\"}]</td>\n",
       "      <td>...</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.121940</td>\n",
       "      <td>0.820490</td>\n",
       "      <td>2009</td>\n",
       "      <td>3/20/2009</td>\n",
       "      <td>0.450642</td>\n",
       "      <td>0.659335</td>\n",
       "      <td>0.278349</td>\n",
       "      <td>2009</td>\n",
       "      <td>0.337437</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 genres      id  \\\n",
       "2770  [{\"id\": 35, \"name\": \"Comedy\"}, {\"id\": 10749, \"...  378200   \n",
       "208   [{\"id\": 878, \"name\": \"Science Fiction\"}, {\"id\"...   80274   \n",
       "650   [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 80, \"nam...    8838   \n",
       "2251  [{\"id\": 18, \"name\": \"Drama\"}, {\"id\": 80, \"name...   10220   \n",
       "979                      [{\"id\": 35, \"name\": \"Comedy\"}]   16538   \n",
       "\n",
       "     original_language     original_title  popularity  \\\n",
       "2770                en  The Perfect Match   -0.400200   \n",
       "208                 en       Ender's Game    0.879321   \n",
       "650                 en     Mercury Rising    0.057314   \n",
       "2251                en           Rounders    0.043474   \n",
       "979                 en    I Love You, Man    0.255117   \n",
       "\n",
       "                                   production_companies  \\\n",
       "2770  [{\"name\": \"Lionsgate\", \"id\": 1632}, {\"name\": \"...   \n",
       "208   [{\"name\": \"Summit Entertainment\", \"id\": 491}, ...   \n",
       "650   [{\"name\": \"Imagine Entertainment\", \"id\": 23}, ...   \n",
       "2251  [{\"name\": \"Miramax Films\", \"id\": 14}, {\"name\":...   \n",
       "979   [{\"name\": \"The Montecito Picture Company\", \"id...   \n",
       "\n",
       "                                   production_countries release_date  \\\n",
       "2770  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2016-03-11   \n",
       "208   [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2013-10-23   \n",
       "650   [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   1998-04-03   \n",
       "2251  [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   1998-09-11   \n",
       "979   [{\"iso_3166_1\": \"US\", \"name\": \"United States o...   2009-03-20   \n",
       "\n",
       "       runtime                                   spoken_languages  \\\n",
       "2770 -0.439008           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]   \n",
       "208   0.439008           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]   \n",
       "650   0.292672           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]   \n",
       "2251  0.780459  [{\"iso_639_1\": \"ru\", \"name\": \"P\\u0443\\u0441\\u0...   \n",
       "979   0.000000           [{\"iso_639_1\": \"en\", \"name\": \"English\"}]   \n",
       "\n",
       "             ...          vote_average  vote_count  metacritic_metascore  \\\n",
       "2770         ...                   5.2   -0.234094             -0.765790   \n",
       "208          ...                   6.6    1.467792             -0.218797   \n",
       "650          ...                   6.0    0.011291             -0.984588   \n",
       "2251         ...                   6.9    0.064733             -0.054699   \n",
       "979          ...                   6.5    0.121940              0.820490   \n",
       "\n",
       "      release_year  Release Date Production Budget  Domestic Gross  \\\n",
       "2770          2016     3/11/2016         -0.379488       -0.259943   \n",
       "208           2013     11/1/2013          2.110903        0.507815   \n",
       "650           1998      4/3/1998          0.925002        0.083828   \n",
       "2251          1998     9/11/1998         -0.213462       -0.064531   \n",
       "979           2009     3/20/2009          0.450642        0.659335   \n",
       "\n",
       "      Worldwide Gross  num_year  log Worldwide Gross  \n",
       "2770        -0.185281      2016            -0.636367  \n",
       "208          0.479793      2013             0.481812  \n",
       "650         -0.056551      1998            -0.117150  \n",
       "2251        -0.113356      1998            -0.277905  \n",
       "979          0.278349      2009             0.337437  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#splitting data set into training and test data set in 0.7/0.3\n",
    "train, test = train_test_split(critic_revenue,test_size=0.30)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Fill the training and test data with require information\n",
    "X_train = train[features].dropna()\n",
    "y_train = train[target].dropna()\n",
    "X_test = test[features].dropna()\n",
    "y_test = test[target].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tatenda/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
       "             learning_rate=0.1, loss='ls', max_depth=3, max_features=None,\n",
       "             max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "             min_impurity_split=None, min_samples_leaf=1,\n",
       "             min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "             n_estimators=100, presort='auto', random_state=None,\n",
       "             subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import ensemble\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "model = ensemble.GradientBoostingRegressor()\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting R squared\": -0.0242\n"
     ]
    }
   ],
   "source": [
    "print('Gradient Boosting R squared\": %.4f' % model.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting RMSE: 1.0098\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "y_pred = model.predict(X_test)\n",
    "model_mse = mean_squared_error(y_pred, y_test)\n",
    "model_rmse = np.sqrt(model_mse)\n",
    "print('Gradient Boosting RMSE: %.4f' % model_rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}