preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn import preprocessing\n",
    "# import seaborn as sns\n",
    "# sns.set()\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy import inf\n",
    "from scipy.linalg import svd\n",
    "from sklearn.decomposition import TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in data\n",
    "season_df = pd.read_pickle(\"./season_df.pkl\")\n",
    "\n",
    "# delete duplicate data\n",
    "del season_df['home_team'], season_df['away_team']\n",
    "del season_df['home_goals_against'], season_df['home_shots_against'], season_df['home_power_play_goals_against']\n",
    "del season_df['away_goals_against'], season_df['away_shots_against'], season_df['away_power_play_goals_against']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turning stats into differentials and percentages\n",
    "main_df = pd.DataFrame()\n",
    "# home\n",
    "home_shots_in_pct = (season_df['home_shots_for'] / season_df['home_goals_for'])\n",
    "away_shots_in_pct = (season_df['away_shots_for'] / season_df['away_goals_for'])\n",
    "main_df['home_shots_in_pct'] = home_shots_in_pct\n",
    "main_df['home_pp_goals_pct'] = season_df['home_power_play_goals_for'] /season_df['home_power_play_opportunities']\n",
    "main_df['home_pp_pk_pct'] = season_df['home_power_play_percent'] / season_df['home_penalty_kill_percent']  \n",
    "main_df['home_power_play_percent'] = season_df['home_power_play_percent']\n",
    "main_df['home_penalty_kill_percent'] = season_df['home_penalty_kill_percent']\n",
    "main_df['home_faceoff_win_pct'] = season_df['home_faceoff_win_pct']\n",
    "main_df['home_faceoff_win_diff'] = season_df['home_faceoff_win_pct'] - season_df['away_faceoff_win_pct']\n",
    "main_df['home_shots_diff'] = season_df['home_shots_for'] - season_df['away_shots_for']\n",
    "main_df['home_power_play_goals_diff'] =  season_df['home_power_play_goals_for'] - season_df['away_power_play_goals_for']\n",
    "main_df['home_pp_opportunity_diff'] = season_df['home_power_play_opportunities'] - season_df['away_power_play_opportunities']\n",
    "main_df['home_shots_in_diff'] = home_shots_in_pct - away_shots_in_pct\n",
    "main_df['home_shots_missed_diff'] = (season_df['home_shots_for']- season_df['home_goals_for']) - (season_df['away_shots_for']- season_df['away_goals_for'])\n",
    "main_df['home_home_away'] = season_df['home_home/away']\n",
    "main_df['home_outcome'] = season_df['home_outcome']\n",
    "# away\n",
    "main_df['away_shots_in_pct'] = away_shots_in_pct\n",
    "main_df['away_pp_goals_pct'] = season_df['away_power_play_goals_for'] /season_df['away_power_play_opportunities']                      \n",
    "main_df['away_pp_pk_pct'] = season_df['away_power_play_percent'] / season_df['away_penalty_kill_percent']\n",
    "main_df['away_power_play_percent'] = season_df['away_power_play_percent']\n",
    "main_df['away_penalty_kill_percent'] = season_df['away_penalty_kill_percent']\n",
    "main_df['away_faceoff_win_pct'] = season_df['away_faceoff_win_pct']\n",
    "main_df['away_faceoff_win_diff'] = season_df['away_faceoff_win_pct'] - season_df['home_faceoff_win_pct']\n",
    "main_df['away_shots_diff'] = season_df['away_shots_for'] - season_df['home_shots_for']\n",
    "main_df['away_power_play_goals_diff'] =  season_df['away_power_play_goals_for'] - season_df['home_power_play_goals_for']\n",
    "main_df['away_pp_opportunity_diff'] = season_df['away_power_play_opportunities'] - season_df['home_power_play_opportunities']\n",
    "main_df['away_shots_in_diff'] = away_shots_in_pct - home_shots_in_pct\n",
    "main_df['away_shots_missed_diff'] = (season_df['away_shots_for']- season_df['away_goals_for']) - (season_df['home_shots_for']- season_df['home_goals_for'])\n",
    "main_df['away_home_away'] = season_df['away_home/away']\n",
    "main_df['away_outcome'] = season_df['away_outcome']\n",
    "\n",
    "# cleaning up data \n",
    "main_df.replace([np.inf, -np.inf], np.nan,inplace=True)\n",
    "main_df.fillna(value=-99999, inplace=True)\n",
    "# filling in target data\n",
    "train_labels = main_df[\"home_outcome\"][:1000]\n",
    "valid_labels = main_df[\"home_outcome\"][1000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mean subtraction\n",
    "main_df -= main_df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardization\n",
    "x = main_df.values\n",
    "scaled_values = preprocessing.RobustScaler().fit_transform(x) \n",
    "main_df.loc[:,:] = scaled_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# value decomposition ????"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# balancing ?????"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 26)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# whitening???"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data split roughly 80/20 and separation of target \n",
    "training = main_df[:1000]\n",
    "del training['home_outcome'], training['away_outcome']\n",
    "\n",
    "validation = main_df[1000:]\n",
    "del validation['home_outcome'], validation['away_outcome']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save\n",
    "training.to_pickle(\"./training.pkl\",'infer',-1)\n",
    "train_labels.to_pickle(\"./train_labels.pkl\", 'infer', -1)\n",
    "validation.to_pickle(\"./validation.pkl\",'infer',-1)\n",
    "valid_labels.to_pickle(\"./valid_labels.pkl\", 'infer', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}