XGBoost_next_row.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bukosabino/envs/deeplearning/local/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import *\n",
    "import xgboost as xgb\n",
    "import operator\n",
    "\n",
    "import settings\n",
    "import utils\n",
    "import get_data\n",
    "from ta import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Get Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "API: http://bitcoincharts.com/charts\n",
    "\n",
    "period = ['1-min', '5-min', '15-min', '30-min', 'Hourly', '2-hour', '6-hour', '12-hour', 'Daily', 'Weekly']\n",
    "\n",
    "market = ['krakenEUR', 'bitstampUSD'] -> list of markets: https://bitcoincharts.com/charts/volumepie/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# get_data.get('data/datas.csv', period=settings.PERIOD, market=settings.MARKET)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/datas.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# add next row\n",
    "last_timestamp = df['Timestamp'].iloc[-1]\n",
    "if settings.PERIOD == 'Hourly':\n",
    "    next_timestamp = last_timestamp + 3600\n",
    "df_next = pd.DataFrame([next_timestamp], columns=['Timestamp'])\n",
    "df = df.append(df_next, ignore_index=True)\n",
    "df.iloc[-1] = df.iloc[-1].fillna(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 51212, Number of columns: 8\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df = utils.dropna(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 45461, Number of columns: 8\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Create column target with class [UP, KEEP, DOWN]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df['Target'] = 0 # 'KEEP'\n",
    "df.loc[df.Open + (df.Open * settings.PERCENT_UP) < df.Close, 'Target'] = 1 # 'UP'\n",
    "df.loc[df.Open - (df.Open * settings.PERCENT_DOWN) > df.Close, 'Target'] = 2 # 'DOWN'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 45461, Number of columns: 9\n",
      "Number of UP rows: 3301, Number of DOWN rows: 3115\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))\n",
    "print('Number of UP rows: {}, Number of DOWN rows: {}'.format(len(df[df.Target == 1]), len(df[df.Target == 2])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Create columns from Timestamp to Date, Year, Month, Hour, etc."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df['Date'] = df['Timestamp'].apply(utils.timestamptodate)\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "\n",
    "df['Year'] = df['Date'].dt.year\n",
    "df['Month'] = df['Date'].dt.month\n",
    "df['Week'] = df['Date'].dt.weekofyear\n",
    "df['Weekday'] = df['Date'].dt.weekday\n",
    "df['Day'] = df['Date'].dt.day\n",
    "df['Hour'] = df['Date'].dt.hour\n",
    "\n",
    "# extra dates\n",
    "# df[\"yearmonth\"] = df[\"Date\"].dt.year*100 + df[\"Date\"].dt.month\n",
    "# df[\"yearweek\"] = df[\"Date\"].dt.year*100 + df[\"Date\"].dt.weekofyear\n",
    "# df[\"yearweekday\"] = df[\"Date\"].dt.year*10 + df[\"Date\"].dt.weekday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# shift\n",
    "cols = ['Open', 'High', 'Low', 'Close', 'Volume_BTC', 'Volume_Currency', 'Weighted_Price']\n",
    "for col in cols:\n",
    "    df[col] = df[col].shift(1)\n",
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "df['High-low'] = df['High'] - df['Low']\n",
    "df['Close-open'] = df['Close'] - df['Open']\n",
    "\n",
    "df['Up_or_Down'] = 0 # 'UP' or 'DOWN' if diff > settings.PERCENT_UP\n",
    "df.loc[( df.Open + (df.Open * settings.PERCENT_UP) ) < df.Close, 'Up_or_Down'] = 1 # 'UP'\n",
    "df.loc[( df.Open - (df.Open * settings.PERCENT_DOWN) ) > df.Close, 'Up_or_Down'] = 2 # 'DOWN'\n",
    "\n",
    "df['Up_or_Down_2'] = 0 # 'UP' or 'DOWN' if diff > settings.PERCENT_UP * 2\n",
    "df.loc[df.Open + (df.Open * settings.PERCENT_UP * 2 ) < df.Close, 'Up_or_Down_2'] = 1 # 'UP'\n",
    "df.loc[df.Open - (df.Open * settings.PERCENT_DOWN * 2) > df.Close, 'Up_or_Down_2'] = 2 # 'DOWN'\n",
    "\n",
    "df['Up_or_Down_3'] = 0 # 'UP' or 'DOWN' if diff > 0\n",
    "df.loc[df.Open < df.Close, 'Up_or_Down_3'] = 1 # 'UP'\n",
    "df.loc[df.Open  > df.Close, 'Up_or_Down_3'] = 2 # 'DOWN'\n",
    "\n",
    "df['Up_or_Down_4'] = 0 # 'UP' or 'DOWN' if diff > settings.PERCENT_UP / 2\n",
    "df.loc[df.Open + (df.Open * settings.PERCENT_UP / 2 ) < df.Close, 'Up_or_Down_4'] = 1 # 'UP'\n",
    "df.loc[df.Open - (df.Open * settings.PERCENT_DOWN / 2) > df.Close, 'Up_or_Down_4'] = 2 # 'DOWN'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Fundamental analysis\n",
    "\n",
    "# daily return\n",
    "df['Daily_return'] = (df['Close'] / df['Close'].shift(1)) - 1\n",
    "df['Daily_return_100'] = ((df['Close'] / df['Close'].shift(1)) - 1) * 100\n",
    "\n",
    "# cumulative return\n",
    "df = df.dropna()\n",
    "df['Cumulative_return'] = (df['Close'] / df['Close'].iloc[0]) - 1\n",
    "df['Cumulative_return_100'] = ((df['Close'] / df['Close'].iloc[0]) - 1) * 100\n",
    "\n",
    "# TODO: cumulative return week, month, year..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 45459, Number of columns: 26\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Technical Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "https://en.wikipedia.org/wiki/Technical_analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Volume-based indicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ta/volume.py:38: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,span=3,min_periods=2,adjust=True).mean()\n",
      "  return pd.Series(pd.ewma(ad, span=3, min_periods=2) - pd.ewma(ad, span=10, min_periods=9))\n",
      "ta/volume.py:38: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,span=10,min_periods=9,adjust=True).mean()\n",
      "  return pd.Series(pd.ewma(ad, span=3, min_periods=2) - pd.ewma(ad, span=10, min_periods=9))\n",
      "ta/volume.py:48: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=20,center=False).mean()\n",
      "  return pd.Series(pd.rolling_mean(mf, n))\n",
      "ta/volume.py:57: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=20,center=False).sum()\n",
      "  return pd.Series(moments.rolling_sum(clv*df[col_volume], n) / moments.rolling_sum(df[col_volume], n))\n",
      "ta/volume.py:74: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  n_positive_mf = pd.rolling_sum(df['1_Period_Positive_Money_Flow'], n)\n",
      "ta/volume.py:75: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  n_negative_mf = pd.rolling_sum(df['1_Period_Negative_Money_Flow'], n)\n",
      "ta/volume.py:102: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=10,center=False).mean()\n",
      "  return pd.Series(pd.rolling_mean(df['OBV'], n))\n"
     ]
    }
   ],
   "source": [
    "# Accumulation/Distribution index\n",
    "df['Acc_Dist_Roc_BTC'] = acc_dist_roc(df, 'Volume_BTC', 2)\n",
    "df['Acc_Dist_Roc_Currency'] = acc_dist_roc(df, 'Volume_Currency', 2)\n",
    "df['Acc_Dist_BTC'] = acc_dist_index(df, 'Volume_BTC')\n",
    "df['Acc_Dist_Currency'] = acc_dist_index(df, 'Volume_Currency')\n",
    "\n",
    "# Chaikin Money Flow\n",
    "df['Chaikin_Money_Flow_1_BTC'] = chaikin_money_flow1(df, 'Volume_BTC')\n",
    "df['Chaikin_Money_Flow_2_BTC'] = chaikin_money_flow2(df, 'Volume_BTC', 20)\n",
    "df['Chaikin_Money_Flow_3_BTC'] = chaikin_money_flow3(df, 'Volume_BTC', 20)\n",
    "df['Chaikin_Money_Flow_1_Currency'] = chaikin_money_flow1(df, 'Volume_Currency')\n",
    "df['Chaikin_Money_Flow_2_Currency'] = chaikin_money_flow2(df, 'Volume_Currency', 20)\n",
    "df['Chaikin_Money_Flow_3_Currency'] = chaikin_money_flow3(df, 'Volume_Currency', 20)\n",
    "\n",
    "# Money Flow Index\n",
    "df['Money_Flow_BTC'] = money_flow_index(df, 'Volume_BTC', 14)\n",
    "df['Money_Flow_Currency'] = money_flow_index(df, 'Volume_Currency', 14)\n",
    "\n",
    "# On-balance volume\n",
    "df['OBV_BTC'] = on_balance_volume(df, 'Volume_BTC')\n",
    "df['OBV_BTC_mean'] = on_balance_volume_mean(df, 'Volume_BTC')\n",
    "df['OBV_Currency'] = on_balance_volume(df, 'Volume_Currency')\n",
    "df['OBV_Currency_mean'] = on_balance_volume_mean(df, 'Volume_Currency')\n",
    "\n",
    "# Force Index\n",
    "df['Force_Index_BTC'] = force(df, 'Volume_BTC', 2)\n",
    "df['Force_Index_Currency'] = force(df, 'Volume_Currency', 2)\n",
    "\n",
    "# delete intermediate columns\n",
    "df.drop('OBV', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Trend indicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ta/trend.py:27: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,span=12,min_periods=25,adjust=True).mean()\n",
      "  EMAfast = pd.Series(pd.ewma(df['Close'], span=n_fast, min_periods=n_slow - 1))\n",
      "ta/trend.py:28: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,span=26,min_periods=25,adjust=True).mean()\n",
      "  EMAslow = pd.Series(pd.ewma(df['Close'], span=n_slow, min_periods=n_slow - 1))\n",
      "ta/trend.py:30: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,span=9,min_periods=8,adjust=True).mean()\n",
      "  MACDsign = pd.Series(pd.ewma(MACD, span=n_sign, min_periods=n_sign - 1), name='MACD_sign_%d_%d' % (n_fast, n_slow))\n",
      "ta/trend.py:57: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  trs = pd.rolling_sum(tr, n)\n",
      "ta/trend.py:68: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  dip = 100 * pd.rolling_sum(pos, n) / trs\n",
      "ta/trend.py:69: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  din = 100 * pd.rolling_sum(neg, n) / trs\n",
      "ta/trend.py:72: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,min_periods=0,adjust=True,com=14).mean()\n",
      "  adx = pd.ewma(dx, n)\n",
      "ta/trend.py:87: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  trn = moments.rolling_sum(tr, n)\n",
      "ta/trend.py:92: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  vip = moments.rolling_sum(vmp, n) / trn\n",
      "ta/trend.py:93: FutureWarning: pd.rolling_sum is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.rolling(window=14,center=False).sum()\n",
      "  vin = moments.rolling_sum(vmm, n) / trn\n"
     ]
    }
   ],
   "source": [
    "# Moving Average Convergence Divergence\n",
    "df[['MACD', 'MACD_sign', 'MACD_diff']] = macd(df, 12, 26, 9)\n",
    "\n",
    "# Average directional movement index\n",
    "df[['ADX', 'ADX_pos', 'ADX_neg']] = adx(df, 14)\n",
    "\n",
    "# Vortex indicator\n",
    "df[['Vortex_pos', 'Vortex_neg']] = vortex(df, 14)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Momentum Indicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ta/momentum.py:16: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,min_periods=0,adjust=True,com=14).mean()\n",
      "  emaup = pd.ewma(up, n)\n",
      "ta/momentum.py:17: FutureWarning: pd.ewm_mean is deprecated for Series and will be removed in a future version, replace with \n",
      "\tSeries.ewm(ignore_na=False,min_periods=0,adjust=True,com=14).mean()\n",
      "  emadn = pd.ewma(dn, n)\n"
     ]
    }
   ],
   "source": [
    "df['RSI'] = rsi(df, 14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfor c in df.columns:\\n    print str(c) + u' - ' + str(df[c].isnull().sum())\\n\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "for c in df.columns:\n",
    "    print str(c) + u' - ' + str(df[c].isnull().sum())\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Price-based indicators "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Momentum\n",
    "for idx in range(9):\n",
    "    m = idx+2\n",
    "    df['Momentum_'+str(m)] = ((df['Close'] / df['Close'].shift(m)) - 1)\n",
    "\n",
    "# Rollings\n",
    "for idx in range(9):\n",
    "    m = idx+2\n",
    "    df['Rolling_mean_'+str(m)] = (df.set_index('Date')['Close'].rolling(window=m).mean()).values\n",
    "    df['Rolling_std_'+str(m)] = (df.set_index('Date')['Close'].rolling(window=m).std()).values\n",
    "    df['Rolling_cov_'+str(m)] = (df.set_index('Date')['Close'].rolling(window=m).cov()).values\n",
    "\n",
    "# Bollinger bands\n",
    "for idx in range(9):\n",
    "    m = idx+2\n",
    "    df['Bollinger_band_mean_'+str(m)+'_max'] = df['Rolling_mean_'+str(m)] + (2*df['Rolling_std_'+str(m)])\n",
    "    df['Bollinger_band_mean_'+str(m)+'_min'] = df['Rolling_mean_'+str(m)] - (2*df['Rolling_std_'+str(m)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows: 45459, Number of columns: 107\n",
      "Number of rows: 45394, Number of columns: 107\n"
     ]
    }
   ],
   "source": [
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))\n",
    "df = df.dropna()\n",
    "print('Number of rows: {}, Number of columns: {}'.format(*df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train, test = utils.split_df(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "excl = ['Target', 'Date', 'Timestamp']\n",
    "cols = [c for c in df.columns if c not in excl]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-mlogloss:1.09319\ttest-mlogloss:1.09409\n",
      "[50]\ttrain-mlogloss:0.865379\ttest-mlogloss:0.907757\n",
      "[100]\ttrain-mlogloss:0.699335\ttest-mlogloss:0.777195\n",
      "[150]\ttrain-mlogloss:0.57352\ttest-mlogloss:0.68265\n",
      "[200]\ttrain-mlogloss:0.47552\ttest-mlogloss:0.612951\n",
      "[250]\ttrain-mlogloss:0.397623\ttest-mlogloss:0.561003\n",
      "[300]\ttrain-mlogloss:0.334694\ttest-mlogloss:0.522111\n",
      "[350]\ttrain-mlogloss:0.283236\ttest-mlogloss:0.492986\n",
      "[400]\ttrain-mlogloss:0.240706\ttest-mlogloss:0.471339\n",
      "[450]\ttrain-mlogloss:0.20543\ttest-mlogloss:0.455488\n",
      "[500]\ttrain-mlogloss:0.176122\ttest-mlogloss:0.44411\n",
      "[550]\ttrain-mlogloss:0.151703\ttest-mlogloss:0.436222\n",
      "[600]\ttrain-mlogloss:0.131343\ttest-mlogloss:0.431155\n",
      "[650]\ttrain-mlogloss:0.11435\ttest-mlogloss:0.428371\n",
      "[700]\ttrain-mlogloss:0.10007\ttest-mlogloss:0.427287\n",
      "[750]\ttrain-mlogloss:0.0881423\ttest-mlogloss:0.427607\n",
      "714\n",
      "Prediction: KEEP\n",
      "\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " ********** WEIGHT ************\n",
      "('Cumulative_return_100', 27)\n",
      "('Cumulative_return', 390)\n",
      "('Up_or_Down_3', 1087)\n",
      "('Rolling_mean_2', 1236)\n",
      "('Rolling_mean_9', 1365)\n",
      "('Rolling_mean_8', 1404)\n",
      "('Rolling_mean_10', 1497)\n",
      "('Rolling_mean_7', 1518)\n",
      "('Rolling_mean_6', 1564)\n",
      "('Weighted_Price', 1776)\n",
      "('Rolling_mean_5', 1790)\n",
      "('Bollinger_band_mean_8_max', 1803)\n",
      "('Rolling_mean_4', 1926)\n",
      "('Bollinger_band_mean_7_max', 1928)\n",
      "('Rolling_mean_3', 2021)\n",
      "('Bollinger_band_mean_9_max', 2102)\n",
      "('Bollinger_band_mean_6_max', 2259)\n",
      "('Bollinger_band_mean_5_max', 2384)\n",
      "('Bollinger_band_mean_7_min', 2430)\n",
      "('Rolling_cov_9', 2481)\n",
      "('Up_or_Down_2', 2482)\n",
      "('Bollinger_band_mean_8_min', 2532)\n",
      "('Rolling_cov_7', 2584)\n",
      "('Bollinger_band_mean_6_min', 2618)\n",
      "('Bollinger_band_mean_10_max', 2631)\n",
      "('Up_or_Down', 2656)\n",
      "('Bollinger_band_mean_9_min', 2711)\n",
      "('Rolling_cov_8', 2752)\n",
      "('Bollinger_band_mean_4_max', 2929)\n",
      "('Rolling_cov_6', 3001)\n",
      "('Rolling_cov_10', 3189)\n",
      "('Bollinger_band_mean_5_min', 3238)\n",
      "('Bollinger_band_mean_3_max', 3329)\n",
      "('Bollinger_band_mean_10_min', 3380)\n",
      "('Bollinger_band_mean_4_min', 3646)\n",
      "('Bollinger_band_mean_3_min', 4040)\n",
      "('Bollinger_band_mean_2_max', 4308)\n",
      "('Up_or_Down_4', 4339)\n",
      "('Bollinger_band_mean_2_min', 4511)\n",
      "('Open', 4651)\n",
      "('Rolling_cov_5', 4746)\n",
      "('OBV_Currency', 4990)\n",
      "('Rolling_cov_4', 5304)\n",
      "('Rolling_cov_3', 6211)\n",
      "('Low', 6276)\n",
      "('Daily_return_100', 8621)\n",
      "('OBV_Currency_mean', 8763)\n",
      "('Year', 10316)\n",
      "('High', 10402)\n",
      "('Rolling_cov_2', 10487)\n",
      "('Acc_Dist_Roc_Currency', 14673)\n",
      "('Chaikin_Money_Flow_3_Currency', 17320)\n",
      "('Rolling_std_9', 23939)\n",
      "('Rolling_std_8', 24036)\n",
      "('Rolling_std_7', 24640)\n",
      "('Month', 27944)\n",
      "('Rolling_std_10', 29021)\n",
      "('Rolling_std_6', 29164)\n",
      "('Rolling_std_5', 32027)\n",
      "('Weekday', 37152)\n",
      "('Money_Flow_Currency', 37467)\n",
      "('Force_Index_Currency', 39773)\n",
      "('Rolling_std_4', 39775)\n",
      "('Week', 40557)\n",
      "('Chaikin_Money_Flow_1_Currency', 41247)\n",
      "('Rolling_std_3', 42046)\n",
      "('Acc_Dist_Currency', 43287)\n",
      "('Chaikin_Money_Flow_2_Currency', 44701)\n",
      "('Rolling_std_2', 46386)\n",
      "('Close', 47521)\n",
      "('OBV_BTC', 47541)\n",
      "('Volume_Currency', 50124)\n",
      "('High-low', 57275)\n",
      "('Close-open', 60216)\n",
      "('MACD', 61742)\n",
      "('MACD_sign', 62046)\n",
      "('RSI', 63136)\n",
      "('Chaikin_Money_Flow_1_BTC', 63638)\n",
      "('Chaikin_Money_Flow_3_BTC', 65609)\n",
      "('Day', 66716)\n",
      "('Vortex_pos', 67091)\n",
      "('Vortex_neg', 68111)\n",
      "('Hour', 69583)\n",
      "('Force_Index_BTC', 69640)\n",
      "('Momentum_8', 69921)\n",
      "('MACD_diff', 70665)\n",
      "('Momentum_9', 71557)\n",
      "('Chaikin_Money_Flow_2_BTC', 71923)\n",
      "('Momentum_6', 72125)\n",
      "('Acc_Dist_BTC', 72244)\n",
      "('Money_Flow_BTC', 72843)\n",
      "('Momentum_7', 73485)\n",
      "('Momentum_10', 73496)\n",
      "('Momentum_5', 75876)\n",
      "('Momentum_2', 78745)\n",
      "('ADX_pos', 79034)\n",
      "('ADX_neg', 80115)\n",
      "('Momentum_3', 81444)\n",
      "('Momentum_4', 82088)\n",
      "('OBV_BTC_mean', 83296)\n",
      "('Volume_BTC', 84223)\n",
      "('Daily_return', 84516)\n",
      "('ADX', 99026)\n",
      "('Acc_Dist_Roc_BTC', 101354)\n",
      "\n",
      " \n",
      " \n",
      " \n",
      " \n",
      " \n",
      " ********** GAIN ************\n"
     ]
    }
   ],
   "source": [
    "y_train = train['Target']\n",
    "y_mean = np.mean(y_train)\n",
    "xgb_params = {\n",
    "    'n_trees': 800,\n",
    "    'eta': 0.0045,\n",
    "    'max_depth': 20,\n",
    "    'subsample': 0.95,\n",
    "    'colsample_bytree': 0.95,\n",
    "    'colsample_bylevel': 0.95,\n",
    "    'objective': 'multi:softmax',\n",
    "    'num_class' : 3,\n",
    "    'eval_metric': 'mlogloss', # 'merror', # 'rmse',\n",
    "    'base_score': 0,\n",
    "    'silent': 1\n",
    "}\n",
    "\n",
    "dtrain = xgb.DMatrix(train[cols], y_train)\n",
    "dtest = xgb.DMatrix(test[cols])\n",
    "\n",
    "cv_result = xgb.cv(xgb_params, dtrain)\n",
    "\n",
    "# xgboost, cross-validation\n",
    "cv_result = xgb.cv(xgb_params,\n",
    "                   dtrain,\n",
    "                   num_boost_round=5000,\n",
    "                   early_stopping_rounds=50,\n",
    "                   verbose_eval=50,\n",
    "                   show_stdv=False\n",
    "                  )\n",
    "num_boost_rounds = len(cv_result)\n",
    "\n",
    "# num_boost_rounds = 1000\n",
    "\n",
    "print(num_boost_rounds)\n",
    "\n",
    "# train\n",
    "model = xgb.train(xgb_params, dtrain, num_boost_round=num_boost_rounds)\n",
    "\n",
    "# predict\n",
    "y_pred = model.predict(dtest)\n",
    "y_true = test['Target']\n",
    "\n",
    "prediction_value = y_true.tolist()[0]\n",
    "\n",
    "if prediction_value == 1.0:\n",
    "    print(\"Prediction: UP\")\n",
    "elif prediction_value == 2.0:\n",
    "    print(\"Prediction: DOWN\")\n",
    "else: # 0.0\n",
    "    print(\"Prediction: KEEP\")\n",
    "\n",
    "print \"\\n \\n \\n \\n \\n \\n ********** WEIGHT ************\"\n",
    "importance = model.get_fscore()\n",
    "importance = sorted(importance.items(), key=operator.itemgetter(1))\n",
    "for i in importance:\n",
    "    print i\n",
    "    \n",
    "print \"\\n \\n \\n \\n \\n \\n ********** GAIN ************\"\n",
    "importance = model.get_score(fmap='', importance_type='gain')\n",
    "importance = sorted(importance.items(), key=operator.itemgetter(1))\n",
    "for i in importance:\n",
    "    print i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}