Data Training using TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Training using TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python import debug as tf_debug\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = pd.read_csv('./data/train_processed.csv')\n",
    "X_predict = pd.read_csv('./data/test_processed.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>UnknownAge</th>\n",
       "      <th>Baby</th>\n",
       "      <th>Child</th>\n",
       "      <th>Young</th>\n",
       "      <th>...</th>\n",
       "      <th>Ticket_SOTONOQ.1</th>\n",
       "      <th>Ticket_SP.1</th>\n",
       "      <th>Ticket_STONO.1</th>\n",
       "      <th>Ticket_STONO2.1</th>\n",
       "      <th>Ticket_STONOQ.1</th>\n",
       "      <th>Ticket_SWPP.1</th>\n",
       "      <th>Ticket_WC.1</th>\n",
       "      <th>Ticket_WEP.1</th>\n",
       "      <th>Ticket_XXX.1</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.273456</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014151</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.473882</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.139136</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.323563</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.015469</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.436302</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.103644</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.436302</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.015713</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex       Age  SibSp  Parch      Fare  UnknownAge  Baby  Child  \\\n",
       "0       3    1  0.273456      1      0  0.014151           0     0      0   \n",
       "1       1    0  0.473882      1      0  0.139136           0     0      0   \n",
       "2       3    0  0.323563      0      0  0.015469           0     0      0   \n",
       "3       1    0  0.436302      1      0  0.103644           0     0      0   \n",
       "4       3    1  0.436302      0      0  0.015713           0     0      0   \n",
       "\n",
       "   Young    ...     Ticket_SOTONOQ.1  Ticket_SP.1  Ticket_STONO.1  \\\n",
       "0      1    ...                    0            0               0   \n",
       "1      0    ...                    0            0               0   \n",
       "2      1    ...                    0            0               0   \n",
       "3      0    ...                    0            0               0   \n",
       "4      0    ...                    0            0               0   \n",
       "\n",
       "   Ticket_STONO2.1  Ticket_STONOQ.1  Ticket_SWPP.1  Ticket_WC.1  Ticket_WEP.1  \\\n",
       "0                0                0              0            0             0   \n",
       "1                0                0              0            0             0   \n",
       "2                1                0              0            0             0   \n",
       "3                0                0              0            0             0   \n",
       "4                0                0              0            0             0   \n",
       "\n",
       "   Ticket_XXX.1  Survived  \n",
       "0             0         0  \n",
       "1             0         1  \n",
       "2             0         1  \n",
       "3             1         1  \n",
       "4             1         0  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>UnknownAge</th>\n",
       "      <th>Baby</th>\n",
       "      <th>Child</th>\n",
       "      <th>Young</th>\n",
       "      <th>...</th>\n",
       "      <th>Ticket_SOTONO2.1</th>\n",
       "      <th>Ticket_SOTONOQ.1</th>\n",
       "      <th>Ticket_SP.1</th>\n",
       "      <th>Ticket_STONO.1</th>\n",
       "      <th>Ticket_STONO2.1</th>\n",
       "      <th>Ticket_STONOQ.1</th>\n",
       "      <th>Ticket_SWPP.1</th>\n",
       "      <th>Ticket_WC.1</th>\n",
       "      <th>Ticket_WEP.1</th>\n",
       "      <th>Ticket_XXX.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.430039</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.015282</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.586622</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.013663</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.774521</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.018909</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.336089</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.016908</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.273456</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.023984</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 105 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass  Sex       Age  SibSp  Parch      Fare  UnknownAge  Baby  Child  \\\n",
       "0       3    1  0.430039      0      0  0.015282           0     0      0   \n",
       "1       3    0  0.586622      1      0  0.013663           0     0      0   \n",
       "2       2    1  0.774521      0      0  0.018909           0     0      0   \n",
       "3       3    1  0.336089      0      0  0.016908           0     0      0   \n",
       "4       3    0  0.273456      1      1  0.023984           0     0      0   \n",
       "\n",
       "   Young      ...       Ticket_SOTONO2.1  Ticket_SOTONOQ.1  Ticket_SP.1  \\\n",
       "0      0      ...                      0                 0            0   \n",
       "1      0      ...                      0                 0            0   \n",
       "2      0      ...                      0                 0            0   \n",
       "3      1      ...                      0                 0            0   \n",
       "4      1      ...                      0                 0            0   \n",
       "\n",
       "   Ticket_STONO.1  Ticket_STONO2.1  Ticket_STONOQ.1  Ticket_SWPP.1  \\\n",
       "0               0                0                0              0   \n",
       "1               0                0                0              0   \n",
       "2               0                0                0              0   \n",
       "3               0                0                0              0   \n",
       "4               0                0                0              0   \n",
       "\n",
       "   Ticket_WC.1  Ticket_WEP.1  Ticket_XXX.1  \n",
       "0            0             0             1  \n",
       "1            0             0             1  \n",
       "2            0             0             1  \n",
       "3            0             0             1  \n",
       "4            0             0             1  \n",
       "\n",
       "[5 rows x 105 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_predict.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split into 3 sets: train, test and valid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = X_train['Survived']\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train.drop('Survived', axis=1), y_train, test_size=0.2)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (569, 105) (569,)\n",
      "Validation set (179, 105) (179,)\n",
      "Test set (143, 105) (143,)\n"
     ]
    }
   ],
   "source": [
    "print('Training set', X_train.shape, y_train.shape)\n",
    "print('Validation set', X_valid.shape, y_valid.shape)\n",
    "print('Test set', X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reformat(X, y):\n",
    "    #X = X.reshape((-1, -1)).astype(np.float32)\n",
    "    y = (np.arange(1) == y[:,None]).astype(np.float32)\n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, y_train = reformat(X_train, y_train)\n",
    "X_valid, y_valid = reformat(X_valid, y_valid)\n",
    "X_test, y_test = reformat(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (569, 105) (569, 1)\n",
      "Validation set (179, 105) (179, 1)\n",
      "Test set (143, 105) (143, 1)\n"
     ]
    }
   ],
   "source": [
    "print('Training set', X_train.shape, y_train.shape)\n",
    "print('Validation set', X_valid.shape, y_valid.shape)\n",
    "print('Test set', X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  correct_pred = tf.equal(tf.round(predictions), labels)\n",
    "  return tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "_startLearningRate = 0.5\n",
    "_learningDecayRate = 0.98\n",
    "_decaySteps = 1000\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    \n",
    "    hidden_units = 100\n",
    "    is_training = tf.Variable(True, dtype=tf.bool)\n",
    "#     learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(None, X_train.shape[1]))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(None, y_train.shape[1]))\n",
    "    tf_valid_dataset = tf.cast(tf.constant(X_valid.values), tf.float32)\n",
    "    #tf_test_dataset = tf.cast(tf.constant(X_test.values), tf.float32)\n",
    "    \n",
    "    initializer = tf.contrib.layers.xavier_initializer()\n",
    "    fc = tf.layers.dense(tf_train_dataset, hidden_units, activation=None, kernel_initializer=initializer)\n",
    "    fc = tf.layers.batch_normalization(fc, training=is_training)\n",
    "    fc = tf.nn.relu(fc)\n",
    "    \n",
    "    logits = tf.layers.dense(fc, 1, activation=None)    \n",
    "\n",
    "    #weights = tf.Variable(tf.truncated_normal([X_train.shape[1], y_train.shape[1]]))\n",
    "#     weights = tf.Variable(tf.truncated_normal([X_train.shape[1], 200]))\n",
    "#     biases = tf.Variable(tf.zeros([200]))\n",
    "#     biases = tf.Variable(tf.constant(0.1, shape=[200]))\n",
    "    \n",
    "    #weights1 = tf.Variable(tf.truncated_normal([y_train.shape[1], y_train.shape[1]]))\n",
    "#     weights1 = tf.Variable(tf.truncated_normal([200, y_train.shape[1]]))\n",
    "#     biases1 = tf.Variable(tf.zeros([y_train.shape[1]]))\n",
    "#     biases1 = tf.Variable(tf.constant(0.1, shape=[y_train.shape[1]]))\n",
    "\n",
    "    #logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "#     hidden = tf.nn.relu(tf.matmul(tf_train_dataset, weights) + biases)   \n",
    "#     hidden = tf.nn.dropout(hidden, 0.5)\n",
    "#     logits = tf.nn.relu(tf.matmul(hidden, weights1) + biases1)\n",
    "\n",
    "    #test = tf.reduce_mean(tf.nn.softmax(logits))\n",
    "    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "#     loss = tf.reduce_mean(tf.nn.softmax(logits))\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(_startLearningRate, global_step, _decaySteps, _learningDecayRate)\n",
    "    \n",
    "    with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "\n",
    "#     optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "    #train_prediction = tf.nn.softmax(logits)\n",
    "    #valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    #test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)\n",
    "    train_prediction = tf.nn.sigmoid(logits)\n",
    "    #valid_prediction = tf.nn.sigmoid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 0.833543\n",
      "Minibatch accuracy: 0.4062\n",
      "Valid accuracy: 0.6145\n",
      "Minibatch loss at step 100: 0.417779\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.6313\n",
      "Minibatch loss at step 200: 0.485480\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 300: 0.371980\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 400: 0.272479\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 500: 0.282525\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 600: 0.235736\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 700: 0.198316\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7039\n",
      "Minibatch loss at step 800: 0.295181\n",
      "Minibatch accuracy: 0.8750\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 900: 0.507925\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.6927\n",
      "Minibatch loss at step 1000: 0.178465\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7207\n",
      "Minibatch loss at step 1100: 0.156280\n",
      "Minibatch accuracy: 0.9531\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 1200: 0.213141\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 1300: 0.188523\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 1400: 0.188021\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7207\n",
      "Minibatch loss at step 1500: 0.208351\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 1600: 0.184401\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 1700: 0.278223\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.6872\n",
      "Minibatch loss at step 1800: 0.220254\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 1900: 0.244136\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 2000: 0.392548\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 2100: 0.213153\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 2200: 0.231944\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 2300: 0.424044\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 2400: 0.157313\n",
      "Minibatch accuracy: 0.9688\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 2500: 0.237575\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 2600: 0.401711\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 2700: 0.153762\n",
      "Minibatch accuracy: 0.9688\n",
      "Valid accuracy: 0.7709\n",
      "Minibatch loss at step 2800: 0.234729\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 2900: 0.419976\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 3000: 0.284710\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 3100: 0.231149\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 3200: 0.436317\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 3300: 0.229566\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 3400: 0.183966\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 3500: 0.471404\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 3600: 0.290327\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 3700: 0.260401\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 3800: 0.525531\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 3900: 0.292318\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 4000: 0.311180\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 4100: 0.535756\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 4200: 0.337430\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 4300: 0.461121\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 4400: 0.504254\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 4500: 0.397604\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 4600: 0.553356\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 4700: 0.411347\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 4800: 0.426848\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 4900: 0.676343\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 5000: 0.479018\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 5100: 0.403227\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 5200: 0.374901\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 5300: 0.348762\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 5400: 0.464826\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 5500: 0.331267\n",
      "Minibatch accuracy: 0.8750\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 5600: 0.266320\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 5700: 0.392397\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 5800: 0.265263\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 5900: 0.240776\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 6000: 0.368325\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.8045\n",
      "Minibatch loss at step 6100: 0.224925\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 6200: 0.295325\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 6300: 0.295727\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 6400: 0.216485\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 6500: 0.379783\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 6600: 0.312730\n",
      "Minibatch accuracy: 0.8750\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 6700: 0.216776\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 6800: 0.287899\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 6900: 0.416491\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 7000: 0.292636\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 7100: 0.343877\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 7200: 0.378260\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 7300: 0.217932\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 7400: 0.345579\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 7500: 0.464721\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 7600: 0.226111\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 7700: 0.292099\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 7800: 0.398861\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 7900: 0.244738\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 8000: 0.275352\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 8100: 0.496705\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 8200: 0.257599\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 8300: 0.256561\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 8400: 0.487635\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 8500: 0.260130\n",
      "Minibatch accuracy: 0.8906\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 8600: 0.276767\n",
      "Minibatch accuracy: 0.9219\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 8700: 0.524392\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 8800: 0.271680\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 8900: 0.267033\n",
      "Minibatch accuracy: 0.9062\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 9000: 0.561952\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 9100: 0.273461\n",
      "Minibatch accuracy: 0.9375\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 9200: 0.341789\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 9300: 0.479743\n",
      "Minibatch accuracy: 0.8125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 9400: 0.405515\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 9500: 0.481126\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 9600: 0.397415\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 9700: 0.427340\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 9800: 0.516559\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 9900: 0.448885\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 10000: 0.388881\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7877\n"
     ]
    }
   ],
   "source": [
    "num_steps = 10001\n",
    "report_interval = 100\n",
    "x_collect = []\n",
    "train_loss_collect = []\n",
    "train_acc_collect = []\n",
    "valid_loss_collect = []\n",
    "valid_acc_collect = []\n",
    "    \n",
    "with tf.Session(graph=graph) as session:\n",
    "    \n",
    "    tf.global_variables_initializer().run()\n",
    "    saver = tf.train.Saver()\n",
    "    #session = tf_debug.LocalCLIDebugWrapperSession(session)\n",
    "    #session.add_tensor_filter(\"has_inf_or_nan\", tf_debug.has_inf_or_nan)\n",
    "    \n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "\n",
    "        offset = (step * batch_size) % (X_train.shape[0] - batch_size)\n",
    "\n",
    "        batch_data = X_train.values[offset:(offset + batch_size), :]\n",
    "        batch_labels = y_train[offset:(offset + batch_size), :]\n",
    "\n",
    "        feed_dict = { \n",
    "            tf_train_dataset: batch_data, \n",
    "            tf_train_labels: batch_labels, \n",
    "            is_training: True\n",
    "        }\n",
    "        \n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "\n",
    "        if (step % report_interval == 0):\n",
    "            x_collect.append(step)\n",
    "            train_loss_collect.append(l)\n",
    "            train_acc_collect.append(accuracy(predictions, batch_labels).eval())\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.4f\" % accuracy(predictions, batch_labels).eval())\n",
    "            #print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), y_valid))\n",
    "            #print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), y_test))\n",
    "            \n",
    "        valid_data = X_valid.values\n",
    "        valid_labels = y_valid\n",
    "        \n",
    "        feed_dict = {\n",
    "            tf_train_dataset: valid_data,\n",
    "            tf_train_labels: valid_labels,\n",
    "            is_training: False\n",
    "        }\n",
    "        \n",
    "        valid_l, valid_predictions = session.run([loss, train_prediction], feed_dict=feed_dict)\n",
    "\n",
    "        if (step % report_interval == 0):\n",
    "            valid_loss_collect.append(valid_l)\n",
    "            valid_acc_collect.append(accuracy(valid_predictions, valid_labels).eval())            \n",
    "            #print(\"Valid loss at step %d: %f\" % (step, valid_l))\n",
    "            print(\"Valid accuracy: %.4f\" % accuracy(valid_predictions, valid_labels).eval())\n",
    "            \n",
    "#         feed_dict = {\n",
    "#             tf_train_dataset: X_predict.values\n",
    "#         }\n",
    "        \n",
    "#         predict_result = session.run([train_prediction], feed_dict=feed_dict)        \n",
    "        \n",
    "    saver.save(session, \"./data/model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOXd//H3d2aSsEOAsMgWNkFFNlNlETcE0dYFiz7W\n2qLVUqvtow9PW/HRn221tna9qFq1VFHcxSLiUhdExQVZgoIFA2XXAIEEEUhISDJz//44JyFIQgJJ\nmDnD53VduWbO/r1nks/cc86dGXPOISIiwReKdwEiItIwFOgiIklCgS4ikiQU6CIiSUKBLiKSJBTo\nIiJJQoEuIpIkFOgiIklCgS4ikiQiR/Ng7du3d5mZmUfzkCIigbd06dIC51xGbesd1UDPzMwkOzv7\naB5SRCTwzGxTXdbTKRcRkSShQBcRSRIKdBGRJKFAFxFJEgp0EZEkoUAXEUkSCnQRkSShQBcRSRKB\nCPSpC6fyzae/Ge8yREQSWiACfcPODXz4+YfxLkNEJKEFItAjoQjlsfJ4lyEiktACE+hRF413GSIi\nCS0wga4euojIodUp0M2sjZn908xWmVmOmQ03s7ZmNtfM1vi36Y1VZLfW3RjSaQjOucY6hIhI4NW1\nh/5X4HXnXH9gEJADTAHmOef6AvP86UZxfdb1LP7hYsyssQ4hIhJ4tQa6mbUGzgAeAXDOlTrnvgIu\nBmb4q80ALmmsIkVEpHZ16aH3BPKBR83sEzN72MyaAx2dc1v9dfKAjtVtbGaTzCzbzLLz8/OPqMgn\nlj/B4IcGU1RadETbi4gcC+oS6BFgKPCgc24IUMTXTq847+R2tSe4nXPTnHNZzrmsjIxav0GpWjuK\nd7B823LKYmVHtL2IyLGgLoGeC+Q65xb50//EC/htZtYZwL/d3jgleqNcAI10ERE5hFoD3TmXB3xh\nZv38WaOBz4CXgIn+vInAnEapEAW6iEhd1PVLon8KPGVmqcB64Bq8F4OZZnYtsAm4vHFKVKCLiNRF\nnQLdObcMyKpm0eiGLad6XVp24Zye51QGu4iIHCwQCXl+3/M5v+/58S5DRCShBeJf/0VEpHaBCPTX\n175O73t7s7pgdbxLERFJWIEI9JLyEtbvXE9xeXG8SxERSViBCHSNchERqZ0CXUQkSSjQRUSSRCAC\nvUPzDlzU7yLaNGkT71JERBJWIMahD+w4kDlXNNonC4iIJIVA9NBFRKR2gQj0ZXnL6Pinjry57s14\nlyIikrACEejOObYXbae4TOPQRURqEohArxjloi+4EBGpWSACPSWcAmjYoojIoQQi0DUOXUSkdoEI\n9FZprfjuyd+lR+se8S5FRCRhBWIceofmHXjy0ifjXYaISEILRA9dRERqF4hAL9hbQLO7m/Hgkgfj\nXYqISMIKRKCHLUxxeTGl0dJ4lyIikrACEega5SIiUjsFuohIkqjTKBcz2wjsAaJAuXMuy8zaAs8B\nmcBG4HLn3M5GKVKBLiJSq8PpoZ/tnBvsnMvyp6cA85xzfYF5/nSjiIQi/OiUHzGk85DGOoSISODV\nZxz6xcBZ/v0ZwLvALfWsp1pmxkPfeqgxdi0ikjTq2kN3wFtmttTMJvnzOjrntvr384CODV5dFTEX\nIxqLNuYhREQCra6BfrpzbjBwPnCjmZ1RdaFzzuGF/kHMbJKZZZtZdn5+/hEX2vJ3LbnlrUZ5AyAi\nkhTqFOjOuc3+7XZgNnAqsM3MOgP4t9tr2Haacy7LOZeVkZFxxIVGQhFdFBUROYRaA93MmptZy4r7\nwFhgBfASMNFfbSLQqF/6qUAXETm0ulwU7QjMNrOK9Z92zr1uZkuAmWZ2LbAJuLzxyvQCXefQRURq\nVmugO+fWA4Oqmb8DGN0YRVVHPXQRkUMLxMfnAtyQdQPHtzs+3mWIiCSswAT6bWfcFu8SREQSWiA+\nywVgz749FJYWxrsMEZGEFZhAH/7IcK5+8ep4lyEikrACE+i6KCoicmgKdBGRJKFAFxFJEgp0EZEk\nEZhhi9cNvY6wheNdhohIwgpMoF89+Op4lyAiktACc8qlYG8BW/dsrX1FEZFjVGAC/eoXr+bCZy6M\ndxkiIgkrMIGui6IiIoemQBcRSRIKdBGRJKFAFxFJEoEZtnjVwKsY3fOofZ+GiEjgBCbQx/UZF+8S\nREQSWmBOueQV5rGqYFW8yxARSViBCfS75t/FqEdHxbsMEZGEFZhAj4QilEXL4l2GiEjCClSga5SL\niEjNAhPoKeEUBbqIyCHUOdDNLGxmn5jZK/50WzOba2Zr/Nv0xitTPXQRkdocTg/9JiCnyvQUYJ5z\nri8wz59uNOP7j2fahdMa8xAiIoFWp0A3s67AN4GHq8y+GJjh358BXNKwpR3olONO4QdDftCYhxAR\nCbS69tCnAr8AYlXmdXTOVXxAeR7QsSEL+7qte7ayMHch0Vi0MQ8jIhJYtQa6mX0L2O6cW1rTOs45\nB7gatp9kZtlmlp2fn3/EhT756ZMMf2Q4JeUlR7wPEZFkVpce+kjgIjPbCDwLnGNmTwLbzKwzgH+7\nvbqNnXPTnHNZzrmsjIyMIy40EvI+pUAXRkVEqldroDvnbnXOdXXOZQJXAG87564CXgIm+qtNBOY0\nWpUo0EVEalOfcej3AGPMbA1wrj/daBToIiKHdliftuicexd417+/Azhqn2erQBcRObTA/Kfo6F6j\nmTlhJulNG/X/l0REAiswn4feK70XvdJ7xbsMEZGEFZge+rbCbcxdN5fC0sJ4lyIikpACE+jvbXqP\nsU+OZeNXG+NdiohIQgpMoOuiqIjIoSnQRUSSROACXZ/lIiJSvcAFunroIiLVC0ygD+k8hNe++xon\nZpwY71JERBJSYMaht2/WnnF9xsW7DBGRhBWYHvrO4p3MzplNXmFevEsREUlIgQn0dTvXcenMS8ne\nkh3vUkREElJgAl0XRUVEDk2BLiKSJBToIiJJQoEuIpIkAjNssUvLLnxwzQcc3+74eJciIpKQAhPo\nTVOaMrL7yHiXISKSsAJzyqW4rJjHlz/OqoJV8S5FRCQhBSbQi8qKmPjiRN5a/1a8SxERSUiBCXRd\nFBUROTQFuohIklCgi4gkiVoD3cyamNliM1tuZivN7Nf+/LZmNtfM1vi36Y1ZqAJdROTQ6jJscR9w\njnOu0MxSgA/M7DXgUmCec+4eM5sCTAFuaaxCwxZm+fXL6dSiU2MdQkQk0GrtoTtPoT+Z4v844GJg\nhj9/BnBJo1ToMzMGdhxIh+YdGvMwIiKBVadz6GYWNrNlwHZgrnNuEdDRObfVXyUP6FjDtpPMLNvM\nsvPz8+tV7N+z/86Hn39Yr32IiCSrOgW6cy7qnBsMdAVONbMBX1vu8Hrt1W07zTmX5ZzLysjIqFex\nk9+czIurXqzXPkREktVhjXJxzn0FvAOMA7aZWWcA/3Z7w5d3oEgoQlmsrLEPIyISSHUZ5ZJhZm38\n+02BMcAq4CVgor/aRGBOYxVZIRKKaJSLiEgN6jLKpTMww8zCeC8AM51zr5jZR8BMM7sW2ARc3oh1\nApASSlGgi4jUoNZAd859CgypZv4OYHRjFFUT9dBFRGoWmI/PBVhw7QKapTSLdxkiIgkpUIHevXX3\neJcgIpKwAvNZLgD/WPoPZufMjncZIiIJKVCBPnXRVJ5e8XS8yxARSUiBCnRdFBURqZkCXUQkSSjQ\nRUSShAJdRCRJBGrY4qtXvkrIAvUaJCJy1AQq0Ns0aRPvEkREElagurszls3ggSUPxLsMEZGEFKhA\nn/nZTKZ/Mj3eZYiIJKRABbouioqI1EyBLiKSJBToIiJJInCBHnXReJchIpKQAjVs8dGLH8WweJch\nIpKQAhXoqeHUeJcgIpKwAnXKZebKmdz61q3xLkNEJCEFKtDf2/Qe//j4H/EuQ0QkIQUq0DXKRUSk\nZgp0EZEkUWugm1k3M3vHzD4zs5VmdpM/v62ZzTWzNf5temMXq0AXEalZXXro5cD/OudOBIYBN5rZ\nicAUYJ5zri8wz59uVKnhVMKhcGMfRkQkkMw5d3gbmM0B7vd/znLObTWzzsC7zrl+h9o2KyvLZWdn\nH3GxIiLHIjNb6pzLqm29wzqHbmaZwBBgEdDRObfVX5QHdDzMGkVEpAHVOdDNrAUwC7jZObe76jLn\ndfOr7eqb2SQzyzaz7Pz8/HoV+9qa17j6xaspjZbWaz8iIsmoToFuZil4Yf6Uc+4Ff/Y2/1QL/u32\n6rZ1zk1zzmU557IyMjLqVezK/JXMWD5DgS4iUo26jHIx4BEgxzn3lyqLXgIm+vcnAnMavrwDRULe\nJxVopIuIyMHq8lkuI4HvAf82s2X+vP8D7gFmmtm1wCbg8sYpcT8FuohIzWoNdOfcB1DjRxyObthy\nDk2BLiJSs0D9p2izlGa0a9qOmIvFuxQRkYRz2OPQ60Pj0EVEDl+jjEMXEZHEFahA//DzD7n0uUvJ\n3Z0b71JERBJOoAJ9y54tzF41m10lu+JdiohIwglUoFeMcimLlcW5EhGRxBOoQE8JpwAatigiUp1A\nBbrGoYuI1CxQgd48pTmZbTIrg11ERPYLVDKO6jGKDTdtiHcZIiIJKVA9dBERqVmgAj0nP4exT4xl\n8ebF8S5FRCThBCrQC0sLmbt+LvlF9fuiDBGRZBSoQNcoFxGRminQRUSShAJdRCRJBCrQm6U0Y0CH\nAbRIbRHvUkREEk6gxqH3aNODf//43/EuQ0QkIQWqhy4iIjULVKDv2LuD4Y8MZ9Zns+JdiohIwglU\noMdcjIW5C9lauDXepYiIJJxABbpGuYiI1EyBLiKSJGoNdDObbmbbzWxFlXltzWyuma3xb9Mbt0yP\nAl1EpGZ16aE/Boz72rwpwDznXF9gnj/d6FLCKQzvOpzjWh53NA4nIhIotY5Dd869Z2aZX5t9MXCW\nf38G8C5wSwPWVa1IKMKCaxc09mFERALpSM+hd3TOVQw1yQM6NlA9IiJyhOp9UdQ55wBX03Izm2Rm\n2WaWnZ9f/4+9PWXaKfx5wZ/rvR8RkWRzpIG+zcw6A/i322ta0Tk3zTmX5ZzLysjIOMLD7be6YLXG\noYuIVONIA/0lYKJ/fyIwp2HKqV0kFNEoFxGRatRl2OIzwEdAPzPLNbNrgXuAMWa2BjjXnz4qFOgi\nItWryyiX79SwaHQD11InCnQRkeoF6j9FAc7ueTb92/ePdxkiIgknUJ+HDvDMt5+JdwkiIgkpcD10\nERGpXuACffTjo7nh1RviXYaISMIJ3CmXbYXbSG9yVD4LTEQkUALXQ9coFxGR6inQRUSShAJdRCRJ\nBO4c+uieo2kSaRLvMkREEk7gAv3u0XfHuwQRkYQUuFMuIiJSvcAF+mXPX8aYJ8bEuwwRkYQTuEDf\nW7aXncU7412GiEjCCVyga5SLiEj1AhfoKaEUBbqISDUCF+jqoYuIVC9wwxbPyjyLzDaZ8S5DRCTh\nBC7Qr8+6Pt4lALAwdyEnZZxEy7SW8S5FRAQI4CmXo6FgbwGTXp5E7u7cape/uOpFhj8ynPOfOp+Y\ni1XO37JnC+c9eR4PZT90tEoV34adG5i/cX68yxCJq8AF+o2v3kiXv3RplH1XnJv/6IuPeHz54/S7\nvx93zr+TfeX7KtfZumcr1710HT3b9GTy8MmEzHsIc/JzGP7IcN5c9yZDOg1plPqOhvJYOdM/mc6M\nZTOY9dks3tv03gEvWhVWF6zm022fsn7n+gMen3goj5XT574+nDXjLNZ9uS6utYjEU+ACHaA0Wtrg\n+ywpL2HUo6O4f/H9XNjvQnJuzOGCvhfwy3d/yfBHhrNmxxpiLsY1c65hb9leXr3yVS494VIA/vjh\nHzn14VPZV76PpZOWclrX0wCY8tYUHlzyIHmFeYddz5LNS3huxXN8vPVj9uzbA4Bzrsb1dxbvZO2X\nayunl+UtO+T6X1fxYhYJRXhmxTNcPedqJjw/gTMfO5MLnrqA/KJ8AIrLirn+levp/7f+DHpoEL3v\n7c2SLUsOu30Nafon04m5GG2btqVpStO41iIST4E7h364o1wW5S7i3Y3vcsWAK+jRpscBy3YW72T+\npvm0TmvNjOUzWJi7kCkjpwDQM70nz1/2PK/85xUmvjiR29+5nVtPv5V3Nr7D1POmckLGCQCs37me\nX7z1CzLbZDLv+/Pold4LgN37dvP62tdZvm05N/7rRoZ1HcaQTkO4pP8ljOk9hh17d/BQ9kPsKd1D\nh+YdGNFtBEM6DSEtkgbAvYvv5clPn6ysNS2cRueWndlw0wYA7px/J7v37aZ5SnPeXP8mizcv5sqT\nr+SJ8U8AMHL6SDLbZDJp6CTG9B5D9pZsxvYeS6cWnVjwxQL+uOCPNIk0YVfJLnaW7GTTV5tYeN1C\nurfuzqzLZ1Gwt4DC0kLmb5zPz+f+nPsW38ctI29h2CPDWLF9BZOHTWZk95EUlhbSr10/AB5c8iAj\nuo1gUKdBR/LUHpHC0kLueOcORnYbyfvXvI+ZHbVjiySaQAb63rK9fPj5h4zsPrLadcqiZTgcqeFU\n5qyew+8++B23v3M73x/4fb5z8nc4ruVxnJhxIjkFOYx/bnzldreNuo2L+198wL6+dfy3WH79cppE\nmtC+WXs+u+GzytAG6JXeiwU/WED/9v1Jb7r/m5RapbXikx99wortK5izeg4vrX6JZ1c+S6/0Xozp\nPYbC0kJuf+d2UkIplMXKAAhZiOwfZjOk8xD+PPbP3HzazWzatYnVBavZWbKTjGYZlftfmb+SOavm\nUBYr4xvHfYPbR91eWbtzjvvPv5+Hlj7EzW/cXLnNE+Of4KqBV7F7327WfbmO4vJiWqe1Jr1pOmN6\nj6GkvKSy9lZprQAY2HEgo3uNpk/bPqSGUxnffzx/GvMnzutz3gGPU2FpIb/94LfkF+Vzx5l3MHn4\n5KPyqZjz1s+jYG8BL17xImbGsrxlPLH8Cf409k8Kdznm2OG8LT9oY7NxwF+BMPCwc+6eQ62flZXl\nsrOzj+xg11wDffrwxrg+XDnvBkZ0G8HL33mZ3ft28+yKZymNlrKtcBsLchewMHchT45/kvEnjGdX\nyS7yCvN4YMkDTPt4GiXlJVx+0uU8N+E5ikqLWL1jNbtKdhEOhTm9++mV58QbW8zFKIuWkRZJY+ue\nrXyU+xELvljAmT3O5MJ+F9ZpH/vK97Evuq8yfKuzLG8Z2VuyObXLqQzoMKBR25dflM/1r17PCzkv\nkNkmk9+f+3suO/GyRg/WL3Z9QbfW3QCYtnQaP3rlR7xw+QuMP2F8zRs5Bw88AP37wymnQJs2Nazm\nqq3fOYfDEXMxwhY+ojZW7ONQz0nMxSqXx1wMw+p0LOccZbEyorFo5WmosqjXcSiNllIaLSXmYjRN\naUqzlGYHbRt10cprJ6nhVIAD3hlX1G4YKeGUyv2bGTEXIxrztk8Np1Yur9hveay88ic1nEqzlGY4\n5ypPpcZcDIeXSymhFFLCKZXtCVmIaCxKSXkJ+6L7aJHaonL7qIseVF/YwoRD4cqaHK6yNoejSaQJ\nkVDkgDZXrBt1UZpGmpIS9v6ZsaJ9Fc9BRftCFqpsT8XjUBYrozxWTtumbYmE6t9vNrOlzrmsWtc7\n0kA3szDwH2AMkAssAb7jnPuspm2OONBLS+HCC+HNNyEtjaKzR/Jl6W66TfoZG887jZ5/7QlAyBmD\nU7pyevogruk9gcGnfBPat4c9e2DtWvLCxczf/Sljjh9H27ZdIeI/0Nu3w0cfweLF3u1nn8GIETB1\nKnTvXn1N0SiEw4fXjmjUO1ZeHqSkwIAB3vzdu6GkxGvn1q2wahW0aAHj/UDasgXS06G8HMrKvOmU\nFOjXzwum2bNh1CjI8HvwRUVe29LS4JNP4LHHYORIOPvs/etUWLIE5syBDRuga1fo0wd69oSzzvL2\nUVAAH37oPTYrV3rLTzsNJkyAimCJRr39vPwyzJvH26N78T/dVpKTn8O+M+ZiTZpw7crf8tQX/yIU\nc4QiKbhwiHZN2vJ5619B585ctfYP/GtXNhaL4dLScAY9LJ1lr/WAb3yDcc1n87ZbD85hKSmELcyg\nsrZ89PeoV3fXrmBGed4WBl21h1UFq4jEvD+67PXnMqi4FdNar+WmbiuwFO+PkOJiyonx2d+gV+tM\n/jgsxq39c7FQqPIPG2Br+c10SmvHL93b3B2bjxcF+/9u9rb7C007dGHyV89xX95LYH6gxGKEzCjr\n8zj07MkP1/yFRze9CFD5YtCKNHa1/zP068d3t97P8xv/5e3dD5huqRl8PngGNG/O+Tm383re+xj7\nA/2EWDtWtpoC6emcufMvfFCUA1BZ+2mtT2Jhj7tg/XoG7/o9y8P5Bzz95zQ9iXlt/hsKC+ld+FvW\nux0HLL+k5/nMPsnbPmP1tRRE9xyw/Hutz+DxS5+AsjKaPHMS+6IHXiD/cehUHhgwhdK+PUmbdfBg\ngSmftuZ3Xw5h55mn0tb+cNDy3+z5Bre9WcKmTk3IHHnwtZqprS7npkXGyvyVDDhjxUHLpw+6g2t2\n9+aj0nWM2HznQcuf7/FzJkT78WZmlPM++NFBy1/fdAbnfZHKrB+OZELOrw9a/uHaMxmR34THfjyM\na5YdvHz5v09nYGo3+NWv4PjjD1peV3UN9Pq8dJwKrHXOrfcP+CxwMVBjoB+x1FR44w0vaO+7j+aL\nFtE8PR0iEbq26srm898i9bvfp/nnW2la+gXwBfAKPPUUXHklLF0KZ59NJ+C/qu53zhy46CJ4/30v\noCIRGDQIRo+GRYugXTtvvZ/9DGbN8pYXFcGuXV5Yfvmlt/x73/P2VaG83AuY//zHm77oIpg/HwoL\nIeaPGBk82Atb8MKz4n6FM87wAt05b938A/8QufZaePhhWLsWvv1tL1y7dPFq2rt3f9vLymDaNLj3\nXm+77t29F5CFC70XhA8+gHvu8erdutV7UQFvH5EI/OY38Ne/evd794bXX/deQC67zFvv3HO9wC8p\n8V7gTjuNc9KH8vGkJ3h59UvYiWcBMPYkyOgMsVYtiY4YTujkgbT4PA+uuw6As4dAeidwBvbNc7Ce\nvWi/KR8K18LUqVw2oIyh3ZtD6zbELptANDWFTrk74ZtAbi6sWQPRKJH+/Zl98TSm5zwNixcTXrOO\njBXroawJJ3fYx09DXXGXTcABrngvke0FtPrfvrByA8NK1zDFdSc2YhRhCxOaOhXbW0zL96ZCGZze\nC6ZcMpjQty7E9u0jPP0xQju+JDx/MkRhdF9ocuXpMGoUoaK92N/+hpWVw6+uAuCC/tDxuvNxg4cQ\n2rKF0PTHaF62Dz78ibf8ZOj+3/+F9eyFrVpFeNZs2pTkw/9dAMBVJ8OpP7sG160rfPwxvPoq7Yt3\nwMKfAfDdU2DU7T/FWrUm9O67pL37AV13r4Tl3sX7G0Y1Y/v/u520Zq1Ife55bPESuuxeCTlekP3k\nrDS+uuMOzIzwU08TWr2GEwpeg5zXAJhycQbFk3/hPe+vvEJo6cecnPse/I93beqXPx1M+bcvJWwh\nQnfeRQhj8MbFsO5SIga/vG0kkTHjCBeXkHLn3URap/ONjEGwcTdpv/0Dd993JXbiAGz1auyxGRgw\nqmAlZI6i9Z587h5+O9G0VMJvzSNt3nzSonDmxpkQOY52Y4Zz1+mXeh2dhx6CzZsxB0MfvBO2Qbex\nw/jNbb+BoiLCDz9COG87AANX/xF2QJ+7JnPX2XdhRXsJvfwy4b0lhIv20n/tF9C0EwNSu/K70b/D\nfb4JN3s2rqSEcDhC9y15QCtOSc30lm/OJWXxUlKLvO27bimE/CWw7+iMBKtPD30CMM45d50//T3g\nNOfcT7623iRgEkD37t1P2bRpU/0qPpRoFLZt83qwBQUwcCAcd5x3/4MPvEDds8cLq7174YoroG9f\n2LHDC8aTT4Zm/ttP5/b3QB95BN57zwu75s2hdWvo3NkLeoDp02HFiv3bRSLeO4NbbvHm/f3vkJMD\nLVt623Xu7PWUTz/dW/7ss14Qp6Z62/XvD716edPRKMyY4fXsU1K8n3btYNgwL2CjUe+P+403vFBr\n3x46dICxY2GI3yMqL/fWefttr862beHnP4du3bzHJByGpk29feXmer31M8/02v/++xAKwdCh3jql\npbB5s9eLB3jwQe+xy8qCceO8dxIVYjF4910v7PftgxNO8F5EKh5X57wXva++2t+2bt0OPv1RWurt\nq0mcvqnKOe8xjEYPrsE573duxw7vOezSxXvuKv6uSkpg40bvMS0q8k7v9OoFxcXe/B49vA7C6tXe\n83fOOd7z+tVX3ryKYxQWei/q553nPX+Fhd78Fi283+kdO7zlQ4d6v38ff+y9o2rXDjp29I5Z9bnJ\nzfX+Tlq18n4vW7TwfvdTvNMjrFkDy5Z563Tv7m3fs6e3foWyMu84n3zibduvn/fuDbzHKxLxalu1\nyttf//5efeC9S+3Uaf++vvrK6yQ1beod8/33vR7twIEHvxPOyfF+5yIRr67jj9//OwXw6afw+efe\n4+Oc95wMHXrgOhs2wPr13mPZtq33N9M0cUdIHY1TLnUK9KrqdQ5dROQYVddAr88Vss1AtyrTXf15\nIiISB/UJ9CVAXzPraWapwBXASw1TloiIHK4jvijqnCs3s58Ab+ANW5zunFvZYJWJiMhhqdcASefc\nv4B/NVAtIiJSD4H8LBcRETmYAl1EJEko0EVEkoQCXUQkSdTrw7kO+2Bm+cCR/qtoe6CgAcsJArX5\n2KA2Hxvq0+YezrmM2lY6qoFeH2aWXZf/lEomavOxQW0+NhyNNuuUi4hIklCgi4gkiSAF+rR4FxAH\navOxQW0+NjR6mwNzDl1ERA4tSD10ERE5hEAEupmNM7PVZrbWzKbEu54jZWbdzOwdM/vMzFaa2U3+\n/LZmNtfM1vi36VW2udVv92ozO6/K/FPM7N/+snstwb8R2czCZvaJmb3iTyd1m82sjZn908xWmVmO\nmQ0/Btr8P/7v9Qoze8bMmiRbm81supltN7MVVeY1WBvNLM3MnvPnLzKzzMMq0DmX0D94n+S4DugF\npALLgRPjXdcRtqUzMNS/3xLvO1lPBP4ATPHnTwF+798/0W9vGtDTfxzC/rLFwDDAgNeA8+Pdvlra\nPhl4Gng/VWSuAAAC20lEQVTFn07qNgMzgOv8+6lAm2RuM9AF2AA09adnAlcnW5uBM4ChwIoq8xqs\njcANwEP+/SuA5w6rvng/QHV4AIcDb1SZvhW4Nd51NVDb5uB9yfZqoLM/rzOwurq24n1U8XB/nVVV\n5n8H+Hu823OIdnYF5gHnVAn0pG0z0NoPN/va/GRucxe8L/Nti/cprq8AY5OxzUDm1wK9wdpYsY5/\nP4L3j0hW19qCcMql4helQq4/L9D8t1JDgEVAR+fcVn9RHtDRv19T27v4978+P1FNBX4BxKrMS+Y2\n9wTygUf900wPm1lzkrjNzrnNwJ+Az4GtwC7n3JskcZuraMg2Vm7jnCsHdgHt6lpIEAI96ZhZC2AW\ncLNzbnfVZc57aU6aoUdm9i1gu3NuaU3rJFub8XpWQ4EHnXNDgCK8t+KVkq3N/nnji/FezI4DmpvZ\nVVXXSbY2VyfebQxCoCfVd5eaWQpemD/lnHvBn73NzDr7yzsD2/35NbV9s3//6/MT0UjgIjPbCDwL\nnGNmT5Lcbc4Fcp1zi/zpf+IFfDK3+Vxgg3Mu3zlXBrwAjCC521yhIdtYuY2ZRfBO3+2oayFBCPSk\n+e5S/0r2I0COc+4vVRa9BEz070/EO7deMf8K/8p3T6AvsNh/e7fbzIb5+/x+lW0SinPuVudcV+dc\nJt5z97Zz7iqSu815wBdm1s+fNRr4jCRuM96plmFm1syvdTSQQ3K3uUJDtrHqvibg/b3Uvccf7wsM\ndbwIcQHeiJB1wG3xrqce7Tgd7+3Yp8Ay/+cCvHNk84A1wFtA2yrb3Oa3ezVVrvYDWcAKf9n9HMaF\nkzi2/yz2XxRN6jYDg4Fs/7l+EUg/Btr8a2CVX+8TeKM7kqrNwDN41wjK8N6JXduQbQSaAM8Da/FG\nwvQ6nPr0n6IiIkkiCKdcRESkDhToIiJJQoEuIpIkFOgiIklCgS4ikiQU6CIiSUKBLiKSJBToIiJJ\n4v8Dsh5X6pdWD1oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12ddc94e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x_collect, train_loss_collect, \"r--\")\n",
    "plt.plot(x_collect, valid_loss_collect, \"g--\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXd4FFX3x783DUILEEINAaQXBemCKChNXhEVUUTFxosN\nf5bX3ogiNmwgKIogCiKKKKCCAQGpitRIDYQaWkiAkEL6nt8fJzc7Ozu72SSbsuF8nifPZmfvztyZ\nnTn33NOuIiIIgiAIFQu/su6AIAiC4H1EuAuCIFRARLgLgiBUQES4C4IgVEBEuAuCIFRARLgLgiBU\nQES4C4IgVEBEuAuCIFRARLgLgiBUQALK6sB16tShpk2bltXhBUEQfJKtW7cmElFYQe3KTLg3bdoU\nW7ZsKavDC4Ig+CRKqaOetBOzjCAIQgVEhLsgCEIFRIS7IAhCBUSEuyAIQgVEhLsgCEIFRIS7IAhC\nBUSEuyAIQgVEhLsgCEIFRIR7RYUISE93/XlmJnDyJJCdXXp9EgSh1BDhXhE5eRLw8wPGjXPdZutW\noFEjYP780uuXIAilhgj3isiPP/LrDz+wBm9FcjK/3ntv6fRJEIRSRYR7ReTCBX5NTQViY63baOFO\nBNhspdMvQRBKDRHuFZGkJPv/UVHWbbRwB4CDB0u2P4IglDoi3IvKzTcD991X1r2wJimJ7enNm7sW\n7lq7B4Do6NLplyAIpYYI96LSvj3w9dfA9u1l3RNnLlwAatYE3ngDGDvWus011/Dnfn4i3AWhAqLI\nlcOthOnatSv5dD33pCSgRQugUydgxQpAqbLukZ1Nm1jADxxYcNvPPgM6dwZ69Cj5fgmCUGyUUluJ\nqGuB7US4F4Fly4BffwUaNgReeQVYuhS44Yay7pU1O3cC584B117ruP3YMXamNmlSNv0SBKFIeCrc\nxSxTFKKigNmzgf/9j7X3Z54BcnPLuld2VqwA9u7l/8eNA5580rnN008DQ4YAKSnAypUcWSMIQoWh\nzJbZ82mio4HLLwcqV2azRk4O266/+QbYt8+x7Q03AH36OG7bvRuIiwMGDy6Z/t15J3DHHcC0acCg\nQcDLLwPx8UC9evY2yclAjRrAxo3cj1WrgH79SqY/QuGJjuYZl/wmJU9KCueEPPBA+TKvFhMR7oWF\nCNixg4UnAPTvb//s55+B336zv8/OBn75hdv7+/O2zEygQwf+PyYGaNXK+/3TDlUAuPpqft2+3XEw\nSU4GQkLYZwCwMPGGIDl1ige7xo2Lv69LGf27FNdsuncvC69u3SqU4PKYOXN4pj13rus2zz4LfP45\n0Lq1/XmpAIhZprDExbEztWNH589+/hnIyrL/nTkDbN5sF+yap57i1xde8H7/Ll5k4RoSwu/r1uXX\n8+cd22nNvV49/vNGxExCAkfhjBlT/H1d6vTs6ag4FJWPP2bz26XKP/+wT8wdLVrwa/v2Jd+fUkSE\ne2E5fRqIiLBrVu4IC2PTTXo6fw8AKlUCPvwQmDCBB4N167zbP53ApDX3WrX49dw5x3YXLrBwB/hc\nduwo/rGnTOGM2JMni7+vS53ERKBOneLvZ8UK4OzZS7eGUFISKzYTJrhuc/Eiv1avXjp9KiVEuBeW\n7t2Bo0eBq67yrL3NxlrYmDHAu++ybQ9gh2ajRsAff3i3fzo5SQv3OnW4SNioUY7tPvjAnoTVsSOw\nZ0/xK0TqhCkpZ1B8YmNZIO/cWbz96Pvh+PHi98kX0crOhx/yjNYKfY0q2AB4aQv3nBz3ZXG9gZ8f\ncNddbIt/6SW7MK9ShR/c11/37vEiItg5qkMf/f05jl1r8JqRI+2O3rFj2bHqV4zbITER0KGtxtIG\nQuExDrKHDhV9P5mZduEWF1e8PvkqWnAnJbGJ1AotA1atKp0+lRKXrnC/cAFo0MBub/OU668H3nuv\ncN/5v/9joRsc7CjMtcDdu5cfRG9QrRo7Ro2RMfPmcVy+JisL2LCBbeQAlyno0sXZN1AY/viDnX89\ne4pwLy7+/nZzXUEat7vZ1pEj9llUedLc27UDHnzQ8/YDBwIDBhTtWDVrAjfdxM7k5cut23z6KStA\n5ekaeYFLV7ivXMna5gMP8PvHHgNefNH9d1JSeHQvrCCuXJltnytX8oBiJCaGwyqnTi3cPl1x4ABP\nL9PS7NsmTQK++ML+Pj6eowIWL+b3Nhswaxbw119FP25UFA9WkyYBH31U9P0IPIPq1QsIDHSvcb/z\nDhAU5FrAN27M9+sVVxRfcB09ygqC8b4qCkSszMya5Vn7nBx+dgoyX37yCUeemaOLlizh+7xbN9d1\nlgCgWTMR7hWG5cvZgfLaa/z+0KGCvera/umJM9VMq1bWKf6tW7Nm8uab7PgqLitWcJy7MSmpdm1H\nh6rWrLVD1c+PZxfaH1AUXn4Z+PZbHjT0gCkUjYMH2Ubs7+9a4CQlAWvX8v/mSChNlSo8i3vkEeDG\nG4vXp2XLgKFDrRPiCkNKiv1/T6qRBgQAw4a5b3P6NPuzDhywO0fN3HQTz2qtkg0nTAAWLeKBtIwy\n9kuCS1O4E/Eoft11rB0BLLD37mWThSt0uKBVGGRxeO89Frhvvln8fWkbow6FBFijNgoALdyNberX\nZ42+qLRowQlb2raZkVH0fV3qbN/OsdedO7PJzIqoKBa4gHMklGblSta2H37YrsQUFS2IXTklPSUr\ni5+7115jU6UndO7Mr65mKEuXAidO8P/Ga5GbC/TtC3z3HSsfy5dbmx5/+IHbhoa6Hhx8kEtTuJ89\nyzfKoEH2bR078rY9e1x/b8cOtuF5O0GnQwe2QU6bVvza6klJHG5ZubJ9W0GaO8DCXYdrFpZlyzg7\nl4gfoO7di+cIvNTRvpCFC1073I0mBlea+0cfce0jgIVWcaKh9KIvZ84UfR8AR2+tXMnnpRUrdzz3\nHLB/P+dPuBK8xmthvs/XrHFUWqyUt+RkXpHsyBGgalWPTsMX8Ei4K6UGK6VilFKxSimnzBulVC2l\n1M9KqX+VUv8opTp4v6tepE4dnoL997/2bVobd5fM06IFR76URKbf66+zsP37b34/aRLw5ZfO7U6f\nBqZPdx1umJRkD4PUaOGup5xauzcK93r1ii7cP/qIp8VK2fdZUk7VY8d8J2Rt8WJg1y73bVautJtX\nNImJ/Boaav0dPfPURd9cae6xsXzP/vEHC61NmzzvuxmtdBQ0u5s9m4WkK/Q9uGsXz07czZQTE4H3\n32eT5po1jjNNTW4umyKbNuX3xmthzvl46SWgTRtn04tO6KtgFCjclVL+AKYBuAFAOwB3KqXamZq9\nBGAHEV0BYDSAyd7uqNdRiu15mlatOHbdqPGaefZZ7zk+zTRowE6ru+7i93Pm8OBjDtX85hu2oc6e\nbb0fK+H+7LO8b02vXjwVNU6Li6O5b9liD6vUD4lxMRBv8p//sE/BaLstj6Sn84Iuw4e7bnPmDGeh\nmit2JiSwIPvxR84w1pq8ZtcuThQbO5bzJazMG7m5wOHDbNbRTvyiOgyJ7MLd3T1y8SJw//3uzYtT\np/I9smkTC253TnwdgWWcYZvZsoVnLg8+yHZ1YyKSFu56UKhTh6+JcQAgYuFus3Em708/uT6Wj+GJ\n5t4dQCwRHSKiLADzAZg9HO0ArAIAItoHoKlSqh7KI5mZLMjnzHHc7u/Psd66ZoyZ7OySr/xonBI+\n8wy/mh/IZ54BrrySNX0ru/akSfYFsjWhoSy89YwjPBwYMcLxQXjtNS5oVljS0vjh0ppTSWvuWqst\n72YfrY1PdqPnuMqa1NmplSuzYDdHzKxeza+jR3My2uWXO+/j+HHWilu0sJsRixPrvmkTMH68+1IG\nWmh27+66zenTPAgMH87PnLsIlqgonnXWrs0atzGcV7NzJytpDz/MM6Wuhkq4Zs09LMyxnwA/Q7Vq\n8QD5++8VauEaT4R7IwDGu+J43jYj0QBuBQClVHcATQCEm3eklBqrlNqilNqSYNZGSov169mrbtZu\nNTabtcf8559ZcMXElGz/NOF5l88s3P38WOM5dozT/c1ERNgLk2liY4HISLvTae9eNgcYqVfPOUzT\nE3T/dH9LWrjrrFpXC3+XF6Ki2PdxzTWu2zzzDJtW/PwcHZUzZ3IeghbK5ntg3DjW3sPDOSrK6lpr\nTbt5c/5NqlcvuuauFN9TkZGOIbVmtO2/enXgq6+sbfzx8TwbqVmTZ5CuhLv23/Tvz1EuMTHW/R8z\nhoW1VamGgAA2t+r6SrVrO/YT4NyTxES27devX6HCIb3lUH0HQE2l1A4AjwPYDsBJzSWiL4ioKxF1\nDdOjaGmzfDk7cqwqIP74I0/htBA0Eh3NmpDWUEsaLSzN2tb99/MAdeONwMSJdk1WM3s28OefjtuO\nHWNNXwvEGTOAW25xbHP0KC+7585eaoW+Vrq/detyBb6SKlX7yivsi+ha4FoFZUtUFM8SBwxwHeLa\npAmH4M2Z4+hDCQ7mwdbdAK+LXIWH252mRvr0YSWmZ09+37hx0TX3zZu5tHVGBgtdV/4erRFv387h\nsDNmOLc5fZqFKMDmlm3brJ20iYl8DQYPtvsezPe6Rs9AmzYFXn3Vvr1PHw6C0NdKC3dXPoriXKNy\niCfC/QQAY3hIeN62fIgomYjuJ6JOYJt7GIDyOW+OigJ692ZtwEz9+qwJWU3NduzgqWGlSiXfR4Af\n2uHDnbXpqCgWxO+9x9Nyc8mAF17g0C8j5pvaWDRMEx/P025XDsCUFOsEluuu4/316sXvK1dmv8Fl\nlzm2s9nsQiEz0339GSI+nisz2H33le8VpFJS+D669lo29ZkjoP78k3MbTp7kENxRozgZSRMZyaWi\n69Zl7dMocNatAx56yC4Qa9e2jpYJDGSTTJUq/P6JJ4Dbby/a+SxZAjz+OAv5ypWdlQeNvr/uvJND\nEMePd/a9GNcVGDSIBff+/c77CgtjwX/ffXxtatRwFu6//MIZ47pQXWame4dvRAT7KfTgAvD6C8OH\n8zMfHn7Jae6bAbRUSjVTSgUBGAlgibGBUqpm3mcAMAbAWiIquxz0w4d5Kvn7747bT5/mH9GVg+aK\nK/jVSrhHR3s/vt0dVarwTMKYdp2VxecQHg60bctZeVpwa4y13DVm4W4VHaAfOFcPR40aPCB++qn1\nZ0bhtGGDs/3+nnvsD+FLL3FtG1ekpfE+tdZp5NprOfHLlYApDdLT+f6yuhYAa5JHjtjt7WYT0htv\nsFCpVYt/0/XreXYF8MD27rtss/fzY0ehsRTtTz/xwuxaWzWHuWpmznSsYT52rOtrvnEjn4+rgT02\nlgfTsDD7PWhFv34c7dW6NZsOExO5TrqR4cOBW2/l/zt3tmdLu0L7icLCnB3Lv//OA462ApjzOaZN\n43tIKxKNGnF/jEmIx4/zNU1OZl9WeVYaCkmBwp2IcgCMAxAFYC+AH4hot1LqYaXUw3nN2gLYpZSK\nAUfVPFFSHfaIf//lV3PiRna2+2y9GjVY4zSXv01MZPNDUTJTi4vRFnvqFD/82habksI3phYMGRn8\nZw4ZsxLu5jZauBsf3ORkriiZns4Pa506LAiMzJoFvPWW47bbb3cuQbBjB2tyAIe1WZUYfuYZzpQN\nCmJBYSVskpNZE773XufPSgtd9+Wdd1y3Ucq+EItRc8/NZefkrbey+SUtjc0HCxfy5xcv8m+obcjT\npwN3323/flQUD3DBwfzelXD/5BPHkNGMDDbTWM2Gtm7lVyuHpe5/8+Z2jdeVcK9Zk7Owq1ThWkVN\nmzorSi+8YA9B9vNzXc+od2/H9Q6GDHF2HB88yAOJjpc3X4vYWM5bMc5ubTbH8iHGhL5XXrEnhlUA\nPLK5E9FSImpFRM2JaGLetulEND3v/7/yPm9NRLcSkYusilJC2yr1OqKaxo1Z2zI7HI107Gitub/+\nuncWTygMo0Y5Rh7o6bk+v8RE1oR03Q1zuV9N1ar8ALjT3CtX5hvc+OBu2MC27a1beb3YK65wNjEs\nXOgcnVOjhrOTLzHRHnras6e1HXrDBv7NgoJ4up2R4VzHR+83Ls57xdYKi/YzNDLHFYCFZ7t2bG8O\nDubfyqi5x8ayANezwJo1WRjq31abHowOQh0LHhfH18c486xVy1m469BFY3brt9/yYGNldnjsMb5H\nTp2yPl8dLx8Swr+Nq9ndunWOUWhXXOEYnJCby7+fcdsHH3AIo7n/27c7OmSnTHFe3Ob4cfuzADgL\nd1c5H8b9WCX0VRAqZoZqly4cEpiaan8QbTaOiS2o1vgdd/D01XgD1qnDs4DS1txDQuxaOcB96tKF\nixwBPFgFBNiFhzn0S6MUT2knTuT3U6c6a9uAcwkCY8SFfjUL9+PHnTN2a9RwtLXabCzMtcCqU4cf\nQnMqe2KifYqtz8Fss9XVPInY/FYW3H8/25WtMiY3b2YBrIVF//6ONl5zCQul+PppoauFu74O48fz\nfaATlwC212vuusu+spfmzBm+940VT105Z3UfrrjCejalF7to3pzbucuH+PZbewgvwKGJ8+bZ38fE\n8LkYaxidOcNBDsbn7fx5ni0WlAkeF+fYZvBgzoMw9t38LISEOJpujAl9e/bw7ECHmvo4FXMNVZuN\np7KDBwMNG/K27du5MtzcufZEISus4tz37WPnltm+XdKEh7NQTE9nLbBPH3vNdIAFe7NmjkL4yBHn\n2u2AoxmmSxfr423a5Bj7HhvLWqUWTiNH8gBHZLeFxsU520zNmvuFC6y1aYGlhfy5c/YwNcBx9SH9\nUCYl2dtkZbE2f+WVrGXGxrKTuzTR5z57tqOfQRMVxZ/rWd5XXzl+HhLC92U7Qx5geLhdc9eCR1+H\n0FA+58REvoY9ejja4M1aL+A8KOtjAM7RIFlZ7Gg8c4avufG3BXibVhoAduYatWUj5865f0b0oGAs\nRx0ezjMw48BunqECrIxMnmxXPrKyeAZ45ZX2No8+6ng8d9namqAgPk716mzm3LWLr18FWJi8Ymru\n48ZxtmmHDvYbVWs9nphWkpMdNdiRI90PCCWF1kqsQjM1zZvbNfeAAHYIWU0xp0/nSoMAR9NY2bND\nQhztk3pqr6/hddfxA6Tf6wQms4YVEuIo3LUjTAus9u3ZnGS0/2Zn88OoH/ARI1ioGRcQt9k4kkRX\nnSxuHZ6iMHs231eu6rlERbEpy1XpgEGD2K5rzIQ2Rmn078+CS1cQNWrcDz3EDkuj8E1L42gT4yxI\n78sTzX3fPr7XP/qIo7CsSmuEhNiVg5de4igtK8zCfds29g/oaqr6mTILd8Bx0DHnTgBsmz9zxh6x\nFRTE1/r++x37kJtrnwV06OBcidUs3B95hI8dGMiKoFIVJmKmYgr3hAR+eDZtYmGUm8s3QqdOjjeW\nFUQ8Zb7xRhYmWVk8XSvNSBmN+YEcO9Z5ZtGiBQs5Ip7yv/22teD59VeeNttsbMs328kBrq43bpz9\nvba1anJz+VroqJeEBB5IzJrcSy85Jrvo2cXQofy+Xz8+vjHMMy2N/Qta2wwKcg47rVyZzRS33sq/\np7uIm5Li999ZOGRl8XXcsMH+WWYmm2Wuu86+7c8/WTPW5hirlb+eftoxfDUw0O5o1APngQPWyXU/\n/MBORaNAuv12NhkZf7uQENZOzZq77tfAgdZriP78Mzsa9bG1ic2Kc+ccZ42BgRz1o31fWnM3mqms\nBp1atXjw1+ZHwK4YuIp1B/ieCwiwt/nkEzbPGjFH1BgJDOS+VZBY94op3PX0/tAhTr5YvZqjPNzV\nqNAoxc7TLVuA77/nGzM7u2wiZVq35vhkbZaIjnacIgNs4/znH/5/0yYWrFa2YK2x6DrvVtp9dDSH\nj2kB9OmnXJdGc/Eia93ffMPvmzZlk4t5fdbOnTnaQRMYyFFIVoWfNDVrcv/1vs6c4dhqY7GrjAwe\nWHJyWBgVNFB7G12kauBAHni++84eaQLw9XnkEcf7rEYNFhaxsSwUq1VzTu654gq7aWv+fP7NNVr4\njRvHg4S55IQWpmaBFRzsnAMxZYrzgLhjBw+aLVtyApC5KNuSJWxa0hp9ZCTfj1ZRN+fPO2ruOtdB\nzyxPn+ZB22gqadKEzStGE1evXjz4G39fs3D/5hvev9H+r3NXXCUpAcBttzmuAhUZyUqTpgLFulds\n4T5gAN+UL77IAsET4Q6wvb5TJ/6eFi5lobk3agR8/LHdPmuODgD44WjVis/TlUMVsAt3q4qQGq1R\n6elzv36OC4FXr84Ptjlu2yxEDhxgjVI7rzdvZg1KT6nj4/n3scpg1GRlseNXh7UCPIg1asShlJs3\ncyx3aaKLVA0axOajatUcr0WtWixAjfZaPRM5eJAHT5vNOcv53DkeKE6cYEXEKGDr1uUFMhIS2Gxg\nLmxnlXU5caLdBGfkvvucF3aPjmbzRUAAKzMLFjh+bo66qV+fz8FKg163zjE8tGpVbq/NZ/37s+Jk\nNP3UrcvFwwYPtm+zGji0uU4f9/Bh/jMOJnqgO3eO+9isGWvvRkaO5JmS5p9/2B+nGTSobJ71EqDi\nCvewMBYgXbrwD/377/YsyoLw8+MQraNH2c6pNZuyICPDXn/+1Cln4Z6WxgJlyxYW7v7+9qxEI7Vq\nsR1cCwF3wv30aY7SWbTIufqiNgMBHPZ2113OEUhLlrD5SM8SVq/m2h2amjX5nIxp57/9xs4xXf7A\n6FDVGGOSf/qJteSSLuZmRDtLtdJgvBYAD1rmeiohIXwfxsa6Xuzl+HGesWzc6OhYBPj3fO01viet\nlBMr4f7dd/ZYfCOnTjmXF+7f3x5H37Gjc8RMbKyjcLfKh9CEh9sDGDQtWtgHwAEDnMMZrRg0iBd+\nMRIRwbkN+tocP859MWr8xmuRmsr3kjlcNiuLr4O+b8xhwRMmFH6N5HJKxRTu991nDxcbNIgfqp49\nC1c64LrrOKyqa1e2VRvLA5cmPXuy00gnMJmFu78/a3ZLl9qzU62cYrVr81Rd2xOthLvxwV2+nOvP\nmO2rRgfuhg3czqy5m4uHaR+IHnQqVeJZgFH7O3bMbiIAWOvz97cW7jVqsNDIzi5d+2i3bsDzz9tN\nBEbBBbBW2Lev8/d0CGl0NPsZjBFCgKPdOSHBuQjW0qU8gFoJd6O2qrGa4QHsVO/b13EAeuEFuxmo\nY0c2ZerrnJbG953Rdm+e3WlSUnjGoJ2nmj597DOVQ4esS0E/8IBjraO4OGf7f3g4O7P1qkxWIbhG\n4e4q5+Obb3gA0n6jClrLHaiowv3tt9m2BrBmYrO5X2HJFT/+yNN/nS5dFugwudxcrg9uDIMDWBiG\nh7PwsAr90jz2GNuE+/VjLd88PQf4wQ0K4gc1NpZt5eYHqEULfrDS053jjDVm4a7NZMZBp04dR+Gu\nI2p0lIlSrPUahYFRuGttsjSrQ95wA99bmssv574Q2R3a5t8H4Pund28evKym/LVq8cB3/LhjOKjm\nf//jV6s1eOvUYfOVttmnpPA1sxLuERHcT10a4sIFx3pB2q+kBfSFCzwQWQl3s+Z+6hQ7Xo1mNIBD\nGL/+mv/v3dsxDl6Tnm4/JpHr+4rIPjDFxTmfY716PFC1auXaRGkeDM11lpYt4zbmQcoHqXjCPTub\nb3Dt3b/2Wvb46xG/MLhbuKO00A6eZs34PKxMS1qbnjXL7lw1o7XrqlXZVGXl3GzQgM1A99zDg0Wz\nZs7p4SNGsNnFz8+1huhKuBsx1wpJTOQ+GZdeCw11XKnHSriXVjjkrFl2bU/z2ms8+OvwufPnrR3v\nzz3H9WTGjnV05mmUsg/ifn6O0SQAhz9u2mQ9e6xUiQduPahoZ6CVcBw2jK+dLsvx6af8XpveOnbk\n91orb9iQ7ytjYlDDhmw313WYNFpYuopzz81lM5z53HRfjx/nZ1YnMFndV02a2KO5+vd3TOYCuO8f\nf8yDoCvhbi7726KFo8m1USP+bkEraPkAFS+JaetW1kqXLmVNS6mCV08vz4SHs+DTiUxWNG/OoY6V\nKrk2PR09ytEQffuyc/nuu51t80bN2hwGqWnblv8AFkbGqBiNHjiMwt1c4lkv1qCxGgBiYhz71Lcv\nF9WqUoWvRaVKpaO5b9vGdcOff95RczdS0OLpOTlcU8XVuqFawFkJlaZN3Zea3rePBwWtsdatay3c\n69ThaKoXXmA/yI4drM1rE0h4OH9fKXZWNm7sbB4JDrZebFsLS7Nwj4nh5/Dll3kGbRXhZExk0iUQ\nrIS70ZT38cfW1yI9nRW86tXZ1GPej9lHYV7XoE0b/o2iozkL2YepeMLdqjaHL6Mf0jvu4LDMAwec\n27RowdrWyy+zqcAq/jszkx2gu3ZxdICrFadee41NMwcP2pfOM6IXUQgNZU3fmGSkufxydg7qKJ9V\nq5zju40OVoDDPs22T7PvoHt3e60dpdi8YFXfxZsQsSmhdm0W7kbOn+cM0Ucesa8MZdZoAR4cunTh\nMNNHHrH2iXz6adFniiNGsPb500+s2Lgre/t//8fHWrXKudKp7ld2NkevtG3LTnUz8fE8wzNWUNTC\n0pwdHRbGA4V25Fpp7kafQ82abFqxWl0qLMyeqauUs68H4BlM7958r1stmVdQTfegIL5vK8KKTERU\nJn9dunShEmHWLLaAHj5cMvsvbWJiiD74gKhPH6I2bazbJCcTpaURNWpE9MAD1m0SEvi61K7Nrzk5\n1u2uv56oZ0+iffusr6HNRhQSQvTYY0U6HQcyMtx//umnRE89ZX9/5Ejp/66//srXa8oU589yc4kq\nVSJ69lmirVuJpk613kd8vLbKE2Vnuz7W0aNEw4YR/f134fp4zTVE117refvz54lSU4mUIho/3vGz\nefPsff3lF+vv9+rF94mRKVP4OwkJzu1r1iRq3pw/X7vW+fPdu4lGjCDatct9v4cPJ2rbligqiigw\n0Po6delCNGSI632kpxO9/TbRjh38u3TuTLRokWOb0aOJ6td335cyBMAW8kDGVjybe0XT3Fu14rjc\nzEzXhZSqV2dThTuHqt5+7hzHZ7sqtaqLh7VubW0KsAoBNJOTw5EN27ezzfzJJ51D895/nzVVq4Qr\nzebNjnHXTz7pWEtl/XrWqs2Zm+fOcTq9OUQzOZmTssaN47/nn7eHa7o6j2efZa34oYecP/fzs2ff\ndu7Mtm/s5Gv6AAAgAElEQVQrjCYpV1FXsbGsVS9eXPjFxY1Zl++8Yy+p64qaNXkGR+SsIevr0aCB\no63dSL16zg7Vhx/m+8bK5t6iBZ/3tGnWtYDateO8iPbt+R62WhsYsDvhjx/n2YU56giw53O88w5/\nbg5NrVyZzVIdO/I127bNeRGaW29l34ixpENGBu/TPAO12dhU6G62BHBdHH3fvf+++7ZeouIJ94QE\ntsUaF5v2dQ4eZEepq4JNublcGTAtzXUWaECA/TN3oV/16vE0esoU14K3eXPOG7j2Wusl0pTi8M0l\nSziUcvJkZ1uyHmzOnmUhExHhnCpuLlplDlvbvp3zEawWcXj6acfSAAD35/33uVLhtGkct+6usuTF\ni2wGevdd6yJhAAuunTvZVOVKKCvFTuo333R9rJgYFuxA4RUTY72UtWsdk3Jc0aABm4rMa7wOGsSm\npaVLrc1HgHVlyMBAFqZWppLmzVnIPvqos+/FiM3GA7irnJIbbuBBRDuNzTH1gL0EcmIiD1RWPo64\nOA69dVXud9gw/q2MA/HHH3NS49Kljm23b+fBwsqEqUlO5vP66itOUFuxwnVbL1LxhHv//lx/xNWN\n6YvoyneuhLu/v32xbFeaO2Cv1eGuDIC2iT7xhGvtXkeqrF1rPVD4+/Pgmpzseial3ycksBCNi3PW\nwGvW5AdUa1DmRUZcFcPSIYPmxcyjo3ngj4/nY+3YYW3b1dSowTMQ83qzRpo3Zz/IoEGcOeuKb75h\nn4grjL9tcYS7VYigFRERHBJrFra6Do67chv169sT6zTffOPayTloEJcKMK/OZaRzZ05SchWBBbDQ\nfeMNPsd69ayDB/Syg1YrkmkGDuQZnzEpzkxysmN01J497Ge6+WbHdlrOHDjAM00r9P05cyY/D64W\nBfcyFU+4DxzII2xFQk8/3WkH5jroVmzfzpr2b7+5bhMRwa9+fq4jb4xRNK6cgLrsr7k+ucZYK8Rc\nNVJjjroxa+7aTGUW7jr70OwUmzCBr4GriBUjc+bwlL0gjIupFCdt3WhyK6xwv/tu7i+RdXKPt9EK\ngHHW9sMPjgt1GLn/fhbGunCcFTrO313/ddGyQ4dcDwA338zmNncmSk9KcXToYHeg22w8MxswwFnh\n6dyZ78u6da1NhACbVg8dcs66LWEqnnA/etS5uJav06YNa+/G9VTNDBjAD0VBpYnDwhyr7Zm54w7O\nirXKtNQMHcpRKlYJOxqdgORKcBuFu6sBoG5dPo62iZoTTlzVKNf2cbNwr1zZHsYJsK3XvNAFwDVe\nHnrIszT0UaO4bc2a9oGxKBijTKzKR7ijUye2E6el8b3vieZeHPr2ZS3UuMi8uSKkmZMn3X/euDH/\nju5mHitW8H3TqJFzqV/NoEGc9OWJcK9RgxUmq/LMxsVLdu7k2d4vvzgXySNin9frr/M9b+XD0esu\nuJsxlwAVT7gPGMDhZhWJ0FB7qJ0rmjdnrce8upGRjz7iaaQ7zR1wLhZlRtft0VX/rNCau9a6zcK9\nYUPWdFq3dj0A3Hmnoyb34YeOtcTr1mUt3Gxz15p8dLRdk0pI4OMZl1602axr5b/2Gmv/rmLazeiQ\nwuKYApViW/OIEYX/7tmzPNU/dow1SavwVG/SqhWXDDAKK3NFSCMJCRxX724mpLOsL150rbnr+2P4\ncNfO64wM3s811zibUDRauPfvz6ZFq8GkY0fOH8jMZGd3tWps7jOvITxgACtEDz3EPh2rssnLlpWa\nE9VIxRPuVrU5fJ2EBNZa3UVRhIayIDMuy2dG26Ct4n81KSl8PKvppZHUVOuYZc28eZzVOWYM22bN\nySs1arADtUsX7vsddxSscY4a5Zg05efH1yQy0r5Nmyb69mV7uTbRbN3KzlejKcHKMXjiBDu+xo1z\nP8PR5OZyBmlhI1ysWLuWVworLFu3clz6+fP8vzsfgTew2VhQG3Mu3K3CpJ9Hd4lYWqC/+KLrVZD0\nzG73btfLZUZFsdnwP/9xzqXQuFpU3EinTqwo7dnDg8m5c1ziwbx27759nNilFN8L27Y5O+kXLrSu\n0lnCVCzhrlfzqWjC/bvveNEHd9O6hx9mT7676pX64bPSLjTVq3OCy0cfue/TvHlsw3ZF8+b2aIaA\nAGutNiWFtc7u3TmKwKyxHTvGi6asWcMhlRs2OGvp5qzdc+dYe7v5ZhZyOuJBm2iMSUb16jmHsOlF\nMYYMcX1uRvz9OSTTGxEQurZPYSkoMcfbELE5Y9o0+/uUFNfCXSkun/DXX6732aMHm8ieesq1M1c/\n1y+/7Fx3XqNNP+b7xMjIkVyPZ+JE12VJtP9Em2YCA3nQsNnslUvT01kZ0LNcIj4PcylrT53cXqZi\nCXd9c1c04R4SwmGH7ggMLNhho22QVuFqRvr1c7SnWtG9u/vFMlat4od/8mS2R1rRowdPZ13NEojY\nhHToENtsr77a2aT07be8qIdG298bN+bYel2PX6faG+2+Vpo7ET/w7rRMM717l+09p88pMpL9JSVd\nBtnfn6OMtOBTiu39rn5ngO8XdzO9Hj04p+DCBddaudEX4cp0oweYwYOtyyToY40ezcqDuV6Qpnlz\nzuRNTWVBbzRV6hwPbSrVAQYBAXzfWC0iL8K9mLhyzAmM1gpLowb64sWsYf36q+vQL52U8vDD1nZi\nPVNJSnId2RAdzZqSHiAaN2bTRo8ebBLSi0eYU+0BHjDvvtvxevTrx6YNdz6H8oYWaNok4CqE1Zt0\n6uTo01DKsygkd3zwAc88XQl3wD7zciUsjbMHV7kuFy6w7fzECdc5H/7+7Ls7cID/GjXivg0ebB9k\nrBYiN5eBBkongsmCiiXc69ZlbbFr17LuSflEa+OlUe1SL5J95oxrrVYL98REa3NEjRr2FaZcJZwY\ni04BbL+/6y5+GLUAysmxrth46618v5SGMCxJQkLsZq/S0hA7duTfJS6ONdgxY4pXSdFmY+EOuF87\nQZfytkpgAhxnZq6iZTZu5NnWX3+5N3WePMkrOfXty89MWBg7R3UkWYMGfN5GU6iu3a8HvdRUvndF\ncy8mYWGcBecuiuNS5oEH2HQ1fnzJH0vXOT9yxLVw14WgXDnB/fx4PxcuuBbuWiPS5pidO9nBCbAA\nOnyYTQYnT3JVTDPZ2Y7JOGPGcDapL+Hnx7OjqlVLT0PUA2V0NP/GM2e6XjjbEwoyFWq0KchVDkZw\nsD1azpXgNvoo3GVrT53Kr8bwWcA+s+jWjWeNxgGlRQu+X/W1qFaNlQ9X0T0liG8L9+RkDm/TzqwT\nJ3ixgNJces2X8POzLwxR0hhruhekubvT7jt1Yg3MneYO2MMfJ02yV7zUAujff63NBjpj1ZhSvm1b\n6TkmvcmAAXx+paUhdurEiT3XXFNwLXdPeegh6wHYSJcu9mUBXaEHZ3dx7gBr3u58WWPGsBXgySft\n2x591G4aSkx0NiHdfDMPtEafVVBQ6TxzJjwq+auUGgxgMgB/AF8S0Tumz0MAzAUQkbfP94noKy/3\n1ZkqVVhjOH+eb+5Zs9iJkpnp+1NtX8cohBs0sG4zZAhr76+/7jpp6s8/+TUujouImTXT8HA2xehE\nJ6PzStvYR43iY33+ueN369Th2YXRqXr8uGPWqa+wahXf9926lc7xgoPtSXW6aJm7JCVPmD694DYv\nvVRwm5QU/r1dRY7pfr74oqMz3sxllzmXFKheHdi/nxXIHj3Ygf3tt/bPzbX3V65k/9PEie6j1EqA\nAjV3pZQ/gGkAbgDQDsCdSql2pmaPAdhDRB0B9AXwgVKqCDFdheTPP9nuN2sWP6SJiSxUihJOJniX\noUNZaObkWGeBAmz3fPJJrmLoIvs2OzcbkzZMwpjtryPn1pudnWQNGoASEvBVmwyMWTIGOceP2YV7\nw4YcLVO3rpM9OCEtAbesfRT7Q2EX7hkZiL+YgJsbrEGvmb3Qa2YvXPf1dTiRbJHoVAjiU+Px4OIH\n0WtmL6w8xItD5NhykJplz2bMyHFRCdFT3nuPI1juvbd4+ykMW7ey6cJbmru3ePNNToZy5RQ3Vkgt\nLHrt3sOHORveKqrql1/spsH16/kaFWb9Zi/hiVmmO4BYIjpERFkA5gMwL21EAKorpRSAagDOAXCT\nKukl9KotACcbWK34I5QNVatyqKS7GVRmJieBvPIKJ4qY2H5qO3pMiMBzfzyHGqnZ8F/1p+VuLmZf\nxANLHsDM7TMRm2ZYf1Mp1mRjY/O1+Bwb35Z1qtTBqqN/4tZR/kiLZ5NOTtxRjLwNiFIHUS2oGqoG\nVcULV7+ARjWKtiAIEeGb6G/QdlpbzN05F1UCqyDAjyfLH//9MTp82gHLDy7HsgPL0Hpqa/wS8wsA\nYNKGSViwewGooEQyI54k5nibpUtZ801Ksi/AXh6oVInDHF1dv4AAXh85MtKzEhNG9ICxciVr71ar\nlT36KIdRAjwTrFevTBROT4R7IwDG4h3H87YZmQqgLYCTAHYCeIKI3MQzeQmjIywqqmJmp/oqiYlc\nxM0q7lfz77/srFq61OFBTM9Ox4t/vIhuM7rhZG4Sflwdhg//DoEaMQLp2VxPO9eWixlbZyAjJwNV\n35qEuSms+R8LznK0O//8M5CcDGrfHgt2L0DLT1riwNkDUEph4e0Lsad2LsYG/Q4iwqaEHVjfVOHz\ny1/C8nuWY8U9KzCwOa/TeTHbXv44NSsVoxaOwqrDq1yePhFhxIIRuHfRvWgb1hY7HtqBP0b/gWub\nso33qvCrEBwYjEFzB2HIvCGoGlgVdavWRY4tBwv3LsTtP96OW76/BSdTXMRhm/n1VyA2Frlxx/DR\nXx9hyLdDsO2UB4XPPORkyknc8v0t+Pv43/aN2qcxdCg7EMtJJdZFx//A8K6HsO+Em9WUDDkhf8X9\nhatmXoVO0zuh0/RO6PNVH6w46CIpTQvzvPDeU+E1cfP8mx2vizEcsowSmADvOVQHAdgBoCGATgCm\nKqWc3NBKqbFKqS1KqS0J7jLIPEUL98su46xEq3U4hbIhLY0d3UePuo5b1rOsO+90yPA8eP4gPvjr\nA4zuOBp7k+/B8OhsIDkZK9oEotnkZvhxz4+4auZVGPvrWCzYvQCIicHVK7i0wtEJz3CIo2b3bpyo\nDtxim4fbf7wdtYNr52vv/S/rjwlVb8S8Wsfx6eZP0bvXHdj3fwcw+pZIh24+vvRx9J3dN1+TXnV4\nFb7b9R1GLBiRvy8zSinc3+l+fDDwA6y7fx3ahjlGXPSO6I3tD23HhH4TMPG6idj+0Hb0CO+BAL8A\nrH9gPd7t/y6iDkah3bR2mLF1RsFafJcuOFMV6LVsOJ5e/jTWHF2D7jO64/kVzyMzJ9P9dwsgOzcb\nty+4HYv2LcKt39+K+NS8rF5zFmc5wEY2PPufIPzUDug4uwcmrp2I7Nxs54Z67dQaNRBSOQQ2sqFp\nzaZoWrMpTqWcwqC5gxCTGOP8vfBwXujc3x/ZfsCIg29jccxi3Pr9rTidmmfe0+GQQJklMAEoeJk9\nAFcBiDK8fxHAi6Y2vwHoY3i/CkB3d/v1yjJ7emmvxER+v3w50Z9/Fn+/QvE5d86+XNv585ZNzp45\nSmOGglqNA7Wa1IS6ftE1/7PD5w/zP6++ysvBDRtGx7u2prqT6hIiQWHvhdH8nfPJZrMRPfssZVcO\nIr/X/ejllS87HCPm9G6qMaEKBb8ZTJM2TKLsXMdl7nJtuXTjvBvp+q+vp+ycLMt+zto2ixAJWrTX\nvhzb1E1TCZGgmdtmOrVPSk/y5AoVyIGzB6jf7H4U+EYgHTh7gIiIxv02jlp90srhr9/sfkTnz1PO\nxvV047wb6bud39G5i+dozOIx1H1Gd8rJdV5S8UTyCRo0ZxDFno112L4rfhf1mdWH5kTP4WtLRONX\njydEgsavHk/BbwbTI78+wo1tNl5CDyB64w1Ky0qjG+be4NS/e366J3///b/pT60+aUXjfhtHFzIu\neOU6OZ3bkZ20+vfpdPuC2wmRoC6f22XN/6L+R+uPridq3JhyFcg2d67T99Oz0x1+6+4zulOrT1rR\nFZ9dQd/s+Iavy5Yt9PTrvRyuy+S/J/MX3n6br0lyMlHr1kRPPOHV84OHy+x5ItwDABwC0AxAEIBo\nAO1NbT4DEJn3fz0AJwDUcbdfrwj3efN4DcSUFH6fZf1wCmVATo5duOcJCSOrDq2iupPqUsCroJvv\nAI38+iZ6cPGDzvv54APeR7duRL1706bjm+j5Fc9TYlqivc3kyUQA/bv4C0qaO9PhPvhs82eESNCO\nUzus+5mRQef2/0svrniBUsaNJWrf3qlJdm42tZzSkhAJWrp/KRER2Ww26vZFN4r4KIIysu1rwR44\ne4BC3w2ludHOQqMo2Gw22nZyW/77Dzd+SCN/HOnwN+63cS6/n56dzqeZneEwsGXlZFHrT1pTp+md\n6GLWRSIiupBxgVp90opUpCJEgm6YewMdOX+EEtIS6PMtnxMR0fqj6/PbExGv3QoQDR1KNpuNpvw9\nhYbOG+rQvwlrJuQ3f2LZEzR03lBSkYrCPwynX2JcrNNaBHJyc/IHJM3ifYvp6d+fJiK+lgPnDCQV\nqWjcHTVo3A2gez+5znLwM/LfJf+lkT+OpCunX0mIBA2eM4iOHthCG45toNf/fJ2IDMoIEdEPP/A1\n2b6d8g7stXMk8qJw531hCID9AA4CeDlv28MAHs77vyGA5WB7+y4Adxe0T68ukJ2bywvntm1LdPKk\n9/YrFA8t3C04cPYA9ZnVh3bUy2uTlma9j59+Irr6aqKmTYluuMG6zcKFvI8hQ4gCAhwW/7bZbJSQ\nluD00Ofz3nt2Leumm4guv9yy2bx/5xEiQVUmVqGsPA0/KjaK6k2qR9Gno4mIaOvJrdT6k9ZU+93a\njg97GZOWlUbdvuhGL6x4gbJzs2nzic1ERPRrzK+ESNB9i+4jm81Ge87socsmX0arDq2iyX9PpqoT\nq9K0f6ZZ7jM+NZ7eWfcOZZ08TtSwISXdd2eh+vR33N/Uflp7QiRo5aGVHn0nPjWe7v7pbho0ZxDF\nJMY4ff7W2rfomq+ucRx8TCRnJNPjSx8nNR6ESNBjczzvd05uDk3+ezKFjK9EWxoYLAam84r4IJzC\n361P4R+EU/iH/Hcq5RQREX2w8QMaOm+ox8e0wqvCvST+vCrciYhateLTmek8TRbKCDfCPZ/w8ILb\nEBFt2UK0bZv1Z9u2EXXsSKv6NKb3htQsXB+//pqPv38/0ZVX8gBhQa4tlyZtmET7Evblb7PZbHQx\n6yJdzLpIzy1/jvxf96f679enPw+XP9PgQ788RIgEDZwzkPxf96c9Z/YQEdGrq14lRIK+2PIFEVH+\nwEVEFHchjnJtuZb7m/L3FEIk6IrPrqBpV1eimuMr0bqj6wrVp8ycTJq1bVb+wHvg7AHLQdhms9HX\nO76m2u/WpqAJQRTydojdNJTH+fTzVPOdmnTjvBs9OvbGl+6h8f0UZRpmXZ6S/PG7fM/cfbdTP99a\n+xY9sOgB/psxlB54PIIemDOCzqezaXLR3kX05po3C31MI5eGcP/kE6KueXbakSP5dKZPp5PJJ6n5\n5Oa0K35X8Y/hgtE/j6aJayeW2P4rBKmpPKsykJ6dTv2/6U+/7f+NNyxbxjZKL/DiA00o4DXlMM1+\nJuoZ+n7X966/FBXF9826dURhYURjxxb6uKsOrSJEgsYsHpP/EJc30rPTqesXXQmRcBCMObk5dP3X\n11OjDxpRZk5mofa5aO8iavhefUIkqPVzVYplQz+ZfJKqv1U93xRk5PMtnxMiQb1m9qI9Z/bQyeST\n+X6NDzd+SKHvhlL1t6oTIkHbT2337IDGGVthmTHDUrg7MHs20aBB3O7w4cIfww2XhnB/7jmiSpX4\n/4MHiXr0IDp+PN/ZZR7dvYXNZiNE8rROKBw7Tu0gRILm75zv2Rf27mVz2333Ee3c6bbpZ0PCCJGg\nuAtxRMSCK2hCED2/4nnXX4qO5sdg7lzSjsGioDXh8szxC8fp/Q3vO/gIiIgS0xLpvfXvFUk4JyXF\n01t3NaEDy74tVt+0yaPqxKpUdWJVmvL3FDqWdIyI2Kz05dYvLWcRvx/4nR777TF67LfHaPrm6Z4f\ncM0alhcZhdfcKSGBlcp9+1y3ueoq+8w1s3CDZkFcGsL9qaeIqld32vzW2rcIkaDkjCKMyh5wMvkk\nIRLFnl6VR1IzU6n9tPbutd1iMDd6LiESns+qYmPtD4k7wTt8OC1twQPuhmMbiIidXIgEzdg6w/X3\n4uN53xMnEj32GNEffxTibARvc+T8ERo0ZxAhEtRiSot8h7DPcffdfF/Vq+f1XXsq3D2qLVNuycqy\nrCEdczYGjao3QvVKJVPLIfYcJyh0bVjxSguvP7YeuxN2I8i/ZDLqdp7ZiUC/QLQK9XCdT2NlP3cV\n/E6fRkSV+gBO42jSUfRq3Cv/d2pR2yKLUBMaCrz7Lpc/uPJKz/oklBhNajbBsruWYcGeBVBQqORf\n+mn7XkEnO9WtW2Zd8O2qkNnZlsJ9X+I+nEg5gdsX3M7TEy9zPJnT1b/f/T1+2P2D1/dfGri6LqsO\nr0KgXyAGXDYAF7Mv4ly6d1Pad53ZhTZ12iDQ38OFHTwV7o0bIyKLK+/FJXNCtUfC3d+f19ps08Yx\n41koM5RSuL397RjRfgRUOcl6LTS6TIG7e7aE8W3h3rIlr+Vo4rdRv+GZq57Bgj0L8gWxN7nz8juR\n9lIaNsRtwJx/53h9/0XF04HsXPo5dJvRDX8c+sPps1VHVqFneE8EBwaj39f9MPLHkci1ea+Ecp0q\ndXBtkwKWDDRiHLzdPSgXLqD6vkNIfHA/nu31LAAgOTMZtYNro2F1Fws7aOLiuAZ45cr26pKCUBy0\n5v7KK2XWBd8W7s88w2VgTYRWCcVNrW8CwJpiSVAlsAp6N+6Nv4//XSKzg8Ky+cRm1H6vNt5c+6Z1\nunUeNrJh9M+j8W/8v/hp70/o+kXXfOF9Pv08tp3ahuubXQ8/5YcxV47BikMrMPyH4TiVcsrlPgvD\n7Jtn45MhnxTuS7oQmDvhnldWODQ3KF/be673c0h8NhF+qoDb/L//Bb7+mvfvamk2QSgMXbrwGgSD\nB5dZF3xbuFuwJ2EPXv/zdYRV5bolJSHcn456Gl9u+xJXhV+FxIuJ+dP/sqRbo264vtn1eHX1q+g6\noyu2nNxi2e7tdW/jtwO/4ePBH+OaJtdg66mt+UWP0rLTMLrjaNzQkosqjek8Bu8PeB9RB6PQdlpb\nfLnty/yBLDkzuXRODOCKn1u3cv1sV0ydCqxfj++SN+LllS/nb/ZoWq8X+i6DdS6FCkpgYKnXbzfj\n28L9kUe4Ip2BdUfXIXJNJKoEVkGj6o2wK6Fwwt1GNqjXFcavtl6Kjojw5bYvEX06Gj3DewKAY0W4\nMkBr6j/e/iMW3bEIiRcT0fPLnvjnxD8O7bae3IpXV7+Kuy6/C490fQQ3tLgBgX6BWLRvEQAgvEY4\nvhr2Fbo34sUqlFL4X6//4d+H/0Wn+p3w2ZbPkEus5beb1g4P//pwoWYts3fMRttpbZGQVsiicdWq\nAZ07u9fcg4OB3r3x1/G/MHXzVNjIhgFzBnjmE6lfn18bFa20ryCUR3xbuMfF8dqYBmLOxqBKYBWE\n1whH74jeCPQr3IrscRfYGffG2jcsP0+4mICUrBS0qN0C7cLaoX61+jiTdqZo/fcCObYcdP6iM97f\n+D4AYFibYdj1yC7UDq6NmdtmOrRdtG8RagfXxqf/+RRKKYRUDkG/Zv2wKGYRiAix52IthXXL0JZY\nde8q/H7X7/n1yK+/7Hp8vvVzfLntS4/7uuP0Dhy7cAyhVUILd5JdunA52fT0Aps2CWmC5Mxk7EnY\ngz8O/eGZQ1gL9zLWtATBm/h+KKSpCP6+xH1oFdoKfsoP39/2faF3GXOWy3zOHz7f8nNtgmkZ2hL+\nfv448fSJgm26XuR8+nnUCrYvZzYneg52ndmFlrXtS4rVCq6F9Q+sR/NajivRTLhuAh7t9ihqVLJr\nwDe3vhmPLn0Ufx75E9d9cx0+HPghnrrKeeUkP+WXb+oCgFk3zUJ8ajzGLRuHKxtc6VFY6K4zu9A+\nrH3hr9e2vLrkHqyNGxESAQD5Kx65jZTRaLNMz56F65cglGN8W3O3CIXcl7gPbeq0KfIu9yXuAwD0\na9bP8nNzeF1pCvbUrFS0ndYWb659E4kXE9Hv634Y/+d4dG3YNd+BrGkV2gr+fv5IyUwBEeU7RBtU\nd1zP9KbWN+H+Tvdj7dG1AICrI672qC/+fv749tZvUb9afdz2w204e/Fsgd/ZdWYXOtTt4NH+LfHA\n2dmkZhMAHPUDeCjcr74amDcPuP/+ovdNEMoZvi/cDZp7Vm4WzqSdQevQ1gCAI0lH0OWLLvh1/68e\n77Jh9YZoU6cNZu+YbbnIQXp2OupVrYemNZsCAPYm7EX3Gd2x7ug6LNi9AAv3LCzeObnhk02fID4t\nHv0v64+jSUdx8NxBxCXH4c1+b1o6Dnec3oEmHzfBsyueRdPJTbHmyBqnNo1qNMKsYbNwIuUEQiqF\n4MoGnifyhFYJxY8jfkTH+h0LbJuQloD4tHhcXvdyj/efz3ffATfd5NFKPxEhEagcUBmbT2xGoF8g\nGtfwwEkaEcELhhR3gWdBKE94ksZaEn9eKT/w7LNEr73msCnXlpufspyamUoqUtEbfxauXsjXO74m\nRMKyrKiZsxfPEiJBjT5olF91ryAyczIpJTPFbZuc3Bw6k3om/71V1bvkjGS3lfgyczKp6cdNCZFw\nqjtuRNfKuXrW1QX23R2nU07T/6L+Z1n2Ie5CHN2/6H76O+7vYh2jIGw2G9lsNnpzzZs0aM6gEj2W\nIJQF8LD8gG9r7u+9B7z+usMmP+WHygGVAQBVg6rislqXFSpiJis3C81qNgMAHD5/uMD2tYNr4/K6\nl+Ns+llMGjAJC0YswAt/vIClB5Zatl92YBlaftIS13x1jct9EhFGLxqNhh82xCurXkF2bjY++usj\nJLVJexMAABZ6SURBVGUk4Y2+dkdv9UrV3ZpRgvyDEHltJADgtWteQ6UA61Tu3Qm7AQB9IpwTwgrD\n8oPL8eFfH2LUT6OcPguvEY5Zw2ahR7ibcEYvoJSCUgovX/Myfr/79xI9liCUZ3zboWpiTvQcrDu2\nDtNvnJ5vC+9QtwN2xu/06PvJmcmo/W5tPN/7eQDA4SRn4d53dl/cfcXdGNN5TP62X0f9Cn/lj0Y1\nGsFGNnwT/Q0OnT+EIS2HOHz3p70/YfgPwxHoF4i4C3HIyMnIH4iMTP1nKubtnIeuDbti/bH1IBBm\nbp+J29rdViizCQCM7jgarUJb5YdtWtGhbgdsHbu1aCYTA/d0vAcnUk7gxZUvYmPcRvRq3Cv/swsZ\nF1CjUo1SSSd/f+P7OJp0tPDJUoJQgfBtzb1nT+DRR/PfRh2MwvKDyx2cnB3qdsD+s/s9WiQ4JjEG\nuZSLLg27IMg/yElzP5d+DmuOrnFK4IkIiUCjGhwj7af8cFPrm7AsdpnTMW9sdSM+HvQxvhj6BQiE\nA2cPOPXhr7i/8PTypzG01VBsGrMJv9/9O4L8g7Dj4R34aNBHBV8TE0opXNX4qgKFaucGnT2v9+KG\nx7s/jnpV6+GVVfa063Pp59Du03YY+t1QN9/0HquPrMbUzVOxeN/iUjmeIJRHfFu4nz7tUAtkX+I+\ntK7T2qHJ1RFXY3i74ZYZlTayObzXYZDtwtqhSUgTJ83do0JUAIa1HobUrFSsOswRG9Gno3H4/GEE\n+QfhiZ5P4Ppm1+PzGz9HvWr1LPvUo1EPfH3z1w4mpjpV6iC8Rhmtol4IqgZVxUt9XsLqI6ux8tBK\n2MiGe36+BwlpCXj1mldLpQ9ZuVkASjeSSRDKG75tljFEyxARYs7G4IHGDzg0GdxiMAa34PoOp1NP\no361+ki8mIgnf38SESEReOv6t/Lb7kvchwC/ADSv1Rxr7lvjlGzjqXC/rtl1qBZUDYv2LcLgFoPx\n31/+i/MZ5xEzLgZ+yg+NQxpjbJexlt/tHdEb6+5f57vV8ACM7TIWubZc9AjvgYlrJ2LpgaWYNmRa\nidvbNQMvG4g/Dv3heVlhQaiA+LZqY6jnfjLlJFKzUl3GuP9z4h80+bgJxiwZg7bT2uKH3T8gOCDY\noc2+xH24rNZlCPQPRIPqDZxqmseei4WCwmW1LnPbrUoBlXB7u9sR6B+IJTFLsPnkZrx49YsOmmRM\nYoxTeYCf9/6MXWd2+bRgB4DKAZXx1FVPYWPcRoz/czzuvuJuPNL1kVI7/v96/Q+HnzjsNIsThEsJ\n3xbuBs39TNqZ/JIAVrSo3QKjLh+Fmdtn4rJal2HbQ9vweI/HMXPbzPzyATe1vgmPd38cAA8Gjy99\nHKlZqfn7qB1cG4NbDLZ0gpqZOWwmptwwBa+ufhUtarfA6I6jHT5/fNnjeGzpY/nvbWTDvYvuxfQt\n0wt3DcoxObYcDGg+ANP/M71UByw/5ZefhyAIlyyexEuWxJ9X4twffJDXviwEh88fzl9A+d/T/zqs\n/G5k/s75hEhQ9OnoIndP7+Pbf53Xl3x86eNU7a1q+au970/cT4gEzdw2s8jHK29cyLjgMrZeEISi\ngUsizv3LL4G77irUV5rWbAp/P38AHEnTrGYzLI5ZjLSsNBxJOpLvZG1WyzHWPdeW67ZOuhW/7P8F\ntYNrY2SHkU6ftanTBqlZqTiZwoXPtp3i+ilX1q84S73VqFTDZWy9IAgli28LdwOT/56MwXMLVxhf\nKYVhrYfhj0N/4LcDv6HZ5Gb4K+4vAMi3q+uImdVHVqPe+/XyhbAnzBg6A8eePGYZtaFLJOgInW2n\ntiHQLxDt67Yv1DkIgiBY4bvCPSuLa3i/z6Vud8TvKNLCHDe3uRmZuZmYvGkyAOQ7ZEODQ1EtqFq+\n5r5o3yJk5GQUqihZcGAwqgZZF7vS+9GFyraf3o7L611eYgtTC4JwaeG7oZBZWUBGBpBXfzw+Nd4y\nbrwgekf0RmhwKDbGbUSdKnXywx+VUmheqzmSMpNARFgSswSDWgxClcAqXul+w+oN8duo39ClQRcA\nwIIRC8q0LrwgCBULj4S7UmowgMkA/AF8SUTvmD5/FoA2fgcAaAsgjIg8WCmhiOiV6vNCIU+nnnYq\nZ+sJAX4BiP2/WAybP8wpqWnL2C0I8AvAtlPbEJcchzf6WS/gURSUUg7lCUIqhyCkcojX9i8IwqVN\ngWYZpZQ/gGkAbgDQDsCdSimHeEMimkREnYioE4AXAawpUcEOsOYO5IdCxqfFo17VwmvuAFCzck3O\nbg11jIvWqw4t2rcIfsoPN7a6sej9tSD6dDRmbJ2BTcc34bXVr+F8+nmv7l8QhEsXT2zu3QHEEtEh\nIsoCMB/AMDft7wTwnTc65xaT5t6hbgdcUe+KIu2KiHBFvSucqiKuPboWN8+/Gb0b98ZHgz5CnSp1\nitVlM7/s/wVjfx2LhXsXYuK6iWJvFwTBa3hilmkEIM7w/jgAyzxypVQVAIMBjCt+1wogOJgXyG7H\nk4iou6OKvCulFJbfvdwp0SYpIwmLYxbjpT4v4f96/F+xumuFdqrO3zUfrUNbu3S+CoIgFBZvR8sM\nBbDBlUlGKTVWKbVFKbUlISGheEcKDQU+/RTo3bt4+7H3zWmbrus+e8dsy4Wji4s2A8Ulx6Fzg85e\n378gCJcungj3EwCMa5WF522zYiTcmGSI6Asi6kpEXcPCwlw18wybDcjJAYiw7ug6tJ7aGttPbS/e\nPk3oRKbPtnxWIunzLUPti1qLcBcEwZt4Itw3A2iplGqmlAoCC/Al5kZKqRAA1wIonSLaO3awvX3J\nEsQlx2H/2f0e1XwpDNWCqqFzg84Oqx95E2N/RbgLguBNCrS5E1GOUmocgChwKOQsItqtlHo473Nd\n6eoWAMuJKM3FrryLwaEanxoPAEWKcy+IrWO3en2fRg4/cRhhVcK8PjAJgnBp41GcOxEtBbDUtG26\n6f1sALO91bEC0cI9KAjxafEI9AtErcq+t3q9VC8UBKEk8O3yAwAQGIjTqadRr1o9n6+DLgiC4C18\nt/yAQXPvULcDqgZKGKEgCILGd4V7kybAs88CjRrhmYhnyro3giAI5QrfFe5t2gDvvVfWvRAEQSiX\n+K7NPTMTSEqCLTcHtd6thUkbJpV1jwRBEMoNvivcf/gBqFUL5/ZtR1JGkqz4IwiCYMB3hXtetEx8\n7gUAKHJFSEEQhIqI7wr3vGiZ+OwkACWTwCQIguCr+K5wz9PcT2eeBQDUr1a/LHsjCIJQrvBd4Z6n\nuUfUbob7Ot2HhtUblnGHBEEQyg++GwrZqxfwxhu4unk/XN1mYFn3RhAEoVzhu8L9qquAq65CRk4G\nKlGAlB4QBEEw4LvC/fx5IC0Nt6z5Ly5kXMDGBzeWdY8EQRDKDb5rc3/nHaBFC5xOPY3QKqFl3RtB\nEIRyhe8K96wsLvebGi8x7oIgCCZ8V7hnZ8MWGIAzaWdEuAuCIJjwaeF+tkYgcilXYtwFQRBM+K5w\nz8qCv38AXunzCnqG9yzr3giCIJQrfDdaZtQo1O7TBxOue6CseyIIglDu8F3NfcAAfN+9KtKz08u6\nJ4IgCOUOnxXua/6ej5ELR2Lm9pll3RVBEIRyh0+aZYgIr/z4KBoGBOLBKx8s6+4IgiCUO3xSuEcd\njML66ufx6Z5mCA4MLuvuCIIglDt8zixDRHhl1StoklEJD8Y3KuvuCIIglEt8TrifSz+HSgGVMD42\nHEGBlcu6O4IgCOUSnxPuoVVCsf7+9bj3aC0gMLCsuyMIglAu8cjmrpQaDGAyAH8AXxLROxZt+gL4\nGEAggEQiutaL/TQfC+qVV4HKorkLgiBYUaBwV0r5A5gGYACA4wA2K6WWENEeQ5uaAD4FMJiIjiml\n6pZUh/O56aYSP4QgCIKv4olZpjuAWCI6RERZAOYDGGZqMwrAT0R0DACI6Ix3u2nB1q3AwYMlfhhB\nEARfxBPh3ghAnOH98bxtRloBqKWU+lMptVUpNdpbHXTJrbcCEyaU+GEEQRB8EW/FuQcA6ALgegDB\nAP5SSv1NRPuNjZRSYwGMBYCIiIjiHTE7WxyqgiAILvBEcz8BoLHhfXjeNiPHAUQRURoRJQJYC6Cj\neUdE9AURdSWirmFhYUXtM5O3WIcgCILgjCfCfTOAlkqpZkqpIAAjASwxtVkM4GqlVIBSqgqAHgD2\nererJkRzFwRBcEmBZhkiylFKjQMQBQ6FnEVEu5VSD+d9Pp2I9iqlfgfwLwAbOFxyV0l2HNnZorkL\ngiC4wCObOxEtBbDUtG266f0kAJO817UCmDsXaNGi1A4nCILgS/hk4TAAHC0jCIIgWOJz5QcAALm5\nQFQUcORIWfdEEAShXOKbwj09HRg8GFiwoKx7IgiCUC7xTeGenc2v4lAVBEGwxDeFe1YWv0oopCAI\ngiW+Kdy15i7CXRAEwRLfFO5acxezjCAIgiW+GQpZvz5Hy7RvX9Y9EQRBKJf4pnCvUgUYOLCseyEI\nglBu8U2zzLlzwMKFwOnTZd0TQRCEcolvCvcDB4DbbgO2by/rngiCIJRLfFO4SyikIAiCW3xbuEu0\njCAIgiW+KdwlQ1UQBMEtvincxSwjCILgFt8U7ldfDWzYALRuXdY9EQRBKJf4Zpx77dpAr15l3QtB\nEIRyi29q7gcOAF9/DaSmlnVPBEEQyiW+KdzXrAHuuw84f76seyIIglAu8U3hLlUhBUEQ3OLbwl1C\nIQVBECzxTeEuoZCCIAhu8U3hLmYZQRAEt/imcB8zBoiOBipVKuueCIIglEt8M849LIz/BEEQBEt8\nU3Nftw6YPr2seyEIglBu8Ui4K6UGK6VilFKxSqkXLD7vq5S6oJTakff3mve7auCnn4DnnivRQwiC\nIPgyBZpllFL+AKYBGADgOIDNSqklRLTH1HQdEd1YAn10JjtbwiAFQRDc4Inm3h1ALBEdIqIsAPMB\nDCvZbhVAVpZEygiCILjBE+HeCECc4f3xvG1meiml/lVKLVNKtfdK71yRnS3CXRAEwQ3eipbZBiCC\niFKVUkMALALQ0txIKTUWwFgAiIiIKPrRsrLELCMIguAGTzT3EwAaG96H523Lh4iSiSg17/+lAAKV\nUnXMOyKiL4ioKxF1DStOKOPkycDq1UX/viAIQgXHE819M4CWSqlmYKE+EsAoYwOlVH0A8URESqnu\n4EHjrLc7m08dp3FDEARBMFCgcCeiHKXUOABRAPwBzCKi3Uqph/M+nw7gNgCPKKVyAKQDGElEVGK9\n/u47Ns3ce2+JHUIQBMGXUSUpg93RtWtX2rJlS9G+PHAgL9SxcaN3OyUIglDOUUptJaKuBbXzzQxV\ncagKgiC4xTeFu4RCCoIguMU3hbto7oIgCG7xXeEumrsgCIJLfLPk77p1Zd0DQRCEco1vCvcaNcq6\nB4IgCOUa3zTLvP02sHBhWfdCEASh3OKbwv2TT4CoqLLuhSAIQrnFN4W7OFQFQRDc4pvCXeLcBUEQ\n3OKbwl3i3AVBENzim8JdNHdBEAS3+GYoZEZGWfdAEAShXOObwj3AN7stCIJQWvieWSYjA3jkEWDV\nqrLuiSAIQrnF94T7xYvA9OnAzp1l3RNBEIRyi+8J9+xsfpVoGUEQBJf4nnDPyuJXiZYRBEFwie8K\nd9HcBUEQXOJ7wj03F6hcGahUqax7IgiCUG7xvZjCVq2A9PSy7oUgCEK5xvc0d0EQBKFAfE+4HzgA\n3HOPhEIKgiC4wfeE+6lTwNy5wJkzZd0TQRCEcovvCXeJcxcEQSgQ3xPuEucuCIJQIB4Jd6XUYKVU\njFIqVin1gpt23ZRSOUqp27zXRRNacxfhLgiC4JIChbtSyh/ANAA3AGgH4E6lVDsX7d4FsNzbnXTA\n3x8IC+NYd0EQBMEST+LcuwOIJaJDAKCUmg9gGIA9pnaPA1gIoJtXe2jmP/8RZ6ogCEIBeGKWaQQg\nzvD+eN62fJRSjQDcAuAz73VNEARBKCrecqh+DOB5IrK5a6SUGquU2qKU2pKQkOClQwuCIAhmPDHL\nnADQ2PA+PG+bka4A5iulAKAOgCFKqRwiWmRsRERfAPgCALp27UpF7bQgCILgHk+E+2YALZVSzcBC\nfSSAUcYGRNRM/6+Umg3gV7NgFwRBEEqPAoU7EeUopcYBiALgD2AWEe1WSj2c9/n0Eu6jIAiCUEg8\nqgpJREsBLDVtsxTqRHRf8bslCIIgFAffy1AVBEEQCkSEuyAIQgVEhLsgCEIFRBGVTUSiUioBwNEi\nfr0OgEQvdscXkHO+NJBzvjQozjk3IaKwghqVmXAvDkqpLUTUtaz7UZrIOV8ayDlfGpTGOYtZRhAE\noQIiwl0QBKEC4qvC/Yuy7kAZIOd8aSDnfGlQ4ufskzZ3QRAEwT2+qrkLgiAIbvA54e7pkn/lHaVU\nY6XUaqXUHqXUbqXUE3nbayulViilDuS91jJ858W8845RSg0ybO+ilNqZ99kUlVees7yilPJXSm1X\nSv2a975Cn7NSqqZS6kel1D6l1F6l1FWXwDk/lXdf71JKfaeUqlzRzlkpNUspdUYptcuwzWvnqJSq\npJT6Pm/7JqVU00J1kIh85g9cuOwggMsABAGIBtCurPtVxHNpAKBz3v/VAewHL2P4HoAX8ra/AODd\nvP//v72zCYkqigLwd8jsx6CyhUy50CCCVulKKyK0giRq08JAMqhVq2gRiKv2ES2CCoqI/n+QkiCK\nbK8UREgqFUYpmtJCoZXBaXHP2GPQnLGB59zOB8J99743nO8Nnpn7M+9uM98VQK3dh2XW1gc0AAI8\nBw6k7beA+xngLuHpocTuDNwETlq5HFgXszNhM59hYJUdPwSOx+YM7Abqgf5EXdEcgVPAFSu3Ag8K\nii/tG1TgzWwEXiSOO4COtOMqkttTYB8wBGSsLgMMzeVKeEpno50zmKg/ClxN2+cvntVAD9CUSO7R\nOgNrLdFJTn3Mztnd2yoJDyd8BuyP0RmoyUnuRXPMnmPlMsKPniTf2EptWGbBLf9KEetu1QG9QJWq\njlnTOFBl5fncN1k5t36pchE4CyR37YrZuRaYBG7YUNQ1EakgYmdVHQXOA1+BMWBKVV8SsXOCYjrO\nXqOqv4ApYEO+gZRaco8OEVlD2Fj8tKpOJ9s0fGRHs5xJRA4CE6r6dr5zYnMmfOOqBy6rah3wk9Bd\nnyU2ZxtnPkz4YNsIVIhIW/Kc2JznIm3HUkvu+Wz5VzKIyHJCYr+jql1W/V1EMtaeASasfj73USvn\n1i9FdgKHROQLcB9oEpHbxO08Aoyoaq8dPyYk+5id9wLDqjqpqjNAF7CDuJ2zFNNx9hoRKSMM8f3I\nN5BSS+6zW/6JSDlhkqE75ZgWhc2IXwcGVPVCoqkbaLdyO2EsPlvfajPotcAWoM+6gNMi0mCveSxx\nzZJCVTtUtVpVawjv3WtVbSNu53Hgm4hstapm4AMROxOGYxpEZLXF2gwMELdzlmI6Jl/rCOH/Jf+e\nQNoTEouYwGghrCz5DHSmHc8/eOwidNneA+/sr4UwptYDfAReAZWJazrNe4jEqgHCBuX91naJAiZd\nUvTfw58J1aidge3AG3uvnwDr/wPnc8CgxXuLsEokKmfgHmFOYYbQQztRTEdgJfAI+ERYUbO5kPj8\nF6qO4zgRUmrDMo7jOE4eeHJ3HMeJEE/ujuM4EeLJ3XEcJ0I8uTuO40SIJ3fHcZwI8eTuOI4TIZ7c\nHcdxIuQ3bJGVYIwdRgYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x13294feb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(x_collect, train_acc_collect, \"r--\")\n",
    "plt.plot(x_collect, valid_acc_collect, \"g--\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./data/model.ckpt\n",
      "Valid accuracy: 0.7877\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "\n",
    "    tf.global_variables_initializer().run()\n",
    "    saver = tf.train.Saver()        \n",
    "    saver.restore(session, \"./data/model.ckpt\")\n",
    "    \n",
    "    valid_data = X_valid.values\n",
    "    valid_labels = y_valid\n",
    "\n",
    "    feed_dict = {\n",
    "        tf_train_dataset: valid_data,\n",
    "        tf_train_labels: valid_labels,\n",
    "        is_training: False\n",
    "    }\n",
    "\n",
    "    valid_l, valid_predictions = session.run([loss, train_prediction], feed_dict=feed_dict)\n",
    "    print(\"Valid accuracy: %.4f\" % accuracy(valid_predictions, valid_labels).eval())\n",
    "\n",
    "    feed_dict = {\n",
    "        tf_train_dataset: X_predict.values\n",
    "    }\n",
    "\n",
    "    predict_result = session.run([train_prediction], feed_dict=feed_dict)            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import Binarizer\n",
    "\n",
    "binarizer = Binarizer(0.5)\n",
    "\n",
    "test_predict_result = binarizer.fit_transform(predict_result[0])\n",
    "test_predict_result = test_predict_result.astype(np.int32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv('./data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r = test['PassengerId'].to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r['Survived'] = test_predict_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r.to_csv('./data/output.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>418.000000</td>\n",
       "      <td>418.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1100.500000</td>\n",
       "      <td>0.727273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>120.810458</td>\n",
       "      <td>0.445895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>892.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>996.250000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1100.500000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1204.750000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1309.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       PassengerId    Survived\n",
       "count   418.000000  418.000000\n",
       "mean   1100.500000    0.727273\n",
       "std     120.810458    0.445895\n",
       "min     892.000000    0.000000\n",
       "25%     996.250000    0.000000\n",
       "50%    1100.500000    1.000000\n",
       "75%    1204.750000    1.000000\n",
       "max    1309.000000    1.000000"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/cc/anaconda/lib/python3.6/site-packages/tensorflow/python/util/tf_should_use.py:170: initialize_all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.\n",
      "Instructions for updating:\n",
      "Use `tf.global_variables_initializer` instead.\n",
      "Initialized\n",
      "Minibatch loss at step 0 : 0.0160054\n",
      "Minibatch accuracy: 0.6250\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 1000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 1500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 2000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 2500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 3000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 3500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 4000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 4500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 5000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 5500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 6000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 6500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 7000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 7500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 8000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 8500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 9000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 9500 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 10000 : nan\n",
      "Minibatch accuracy: 0.0000\n",
      "Valid accuracy: 0.5978\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "_imageSize = X_train.shape[1]\n",
    "_numLabels = y_train.shape[1]\n",
    "_trainSubset = 100\n",
    "_batchSize = 24\n",
    "_hiddenLayers = [2048, 1]\n",
    "_numInputs = _imageSize\n",
    "_startLearningRate = 0.5\n",
    "_learningDecayRate = 0.98\n",
    "_decaySteps = 1000\n",
    "_numSteps = 10001\n",
    "_regularizationRate = 0.00001\n",
    "_dropoutKeepRate = 0.5\n",
    "\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    correct_pred = tf.equal(tf.round(predictions), labels)\n",
    "    return tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "def validateNumHiddenLayers(numHiddenLayers):\n",
    "    if numHiddenLayers < 1:\n",
    "        raise ValueError('Number of hidden layers must be >= 1')\n",
    "\n",
    "def generateHiddenLayerKey(layerNum):\n",
    "    return 'h' + str(layerNum)\n",
    "\n",
    "def generateHiddenLayer(layerNum, previousLayer, weights, biases, training, dropoutKeepRate):\n",
    "    key = generateHiddenLayerKey(layerNum)\n",
    "    if training:\n",
    "        hiddenLayer = tf.nn.relu(tf.matmul(previousLayer, weights[key]) + biases[key])\n",
    "        hiddenLayer = tf.nn.dropout(hiddenLayer, dropoutKeepRate)\n",
    "        return hiddenLayer\n",
    "    else:\n",
    "        hiddenLayer = tf.nn.relu(tf.matmul(tf.cast(previousLayer, tf.float32), weights[key]) + biases[key])\n",
    "        return hiddenLayer\n",
    "\n",
    "\n",
    "def multilayerNetwork(inputs, weights, biases, numHiddenLayers, training, dropoutKeepRate):\n",
    "    validateNumHiddenLayers(numHiddenLayers)\n",
    "\n",
    "    hiddenLayer = generateHiddenLayer(1, inputs, weights, biases, training, dropoutKeepRate)\n",
    "\n",
    "    for layerNum in range(numHiddenLayers+1):\n",
    "        if layerNum > 1:\n",
    "            hiddenLayer = generateHiddenLayer(layerNum, hiddenLayer, weights, biases, training, dropoutKeepRate)\n",
    "\n",
    "    return tf.matmul(hiddenLayer, weights['out']) + biases['out']\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, _imageSize * _imageSize)).astype(np.float32)\n",
    "    # Map 2 to [0.0, 1.0, 0.0 ...], 3 to [0.0, 0.0, 1.0 ...]\n",
    "    labels = (np.arange(_numLabels) == labels[:, None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "\n",
    "#source:  http://arxiv.org/pdf/1502.01852v1.pdf\n",
    "def calculateOptimalWeightStdDev(numPreviousLayerParams):\n",
    "    return math.sqrt(2.0/numPreviousLayerParams)\n",
    "\n",
    "def generateWeights(hiddenLayers, numInputs, numLabels):\n",
    "    numHiddenLayers = hiddenLayers.__len__()\n",
    "    validateNumHiddenLayers(numHiddenLayers)\n",
    "    weights = {}\n",
    "\n",
    "    numHiddenFeatures = hiddenLayers[0]\n",
    "    stddev = calculateOptimalWeightStdDev(numInputs)\n",
    "    weights[generateHiddenLayerKey(1)] = tf.Variable(tf.truncated_normal([numInputs, numHiddenFeatures], 0, stddev))\n",
    "\n",
    "    for layerNum in range(numHiddenLayers+1):\n",
    "        if layerNum > 1:\n",
    "            previousNumHiddenFeatures = numHiddenFeatures\n",
    "            numHiddenFeatures = hiddenLayers[layerNum-1]\n",
    "            stddev = calculateOptimalWeightStdDev(previousNumHiddenFeatures)\n",
    "            weights[generateHiddenLayerKey(layerNum)] = tf.Variable(tf.truncated_normal([previousNumHiddenFeatures, numHiddenFeatures], 0, stddev))\n",
    "\n",
    "    stddev = calculateOptimalWeightStdDev(numHiddenFeatures)\n",
    "    weights['out'] = tf.Variable(tf.truncated_normal([numHiddenFeatures, numLabels], 0, stddev))\n",
    "    return weights\n",
    "\n",
    "def generateBiases(hiddenLayers,  numLabels):\n",
    "    numHiddenLayers = hiddenLayers.__len__()\n",
    "    validateNumHiddenLayers(numHiddenLayers)\n",
    "    biases = {}\n",
    "\n",
    "    numHiddenFeatures = hiddenLayers[0]\n",
    "    biases[generateHiddenLayerKey(1)] = tf.Variable(tf.zeros([numHiddenFeatures]))\n",
    "\n",
    "    for layerNum in range(numHiddenLayers+1):\n",
    "        if layerNum > 1:\n",
    "            numHiddenFeatures = hiddenLayers[layerNum-1]\n",
    "            biases[generateHiddenLayerKey(layerNum)] = tf.Variable(tf.zeros([numHiddenFeatures]))\n",
    "\n",
    "    biases['out'] = tf.Variable(tf.zeros([numLabels]))\n",
    "    return biases\n",
    "\n",
    "def generateRegularizers(weights, biases, numHiddenLayers):\n",
    "    validateNumHiddenLayers(numHiddenLayers)\n",
    "    regularizers = tf.nn.l2_loss(weights['h1']) + tf.nn.l2_loss(biases['h1'])\n",
    "\n",
    "    for layerNum in range(numHiddenLayers+1):\n",
    "        if layerNum > 1:\n",
    "            regularizers = regularizers + tf.nn.l2_loss(weights['h' + str(layerNum)]) + tf.nn.l2_loss(biases['h' + str(layerNum)])\n",
    "\n",
    "    regularizers = regularizers + tf.nn.l2_loss(weights['out']) + tf.nn.l2_loss(biases['out'])\n",
    "    return regularizers\n",
    "\n",
    "def generateLossCalc(weights, biases, numHiddenLayers, trainingNetwork, trainingLabels, regularizationRate):\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=trainingNetwork, labels=trainingLabels))\n",
    "    regularizers = generateRegularizers(weights, biases, numHiddenLayers)\n",
    "    loss += regularizationRate * regularizers\n",
    "    return loss\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(None, X_train.shape[1]))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(None, y_train.shape[1]))\n",
    "    tf_valid_dataset = tf.constant(X_valid.values)\n",
    "    tf_valid_labels = tf.constant(y_valid)\n",
    "\n",
    "    numHiddenLayers = _hiddenLayers.__len__()\n",
    "    weights = generateWeights(_hiddenLayers, _numInputs, _numLabels)\n",
    "    biases = generateBiases(_hiddenLayers, _numLabels)\n",
    "    trainingNetwork = multilayerNetwork(tf_train_dataset, weights, biases, numHiddenLayers, True, _dropoutKeepRate)\n",
    "    loss = generateLossCalc(weights, biases, numHiddenLayers, trainingNetwork, tf_train_labels, _regularizationRate)\n",
    "    global_step = tf.Variable(0)  # count the number of steps taken.\n",
    "    learning_rate = tf.train.exponential_decay(_startLearningRate, global_step, _decaySteps, _learningDecayRate)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "\n",
    "    train_prediction = tf.nn.softmax(multilayerNetwork(tf_train_dataset, weights, biases, numHiddenLayers, True, _dropoutKeepRate))\n",
    "    valid_prediction = tf.nn.softmax(multilayerNetwork(tf_valid_dataset, weights, biases, numHiddenLayers, False, _dropoutKeepRate))\n",
    "    #test_prediction = tf.nn.softmax(multilayerNetwork(tf_test_dataset, weights, biases, numHiddenLayers, False, _dropoutKeepRate))\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.initialize_all_variables().run()\n",
    "    print(\"Initialized\")\n",
    "    for step in range(_numSteps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (np.random.randint(1, _trainSubset) * _batchSize) % (X_train.shape[0] - _batchSize)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = X_train.values[offset:(offset + _batchSize), :]\n",
    "        batch_labels = y_train[offset:(offset + _batchSize), :]\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "        _, l, predictions = session.run(\n",
    "            [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        \n",
    "#         valid_data = X_valid.values\n",
    "#         valid_labels = y_valid\n",
    "        \n",
    "#         feed_dict = {\n",
    "#             tf_train_dataset: valid_data,\n",
    "#             tf_train_labels: valid_labels\n",
    "#         }\n",
    "        \n",
    "#         valid_l, valid_predictions = session.run([loss, train_prediction], feed_dict=feed_dict)\n",
    "\n",
    "        if (step % 500 == 0):\n",
    "            print(\"Minibatch loss at step\", step, \":\", l)\n",
    "            print(\"Minibatch accuracy: %.4f\" % accuracy(predictions, batch_labels).eval())\n",
    "            print(\"Valid accuracy: %.4f\" % accuracy(valid_predictions, valid_labels).eval())            \n",
    "            #print \"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), valid_labels)\n",
    "\n",
    "#     print \"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 179 entries, 820 to 766\n",
      "Data columns (total 16 columns):\n",
      "Pclass         179 non-null int64\n",
      "Sex            179 non-null int64\n",
      "Age            179 non-null float64\n",
      "SibSp          179 non-null int64\n",
      "Parch          179 non-null int64\n",
      "Fare           179 non-null float64\n",
      "Cabin          179 non-null int64\n",
      "Embarked       179 non-null int64\n",
      "UnknownAge     179 non-null int64\n",
      "Baby           179 non-null int64\n",
      "Child          179 non-null int64\n",
      "Young          179 non-null int64\n",
      "FamilySize     179 non-null int64\n",
      "Alone          179 non-null int64\n",
      "Title          179 non-null int64\n",
      "TicketGroup    179 non-null float64\n",
      "dtypes: float64(3), int64(13)\n",
      "memory usage: 23.8 KB\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "_startLearningRate = 0.5\n",
    "_learningDecayRate = 0.98\n",
    "_decaySteps = 1000\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    \n",
    "    hidden_units = 128\n",
    "    is_training = tf.Variable(True, dtype=tf.bool)\n",
    "\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(None, X_train.shape[1]))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(None, y_train.shape[1]))\n",
    "\n",
    "    initializer = tf.contrib.layers.xavier_initializer()\n",
    "    fc = tf.layers.dense(tf_train_dataset, hidden_units, activation=None, kernel_initializer=initializer)\n",
    "    fc = tf.layers.batch_normalization(fc, training=is_training)\n",
    "    fc = tf.nn.relu(fc)\n",
    "    \n",
    "    dropout = tf.layers.dropout(inputs=fc, rate=0.4, training=is_training)        \n",
    "    logits = tf.layers.dense(dropout, units=1)    \n",
    "\n",
    "    loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "    global_step = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(_startLearningRate, global_step, _decaySteps, _learningDecayRate)\n",
    "    \n",
    "    with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "\n",
    "    train_prediction = tf.nn.sigmoid(logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 1.221464\n",
      "Minibatch accuracy: 0.3594\n",
      "Valid accuracy: 0.6313\n",
      "Minibatch loss at step 10: 1.595251\n",
      "Minibatch accuracy: 0.6797\n",
      "Valid accuracy: 0.6536\n",
      "Minibatch loss at step 20: 0.816950\n",
      "Minibatch accuracy: 0.6641\n",
      "Valid accuracy: 0.6034\n",
      "Minibatch loss at step 30: 0.488417\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.5978\n",
      "Minibatch loss at step 40: 0.516759\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.6034\n",
      "Minibatch loss at step 50: 0.511755\n",
      "Minibatch accuracy: 0.7266\n",
      "Valid accuracy: 0.6089\n",
      "Minibatch loss at step 60: 0.539856\n",
      "Minibatch accuracy: 0.7734\n",
      "Valid accuracy: 0.6034\n",
      "Minibatch loss at step 70: 0.448314\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.6369\n",
      "Minibatch loss at step 80: 0.446801\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.6313\n",
      "Minibatch loss at step 90: 0.358999\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.6034\n",
      "Minibatch loss at step 100: 0.478866\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.6592\n",
      "Minibatch loss at step 110: 0.416218\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.6536\n",
      "Minibatch loss at step 120: 0.363065\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.6592\n",
      "Minibatch loss at step 130: 0.377224\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.6536\n",
      "Minibatch loss at step 140: 0.419068\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.6592\n",
      "Minibatch loss at step 150: 0.472550\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.6480\n",
      "Minibatch loss at step 160: 0.434415\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.6313\n",
      "Minibatch loss at step 170: 0.435807\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.6983\n",
      "Minibatch loss at step 180: 0.376088\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 190: 0.352627\n",
      "Minibatch accuracy: 0.8594\n",
      "Valid accuracy: 0.7039\n",
      "Minibatch loss at step 200: 0.435364\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.6983\n",
      "Minibatch loss at step 210: 0.390932\n",
      "Minibatch accuracy: 0.8516\n",
      "Valid accuracy: 0.7095\n",
      "Minibatch loss at step 220: 0.400390\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.6760\n",
      "Minibatch loss at step 230: 0.340964\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7095\n",
      "Minibatch loss at step 240: 0.358786\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 250: 0.420889\n",
      "Minibatch accuracy: 0.7578\n",
      "Valid accuracy: 0.6704\n",
      "Minibatch loss at step 260: 0.322304\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 270: 0.383027\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.6816\n",
      "Minibatch loss at step 280: 0.383776\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.6760\n",
      "Minibatch loss at step 290: 0.468106\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.6983\n",
      "Minibatch loss at step 300: 0.405728\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.6760\n",
      "Minibatch loss at step 310: 0.416841\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.6872\n",
      "Minibatch loss at step 320: 0.341054\n",
      "Minibatch accuracy: 0.8516\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 330: 0.323603\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 340: 0.384437\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 350: 0.377620\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 360: 0.393406\n",
      "Minibatch accuracy: 0.7734\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 370: 0.343509\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 380: 0.331929\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 390: 0.411205\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7933\n",
      "Minibatch loss at step 400: 0.298360\n",
      "Minibatch accuracy: 0.8828\n",
      "Valid accuracy: 0.7207\n",
      "Minibatch loss at step 410: 0.358610\n",
      "Minibatch accuracy: 0.8516\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 420: 0.369423\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 430: 0.360171\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 440: 0.426413\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7039\n",
      "Minibatch loss at step 450: 0.429086\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.6983\n",
      "Minibatch loss at step 460: 0.426079\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 470: 0.373446\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 480: 0.388015\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7207\n",
      "Minibatch loss at step 490: 0.324364\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 500: 0.393757\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 510: 0.464922\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 520: 0.376218\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 530: 0.446888\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 540: 0.369360\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 550: 0.388900\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 560: 0.449119\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 570: 0.443555\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 580: 0.503924\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 590: 0.346073\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 600: 0.432762\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 610: 0.402633\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 620: 0.465302\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.7039\n",
      "Minibatch loss at step 630: 0.408123\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 640: 0.430096\n",
      "Minibatch accuracy: 0.7812\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 650: 0.428581\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 660: 0.501731\n",
      "Minibatch accuracy: 0.7109\n",
      "Valid accuracy: 0.6704\n",
      "Minibatch loss at step 670: 0.400196\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 680: 0.402558\n",
      "Minibatch accuracy: 0.7734\n",
      "Valid accuracy: 0.7207\n",
      "Minibatch loss at step 690: 0.395109\n",
      "Minibatch accuracy: 0.8516\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 700: 0.369250\n",
      "Minibatch accuracy: 0.8516\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 710: 0.347062\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.7318\n",
      "Minibatch loss at step 720: 0.399343\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 730: 0.358653\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 740: 0.337073\n",
      "Minibatch accuracy: 0.8359\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 750: 0.364741\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 760: 0.410835\n",
      "Minibatch accuracy: 0.7578\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 770: 0.385421\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7877\n",
      "Minibatch loss at step 780: 0.570804\n",
      "Minibatch accuracy: 0.7422\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 790: 0.375354\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.6872\n",
      "Minibatch loss at step 800: 0.393632\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 810: 0.354793\n",
      "Minibatch accuracy: 0.8281\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 820: 0.569886\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 830: 0.411760\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 840: 0.394888\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7151\n",
      "Minibatch loss at step 850: 0.365515\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 860: 0.456734\n",
      "Minibatch accuracy: 0.7578\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 870: 0.438441\n",
      "Minibatch accuracy: 0.7344\n",
      "Valid accuracy: 0.7989\n",
      "Minibatch loss at step 880: 0.366709\n",
      "Minibatch accuracy: 0.8125\n",
      "Valid accuracy: 0.7709\n",
      "Minibatch loss at step 890: 0.363239\n",
      "Minibatch accuracy: 0.8047\n",
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 900: 0.387943\n",
      "Minibatch accuracy: 0.8672\n",
      "Valid accuracy: 0.7765\n",
      "Minibatch loss at step 910: 0.310072\n",
      "Minibatch accuracy: 0.8672\n",
      "Valid accuracy: 0.7263\n",
      "Minibatch loss at step 920: 0.409305\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7654\n",
      "Minibatch loss at step 930: 0.393610\n",
      "Minibatch accuracy: 0.7891\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 940: 0.400342\n",
      "Minibatch accuracy: 0.7969\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid accuracy: 0.7486\n",
      "Minibatch loss at step 950: 0.423833\n",
      "Minibatch accuracy: 0.7109\n",
      "Valid accuracy: 0.7430\n",
      "Minibatch loss at step 960: 0.429996\n",
      "Minibatch accuracy: 0.7656\n",
      "Valid accuracy: 0.7821\n",
      "Minibatch loss at step 970: 0.367213\n",
      "Minibatch accuracy: 0.7969\n",
      "Valid accuracy: 0.7598\n",
      "Minibatch loss at step 980: 0.412767\n",
      "Minibatch accuracy: 0.7500\n",
      "Valid accuracy: 0.7542\n",
      "Minibatch loss at step 990: 0.314481\n",
      "Minibatch accuracy: 0.8438\n",
      "Valid accuracy: 0.7374\n",
      "Minibatch loss at step 1000: 0.349624\n",
      "Minibatch accuracy: 0.8203\n",
      "Valid accuracy: 0.7598\n"
     ]
    }
   ],
   "source": [
    "num_steps = 1001\n",
    "report_interval = 10\n",
    "x_collect = []\n",
    "train_loss_collect = []\n",
    "train_acc_collect = []\n",
    "valid_loss_collect = []\n",
    "valid_acc_collect = []\n",
    "    \n",
    "with tf.Session(graph=graph) as session:\n",
    "    \n",
    "    tf.global_variables_initializer().run()\n",
    "    saver = tf.train.Saver()\n",
    "    #session = tf_debug.LocalCLIDebugWrapperSession(session)\n",
    "    #session.add_tensor_filter(\"has_inf_or_nan\", tf_debug.has_inf_or_nan)\n",
    "    \n",
    "    print(\"Initialized\")\n",
    "    for step in range(num_steps):\n",
    "\n",
    "        offset = (step * batch_size) % (X_train.shape[0] - batch_size)\n",
    "\n",
    "        batch_data = X_train.values[offset:(offset + batch_size), :]\n",
    "        batch_labels = y_train[offset:(offset + batch_size), :]\n",
    "\n",
    "        feed_dict = { \n",
    "            tf_train_dataset: batch_data, \n",
    "            tf_train_labels: batch_labels, \n",
    "            is_training: True\n",
    "        }\n",
    "        \n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "\n",
    "        if (step % report_interval == 0):\n",
    "            x_collect.append(step)\n",
    "            train_loss_collect.append(l)\n",
    "            train_acc_collect.append(accuracy(predictions, batch_labels).eval())\n",
    "            print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "            print(\"Minibatch accuracy: %.4f\" % accuracy(predictions, batch_labels).eval())\n",
    "            #print(\"Validation accuracy: %.1f%%\" % accuracy(valid_prediction.eval(), y_valid))\n",
    "            #print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), y_test))\n",
    "            \n",
    "        valid_data = X_valid.values\n",
    "        valid_labels = y_valid\n",
    "        \n",
    "        feed_dict = {\n",
    "            tf_train_dataset: valid_data,\n",
    "            tf_train_labels: valid_labels,\n",
    "            is_training: False\n",
    "        }\n",
    "        \n",
    "        valid_l, valid_predictions = session.run([loss, train_prediction], feed_dict=feed_dict)\n",
    "\n",
    "        if (step % report_interval == 0):\n",
    "            valid_loss_collect.append(valid_l)\n",
    "            valid_acc_collect.append(accuracy(valid_predictions, valid_labels).eval())            \n",
    "            #print(\"Valid loss at step %d: %f\" % (step, valid_l))\n",
    "            print(\"Valid accuracy: %.4f\" % accuracy(valid_predictions, valid_labels).eval())\n",
    "            \n",
    "#         feed_dict = {\n",
    "#             tf_train_dataset: X_predict.values\n",
    "#         }\n",
    "        \n",
    "#         predict_result = session.run([train_prediction], feed_dict=feed_dict)        \n",
    "        \n",
    "    saver.save(session, \"./data/model.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}