Week7.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Esophagus Cancer Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "dataset_train = np.loadtxt('lung.csv',delimiter=',')\n",
    "dataset_test = np.loadtxt('lung_test.csv',delimiter=',')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split Label and Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = dataset_train[:,1:]\n",
    "y_train = dataset_train[:,0]\n",
    "X_test = dataset_test[:,1:]\n",
    "y_test = dataset_test[:,0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create SVM Model and Perform GridSearch with 10-CV Train Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': [1, 2, 4, 8, 16], 'gamma': [0.0625, 0.125, 0.25, 0.5, 0, 1, 2, 4, 8]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "clf = SVC()\n",
    "parameters = {'C':[1,2,4,8,16],\n",
    "              'gamma':[2**(-4),2**(-3),2**(-2),2**(-1),0,1,2,4,8]}\n",
    "clf = GridSearchCV(clf,parameters,scoring='accuracy',cv=10)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Best Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 8, 'gamma': 0.0625}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Best 10-CV Training Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.555379746835443"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5845588235294118"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove Low Variance Features\n",
    "\n",
    "If features having nearly constant or almost the same values for each sample data form different labels, this features is meaningless and should be remove "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(632, 734)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "sel = VarianceThreshold(threshold=0.05)\n",
    "sel.fit(X_train)\n",
    "X_train_sel = sel.transform(X_train)\n",
    "X_test_sel = sel.transform(X_test)\n",
    "X_train_sel.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try perform Grid-Search again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': [1, 2, 4, 8, 16], 'gamma': [0.0625, 0.125, 0.25, 0.5, 0, 1, 2, 4, 8, 16]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = SVC()\n",
    "parameters = {'C':[1,2,4,8,16],\n",
    "              'gamma':[2**(-4),2**(-3),2**(-2),2**(-1),0,1,2,4,8,16]}\n",
    "clf = GridSearchCV(clf,parameters,scoring='accuracy',cv=10)\n",
    "clf.fit(X_train_sel, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 8, 'gamma': 0.0625}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check best 10-CV Training Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5537974683544303"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6066176470588235"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test_sel)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter Method\n",
    "\n",
    "Select feature based-on external measurement of each features between different labels and try to find feature that significantly different between labels such as chi2, F-Score or Mutual Information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use SelectionKBest to select 80 Best feature based-on F-Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(632, 80)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import f_classif\n",
    "sel2 = SelectKBest(f_classif, k=80)\n",
    "X_train_sel2 = sel2.fit(X_train_sel,y_train)\n",
    "X_train_sel2 = sel2.transform(X_train_sel)\n",
    "X_test_sel2 = sel2.transform(X_test_sel)\n",
    "X_train_sel2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try to perform Grid-Search again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': [1, 2, 4, 8, 16], 'gamma': [0.0625, 0.125, 0.25, 0.5, 0, 1, 2, 4, 8, 16]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = SVC()\n",
    "parameters = {'C':[1,2,4,8,16],\n",
    "              'gamma':[2**(-4),2**(-3),2**(-2),2**(-1),0,1,2,4,8,16]}\n",
    "clf = GridSearchCV(clf,parameters,scoring='accuracy',cv=10)\n",
    "clf.fit(X_train_sel2, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check best parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 2, 'gamma': 0.0625}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check best 10-CV Training Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.564873417721519"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6470588235294118"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test_sel2)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrapper Method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluated best set of features based on model performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Feed Forward Feature Selection to choose features based on SVM 10-CV Training Accuracy Performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note: This is my own Feed Forward Feature Selection Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "def FF_selection(clf, X, y, cv, k):\n",
    "    feature_choose = []\n",
    "    feature_index = list(range(0,X.shape[1]))\n",
    "    while(len(feature_choose) != k):\n",
    "        feature_accuracy = [cross_val_score(clf, X[:,feature_choose+[i]], y, cv = cv, \n",
    "                                            scoring = 'accuracy').mean() for i in feature_index]\n",
    "        current_max_index = np.argmax(feature_accuracy)\n",
    "        feature_choose.append(feature_index[current_max_index])\n",
    "        feature_index.pop(current_max_index)\n",
    "    return feature_choose, feature_accuracy[current_max_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creat simple SVM model to perform feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = SVC(C=1, gamma='auto')\n",
    "feature_choose, accuracy = FF_selection(clf, X_train_sel2, y_train, 10, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check choosing features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[14, 30, 19, 51, 38, 35, 11, 62, 45, 74]"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_choose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check 10-CV Training Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6208701356886841"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = SVC(C=1, gamma='auto')\n",
    "cross_val_score(clf,X_train_sel2[:,feature_choose], y_train, cv = 10, scoring = 'accuracy').mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6433823529411765"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train_sel2[:,feature_choose], y_train)\n",
    "y_pred = clf.predict(X_test_sel2[:,feature_choose])\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimize parameters by grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'C': [1, 2, 4, 8, 16], 'gamma': [0.0625, 0.125, 0.25, 0.5, 0, 1, 2, 4, 8, 16]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = SVC()\n",
    "parameters = {'C':[1,2,4,8,16],\n",
    "              'gamma':[2**(-4),2**(-3),2**(-2),2**(-1),0,1,2,4,8,16]}\n",
    "clf = GridSearchCV(clf,parameters,scoring='accuracy',cv=10)\n",
    "clf.fit(X_train_sel2[:, feature_choose], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 2, 'gamma': 0.0625}"
      ]
     },
     "execution_count": 299,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6186708860759493"
      ]
     },
     "execution_count": 300,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6617647058823529"
      ]
     },
     "execution_count": 301,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test_sel2[:, feature_choose])\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ex 1. Write program to perform Backward Feature Selection\n",
    "1. Start with all features\n",
    "2. Try to remove features 1 by 1 to find a best removing feature\n",
    "3. If number of features = k Then end Else go to 2.\n",
    "   \n",
    "Hint: use set to subtract list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BW_selection(clf, X, y, cv, k):\n",
    "    feature_index = \"\"\" ### Your Code ### \"\"\"\n",
    "    while(len(feature_index) != k):\n",
    "        feature_accuracy = [cross_val_score(clf, X[:, \"\"\" ### Your Code ### \"\"\" ], y, cv = cv, \n",
    "                                            scoring = 'accuracy').mean() for i in feature_index]\n",
    "        \"\"\" ### Your Code ### \"\"\"\n",
    "        \"\"\" ### Your Code ### \"\"\"\n",
    "    return feature_index, feature_accuracy[current_max_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = SVC(C=1, gamma='auto')\n",
    "feature_index, accuracy = BW_selection(clf, X_train_sel2, y_train, 10,50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(feature_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6034554211469534"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning using Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Neural Network vs. Deep Learning Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Deep](https://ackstorm.com/wp-content/uploads/2019/02/deep-learning.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Component of neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![component](https://upload.wikimedia.org/wikipedia/commons/6/60/ArtificialNeuronModel_english.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Activation Function\n",
    "\n",
    "https://en.wikipedia.org/wiki/Activation_function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keras is a high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano. It was developed with a focus on enabling fast experimentation. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![Keras](https://raw.github.com/plenoi/CMU_DataScience/master/keras.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Install Keras by <br>\n",
    "1. Open Anaconda-Navigator <br>\n",
    "2. Create new environment called \"Keras\" using python 3.5 <br>\n",
    "3. Install Keras "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load Keras component for create model & KerasClassifier to work with Scikit-Learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Deep Learning model <br> \n",
    "Model Type = Sequential <br>\n",
    "Model Config = [734, 12, 8, 1] <br>\n",
    "Binary Classifier Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(12, input_dim=X_train_sel.shape[1], activation='relu') )\n",
    "    model.add(Dense(8, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Model with training configuration <br>\n",
    "epochs = 200 <br>\n",
    "batch_size = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = KerasClassifier(build_fn=create_model, epochs=200, batch_size=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Deep Learning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "632/632 [==============================] - 0s 728us/step - loss: 0.6993 - acc: 0.4968\n",
      "Epoch 2/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.6864 - acc: 0.5823\n",
      "Epoch 3/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6756 - acc: 0.6123\n",
      "Epoch 4/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.6724 - acc: 0.6076\n",
      "Epoch 5/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.6677 - acc: 0.6250\n",
      "Epoch 6/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.6613 - acc: 0.6108\n",
      "Epoch 7/200\n",
      "632/632 [==============================] - 0s 98us/step - loss: 0.6580 - acc: 0.6234\n",
      "Epoch 8/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.6556 - acc: 0.6218\n",
      "Epoch 9/200\n",
      "632/632 [==============================] - 0s 112us/step - loss: 0.6509 - acc: 0.6203\n",
      "Epoch 10/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.6452 - acc: 0.6313\n",
      "Epoch 11/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.6469 - acc: 0.6187\n",
      "Epoch 12/200\n",
      "632/632 [==============================] - 0s 89us/step - loss: 0.6428 - acc: 0.6313\n",
      "Epoch 13/200\n",
      "632/632 [==============================] - 0s 120us/step - loss: 0.6386 - acc: 0.6282\n",
      "Epoch 14/200\n",
      "632/632 [==============================] - 0s 108us/step - loss: 0.6399 - acc: 0.6171\n",
      "Epoch 15/200\n",
      "632/632 [==============================] - 0s 104us/step - loss: 0.6392 - acc: 0.6218\n",
      "Epoch 16/200\n",
      "632/632 [==============================] - 0s 128us/step - loss: 0.6354 - acc: 0.6361\n",
      "Epoch 17/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.6353 - acc: 0.6203\n",
      "Epoch 18/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.6324 - acc: 0.6503\n",
      "Epoch 19/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.6287 - acc: 0.6361\n",
      "Epoch 20/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.6309 - acc: 0.6203\n",
      "Epoch 21/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.6243 - acc: 0.6440\n",
      "Epoch 22/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.6255 - acc: 0.6139\n",
      "Epoch 23/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.6213 - acc: 0.6361\n",
      "Epoch 24/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.6201 - acc: 0.6519\n",
      "Epoch 25/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6259 - acc: 0.6329\n",
      "Epoch 26/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.6178 - acc: 0.6503\n",
      "Epoch 27/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.6069 - acc: 0.6741\n",
      "Epoch 28/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.6234 - acc: 0.6313\n",
      "Epoch 29/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.6151 - acc: 0.6661\n",
      "Epoch 30/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.6087 - acc: 0.6582\n",
      "Epoch 31/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6138 - acc: 0.6566\n",
      "Epoch 32/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6070 - acc: 0.6820\n",
      "Epoch 33/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.6084 - acc: 0.6709\n",
      "Epoch 34/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.6066 - acc: 0.6582\n",
      "Epoch 35/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.6059 - acc: 0.6551\n",
      "Epoch 36/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.6037 - acc: 0.6677\n",
      "Epoch 37/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.6116 - acc: 0.6487\n",
      "Epoch 38/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.6153 - acc: 0.6503\n",
      "Epoch 39/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.6065 - acc: 0.6472\n",
      "Epoch 40/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.6079 - acc: 0.6804\n",
      "Epoch 41/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6040 - acc: 0.6630\n",
      "Epoch 42/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.6041 - acc: 0.6456\n",
      "Epoch 43/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5966 - acc: 0.6804\n",
      "Epoch 44/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5997 - acc: 0.6456\n",
      "Epoch 45/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5960 - acc: 0.6566\n",
      "Epoch 46/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5981 - acc: 0.6772\n",
      "Epoch 47/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5999 - acc: 0.6472\n",
      "Epoch 48/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5954 - acc: 0.6630\n",
      "Epoch 49/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5943 - acc: 0.6725\n",
      "Epoch 50/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5983 - acc: 0.6566\n",
      "Epoch 51/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5892 - acc: 0.6899\n",
      "Epoch 52/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5887 - acc: 0.6598\n",
      "Epoch 53/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.6045 - acc: 0.6566\n",
      "Epoch 54/200\n",
      "632/632 [==============================] - 0s 119us/step - loss: 0.5995 - acc: 0.6456\n",
      "Epoch 55/200\n",
      "632/632 [==============================] - 0s 98us/step - loss: 0.5922 - acc: 0.6582\n",
      "Epoch 56/200\n",
      "632/632 [==============================] - 0s 108us/step - loss: 0.5877 - acc: 0.6915\n",
      "Epoch 57/200\n",
      "632/632 [==============================] - 0s 122us/step - loss: 0.5871 - acc: 0.6646\n",
      "Epoch 58/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.6040 - acc: 0.6551\n",
      "Epoch 59/200\n",
      "632/632 [==============================] - 0s 104us/step - loss: 0.6049 - acc: 0.6361\n",
      "Epoch 60/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5919 - acc: 0.6709\n",
      "Epoch 61/200\n",
      "632/632 [==============================] - 0s 90us/step - loss: 0.5813 - acc: 0.6804\n",
      "Epoch 62/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5910 - acc: 0.6883\n",
      "Epoch 63/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5829 - acc: 0.6772\n",
      "Epoch 64/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5882 - acc: 0.6772\n",
      "Epoch 65/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5795 - acc: 0.6867\n",
      "Epoch 66/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.5925 - acc: 0.6614\n",
      "Epoch 67/200\n",
      "632/632 [==============================] - 0s 122us/step - loss: 0.5837 - acc: 0.6772\n",
      "Epoch 68/200\n",
      "632/632 [==============================] - 0s 108us/step - loss: 0.5820 - acc: 0.6709\n",
      "Epoch 69/200\n",
      "632/632 [==============================] - 0s 101us/step - loss: 0.5815 - acc: 0.6899\n",
      "Epoch 70/200\n",
      "632/632 [==============================] - 0s 98us/step - loss: 0.5883 - acc: 0.6820\n",
      "Epoch 71/200\n",
      "632/632 [==============================] - 0s 93us/step - loss: 0.5909 - acc: 0.6788\n",
      "Epoch 72/200\n",
      "632/632 [==============================] - 0s 111us/step - loss: 0.5792 - acc: 0.6867\n",
      "Epoch 73/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.5812 - acc: 0.6677\n",
      "Epoch 74/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5812 - acc: 0.6946\n",
      "Epoch 75/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5720 - acc: 0.6915\n",
      "Epoch 76/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5851 - acc: 0.6804\n",
      "Epoch 77/200\n",
      "632/632 [==============================] - 0s 122us/step - loss: 0.5878 - acc: 0.6756\n",
      "Epoch 78/200\n",
      "632/632 [==============================] - 0s 103us/step - loss: 0.5827 - acc: 0.6693\n",
      "Epoch 79/200\n",
      "632/632 [==============================] - 0s 115us/step - loss: 0.5833 - acc: 0.7057\n",
      "Epoch 80/200\n",
      "632/632 [==============================] - 0s 103us/step - loss: 0.5712 - acc: 0.7073\n",
      "Epoch 81/200\n",
      "632/632 [==============================] - 0s 109us/step - loss: 0.5852 - acc: 0.6772\n",
      "Epoch 82/200\n",
      "632/632 [==============================] - 0s 85us/step - loss: 0.5828 - acc: 0.6915\n",
      "Epoch 83/200\n",
      "632/632 [==============================] - 0s 106us/step - loss: 0.5792 - acc: 0.6661\n",
      "Epoch 84/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "632/632 [==============================] - 0s 122us/step - loss: 0.5800 - acc: 0.6851\n",
      "Epoch 85/200\n",
      "632/632 [==============================] - 0s 98us/step - loss: 0.5685 - acc: 0.6899\n",
      "Epoch 86/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.5799 - acc: 0.6851\n",
      "Epoch 87/200\n",
      "632/632 [==============================] - 0s 95us/step - loss: 0.5754 - acc: 0.6661\n",
      "Epoch 88/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.5642 - acc: 0.6946\n",
      "Epoch 89/200\n",
      "632/632 [==============================] - 0s 95us/step - loss: 0.5796 - acc: 0.6677\n",
      "Epoch 90/200\n",
      "632/632 [==============================] - 0s 108us/step - loss: 0.5678 - acc: 0.7120\n",
      "Epoch 91/200\n",
      "632/632 [==============================] - 0s 103us/step - loss: 0.5753 - acc: 0.6677\n",
      "Epoch 92/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.5683 - acc: 0.6883\n",
      "Epoch 93/200\n",
      "632/632 [==============================] - 0s 89us/step - loss: 0.5805 - acc: 0.6867\n",
      "Epoch 94/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5951 - acc: 0.6788\n",
      "Epoch 95/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.5655 - acc: 0.6835\n",
      "Epoch 96/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5715 - acc: 0.6851\n",
      "Epoch 97/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5743 - acc: 0.6946\n",
      "Epoch 98/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.5678 - acc: 0.6867\n",
      "Epoch 99/200\n",
      "632/632 [==============================] - 0s 93us/step - loss: 0.5853 - acc: 0.6693\n",
      "Epoch 100/200\n",
      "632/632 [==============================] - 0s 136us/step - loss: 0.5662 - acc: 0.7041\n",
      "Epoch 101/200\n",
      "632/632 [==============================] - 0s 112us/step - loss: 0.5586 - acc: 0.6946\n",
      "Epoch 102/200\n",
      "632/632 [==============================] - 0s 123us/step - loss: 0.5667 - acc: 0.6930\n",
      "Epoch 103/200\n",
      "632/632 [==============================] - 0s 96us/step - loss: 0.5594 - acc: 0.6867\n",
      "Epoch 104/200\n",
      "632/632 [==============================] - 0s 95us/step - loss: 0.5564 - acc: 0.7073\n",
      "Epoch 105/200\n",
      "632/632 [==============================] - 0s 85us/step - loss: 0.5628 - acc: 0.6930\n",
      "Epoch 106/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5747 - acc: 0.6804\n",
      "Epoch 107/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5686 - acc: 0.6788\n",
      "Epoch 108/200\n",
      "632/632 [==============================] - 0s 79us/step - loss: 0.5591 - acc: 0.6899\n",
      "Epoch 109/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5565 - acc: 0.6899\n",
      "Epoch 110/200\n",
      "632/632 [==============================] - 0s 62us/step - loss: 0.5557 - acc: 0.6994\n",
      "Epoch 111/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5570 - acc: 0.6962\n",
      "Epoch 112/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5701 - acc: 0.7025\n",
      "Epoch 113/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5579 - acc: 0.6835\n",
      "Epoch 114/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5594 - acc: 0.7104\n",
      "Epoch 115/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5591 - acc: 0.6915\n",
      "Epoch 116/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5571 - acc: 0.7025\n",
      "Epoch 117/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5739 - acc: 0.6867\n",
      "Epoch 118/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5660 - acc: 0.6867\n",
      "Epoch 119/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5745 - acc: 0.6867\n",
      "Epoch 120/200\n",
      "632/632 [==============================] - 0s 89us/step - loss: 0.5490 - acc: 0.7152\n",
      "Epoch 121/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5576 - acc: 0.6978\n",
      "Epoch 122/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5509 - acc: 0.6978\n",
      "Epoch 123/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5612 - acc: 0.6978\n",
      "Epoch 124/200\n",
      "632/632 [==============================] - 0s 98us/step - loss: 0.5565 - acc: 0.7152\n",
      "Epoch 125/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5556 - acc: 0.7009\n",
      "Epoch 126/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5661 - acc: 0.6978\n",
      "Epoch 127/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5585 - acc: 0.6835\n",
      "Epoch 128/200\n",
      "632/632 [==============================] - 0s 100us/step - loss: 0.5520 - acc: 0.7089\n",
      "Epoch 129/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5563 - acc: 0.7152\n",
      "Epoch 130/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5558 - acc: 0.6899\n",
      "Epoch 131/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5400 - acc: 0.7104\n",
      "Epoch 132/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5512 - acc: 0.7120\n",
      "Epoch 133/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5475 - acc: 0.7009\n",
      "Epoch 134/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5515 - acc: 0.6946\n",
      "Epoch 135/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5404 - acc: 0.7089\n",
      "Epoch 136/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5378 - acc: 0.7168\n",
      "Epoch 137/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5537 - acc: 0.6930\n",
      "Epoch 138/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5484 - acc: 0.6962\n",
      "Epoch 139/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5542 - acc: 0.6915\n",
      "Epoch 140/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5582 - acc: 0.6930\n",
      "Epoch 141/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5608 - acc: 0.6930\n",
      "Epoch 142/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5460 - acc: 0.7009\n",
      "Epoch 143/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5467 - acc: 0.7089\n",
      "Epoch 144/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5421 - acc: 0.7073\n",
      "Epoch 145/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5419 - acc: 0.7009\n",
      "Epoch 146/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5524 - acc: 0.6835\n",
      "Epoch 147/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5393 - acc: 0.7152\n",
      "Epoch 148/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5568 - acc: 0.6946\n",
      "Epoch 149/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5452 - acc: 0.7057\n",
      "Epoch 150/200\n",
      "632/632 [==============================] - 0s 85us/step - loss: 0.5382 - acc: 0.7073\n",
      "Epoch 151/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5431 - acc: 0.6883\n",
      "Epoch 152/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5318 - acc: 0.7152\n",
      "Epoch 153/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5385 - acc: 0.7057\n",
      "Epoch 154/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5505 - acc: 0.7120\n",
      "Epoch 155/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5383 - acc: 0.7231\n",
      "Epoch 156/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5380 - acc: 0.6978\n",
      "Epoch 157/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5495 - acc: 0.7089\n",
      "Epoch 158/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5464 - acc: 0.6915\n",
      "Epoch 159/200\n",
      "632/632 [==============================] - 0s 96us/step - loss: 0.5451 - acc: 0.6930\n",
      "Epoch 160/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5409 - acc: 0.7041\n",
      "Epoch 161/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5393 - acc: 0.7104\n",
      "Epoch 162/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5315 - acc: 0.7215\n",
      "Epoch 163/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5446 - acc: 0.6962\n",
      "Epoch 164/200\n",
      "632/632 [==============================] - 0s 74us/step - loss: 0.5415 - acc: 0.6978\n",
      "Epoch 165/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5265 - acc: 0.7358\n",
      "Epoch 166/200\n",
      "632/632 [==============================] - 0s 62us/step - loss: 0.5228 - acc: 0.7278\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 167/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5355 - acc: 0.7104\n",
      "Epoch 168/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5379 - acc: 0.7025\n",
      "Epoch 169/200\n",
      "632/632 [==============================] - 0s 62us/step - loss: 0.5231 - acc: 0.7247\n",
      "Epoch 170/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5366 - acc: 0.7215\n",
      "Epoch 171/200\n",
      "632/632 [==============================] - 0s 63us/step - loss: 0.5323 - acc: 0.7231\n",
      "Epoch 172/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5272 - acc: 0.7215\n",
      "Epoch 173/200\n",
      "632/632 [==============================] - 0s 62us/step - loss: 0.5361 - acc: 0.7073\n",
      "Epoch 174/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5335 - acc: 0.7136\n",
      "Epoch 175/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5296 - acc: 0.7120\n",
      "Epoch 176/200\n",
      "632/632 [==============================] - 0s 76us/step - loss: 0.5328 - acc: 0.7104\n",
      "Epoch 177/200\n",
      "632/632 [==============================] - 0s 100us/step - loss: 0.5516 - acc: 0.6899\n",
      "Epoch 178/200\n",
      "632/632 [==============================] - 0s 100us/step - loss: 0.5271 - acc: 0.7168\n",
      "Epoch 179/200\n",
      "632/632 [==============================] - 0s 84us/step - loss: 0.5262 - acc: 0.7120\n",
      "Epoch 180/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5209 - acc: 0.7152\n",
      "Epoch 181/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5336 - acc: 0.7073\n",
      "Epoch 182/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5276 - acc: 0.7231\n",
      "Epoch 183/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5223 - acc: 0.7168\n",
      "Epoch 184/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5359 - acc: 0.7168\n",
      "Epoch 185/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5239 - acc: 0.7104\n",
      "Epoch 186/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5182 - acc: 0.7199\n",
      "Epoch 187/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5396 - acc: 0.7009\n",
      "Epoch 188/200\n",
      "632/632 [==============================] - 0s 63us/step - loss: 0.5240 - acc: 0.7231\n",
      "Epoch 189/200\n",
      "632/632 [==============================] - 0s 70us/step - loss: 0.5130 - acc: 0.7215\n",
      "Epoch 190/200\n",
      "632/632 [==============================] - 0s 60us/step - loss: 0.5183 - acc: 0.7215\n",
      "Epoch 191/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5206 - acc: 0.7484\n",
      "Epoch 192/200\n",
      "632/632 [==============================] - 0s 65us/step - loss: 0.5164 - acc: 0.7168\n",
      "Epoch 193/200\n",
      "632/632 [==============================] - 0s 63us/step - loss: 0.5430 - acc: 0.7025\n",
      "Epoch 194/200\n",
      "632/632 [==============================] - 0s 71us/step - loss: 0.5096 - acc: 0.7373\n",
      "Epoch 195/200\n",
      "632/632 [==============================] - 0s 66us/step - loss: 0.5209 - acc: 0.7326\n",
      "Epoch 196/200\n",
      "632/632 [==============================] - 0s 68us/step - loss: 0.5150 - acc: 0.7263\n",
      "Epoch 197/200\n",
      "632/632 [==============================] - 0s 73us/step - loss: 0.5150 - acc: 0.7231\n",
      "Epoch 198/200\n",
      "632/632 [==============================] - 0s 81us/step - loss: 0.5051 - acc: 0.7468\n",
      "Epoch 199/200\n",
      "632/632 [==============================] - 0s 77us/step - loss: 0.5081 - acc: 0.7342\n",
      "Epoch 200/200\n",
      "632/632 [==============================] - 0s 82us/step - loss: 0.5180 - acc: 0.7120\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x24f1be033c8>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train_sel, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss & Accuracy\n",
    "Loss is often used in the training process to find the \"best\" parameter values for your model <br>\n",
    "Accuracy is when you apply model using the optimized parameters above to see how accurate of your model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Loss Trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x24f25e18400>]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNXdx/HPL5ONhCRsYV8CsiOLEFlUcENEVNDWqlj3tVVcHm19rPZR3GqrrVpbraJSF9yoKyriQlEUZd93QsIStkACCWQhycx5/pghDZDAoGESbr/v1yuvzL33cOfHmck3N+fee8acc4iIiLdE1XYBIiJS8xTuIiIepHAXEfEghbuIiAcp3EVEPEjhLiLiQYcNdzMbb2Y5Zra0mu1mZs+YWYaZLTazvjVfpoiIHIlwjtxfAYYfYvs5QKfQ143AP356WSIi8lMcNtydc9OBvEM0GQW85oJmAg3MrEVNFSgiIkcuugb20QrYWGk5O7Ruy4ENzexGgkf3JCYm9uvatWsNPL2IyH+PefPm7XDOpR6uXU2Eu1Wxrso5DZxz44BxAOnp6W7u3Lk18PQiIv89zGx9OO1q4mqZbKBNpeXWwOYa2K+IiPxINRHuk4ArQ1fNDATynXMHDcmIiEjkHHZYxszeAk4DmphZNvAAEAPgnHsemAyMADKAIuCao1WsiIiE57Dh7pwbfZjtDrilxioSEZGfTHeoioh4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAXEfEghbuIiAcp3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIeFFa4m9lwM1tlZhlmdk8V29ua2TQzW2Bmi81sRM2XKiIi4TpsuJuZD3gWOAfoDow2s+4HNPs9MNE5dwJwKfBcTRcqIiLhC+fIvT+Q4ZzLdM6VAm8Dow5o44Dk0OMUYHPNlSgiIkcqnHBvBWystJwdWlfZWOByM8sGJgO3VrUjM7vRzOaa2dzt27f/iHJFRCQc4YS7VbHOHbA8GnjFOdcaGAG8bmYH7ds5N845l+6cS09NTT3yakVEJCzhhHs20KbScmsOHna5DpgI4Jz7AYgHmtREgSIicuTCCfc5QCcza29msQRPmE46oM0G4EwAM+tGMNw17iIiUksOG+7OuXJgDPA5sILgVTHLzOwhMxsZanYXcIOZLQLeAq52zh04dCMiIhESHU4j59xkgidKK6+7v9Lj5cDJNVuaiIj8WLpDVUTEgxTuIiIepHAXEfEghbuIiAcp3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kHHZLjnFJTUdgkiInXaMRfu//h6LWf+5RvyCktruxQRkTrrmAv3od2aUlhazt/+vaa2SxERqbOOuXDv1CyJS05sw4SZ69mQW1Tb5YiI1EnHXLgD3DG0M9FRUTz0yXL0Ua0iIgc7JsO9WXI8/3NWJ75asY1Pl2yp7XJEROqcYzLcAa49uT29Wqfw+w+XMmHmekrK/LVdkohInXHMhnu0L4qnLulD20YJ/P7DpQx/ejpz1uXVdlkiInXCMRvuAMel1uejW07mtWv7Ux5wXPzCD7w+c31tlyUiUuuO6XAHMDOGdE7l8zuGcEaXpvzfh0u594MlbNpVXNuliYjUmmM+3PdJjIvmhSv6cfVJaUycs5Ehj0/jipdn8cWyrbVdmohIxFltXUqYnp7u5s6de1T2nb2ziDdmbeCTxZvZmFfMub1a8IcLe5JSL+aoPJ+ISKSY2TznXPph23kx3Pcp9wd4YXomT325mraNErh+cAcyt+/h/N4t6d2mwVF9bhGRo0HhXsnsrDx+PWEeuYWlmAXXXZLeht+e3YUGCbEUl/lJjPVh+zaKiNRRCvcD5BeVsW13Cc2S4/n7v9fwzxnriIuOwu8cJWUBEmJ9jOzdkhuGdCClXgyNE2MV9iJS5yjcDyMjZzfjpmeSFB9DalIcmdv38MGCTZT5g/1xXGoiVw5K45IT2xDri6LUHyA+xldr9YqIgML9R9mYV8S3a3ZQVFrOJ4u3sHDjLponxxNwjp1Fpfz5F70Z1afVQf9uXx/qSF9EjjaFew34fu0Onv8mk4QYHzv27GXehp10aZZEQXEZ1w/uwBWD2rGtoIRfT5hPQqyPV6/tT3yMD3/A8fGizZzepSkpCbpCR0RqjsK9hpWU+Xnw4+VsyS+muNTPrKw84mOi8JlhZhSWljO8R3Oevawvk5duYcybC+if1ojXr+9PXLSGc0SkZoQb7tGRKMYL4mN8PPaznkBwGObr1dv5bs0OdhaWcssZHZm2ModHPl3B+BlZTFm6leT4aGavy+O3/1rMkxf3ZtOuYjbtKuak45oc8nnW7Sjkhelr6deuERf1ax2J/5qIeFBY4W5mw4G/Aj7gJefcH6toczEwFnDAIufcZTVYZ51iZpzepSmnd2lasa5Dk0R+WJvL41NWUeoP8Ptzu1Hmd/xpykq25BezbHMBRaV+Hji/O+cc34L84jI6Nq2PL+o/4/QT527kd+8vwR9wTJybTcsG8fRq3YC46ChifJ65mVhEIuCwwzJm5gNWA2cB2cAcYLRzbnmlNp2AicAZzrmdZtbUOZdzqP0ea8My4diSX8ywJ6dTHnDMvPdMUurF8Or36xj78TL6pzUiuV4MXy7fVtE+KT6aDqn1adcogU5N6/P01DUM6tCYhy84nhtem8v63ELK/I7myfGMOaMjzZLjOb5VMi1S6v3oGhdu3EXjxFjaNEqocvv3a3ewautuerZKIT2t0Y9+HhE5OmpszN3MBgFjnXNnh5Z/B+Cce6xSm8eB1c65l8It0IvhDjBnXR57Sso5vet/juq35peQmhRHwDle/2E9viijflw08zbsZGNeESu37mb77r30aJnMOzcNon5cNOt2FPL3aRm0bZTA1BXbWJSdD0CDhBj+ddMgHFDud3RvmQwEf7Es2riLAe0b0zAxtuK5nXMVV/Hk7C7hlD9OI9pnjB3Zg4vT2+xX+6RFm7ntrQUVzzP97tNJjtcJYZG6pCbD/SJguHPu+tDyFcAA59yYSm0+JHh0fzLBoZuxzrkpVezrRuBGgLZt2/Zbv17T8wIEAo7lWwpo2zihyjB1Lrg9v6iM299ZyJ6ScopDH05ycsfG5BTsZU3OHgAaJ8Zy+9BOtGmUwEvfZpK5vZD3bz6JFin1ePKLVfxtWgZ92zZk3vqd3DuiKzcOOa7ieX723Ax2FpXx6IXHc9mLs7j5tOO4e3jXyHTCUeacY2ZmHgPaNyIqSpesyrEr3HAPZyC3qp+EA38jRAOdgNOA0cBLZnbQ5C3OuXHOuXTnXHpqamoYT/3fISrKOL5VSrVHyWZGj5YpnNSxCa9d259erVO4e3gXfjOsM+t2FNEsOZ57R3TllWtOpHXDetz/0TKu+ecclm0uoKC4jJvfmE9+URkTZm3gzK7NmHjTIM7r1YI/TF7JM1PXUOYPsHRTPvM37OLyge046bgmXNCnJeNnZLElvxh/wPHZki3sLinbr6695X7umriI6au3V1n3ii0FfLJ4c534nNtJizYz+sWZfFFpWEzEy8I5oZoNVP77vTWwuYo2M51zZUCWma0iGPZzaqRKqdCtRXDoZp8xZ3Tab/uQTqlk5RaSvbOYPq0bMGPtDm5+Yz69H/oCgOsHt8cXZTx1SR/MjCe/XM2HCzcRExVFfEwUF/UNXqFz17AuTFm2ld9/sJQ+bRrwly9Xc9JxjblrWGee+HwVvzr1ONZuL+S9+dl8vmwrH95yMh2b1gdgV1EpN70+j1lZwU/GCoyGkb1bHvR/2V1SxocLNjErK48rBrZjQIfG+23fVlBCw4RYYqN/2snkQMDx939nADAzM5fhxzc/qM3ecj97SsppXD/uJz2XSF0RzrBMNMEhlzOBTQQD+zLn3LJKbYYTPMl6lZk1ARYAfZxzudXt16tj7nXRtJU5LNmUT/24aK45OW2/O2mnLN3K+BlZbCsoYVSfVtx5VueKbS99m8kjn64AoHuLZJZvKajYVj8ummif0b5JIhvzijAzzu7RjLTGibw3fxNrt+/h7rO78OHCTeQU7OWtGweSkbOHQMCRntaIxomxXPjcDBZl55MQ66O4zM/JxzWh1B/gwZE9iDLj3Ge+JTEumnN7teDXpx5Hi5R4/M5V3DdQ+XzCoUxZupVfTZhHYqyPtCaJfHrb4IPa3PvBEj5fupUZ95yhaSakTqvRm5jMbATwNMHx9PHOuUfN7CFgrnNukgV/wv4CDAf8wKPOubcPtU+Fe93nDzhGvziT7bv3MmnMybw3L5vFm/K57pT2XPnybHILS/l4zCkEnOOpr1YzJyuPwlI/9WJ8jLuyH4M7pbJgw04ufO77/fbbqkE9bhzSgQcmLeMPF/ZkVJ+W/PGzlczfsJMNuUV0bp5Es+Q4pq/ewVndm/Hpki34A8H3acOEGD6/YwiTFm3myS9Xc9dZnbliUBpRBleOn023FsncO6JbxVBQwMG5z3xLcZmf83u15LmvM1j0wDCSKg2BFZSUMeDRqRSX+Xn2sr6c26tF5DpZ5AjpDlWpEWX+AP6AO+hodtXW3azJ2c15vf4z3BIIOHbvLSfGZyTE/mfEb+KcjeQVlTKgfSPyCkv59YT5lPoDdGuRzKe3nrLfCc535mzgf99bAsCtZ3TkrmFd2JJfzISZ6yn3O17+LoshnVP5fu0O4qJ95BeXcfnAtgzr3pwrx8/GDF69pj9PfL6KBgkxnNo5lUc+XcGzl/UlpV4Ml788i1ev7U+9GB8TZq7HAT1aJvPHz1aSGOtjQIfGjL/6xGr7IxBw3PLmfBolxvLA+T1+1JBRuT/AI5+uYFj3ZvRsncKtby3g0hPbVjlcJHIg3aEqNSLGF0VVoxRdmifRpXnSfuuioqzKT7u6+MT9L7m8//zujJ20jHtHdD3oypWL+rVhwswNrNtRyHWntAegRUo9fnt214rn+MfXa0mI9TH59sG8OD2TV75fx/cZuTRNiiPggkfwsb4oygMBvl2zg/5pjRjRszlFpX58UcbYScvI2lFIcnw0haV+Pl60meNbJTO4UyrjpmeyffdeUpPiyC8uIykuer8a352fzWdLgx/duD63iJeuSq9yGOfpr1azeVcxf/p5r4OGjqauzOGV79fxzpyN9GiZzNz1O3GOsMI93KEoEd32KBF3+cB2LLj/LAZ3OviKKV+UMeG6AUy69RQaJMQetP3WMzoyqENjxp7fg1YN6nH38C60alCPzB2FXHNyex44vzsJsT7+ftkJ/PXSE2jbKIEHRnbHzEiMi6ZHy2SydhTyywFtmX3fUJ77ZV8SYn3cMLgDP+/bCn/A8fRXq1m0cReDHpvKta/OYW958LLTgpIyHp+ykr5tG/D4Rb34LmMHD3+y/KAaM3J288zUNUycm13l1TkTZq6neXI8jRJjmbt+Jx2aJDIrK5eS0OWt1flm9Xb6PfIVU5bqc4Hl8DQsI8e8mZm5jJueyVOX9CGlXgzl/gDR1UzXMHddHpt2FTOyd8uKI+DK7R/5ZDkvfZdFfEwUCbHR5BWWcnqXVG4f2pmHP1nOgg07+eiWU+jZOoXHPlvBC99kct+Iblw2oC2JccE/hG96fS4zMnJpmhxHud9xaf825BTsJT7GR9tGCdz7wRJ+M6wzF5zQiqWb8omL9nHNK3OYcN0AerZKITHOh985rnhpNglxPh69sCf1Y6MZ9vQ3bCvYS6wvivFXn8gpnQ49T1G4lmTn8+bs9SzdVMBLV6XTLDm+RvYrR4fG3EV+hEDAcc/7i5m6Iod3bhrI92tzefiT5ZT5HbG+KJ6+tA8jegZPuJb5A1z58mx+yMwl1hdF64b1KCnzszm/hDvP6kzftg25/OVZACTFRVNS7qfM74jxGd/fcyapScHLLotKy+nz4Jec3jWVH9bm0j61Pr1bp/DaD+uJj4ki4CClXgx5haW8dm1/Hv5kOVk7CnlwZA+mrcphcXY+zVPieeKi3hWXo1anpMzP0k359GvXkO179vLQx8v5ZPEW6sX4KPUHuOakNH5/XveD/p1zDueCw2KPTV7Bdxk7eP/mkzTjaS1QuIv8BGX+QMVkbTm7S/hg/ibS0xrSr93+8+0455i3fidfrchhQ14hMb4oerduwOUD2xEbHUVGzm6a1I+jQUIsZf4Ac9bl4TM76Jr+y16cyfdrc2mQEEPRXj+l/gAXp7fm1jM68doP61iXW8RpXVL55YB27Cws5ep/zq64jHRY92Z8tSKHU7uk8tjPenLnO4tIT2tIhyaJvPRtFjcM6cBZ3ZsBMHbSMl75fh3n9mzB/A07ySss5aYhHbhhSAce+GgZn4UuB210wBQWY95cwI49e3nxqnQG/WEqhaV+7h7ehZtP61hl/2Xk7OGzJVu4+fSOFZPjzV2Xhxn0bdswoucNsnYUcufEhTz3y74/aV6mukLhLnIM+eeMLB6bvJIJ1w+g3B/g3fnZPDiyx36XbFa2Z285b8/ewIieLWjZoB6PTV7BS99lMbRbUz5f9p9x/iiDhgmxfHXnqURFGYMem0rz5HjW5RbSPDmeF69Kp0fLFADWbNvNWU9Np2PT+jSoF8OvTj2Ood2bMXHORu5+bzEAgzs14ds1O+jaPIn1uUV88T9DaNMogUDAVZx49gcco579jqWbCnjsZz0Z3b8t63MLOeup6ZSWB+jeIpkT0xpydo/mnNRx/6Gl7J1FbCvYS9+2DWrsF8BDHy9n/Iws/u+87hUn6Y9lCneRY4g/4MgvLtvviPlIbN5VzODHp+EPOK45OY0RPVuweVcxHZrU58LnZnD28c05LrU+z0xdwye3nkJcdBSpSXEHnbR+7LMVzMnKY2dRGVk7CunUtD7ZO4vp2TqFPSXlLN9SQNfmSbx0VTrnPP0tjerHctWgNJ7+ajW3D+3Mdae05+3ZG7jn/SXByfICjn//5jRue2sB89bv5I6hnZiydCvLtwSnwD6rezNG9GzOqZ2bkhwfzfC/fktGzh56tEzm+cv7VTt7abhKywMMfGwqeYWlDO7UhNevG3DE+1ifW0iz5PhD3tx2y5vzaZYUz/3nHzykVdN0KaTIMcQXZT862AFaNqjHqD4tmZWZx13DulA/7j8/2rec3pG/Tl0DwID2jTi+VUq1+/ndOd2AYCiOn5HFvPU76dI8iXvO6cra7YVcNX42vxzYjtYNE3j1uv5c9fJsHvpkOfXjonl8ykpaN6zHH6es5MS0hjxwfg/O//t39Hv4S8oDjvvP6861p7Tn+sEdKCnz8/J3WTz/zVq+XL6NFinxXHpiWzJy9nDVoHZMnJvNU1+u5slL+gCQU1DCy99l8fWq7fzPWZ05u0czxk3PJK1JIsO6N8PMyCko4YMFmzizW7OKcw9fr8ohr7CUrs2TmJWZR+He8ooT31XZvKuYjJw9DOkcvJJrd0kZw5/+lgtOaFXxYT0H2l1SxpSlW2meHJlwD5eO3EU8oswfoMwf2O8GMgiOmS/bXMCUpVsZ0bNFxTTRP8byzcEj931DMCu2FLBkUz4nd2zCsCe/obDUT8uUeN68YSBpTRL5avk25m3YSePEWK4+Ke2gq5j8AcfsrDxueG0ue/aWc0LbBrz/65N4+JMVvPbDOr7+7Wm0alCPS8bNZP76nSTE+mhSP47/O7871/wzOHXV0G5NeXDU8dz8xnwWbdwFBIePLuvflme/zmBr/l6evLg3V46fzYtXplecf9iQW8TVr8zm4vQ23DSkAwAXPvc9i7J3MSl0RdSHCzZxxzsLifVF8d09p9M06eAriaau2MZ1rwazbM59QytOlB8tGpYRkYh6d142b83ewF8v7UPrhkc2nDIjYwf3frCEJy/uTb92jdi8q5ghj0/jF+mtObVzU341YR6PXHA8jRJjufmN+STHR5NcL4YrBrbjyS9X4w84ygOOx3/ei7yiUp7/Zi27ispIjo/mwVE9OLdnS0546AuOb5XCxeltOLdXC+6auIhPl2wB4NxeLTi7R3Nue2sBviijT5sGvPurQRUT4BWUlDHm9OAd0wd68ONl/HPGOgBeviqdM7s1+8l9eSgKdxE5pt37wRLenLUBX5SR1jiBz+8Ygplx1lPfkLm9kCcu6sUv0tuwautufvf+Ys7o2rRiltT8ojLmrMtj0HGNK4Zhxk5axms/rCPgoENqIpnbC7n9zE7Ui/Xx+JSVBBy0a5zAjUM6cN8HS7ntjI68MD2T0f3bsnlXMbPX5fHNb04nJSF4krugpIz4aB/n/e1bkuJjWLBhJ2NO78j1QzrgQpevHg0KdxE5ppX7A3y0cDNvzFrPb87uUvHh8tNW5fDRgk38+Re9q71ZrTpl/gDTVuZw93uLiYuOYtpvTiMhNprpq7dz/0dLuXdEN4Z2a8aYt+YzeUnwTuCJNw0iIdbHqGdncHF6a36R3oYnpqxiVlYuzZLj2ZJfwj3ndOXDBZtomBBLbuFecveUMu7KfgddOlvmD/C/7y7m2lPaH/Lcx6Eo3EVEqpFXWEqZP1Dt3biBgOMf36xl4cZdPH95P3xRxh8mr2Dc9Eyio4zUpDguOKEV01bmsGrbbibfNphXv1/H23M2ApCaFEdeYSmJsT5O6dSEv43uiy/KePKLVTzz74yfNPuorpYREanG4a5Miooybjl9/xu07hjaie/W7KBtowT+9PNepCTEcNdZncneWUxak0R6t2nA23M2MqJncx69oCcvfZfJpp3FfLhwMz1arqVLsyT+Pi2Dn/dtHZFppRXuIiJhSIiNZvLt+3/QS7QvirQmiQAM7daMH3rnct+53WiYGMtvz+6Kc8ETvU98vgqADk0SGTsyMpdLKtxFRGpAalIcz4w+Yb91ZsajF/QkPsZH37YNueCElgddqnq0KNxFRI6ilIQY/vyL3hF/Xs3nLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ8KK9zNbLiZrTKzDDO75xDtLjIzZ2bpNVeiiIgcqcOGu5n5gGeBc4DuwGgzO+jju80sCbgNmFXTRYqIyJEJ58i9P5DhnMt0zpUCbwOjqmj3MPA4UFKD9YmIyI8QTri3AjZWWs4OratgZicAbZxznxxqR2Z2o5nNNbO527dvP+JiRUQkPOGEu1WxzlVsNIsCngLuOtyOnHPjnHPpzrn01NTU8KsUEZEjEk64ZwNtKi23BjZXWk4Cjge+NrN1wEBgkk6qiojUnnDCfQ7Qyczam1kscCkwad9G51y+c66Jcy7NOZcGzARGOufmHpWKRUTksA4b7s65cmAM8DmwApjonFtmZg+Z2cijXaCIiBy56HAaOecmA5MPWHd/NW1P++lliYjIT6E7VEVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAXEfEghbuIiAcp3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kFhhbuZDTezVWaWYWb3VLH9TjNbbmaLzWyqmbWr+VJFRCRchw13M/MBzwLnAN2B0WbW/YBmC4B051wv4F3g8ZouVEREwhfOkXt/IMM5l+mcKwXeBkZVbuCcm+acKwotzgRa12yZIiJyJMIJ91bAxkrL2aF11bkO+KyqDWZ2o5nNNbO527dvD79KERE5IuGEu1WxzlXZ0OxyIB14oqrtzrlxzrl051x6ampq+FWKiMgRiQ6jTTbQptJya2DzgY3MbChwH3Cqc25vzZQnIiI/RjhH7nOATmbW3sxigUuBSZUbmNkJwAvASOdcTs2XKSIiR+Kw4e6cKwfGAJ8DK4CJzrllZvaQmY0MNXsCqA/8y8wWmtmkanYnIiIREM6wDM65ycDkA9bdX+nx0BquS0REfgLdoSoi4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAXEfEghbuIiAcp3EVEPEjhLiLiQQp3EREPUriLiHiQwl1ExIMU7iIiHqRwFxHxIIW7iIgHKdxFRDxI4S4i4kEKdxERD1K4i4h4kMJdRMSDFO4iIh6kcBcR8SCFu4iIByncRUQ8SOEuIuJBCncREQ9SuIuIeFBY4W5mw81slZllmNk9VWyPM7N3QttnmVlaTRcqIiLhO2y4m5kPeBY4B+gOjDaz7gc0uw7Y6ZzrCDwF/KmmCxURkfCFc+TeH8hwzmU650qBt4FRB7QZBbwaevwucKaZWc2VKSIiRyI6jDatgI2VlrOBAdW1cc6Vm1k+0BjYUbmRmd0I3Bha3GNmq35M0UCTA/ddh9TV2lTXkVFdR66u1ua1utqF0yiccK/qCNz9iDY458YB48J4zkMXZDbXOZf+U/dzNNTV2lTXkVFdR66u1vbfWlc4wzLZQJtKy62BzdW1MbNoIAXIq4kCRUTkyIUT7nOATmbW3sxigUuBSQe0mQRcFXp8EfBv59xBR+4iIhIZhx2WCY2hjwE+B3zAeOfcMjN7CJjrnJsEvAy8bmYZBI/YLz2aRVMDQztHUV2tTXUdGdV15Opqbf+VdZkOsEVEvEd3qIqIeJDCXUTEg465cD/cVAgRrKONmU0zsxVmtszMbg+tH2tmm8xsYehrRC3Uts7MloSef25oXSMz+9LM1oS+N4xwTV0q9clCMyswsztqq7/MbLyZ5ZjZ0krrquwjC3om9J5bbGZ9I1zXE2a2MvTcH5hZg9D6NDMrrtR3z0e4rmpfOzP7Xai/VpmsqeoKAAADvklEQVTZ2UerrkPU9k6lutaZ2cLQ+oj02SHyIXLvMefcMfNF8ITuWqADEAssArrXUi0tgL6hx0nAaoLTM4wFflPL/bQOaHLAuseBe0KP7wH+VMuv41aCN2PUSn8BQ4C+wNLD9REwAviM4P0cA4FZEa5rGBAdevynSnWlVW5XC/1V5WsX+jlYBMQB7UM/s75I1nbA9r8A90eyzw6RDxF7jx1rR+7hTIUQEc65Lc65+aHHu4EVBO/UrasqTxHxKnBBLdZyJrDWObe+tgpwzk3n4HsxquujUcBrLmgm0MDMWkSqLufcF8658tDiTIL3mkRUNf1VnVHA2865vc65LCCD4M9uxGsLTYNyMfDW0Xr+amqqLh8i9h471sK9qqkQaj1QLTgL5gnArNCqMaE/rcZHevgjxAFfmNk8C075ANDMObcFgm88oGkt1LXPpez/w1bb/bVPdX1Ul9531xI8wtunvZktMLNvzGxwLdRT1WtXl/prMLDNObem0rqI9tkB+RCx99ixFu5hTXMQSWZWH3gPuMM5VwD8AzgO6ANsIfgnYaSd7JzrS3Amz1vMbEgt1FAlC94INxL4V2hVXeivw6kT7zszuw8oB94IrdoCtHXOnQDcCbxpZskRLKm6165O9FfIaPY/kIhon1WRD9U2rWLdT+qzYy3cw5kKIWLMLIbgC/eGc+59AOfcNuec3zkXAF7kKP45Wh3n3ObQ9xzgg1AN2/b9mRf6nhPpukLOAeY757aFaqz1/qqkuj6q9fedmV0FnAf80oUGaUPDHrmhx/MIjm13jlRNh3jtar2/oGIqlJ8B7+xbF8k+qyofiOB77FgL93CmQoiI0Fjey8AK59yTldZXHie7EFh64L89ynUlmlnSvscET8YtZf8pIq4CPopkXZXsdyRV2/11gOr6aBJwZeiKhoFA/r4/rSPBzIYD/wuMdM4VVVqfasHPW8DMOgCdgMwI1lXdazcJuNSCH+LTPlTX7EjVVclQYKVzLnvfikj1WXX5QCTfY0f7rHFNfxE8q7ya4G/c+2qxjlMI/tm0GFgY+hoBvA4sCa2fBLSIcF0dCF6psAhYtq+PCE7BPBVYE/reqBb6LAHIBVIqrauV/iL4C2YLUEbwqOm66vqI4J/Mz4bec0uA9AjXlUFwPHbf++z5UNufh17jRcB84PwI11XtawfcF+qvVcA5kX4tQ+tfAX51QNuI9Nkh8iFi7zFNPyAi4kHH2rCMiIiEQeEuIuJBCncREQ9SuIuIeJDCXUTEgxTuIiIepHAXEfGg/wfXnNSAQXNnqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.ylim(0, 1)\n",
    "plt.plot(clf.model.history.history['loss'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Accuracy Trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x24f25dd7e10>]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VNX9//HXyWSD7CEJhEAWIOyLQFgERFSsuFK0dbdurVVr1dpabfVrbfvtr3bT1n7Val2rFtwV941VZAv7DiEkJIQle0JC9vP7Y4YxK5loCOT6fj4eeTBz5+TOhzuT95w599x7jbUWERFxFr8TXYCIiHQ+hbuIiAMp3EVEHEjhLiLiQAp3EREHUriLiDhQu+FujHnWGHPIGLO5jceNMeZRY0yGMWajMWZc55cpIiId4UvP/Xlg1jEePxdI9fzcBDzxzcsSEZFvot1wt9YuAYqO0WQ28B/rtgKINMbEd1aBIiLScf6dsI4EIKfR/VzPsv3NGxpjbsLduyckJGT80KFDO+HpRUS+PdasWVNgrY1tr11nhLtpZVmr5zSw1j4FPAWQlpZm09PTO+HpRUS+PYwx2b6064zZMrlA/0b3+wF5nbBeERH5mjoj3OcDP/DMmpkMlFprWwzJiIhI12l3WMYYMxeYAcQYY3KB3wABANbafwEfAOcBGUAlcP3xKlZERHzTbrhba69o53EL/KTTKhIRkW9MR6iKiDiQwl1ExIEU7iIiDqRwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg6kcBcRcSCFu4iIAyncRUQcSOEuIuJACncREQdSuIuIOJDCXUTEgRTuIiIOpHAXEXEghbuIiAMp3EVEHEjhLiLiQAp3EREHUriLiDiQwl1ExIEU7iIiDqRwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg7kU7gbY2YZY3YYYzKMMfe28niiMWahMWadMWajMea8zi9VRER81W64G2NcwGPAucBw4ApjzPBmze4HXrXWjgUuBx7v7EJFRMR3vvTcJwIZ1tpMa20NMA+Y3ayNBcI9tyOAvM4rUUREOsqXcE8Achrdz/Usa+xB4GpjTC7wAfDT1lZkjLnJGJNujEnPz8//GuWKiIgvfAl308oy2+z+FcDz1tp+wHnAi8aYFuu21j5lrU2z1qbFxsZ2vFoREfGJL+GeC/RvdL8fLYddbgReBbDWLgeCgZjOKFBERDrOl3BfDaQaY1KMMYG4d5jOb9ZmL3AWgDFmGO5w17iLiMgJ0m64W2vrgNuAj4FtuGfFbDHG/M4Yc5Gn2c+BHxljNgBzgeustc2HbkREpIv4+9LIWvsB7h2ljZc90Oj2VmBq55YmIiJfl45QFRFxIIW7iIgDKdxFRBxI4S4i4kAKdxGRr+HF5Vks2nHIp7ars4qorW84vgU1o3AXEemgqtp6fv/+Nh5fuLvdthtySvj+v5bzz893dUFlX1G4i4h00JrsYmrqGti4r6RJj3zXwXIaGpoe4vPZtoMAPLU0kwOlVV1Wo8JdRE5KTy/N5KqnV3C4uu5El9LCsowCAKpqG9i2vwyALzMKOPuRJVz9zEpyiiq9bRdsP8TA2BAaGuDhT3d0WY0KdxE5KX24+QDLMgq59eW1XT5e3Z5luwvpH90DgHV7SwBYnVWMMbAxt5TZjy1jfU4JB0qr2JJXxvfT+nPZhP68vS6P8qraLqlR4S4ix01pZS01dR0P5oYGy/b9ZQyICWHJznxe+DLL+1hOUSX/WZ5FUUUNlTV17C2sbDEU0hE1dQ1UtPPtoL7Bsq/kCA0NltIjtWzKLWHO2H70Dg9i7d5iADbtK2FgbCjv/nQaIUEurnhqBXe/vgGAM4fGceGYvtTUN7BoR9ecdkvhLiLHRX55NWf8bREPfbi93bY5RZVc+e8V7C10D2fkFFdSUVPPTdMHMD4pipdX7uXo6aoe/nQnD7yzhcl//JxRD37C9L8sZOzvP+Xtdfu+Vp33vLGRyX/8nJWZhU2W7y89woLt7vHyF77MYupDCxj7+085/9GlNFiYOrAX4xKjvOG+MbeU0QkRpMSE8MYtUzgtNYYvMgpI6tWT1LhQxidF0SskkI+3HPhadXaUwl3kJFBUUcMnrfzRH6mp5+11+6hvo2e6Na+Mjbkl7a6/orqOu15dz5e7C752jdZa5m/I40hNfauPHyyravJ/+N17W93/r60HONZ5BK21/PqtTXy5u5B3N7rPJr41zz2OPbxvOFdNSmRPQQXLdxdSW9/A59sOMmNILFdNSuSW0wfyhzkjSYjswe/f20plTR119Q1U17Ws8dkv9vA/b2+mtLKWpbvyWb67kJyiSt5Zv4+q2nqueXYV5z+6lNv+u5aDZVVc88wqbng+nezCCuZvyCMlJoTzRvVhTL9IfnBqEuOTohibGElO0RE25pZwqLyaUf0iAIgLC+apH6Sx+Bdn8NKNkzDG4PIznD28Nwu3H6KqtvVt2Jl8OnGYSHeRW1zJnz/awUOXjKJn4Mnx9s4pqiT/cDXjEqPabPPvpZk8sWg37/10GiMTIrzLn/kik79+shOA745tegG0Dzft545564nsGcDKX5/F/tIqXliexfb95dw5M5WxnuerrW/g1pfXsnhnPoWHa5gysP1LLdTWNxDgatr3W5dTwu1z13HvuUO5+fSBLX7n0c938fLKvTx25TjqGhp4d0Meg+JCyTh0mD0FFSRG98TlZzCm6fV/3ly7j6W7Cgh0+bF4Zz4/OWMQ2/aX4WdgcO8wBvcO47fvbuWlldnUW0tZVR1XTkzkOyP6eNcxtE8YlzyxnN+/t5UF2w9RXFHLiIRwxidGcdXkJJKie/LYwgwKK2qYt3ovtfUWl58hLSkKP2N45yfTeOHLLPIPV/PxlgN8tu0g1XUNGAOPL9zN+pwS7j5nCD85Y1CT2mcO680fP9zOvW9sAmBUo9cOILFXzyb3zxnZh3mrc/hydwFnDu3d7uvwTajnLicNay1vrs3lF69t+Npzgj/depD5G/JYv7f93mxr2hq7/fNH2/ncM6Wto3777haueXollTVtj+uu8AwJvLxyr3dZfYNl7ir3FS4fXbCrSe99x4Fybv3vWkKCXBwqr2b7gXJ++fpGnlm6h/SsIn7+2gZv7/CfCzJYvDOfwb1DWb67sN3x5VfTcxjxwMdMfWgBD3+607tN1ma7hx/e25iHtZYPNu3nntc38scPtgFfzSD5xWsbuPOV9aQlRfGvq8cB8MnWg8z6x1Lue3tzk+eavyGPe9/cSFpSFNdPS2ZtdjHlVbVs3V/OwNhQggNcBAe4uHxifz7YdIA/vL+NHgEupg9ueiW38UnRTBsUw9xVOfgZw7VTkvD3M/xneTY/nbuWHQfLKayo4cZpKcw+JYE/f280g3uHsXJPEReMjmd433D+9L3RPHvdBJ6/fiJB/i7uPGsw0wbF8Eq6+zU4Z0TLMB4QG8qFo/uy1fNhNLxveIs2jU0Z2Iu0pCgaumD/sMJdThpb8sq469UNvL1uH48u2PW1vrpmHDoMwO78wx3+3araeib98XMeW5jRZHlWQQWPL9rN/W9vbvXrfnPWWl5emc1LK7KprW9wB2pNPZ9sOciXGQUt1l9ZU8em3FICXIZ31u/zzqZYsjOffSVHuGB0PJn5FbyxNtf7O59uPYC18OKNkwB4c20uX+4u4JYZA3ni6vFk5lfw2MIMSiprePaLPZw7sg8PXjiCmvoGbwi3ZuH2Q/zqzU2M6R/B4N6hPPr5Lu58ZT219Q3eseXN+8p4fNFubn15LW+t38eTSzJZvDOfrMJKbpiaQs9AFzOH9ebFGycxKC6MpF49efiTnWQcOszra3IpqaxxP9eOQ9w+dx1jE6N45roJzBgcR12D5cvdhWzbX8aw+K+C8q6zBzN5QDTbD5QzY0gswQGuFrXff8Ew5oxN4M1bp3Df+cN57eYp/HLWEDbvK+O/ng/NG6el8Nfvj+HStP68cP0ELh6XwJ0zBzdZz9RBMaz9n7O5Y2Yql4zrB8DA2BAGxYW1us1uO3MQxkBqXFi73xaD/F28fssUZg4/vr120LCMnEQ25pYC8KvzhvH797ayMbeUiSnRHVrH0VA/GvIdsXV/Gfnl1fztkx2MS4zi1IG9AHh/034A9pdW8fqaXK6alNTid+vqG7jhhXT2FlYwLD6cDzcfINDfj4TIHlTU1GMMPP9lFtmFFRRX1jJtUAxj+kcC7gNi6hosP5s5mEc+28n1z60mJjSI7QfKiAkN4m+XjiEzv4Jfvr6R+evzeOzKcSzZWcCIvuGMTIhgSO8wnluWRYOFC0b3ZUifMC4el8A/F2SwLKOAw9V13DEzlQExoYQF+fP5tkPER/QgPjKYmNAg7/9hQ04Jt768lmHxYTx//UR6Brp4YvFu/vzRDqYM7MXa7BImJkezKquIv3y8g5EJ4Txz7QSmPLSA+95yD0tcOqEf95w7hCD/r8J3emosL67IZvKAaFZkFvH6mlyumJjI/W9tJjUulP/cMJHgABfjk6IICXTx5OLd7Cs5wtWTv9rOQf4unrwmjQfnb+HaKcmtvn5D+4TzyGWnNFl2/uh4/vf9bby8MpuUmBD6RvbwPhYXHszDl57SfDUAuPzcQ0ffGdGbmNBAvntKQqvtwD10dOdZg4kNC2qzzYmgnvu3XEOD9e686gp3zFvHhf/8gnfW72uxk21LXilhwf7M8Ywtr84qoryqlj0FFcdc5xe7Cpjxl4WUVtaSccjdNsOHnvuD87cw5/FllFa6e8qb97k/XPqEB3PD86u55pmVfJlRwLsb8hifFMUp/SN5fOHuVqf2PboggyU78wlw+fHh5gOcM6I3NXUN/PbdLRgD10xOYn1OCYer6wgJdPHMF3u8v7syswiXn+HG01K4eGwC5VV17CmoIMjfxV1nDybI38W8H0/mnllDWba7gL99uoO1e4s53TM0MX1wDHUNlsG9QxnSx927/H9zRjFrRB/W7i1h1og+DO0TTqC/H9MHx/JKeg4X/t8XnPnXRTy+KIPfvLOZO+at4/rnV9MrNJBnr5tASJA/xhhuOX0gQ3qH8diiDA6UVXHuqD5MSI7C5Wd46OLR9A4PZsbgWHKLjxATGsiQ3mFNgh3gsgn9OWNILP+6ejxjEyM9QyXr2FdyhIcuGeXthQf6+3HG0DjW7i0huVdPzhwa12Q9ET0CeOSyUzjF86Hoi/iIHkxMjqbBuodEOqpnoD9LfnlGi7H25u6YmcqVkxI7vP7jST13h2tosFz21HKunpzE7FZ6H+9t2s/tc9fx/u3TGNE3opU1wOHqOp79Yg+XT+hPXHgw4B5KKDxcQ//opjuMPtt6kEc+28kTV4337kzKL6+mZ6D7D/j9jfsJcPlxx7z1VNbUc8XEr/4gtuSVMaJvONEhgQyKC2VNdjEbc0tYvruQ9PvPJtDfj2UZBTzwzmbGJ0Vx/wXDCQ8O4P1NeWQVVvLRlv0UHK7GGNh9qILSI7X834JdJMeEMHNYb3p7agf3MMbznrnTP3oxnf/cMJGNuaXEhAYy96bJPLUkk0U78rnm2VXUN1gevHA4yTEhXPfcal5akc0N01JoaLD8dO46Fmw/xJHaei4Z14+/XTqGQ+VVxIYGcc7fl7Dz4GFG9A3nh9MGMHfVXm6dMYjKmjqeXZbFeaPiCQlysWjnIUYmRBAa5M/Dl7XekwwPDuCWGQPZnFfKf5ZnA3jHnacPjuXfS/dwwei+3vbBAS4eu2ocb6zNZcaQr8anr5qcSH55NeePjufdDXn8+aMdhAS6iA0LIjG6J3+7dAxxYV9tJ2MMV05K5DfztwAwPimKyQN6sa/4iHfH7yXj+/H59kOcOjCmxc5SgJEJETx3/UQArj01mTtfWU9+eTV3zkxlfFLTb2Z//f4YfnvRCHqFdl4v+IIx8azKKmLqoPZ3JLfmZNkx31Hds+puLKeokviIYPxdXfOlaU9hBauzigkPDmg13NOzigA8X/NbhvvRmRZLduazdm8xz103gfoGy3XPrmbnoXJW3zezyayKfy3ezZa8Mq59bhWv/vhUDpZVcdmTy5k+OJZL0/pT12B57vrxPLYwgz9+sI2zhsURFxZMXb37MO5rPF/FJyRH8ebafVR7eslrsovJKznCz1/bQN+IYN5Yu48VmUV88rPprMx0/x9e+NIdeuMTo0jPLubZL/bw76XuHvLfP9vFm7dMYX1OCW+v28cXGQVMTInmyomJ3PnKep5blsWm3FJGJUSQ1CuEP8wZRVlVLTf9J511e0s4b1Q8sWFBTBsUw6MLdnHJuH58sHk/72/az3dP6Utq7zCun5oM4A3HS8b1448fbmfqoBgSe/Vk6S/PpHd4EHmlVTz/ZRY3v7TGu91+ckbL2Set+emZg3h/435CAl3e2TdTBsbwmwuHc7FnfPgol5/h0rT+TZZNGRjjnS1z9eQk8kqO0Deyh3cYojVzxiXw0IfbsViGxYcT4PJrMh5+1rA4pg2K4Xvj+7W5jqNmn9KXsYmRJET2aPVv4OgO1M70/fH9qa23zBx2/Me5TyYK9y5UVFHDWQ8v5rK0/vz+uyN9+p3fvbuV6JAAbjszFYBDZVXc9OIaHr18rLdnXFVbT3VtAxE9A1r8/oYc96yR9OxiGhosfs3+iI8+fnRnXHMPvLOZJTvzmTEklkU73EcK5hYfYZXnQ2FtdjFpydGUVNZQXlVHenYx54+O57OtB5nxl4UE+vtRUVPPp1sP0iPARaC/HxOSo93DBv9Yym3/Xcejl4+lrKqW6roGRiS4QyMtKZq5q3IIDfKnqraeJbvyWbQjn+Hx4bx56xSW7MznphfX8NKKbDILKjDGPWYOMGtkH3e4L9vDyIRw/vDdUVzzzEpm/X0JFTX1JEb3ZM7YBO76zmDiwoKZu2ovL63IZn/pEc4Z+dX0uvDgAF66cRJFFTXebyy/Pm8Y5/9zKZc+uZy80iNMHhDNI5ed0mqP9eJx/Zi/IY+Lxrh71H0i3OtIiOzBJz87ncLD1YC7dzyinVkWRw3tE851U5IJcBkC/d3h6PIzXD81xaffb8zlZ1p882pNeHAAN00fQMHh6hbTI8E9Hv7SDyf59JzGGJJ6hXS41m+iR6CLG6d1fPt0dxpz72Sbcku909qaW7orn5q6Bl5cke3tMR9LXX0Dc1ft5fFFuynzzKD4IqOA9TklLGx0Hulfv7mJi59Y1uR3y6tqsdZ6w7v0SG2Lcejqunq27i/D38+wOquoxUyQd9bvY+6qHG4+fSDPXDuBMf0iePDdrTz9xR7OHxWPy8+wZFc+f/poO6f+cQG/fmsTfgb+5/zhvHPbVM4e3pvInoGeec+WN9ftY3xiFMEBLgbEhvLQxaPYmFvCdx5ZzNNLMwEY6fn2MDElGmPghqnJjEuK4tXVOWzbX8blE/sTHOCejZEQ2YNHPnXPAb/QMyQR6O/nHYsur6rjwtF9GdM/kqevnUBMWBC/Pm8oC38xg4cuGe3tYV85KdF9aLmF0c3mKfu7/LzBDu6pbg9dPIrwHv5E9Ajg/80Z1WqwA8SGBfH+7ac1mbd+VEpMCGnJ0aQlRzM+KapDvdUHLxrBfecP97l9Z/jZ2YP5w5xRXfqc8s2o597J7n1zI+VVdSz55RktHlu8M5/IngGEBPrzqzc38cEdp3l7QhXVdQT5+zX5qpqRf5gjnumA76zbxzWnJntnlBz9t7Kmjg83H+BIbb33K/am3FK+/+SX3DlzMOtzS0mI7MG+kiOszipicO+vpnNt219Obb3lknH9eGNtLmuzS7wzRA6VVfHrNzeRlhTFL74zGJef4eUfTWbVnkICXS4mD4jm8qdW8NHmA+wvrQIDX+4u5LTUGPpEBNMnIpi/Xz7W+1xPLA5n874ypg76aqfWxeP6MS4xiptfWsOr6bkEB/gxIDYUgP7RPfng9tNIjQslaEkmq/YUEeAy3hD38zPeGSGhQf7cNH0A8zfkMSAmhOSYEPz9DHUNlvNHxwPuD4vFd7d8TcDd048OCaSoosZ7hOGxXDYhkcsmnFw7z0SaU8+9Ex09A1xOcaX3sPH73tpEXX0D1lqW7irgtNRYfnPhcHYdOsy8Ve65t2VVtZzx15bn4Dja644LC/KeW2OTZ0bHpn3uxz7bdsj7AZDuOQDktrlrqapt4OmlmWzLK+OC0e7x4vSs4lbX/+PTB+BnaHJo+vJM99zsBy8a4f3ACQ3y58yhvZmWGoO/yz3zYnd+BZU19cz90WRuO2MQ98wa2uq2+Z5nPHhaatODT5JjQnj15lM5Y0gs54zo02Tsd1h8uPt5PL9z5tA4okICvY8fnVWTlhzF8PhwYkKDGNonjACXH4M85/LoF9X+sEOQv4sfnpbCKf0jm+x0FenO1HPvRAu2u4dKrHXPt355ZTars4qpb7D84NRk8surmZ4aw9nDezMpJZpHPtvF7LEJPL5wN4fKq3ltTS53z/pqjvD6nFLCg/25Y2Yq9721mfTsYrbkuQ92yTh0mMqaOt7bkEdcWBAV1XWkZxWxNruY3OIj3jnTAKf0jySnuJJVe4qoqq33DgFsyCkhLiyI1LhQJqZE89KKbK6clEh8RA825pYS5O/H0D6tH7gB7lkaD3+6k8kD3EML45PaPrz+6slJDI0Pb3UaW3hwgHc2RWvcs01SmDOu6Q7hAbGh/PzswYxPisLPzzDvpkmE93Dvd3ji6vEE+fved7l1xiBunXHs6W4i3Yl67p1owfaDBAe4N+mOA+VsySsjNiyIeatzvGPi0wfHYozh/vOHU1RRw/efWM6zy/YwKC6U0iO1LNj21Vj6hpwSxvSPZPYpCYQG+fO/722lqraBWSPjabCwLKOQRTvyuWB0X8YlRbF4Zz6vrM7h4rEJ3H7WIG8wj+kfycxhvdlXcoQpDy3gky0HqKlrYHlmIWP6R2KM4Q9zRlFd18Ad89ZT32DZlFvKiL7hx5zVMyohgqsnJ/Krc4e1u238XX5MHtDxecbgHoK5/4Lhrc7m+elZqUzxTHEbFBfmHUdvfsCKyLeNwr0NFdV1rNtbfMw2h8qr+NWbmyg9UktVbT3LMgqZMzYBfz/Dx1sOUFlTzz2zhvK374/hvJHx3Dgtxfu1f1S/CB670n3ejSCXH89dN4G4sCDmrs5h8c58Nu8rZcfBcsb0iyQ0yJ/Zp/Rlg2ec/UrP3PB73thIvbVcNqE/45OiyC6s5EhtPT88bQDGGP7nguFcPTmR+Ihg5oxNYN5Nk+kTHszdr2/k75/tZH9plffAi4GxoTx40QhW7Snio80H2JxX2uIkSM25/Az/+91R3iMtReTkoWGZNjy9dA+PfLbzmAf3PLU4k7mr9jIoLpT4iGCO1NYza2Q8q7OKvbNZxvSLILV3GJe0Mgf4/NHxnDeqD1W1DfQIdDFnXAJPLs5kyc6vTuZ/NDivnJTIyyv3EhLoYlJKNL3DgzhYVs2PTx/AkD5hFHim1Z2WGuM9SnHqoJgmB25MHtCLR68Yy3n/WMrji3ZzWmoMMxqdgOmScf3468c7eOSznVTW1DOqn0JbpLtSz70NK/e4pzP+8/OvTvL0wxdWe0/6VF5Vy7zV7rPFvbEml/+u3EvfiGCmDYohNS6U2npLz0CXd/ZHW4wx9PAcvXnL6QP5zYXDeenGSfxs5mCmD45l0gD3EXwj+kYwMSWaiSnR+PkZJqb0IiUmhDvPcp/0aHxSFGcOjeOuswe3+VwAg+JCufPsVAL9/fjVucOaTONz+RnmjE3wnpdltA8zR0Tk5KSeeyNVtfXkl1fTJyKYdXtLCA3y56MtB9i2v4ywYH8+23aI3OIj/OSMQbyyOofD1XXeaYTgPnOdy8+QGhfKh7h3BB7ryL/mInsGeg9GmZba8lDp56+fgMG9vr98bzR1Ddb7wRAc4OLZ6yb49Dy3zhjENZOTCAtuedDTJeP78eSSTHoEuBjYzgeTiJy8vjU996raeu+BQBXVdRworWrR5rfvbuHsRxazZGc+R2rruffcoYQG+fPEot0s9MyE2XGwnNLKWv67ci8TkqO47/xhBLjcV1m5bIL7UO9BnrnkoxI6d1ijZ6B/kzAPDfr6n82tBTu4z3A3NjGScUmRHfpgEpGTy7ei515X38ClTy6npLKWT342nZtfWsMXGQV8Z3hvfjd7JL3Dg8kpquS19FzqGiz3ey4oMHNYb7IKKnj+yywyCw4T4DLU1lvmrd5LZkEFN0xLITokkKsnJ2Et3p2lI/qGY4x7/nV39JyP3wBE5OTlmJ771rwyrnhqBQWHq6moruPppZl8vu0gxRU1PLtsDxtzS9lbVMktL61h6a4CpqfGsnhnPne/vhFrLY8v2o2fMYxLjGR/aRX9o3vQJyKY66Ym02Atm/eV8b3x/fH3M/zfwgyMge94Trj/mwtH8OBFI7y1DIwN5dOfnc65jc5T0p1E9gwksmdg+w1F5KTlmJ77k0t2szyzkKeWZGIMPLk40/uYn3H3wusbGli4I5/kXj359w/SeHllNr99dys3v7SGT7ce5MpJiZw3Kp4r/72SNM+pSPtF9eTckfG8v2k/F4yOZ9v+MtbnlDAuMbLJOUeaGxSn8WoROXG6fbjXN1hKj9Ty4Sb3lW9eXO6+iO7sU/py5cRE1uwtJjO/grvPGUJ5VS07Dx7mNxeOINDfjx+cmsw76/P4eMtBLh6XwD2z3GPst5+VylmNLhTwi3OG0CcimIkp0UxIjmJ9TgmzummvXES+HUzzq+G02siYWcA/ABfwtLX2oVbaXAo8CFhgg7X2ymOtMy0tzaanp3+dmr0+2nzAcw3GSFbuKeKJq8Zx63/XEujyY9HdM4iPaHmEorW2yfS/gsPVZBdWtLhoQFtWZBbyoxfS+fDO03w6b4mISGcyxqyx1qa1167dnrsxxgU8BpwN5AKrjTHzrbVbG7VJBX4FTLXWFhtj4lpfW+f6bNtB6hoaWLmniLSkKM4dFc8vzxlKdEhAq8HuqbXJ/ZjQoCbXkWzP5AG92PTbc75R3SIix5svwzITgQxrbSaAMWYeMBvY2qjNj4DHrLXFANbaQy3WchysyS7mzKG9uWFqsveiA61dcEJE5NvGl9kyCUBOo/u5nmWNDQa6L8hRAAAL60lEQVQGG2OWGWNWeIZxWjDG3GSMSTfGpOfn57fWxGf55dXsKahgQnIUUwbF+HRFGRGRbwtfwr21I1maD9T7A6nADOAK4GljTIsjeKy1T1lr06y1abGxsc0f7pA12e4rGaUl+zZWLiLybeJLuOcCja+y2w/Ia6XNO9baWmvtHmAH7rA/blZnFRPk78fIBN+uPSki8m3iS7ivBlKNMSnGmEDgcmB+szZvA2cAGGNicA/TZHIcpWcVMaZ/pPfCFiIi8pV2w91aWwfcBnwMbANetdZuMcb8zhhzkafZx0ChMWYrsBC421rb+lWiO8nOg4e9F1MWEZGmfDqIyVr7AfBBs2UPNLptgbs8P8ddZU0dR2rriQnTIfIiIq3plueWKTxcA0CvEIW7iEhrumW4F1W4wz06xPeDj0REvk26ebir5y4i0ppuGe6FFRqWERE5lm4Z7kUV7otB9wpVuIuItKZbhnthRQ2BLr9vdJk5EREn65bhXnS4huiQwBZneBQREbfuGe4VNdqZKiJyDN0y3AsrajTeLiJyDN0y3NVzFxE5tm4Z7oWHqxXuIiLH0O3Cvaq2noqaes1xFxE5hm4X7jr1gIhI+7pxuKvnLiLSlm4X7t5TD2i2jIhIm7pduB899YB67iIibet24a5zuYuItK/bhfvofpH8ePoAwoMDTnQpIiInrW535q2JKdFMTIk+0WWIiJzUul3PXURE2qdwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg6kcBcRcSCFu4iIAyncRUQcSOEuIuJACncREQdSuIuIOJDCXUTEgXwKd2PMLGPMDmNMhjHm3mO0+54xxhpj0jqvRBER6ah2w90Y4wIeA84FhgNXGGOGt9IuDLgdWNnZRYqISMf40nOfCGRYazOttTXAPGB2K+1+D/wZqOrE+kRE5GvwJdwTgJxG93M9y7yMMWOB/tba9461ImPMTcaYdGNMen5+foeLFRER3/gS7qaVZdb7oDF+wCPAz9tbkbX2KWttmrU2LTY21vcqRUSkQ3wJ91ygf6P7/YC8RvfDgJHAImNMFjAZmK+dqiIiJ44v4b4aSDXGpBhjAoHLgflHH7TWllprY6y1ydbaZGAFcJG1Nv24VCwiIu1qN9yttXXAbcDHwDbgVWvtFmPM74wxFx3vAkVEpOP8fWlkrf0A+KDZsgfaaDvjm5clIiLfhI5QFRFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg6kcBcRcSCFu4iIAyncRUQcSOEuIuJACncREQdSuIuIOJDCXUTEgRTuIiIOpHAXEXEghbuIiAMp3EVEHEjhLiLiQAp3EREHUriLiDiQwl1ExIEU7iIiDqRwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg6kcBcRcSCFu4iIA/kU7saYWcaYHcaYDGPMva08fpcxZqsxZqMx5nNjTFLnlyoiIr5qN9yNMS7gMeBcYDhwhTFmeLNm64A0a+1o4HXgz51dqIiI+M6XnvtEIMNam2mtrQHmAbMbN7DWLrTWVnrurgD6dW6ZIiLSEb6EewKQ0+h+rmdZW24EPmztAWPMTcaYdGNMen5+vu9ViohIh/gS7qaVZbbVhsZcDaQBf2ntcWvtU9baNGttWmxsrO9ViohIh/j70CYX6N/ofj8gr3kjY8xM4D7gdGttdeeUJyIiX4cvPffVQKoxJsUYEwhcDsxv3MAYMxZ4ErjIWnuo88sUEZGOaDfcrbV1wG3Ax8A24FVr7RZjzO+MMRd5mv0FCAVeM8asN8bMb2N1IiLSBXwZlsFa+wHwQbNlDzS6PbOT6xIRkW9AR6iKiDiQwl1ExIEU7iIiDqRwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg6kcBcRcSCFu4iIAyncRUQcSOEuIuJACncREQdSuIuIOJDCXUTEgRTuIiIOpHAXEXEghbuIiAMp3EVEHEjhLiLiQAp3EREHUriLiDiQwl1ExIEU7iIiDqRwFxFxIIW7iIgDKdxFRBxI4S4i4kAKdxERB1K4i4g4kMJdRMSBFO4iIg7kU7gbY2YZY3YYYzKMMfe28niQMeYVz+MrjTHJnV2oiIj4rt1wN8a4gMeAc4HhwBXGmOHNmt0IFFtrBwGPAH/q7EJFRMR3vvTcJwIZ1tpMa20NMA+Y3azNbOAFz+3XgbOMMabzyhQRkY7w96FNApDT6H4uMKmtNtbaOmNMKdALKGjcyBhzE3CT5+5hY8yOr1M0ENN83SeRk7U21dUxqqvjTtbanFZXki+NfAn31nrg9mu0wVr7FPCUD8957IKMSbfWpn3T9RwPJ2ttqqtjVFfHnay1fVvr8mVYJhfo3+h+PyCvrTbGGH8gAijqjAJFRKTjfAn31UCqMSbFGBMIXA7Mb9ZmPnCt5/b3gAXW2hY9dxER6RrtDst4xtBvAz4GXMCz1totxpjfAenW2vnAM8CLxpgM3D32y49n0XTC0M5xdLLWpro6RnV13Mla27eyLqMOtoiI8+gIVRERB1K4i4g4ULcL9/ZOhdCFdfQ3xiw0xmwzxmwxxtzhWf6gMWafMWa95+e8E1BbljFmk+f50z3Loo0xnxpjdnn+jerimoY02ibrjTFlxpg7T9T2MsY8a4w5ZIzZ3GhZq9vIuD3qec9tNMaM6+K6/mKM2e557reMMZGe5cnGmCONtt2/uriuNl87Y8yvPNtrhzHmnONV1zFqe6VRXVnGmPWe5V2yzY6RD133HrPWdpsf3Dt0dwMDgEBgAzD8BNUSD4zz3A4DduI+PcODwC9O8HbKAmKaLfszcK/n9r3An07w63gA98EYJ2R7AdOBccDm9rYRcB7wIe7jOSYDK7u4ru8A/p7bf2pUV3Ljdidge7X62nn+DjYAQUCK52/W1ZW1NXv8b8ADXbnNjpEPXfYe6249d19OhdAlrLX7rbVrPbfLgW24j9Q9WTU+RcQLwHdPYC1nAbuttdknqgBr7RJaHovR1jaaDfzHuq0AIo0x8V1Vl7X2E2ttnefuCtzHmnSpNrZXW2YD86y11dbaPUAG7r/dLq/NcxqUS4G5x+v526iprXzosvdYdwv31k6FcMID1bjPgjkWWOlZdJvnq9WzXT384WGBT4wxa4z7lA8Ava21+8H9xgPiTkBdR11O0z+2E729jmprG51M77sbcPfwjkoxxqwzxiw2xpx2Aupp7bU7mbbXacBBa+2uRsu6dJs1y4cue491t3D36TQHXckYEwq8AdxprS0DngAGAqcA+3F/JexqU62143CfyfMnxpjpJ6CGVhn3gXAXAa95Fp0M26s9J8X7zhhzH1AHvOxZtB9ItNaOBe4C/muMCe/Cktp67U6K7eVxBU07El26zVrJhzabtrLsG22z7hbuvpwKocsYYwJwv3AvW2vfBLDWHrTW1ltrG4B/cxy/jrbFWpvn+fcQ8JanhoNHv+Z5/j3U1XV5nAustdYe9NR4wrdXI21toxP+vjPGXAtcAFxlPYO0nmGPQs/tNbjHtgd3VU3HeO1O+PYC76lQLgZeObqsK7dZa/lAF77Hulu4+3IqhC7hGct7BthmrX240fLG42RzgM3Nf/c41xVijAk7ehv3zrjNND1FxLXAO11ZVyNNelInens109Y2mg/8wDOjYTJQevSrdVcwxswC7gEustZWNloea9zXW8AYMwBIBTK7sK62Xrv5wOXGfRGfFE9dq7qqrkZmAtuttblHF3TVNmsrH+jK99jx3mvc2T+49yrvxP2Je98JrGMa7q9NG4H1np/zgBeBTZ7l84H4Lq5rAO6ZChuALUe3Ee5TMH8O7PL8G30CtllPoBCIaLTshGwv3B8w+4Fa3L2mG9vaRri/Mj/mec9tAtK6uK4M3OOxR99n//K0vcTzGm8A1gIXdnFdbb52wH2e7bUDOLerX0vP8ueBm5u17ZJtdox86LL3mE4/ICLiQN1tWEZERHygcBcRcSCFu4iIAyncRUQcSOEuIuJACncREQdSuIuIOND/B3moMgJsqrZ6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "clf.model.history.history['acc']\n",
    "plt.ylim(0, 1)\n",
    "plt.plot(clf.model.history.history['acc'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test our Deep Learning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6286764705882353"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test_sel)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolution Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CNN are biologically-inspired from the animal visual cortex and CNN considered to be one kinds of multilayer perceptions. In animal visual cortex, each neuron cell is responsible for a small sub-regions of the visualisation called a receptive field. The receptive fields of different neurons are combined and partially overlapped to make it cover the all area of visualization. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![CNN](https://raw.github.com/plenoi/CMU_DataScience/master/convolutional_neural_network.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![CNN2](https://raw.github.com/plenoi/CMU_DataScience/master/cnn.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load digits image dataset from sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1797, 8, 8)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "digit = datasets.load_digits()\n",
    "X = digit.images\n",
    "y = digit.target\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split Train & Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(360, 8, 8)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load skimage library for image processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEYCAYAAAA9LWHsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE0BJREFUeJzt3X+spmWd3/H3hwEWQQjTjlo6QxbHGBJjWnEmtJbEssKaUQnuH2og0bi2m9km1UK7jR1305D+N6aN1T82m5wAwkYW10HJmo1Fh1VC/UPKHMQFHNy6p1ZmQIcJs0X2Rynw7R/nGXI8Hjz3c/P8uJ/rvF/JkznPj3N9r4fRz1z3fd3XfaWqkKRWnDHvDkjSJBlqkppiqElqiqEmqSmGmqSmGGqSmmKoSWqKoSapKYaapKacOY1Gk2y5ZQrbt2+fW+2dO3fOrfazzz47t9rHjx+fS90XX3xxLnXnrapy+ud9+/bVyZMnO//u8vLy16tq31Q6ts5UQm0ruvrqq+dW++DBg3Orfe+9986t9oEDB+ZS99SpU3OpOyQnT57kwQcf7Pz5M844Y8cUu/NzDDVJvQx13bihJqkXQ01SM6rKUJPUFkNNUlMMNUlNMdQkNcVQk9QMJwokNcdQk9SUoYZapwXtSfYl+UGSHyaZz9oUSYNy+hC0y2OWNg21JNuA3wfeA7wFuD7JW6bdMUnDNslQS3JrkhNJHt3gvX+fpJJ0Wj/aZaR2OfDDqlqpqueBLwLv79K4pDaNE2gdR2q3Ab9wF48kFwO/Dvy4a9+6hNpO4Ik1z4+NXltffH+SI0mOdC0uaXFNMtSq6n7gmQ3e+q/AJ4HOx7BdJgqywWu/UKCqloAl2Jr3U5O2mjHPle1YN+BZGmXGK0pyLXC8qr6XbBRDG+sSaseAi9c83wU82bmCpCaNGWonq2pv1w8nORf4PeDd4/ary+Hng8Cbk7wxydnAdcBXxy0kqR1TOKe23puANwLfS/IjVgdTDyX5B5v94qYjtap6IcnHga8D24Bbq+qxPr2U1I5pXqpRVY8Arz/9fBRse6tq03uId7r4tqq+BnytbwcltWeSoZbkTuBKVs+9HQNuqqpb+rTligJJvUwy1Krq+k3ev6RrW4aapLFVFS+99NK8u7EhQ01SL0Nd+2moSerFUJPUFENNUlMMNUnN8M63kppjqElqiqEmqSmGWuMOHjw4t9q7d++eW+3t27fPrfYzz2x0+63p+9CHPjSXugCHDh2aW+31DDVJzXCiQFJzDDVJTTHUJDXFUJPUFENNUjOcKJDUHENNUlMMNUlNMdQkNcVQk9SMIU8UbLqZcZJbk5xI8ugsOiRpMUx5M+PeuuzQfhuwb8r9kLRghhpqXXZovz/JJdPviqRFsrCHn10l2Z/kSJIjk2pT0nBNcqS20WmuJP85yeNJ/jzJ3Uku7NKviYVaVS1V1d6q2jupNiUN0+nNjLs+OriNXzzNdRh4a1X9I+AvgE91aWhioSZpa5nkSK2q7geeWffaN6rqhdHT7wC7uvTLSzok9TLmObUd605NLVXV0hi//y+AP+7ywU1DLcmdwJWjTh0DbqqqW8bojKQGjRlqJ/uemkrye8ALwB1dPt9l9vP6Ph2R1K5ZXaqR5KPANcBV1bGgh5+Sepl2qCXZB/wH4J9X1d90/T1DTVIvkwy1jU5zsTrb+SvA4SQA36mqf7VZW4aapF4mGWqvcJqr17l7Q01SL0NdUWCoSRrbkO/SYahJ6sVQk9QUQ01SUww1SU0x1GZkz549c6m7e/fuudQFeNOb3jS32isrK3Orffjw4bnUndf/xgAOHTo0t9prOVEgqTmGmqSmGGqSmmKoSWqKoSapGU4USGqOoSapKYaapKYYapKaYqhJasbpfT+HyFCT1IsjNUlNGWqobbpDe5KLk3wrydEkjyW5YRYdkzRsk9yhfZK6jNReAH6nqh5Kcj6wnORwVX1/yn2TNFALffFtVT0FPDX6+WdJjgI7AUNN2sIWNtTWSnIJcBnwwAbv7Qf2T6RXkgZv4UMtyWuBLwM3VtWz69+vqiVgafTZYX5bSRMz1FDbdKIAIMlZrAbaHVX1lel2SdIimOREQZJbk5xI8uia1/5eksNJ/ufoz+1d+tVl9jOs7pR8tKo+06VRSW0bJ9A6juhuA/ate+0A8GdV9Wbgz0bPN9VlpHYF8BHgXUkeHj3e26VxSe2aZKhV1f3AM+tefj9w++jn24Hf6NKvLrOf3wbSpTFJW8eY59R2JDmy5vnS6Dz8L/OG0dUXVNVTSV7fpZArCiT1MmaonayqvdPqy1qGmqReZjD7+dMkF41GaRcBJ7r8UqfZT0laawoTBRv5KvDR0c8fBf6kyy85UpPUyyRHaknuBK5k9dzbMeAm4CDwpST/Evgx8MEubRlqknqZZKhV1fWv8NZV47ZlqEnqZagrCgw1Sb0YapKasdC3HpKkjRhqM7J9e6c1rxO3vLw8l7oAKysrc6s9T/P8by5DTVJjDDVJzfCcmqTmuO+npKY4UpPUFENNUjM8pyapOYaapKYYapKaYqhJaoqhJqkZCz1RkOQc4H7gV0afv6uqbpp2xyQN28KGGvB/gXdV1XOjndq/neS/VdV3ptw3SQO2sKFWqz1/bvT0rNFjmN9G0swMNdQ67SaVZFuSh1ndoupwVT2wwWf2JzmybsNSSY2awW5SvXQKtap6sareBuwCLk/y1g0+s1RVe2e1Yamk+ZnRFnm9jLXvZ1X9FXAfsG8qvZG0MBY21JK8LsmFo59fA1wNPD7tjkkatqGGWpfZz4uA25NsYzUEv1RVfzrdbkkaukmHVZJ/C/wWqxORjwAfq6q/G7edLrOffw5cNnYPJTVtwju07wT+DfCWqvrbJF8CrgNuG7ctVxRIGtuUDivPBF6T5P8B5wJP9mlkrIkCSTptkufUquo48F+AHwNPAf+nqr7Rp1+GmqRexgy1HaevYx099q9tK8l24P3AG4F/CJyX5MN9+uXhp6Rexjz8PLnJNaxXA/+rqp4GSPIV4J8BXxi3X4aapF4mfE7tx8A/TXIu8LfAVUCv1UmGmqSxTXqioKoeSHIX8BDwAvBdYKlPW4aapF4mPfs5uqXZq76tmaEmqRc3M5bUlKHeeshQkzS2hb6d96LZvn37XOree++9c6m7lc3r7/rUqVNzqTs0hpqkphhqkppiqElqiqEmqRlOFEhqjqEmqSmGmqSmGGqSmmKoSWqGEwWSmmOoSWrKwofaaN/PI8Dxqrpmel2StAgWPtSAG4CjwAVT6oukBTLUUOu0m1SSXcD7gJun2x1Ji2CcnaRmHX5dR2qfBT4JnP9KHxhtebX/ld6X1JaFHakluQY4UVXLv+xzVbVUVXs32QZLUiMWeaR2BXBtkvcC5wAXJPlCVfXaaFRSGxZ2pFZVn6qqXVV1CXAd8E0DTdIij9Qk6ec0s6Kgqu4D7ptKTyQtlKGGWqdLOiRpvZdeeqnzo4skFya5K8njSY4meUeffnn4KamXKYzUPgfcU1UfSHI2cG6fRgw1SWOb9Dm1JBcA7wR+c9T+88Dzfdry8FNSL2POfu5IcmTNY/2F+ruBp4HPJ/lukpuTnNenX47UJPUy5kjt5CYX5p8JvB34RFU9kORzwAHgP47bL0dqknqZ8HVqx4BjVfXA6PldrIbc2Aw1Sb1MMtSq6ifAE0kuHb10FfD9Pv3y8FPS2KZ08e0ngDtGM58rwMf6NGKoSepl0qFWVQ8Dr/qGGIaapF6GuqKguVA7derUXOru2bNnLnXnbfv27XOrPa//5ocOHZpL3aEx1CQ1xVCT1Ixm7tIhSacZapKaYqhJaoqhJqkphpqkZjhRIKk5hpqkphhqkppiqElqykKHWpIfAT8DXgRe2OQOlpIa18pEwa9V1cmp9UTSQmkh1CTpZV3385y1rrfzLuAbSZY32AUGgCT7T+8UM7nuSRqicW7lPesRXdeR2hVV9WSS1wOHkzxeVfev/UBVLQFLAEmGOS6VNDFDPfzsNFKrqidHf54A7gYun2anJA3fUEdqm4ZakvOSnH/6Z+DdwKPT7pikYRtqqHU5/HwDcHeS05//o6q6Z6q9kjR4Qz383DTUqmoF+Mcz6IukBdHKdWqS9DJDTVJTDDVJTZlGqCXZBhwBjlfVNX3aMNQk9TKlkdoNwFHggr4NdF1RIEkvm8aKgiS7gPcBN7+avjlSk9TLmCO1HeuWUC6NViGt9Vngk8D5r6ZfhpqkXsYMtZO/7JZlSa4BTlTVcpIrX02/DDVJvUz4nNoVwLVJ3gucA1yQ5AtV9eFxG/KcmqReJnlOrao+VVW7quoS4Drgm30CDRocqa2srMyl7p49e+ZSF+CDH/zglqw9L5/+9Kfn3YW5c0WBpOZMK9Sq6j7gvr6/b6hJ6sWRmqSmGGqSmmKoSWqGEwWSmmOoSWqKoSapKYaapGZU1WA3MzbUJPXiSE1SU4Yaap0WtCe5MMldSR5PcjTJO6bdMUnDtsj7fgJ8Drinqj6Q5Gzg3Cn2SdICGOpIbdNQS3IB8E7gNwGq6nng+el2S9KQDfni2y6Hn7uBp4HPJ/lukpuTnDflfkkauKEefnYJtTOBtwN/UFWXAX8NHFj/oST7kxxZdx9ySY1a5FA7BhyrqgdGz+9iNeR+TlUtVdXeX3YfckntWNhQq6qfAE8kuXT00lXA96faK0mDN9RQ6zr7+QngjtHM5wrwsel1SdLQDXmioFOoVdXDgIeVkl620KEmSesZapKaYqhJaoqhJqkZCz9RIEnrDTXUOt2lQ5LWm+R1akkuTvKt0V2AHktyQ99+OVKT1MuER2ovAL9TVQ8lOR9YTnK4qsa+0N9Qk9TLJEOtqp4Cnhr9/LMkR4Gd9Fi9ZKhJGluPiYId6252sVRVSxt9MMklwGXAAxu9vxlDTVIvY4bayS43u0jyWuDLwI1V9WyffjUXaisrK3Ope+DAL9yNaWYOHjw4t9rLy8tzq713ryv35mnSs59JzmI10O6oqq/0bae5UJM0G5MMtSQBbgGOVtVnXk1bhpqksU1h388rgI8AjyR5ePTa71bV18ZtyFCT1MuEZz+/DWQSbRlqknoZ6ooCQ01SL4aapKYYapKa4V06JDXHUJPUFENNUlMMNUlNGWqobXqTyCSXJnl4zePZJDfOonOShmmcG0QObjPjqvoB8DaAJNuA48DdU+6XpIEb6kht3MPPq4C/rKr/PY3OSFocrYTadcCdG72RZD+w/1X3SNJCGGqodd54JcnZwLXAoY3er6qlqtrb5UZwkhbfwp5TW+M9wENV9dNpdUbSYmhlRcH1vMKhp6StZ6FDLcm5wK8Dvz3d7khaFAsdalX1N8Dfn3JfJC2QhQ41SVrPUJPUjFYmCiTpZYaapKYYapKaYqhJasYU9v2cGENNUi9DHal1XvspSWtNeu1nkn1JfpDkh0kO9O2XIzVJvUxypDa6V+Pvs7py6RjwYJKvVtX3x23LkZqkXiY8Ursc+GFVrVTV88AXgff36VemcVyc5Gmg740kdwAnJ9idRai9Fb/zVq29qN/5V6vqdaefJLln1F5X5wB/t+b5UlUtrWnvA8C+qvqt0fOPAP+kqj4+bkencvi59suPK8mRed2TbV61t+J33qq1W/nOVbVvEu2skY3K9GnIw09JQ3AMuHjN813Ak30aMtQkDcGDwJuTvHF0l+3rgK/2aWiIs59Lm3+kudpb8Ttv1dpb8TtvqqpeSPJx4OvANuDWqnqsT1tTmSiQpHnx8FNSUww1SU0ZTKhNaolEz9q3JjmR5NEZ1704ybeSHE3yWJIbZlj7nCT/I8n3RrX/06xqj+pvS/LdJH8647o/SvJIkoeTHJlx7QuT3JXk8dHf+TtmVPfS0fc9/Xg2yY2zqD0PgzinNloi8ResWSIBXN9niUTP+u8EngP+sKreOouao7oXARdV1UNJzgeWgd+YxfdOEuC8qnouyVnAt4Ebquo70649qv/vgL3ABVV1zSxqjur+CNhbVTO/ADbJ7cB/r6qbRzN851bVX824D9uA46xe2Nr3AvlBG8pIbWJLJPqoqvuBZ2ZVb03dp6rqodHPPwOOAjtnVLuq6rnR07NGj5n8C5dkF/A+4OZZ1BuCJBcA7wRuAaiq52cdaCNXAX/ZaqDBcEJtJ/DEmufHmNH/uYciySXAZcADM6y5LcnDwAngcFXNqvZngU8C87ghVwHfSLKcZP8M6+4GngY+PzrsvjnJeTOsf9p1NL5/71BCbWJLJBZRktcCXwZurKpnZ1W3ql6sqrexevX25Ummfuid5BrgRFUtT7vWK7iiqt4OvAf416NTD7NwJvB24A+q6jLgr4FZnzs+G7gWODTLurM2lFCb2BKJRTM6n/Vl4I6q+so8+jA6DLoPmPR6vo1cAVw7Orf1ReBdSb4wg7oAVNWToz9PAHezeupjFo4Bx9aMhu9iNeRm6T3AQ1X10xnXnamhhNrElkgsktHJ+luAo1X1mRnXfl2SC0c/vwa4Gnh82nWr6lNVtauqLmH17/mbVfXhadcFSHLeaEKG0aHfu4GZzHhX1U+AJ5JcOnrpKmAmE2FrXE/jh54wkGVSk1wi0UeSO4ErgR1JjgE3VdUtMyh9BfAR4JHRuS2A362qr82g9kXA7aPZsDOAL1XVTC+vmIM3AHev/lvCmcAfVdU9M6z/CeCO0T/cK8DHZlU4ybmsXl3w27OqOS+DuKRDkiZlKIefkjQRhpqkphhqkppiqElqiqEmqSmGmqSmGGqSmvL/ARzRSx8Jaw76AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from skimage import io\n",
    "io.imshow(X[0], cmap='gray')\n",
    "io.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Convolution2D, MaxPool2D, Dropout, Flatten\n",
    "from keras import backend as K\n",
    "def create_cnn_model(h,w):\n",
    "    K.set_image_dim_ordering('th')\n",
    "    model = Sequential()\n",
    "    model.add(Convolution2D(32,(3,3), activation='relu', input_shape=(1,h,w)))\n",
    "    model.add(MaxPool2D(pool_size=(2,2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128,activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(128,activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(10,activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy',optimizer='sgd', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reshape input data to 1 channel images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "h = 8\n",
    "w = 8\n",
    "X_train = X_train.reshape(X_train.shape[0],1,h,w)\n",
    "X_test = X_test.reshape(X_test.shape[0],1,h,w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create CNN Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = KerasClassifier(build_fn = create_cnn_model, h=h, w=w, epochs=5, batch_size=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change label to Neural Network friendly format (Dummy Coding, OneHotEncoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import np_utils\n",
    "y_train_oh = np_utils.to_categorical(y_train)\n",
    "y_test_oh = np_utils.to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1437/1437 [==============================] - 1s 618us/step - loss: 2.2514 - acc: 0.2477\n",
      "Epoch 2/5\n",
      "1437/1437 [==============================] - 0s 275us/step - loss: 1.6213 - acc: 0.4356\n",
      "Epoch 3/5\n",
      "1437/1437 [==============================] - 0s 262us/step - loss: 1.2166 - acc: 0.5839\n",
      "Epoch 4/5\n",
      "1437/1437 [==============================] - 0s 305us/step - loss: 0.9944 - acc: 0.6667\n",
      "Epoch 5/5\n",
      "1437/1437 [==============================] - 0s 317us/step - loss: 0.8138 - acc: 0.7265\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x24f2c4fc828>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train_oh)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Continue Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1437/1437 [==============================] - 0s 265us/step - loss: 0.6996 - acc: 0.7648\n",
      "Epoch 2/10\n",
      "1437/1437 [==============================] - 0s 230us/step - loss: 0.6027 - acc: 0.7996\n",
      "Epoch 3/10\n",
      "1437/1437 [==============================] - 0s 220us/step - loss: 0.5870 - acc: 0.7996\n",
      "Epoch 4/10\n",
      "1437/1437 [==============================] - 0s 225us/step - loss: 0.5358 - acc: 0.8260\n",
      "Epoch 5/10\n",
      "1437/1437 [==============================] - 0s 266us/step - loss: 0.5018 - acc: 0.8372\n",
      "Epoch 6/10\n",
      "1437/1437 [==============================] - 0s 241us/step - loss: 0.4340 - acc: 0.8664\n",
      "Epoch 7/10\n",
      "1437/1437 [==============================] - 0s 229us/step - loss: 0.4343 - acc: 0.8636\n",
      "Epoch 8/10\n",
      "1437/1437 [==============================] - 0s 221us/step - loss: 0.3628 - acc: 0.8831\n",
      "Epoch 9/10\n",
      "1437/1437 [==============================] - 0s 242us/step - loss: 0.3981 - acc: 0.8782\n",
      "Epoch 10/10\n",
      "1437/1437 [==============================] - 0s 237us/step - loss: 0.3440 - acc: 0.8914\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x24f2a711048>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.model.fit(X_train, y_train_oh, epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try Test Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9777777777777777"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ex.3 Find Best Model for Esophagus Cancer Prediction <br>\n",
    "You must have 10-CV Training Accuracy & Test Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ex.4 Find Best Model for Digits Image Prediction using CNN <br>\n",
    "You must have 10-CV Training Accuracy & Test Accuracy <br>\n",
    "Hint : Same as Scikit Learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Scikit Learn Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ex.5 Write code to store Scikit Learn Classifier Model (Model Persistence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Scikit Learn Model\n",
    "Ex.6 Write code to load Scikit Learn Classifier Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Deep Learning Model\n",
    "Store Architecture of Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2696"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_model = clf.model.to_json()\n",
    "open('model_architecture.json', 'w').write(json_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store Weight of Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.model.save_weights('model_weights.h5', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(clf.classes_, open('classes.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Deep Learning Model\n",
    "Load Model & Weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import model_from_json\n",
    "model = model_from_json(open('model_architecture.json').read())\n",
    "model.load_weights('model_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compile Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='sgd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create Model & Load Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf2 = KerasClassifier(build_fn = create_cnn_model, h=h, w=w, epochs=5, batch_size=20)\n",
    "clf2.model = model\n",
    "clf2.classes_ = pickle.load(open('classes.pkl','rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try Test Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9777777777777777"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = clf2.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}