Predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "import torch\n",
    "import transformers as ppb\n",
    "import warnings\n",
    "import string\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tensorflow\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/d4/c0cd1057b331bc38b65478302114194bd8e1b9c2bbc06e300935c0e93d90/tensorflow-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl (421.8MB)\n",
      "\u001b[K     |████████████████████████████████| 421.8MB 123kB/s  eta 0:00:01\n",
      "\u001b[?25hCollecting as\n",
      "  Downloading https://files.pythonhosted.org/packages/b4/08/226c133ec497d25a63edb38527c02db093c7d89e6d4cdc91078834486a5d/as-0.1-py3-none-any.whl\n",
      "Collecting tf\n",
      "  Downloading https://files.pythonhosted.org/packages/6a/45/bb735169c261b0711570ee2f63d3a2f552a156eb859a408bb8e1cdff4aab/tf-1.0.0.tar.gz\n",
      "Collecting keras-applications>=1.0.8 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 23.2MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting termcolor>=1.1.0 (from tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz\n",
      "Collecting absl-py>=0.7.0 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/53/9243c600e047bd4c3df9e69cfabc1e8004a82cac2e0c484580a78a94ba2a/absl-py-0.9.0.tar.gz (104kB)\n",
      "\u001b[K     |████████████████████████████████| 112kB 44.8MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting scipy==1.4.1; python_version >= \"3\" (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dc/29/162476fd44203116e7980cfbd9352eef9db37c49445d1fec35509022f6aa/scipy-1.4.1-cp36-cp36m-manylinux1_x86_64.whl (26.1MB)\n",
      "\u001b[K     |████████████████████████████████| 26.1MB 53.2MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting opt-einsum>=2.3.2 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b8/83/755bd5324777875e9dff19c2e59daec837d0378c09196634524a3d7269ac/opt_einsum-3.1.0.tar.gz (69kB)\n",
      "\u001b[K     |████████████████████████████████| 71kB 32.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting tensorboard<2.2.0,>=2.1.0 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/23/53ffe290341cd0855d595b0a2e7485932f473798af173bbe3a584b99bb06/tensorboard-2.1.0-py3-none-any.whl (3.8MB)\n",
      "\u001b[K     |████████████████████████████████| 3.8MB 15.9MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting tensorflow-estimator<2.2.0,>=2.1.0rc0 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/90/b77c328a1304437ab1310b463e533fa7689f4bfc41549593056d812fab8e/tensorflow_estimator-2.1.0-py2.py3-none-any.whl (448kB)\n",
      "\u001b[K     |████████████████████████████████| 450kB 53.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting keras-preprocessing>=1.1.0 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/28/6a/8c1f62c37212d9fc441a7e26736df51ce6f0e38455816445471f10da4f0a/Keras_Preprocessing-1.1.0-py2.py3-none-any.whl (41kB)\n",
      "\u001b[K     |████████████████████████████████| 51kB 28.6MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.0)\n",
      "Collecting astor>=0.6.0 (from tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl\n",
      "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.9.1)\n",
      "Collecting grpcio>=1.8.6 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/17/8f/f79c5c174bebece41f824dd7b1ba98da45dc2d4c373b38ac6a7f6a5acb5e/grpcio-1.26.0-cp36-cp36m-manylinux2010_x86_64.whl (2.4MB)\n",
      "\u001b[K     |████████████████████████████████| 2.4MB 55.8MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.33.4)\n",
      "Requirement already satisfied: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.17.0)\n",
      "Collecting gast==0.2.2 (from tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz\n",
      "Collecting wrapt>=1.11.1 (from tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/23/84/323c2415280bc4fc880ac5050dddfb3c8062c2552b34c2e512eb4aa68f79/wrapt-1.11.2.tar.gz\n",
      "Collecting google-pasta>=0.1.6 (from tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c3/fd/1e86bc4837cc9a3a5faf3db9b1854aa04ad35b5f381f9648fbe81a6f94e4/google_pasta-0.1.8-py3-none-any.whl (57kB)\n",
      "\u001b[K     |████████████████████████████████| 61kB 20.6MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow) (2.10.0)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (41.0.1)\n",
      "Collecting werkzeug>=0.11.15 (from tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c2/e4/a859d2fe516f466642fa5c6054fd9646271f9da26b0cac0d2f37fc858c8f/Werkzeug-0.16.1-py2.py3-none-any.whl (327kB)\n",
      "\u001b[K     |████████████████████████████████| 327kB 38.3MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting markdown>=2.6.8 (from tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c0/4e/fd492e91abdc2d2fcb70ef453064d980688762079397f779758e055f6575/Markdown-3.1.1-py2.py3-none-any.whl (87kB)\n",
      "\u001b[K     |████████████████████████████████| 92kB 38.8MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting google-auth-oauthlib<0.5,>=0.4.1 (from tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/7b/b8/88def36e74bee9fce511c9519571f4e485e890093ab7442284f4ffaef60b/google_auth_oauthlib-0.4.1-py2.py3-none-any.whl\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (2.22.0)\n",
      "Collecting google-auth<2,>=1.6.3 (from tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1c/6d/7aae38a9022f982cf8167775c7fc299f203417b698c27080ce09060bba07/google_auth-1.11.0-py2.py3-none-any.whl (76kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 37.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting requests-oauthlib>=0.7.0 (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/a3/12/b92740d845ab62ea4edf04d2f4164d82532b5a0b03836d4d4e71c6f3d379/requests_oauthlib-1.3.0-py2.py3-none-any.whl\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2019.11.28)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (1.25.8)\n",
      "Collecting cachetools<5.0,>=2.0.0 (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/08/6a/abf83cb951617793fd49c98cb9456860f5df66ff89883c8660aa0672d425/cachetools-4.0.0-py3-none-any.whl\n",
      "Collecting rsa<4.1,>=3.1.4 (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "  Downloading https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl\n",
      "Collecting pyasn1-modules>=0.2.1 (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/de/214830a981892a3e286c3794f41ae67a4495df1108c3da8a9f62159b9a9d/pyasn1_modules-0.2.8-py2.py3-none-any.whl (155kB)\n",
      "\u001b[K     |████████████████████████████████| 163kB 56.6MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting oauthlib>=3.0.0 (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/05/57/ce2e7a8fa7c0afb54a0581b14a65b56e62b5759dbc98e80627142b8a3704/oauthlib-3.1.0-py2.py3-none-any.whl (147kB)\n",
      "\u001b[K     |████████████████████████████████| 153kB 56.0MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting pyasn1>=0.1.3 (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/62/1e/a94a8d635fa3ce4cfc7f506003548d0a2447ae76fd5ca53932970fe3053f/pyasn1-0.4.8-py2.py3-none-any.whl (77kB)\n",
      "\u001b[K     |████████████████████████████████| 81kB 31.6MB/s eta 0:00:01\n",
      "\u001b[?25hBuilding wheels for collected packages: tf, termcolor, absl-py, opt-einsum, gast, wrapt\n",
      "  Building wheel for tf (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for tf: filename=tf-1.0.0-cp36-none-any.whl size=1285 sha256=a9b172355594cbb224e6dde6f634778bf898cce16beea087e46883e49ad1f7b8\n",
      "  Stored in directory: /root/.cache/pip/wheels/7c/51/42/b034446bd6c8252825a1a0e7799b6ff633de376967db5ed5a6\n",
      "  Building wheel for termcolor (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for termcolor: filename=termcolor-1.1.0-cp36-none-any.whl size=4833 sha256=9194d98f229fbc5fdef48a4feaceea848ea942e2c31dfad93c3a3bced52d2d03\n",
      "  Stored in directory: /root/.cache/pip/wheels/7c/06/54/bc84598ba1daf8f970247f550b175aaaee85f68b4b0c5ab2c6\n",
      "  Building wheel for absl-py (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for absl-py: filename=absl_py-0.9.0-cp36-none-any.whl size=121933 sha256=648b22e73574a9d5916a2f993323437bf282fefbe520d0bf80590e11c3277ca3\n",
      "  Stored in directory: /root/.cache/pip/wheels/8e/28/49/fad4e7f0b9a1227708cbbee4487ac8558a7334849cb81c813d\n",
      "  Building wheel for opt-einsum (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for opt-einsum: filename=opt_einsum-3.1.0-cp36-none-any.whl size=61682 sha256=18511bfba1e8ad70b73ac716974ab2d04b4762a4151cd4b9b8dc9e524f37269d\n",
      "  Stored in directory: /root/.cache/pip/wheels/2c/b1/94/43d03e130b929aae7ba3f8d15cbd7bc0d1cb5bb38a5c721833\n",
      "  Building wheel for gast (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for gast: filename=gast-0.2.2-cp36-none-any.whl size=7541 sha256=6db660719509892347e6ef668e6c0a569fb7857088fa3eae80dfbe6da0dbdb61\n",
      "  Stored in directory: /root/.cache/pip/wheels/5c/2e/7e/a1d4d4fcebe6c381f378ce7743a3ced3699feb89bcfbdadadd\n",
      "  Building wheel for wrapt (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for wrapt: filename=wrapt-1.11.2-cp36-cp36m-linux_x86_64.whl size=67517 sha256=ee7641e38f9a189f40f6550364b3579e95ae5e3ae07badc26374bd68bef52112\n",
      "  Stored in directory: /root/.cache/pip/wheels/d7/de/2e/efa132238792efb6459a96e85916ef8597fcb3d2ae51590dfd\n",
      "Successfully built tf termcolor absl-py opt-einsum gast wrapt\n",
      "Installing collected packages: keras-applications, termcolor, absl-py, scipy, opt-einsum, werkzeug, markdown, cachetools, pyasn1, rsa, pyasn1-modules, google-auth, oauthlib, requests-oauthlib, google-auth-oauthlib, grpcio, tensorboard, tensorflow-estimator, keras-preprocessing, astor, gast, wrapt, google-pasta, tensorflow, as, tf\n",
      "  Found existing installation: scipy 1.3.0\n",
      "    Uninstalling scipy-1.3.0:\n",
      "      Successfully uninstalled scipy-1.3.0\n",
      "Successfully installed absl-py-0.9.0 as-0.1 astor-0.8.1 cachetools-4.0.0 gast-0.2.2 google-auth-1.11.0 google-auth-oauthlib-0.4.1 google-pasta-0.1.8 grpcio-1.26.0 keras-applications-1.0.8 keras-preprocessing-1.1.0 markdown-3.1.1 oauthlib-3.1.0 opt-einsum-3.1.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 requests-oauthlib-1.3.0 rsa-4.0 scipy-1.4.1 tensorboard-2.1.0 tensorflow-2.1.0 tensorflow-estimator-2.1.0 termcolor-1.1.0 tf-1.0.0 werkzeug-0.16.1 wrapt-1.11.2\n",
      "\u001b[33mWARNING: You are using pip version 19.2.1, however version 20.0.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, Input, Dropout\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.optimizers import RMSprop\n",
    "from tensorflow.keras.models import Model\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission=pd.read_csv('sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "features=np.load('fetures_distill.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_test=np.load('features_test.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613, 768)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=train['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer dense_3 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model=Sequential()\n",
    "model.add(Input(shape=(768,), dtype=tf.float64, name=\"input_feature\"))\n",
    "model.add(Dense(384, activation='relu'))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n",
    "model.compile(optimizer = optimizer , loss = \"binary_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n",
    "                                            patience=3, \n",
    "                                            verbose=1, \n",
    "                                            factor=0.5, \n",
    "                                            min_lr=0.00001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 100\n",
    "batch_size = 86\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_3 (Dense)              (None, 384)               295296    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 384)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 385       \n",
      "=================================================================\n",
      "Total params: 295,681\n",
      "Trainable params: 295,681\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, Y_train, Y_val=train_test_split(features, labels, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6851, 768)\n",
      "(762, 768)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6851 samples, validate on 762 samples\n",
      "Epoch 1/100\n",
      "6851/6851 - 0s - loss: 0.1184 - accuracy: 0.9559 - val_loss: 0.7862 - val_accuracy: 0.7913\n",
      "Epoch 2/100\n",
      "6851/6851 - 0s - loss: 0.1119 - accuracy: 0.9581 - val_loss: 0.7425 - val_accuracy: 0.7835\n",
      "Epoch 3/100\n",
      "6851/6851 - 0s - loss: 0.1140 - accuracy: 0.9583 - val_loss: 0.7382 - val_accuracy: 0.7913\n",
      "Epoch 4/100\n",
      "6851/6851 - 0s - loss: 0.1154 - accuracy: 0.9546 - val_loss: 0.7525 - val_accuracy: 0.7887\n",
      "Epoch 5/100\n",
      "6851/6851 - 0s - loss: 0.1108 - accuracy: 0.9603 - val_loss: 0.7767 - val_accuracy: 0.8071\n",
      "Epoch 6/100\n",
      "6851/6851 - 0s - loss: 0.1065 - accuracy: 0.9594 - val_loss: 0.8262 - val_accuracy: 0.8058\n",
      "Epoch 7/100\n",
      "6851/6851 - 0s - loss: 0.1027 - accuracy: 0.9593 - val_loss: 0.8514 - val_accuracy: 0.7612\n",
      "Epoch 8/100\n",
      "6851/6851 - 0s - loss: 0.1104 - accuracy: 0.9590 - val_loss: 0.7803 - val_accuracy: 0.7992\n",
      "Epoch 9/100\n",
      "6851/6851 - 0s - loss: 0.1019 - accuracy: 0.9618 - val_loss: 0.7976 - val_accuracy: 0.7966\n",
      "Epoch 10/100\n",
      "6851/6851 - 0s - loss: 0.1009 - accuracy: 0.9618 - val_loss: 0.8308 - val_accuracy: 0.7874\n",
      "Epoch 11/100\n",
      "6851/6851 - 0s - loss: 0.0963 - accuracy: 0.9635 - val_loss: 0.8101 - val_accuracy: 0.7782\n",
      "Epoch 12/100\n",
      "6851/6851 - 0s - loss: 0.0979 - accuracy: 0.9626 - val_loss: 0.8497 - val_accuracy: 0.7612\n",
      "Epoch 13/100\n",
      "6851/6851 - 0s - loss: 0.0901 - accuracy: 0.9663 - val_loss: 0.7997 - val_accuracy: 0.7703\n",
      "Epoch 14/100\n",
      "6851/6851 - 0s - loss: 0.0899 - accuracy: 0.9679 - val_loss: 0.8113 - val_accuracy: 0.7900\n",
      "Epoch 15/100\n",
      "6851/6851 - 0s - loss: 0.0893 - accuracy: 0.9683 - val_loss: 0.8749 - val_accuracy: 0.7507\n",
      "Epoch 16/100\n",
      "6851/6851 - 0s - loss: 0.0950 - accuracy: 0.9666 - val_loss: 0.8763 - val_accuracy: 0.7677\n",
      "Epoch 17/100\n",
      "6851/6851 - 0s - loss: 0.0898 - accuracy: 0.9689 - val_loss: 0.9330 - val_accuracy: 0.7625\n",
      "Epoch 18/100\n",
      "6851/6851 - 0s - loss: 0.0859 - accuracy: 0.9695 - val_loss: 0.8475 - val_accuracy: 0.7795\n",
      "Epoch 19/100\n",
      "6851/6851 - 0s - loss: 0.0863 - accuracy: 0.9693 - val_loss: 0.9204 - val_accuracy: 0.7992\n",
      "Epoch 20/100\n",
      "6851/6851 - 0s - loss: 0.0850 - accuracy: 0.9708 - val_loss: 0.9391 - val_accuracy: 0.8058\n",
      "Epoch 21/100\n",
      "6851/6851 - 0s - loss: 0.0803 - accuracy: 0.9708 - val_loss: 1.0716 - val_accuracy: 0.7940\n",
      "Epoch 22/100\n",
      "6851/6851 - 0s - loss: 0.0828 - accuracy: 0.9705 - val_loss: 0.8749 - val_accuracy: 0.7703\n",
      "Epoch 23/100\n",
      "6851/6851 - 0s - loss: 0.0837 - accuracy: 0.9701 - val_loss: 0.8905 - val_accuracy: 0.8031\n",
      "Epoch 24/100\n",
      "6851/6851 - 0s - loss: 0.0841 - accuracy: 0.9683 - val_loss: 0.9675 - val_accuracy: 0.7402\n",
      "Epoch 25/100\n",
      "6851/6851 - 0s - loss: 0.0820 - accuracy: 0.9717 - val_loss: 0.9836 - val_accuracy: 0.7913\n",
      "Epoch 26/100\n",
      "6851/6851 - 0s - loss: 0.0794 - accuracy: 0.9702 - val_loss: 0.9063 - val_accuracy: 0.7874\n",
      "Epoch 27/100\n",
      "6851/6851 - 0s - loss: 0.0801 - accuracy: 0.9705 - val_loss: 0.8956 - val_accuracy: 0.7756\n",
      "Epoch 28/100\n",
      "6851/6851 - 0s - loss: 0.0755 - accuracy: 0.9711 - val_loss: 0.9269 - val_accuracy: 0.7808\n",
      "Epoch 29/100\n",
      "6851/6851 - 0s - loss: 0.0769 - accuracy: 0.9701 - val_loss: 0.9569 - val_accuracy: 0.7835\n",
      "Epoch 30/100\n",
      "6851/6851 - 0s - loss: 0.0808 - accuracy: 0.9704 - val_loss: 0.9382 - val_accuracy: 0.7756\n",
      "Epoch 31/100\n",
      "6851/6851 - 0s - loss: 0.0725 - accuracy: 0.9746 - val_loss: 0.9563 - val_accuracy: 0.7835\n",
      "Epoch 32/100\n",
      "6851/6851 - 0s - loss: 0.0715 - accuracy: 0.9755 - val_loss: 0.9510 - val_accuracy: 0.7861\n",
      "Epoch 33/100\n",
      "6851/6851 - 0s - loss: 0.0710 - accuracy: 0.9733 - val_loss: 0.9702 - val_accuracy: 0.7822\n",
      "Epoch 34/100\n",
      "6851/6851 - 0s - loss: 0.0707 - accuracy: 0.9746 - val_loss: 0.9797 - val_accuracy: 0.7743\n",
      "Epoch 35/100\n",
      "6851/6851 - 0s - loss: 0.0743 - accuracy: 0.9737 - val_loss: 0.9661 - val_accuracy: 0.7808\n",
      "Epoch 36/100\n",
      "6851/6851 - 0s - loss: 0.0711 - accuracy: 0.9747 - val_loss: 0.9749 - val_accuracy: 0.7808\n",
      "Epoch 37/100\n",
      "6851/6851 - 0s - loss: 0.0697 - accuracy: 0.9761 - val_loss: 1.0108 - val_accuracy: 0.7940\n",
      "Epoch 38/100\n",
      "6851/6851 - 0s - loss: 0.0720 - accuracy: 0.9769 - val_loss: 1.0205 - val_accuracy: 0.8018\n",
      "Epoch 39/100\n",
      "6851/6851 - 0s - loss: 0.0673 - accuracy: 0.9747 - val_loss: 1.0025 - val_accuracy: 0.7769\n",
      "Epoch 40/100\n",
      "6851/6851 - 0s - loss: 0.0679 - accuracy: 0.9758 - val_loss: 1.0343 - val_accuracy: 0.7874\n",
      "Epoch 41/100\n",
      "6851/6851 - 0s - loss: 0.0672 - accuracy: 0.9780 - val_loss: 1.0884 - val_accuracy: 0.7940\n",
      "Epoch 42/100\n",
      "6851/6851 - 0s - loss: 0.0660 - accuracy: 0.9768 - val_loss: 1.0416 - val_accuracy: 0.7900\n",
      "Epoch 43/100\n",
      "6851/6851 - 0s - loss: 0.0646 - accuracy: 0.9759 - val_loss: 1.0327 - val_accuracy: 0.7808\n",
      "Epoch 44/100\n",
      "6851/6851 - 0s - loss: 0.0615 - accuracy: 0.9772 - val_loss: 1.0168 - val_accuracy: 0.7861\n",
      "Epoch 45/100\n",
      "6851/6851 - 0s - loss: 0.0612 - accuracy: 0.9769 - val_loss: 1.0437 - val_accuracy: 0.7953\n",
      "Epoch 46/100\n",
      "6851/6851 - 0s - loss: 0.0618 - accuracy: 0.9778 - val_loss: 1.0575 - val_accuracy: 0.7638\n",
      "Epoch 47/100\n",
      "6851/6851 - 0s - loss: 0.0657 - accuracy: 0.9766 - val_loss: 1.0756 - val_accuracy: 0.7913\n",
      "Epoch 48/100\n",
      "6851/6851 - 0s - loss: 0.0686 - accuracy: 0.9759 - val_loss: 1.0936 - val_accuracy: 0.8045\n",
      "Epoch 49/100\n",
      "6851/6851 - 0s - loss: 0.0614 - accuracy: 0.9759 - val_loss: 1.1341 - val_accuracy: 0.7612\n",
      "Epoch 50/100\n",
      "6851/6851 - 0s - loss: 0.0659 - accuracy: 0.9743 - val_loss: 1.0580 - val_accuracy: 0.7874\n",
      "Epoch 51/100\n",
      "6851/6851 - 0s - loss: 0.0606 - accuracy: 0.9778 - val_loss: 1.1263 - val_accuracy: 0.8005\n",
      "Epoch 52/100\n",
      "6851/6851 - 0s - loss: 0.0548 - accuracy: 0.9819 - val_loss: 1.1105 - val_accuracy: 0.7493\n",
      "Epoch 53/100\n",
      "6851/6851 - 0s - loss: 0.0602 - accuracy: 0.9778 - val_loss: 1.0481 - val_accuracy: 0.7940\n",
      "Epoch 54/100\n",
      "6851/6851 - 0s - loss: 0.0555 - accuracy: 0.9801 - val_loss: 1.2275 - val_accuracy: 0.7927\n",
      "Epoch 55/100\n",
      "6851/6851 - 0s - loss: 0.0635 - accuracy: 0.9746 - val_loss: 1.1062 - val_accuracy: 0.7835\n",
      "Epoch 56/100\n",
      "6851/6851 - 0s - loss: 0.0644 - accuracy: 0.9749 - val_loss: 1.0722 - val_accuracy: 0.7782\n",
      "Epoch 57/100\n",
      "6851/6851 - 0s - loss: 0.0576 - accuracy: 0.9807 - val_loss: 1.2210 - val_accuracy: 0.7336\n",
      "Epoch 58/100\n",
      "6851/6851 - 0s - loss: 0.0602 - accuracy: 0.9781 - val_loss: 1.1074 - val_accuracy: 0.7900\n",
      "Epoch 59/100\n",
      "6851/6851 - 0s - loss: 0.0582 - accuracy: 0.9787 - val_loss: 1.1301 - val_accuracy: 0.7887\n",
      "Epoch 60/100\n",
      "6851/6851 - 0s - loss: 0.0596 - accuracy: 0.9771 - val_loss: 1.0855 - val_accuracy: 0.7887\n",
      "Epoch 61/100\n",
      "6851/6851 - 0s - loss: 0.0572 - accuracy: 0.9784 - val_loss: 1.1363 - val_accuracy: 0.7441\n",
      "Epoch 62/100\n",
      "6851/6851 - 0s - loss: 0.0542 - accuracy: 0.9804 - val_loss: 1.1617 - val_accuracy: 0.7966\n",
      "Epoch 63/100\n",
      "6851/6851 - 0s - loss: 0.0537 - accuracy: 0.9794 - val_loss: 1.2482 - val_accuracy: 0.7913\n",
      "Epoch 64/100\n",
      "6851/6851 - 0s - loss: 0.0527 - accuracy: 0.9800 - val_loss: 1.1260 - val_accuracy: 0.7782\n",
      "Epoch 65/100\n",
      "6851/6851 - 0s - loss: 0.0602 - accuracy: 0.9777 - val_loss: 1.1168 - val_accuracy: 0.7822\n",
      "Epoch 66/100\n",
      "6851/6851 - 0s - loss: 0.0534 - accuracy: 0.9784 - val_loss: 1.1372 - val_accuracy: 0.7664\n",
      "Epoch 67/100\n",
      "6851/6851 - 0s - loss: 0.0552 - accuracy: 0.9800 - val_loss: 1.1095 - val_accuracy: 0.7874\n",
      "Epoch 68/100\n",
      "6851/6851 - 0s - loss: 0.0546 - accuracy: 0.9801 - val_loss: 1.1415 - val_accuracy: 0.7743\n",
      "Epoch 69/100\n",
      "6851/6851 - 0s - loss: 0.0529 - accuracy: 0.9816 - val_loss: 1.2226 - val_accuracy: 0.7533\n",
      "Epoch 70/100\n",
      "6851/6851 - 0s - loss: 0.0497 - accuracy: 0.9831 - val_loss: 1.3210 - val_accuracy: 0.8045\n",
      "Epoch 71/100\n",
      "6851/6851 - 0s - loss: 0.0565 - accuracy: 0.9784 - val_loss: 1.1129 - val_accuracy: 0.7717\n",
      "Epoch 72/100\n",
      "6851/6851 - 0s - loss: 0.0513 - accuracy: 0.9820 - val_loss: 1.2142 - val_accuracy: 0.7966\n",
      "Epoch 73/100\n",
      "6851/6851 - 0s - loss: 0.0503 - accuracy: 0.9810 - val_loss: 1.2047 - val_accuracy: 0.7730\n",
      "Epoch 74/100\n",
      "6851/6851 - 0s - loss: 0.0491 - accuracy: 0.9820 - val_loss: 1.3110 - val_accuracy: 0.7966\n",
      "Epoch 75/100\n",
      "6851/6851 - 0s - loss: 0.0513 - accuracy: 0.9790 - val_loss: 1.2123 - val_accuracy: 0.7743\n",
      "Epoch 76/100\n",
      "6851/6851 - 0s - loss: 0.0517 - accuracy: 0.9812 - val_loss: 1.2100 - val_accuracy: 0.7848\n",
      "Epoch 77/100\n",
      "6851/6851 - 0s - loss: 0.0513 - accuracy: 0.9807 - val_loss: 1.2579 - val_accuracy: 0.7638\n",
      "Epoch 78/100\n",
      "6851/6851 - 0s - loss: 0.0473 - accuracy: 0.9809 - val_loss: 1.2607 - val_accuracy: 0.7664\n",
      "Epoch 79/100\n",
      "6851/6851 - 0s - loss: 0.0514 - accuracy: 0.9800 - val_loss: 1.3191 - val_accuracy: 0.7467\n",
      "Epoch 80/100\n",
      "6851/6851 - 0s - loss: 0.0462 - accuracy: 0.9828 - val_loss: 1.2951 - val_accuracy: 0.7861\n",
      "Epoch 81/100\n",
      "6851/6851 - 0s - loss: 0.0519 - accuracy: 0.9800 - val_loss: 1.2339 - val_accuracy: 0.7913\n",
      "Epoch 82/100\n",
      "6851/6851 - 0s - loss: 0.0487 - accuracy: 0.9823 - val_loss: 1.2380 - val_accuracy: 0.7927\n",
      "Epoch 83/100\n",
      "6851/6851 - 0s - loss: 0.0449 - accuracy: 0.9828 - val_loss: 1.2423 - val_accuracy: 0.8045\n",
      "Epoch 84/100\n",
      "6851/6851 - 0s - loss: 0.0488 - accuracy: 0.9816 - val_loss: 1.1988 - val_accuracy: 0.7979\n",
      "Epoch 85/100\n",
      "6851/6851 - 0s - loss: 0.0502 - accuracy: 0.9815 - val_loss: 1.3134 - val_accuracy: 0.8110\n",
      "Epoch 86/100\n",
      "6851/6851 - 0s - loss: 0.0442 - accuracy: 0.9831 - val_loss: 1.2769 - val_accuracy: 0.7730\n",
      "Epoch 87/100\n",
      "6851/6851 - 0s - loss: 0.0500 - accuracy: 0.9815 - val_loss: 1.4807 - val_accuracy: 0.7953\n",
      "Epoch 88/100\n",
      "6851/6851 - 0s - loss: 0.0518 - accuracy: 0.9812 - val_loss: 1.2746 - val_accuracy: 0.7927\n",
      "Epoch 89/100\n",
      "6851/6851 - 0s - loss: 0.0471 - accuracy: 0.9834 - val_loss: 1.2230 - val_accuracy: 0.7585\n",
      "Epoch 90/100\n",
      "6851/6851 - 0s - loss: 0.0430 - accuracy: 0.9847 - val_loss: 1.3765 - val_accuracy: 0.7585\n",
      "Epoch 91/100\n",
      "6851/6851 - 0s - loss: 0.0505 - accuracy: 0.9809 - val_loss: 1.2631 - val_accuracy: 0.7874\n",
      "Epoch 92/100\n",
      "6851/6851 - 0s - loss: 0.0441 - accuracy: 0.9834 - val_loss: 1.3205 - val_accuracy: 0.7835\n",
      "Epoch 93/100\n",
      "6851/6851 - 0s - loss: 0.0474 - accuracy: 0.9810 - val_loss: 1.3054 - val_accuracy: 0.7743\n",
      "Epoch 94/100\n",
      "6851/6851 - 0s - loss: 0.0422 - accuracy: 0.9845 - val_loss: 1.3670 - val_accuracy: 0.8005\n",
      "Epoch 95/100\n",
      "6851/6851 - 0s - loss: 0.0489 - accuracy: 0.9820 - val_loss: 1.3001 - val_accuracy: 0.7927\n",
      "Epoch 96/100\n",
      "6851/6851 - 0s - loss: 0.0453 - accuracy: 0.9835 - val_loss: 1.2983 - val_accuracy: 0.7835\n",
      "Epoch 97/100\n",
      "6851/6851 - 0s - loss: 0.0480 - accuracy: 0.9818 - val_loss: 1.3711 - val_accuracy: 0.7992\n",
      "Epoch 98/100\n",
      "6851/6851 - 0s - loss: 0.0464 - accuracy: 0.9822 - val_loss: 1.3308 - val_accuracy: 0.7835\n",
      "Epoch 99/100\n",
      "6851/6851 - 0s - loss: 0.0465 - accuracy: 0.9826 - val_loss: 1.3358 - val_accuracy: 0.7913\n",
      "Epoch 100/100\n",
      "6851/6851 - 0s - loss: 0.0452 - accuracy: 0.9835 - val_loss: 1.3145 - val_accuracy: 0.7887\n"
     ]
    }
   ],
   "source": [
    "trained_model = model.fit(\n",
    "    X_train, Y_train,\n",
    "    batch_size=batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=(X_val, Y_val),\n",
    "    verbose=2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=model.predict(features_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3263, 1)\n"
     ]
    }
   ],
   "source": [
    "print(predictions.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['target']= predictions.round().astype(int)\n",
    "submission.to_csv('submission_NN.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='submission_NN.csv' target='_blank'>submission_NN.csv</a><br>"
      ],
      "text/plain": [
       "/storage/submission_NN.csv"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import FileLink\n",
    "FileLink('submission_NN.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf=LogisticRegression()\n",
    "clf.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8228346456692913"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction=clf.predict(features_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission['target']=prediction.round().astype(int)\n",
    "submission.to_csv('submission_LR.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='submission_LR.csv' target='_blank'>submission_LR.csv</a><br>"
      ],
      "text/plain": [
       "/storage/submission_LR.csv"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FileLink('submission_LR.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB(priors=None, var_smoothing=1e-09)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GNB= GaussianNB()\n",
    "GNB.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7637795275590551"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "GNB.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "                       max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=10,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=None, splitter='best')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DTC= DecisionTreeClassifier(criterion='gini', \n",
    "                             min_samples_split=10,min_samples_leaf=1,\n",
    "                             max_features='auto')\n",
    "DTC.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6811023622047244"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DTC.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,\n",
       "                   n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model= AdaBoostClassifier()\n",
    "model.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7742782152230971"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearDiscriminantAnalysis(n_components=None, priors=None, shrinkage=None,\n",
       "                           solver='svd', store_covariance=False, tol=0.0001)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model= LinearDiscriminantAnalysis()\n",
    "model.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8228346456692913"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='submission_LDA.csv' target='_blank'>submission_LDA.csv</a><br>"
      ],
      "text/plain": [
       "/storage/submission_LDA.csv"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission['target']=prediction.round().astype(int)\n",
    "submission.to_csv('submission_LDA.csv', index=False)\n",
    "FileLink('submission_LDA.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "                           learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "                           max_features=None, max_leaf_nodes=None,\n",
       "                           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                           min_samples_leaf=1, min_samples_split=2,\n",
       "                           min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                           n_iter_no_change=None, presort='auto',\n",
       "                           random_state=None, subsample=1.0, tol=0.0001,\n",
       "                           validation_fraction=0.1, verbose=0,\n",
       "                           warm_start=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model= GradientBoostingClassifier()\n",
    "model.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8018372703412073"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=10,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=700,\n",
       "                       n_jobs=-1, oob_score=True, random_state=1, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RandomForestClassifier(criterion='gini', n_estimators=700,\n",
    "                             min_samples_split=10,min_samples_leaf=1,\n",
    "                             max_features='auto',oob_score=True,\n",
    "                             random_state=1,n_jobs=-1)\n",
    "model.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8070866141732284"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "    kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "    shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = SVC()\n",
    "model.fit(X_train,Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8018372703412073"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=400, n_jobs=1,\n",
       "                       oob_score=False, random_state=None, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "random_forest = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
    "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, n_estimators=400, n_jobs=1,\n",
    "            oob_score=False, random_state=None, verbose=0,\n",
    "            warm_start=False)\n",
    "random_forest.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7979002624671916"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_forest.score(X_val, Y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}