GradientDescent  iris_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementing the Gradient Descent Algorithm\n",
    "\n",
    "In this lab, we'll implement the basic functions of the Gradient Descent algorithm to find the boundary in a small dataset. First, we'll start with some functions that will help us plot and visualize the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "#Some helper functions for plotting and drawing lines\n",
    "\n",
    "def plot_points(X, y):\n",
    "    admitted = X[np.argwhere(y==1)]\n",
    "    rejected = X[np.argwhere(y==0)]\n",
    "    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'blue', edgecolor = 'k')\n",
    "    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'red', edgecolor = 'k')\n",
    "\n",
    "def display(m, b, color='g--'):\n",
    "    plt.xlim(-0.05,1.05)\n",
    "    plt.ylim(-0.05,1.05)\n",
    "    x = np.arange(-10, 10, 0.1)\n",
    "    plt.plot(x, m*x+b, color)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading and plotting the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cross-entropy.py', '.DS_Store', 'student_data.csv', 'Learning Pytorch Basics.ipynb', 'fc_model.py', '__pycache__', 'Neural Networks for MNIST PyTorch.ipynb', 'data.csv', 'Single Layer neural network.ipynb', 'README.md', 'Iris.csv', 'Perceptron_Not.py', '.gitignore', 'helper.py', 'Training Neural Network.ipynb', '.ipynb_checkpoints', '.git', 'deep-learning-v2-pytorch', 'GradientDescent.ipynb', 'StudentAdmissions.ipynb', 'assets', 'softmax.py', 'Perceptron_And.py', 'GradientDescent  iris_dataset.ipynb']\n"
     ]
    }
   ],
   "source": [
    "print(os.listdir())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>Iris-virginica</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm         Species\n",
       "145            6.7           3.0            5.2           2.3  Iris-virginica\n",
       "146            6.3           2.5            5.0           1.9  Iris-virginica\n",
       "147            6.5           3.0            5.2           2.0  Iris-virginica\n",
       "148            6.2           3.4            5.4           2.3  Iris-virginica\n",
       "149            5.9           3.0            5.1           1.8  Iris-virginica"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('Iris.csv')\n",
    "del data['Id']\n",
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numerical_label(df):\n",
    "    if(str(df['Species'])=='Iris-setosa'):\n",
    "        return 1\n",
    "    if(df['Species']=='Iris-virginica'):\n",
    "        return 2\n",
    "    if(df['Species']=='Iris-versicolor'):\n",
    "        return 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['label'] = data.apply(numerical_label, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Iris-setosa' 'Iris-versicolor' 'Iris-virginica']\n",
      "[1 3 2]\n"
     ]
    }
   ],
   "source": [
    "data.tail()\n",
    "unique_species = data['Species'].unique()\n",
    "unique_labels = data['label'].unique()\n",
    "print(unique_species)\n",
    "print(unique_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFqxJREFUeJzt3W1wXOdZxvH/VTmmijZpPY2SdKwYTyBtgFCsWPZSi4EQz8TZNnXoNBQBBaIpBBhewsvQEqZ2iy2mkw8wHSa0HRMmDS3QlpaALRBKh7a0tbFsuVKdJoE20EJjCHETFWMhrGh782HXQZZl7a50dvfsOddvRuNdncdnr90j3z569tz7KCIwM7NseUm7A5iZWfJc3M3MMsjF3cwsg1zczcwyyMXdzCyDXNzNzDKo7uIuqUvSlKTRZbbdLem0pOnq108nG9PMzBqxroGx9wJPAldeYvtHIuIX1x7JzMzWqq4zd0l9wOuBB5sbx8zMklDvmft7gLcBV6ww5k2Svh/4EvCrEfG1lXZ41VVXxebNm+t8eDMzAzhx4sTXI6K31riaxV3SHcCzEXFC0i2XGHYI+LOIOCfpZ4GHgVuX2dc9wD0AmzZtYnJystbDm5nZIpL+tZ5x9UzLDAK7JX0V+DBwq6QPLR4QEc9FxLnq3QeBrcvtKCIORMRARAz09tb8j8fMzFapZnGPiPsioi8iNgNDwCcj4i2Lx0h65aK7u6m88WpmZm3SyNUyF5C0D5iMiIPAL0vaDSwAzwN3JxPPzMxWQ+36yN+BgYHwnLuZWWMknYiIgVrj3KFqZpZBq56WMTPLsnK5zNjYGFNTU/T391Mqlejq6mp3rLq5uJuZLVEul3njrl2cmpjgttlZ3tnTw4FikUfGxzumwHtaxsxsibGxMU5NTHD07FneHcHRs2d5emKCsbGxdkerm4u7mdkSU1NT3DY7y2XV+5cBu2ZnmZ6ebmeshri4m5kt0d/fz6M9PbxQvf8CMN7Tw5YtW9oZqyEu7mZmS5RKJTYWixQLBe6TKBYK9BWLlEqldkerm99QNTNboquri0fGxxkbG2N6epp9W7Z03NUybmIyM+sgbmIyM8sxF3czswxycTczyyAXdzOzDHJxNzPLIBd3M7MMcnE3M8sgF3czswxycTczyyAXdzOzDHJxNzPLIH9wmJm1RacvY5d2Lu5m1nJZWMYu7TwtY2Ytl4Vl7NLOxd3MWi4Ly9ilnYu7mbVcFpaxSzsXdzNruSwsY5d2fkPVzFouC8vYpZ2X2TMz6yBeZs/MLMdc3M3MMshz7mYp4G5NS5qLu1mbuVvTmsHTMmZt5m5NawYXd7M2c7emNYOLu1mbuVvTmsHF3azN3K1pzeA3VM3azN2a1gzuUDUz6yCJd6hK6pI0JWl0mW3fIukjkp6SNCFpc2NxzcwsSY1My9wLPAlcucy2twIzEfHtkoaA+4EfSSCfmVlmtLJZra7iLqkPeD3wO8CvLTPkTuBd1dsfAx6QpGjXnI+ZWcq0ulmt3mmZ9wBvA755ie0bga8BRMQC8F/AK9aczswsI1rdrFazuEu6A3g2Ik6s9cEk3SNpUtLk6dOn17o7M7OO0epmtXrO3AeB3ZK+CnwYuFXSh5aMOQVcByBpHfAy4LmlO4qIAxExEBEDvb29awpuZtZJWt2sVrO4R8R9EdEXEZuBIeCTEfGWJcMOAj9VvX1XdYzn283MqlrdrLbqJiZJ+4DJiDgI/BHwQUlPAc9T+U/AzMyqWt2s5iYmM7MO4mX2zMxyzMXdzOoyPz/P3r172blzJ3v37mV+fr7dkWwF/uAwM6tpfn6eG669lstnZtgN/PknP8nDDzzAl595hvXr17c7ni3DZ+5mVtPIyAiXz8xwkspni5wEumdmGBkZaXMyuxQXdzOr6fDhw+yGCxpw7gSOHDnSvlC2Ihd3M6tpcHCQg3BBA85fATt27GhfKFuRL4U0s5rOz7l3z8xwJ5XCPrdhg+fc28CXQppZYtavX8+Xn3mGN+/Zw4mdO3nznj0u7Cnnq2XMrC7r169n37597Y5hdfKZu5lZBrm4m5llkIu72SrMzc0xNDTE9ddfz9DQEHNzc+2OZAkrl8uMjo6yf/9+RkdHKZfL7Y7UEM+5mzVobm6OjVdeyTULC/wwcPArX2Hjxz/OqTNn6O7ubnc8S0Crl8RrBp+5mzVoeHiYaxYWLujWvHphgeHh4TYns6S0ekm8ZnBxN2vQsWPHlu3WPH78ePtCWaJavSReM7i4mzVo+/bty3Zrbtu2rX2hLFGtXhKvGdyhatag83PuVy8svNit+ey6dZ5zz5Dzc+5PT0ywa3aW8Z4e+lIy515vh6rfUDVrUHd3N6fOnGF4eJiPHT/Otm3beOihh1zYM6TVS+I1g8/czcw6iD9bxswsxzwtY7YK5XKZsbExpqam6O/vT92v7M3Il/bnbBdycTdrUNobXJqRL+3P2S7maRmzBqW9waUZ+dL+nO1iLu5mDUp7g0sz8qX9OdvFXNzNGpT2Bpdm5Ev7c7aLubibNahUKrGxWKRYKHCfRLFQoK9YpFQqtTsa0Jx8aX/OdjFf5262CuevHJmenmZLChtcmpEv7c85L+q9zt3F3cysg7iJycwsx1zczcwyyE1MKeeuwLXza2h55OKeYu4KXDu/hpZXnpZJMXcFrp1fQ8srF/cUc1fg2vk1tLxycU8xdwWunV9DyysX9xRzV+Da+TW0vHITU8q5K3Dt/BpalrhD1cwsgxLrUJX0UknHJH1B0uOSfnuZMXdLOi1puvr106sNbmZma1fPde7ngFsj4qyky4DPSRqLiKNLxn0kIn4x+YhmazM/P8/IyAiHDx9mcHCQd7zjHaxfvz5V+8xjo1Uen3MrNTQtI+ly4HPAz0fExKLv3w0MNFLcPS1jrTA/P88N117L5TMz7AYOAv+zYQNffuaZVRfjpPe5tNHq0Z4eNma80SqPzzkpiX5wmKQuSdPAs8AnFhf2Rd4k6aSkj0m6rsG8Zk0xMjLC5TMznATuB04C3TMzjIyMpGafeWy0yuNzbrW6intElCNiC9AHbJd005Ihh4DNEfEa4BPAw8vtR9I9kiYlTZ4+fXotuc3qcvjwYXbDBU1MdwJHjhxJzT7z2GiVx+fcag1d5x4R3wA+Bdy+5PvPRcS56t0Hga2X+PsHImIgIgZ6e3tXk9esIYODgxyEC5qY/grYsWNHavaZx0arPD7nVqs55y6pF3ghIr4hqRt4FLg/IkYXjXllRPxH9fYbgbdHxPeutF/PuVsrnJ8f756Z4U4qRXguoTn3pPZ5fv756YkJds3OMt7TQ1/G55/z+JyTkth17pJeQ2WapYvKmf5HI2KfpH3AZEQclPRuYDewADxP5Q3Xf1xpvy7u1irnr2w5cuQIO3bsSPRqmaT2mcdGqzw+5yS4icnMLIO8zJ6ZWY65uKdcuVxmdHSU/fv3Mzo6Srlcbneki8zPz7N371527tzJ3r17mZ+fb3ekCzQjX9LHxcfZkuZpmRTrhEaPZjQJJakZ+ZI+Lj7O1oh6p2WIiLZ8bd26NWxlhw4dipsLhZiHCIh5iP5CIQ4dOtTuaC/as2dP3FjNdj7jqyH27NnT7mgR0Zx8SR8XH2drBJULWWrWWE/LpFgnNHo0o0koSc3Il/Rx8XG2ZnBxT7FOaPRoRpNQkpqRL+nj4uNszeA59xTrhEaPZjQJJakZ+ZI+Lj7O1ghf554RndDo0YwmoSQ1I1/Sx8XH2erl4m5mlkFuYjIzyzEXdzOzDKpnmT2zjtaM5dy8RJylnYu7ZdrS7s939vRwYI1XojRjn2ZJ87SMZVozlnPzEnHWCVzcLdOa0f3ZCR2lZi7ulmnN6P7shI5SMxd3y7RSqcTGYpFiocB9EsVCgb5ikVKplKp9miXNTUyWec3o/uyEjlLLJneompllkDtUzcxyLPfXuae9GaUZ+c5/ANThw4cZHBzM/AdApf0YmzVDrqdl0r68WTPy5W25tLQfY7NGeVqmDmlvRmlGvpGRES6fmeEkcD9wEuiemWFkZCSp2KmS9mNs1iy5Lu5pb0ZpRr68LZeW9mNs1iy5Lu5pb0ZpRr68LZeW9mNs1iyec0/x8mbNyJe35dLSfozNGuXr3OuU9maUZuTL23JpaT/GZo1wcTczyyBfLWNmlmMu7mZmGeTinnLlcpnR0VH279/P6Ogo5XK53ZEuknTGubk5hoaGuP766xkaGmJubi6hpGb54Tn3FOuE7sqkM87NzbHxyiu5ZmHhxQ7a/1y3jlNnztDd3Z14frNO4zn3DOiE7sqkMw4PD3PNwsIFHbRXLywwPDycZGyzzHNxT7FO6K5MOuOxY8eW7aA9fvz4mrOa5YmLe4p1Qndl0hm3b9++bAfttm3b1pzVLE88555indBdmXTG83PuVy8svNhB+6zn3M1e5CamjOiE7sqkM87NzTE8PMzx48fZtm0bDz30kAu7WZWLu5lZBiV2tYykl0o6JukLkh6X9NvLjPkWSR+R9JSkCUmbVxfbzMySUM8ye+eAWyPirKTLgM9JGouIo4vGvBWYiYhvlzRE5Sq2H2lC3sSXTPMSbGuX9LJ9PsZmCYiIur+Ay4HPA8Ul3x8HXlu9vQ74OtUpn0t9bd26NRq1sLAQb9i5M24uFOI3pbi5UIg37NwZCwsLDe+rGfvLo3PnzsWmDRviRoi3QdwIsWnDhjh37tyq9udjbLYyYDLqqdd1DYIuYBo4C9y/zPYvAn2L7v8zcNVK+1xNcT906FDcXCjEPERAzEP0Fwpx6NCh1bxGie8vj/bs2RM3Vl+786/hqyH27Nmzqv35GJutrN7iXtd17hFRjogtQB+wXdJNq/ktQdI9kiYlTZ4+fbrhv590w0wnNAmlXdLL9vkYmyWjoSamiPgG8Cng9iWbTgHXAUhaB7wMeG6Zv38gIgYiYqC3t7fhsEk3zHRCk1DaJb1sn4+xWUJqndoDvcDLq7e7gc8CdywZ8wvA+6u3h4CP1trvWubc+6vzp/0Jzccmtb88Oj/n/urqnPurE5pz9zE2Wx51TsvUvM5d0muAh6nMu7+kWrj3SdpXfZCDkl4KfBDoB54HhiLiX1ba72qvc0+6YaYTmoTSLull+3yMzS7NTUxmZhnkj/w1M8uxepqYMs0NLmaWRbku7ktXEXpnTw8HUvapi2Zmq5HraZlOWOnIzGw1cl3c3eBiZlmV6+LuBhczy6pcF/dSqcTGYpFiocB9EsVCgb5ikVKp1O5oZmZrkus3VLu6unhkfPzFBpd9bnAxs4xwE5OZWQdxE5OZWY65uJuZZZCLu5lZBrm4m5llkIu7mVkGubibmWWQi7uZWQa5uJuZZZCLu5lZBrm4m5llkIu7mVkGubibmWWQi7uZWQa5uJuZZZCLu5lZBrm4m5llkIu7mVkGubibmWWQi7uZWQa5uJuZZZCLu5lZBrm4m5llkIu7mVkGubibmWWQi7uZWQa5uJuZZZCLu5lZBrm4m5llkIu7mVkG1Szukq6T9ClJT0h6XNK9y4y5RdJ/SZqufu1tTlwzM6vHujrGLAC/HhGfl3QFcELSJyLiiSXjPhsRdyQf0czMGlXzzD0i/iMiPl+9/d/Ak8DGZgczM7PVa2jOXdJmoB+YWGbzayV9QdKYpO9KIJuZma1SPdMyAEgqAB8HfiUizizZ/HngWyPirKTXAX8J3LDMPu4B7gHYtGnTqkObmdnK6jpzl3QZlcL+JxHxF0u3R8SZiDhbvf03wGWSrlpm3IGIGIiIgd7e3jVGNzOzS6nnahkBfwQ8GRG/d4kx11bHIWl7db/PJRnUzMzqV8+0zCDwE8Bjkqar3/stYBNARLwfuAv4eUkLwBwwFBHRhLxmZlaHmsU9Ij4HqMaYB4AHkgplZmZr4w5VM7MMcnE3M8sgF3czswxycTczyyAXdzOzDHJxNzPLIBd3M7MMcnE3M8sgF3czswxycTczyyAXdzOzDKr789ytPuVymbGxMaampujv76dUKtHV1dXuWGaWMy7uCSqXy7xx1y5OTUxw2+ws7+zp4UCxyCPj4y7wZtZSnpZJ0NjYGKcmJjh69izvjuDo2bM8PTHB2NhYu6OZWc64uCdoamqK22Znuax6/zJg1+ws09PTK/01M7PEubgnqL+/n0d7enihev8FYLynhy1btrQzlpnlkIt7gkqlEhuLRYqFAvdJFAsF+opFSqVSu6OZWc74DdUEdXV18cj4OGNjY0xPT7NvyxZfLWNmbaF2LXU6MDAQk5OTbXlsM7NOJelERAzUGudpGTOzDHJxNzPLIBd3M7MMcnE3M8sgF3czswxq29Uykk4D/9qWB1/eVcDX2x1iBWnPB+nPmPZ84IxJSHs+WFvGb42I3lqD2lbc00bSZD2XF7VL2vNB+jOmPR84YxLSng9ak9HTMmZmGeTibmaWQS7u/+9AuwPUkPZ8kP6Mac8HzpiEtOeDFmT0nLuZWQb5zN3MLINyV9wldUmakjS6wpg3SQpJbXnHvVZGSW+W9ISkxyX9aZrySdok6VPV7Sclva4N+b4q6TFJ05Iu+nQ6Vfy+pKeqGW9OYcYfr2Z7TNIRSd+TpnyLxm2TtCDprlbmqz52zYySbqluf1zS36cpn6SXSTok6QvVfMNJPn4eP/L3XuBJ4MrlNkq6ojpmopWhlrhkRkk3APcBgxExI+nqVodj5dfwHcBHI+J9kr4T+BtgcwuznfeDEXGp64hLwA3VryLwvuqfrbZSxq8AP1A9xiUqc7StzrhSPiR1AfcDj7Yu0kUumVHSy4H3ArdHxL+16d/KSq/hLwBPRMQbJPUC/yTpTyJiPokHztWZu6Q+4PXAgysM20/lB/Z/WxJqiToy/gzwBxExAxARz7YqG9SVL/j/ov8y4N9bkatBdwJ/HBVHgZdLemW7Qy0WEUfOH2PgKNDXzjyX8EvAx4GW/gw24MeAv4iIf4PW/1upQwBXSBJQAJ4HFpLaea6KO/Ae4G3AN5fbWP31/LqI+OuWprrQihmBVwGvknRY0lFJt7cuGlA737uAt0h6mspZ+y+1KNdiATwq6YSke5bZvhH42qL7T1e/10q1Mi72VqDVq6yvmE/SRuCNVH7raZdar+GrgA2SPl0d85Mpy/cA8B1UToAeA+6NiEv9u2pYbqZlJN0BPBsRJyTdssz2lwC/B9zd4miLM6yYsWodlemEW6iczX1G0ndHxDdSku9HgQ9ExO9Kei3wQUk3JflDW4fvi4hT1V/DPyHpHyPiMy18/HrUlVHSD1Ip7t+XsnzvAd4eEd+snHi2Ra2M64CtwE6gG/gHSUcj4kspybcLmAZuBb6tOuazEXEmiQfP05n7ILBb0leBDwO3SvrQou1XADcBn66O+V7gYIvfVK2VESpnmQcj4oWI+ArwJSrFPi353gp8FCAi/gF4KZXP0WiZiDhV/fNZ4BFg+5Ihp4DrFt3vq36vZerIiKTXUJn+ujMinktZvgHgw9WfhbuA90r6oZRlfBoYj4jZ6rz3Z4CWvTFdR75hKtNGERFPUXmf5cYkA+Tui8pZ72iNMZ8GBtKWEbgdeLh6+yoq0wuvSFG+MeDu6u3zv3Kqhbl6gCsW3T5C5Q21xWNeX80pKv+JH2vxa1dPxk3AU8CONhzbmvmWjP8AcFfaMlZ//v6Oyhn85cAXgZtSlO99wLuqt6+hcoJxVVIZcjMtcymS9gGTEXGw3VkuZUnGceA2SU8AZeA3osVndTXy/Trwh5J+lcqc491R/eltkWuAR6pTBeuAP42Iv5X0cwAR8X4q7wW8jkrx/B8qZ1CtVE/GvcArqJwRAyxE6z4Mq5587VYzY0Q8KelvgZNU3iN6MCK+mJZ8VC7e+ICkx6icaLw9Vrg6qVHuUDUzy6A8zbmbmeWGi7uZWQa5uJuZZZCLu5lZBrm4m5llkIu7mVkGubibmWWQi7uZWQb9Hz/1S7zfGEoDAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = np.array(data[['SepalLengthCm','SepalWidthCm','PetalLengthCm','PetalWidthCm']])\n",
    "y = np.array(data['label'])\n",
    "plot_points(X,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "      <td>Iris-virginica</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Iris-virginica</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "      <td>Iris-virginica</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "      <td>Iris-virginica</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm         Species  \\\n",
       "145            6.7           3.0            5.2           2.3  Iris-virginica   \n",
       "146            6.3           2.5            5.0           1.9  Iris-virginica   \n",
       "147            6.5           3.0            5.2           2.0  Iris-virginica   \n",
       "148            6.2           3.4            5.4           2.3  Iris-virginica   \n",
       "149            5.9           3.0            5.1           1.8  Iris-virginica   \n",
       "\n",
       "     label  \n",
       "145      2  \n",
       "146      2  \n",
       "147      2  \n",
       "148      2  \n",
       "149      2  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TODO: Implementing the basic functions\n",
    "Here is your turn to shine. Implement the following formulas, as explained in the text.\n",
    "- Sigmoid activation function\n",
    "\n",
    "$$\\sigma(x) = \\frac{1}{1+e^{-x}}$$\n",
    "\n",
    "- Output (prediction) formula\n",
    "\n",
    "$$\\hat{y} = \\sigma(w_1 x_1 + w_2 x_2 + b)$$\n",
    "\n",
    "- Error function\n",
    "\n",
    "$$Error(y, \\hat{y}) = - y \\log(\\hat{y}) - (1-y) \\log(1-\\hat{y})$$\n",
    "\n",
    "- The function that updates the weights\n",
    "\n",
    "$$ w_i \\longrightarrow w_i + \\alpha (y - \\hat{y}) x_i$$\n",
    "\n",
    "$$ b \\longrightarrow b + \\alpha (y - \\hat{y})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement the following functions\n",
    "\n",
    "# Activation (sigmoid) function\n",
    "def sigmoid(x):\n",
    "    return (1/(1+np.exp(-x)))\n",
    "\n",
    "# Output (prediction) formula\n",
    "def output_formula(features, weights, bias):\n",
    "    return (sigmoid(np.dot(features, weights) + bias))\n",
    "\n",
    "# Error (log-loss) formula\n",
    "def error_formula(y, output):\n",
    "    return (-y*(np.log(output)-(1-y)*np.log(1-output)))\n",
    "\n",
    "# Gradient descent step\n",
    "def update_weights(x, y, weights, bias, learnrate):\n",
    "    output = output_formula(x, weights, bias)\n",
    "    d_error = y - output\n",
    "    weights += learnrate * d_error * x\n",
    "    bias += learnrate * d_error\n",
    "    return weights, bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training function\n",
    "This function will help us iterate the gradient descent algorithm through all the data, for a number of epochs. It will also plot the data, and some of the boundary lines obtained as we run the algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(44)\n",
    "\n",
    "epochs = 100\n",
    "learnrate = 0.01\n",
    "\n",
    "def train(features, targets, epochs, learnrate, graph_lines=False):\n",
    "    \n",
    "    errors = []\n",
    "    n_records, n_features = features.shape\n",
    "    last_loss = None\n",
    "    weights = np.random.normal(scale=1 / n_features**.5, size=n_features)\n",
    "    bias = 0\n",
    "    for e in range(epochs):\n",
    "        del_w = np.zeros(weights.shape)\n",
    "        for x, y in zip(features, targets):\n",
    "            output = output_formula(x, weights, bias)\n",
    "            error = error_formula(y, output)\n",
    "            weights, bias = update_weights(x, y, weights, bias, learnrate)\n",
    "        \n",
    "        # Printing out the log-loss error on the training set\n",
    "        out = output_formula(features, weights, bias)\n",
    "        loss = np.mean(error_formula(targets, out))\n",
    "        errors.append(loss)\n",
    "        if e % (epochs / 10) == 0:\n",
    "            print(\"\\n========== Epoch\", e,\"==========\")\n",
    "            if last_loss and last_loss < loss:\n",
    "                print(\"Train loss: \", loss, \"  WARNING - Loss Increasing\")\n",
    "            else:\n",
    "                print(\"Train loss: \", loss)\n",
    "            last_loss = loss\n",
    "            predictions = out > 0.5\n",
    "            accuracy = np.mean(predictions == targets)\n",
    "            print(\"Accuracy: \", accuracy)\n",
    "        if graph_lines and e % (epochs / 100) == 0:\n",
    "            display(-weights[0]/weights[1], -bias/weights[1])\n",
    "            \n",
    "\n",
    "    # Plotting the solution boundary\n",
    "    plt.title(\"Solution boundary\")\n",
    "    display(-weights[0]/weights[1], -bias/weights[1], 'black')\n",
    "\n",
    "    # Plotting the data\n",
    "    plot_points(features, targets)\n",
    "    plt.show()\n",
    "\n",
    "    # Plotting the error\n",
    "    plt.title(\"Error Plot\")\n",
    "    plt.xlabel('Number of epochs')\n",
    "    plt.ylabel('Error')\n",
    "    plt.plot(errors)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time to train the algorithm!\n",
    "When we run the function, we'll obtain the following:\n",
    "- 10 updates with the current training loss and accuracy\n",
    "- A plot of the data and some of the boundary lines obtained. The final one is in black. Notice how the lines get closer and closer to the best fit, as we go through more epochs.\n",
    "- A plot of the error function. Notice how it decreases as we go through more epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:13: RuntimeWarning: divide by zero encountered in log\n",
      "  del sys.path[0]\n",
      "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:13: RuntimeWarning: invalid value encountered in multiply\n",
      "  del sys.path[0]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "========== Epoch 0 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 10 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 20 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 30 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 40 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 50 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 60 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 70 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 80 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n",
      "\n",
      "========== Epoch 90 ==========\n",
      "Train loss:  nan\n",
      "Accuracy:  0.3333333333333333\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEjFJREFUeJzt3X+MZWddx/H3p11Wg5SC7IDQXbpFt8oGVMhYiz+ghmrail0TFXelQElDA1pioGqK/BBL1KABDboqi1YKSH+ACZnAmuKP1obKwk4DVtpSWJdity12Wkr5UWlZ/PrHOauX6czeO7N35+48834lk73nnOee5/vsnfncc59z7z2pKiRJbTlu0gVIksbPcJekBhnuktQgw12SGmS4S1KDDHdJapDhrhWXZHOSSrJumfd/UZKPHIW6zkhyYNz7XY4ktyc5c9J1aPUy3LVsSX4iyb8meSDJl5LckORHxtzHI54Iqupvq+pnxtmP1JplHTlJSR4LfAh4JXA1sB74SeChSda11iVZV1UHJ12HJs8jdy3XqQBVdUVVfauq/ruqPlJVNwEkOS7J65N8Ick9Sd6d5MSFdjR/CiLJm5K8t1+8vv/3y0m+luQ5Sc5P8tGB9j+WZG//CmJvkh8b2HZdkjf3ryq+muQjSTYcbmBJfjvJvX1dLxpYf2I/jrl+XK9PctwCNT/iFcewOpK8uN/nfUleN6+e05J8LMmXk9yd5M+SrB/YXkl+LcnngM8l2ZnkrfP2MZPk1Ycbt9piuGu5Pgt8K8nlSc5O8vh528/vf34KeBrwGODPltHPc/t/H1dVj6mqjw1uTPLdwIeBtwNPAN4GfDjJEwaa/QrwMuCJdK8wfuMw/X0PsAE4CXgpsCvJ9/fb/hQ4sR/P84CX9Psd1YJ1JNkK/AXwYuAp/Tg2DtzvW8Cr+7qeAzwf+NV5+/554EeBrcDlwI6BJ54NwJnA+5ZQq1Y5w13LUlVfAX4CKOCdwFx/dPikvsmLgLdV1f6q+hrwWmD7ck+iHsbPAp+rqvdU1cGqugL4DPBzA23+pqo+W1X/TTeF9MND9vmGqnqoqv6F7onjhUmOB7YDr62qr1bV7cBb6QJ5VIvV8YvAh6rq+qp6CHgD8D+H7lRVN1bVnn58twPvoHtyGfQHVfWl/hXUJ4AH6J4E6Ou+rqr+awm1apUz3LVsVXVrVZ1fVRuBZ9Addf5Jv/kpwBcGmn+B7hzPkxiv+f0c6uukgeUvDtx+kO5VxGLur6qvz9vXU+iOmh/FI8c02M8wi9XxFOCOQxv6/u87tJzk1CQfSvLFJF8Bfr+vZ9Ad85YvB87rb58HvGcJdaoBhrvGoqo+A7yLLuQB7gJOHmjyVOAgsNDR49eBRw8sf8/grod0Pb+fQ33dOeR+i3l8ku+at6+7gHuBb/LIMR3q53BjGOZuYNOhhSSPppuaOeQv6F6NbKmqxwK/DWTePub/P70X2Jbkh4CnAx9cQj1qgOGuZUnyA0kuTrKxX94E7AD29E2uAF6d5JQkj6E72rxqkXdyfIpuyuZRSabppikOmaObonjaIqXsBk5N8itJ1iX5Zbp55w8dwfB+N8n6JD8JvAB4f1V9i24q5feSnJDkZOA1dCF6aAzPTfLU/sTxa5fQ3weAF/RvLV0PXMq3/22eAHwF+FqSH6B7h9JhVdUBYC/dEfvf9VNBWkMMdy3XV+lO4H08ydfpQv3TwMX99svoguV64PPAN4BXLbKvNwDfC9wP/C4DJ/6q6kHg94Ab+neLnD54x6q6jy6AL6abyvgt4AVVde8yx/XFvo67gL8FXtG/KqGv/+vAfuCjfZ2X9XX8A3AVcBNwI0t4cqmqm4Ff6/d3d9//4IepfoPuZOxX6c5vXDXiri8HnolTMmtSvFiH1KYkz6V7ZXFy+Ye+5njkLjUoyaOAXwf+ymBfmwx3qTFJng58GXgy///uJa0xTstIUoM8cpekBk3si8M2bNhQmzdvnlT3krQq3XjjjfdW1dSwdhML982bNzM7Ozup7iVpVUoy/xPZC3JaRpIaZLhLUoMMd0lqkOEuSQ0y3CWpQUPDPcll/WXSPr3I9iR5e5J9SW5K8uzxlylJWopRjtzfBZx1mO1nA1v6nwvpvntakjRBQ8O9qq4HvnSYJtuAd1dnD/C4JE8eV4GSpKUbx5z7SXz7Jb4OsMilx5JcmGQ2yezc3NwYupYkLWRFT6hW1a6qmq6q6ampoZ+elSQt0zjC/U4Grv8IbGT516+UJI3BOMJ9BnhJ/66Z04EHquruMexXkrRMQ784LMkVwBnAhiQHgN8BHgVQVX9Jd4Hic4B9wIPAy45WsZKk0QwN96raMWR70V3cV5J0jPATqpLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNWikcE9yVpLbkuxLcskC25+a5Nokn0xyU5Jzxl+qJGlUQ8M9yfHATuBsYCuwI8nWec1eD1xdVc8CtgN/Pu5CJUmjG+XI/TRgX1Xtr6qHgSuBbfPaFPDY/vaJwF3jK1GStFSjhPtJwB0Dywf6dYPeBJyX5ACwG3jVQjtKcmGS2SSzc3NzyyhXkjSKcZ1Q3QG8q6o2AucA70nyiH1X1a6qmq6q6ampqTF1LUmab5RwvxPYNLC8sV836ALgaoCq+hjwncCGcRQoSVq6UcJ9L7AlySlJ1tOdMJ2Z1+Y/gecDJHk6Xbg77yJJEzI03KvqIHARcA1wK927Ym5OcmmSc/tmFwMvT/JvwBXA+VVVR6toSdLhrRulUVXtpjtROrjujQO3bwF+fLylSZKWy0+oSlKDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAaNFO5JzkpyW5J9SS5ZpM0Lk9yS5OYk7xtvmZKkpVg3rEGS44GdwE8DB4C9SWaq6paBNluA1wI/XlX3J3ni0SpYkjTcKEfupwH7qmp/VT0MXAlsm9fm5cDOqrofoKruGW+ZkqSlGCXcTwLuGFg+0K8bdCpwapIbkuxJctZCO0pyYZLZJLNzc3PLq1iSNNS4TqiuA7YAZwA7gHcmedz8RlW1q6qmq2p6ampqTF1LkuYbJdzvBDYNLG/s1w06AMxU1Ter6vPAZ+nCXpI0AaOE+15gS5JTkqwHtgMz89p8kO6onSQb6KZp9o+xTknSEgwN96o6CFwEXAPcClxdVTcnuTTJuX2za4D7ktwCXAv8ZlXdd7SKliQdXqpqIh1PT0/X7OzsRPqWpNUqyY1VNT2snZ9QlaQGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQSOFe5KzktyWZF+SSw7T7heSVJLp8ZUoSVqqoeGe5HhgJ3A2sBXYkWTrAu1OAH4d+Pi4i5QkLc0oR+6nAfuqan9VPQxcCWxboN2bgbcA3xhjfZKkZRgl3E8C7hhYPtCv+z9Jng1sqqoPH25HSS5MMptkdm5ubsnFSpJGc8QnVJMcB7wNuHhY26raVVXTVTU9NTV1pF1LkhYxSrjfCWwaWN7YrzvkBOAZwHVJbgdOB2Y8qSpJkzNKuO8FtiQ5Jcl6YDswc2hjVT1QVRuqanNVbQb2AOdW1exRqViSNNTQcK+qg8BFwDXArcDVVXVzkkuTnHu0C5QkLd26URpV1W5g97x1b1yk7RlHXpYk6Uj4CVVJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoJHCPclZSW5Lsi/JJQtsf02SW5LclOSfkpw8/lIlSaMaGu5Jjgd2AmcDW4EdSbbOa/ZJYLqqfhD4APCH4y5UkjS6UY7cTwP2VdX+qnoYuBLYNtigqq6tqgf7xT3AxvGWKUlailHC/STgjoHlA/26xVwA/P1CG5JcmGQ2yezc3NzoVUqSlmSsJ1STnAdMA3+00Paq2lVV01U1PTU1Nc6uJUkD1o3Q5k5g08Dyxn7dt0lyJvA64HlV9dB4ypMkLccoR+57gS1JTkmyHtgOzAw2SPIs4B3AuVV1z/jLlCQtxdBwr6qDwEXANcCtwNVVdXOSS5Oc2zf7I+AxwPuTfCrJzCK7kyStgFGmZaiq3cDueeveOHD7zDHXJUk6An5CVZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBo0U7knOSnJbkn1JLllg+3ckuarf/vEkm8ddqCRpdEPDPcnxwE7gbGArsCPJ1nnNLgDur6rvA/4YeMu4C5UkjW6UI/fTgH1Vtb+qHgauBLbNa7MNuLy//QHg+UkyvjIlSUsxSrifBNwxsHygX7dgm6o6CDwAPGH+jpJcmGQ2yezc3NzyKpYkDbWiJ1SraldVTVfV9NTU1Ep2LUlryijhfiewaWB5Y79uwTZJ1gEnAveNo0BJ0tKNEu57gS1JTkmyHtgOzMxrMwO8tL/9i8A/V1WNr0xJ0lKsG9agqg4muQi4BjgeuKyqbk5yKTBbVTPAXwPvSbIP+BLdE4AkaUKGhjtAVe0Gds9b98aB298Afmm8pUmSlstPqEpSgwx3SWqQ4S5JDTLcJalBmdQ7FpPMAV9Y5t03APeOsZzVwDGvDY55bTiSMZ9cVUM/BTqxcD8SSWaranrSdawkx7w2OOa1YSXG7LSMJDXIcJekBq3WcN816QImwDGvDY55bTjqY16Vc+6SpMNbrUfukqTDMNwlqUHHdLivxQtzjzDm1yS5JclNSf4pycmTqHOcho15oN0vJKkkq/5tc6OMOckL+8f65iTvW+kax22E3+2nJrk2ySf73+9zJlHnuCS5LMk9ST69yPYkeXv//3FTkmePtYCqOiZ/6L5e+D+ApwHrgX8Dts5r86vAX/a3twNXTbruFRjzTwGP7m+/ci2MuW93AnA9sAeYnnTdK/A4bwE+CTy+X37ipOtegTHvAl7Z394K3D7puo9wzM8Fng18epHt5wB/DwQ4Hfj4OPs/lo/c1+KFuYeOuaquraoH+8U9dFfGWs1GeZwB3gy8BfjGShZ3lIwy5pcDO6vqfoCqumeFaxy3UcZcwGP72ycCd61gfWNXVdfTXd9iMduAd1dnD/C4JE8eV//HcriP7cLcq8goYx50Ad0z/2o2dMz9y9VNVfXhlSzsKBrlcT4VODXJDUn2JDlrxao7OkYZ85uA85IcoLt+xKtWprSJWerf+5KMdLEOHXuSnAdMA8+bdC1HU5LjgLcB50+4lJW2jm5q5gy6V2fXJ3lmVX15olUdXTuAd1XVW5M8h+7qbs+oqv+ZdGGr0bF85L4WL8w9yphJcibwOuDcqnpohWo7WoaN+QTgGcB1SW6nm5ucWeUnVUd5nA8AM1X1zar6PPBZurBfrUYZ8wXA1QBV9THgO+m+YKtVI/29L9exHO5r8cLcQ8ec5FnAO+iCfbXPw8KQMVfVA1W1oao2V9VmuvMM51bV7GTKHYtRfrc/SHfUTpINdNM0+1eyyDEbZcz/CTwfIMnT6cJ9bkWrXFkzwEv6d82cDjxQVXePbe+TPqM85GzzOXRHLP8BvK5fdyndHzd0D/77gX3AJ4CnTbrmFRjzPwL/BXyq/5mZdM1He8zz2l7HKn+3zIiPc+imo24B/h3YPumaV2DMW4Eb6N5J8yngZyZd8xGO9wrgbuCbdK/ELgBeAbxi4DHe2f9//Pu4f6/9+gFJatCxPC0jSVomw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ16H8BWCtqRNwVR2oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEWCAYAAABBvWFzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFgtJREFUeJzt3X+0XWV95/H3h8SijhYIBCSEGBBGDc5oO1dYrjotCkKYNRoWxQJWiUplOhVnqosZw9BVEF0WsIqrYu0wiEbbCohDSeuqGBA6HepAbgALaENCEPklIKHYiMBK+c4fZ8cerie5N7nPvSc3eb/WOuvs/ezn7P197oV87t7POfukqpAkabJ2G3YBkqSdg4EiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUaYZKsjBJJZk97FokMFCkLUry/SQ/TbKx73HxNNdwZJLnumP/U5I1Sd6zHfs5N8mfTkWN0mb+ZSNt3Vur6rrxOiWZXVWbxmvb1n10Hqqq+UkCLAGuSnIz8NRE9y1NB89QpO2Q5N1JbkpyUZLHgXO30LZbkt9Lcl+SR5N8Kcke3T42X7I6LckPgG9t7ZjV8xfAE8CiATXNS7IiyYYk65K8r2tfDPwP4KTuTOc7jX8cEuAZijQZRwCXA/sBLwBOGtD27u7xJuBR4EvAxcC7+vbza8Crgee2drAku9E7Q9kTuGNAl8uBO4F5wKuAlUnuqapvJPk4cEhVvXM7xilNiGco0tb9RZJ/7Hu8r2/bQ1X1maraVFU/3ULbbwKfqqr1VbUROAs4ecxE+rlV9ZO+fYw1L8k/Aj8CzgHeVVVr+jskORD4FeDDVfV0Vd0OXAqcOumfgDRBnqFIW3f8VuZQ7p9A2zzgvr71++j9f7ffOPvp91BVzR+nzzxgQ1X905hjjYzzOqkZz1Ck7TfoVt1j2x4CXt63vgDYBDwyzn621UPAnCQvHXOsBxseQ9oqA0WaWl8BPpjkoCQvAT4OXLEt7/6aiKq6H/g74A+SvDDJvwVOAza/VfgRYGE3DyNNCf/jkrbuL8d8DuXqbXz9ZcCXgf8D3As8DXygdZGdU4CF9M5WrgbO6btc99Xu+fEkt07R8bWLi1+wJUlqwTMUSVITBookqQkDRZLUhIEiSWpil/pg4z777FMLFy4cdhmSNKOsXr36R1U1d7x+u1SgLFy4kNHR0WGXIUkzSpL7xu/lJS9JUiMGiiSpCQNFktSEgSJJasJAkSQ1YaBIkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ1YaBIkpowUCRJTRgokqQmDBRJUhMGiiSpCQNFktSEgSJJasJAkSQ1YaBIkpowUCRJTRgokqQmDBRJUhNDDZQki5OsSbIuybIB23dPckW3/eYkC8dsX5BkY5Izp6tmSdJgQwuUJLOAzwLHAYuAU5IsGtPtNOCJqjoEuAi4YMz2TwF/PdW1SpLGN8wzlMOBdVW1vqqeBS4HlozpswRY3i1fBRyVJABJjgfuBe6apnolSVsxzEA5ALi/b/2Brm1gn6raBDwJ7J3kJcCHgY+Md5AkpycZTTL62GOPNSlckvTzZuqk/LnARVW1cbyOVXVJVY1U1cjcuXOnvjJJ2kXNHuKxHwQO7Fuf37UN6vNAktnAHsDjwBHAiUkuBPYEnkvydFVdPPVlS5IGGWagrAIOTXIQveA4GXjHmD4rgKXAt4ETgW9VVQH/fnOHJOcCGw0TSRquoQVKVW1KcgZwLTALuKyq7kpyHjBaVSuAzwNfTrIO2EAvdCRJO6D0/uDfNYyMjNTo6Oiwy5CkGSXJ6qoaGa/fTJ2UlyTtYAwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNTHUQEmyOMmaJOuSLBuwffckV3Tbb06ysGt/S5LVSe7ont883bVLkp5vaIGSZBbwWeA4YBFwSpJFY7qdBjxRVYcAFwEXdO0/At5aVf8GWAp8eXqqliRtyTDPUA4H1lXV+qp6FrgcWDKmzxJgebd8FXBUklTVbVX1UNd+F/CiJLtPS9WSpIGGGSgHAPf3rT/QtQ3sU1WbgCeBvcf0+XXg1qp6ZorqlCRNwOxhFzAZSQ6jdxnsmK30OR04HWDBggXTVJkk7XqGeYbyIHBg3/r8rm1gnySzgT2Ax7v1+cDVwKlVdc+WDlJVl1TVSFWNzJ07t2H5kqR+wwyUVcChSQ5K8gvAycCKMX1W0Jt0BzgR+FZVVZI9ga8Dy6rqpmmrWJK0RUMLlG5O5AzgWuB7wJVVdVeS85K8rev2eWDvJOuADwGb31p8BnAI8PtJbu8e+07zECRJfVJVw65h2oyMjNTo6Oiwy5CkGSXJ6qoaGa+fn5SXJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqwkCRJDVhoEiSmjBQJElNGCiSpCYMFElSEwaKJKkJA0WS1ISBIklqYtxASTIryR9ORzGSpJlr3ECpqn8G3jgNtUiSZrDZE+x3W5IVwFeBn2xurKr/PSVVSZJmnIkGyguBx4E397UVYKBIkoAJBkpVvWeqC5EkzWwTepdXkvlJrk7yaPf4WpL5U12cJGnmmOjbhr8ArADmdY+/7NokSQImHihzq+oLVbWpe3wRmDvZgydZnGRNknVJlg3YvnuSK7rtNydZ2LftrK59TZJjJ1uLJGlyJhoojyd5Z/eZlFlJ3klvkn67JZkFfBY4DlgEnJJk0ZhupwFPVNUhwEXABd1rFwEnA4cBi4E/7vYnSRqSiQbKe4HfAH4IPAycCEx2ov5wYF1Vra+qZ4HLgSVj+iwBlnfLVwFHJUnXfnlVPVNV9wLruv1JkoZk3Hd5dX/5n1BVb2t87AOA+/vWHwCO2FKfqtqU5Elg7679/4157QGDDpLkdOB0gAULFjQpXJL08yb6SflTpqGWKVFVl1TVSFWNzJ076WkfSdIWTPSDjTcluRi4gud/Uv7WSRz7QeDAvvX5XdugPg8kmQ3sQW/uZiKvlSRNo4kGyuu65/P62ornf3J+W60CDk1yEL0wOBl4x5g+K4ClwLfpzdt8q6qquw3Mnyf5FL23MR8K3DKJWiRJkzSROZTdgM9V1ZUtD9zNiZwBXAvMAi6rqruSnAeMVtUK4PPAl5OsAzbQCx26flcC3wU2Ae/vLs1JkoYkVTV+p2S0qkamoZ4pNTIyUqOjo8MuQ5JmlCSrJ5IBE33b8HVJzkxyYJI5mx+TrFGStBOZ6BzKSd3z+/vaCji4bTmSpJlqoncbPmiqC5EkzWxbveSV5L/3Lb99zLaPT1VRkqSZZ7w5lJP7ls8as21x41okSTPYeIGSLSwPWpck7cLGC5TawvKgdUnSLmy8SfnXJvkxvbORF3XLdOsvnNLKJEkzylYDpar8jhFJ0oRM9IONkiRtlYEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0MJVCSzEmyMsna7nmvLfRb2vVZm2Rp1/biJF9P8g9J7kpy/vRWL0kaZFhnKMuA66vqUOD6bv15kswBzgGOAA4HzukLnj+sqlcBvwT8SpLjpqdsSdKWDCtQlgDLu+XlwPED+hwLrKyqDVX1BLASWFxVT1XVDQBV9SxwKzB/GmqWJG3FsAJlv6p6uFv+IbDfgD4HAPf3rT/Qtf1Mkj2Bt9I7y5EkDdHsqdpxkuuAlw3YdHb/SlVVktqO/c8GvgL8UVWt30q/04HTARYsWLCth5EkTdCUBUpVHb2lbUkeSbJ/VT2cZH/g0QHdHgSO7FufD9zYt34JsLaqPj1OHZd0fRkZGdnm4JIkTcywLnmtAJZ2y0uBawb0uRY4Jsle3WT8MV0bST4G7AH87jTUKkmagGEFyvnAW5KsBY7u1kkykuRSgKraAHwUWNU9zquqDUnm07tstgi4NcntSX5rGIOQJP2LVO06V4FGRkZqdHR02GVI0oySZHVVjYzXz0/KS5KaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWpiKIGSZE6SlUnWds97baHf0q7P2iRLB2xfkeTOqa9YkjSeYZ2hLAOur6pDgeu79edJMgc4BzgCOBw4pz94kpwAbJyeciVJ4xlWoCwBlnfLy4HjB/Q5FlhZVRuq6glgJbAYIMlLgA8BH5uGWiVJEzCsQNmvqh7uln8I7DegzwHA/X3rD3RtAB8FPgk8Nd6BkpyeZDTJ6GOPPTaJkiVJWzN7qnac5DrgZQM2nd2/UlWVpLZhv68DXlFVH0yycLz+VXUJcAnAyMjIhI8jSdo2UxYoVXX0lrYleSTJ/lX1cJL9gUcHdHsQOLJvfT5wI/AGYCTJ9+nVv2+SG6vqSCRJQzOsS14rgM3v2loKXDOgz7XAMUn26ibjjwGurarPVdW8qloIvBG42zCRpOEbVqCcD7wlyVrg6G6dJCNJLgWoqg305kpWdY/zujZJ0g4oVbvOtMLIyEiNjo4OuwxJmlGSrK6qkfH6+Ul5SVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJgwUSVITBookqQkDRZLUhIEiSWrCQJEkNWGgSJKaMFAkSU0YKJKkJlJVw65h2iR5DLhv2HVso32AHw27iGnmmHcNjnnmeHlVzR2v0y4VKDNRktGqGhl2HdPJMe8aHPPOx0tekqQmDBRJUhMGyo7vkmEXMASOedfgmHcyzqFIkprwDEWS1ISBIklqwkDZASSZk2RlkrXd815b6Le067M2ydIB21ckuXPqK568yYw5yYuTfD3JPyS5K8n501v9tkmyOMmaJOuSLBuwffckV3Tbb06ysG/bWV37miTHTmfdk7G9Y07yliSrk9zRPb95umvfHpP5HXfbFyTZmOTM6ap5SlSVjyE/gAuBZd3yMuCCAX3mAOu757265b36tp8A/Dlw57DHM9VjBl4MvKnr8wvA3wLHDXtMWxjnLOAe4OCu1u8Ai8b0+R3gT7rlk4EruuVFXf/dgYO6/cwa9pimeMy/BMzrll8DPDjs8UzlePu2XwV8FThz2OOZzMMzlB3DEmB5t7wcOH5An2OBlVW1oaqeAFYCiwGSvAT4EPCxaai1le0ec1U9VVU3AFTVs8CtwPxpqHl7HA6sq6r1Xa2X0xt7v/6fxVXAUUnStV9eVc9U1b3Aum5/O7rtHnNV3VZVD3XtdwEvSrL7tFS9/SbzOybJ8cC99MY7oxkoO4b9qurhbvmHwH4D+hwA3N+3/kDXBvBR4JPAU1NWYXuTHTMASfYE3gpcPxVFNjDuGPr7VNUm4Elg7wm+dkc0mTH3+3Xg1qp6ZorqbGW7x9v9Mfhh4CPTUOeUmz3sAnYVSa4DXjZg09n9K1VVSSb8Xu4krwNeUVUfHHtddtimasx9+58NfAX4o6pav31VakeU5DDgAuCYYdcyxc4FLqqqjd0Jy4xmoEyTqjp6S9uSPJJk/6p6OMn+wKMDuj0IHNm3Ph+4EXgDMJLk+/R+n/smubGqjmTIpnDMm10CrK2qTzcod6o8CBzYtz6/axvU54EuJPcAHp/ga3dEkxkzSeYDVwOnVtU9U1/upE1mvEcAJya5ENgTeC7J01V18dSXPQWGPYnjowA+wfMnqC8c0GcOveuse3WPe4E5Y/osZOZMyk9qzPTmi74G7DbssYwzztn03kxwEP8yYXvYmD7v5/kTtld2y4fx/En59cyMSfnJjHnPrv8Jwx7HdIx3TJ9zmeGT8kMvwEdB79rx9cBa4Lq+fzRHgEv7+r2X3sTsOuA9A/YzkwJlu8dM7y/AAr4H3N49fmvYY9rKWP8DcDe9dwKd3bWdB7ytW34hvXf4rANuAQ7ue+3Z3evWsIO+k63lmIHfA37S93u9Hdh32OOZyt9x3z5mfKB46xVJUhO+y0uS1ISBIklqwkCRJDVhoEiSmjBQJElNGCja6SSpJJ/sWz8zybmN9v3FJCe22Nc4x3l7ku8luWGqjzXmuO9OMjM/VKehM1C0M3oGOCHJPsMupF/3CemJOg14X1W9aarqkVozULQz2kTvtiwfHLth7BlGko3d85FJ/ibJNUnWJzk/yW8muaX7bo5X9O3m6CSjSe5O8h+7189K8okkq5L8fZL/1Lffv02yAvjugHpO6fZ/Z5ILurbfB94IfD7JJwa85r/1HecjXdvC7vth/qw7s7kqyYu7bUclua07zmWb796b5PVJ/i7Jd7pxvrQ7xLwk30jvO2gu7BvfF7s670jycz9byXt5aWf1WeDvN/+DOEGvBV4NbKB3K41Lq+rwJP8V+ADwu12/hfRuWf4K4IYkhwCnAk9W1eu7f7BvSvLNrv8vA6+p3i3ofybJPHo3QPx3wBPAN5McX1XndV8sdWZVjY55zTHAod3xA6xI8qvAD4BXAqdV1U1JLgN+p7t89UXgqKq6O8mXgP+c5I+BK4CTqmpVkl8Eftod5nX0vpfkGWBNks8A+wIHVNVrujr23Iafq3YRnqFop1RVPwa+BPyXbXjZqqp6uHq3S78H2BwId9ALkc2urKrnqmotveB5Fb274p6a5HbgZnq3ljm063/L2DDpvB64saoeq94tzf8M+NVxajyme9xG73tgXtV3nPur6qZu+U/pneW8Eri3qu7u2pd3x3gl8HBVrYLez6urAeD6qnqyqp6md1b18m6cByf5TJLFwI/HqVO7IM9QtDP7NL1/dL/Q17aJ7g+pJLvRu5nfZv3fu/Fc3/pzPP//lbH3Kyp6ZwsfqKpr+zckOZLevalaCfAHVfU/xxxn4Rbq2h79P4d/BmZX1RNJXkvvS89+G/gNevdZk37GMxTttKpqA3AlvQnuzb5P7xITwNuAF2zHrt+eZLduXuVgejduvJbepaQXACT510n+1Tj7uQX4tST7JJkFnAL8zTivuRZ4b/fFTCQ5IMm+3bYFSd7QLb8D+L9dbQu7y3IA7+qOsQbYP8nru/28dGtvGuje4LBbVX2N3g0cf3mcOrUL8gxFO7tPAmf0rf8v4Jok3wG+wfadPfyAXhj8IvDbVfV0kkvpXRa7NUmAxxj8tcY/U73vglkG3EDvzOPrVXXNOK/5ZpJXA9/uHYaNwDvpnUmsAd7fzZ98F/hcV9t7gK92gbGK3m3Un01yEvCZJC+iN3+yxe+vofeNg1/ozuoAztpando1ebdhaSfQXfL6q82T5tIweMlLktSEZyiSpCY8Q5EkNWGgSJKaMFAkSU0YKJKkJgwUSVIT/x8TAO3WDp/HAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train(X, y, epochs, learnrate, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}