ensembling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ensembling",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/keshab97/100DaysOfMLcode/blob/master/ensembling.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "aaaUApoyCl8s",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np \n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn import datasets\n",
        "from sklearn.ensemble import RandomForestClassifier, BaggingClassifier, AdaBoostClassifier, VotingClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "c2DnZj3hDmJg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "iris_dataset=datasets.load_iris()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bqovE6uGD-YW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X=iris_dataset.data[:,0:4]\n",
        "y=iris_dataset.target"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OhsHsXpzEr9F",
        "colab_type": "code",
        "outputId": "7cf79dc6-1504-4055-cdbd-e9b4e34ce032",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "print(len(X))\n",
        "print(len(y))"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "150\n",
            "150\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LT84CAMYE6I8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.8,random_state=4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "olOwXGFfFJX7",
        "colab_type": "code",
        "outputId": "bca1d7e0-8a8d-4fe5-b463-09efaf13081f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "cell_type": "code",
      "source": [
        "#descision tree\n",
        "d_tree = DecisionTreeClassifier()\n",
        "d_tree.fit(X_train,y_train)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
              "            max_features=None, max_leaf_nodes=None,\n",
              "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "            min_samples_leaf=1, min_samples_split=2,\n",
              "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
              "            splitter='best')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "metadata": {
        "id": "9c3YMlI0FPWI",
        "colab_type": "code",
        "outputId": "b4f3dae8-6b11-4379-d182-243226461232",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "d_tree.score(X_test,y_test)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9583333333333334"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "metadata": {
        "id": "qyTWfQ7PFain",
        "colab_type": "code",
        "outputId": "bed06349-2d91-4027-9cb7-3e8b7149da0f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "d_tree.score(X_train,y_train)\n"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "metadata": {
        "id": "D1LOHl1mFh6I",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#ensembling methods\n",
        "#execution of random forest classifier\n",
        "random_forest=RandomForestClassifier(n_estimators=20)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vB9MZQNtF20l",
        "colab_type": "code",
        "outputId": "0311d17e-41d8-4b11-c454-ffaae22270ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "cell_type": "code",
      "source": [
        "random_forest.fit(X_train,y_train)"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
              "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "            min_samples_leaf=1, min_samples_split=2,\n",
              "            min_weight_fraction_leaf=0.0, n_estimators=20, n_jobs=None,\n",
              "            oob_score=False, random_state=None, verbose=0,\n",
              "            warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "metadata": {
        "id": "rFBev2uoF69Q",
        "colab_type": "code",
        "outputId": "43418526-4824-4a95-e8e5-e40eb2ca37a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "random_forest.score(X_test,y_test)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9583333333333334"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "metadata": {
        "id": "wNhhU-P9GAFt",
        "colab_type": "code",
        "outputId": "310de657-ccbb-47df-b6ac-a68abce78b7f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "#Bagging \n",
        "\n",
        "bg = BaggingClassifier(DecisionTreeClassifier(), max_samples= 0.5, max_features = 1.0, n_estimators = 20)\n",
        "bg.fit(X_train,y_train)"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
              "            max_features=None, max_leaf_nodes=None,\n",
              "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "            min_samples_leaf=1, min_samples_split=2,\n",
              "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
              "            splitter='best'),\n",
              "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
              "         max_samples=0.5, n_estimators=20, n_jobs=None, oob_score=False,\n",
              "         random_state=None, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "metadata": {
        "id": "DhVO_cESGSGs",
        "colab_type": "code",
        "outputId": "b240b192-735e-4717-d4b6-e2dd48715733",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "bg.score(X_test,y_test)"
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.95"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "metadata": {
        "id": "W1nj2sxsGW9Y",
        "colab_type": "code",
        "outputId": "72919051-b553-4c1c-d69c-d22c2d13f9bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "bg.score(X_train,y_train)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9666666666666667"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "metadata": {
        "id": "srtdVz_vGbBC",
        "colab_type": "code",
        "outputId": "37768d75-0a4a-4a00-a5d4-45ccec167b5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "cell_type": "code",
      "source": [
        "#Boosting - Ada Boost\n",
        "\n",
        "adb = AdaBoostClassifier(DecisionTreeClassifier(),n_estimators = 5, learning_rate = 0.9)\n",
        "adb.fit(X_train,y_train)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AdaBoostClassifier(algorithm='SAMME.R',\n",
              "          base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
              "            max_features=None, max_leaf_nodes=None,\n",
              "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "            min_samples_leaf=1, min_samples_split=2,\n",
              "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
              "            splitter='best'),\n",
              "          learning_rate=0.9, n_estimators=5, random_state=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "metadata": {
        "id": "XHOTEqR9GhIW",
        "colab_type": "code",
        "outputId": "a218240b-3459-49e5-b438-35e3f34b5c0e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "adb.score(X_test,y_test)\n"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.975"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "metadata": {
        "id": "yNiUVHP-Gjfn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Voting Classifier - Multiple Model Ensemble \n",
        "\n",
        "lr = LogisticRegression()\n",
        "dt = DecisionTreeClassifier()\n",
        "nv=GaussianNB()\n",
        "svm = SVC(kernel = 'poly', degree = 2 )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Nkk0au8kGubC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "evc = VotingClassifier( estimators= [('lr',lr),('dt',dt),('svm',svm),('nv',nv)], voting = 'hard')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XFmcsLKKG4x8",
        "colab_type": "code",
        "outputId": "e905ce6d-55d1-4d8b-a78b-b8cf046cfafc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "cell_type": "code",
      "source": [
        "evc.fit(X_train,y_train)\n"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
            "  \"this warning.\", FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
            "  \"avoid this warning.\", FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VotingClassifier(estimators=[('lr', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
              "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
              "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
              "          tol=0.0001, verbose=0, warm_start=False)), ('dt', Decision...\n",
              "  shrinking=True, tol=0.001, verbose=False)), ('nv', GaussianNB(priors=None, var_smoothing=1e-09))],\n",
              "         flatten_transform=None, n_jobs=None, voting='hard', weights=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "metadata": {
        "id": "qdCdRLxbHMio",
        "colab_type": "code",
        "outputId": "e4b4c1b7-c276-4bb5-93c8-a46c361a9dda",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "evc.score(X_test[0:100],y_test[0:100])"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.97"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "metadata": {
        "id": "1IQgeXSFHTVR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}