Supervised learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['DESCR', 'feature_names', 'target_names', 'data', 'target'])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "cancer.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(569, 30)\n"
     ]
    }
   ],
   "source": [
    "print(cancer.data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['malignant' 'benign']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([212, 357], dtype=int64)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "print(cancer.target_names)\n",
    "np.bincount(cancer.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['mean radius', 'mean texture', 'mean perimeter', 'mean area',\n",
       "       'mean smoothness', 'mean compactness', 'mean concavity',\n",
       "       'mean concave points', 'mean symmetry', 'mean fractal dimension',\n",
       "       'radius error', 'texture error', 'perimeter error', 'area error',\n",
       "       'smoothness error', 'compactness error', 'concavity error',\n",
       "       'concave points error', 'symmetry error', 'fractal dimension error',\n",
       "       'worst radius', 'worst texture', 'worst perimeter', 'worst area',\n",
       "       'worst smoothness', 'worst compactness', 'worst concavity',\n",
       "       'worst concave points', 'worst symmetry', 'worst fractal dimension'], \n",
       "      dtype='<U23')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cancer.feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(506, 13)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "print(boston.data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(506, 105)\n"
     ]
    }
   ],
   "source": [
    "from mglearn.datasets import load_extended_boston\n",
    "X, y = load_extended_boston()\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mglearn\n",
    "import matplotlib.pyplot as plt\n",
    "mglearn.plots.plot_knn_classification(n_neighbors=1)\n",
    "plt.title(\"forge_one_neighbor\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_classification(n_neighbors=3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y,random_state=0 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "clf = KNeighborsClassifier(n_neighbors=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=3, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 1, 0, 1, 0, 0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8571428571428571"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(10, 3))\n",
    "\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors).fit(X, y)\n",
    "    mglearn.plots.plot_2d_separator(clf, X, fill=True, eps=0.5, ax=ax, alpha=.4)\n",
    "    ax.scatter(X[:, 0], X[:, 1], c=y, s=60, cmap=mglearn.cm2)\n",
    "    ax.set_title(\"%d neighbor(s)\" % n_neighbors)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=66)\n",
    "\n",
    "training_accuracy = []\n",
    "test_accuracy = []\n",
    "neighbors_settings = range(1, 11)\n",
    "\n",
    "for n_neighbors in neighbors_settings:\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors)\n",
    "    clf.fit(X_train, y_train)\n",
    "    training_accuracy.append(clf.score(X_train, y_train))\n",
    "    test_accuracy.append(clf.score(X_test, y_test))\n",
    "\n",
    "plt.plot(neighbors_settings, training_accuracy, label=\"training accuracy\")\n",
    "plt.plot(neighbors_settings, test_accuracy, label=\"test accuracy\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_regression(n_neighbors=3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsRegressor(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "          metric_params=None, n_jobs=1, n_neighbors=3, p=2,\n",
       "          weights='uniform')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "X, y = mglearn.datasets.make_wave(n_samples=40)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "\n",
    "reg = KNeighborsRegressor(n_neighbors=3)\n",
    "reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.05396539,  0.35686046,  1.13671923, -1.89415682, -1.13881398,\n",
       "       -1.63113382,  0.35686046,  0.91241374, -0.44680446, -1.13881398])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.83441724462496036"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(15,4))\n",
    "\n",
    "line = np.linspace(-3, 3, 1000).reshape(-1, 1)\n",
    "plt.suptitle(\"nearest_neigbor_regression\")\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    reg = KNeighborsRegressor(n_neighbors=n_neighbors).fit(X, y)\n",
    "    ax.plot(X, y, 'o')\n",
    "    ax.plot(X, -3 * np.ones(len(X)), 'o')\n",
    "    ax.plot(line, reg.predict(line))\n",
    "    ax.set_title(\"%d neighbor(s)\" % n_neighbors)\n",
    "    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w[0]: 0.393906  b: -0.031804\n"
     ]
    }
   ],
   "source": [
    "mglearn.plots.plot_linear_regression_wave()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "X, y = mglearn.datasets.make_wave(n_samples=60)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
    "\n",
    "lr = LinearRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr.coef_: [ 0.39390555]\n",
      "lr.intercept_: -0.0318043430268\n"
     ]
    }
   ],
   "source": [
    "print(\"lr.coef_: %s\" % lr.coef_)\n",
    "print(\"lr.intercept_: %s\" % lr.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.670089\n",
      "test set score: 0.659337\n"
     ]
    }
   ],
   "source": [
    "print(\"training set score: %f\" % lr.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % lr.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.load_extended_boston()\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "lr = LinearRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.952353\n",
      "test set score: 0.605775\n"
     ]
    }
   ],
   "source": [
    "print(\"training set score: %f\" % lr.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % lr.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.886058\n",
      "test set score: 0.752714\n"
     ]
    }
   ],
   "source": [
    "from  sklearn.linear_model import Ridge\n",
    "\n",
    "ridge = Ridge().fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % ridge.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % ridge.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.788346\n",
      "test set score: 0.635897\n"
     ]
    }
   ],
   "source": [
    "ridge10 = Ridge(alpha=10).fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % ridge10.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % ridge10.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.928578\n",
      "test set score: 0.771793\n"
     ]
    }
   ],
   "source": [
    "ridge01 = Ridge(alpha=0.1).fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % ridge01.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % ridge01.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.title(\"ridge_coefficients\")\n",
    "plt.plot(ridge.coef_, 'o', label=\"ridge alpa=1\")\n",
    "plt.plot(ridge10.coef_, 'o', label=\"ridge alpa=10\")\n",
    "plt.plot(ridge01.coef_, 'o', label=\"ridge alpa=01\")\n",
    "\n",
    "plt.plot(lr.coef_, 'o', label=\"LinearRegression\")\n",
    "plt.ylim(-25, 25)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.293238\n",
      "Test set score: 0.209375\n",
      "number of features used: 4\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "lasso = Lasso().fit(X_train, y_train)\n",
    "print(\"Training set score: %f\" % lasso.score(X_train, y_train))\n",
    "print(\"Test set score: %f\" % lasso.score(X_test, y_test))\n",
    "print(\"number of features used: %d\" % np.sum(lasso.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.896408\n",
      "Test set score: 0.767806\n",
      "number of features used: 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\tools\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:474: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations\n",
      "  ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "lasso001 = Lasso(alpha=0.01).fit(X_train, y_train)\n",
    "print(\"Training set score: %f\" % lasso001.score(X_train, y_train))\n",
    "print(\"Test set score: %f\" % lasso001.score(X_test, y_test))\n",
    "print(\"number of features used: %d\" % np.sum(lasso001.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set score: 0.942433\n",
      "Test set score: 0.695635\n",
      "number of features used: 100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\tools\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:474: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations\n",
      "  ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "\n",
    "lasso00001 = Lasso(alpha=0.0001).fit(X_train, y_train)\n",
    "print(\"Training set score: %f\" % lasso00001.score(X_train, y_train))\n",
    "print(\"Test set score: %f\" % lasso00001.score(X_test, y_test))\n",
    "print(\"number of features used: %d\" % np.sum(lasso00001.coef_ != 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(lasso.coef_, 'o', label=\"Lasso alpha=1\")\n",
    "plt.plot(lasso001.coef_, 'o', label=\"Lasso alpha=0.01\")\n",
    "plt.plot(lasso00001.coef_, 'o', label=\"Lasso alpha=0.0001\")\n",
    "\n",
    "plt.plot(ridge01.coef_, 'o', label=\"Ridge alpha=0.1\")\n",
    "plt.ylim(-25, 25)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(10,3))\n",
    "plt.suptitle(\"linear_classifiers\")\n",
    "\n",
    "for model, ax in zip([LinearSVC(), LogisticRegression()], axes):\n",
    "        clf = model.fit(X, y)\n",
    "        mglearn.plots.plot_2d_separator(clf, X, fill=False, eps=0.5, ax=ax, alpha=.7)\n",
    "        ax.scatter(X[:,0],X[:,1], c=y, s=60, cmap=mglearn.cm2)\n",
    "        ax.set_title(\"%s\" % clf.__class__.__name__)\n",
    "        \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_linear_svc_regularization()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.953052\n",
      "test set score: 0.958042\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=42)\n",
    "logisticregression = LogisticRegression().fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % logisticregression.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % logisticregression.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.971831\n",
      "test set score: 0.965035\n"
     ]
    }
   ],
   "source": [
    "logisticregression100 = LogisticRegression(C=100).fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % logisticregression100.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % logisticregression100.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training set score: 0.934272\n",
      "test set score: 0.930070\n"
     ]
    }
   ],
   "source": [
    "logisticregression001 = LogisticRegression(C=0.01).fit(X_train, y_train)\n",
    "print(\"training set score: %f\" % logisticregression001.score(X_train, y_train))\n",
    "print(\"test set score: %f\" % logisticregression001.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(logisticregression.coef_.T, 'o', label=\"C=1\")\n",
    "plt.plot(logisticregression100.coef_.T, 'o', label=\"C=100\")\n",
    "plt.plot(logisticregression001.coef_.T, 'o', label=\"C=0.001\")\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "plt.ylim(-5, 5)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy of L1 logreg with C=0.001000: 0.913146\n",
      "test accuracy of L1 logreg with C=0.001000: 0.923077\n",
      "training accuracy of L1 logreg with C=1.000000: 0.960094\n",
      "test accuracy of L1 logreg with C=1.000000: 0.958042\n",
      "training accuracy of L1 logreg with C=100.000000: 0.985915\n",
      "test accuracy of L1 logreg with C=100.000000: 0.979021\n"
     ]
    }
   ],
   "source": [
    "for C in [0.001, 1, 100]:\n",
    "    lr_l1 = LogisticRegression(C=C, penalty=\"l1\").fit(X_train, y_train)\n",
    "    print(\"training accuracy of L1 logreg with C=%f: %f\"\n",
    "          % (C, lr_l1.score(X_train, y_train)))\n",
    "    print(\"test accuracy of L1 logreg with C=%f: %f\"\n",
    "          % (C, lr_l1.score(X_test, y_test)))\n",
    "    plt.plot(lr_l1.coef_.T, 'o', label=\"C=%f\" % C)\n",
    "\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "\n",
    "plt.ylim(-5, 5)\n",
    "plt.legend(loc=2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "\n",
    "X, y = make_blobs(random_state=42)\n",
    "plt.scatter(X[:, 0], X[:,1], c=y, s=120, cmap=mglearn.cm3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2)\n",
      "(3,)\n"
     ]
    }
   ],
   "source": [
    "linear_svm = LinearSVC().fit(X, y)\n",
    "print(linear_svm.coef_.shape)\n",
    "print(linear_svm.intercept_.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=60, cmap=mglearn.cm3)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept in zip(linear_svm.coef_, linear_svm.intercept_):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1])\n",
    "plt.ylim(-10, 15)\n",
    "plt.xlim(-10, 8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_2d_classification(linear_svm, X, fill=True, alpha=.7)\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=60)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept in zip(linear_svm.coef_, linear_svm.intercept_):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = np.array([[0, 1, 0, 1],\n",
    "              [1, 0, 1, 1],\n",
    "              [0, 0, 0, 1],\n",
    "              [1, 0, 1, 0]])\n",
    "y = np.array([0, 1, 0, 1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: array([0, 1, 0, 2]), 1: array([2, 0, 2, 1])}\n"
     ]
    }
   ],
   "source": [
    "counts = {}\n",
    "for label in np.unique(y): \n",
    "    counts[label] = X[y == label].sum(axis=0)\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_animal_tree()\n",
    "plt.suptitle(\"animal_tree\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mglearn.plots.plot_tree_progressive()\n",
    "plt.suptitle(\"tree building\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 1.000000\n",
      "accuracy on test set: 0.937063\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=42)\n",
    "tree = DecisionTreeClassifier(random_state=0)\n",
    "tree.fit(X_train, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % tree.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % tree.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.988263\n",
      "accuracy on test set: 0.951049\n"
     ]
    }
   ],
   "source": [
    "tree = DecisionTreeClassifier(max_depth=4, random_state=0)\n",
    "tree.fit(X_train,y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % tree.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % tree.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import export_graphviz\n",
    "export_graphviz(tree, out_file=\"mytree.dot\", class_names=[\"malignant\", \"benign\"], feature_names=cancer.feature_names, impurity=False, filled=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: Tree Pages: 1 -->\r\n",
       "<svg width=\"1036pt\" height=\"477pt\"\r\n",
       " viewBox=\"0.00 0.00 1036.00 477.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 473)\">\r\n",
       "<title>Tree</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-473 1032,-473 1032,4 -4,4\"/>\r\n",
       "<!-- 0 -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>0</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.403922\" stroke=\"black\" points=\"638,-469 491,-469 491,-401 638,-401 638,-469\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"564.5\" y=\"-453.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst radius &lt;= 16.795</text>\r\n",
       "<text text-anchor=\"middle\" x=\"564.5\" y=\"-438.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 426</text>\r\n",
       "<text text-anchor=\"middle\" x=\"564.5\" y=\"-423.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [159, 267]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"564.5\" y=\"-408.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 1 -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>1</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.901961\" stroke=\"black\" points=\"565.5,-365 367.5,-365 367.5,-297 565.5,-297 565.5,-365\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-349.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst concave points &lt;= 0.1359</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-334.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 284</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-319.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [25, 259]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-304.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 0&#45;&gt;1 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>0&#45;&gt;1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M532.682,-400.884C524.039,-391.887 514.578,-382.041 505.583,-372.678\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"507.946,-370.086 498.494,-365.299 502.898,-374.935 507.946,-370.086\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"497.994\" y=\"-386.594\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">True</text>\r\n",
       "</g>\r\n",
       "<!-- 14 -->\r\n",
       "<g id=\"node15\" class=\"node\"><title>14</title>\r\n",
       "<polygon fill=\"#e58139\" fill-opacity=\"0.941176\" stroke=\"black\" points=\"736.5,-365 590.5,-365 590.5,-297 736.5,-297 736.5,-365\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-349.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">texture error &lt;= 0.4732</text>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-334.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 142</text>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-319.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [134, 8]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-304.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 0&#45;&gt;14 -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>0&#45;&gt;14</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M596.642,-400.884C605.374,-391.887 614.931,-382.041 624.018,-372.678\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"626.727,-374.913 631.18,-365.299 621.704,-370.038 626.727,-374.913\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"631.553\" y=\"-386.597\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">False</text>\r\n",
       "</g>\r\n",
       "<!-- 2 -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>2</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.984314\" stroke=\"black\" points=\"358,-261 215,-261 215,-193 358,-193 358,-261\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-245.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">radius error &lt;= 1.0475</text>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-230.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 252</text>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-215.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [4, 248]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 1&#45;&gt;2 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>1&#45;&gt;2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M408.059,-296.884C390.838,-287.125 371.852,-276.366 354.106,-266.31\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"355.381,-263.01 344.955,-261.125 351.93,-269.1 355.381,-263.01\"/>\r\n",
       "</g>\r\n",
       "<!-- 7 -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>7</title>\r\n",
       "<polygon fill=\"#e58139\" fill-opacity=\"0.474510\" stroke=\"black\" points=\"538.5,-261 394.5,-261 394.5,-193 538.5,-193 538.5,-261\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-245.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst texture &lt;= 25.62</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-230.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 32</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-215.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [21, 11]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"466.5\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 1&#45;&gt;7 -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>1&#45;&gt;7</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M466.5,-296.884C466.5,-288.778 466.5,-279.982 466.5,-271.472\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"470,-271.299 466.5,-261.299 463,-271.299 470,-271.299\"/>\r\n",
       "</g>\r\n",
       "<!-- 3 -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>3</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.988235\" stroke=\"black\" points=\"211.5,-157 35.5,-157 35.5,-89 211.5,-89 211.5,-157\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"123.5\" y=\"-141.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">smoothness error &lt;= 0.0033</text>\r\n",
       "<text text-anchor=\"middle\" x=\"123.5\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 251</text>\r\n",
       "<text text-anchor=\"middle\" x=\"123.5\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 248]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"123.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 2&#45;&gt;3 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>2&#45;&gt;3</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M233.579,-192.884C218.127,-183.214 201.106,-172.563 185.164,-162.587\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"186.768,-159.462 176.435,-157.125 183.055,-165.396 186.768,-159.462\"/>\r\n",
       "</g>\r\n",
       "<!-- 6 -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>6</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"343,-149.5 230,-149.5 230,-96.5 343,-96.5 343,-149.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-134.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 1</text>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-119.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 0]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"286.5\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 2&#45;&gt;6 -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>2&#45;&gt;6</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M286.5,-192.884C286.5,-182.326 286.5,-170.597 286.5,-159.854\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"290,-159.52 286.5,-149.52 283,-159.52 290,-159.52\"/>\r\n",
       "</g>\r\n",
       "<!-- 4 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>4</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.666667\" stroke=\"black\" points=\"97,-53 0,-53 0,-0 97,-0 97,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 4</text>\r\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 3]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"48.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 3&#45;&gt;4 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>3&#45;&gt;4</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M97.2686,-88.9485C90.1535,-79.9834 82.4418,-70.2666 75.296,-61.2629\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"77.8841,-58.8938 68.926,-53.2367 72.4011,-63.2454 77.8841,-58.8938\"/>\r\n",
       "</g>\r\n",
       "<!-- 5 -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>5</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.992157\" stroke=\"black\" points=\"219.5,-53 115.5,-53 115.5,-0 219.5,-0 219.5,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"167.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 247</text>\r\n",
       "<text text-anchor=\"middle\" x=\"167.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [2, 245]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"167.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 3&#45;&gt;5 -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>3&#45;&gt;5</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M138.889,-88.9485C142.893,-80.3494 147.219,-71.0586 151.265,-62.3689\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"154.469,-63.7796 155.517,-53.2367 148.123,-60.8249 154.469,-63.7796\"/>\r\n",
       "</g>\r\n",
       "<!-- 8 -->\r\n",
       "<g id=\"node9\" class=\"node\"><title>8</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.666667\" stroke=\"black\" points=\"541.5,-157 361.5,-157 361.5,-89 541.5,-89 541.5,-157\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"451.5\" y=\"-141.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst smoothness &lt;= 0.1786</text>\r\n",
       "<text text-anchor=\"middle\" x=\"451.5\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 12</text>\r\n",
       "<text text-anchor=\"middle\" x=\"451.5\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [3, 9]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"451.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 7&#45;&gt;8 -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>7&#45;&gt;8</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M461.63,-192.884C460.438,-184.778 459.144,-175.982 457.893,-167.472\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"461.315,-166.684 456.397,-157.299 454.389,-167.702 461.315,-166.684\"/>\r\n",
       "</g>\r\n",
       "<!-- 11 -->\r\n",
       "<g id=\"node12\" class=\"node\"><title>11</title>\r\n",
       "<polygon fill=\"#e58139\" fill-opacity=\"0.890196\" stroke=\"black\" points=\"727.5,-157 559.5,-157 559.5,-89 727.5,-89 727.5,-157\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"643.5\" y=\"-141.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst symmetry &lt;= 0.2682</text>\r\n",
       "<text text-anchor=\"middle\" x=\"643.5\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 20</text>\r\n",
       "<text text-anchor=\"middle\" x=\"643.5\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [18, 2]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"643.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 7&#45;&gt;11 -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>7&#45;&gt;11</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M523.967,-192.884C540.901,-183.125 559.57,-172.366 577.02,-162.31\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"579.102,-165.15 586.019,-157.125 575.607,-159.085 579.102,-165.15\"/>\r\n",
       "</g>\r\n",
       "<!-- 9 -->\r\n",
       "<g id=\"node10\" class=\"node\"><title>9</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.890196\" stroke=\"black\" points=\"378,-53 281,-53 281,-0 378,-0 378,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 10</text>\r\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 9]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"329.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 8&#45;&gt;9 -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>8&#45;&gt;9</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M408.83,-88.9485C396.548,-79.4346 383.172,-69.074 370.964,-59.6175\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"372.775,-56.5934 362.726,-53.2367 368.489,-62.1274 372.775,-56.5934\"/>\r\n",
       "</g>\r\n",
       "<!-- 10 -->\r\n",
       "<g id=\"node11\" class=\"node\"><title>10</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"509,-53 396,-53 396,-0 509,-0 509,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"452.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 2</text>\r\n",
       "<text text-anchor=\"middle\" x=\"452.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [2, 0]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"452.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 8&#45;&gt;10 -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>8&#45;&gt;10</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M451.85,-88.9485C451.937,-80.7153 452.031,-71.848 452.119,-63.4814\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"455.622,-63.2732 452.228,-53.2367 448.622,-63.1991 455.622,-63.2732\"/>\r\n",
       "</g>\r\n",
       "<!-- 12 -->\r\n",
       "<g id=\"node13\" class=\"node\"><title>12</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.498039\" stroke=\"black\" points=\"624,-53 527,-53 527,-0 624,-0 624,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"575.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 3</text>\r\n",
       "<text text-anchor=\"middle\" x=\"575.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [1, 2]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"575.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 11&#45;&gt;12 -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>11&#45;&gt;12</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M619.717,-88.9485C613.332,-80.0749 606.416,-70.4648 599.993,-61.5388\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"602.701,-59.3094 594.02,-53.2367 597.019,-63.398 602.701,-59.3094\"/>\r\n",
       "</g>\r\n",
       "<!-- 13 -->\r\n",
       "<g id=\"node14\" class=\"node\"><title>13</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"755,-53 642,-53 642,-0 755,-0 755,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 17</text>\r\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [17, 0]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"698.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 11&#45;&gt;13 -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>11&#45;&gt;13</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M662.736,-88.9485C667.794,-80.2579 673.264,-70.8608 678.367,-62.0917\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"681.516,-63.6401 683.521,-53.2367 675.466,-60.1189 681.516,-63.6401\"/>\r\n",
       "</g>\r\n",
       "<!-- 15 -->\r\n",
       "<g id=\"node16\" class=\"node\"><title>15</title>\r\n",
       "<polygon fill=\"#399de5\" stroke=\"black\" points=\"712,-253.5 615,-253.5 615,-200.5 712,-200.5 712,-253.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-238.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 5</text>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-223.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 5]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"663.5\" y=\"-208.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 14&#45;&gt;15 -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>14&#45;&gt;15</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M663.5,-296.884C663.5,-286.326 663.5,-274.597 663.5,-263.854\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"667,-263.52 663.5,-253.52 660,-263.52 667,-263.52\"/>\r\n",
       "</g>\r\n",
       "<!-- 16 -->\r\n",
       "<g id=\"node17\" class=\"node\"><title>16</title>\r\n",
       "<polygon fill=\"#e58139\" fill-opacity=\"0.976471\" stroke=\"black\" points=\"905,-261 738,-261 738,-193 905,-193 905,-261\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-245.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst concavity &lt;= 0.1907</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-230.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 137</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-215.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [134, 3]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 14&#45;&gt;16 -->\r\n",
       "<g id=\"edge16\" class=\"edge\"><title>14&#45;&gt;16</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M714.798,-296.884C729.776,-287.214 746.274,-276.563 761.728,-266.587\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"763.686,-269.489 770.189,-261.125 759.889,-263.608 763.686,-269.489\"/>\r\n",
       "</g>\r\n",
       "<!-- 17 -->\r\n",
       "<g id=\"node18\" class=\"node\"><title>17</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.333333\" stroke=\"black\" points=\"897,-157 746,-157 746,-89 897,-89 897,-157\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-141.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">worst texture &lt;= 30.975</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-126.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 5</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [2, 3]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 16&#45;&gt;17 -->\r\n",
       "<g id=\"edge17\" class=\"edge\"><title>16&#45;&gt;17</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M821.5,-192.884C821.5,-184.778 821.5,-175.982 821.5,-167.472\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"825,-167.299 821.5,-157.299 818,-167.299 825,-167.299\"/>\r\n",
       "</g>\r\n",
       "<!-- 20 -->\r\n",
       "<g id=\"node21\" class=\"node\"><title>20</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"1028,-149.5 915,-149.5 915,-96.5 1028,-96.5 1028,-149.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"971.5\" y=\"-134.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 132</text>\r\n",
       "<text text-anchor=\"middle\" x=\"971.5\" y=\"-119.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [132, 0]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"971.5\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 16&#45;&gt;20 -->\r\n",
       "<g id=\"edge20\" class=\"edge\"><title>16&#45;&gt;20</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M870.201,-192.884C887.991,-180.786 908.043,-167.151 925.583,-155.224\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"927.67,-158.037 933.971,-149.52 923.733,-152.249 927.67,-158.037\"/>\r\n",
       "</g>\r\n",
       "<!-- 18 -->\r\n",
       "<g id=\"node19\" class=\"node\"><title>18</title>\r\n",
       "<polygon fill=\"#399de5\" stroke=\"black\" points=\"870,-53 773,-53 773,-0 870,-0 870,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 3</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 3]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"821.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = benign</text>\r\n",
       "</g>\r\n",
       "<!-- 17&#45;&gt;18 -->\r\n",
       "<g id=\"edge18\" class=\"edge\"><title>17&#45;&gt;18</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M821.5,-88.9485C821.5,-80.7153 821.5,-71.848 821.5,-63.4814\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"825,-63.2367 821.5,-53.2367 818,-63.2367 825,-63.2367\"/>\r\n",
       "</g>\r\n",
       "<!-- 19 -->\r\n",
       "<g id=\"node20\" class=\"node\"><title>19</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"1001,-53 888,-53 888,-0 1001,-0 1001,-53\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"944.5\" y=\"-37.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 2</text>\r\n",
       "<text text-anchor=\"middle\" x=\"944.5\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [2, 0]</text>\r\n",
       "<text text-anchor=\"middle\" x=\"944.5\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">class = malignant</text>\r\n",
       "</g>\r\n",
       "<!-- 17&#45;&gt;19 -->\r\n",
       "<g id=\"edge19\" class=\"edge\"><title>17&#45;&gt;19</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M864.519,-88.9485C877.022,-79.3431 890.647,-68.8747 903.051,-59.345\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"905.204,-62.1046 911.001,-53.2367 900.939,-56.5537 905.204,-62.1046\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x16581530da0>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import graphviz\n",
    "with open(\"mytree.dot\") as f:\n",
    "    dot_graph = f.read()\n",
    "graphviz.Source(dot_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.01019737,  0.04839825,  0.        ,  0.        ,  0.0024156 ,\n",
       "        0.        ,  0.        ,  0.        ,  0.        ,  0.        ,\n",
       "        0.72682851,  0.0458159 ,  0.        ,  0.        ,  0.0141577 ,\n",
       "        0.        ,  0.018188  ,  0.1221132 ,  0.01188548,  0.        ])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(tree.feature_importances_, 'o')\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "plt.ylim={0,1}\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature importances: [ 0.  1.]\n"
     ]
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: Tree Pages: 1 -->\r\n",
       "<svg width=\"283pt\" height=\"224pt\"\r\n",
       " viewBox=\"0.00 0.00 283.00 224.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 220)\">\r\n",
       "<title>Tree</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-220 279,-220 279,4 -4,4\"/>\r\n",
       "<!-- 0 -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>0</title>\r\n",
       "<polygon fill=\"none\" stroke=\"black\" points=\"162,-216 54,-216 54,-163 162,-163 162,-216\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"108\" y=\"-200.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[1] &lt;= &#45;5.8141</text>\r\n",
       "<text text-anchor=\"middle\" x=\"108\" y=\"-185.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 100</text>\r\n",
       "<text text-anchor=\"middle\" x=\"108\" y=\"-170.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [50, 50]</text>\r\n",
       "</g>\r\n",
       "<!-- 1 -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>1</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"98,-119.5 0,-119.5 0,-81.5 98,-81.5 98,-119.5\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"49\" y=\"-104.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 25</text>\r\n",
       "<text text-anchor=\"middle\" x=\"49\" y=\"-89.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [25, 0]</text>\r\n",
       "</g>\r\n",
       "<!-- 0&#45;&gt;1 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>0&#45;&gt;1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M90.6172,-162.868C83.1609,-151.873 74.4545,-139.035 66.9379,-127.951\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"69.7209,-125.819 61.2114,-119.507 63.9274,-129.747 69.7209,-125.819\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"56.5027\" y=\"-140.359\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">True</text>\r\n",
       "</g>\r\n",
       "<!-- 2 -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>2</title>\r\n",
       "<polygon fill=\"#399de5\" fill-opacity=\"0.498039\" stroke=\"black\" points=\"220,-127 116,-127 116,-74 220,-74 220,-127\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-111.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">X[1] &lt;= 5.3475</text>\r\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 75</text>\r\n",
       "<text text-anchor=\"middle\" x=\"168\" y=\"-81.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [25, 50]</text>\r\n",
       "</g>\r\n",
       "<!-- 0&#45;&gt;2 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>0&#45;&gt;2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M125.677,-162.868C131.545,-154.359 138.174,-144.747 144.415,-135.698\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"147.445,-137.469 150.242,-127.25 141.683,-133.495 147.445,-137.469\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"154.758\" y=\"-148.138\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">False</text>\r\n",
       "</g>\r\n",
       "<!-- 3 -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>3</title>\r\n",
       "<polygon fill=\"#399de5\" stroke=\"black\" points=\"159,-38 61,-38 61,-0 159,-0 159,-38\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"110\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 50</text>\r\n",
       "<text text-anchor=\"middle\" x=\"110\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [0, 50]</text>\r\n",
       "</g>\r\n",
       "<!-- 2&#45;&gt;3 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>2&#45;&gt;3</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M149.329,-73.9081C142.905,-65.1022 135.702,-55.2299 129.234,-46.3636\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"132.028,-44.2546 123.306,-38.2389 126.373,-48.3803 132.028,-44.2546\"/>\r\n",
       "</g>\r\n",
       "<!-- 4 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>4</title>\r\n",
       "<polygon fill=\"#e58139\" stroke=\"black\" points=\"275,-38 177,-38 177,-0 275,-0 275,-38\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"226\" y=\"-22.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">samples = 25</text>\r\n",
       "<text text-anchor=\"middle\" x=\"226\" y=\"-7.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">value = [25, 0]</text>\r\n",
       "</g>\r\n",
       "<!-- 2&#45;&gt;4 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>2&#45;&gt;4</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M186.671,-73.9081C193.095,-65.1022 200.298,-55.2299 206.766,-46.3636\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"209.627,-48.3803 212.694,-38.2389 203.972,-44.2546 209.627,-48.3803\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.files.Source at 0x16582fbc828>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree = mglearn.plots.plot_tree_not_monotone()\n",
    "plt.suptitle(\"tree_not_monotone\")\n",
    "tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=5, n_jobs=1, oob_score=False, random_state=2,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42)\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators=5, random_state=2)\n",
    "forest.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 3, figsize=(20, 10))\n",
    "for i, (ax, tree) in enumerate(zip(axes.ravel(), forest.estimators_)):\n",
    "    ax.set_title(\"tree %d\" % i)\n",
    "    mglearn.plots.plot_tree_partition(X_train, y_train, tree, ax=ax)\n",
    "mglearn.plots.plot_2d_separator(forest, X_train, fill=True, ax=axes[-1,-1], alpha=.4)\n",
    "axes[-1,-1].set_title(\"random forest\")\n",
    "plt.scatter(X_train[:, 0], X_train[:, 1], c=np.array(['r', 'b'])[y_train], s=60)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 1.000000\n",
      "accuracy on test set: 0.972028\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "forest = RandomForestClassifier(n_estimators=100, random_state=0)\n",
    "forest.fit(X_train, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % forest.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % forest.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(forest.feature_importances_, 'o')\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 1.000000\n",
      "accuracy on test set: 0.958042\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "gbrt = GradientBoostingClassifier(random_state=0)\n",
    "gbrt.fit(X_train, y_train)\n",
    "print(\"accuracy on training set: %f\" % gbrt.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % gbrt.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.990610\n",
      "accuracy on test set: 0.972028\n"
     ]
    }
   ],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, max_depth=1)\n",
    "gbrt.fit(X_train, y_train)\n",
    "print(\"accuracy on training set: %f\" % gbrt.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % gbrt.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.988263\n",
      "accuracy on test set: 0.965035\n"
     ]
    }
   ],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, learning_rate=0.01)\n",
    "gbrt.fit(X_train, y_train)\n",
    "print(\"accuracy on training set: %f\" % gbrt.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % gbrt.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=0, max_depth=1)\n",
    "gbrt.fit(X_train, y_train)\n",
    "plt.plot(gbrt.feature_importances_, 'o')\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = make_blobs(centers=4, random_state=8)\n",
    "y = y % 2\n",
    "\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=60, cmap=mglearn.cm2)\n",
    "plt.xlabel(\"feature1\")\n",
    "plt.ylabel(\"feature2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "linear_svm = LinearSVC().fit(X, y)\n",
    "\n",
    "mglearn.plots.plot_2d_separator(linear_svm, X)\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=60, cmap=mglearn.cm2)\n",
    "plt.xlabel(\"feature1\")\n",
    "plt.ylabel(\"feature2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_new = np.hstack([X, X[:, 1:] ** 2])\n",
    "\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D, axes3d\n",
    "figure = plt.figure()\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "ax.scatter(X_new[:, 0], X_new[:, 1], X_new[:, 2], c=y, cmap=mglearn.cm2, s=60)\n",
    "ax.set_xlabel(\"feature1\")\n",
    "ax.set_ylabel(\"feature2\")\n",
    "ax.set_zlabel(\"feature1 ** 2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "linear_svm_3d = LinearSVC().fit(X_new, y)\n",
    "coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_\n",
    "\n",
    "figure = plt.figure()\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "xx = np.linspace(X_new[:, 0].min(), X_new[:, 0].max(), 50)\n",
    "yy = np.linspace(X_new[:, 1].min(), X_new[:, 1].max(), 50)\n",
    "\n",
    "xx, yy = np.meshgrid(xx, yy)\n",
    "zz = (coef[0] * xx + coef[1] * yy + intercept) / -coef[2]\n",
    "ax.scatter(X_new[:,0], X_new[:, 1], X_new[:,2], c=y, cmap=mglearn.cm2, s=60)\n",
    "ax.plot_surface(xx, yy,zz, rstride=8, cstride=8, alpha=0.3)\n",
    "\n",
    "ax.set_xlabel(\"feature1\")\n",
    "ax.set_ylabel(\"feature2\")\n",
    "ax.set_zlabel(\"feature ** 2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "zz = yy ** 2\n",
    "dec = linear_svm_3d.decision_function(np.c_[xx.ravel(), yy.ravel(), zz.ravel()])\n",
    "plt.contourf(xx, yy, dec.reshape(xx.shape), levels=[dec.min(), 0, dec.max()],\n",
    "             cmap=mglearn.cm2, alpha=0.5)\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, s=60, cmap=mglearn.cm2)\n",
    "plt.xlabel(\"feature1\")\n",
    "plt.ylabel(\"feature2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x1658244b080>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "X, y = mglearn.tools.make_handcrafted_dataset()\n",
    "svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X, y)\n",
    "mglearn.plots.plot_2d_separator(svm, X, eps=.5)\n",
    "plt.scatter(X[:, 0], X[:, 1], s=60, c=y, cmap=mglearn.cm2)\n",
    "sv = svm.support_vectors_\n",
    "plt.scatter(sv[:, 0], sv[:, 1], s=200, facecolors='none', zorder=10, linewidth=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3,3, figsize=(15, 10))\n",
    "\n",
    "for ax, C in zip(axes, [-1, 0, 3]):\n",
    "    for a, gamma in zip(ax, range(-1, 2)):\n",
    "        mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 1.000000\n",
      "accuracy on test set: 0.629371\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "svc = SVC()\n",
    "svc.fit(X_train, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % svc.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % svc.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(X_train.min(axis=0), 'o', label=\"min\")\n",
    "plt.plot(X_train.max(axis=0), 'o', label=\"max\")\n",
    "plt.legend(loc=\"best\")\n",
    "plt.yscale(\"log\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum for each feature\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "Maximum for each feature\n",
      " [ 1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.]\n"
     ]
    }
   ],
   "source": [
    "min_on_training = X_train.min(axis=0)\n",
    "range_on_training = (X_train - min_on_training).max(axis=0)\n",
    "X_train_scaled = (X_train - min_on_training) / range_on_training\n",
    "print(\"Minimum for each feature\\n%s\" % X_train_scaled.min(axis=0))\n",
    "print(\"Maximum for each feature\\n %s\" % X_train_scaled.max(axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_scaled = (X_test - min_on_training) / range_on_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.948357\n",
      "accuracy on test set: 0.951049\n"
     ]
    }
   ],
   "source": [
    "svc = SVC()\n",
    "svc.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % svc.score(X_train_scaled, y_train))\n",
    "print(\"accuracy on test set: %f\" % svc.score(X_test_scaled, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.988263\n",
      "accuracy on test set: 0.972028\n"
     ]
    }
   ],
   "source": [
    "svc = SVC(C=1000)\n",
    "svc.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % svc.score(X_train_scaled, y_train))\n",
    "print(\"accuracy on test set: %f\" % svc.score(X_test_scaled, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: %3 Pages: 1 -->\r\n",
       "<svg width=\"174pt\" height=\"261pt\"\r\n",
       " viewBox=\"0.00 0.00 174.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\r\n",
       "<title>%3</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-257 170,-257 170,4 -4,4\"/>\r\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_2</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"106,-89 106,-164 159,-164 159,-89 106,-89\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"132.5\" y=\"-148.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">output</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"8,-8 8,-245 60,-245 60,-8 8,-8\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-229.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">inputs</text>\r\n",
       "</g>\r\n",
       "<!-- x[0] -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>x[0]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-196\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-192.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[0]</text>\r\n",
       "</g>\r\n",
       "<!-- y -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>y</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"132\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"132\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">y</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;y -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>x[0]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.3484,-184.737C64.3252,-171.257 91.2691,-148.523 110.034,-132.69\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"112.379,-135.291 117.765,-126.167 107.865,-129.941 112.379,-135.291\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-172.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w[0]</text>\r\n",
       "</g>\r\n",
       "<!-- x[1] -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>x[1]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-142\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-138.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;y -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>x[1]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.7219,-137.297C66.3136,-133.193 87.7357,-127.168 104.634,-122.415\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"105.791,-125.726 114.47,-119.649 103.896,-118.987 105.791,-125.726\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-135.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[2] -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>x[2]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-88\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-84.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;y -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>x[2]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.7219,-92.703C66.3136,-96.807 87.7357,-102.832 104.634,-107.585\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.896,-111.013 114.47,-110.351 105.791,-104.274 103.896,-111.013\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-108.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[3] -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>x[3]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-34\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-30.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[3]</text>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;y -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>x[3]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.3484,-45.2627C64.3252,-58.7431 91.2691,-81.477 110.034,-97.3098\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.865,-100.059 117.765,-103.833 112.379,-94.7092 107.865,-100.059\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"83\" y=\"-83.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w[3]</text>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x16581958ef0>"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mglearn.plots.plot_logistic_regression_graph()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: %3 Pages: 1 -->\r\n",
       "<svg width=\"252pt\" height=\"261pt\"\r\n",
       " viewBox=\"0.00 0.00 252.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\r\n",
       "<title>%3</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-257 248,-257 248,4 -4,4\"/>\r\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"8,-8 8,-245 60,-245 60,-8 8,-8\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-229.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">inputs</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_1</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"80,-35 80,-218 164,-218 164,-35 80,-35\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"122\" y=\"-202.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">hidden layer</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_2</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"184,-89 184,-164 237,-164 237,-89 184,-89\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"210.5\" y=\"-148.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">output</text>\r\n",
       "</g>\r\n",
       "<!-- x[0] -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>x[0]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-196\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-192.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[0]</text>\r\n",
       "</g>\r\n",
       "<!-- h0 -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>h0</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"122\" cy=\"-61\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"122\" y=\"-57.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h[0]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h0 -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>x[0]&#45;&gt;h0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.5321,-181.037C59.3841,-157.723 88.1768,-112.525 105.882,-84.7317\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"109.004,-86.3447 111.425,-76.0302 103.1,-82.5838 109.004,-86.3447\"/>\r\n",
       "</g>\r\n",
       "<!-- h1 -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>h1</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"122\" cy=\"-169\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"122\" y=\"-165.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h1 -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>x[0]&#45;&gt;h1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-190.794C63.836,-186.947 80.7639,-181.632 94.8355,-177.214\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"95.9768,-180.525 104.469,-174.19 93.88,-173.846 95.9768,-180.525\"/>\r\n",
       "</g>\r\n",
       "<!-- h2 -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>h2</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"122\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"122\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h2 -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>x[0]&#45;&gt;h2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47.6896,-184.048C61.6628,-170.887 84.3244,-149.543 100.757,-134.066\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.379,-136.405 108.258,-127.001 98.5791,-131.309 103.379,-136.405\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1] -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>x[1]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-142\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-138.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h0 -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>x[1]&#45;&gt;h0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47.6896,-130.048C61.6628,-116.887 84.3244,-95.5433 100.757,-80.066\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.379,-82.405 108.258,-73.0009 98.5791,-77.3093 103.379,-82.405\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h1 -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>x[1]&#45;&gt;h1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-147.206C63.836,-151.053 80.7639,-156.368 94.8355,-160.786\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.88,-164.154 104.469,-163.81 95.9768,-157.475 93.88,-164.154\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h2 -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>x[1]&#45;&gt;h2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-136.794C63.836,-132.947 80.7639,-127.632 94.8355,-123.214\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"95.9768,-126.525 104.469,-120.19 93.88,-119.846 95.9768,-126.525\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2] -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>x[2]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-88\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-84.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h0 -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>x[2]&#45;&gt;h0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-82.7941C63.836,-78.9468 80.7639,-73.6323 94.8355,-69.2144\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"95.9768,-72.5246 104.469,-66.1899 93.88,-65.846 95.9768,-72.5246\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h1 -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>x[2]&#45;&gt;h1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47.6896,-99.9518C61.6628,-113.113 84.3244,-134.457 100.757,-149.934\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.5791,-152.691 108.258,-156.999 103.379,-147.595 98.5791,-152.691\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h2 -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>x[2]&#45;&gt;h2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-93.2059C63.836,-97.0532 80.7639,-102.368 94.8355,-106.786\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.88,-110.154 104.469,-109.81 95.9768,-103.475 93.88,-110.154\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3] -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>x[3]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-34\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-30.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[3]</text>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h0 -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>x[3]&#45;&gt;h0</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.5817,-39.2059C63.836,-43.0532 80.7639,-48.3677 94.8355,-52.7856\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.88,-56.154 104.469,-55.8101 95.9768,-49.4754 93.88,-56.154\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h1 -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>x[3]&#45;&gt;h1</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.5321,-48.9632C59.3841,-72.2774 88.1768,-117.475 105.882,-145.268\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.1,-147.416 111.425,-153.97 109.004,-143.655 103.1,-147.416\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h2 -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>x[3]&#45;&gt;h2</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M47.6896,-45.9518C61.6628,-59.1126 84.3244,-80.4567 100.757,-95.934\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.5791,-98.6907 108.258,-102.999 103.379,-93.595 98.5791,-98.6907\"/>\r\n",
       "</g>\r\n",
       "<!-- y -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>y</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"210\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"210\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">y</text>\r\n",
       "</g>\r\n",
       "<!-- h0&#45;&gt;y -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>h0&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M137.589,-70.1604C150.685,-78.3836 170.145,-90.6029 185.445,-100.21\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"184.01,-103.441 194.34,-105.795 187.732,-97.5129 184.01,-103.441\"/>\r\n",
       "</g>\r\n",
       "<!-- h1&#45;&gt;y -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>h1&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M137.589,-159.84C150.685,-151.616 170.145,-139.397 185.445,-129.79\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"187.732,-132.487 194.34,-124.205 184.01,-126.559 187.732,-132.487\"/>\r\n",
       "</g>\r\n",
       "<!-- h2&#45;&gt;y -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>h2&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M140.403,-115C152.254,-115 168.183,-115 181.708,-115\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"181.916,-118.5 191.916,-115 181.916,-111.5 181.916,-118.5\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x16581958748>"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mglearn.plots.plot_single_hidden_layer_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "line = np.linspace(-3, 3, 100)\n",
    "plt.plot(line, np.tanh(line), label=\"tanh\")\n",
    "plt.plot(line, np.maximum(line, 0), label=\"relu\")\n",
    "plt.legend(loc=\"best\")\n",
    "plt.title(\"activation_functions\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\r\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\r\n",
       " -->\r\n",
       "<!-- Title: %3 Pages: 1 -->\r\n",
       "<svg width=\"378pt\" height=\"261pt\"\r\n",
       " viewBox=\"0.00 0.00 378.00 261.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 257)\">\r\n",
       "<title>%3</title>\r\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-257 374,-257 374,4 -4,4\"/>\r\n",
       "<g id=\"clust4\" class=\"cluster\"><title>cluster_3</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"310,-89 310,-164 363,-164 363,-89 310,-89\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"336.5\" y=\"-148.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">output</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust3\" class=\"cluster\"><title>cluster_2</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"195,-35 195,-218 290,-218 290,-35 195,-35\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"242.5\" y=\"-202.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">hidden layer 2</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust1\" class=\"cluster\"><title>cluster_0</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"8,-8 8,-245 60,-245 60,-8 8,-8\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-229.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">inputs</text>\r\n",
       "</g>\r\n",
       "<g id=\"clust2\" class=\"cluster\"><title>cluster_1</title>\r\n",
       "<polygon fill=\"none\" stroke=\"white\" points=\"80,-35 80,-218 175,-218 175,-35 80,-35\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"127.5\" y=\"-202.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">hidden layer 1</text>\r\n",
       "</g>\r\n",
       "<!-- x[0] -->\r\n",
       "<g id=\"node1\" class=\"node\"><title>x[0]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-196\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-192.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[0]</text>\r\n",
       "</g>\r\n",
       "<!-- h1[0] -->\r\n",
       "<g id=\"node5\" class=\"node\"><title>h1[0]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"127\" cy=\"-61\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"127\" y=\"-57.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h1[0]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h1[0] -->\r\n",
       "<g id=\"edge1\" class=\"edge\"><title>x[0]&#45;&gt;h1[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.7386,-181.553C60.4603,-158.229 91.5706,-112.077 110.406,-84.1334\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"113.368,-86.0011 116.056,-75.7527 107.564,-82.0885 113.368,-86.0011\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[1] -->\r\n",
       "<g id=\"node6\" class=\"node\"><title>h1[1]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"127\" cy=\"-169\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"127\" y=\"-165.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h1[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h1[1] -->\r\n",
       "<g id=\"edge2\" class=\"edge\"><title>x[0]&#45;&gt;h1[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-191.048C65.0315,-187.09 83.9854,-181.466 99.4114,-176.889\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"100.824,-180.121 109.415,-173.921 98.8325,-173.41 100.824,-180.121\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[2] -->\r\n",
       "<g id=\"node7\" class=\"node\"><title>h1[2]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"127\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"127\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h1[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[0]&#45;&gt;h1[2] -->\r\n",
       "<g id=\"edge3\" class=\"edge\"><title>x[0]&#45;&gt;h1[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.0381,-184.395C63.0396,-171.042 87.8647,-148.945 105.468,-133.276\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.844,-135.847 112.986,-126.584 103.189,-130.618 107.844,-135.847\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1] -->\r\n",
       "<g id=\"node2\" class=\"node\"><title>x[1]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-142\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-138.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[1]</text>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h1[0] -->\r\n",
       "<g id=\"edge4\" class=\"edge\"><title>x[1]&#45;&gt;h1[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.0381,-130.395C63.0396,-117.042 87.8647,-94.9446 105.468,-79.276\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.844,-81.8469 112.986,-72.5838 103.189,-76.6182 107.844,-81.8469\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h1[1] -->\r\n",
       "<g id=\"edge5\" class=\"edge\"><title>x[1]&#45;&gt;h1[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-146.952C65.0315,-150.91 83.9854,-156.534 99.4114,-161.111\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.8325,-164.59 109.415,-164.079 100.824,-157.879 98.8325,-164.59\"/>\r\n",
       "</g>\r\n",
       "<!-- x[1]&#45;&gt;h1[2] -->\r\n",
       "<g id=\"edge6\" class=\"edge\"><title>x[1]&#45;&gt;h1[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-137.048C65.0315,-133.09 83.9854,-127.466 99.4114,-122.889\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"100.824,-126.121 109.415,-119.921 98.8325,-119.41 100.824,-126.121\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2] -->\r\n",
       "<g id=\"node3\" class=\"node\"><title>x[2]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-88\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-84.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[2]</text>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h1[0] -->\r\n",
       "<g id=\"edge7\" class=\"edge\"><title>x[2]&#45;&gt;h1[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-83.0478C65.0315,-79.0896 83.9854,-73.4659 99.4114,-68.8889\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"100.824,-72.1208 109.415,-65.9208 98.8325,-65.4099 100.824,-72.1208\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h1[1] -->\r\n",
       "<g id=\"edge8\" class=\"edge\"><title>x[2]&#45;&gt;h1[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.0381,-99.6053C63.0396,-112.958 87.8647,-135.055 105.468,-150.724\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.189,-153.382 112.986,-157.416 107.844,-148.153 103.189,-153.382\"/>\r\n",
       "</g>\r\n",
       "<!-- x[2]&#45;&gt;h1[2] -->\r\n",
       "<g id=\"edge9\" class=\"edge\"><title>x[2]&#45;&gt;h1[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-92.9522C65.0315,-96.9104 83.9854,-102.534 99.4114,-107.111\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.8325,-110.59 109.415,-110.079 100.824,-103.879 98.8325,-110.59\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3] -->\r\n",
       "<g id=\"node4\" class=\"node\"><title>x[3]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"34\" cy=\"-34\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"34\" y=\"-30.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x[3]</text>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h1[0] -->\r\n",
       "<g id=\"edge10\" class=\"edge\"><title>x[3]&#45;&gt;h1[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M51.6908,-38.9522C65.0315,-42.9104 83.9854,-48.5341 99.4114,-53.1111\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"98.8325,-56.5901 109.415,-56.0792 100.824,-49.8792 98.8325,-56.5901\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h1[1] -->\r\n",
       "<g id=\"edge11\" class=\"edge\"><title>x[3]&#45;&gt;h1[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M44.7386,-48.4474C60.4603,-71.7708 91.5706,-117.923 110.406,-145.867\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"107.564,-147.912 116.056,-154.247 113.368,-143.999 107.564,-147.912\"/>\r\n",
       "</g>\r\n",
       "<!-- x[3]&#45;&gt;h1[2] -->\r\n",
       "<g id=\"edge12\" class=\"edge\"><title>x[3]&#45;&gt;h1[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M48.0381,-45.6053C63.0396,-58.9584 87.8647,-81.0554 105.468,-96.724\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"103.189,-99.3818 112.986,-103.416 107.844,-94.1531 103.189,-99.3818\"/>\r\n",
       "</g>\r\n",
       "<!-- h2[0] -->\r\n",
       "<g id=\"node8\" class=\"node\"><title>h2[0]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"242\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"242\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h2[0]</text>\r\n",
       "</g>\r\n",
       "<!-- h1[0]&#45;&gt;h2[0] -->\r\n",
       "<g id=\"edge13\" class=\"edge\"><title>h1[0]&#45;&gt;h2[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.712,-68.5085C162.518,-77.4954 194.233,-92.6514 216.321,-103.207\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"214.889,-106.401 225.421,-107.555 217.908,-100.086 214.889,-106.401\"/>\r\n",
       "</g>\r\n",
       "<!-- h2[1] -->\r\n",
       "<g id=\"node9\" class=\"node\"><title>h2[1]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"242\" cy=\"-61\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"242\" y=\"-57.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h2[1]</text>\r\n",
       "</g>\r\n",
       "<!-- h1[0]&#45;&gt;h2[1] -->\r\n",
       "<g id=\"edge14\" class=\"edge\"><title>h1[0]&#45;&gt;h2[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145.162,-61C163.331,-61 192.275,-61 213.606,-61\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"213.676,-64.5001 223.676,-61 213.676,-57.5001 213.676,-64.5001\"/>\r\n",
       "</g>\r\n",
       "<!-- h2[2] -->\r\n",
       "<g id=\"node10\" class=\"node\"><title>h2[2]</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"242\" cy=\"-169\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"242\" y=\"-165.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">h2[2]</text>\r\n",
       "</g>\r\n",
       "<!-- h1[0]&#45;&gt;h2[2] -->\r\n",
       "<g id=\"edge15\" class=\"edge\"><title>h1[0]&#45;&gt;h2[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M140.525,-72.9705C160.04,-91.6219 197.872,-127.78 221.136,-150.015\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"218.772,-152.597 228.419,-156.976 223.608,-147.536 218.772,-152.597\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[1]&#45;&gt;h2[0] -->\r\n",
       "<g id=\"edge16\" class=\"edge\"><title>h1[1]&#45;&gt;h2[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.712,-161.492C162.518,-152.505 194.233,-137.349 216.321,-126.793\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.908,-129.914 225.421,-122.445 214.889,-123.599 217.908,-129.914\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[1]&#45;&gt;h2[1] -->\r\n",
       "<g id=\"edge17\" class=\"edge\"><title>h1[1]&#45;&gt;h2[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M140.525,-157.029C160.04,-138.378 197.872,-102.22 221.136,-79.9854\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"223.608,-82.4637 228.419,-73.0242 218.772,-77.4033 223.608,-82.4637\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[1]&#45;&gt;h2[2] -->\r\n",
       "<g id=\"edge18\" class=\"edge\"><title>h1[1]&#45;&gt;h2[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145.162,-169C163.331,-169 192.275,-169 213.606,-169\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"213.676,-172.5 223.676,-169 213.676,-165.5 213.676,-172.5\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[2]&#45;&gt;h2[0] -->\r\n",
       "<g id=\"edge19\" class=\"edge\"><title>h1[2]&#45;&gt;h2[0]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M145.162,-115C163.331,-115 192.275,-115 213.606,-115\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"213.676,-118.5 223.676,-115 213.676,-111.5 213.676,-118.5\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[2]&#45;&gt;h2[1] -->\r\n",
       "<g id=\"edge20\" class=\"edge\"><title>h1[2]&#45;&gt;h2[1]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.712,-107.492C162.518,-98.5046 194.233,-83.3486 216.321,-72.7935\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"217.908,-75.9144 225.421,-68.4447 214.889,-69.5985 217.908,-75.9144\"/>\r\n",
       "</g>\r\n",
       "<!-- h1[2]&#45;&gt;h2[2] -->\r\n",
       "<g id=\"edge21\" class=\"edge\"><title>h1[2]&#45;&gt;h2[2]</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M143.712,-122.508C162.518,-131.495 194.233,-146.651 216.321,-157.207\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"214.889,-160.401 225.421,-161.555 217.908,-154.086 214.889,-160.401\"/>\r\n",
       "</g>\r\n",
       "<!-- y -->\r\n",
       "<g id=\"node11\" class=\"node\"><title>y</title>\r\n",
       "<ellipse fill=\"none\" stroke=\"black\" cx=\"336\" cy=\"-115\" rx=\"18\" ry=\"18\"/>\r\n",
       "<text text-anchor=\"middle\" x=\"336\" y=\"-111.3\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">y</text>\r\n",
       "</g>\r\n",
       "<!-- h2[0]&#45;&gt;y -->\r\n",
       "<g id=\"edge22\" class=\"edge\"><title>h2[0]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M260.304,-115C273.54,-115 292.053,-115 307.337,-115\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"307.773,-118.5 317.773,-115 307.773,-111.5 307.773,-118.5\"/>\r\n",
       "</g>\r\n",
       "<!-- h2[1]&#45;&gt;y -->\r\n",
       "<g id=\"edge23\" class=\"edge\"><title>h2[1]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M257.781,-69.676C272.181,-78.1282 294.309,-91.1164 311.192,-101.026\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"309.554,-104.123 319.95,-106.166 313.097,-98.0856 309.554,-104.123\"/>\r\n",
       "</g>\r\n",
       "<!-- h2[2]&#45;&gt;y -->\r\n",
       "<g id=\"edge24\" class=\"edge\"><title>h2[2]&#45;&gt;y</title>\r\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M257.781,-160.324C272.181,-151.872 294.309,-138.884 311.192,-128.974\"/>\r\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"313.097,-131.914 319.95,-123.834 309.554,-125.877 313.097,-131.914\"/>\r\n",
       "</g>\r\n",
       "</g>\r\n",
       "</svg>\r\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x165828853c8>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mglearn.plots.plot_two_hidden_layer_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X, y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=42)\n",
    "\n",
    "mlp = MLPClassifier(algorithm='l-bfgs', random_state=0).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "plt.scatter(X_train[:,0], X_train[:,1], c=y_train, s=69, cmap=mglearn.cm2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(algorithm='l-bfgs', random_state=0, hidden_layer_sizes=[10]).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "plt.scatter(X_train[:,0], X_train[:,1], c=y_train, s=69, cmap=mglearn.cm2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(algorithm='l-bfgs', random_state=0, hidden_layer_sizes=[10, 10]).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "plt.scatter(X_train[:,0], X_train[:,1], c=y_train, s=69, cmap=mglearn.cm2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(algorithm='l-bfgs', activation='tanh',random_state=0, hidden_layer_sizes=[10, 10]).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "plt.scatter(X_train[:,0], X_train[:,1], c=y_train, s=69, cmap=mglearn.cm2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for ax, n_hidden_nodes in zip(axes, [10, 100]):\n",
    "    for axx, alpha in zip(ax, [0.0001, 0.01, 0.1, 1]):\n",
    "        mlp = MLPClassifier(algorithm='l-bfgs', random_state=0,\n",
    "                            hidden_layer_sizes=[n_hidden_nodes, n_hidden_nodes],\n",
    "                            alpha=alpha)\n",
    "        mlp.fit(X_train, y_train)\n",
    "        mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=axx)\n",
    "        axx.scatter(X_train[:, 0], X_train[:, 1], c=y_train, s=60, cmap=mglearn.cm2)\n",
    "        axx.set_title(\"n_hidden=[%d, %d]\\nalpha=%.4f\"\n",
    "                      % (n_hidden_nodes, n_hidden_nodes, alpha))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for i, ax in enumerate(axes.ravel()):\n",
    "    mlp = MLPClassifier(algorithm='l-bfgs', random_state=i,\n",
    "                        hidden_layer_sizes=[100, 100])\n",
    "    mlp.fit(X_train, y_train)\n",
    "    mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=ax)\n",
    "    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, s=60, cmap=mglearn.cm2)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.917840\n",
      "accuracy on test set: 0.909091\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    cancer.data, cancer.target, random_state=0)\n",
    "\n",
    "mlp = MLPClassifier()\n",
    "mlp.fit(X_train, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % mlp.score(X_train, y_train))\n",
    "print(\"accuracy on test set: %f\" % mlp.score(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.990610\n",
      "accuracy on test set: 0.965035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\tools\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:560: ConvergenceWarning: Stochastic Optimizer: Maximum iterations reached and the optimization hasn't converged yet.\n",
      "  % (), ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "mean_on_train = X_train.mean(axis=0)\n",
    "\n",
    "std_on_train = X_train.std(axis=0)\n",
    "\n",
    "\n",
    "X_train_scaled = (X_train - mean_on_train) / std_on_train\n",
    "X_test_scaled = (X_test - mean_on_train) / std_on_train\n",
    "\n",
    "mlp = MLPClassifier(random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % mlp.score(X_train_scaled, y_train))\n",
    "print(\"accuracy on test set: %f\" % mlp.score(X_test_scaled, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.995305\n",
      "accuracy on test set: 0.965035\n"
     ]
    }
   ],
   "source": [
    "mlp = MLPClassifier(max_iter=1000, random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % mlp.score(X_train_scaled, y_train))\n",
    "print(\"accuracy on test set: %f\" % mlp.score(X_test_scaled, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy on training set: 0.988263\n",
      "accuracy on test set: 0.972028\n"
     ]
    }
   ],
   "source": [
    "mlp = MLPClassifier(max_iter=1000, alpha=1, random_state=0)\n",
    "mlp.fit(X_train_scaled, y_train)\n",
    "\n",
    "print(\"accuracy on training set: %f\" % mlp.score(X_train_scaled, y_train))\n",
    "print(\"accuracy on test set: %f\" % mlp.score(X_test_scaled, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20, 5))\n",
    "plt.imshow(mlp.coefs_[0], interpolation='none', cmap='viridis')\n",
    "plt.yticks(range(30), cancer.feature_names)\n",
    "plt.colorbar()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "              min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=0,\n",
       "              subsample=1.0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.datasets import make_blobs, make_circles\n",
    "X, y = make_circles(noise=0.25, factor=0.5, random_state=1)\n",
    "\n",
    "y_named = np.array([\"blue\", \"red\"])[y]\n",
    "\n",
    "X_train, X_test, y_train_named, y_test_named, y_train, y_test = \\\n",
    "    train_test_split(X, y_named, y, random_state=0)\n",
    "\n",
    "gbrt = GradientBoostingClassifier(random_state=0)\n",
    "gbrt.fit(X_train, y_train_named)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25, 2)\n",
      "(25,)\n"
     ]
    }
   ],
   "source": [
    "print(X_test.shape)\n",
    "print(gbrt.decision_function(X_test).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 4.13592629, -1.68343075, -3.95106099, -3.6261613 ,  4.28986668,\n",
       "        3.66166106])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbrt.decision_function(X_test)[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ True False False False  True  True False  True  True  True False  True\n",
      "  True False  True False False False  True  True  True  True  True False\n",
      " False]\n",
      "['red' 'blue' 'blue' 'blue' 'red' 'red' 'blue' 'red' 'red' 'red' 'blue'\n",
      " 'red' 'red' 'blue' 'red' 'blue' 'blue' 'blue' 'red' 'red' 'red' 'red'\n",
      " 'red' 'blue' 'blue']\n"
     ]
    }
   ],
   "source": [
    "print(gbrt.decision_function(X_test) > 0)\n",
    "print(gbrt.predict(X_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "greater_zero = (gbrt.decision_function(X_test) > 0).astype(int)\n",
    "pred = gbrt.classes_[greater_zero]\n",
    "np.all(pred == gbrt.predict(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-7.6909717730121798, 4.289866676868515)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decision_function = gbrt.decision_function(X_test)\n",
    "np.min(decision_function), np.max(decision_function)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "\n",
    "mglearn.tools.plot_2d_separator(gbrt, X, ax=axes[0], alpha=.4, fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(gbrt, X, ax=axes[1], alpha=.4, cm='bwr')\n",
    "\n",
    "for ax in axes:\n",
    "    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=mglearn.cm2, s=60, marker='^')\n",
    "    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=mglearn.cm2, s=60)\n",
    "plt.colorbar(scores_image, ax=axes.tolist())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25, 2)"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbrt.predict_proba(X_test).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.016,  0.984],\n",
       "       [ 0.843,  0.157],\n",
       "       [ 0.981,  0.019],\n",
       "       [ 0.974,  0.026],\n",
       "       [ 0.014,  0.986],\n",
       "       [ 0.025,  0.975]])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True, precision=3)\n",
    "gbrt.predict_proba(X_test[:6])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "\n",
    "mglearn.tools.plot_2d_separator(gbrt, X, ax=axes[0], alpha=.4,\n",
    "                                fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(gbrt, X, ax=axes[1], alpha=.4,\n",
    "                                            cm='bwr', function='predict_proba')\n",
    "\n",
    "for ax in axes:\n",
    "    # plot training and test points\n",
    "    ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=mglearn.cm2, s=60, marker='^')\n",
    "    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=mglearn.cm2, s=60)\n",
    "plt.colorbar(scores_image, ax=axes.tolist())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}