fauna-dataset-tfidf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jeremy/Development/anaconda/envs/ml/lib/python3.6/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated since IPython 4.0. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.\n",
      "  \"`IPython.html.widgets` has moved to `ipywidgets`.\", ShimWarning)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib.pyplot import *\n",
    "from ml.tfidf import TFIDFModel\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean up the data to remove duplicates, nulls and noise (wikipedia redirects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#If new records have been added, run this routine to clean up and remove dups\n",
    "#fauna = pd.read_csv('./ml/datasets/fauna.csv.gz', compression='gzip')\n",
    "#print('fauna.shape before nulls removed', fauna.shape)\n",
    "#fauna = fauna[fauna['class'].notnull()]\n",
    "#fauna = fauna[fauna['text'].notnull()]\n",
    "#fauna = fauna[fauna['order'].notnull()]\n",
    "#print('fauna.shape after nulls removed', fauna.shape)\n",
    "#print('fauna.shape before duplicates removed', fauna.shape)\n",
    "#fauna = fauna.drop_duplicates(['page-id'], keep='first')\n",
    "#print('fauna.shape after duplicates removed', fauna.shape)\n",
    "#fauna = fauna.drop(fauna[fauna['text'].str.contains('This is a redirect')].index)\n",
    "#print('fauna.shape after redirects dropped', fauna.shape)\n",
    "#print(fauna.shape)\n",
    "#fauna.to_csv('./ml/datasets/fauna-{}.csv.gz'.format(fauna.shape[0]), compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the fauna data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fauna = pd.read_csv('./ml/datasets/fauna-53699.csv.gz', compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(53699, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>family</th>\n",
       "      <th>genus</th>\n",
       "      <th>kingdom</th>\n",
       "      <th>order</th>\n",
       "      <th>page-id</th>\n",
       "      <th>phylum</th>\n",
       "      <th>species</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>muridae</td>\n",
       "      <td>acomysigeoffroy1838</td>\n",
       "      <td>animalia</td>\n",
       "      <td>rodentia</td>\n",
       "      <td>2375794</td>\n",
       "      <td>chordata</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The term spiny mouse refers to any species of ...</td>\n",
       "      <td>Spiny mouse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>muridae</td>\n",
       "      <td>lophuromys</td>\n",
       "      <td>animalia</td>\n",
       "      <td>rodentia</td>\n",
       "      <td>12173296</td>\n",
       "      <td>chordata</td>\n",
       "      <td>lmelanonyx</td>\n",
       "      <td>The black-clawed brush-furred rat (Lophuromys ...</td>\n",
       "      <td>Black-clawed brush-furred rat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>erinaceidae</td>\n",
       "      <td>NaN</td>\n",
       "      <td>animalia</td>\n",
       "      <td>eulipotyphla</td>\n",
       "      <td>46590</td>\n",
       "      <td>chordata</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A hedgehog is any of the spiny mammals of the ...</td>\n",
       "      <td>Hedgehog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>erinaceidae</td>\n",
       "      <td>hylomys</td>\n",
       "      <td>animalia</td>\n",
       "      <td>eulipotyphla</td>\n",
       "      <td>3422827</td>\n",
       "      <td>chordata</td>\n",
       "      <td>hparvus</td>\n",
       "      <td>The dwarf gymnure (Hylomys parvus) is a gymnur...</td>\n",
       "      <td>Dwarf gymnure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>muridae</td>\n",
       "      <td>acomys</td>\n",
       "      <td>animalia</td>\n",
       "      <td>rodentia</td>\n",
       "      <td>12171686</td>\n",
       "      <td>chordata</td>\n",
       "      <td>amullah</td>\n",
       "      <td>The Mullah spiny mouse (Acomys mullah) is a sp...</td>\n",
       "      <td>Mullah spiny mouse</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      class       family                genus   kingdom         order  \\\n",
       "0  mammalia      muridae  acomysigeoffroy1838  animalia      rodentia   \n",
       "1  mammalia      muridae           lophuromys  animalia      rodentia   \n",
       "2  mammalia  erinaceidae                  NaN  animalia  eulipotyphla   \n",
       "3  mammalia  erinaceidae              hylomys  animalia  eulipotyphla   \n",
       "4  mammalia      muridae               acomys  animalia      rodentia   \n",
       "\n",
       "    page-id    phylum     species  \\\n",
       "0   2375794  chordata         NaN   \n",
       "1  12173296  chordata  lmelanonyx   \n",
       "2     46590  chordata         NaN   \n",
       "3   3422827  chordata     hparvus   \n",
       "4  12171686  chordata     amullah   \n",
       "\n",
       "                                                text  \\\n",
       "0  The term spiny mouse refers to any species of ...   \n",
       "1  The black-clawed brush-furred rat (Lophuromys ...   \n",
       "2  A hedgehog is any of the spiny mammals of the ...   \n",
       "3  The dwarf gymnure (Hylomys parvus) is a gymnur...   \n",
       "4  The Mullah spiny mouse (Acomys mullah) is a sp...   \n",
       "\n",
       "                           title  \n",
       "0                    Spiny mouse  \n",
       "1  Black-clawed brush-furred rat  \n",
       "2                       Hedgehog  \n",
       "3                  Dwarf gymnure  \n",
       "4             Mullah spiny mouse  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(fauna.shape)\n",
    "fauna.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize the number of records in each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x111a38d68>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAE9CAYAAAAbNJn3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnXmYJFWVt99umkWwRZYahGFxRDig4gIiMIIsn8jghuIG\nbgPYKIijoAwisjSCiKgoKAiyDKDgAriziA6yCCio6LAeQARUFhtooNmXru+Pc6MzKisiMqI6syo7\n+L3P009nRZ26eTMy4sS5Z7vTRkdHEUIIsegzfaonIIQQoj9IoQshREuQQhdCiJYghS6EEC1BCl0I\nIVrCjKl88zlz5o1LsVluuaWZO/fR2mM0kW+z7LDMY1GTHZZ5DIPssMxjGGSHZR5lsiMjM6cVyQ+d\nhT5jxmIDk2+z7LDMY1GTHZZ5DIPssMxjGGSHZR5N5zx0Cl0IIcTEkEIXQoiWIIUuhBAtQQpdCCFa\nQq0sFzO7Gngw/fhX4HjgKOBp4AJ3P9jMpgPHAq8AngBmufst/Z+yEEKIInoqdDNbCsDdt8gd+xPw\nDuBW4BwzWx94IbCUu29iZhsDXwG2G8CchRBCFFDHQn8FsLSZXZDkZwNLuvtfAMzsF8D/A1YGzgdw\n99+a2asHMmMhhBCFTOvVPtfM1gM2Bk4E1gLOAx5w9w3S73cBXgS8ADjb3c9Lx+8AXuTuT5eN/fTT\nz4w2zbMUQghBYWFRHQv9JuAWdx8FbjKzB4Hlc7+fCTwALJ1eZ0yvUuZAWQUUc+bMqzGt5vJtlh2W\neSxqssMyj2GQHZZ5DIPssMyjTHZkZGaBdD2FvguwHvBRM1uFUNyPmNmahA99G+BgYFXgLcAPkg/9\nmlozzt7k8AvHHTt5360KZfe4cJ9xx47Z6ogmbyeEEK2jjkI/CTjFzH4DjBIKfj5wOrAYkeXyOzO7\nCtjazC4nlgM7D2jOQgghCuip0N39SeC9Bb/auEtuPrBbn+YlhBCiISosEkKIliCFLoQQLUEKXQgh\nWoIUuhBCtAQpdCGEaAlS6EII0RKk0IUQoiVIoQshREuo1Q99UeamWTt1XueOr33iKZM9FSGEGCiy\n0IUQoiVIoQshREuQQhdCiJYghS6EEC1BCl0IIVqCFLoQQrQEKXQhhGgJUuhCCNESpNCFEKIlSKEL\nIURLkEIXQoiWIIUuhBAtQQpdCCFaghS6EEK0BCl0IYRoCVLoQgjREqTQhRCiJUihCyFES5BCF0KI\nliCFLoQQLUEKXQghWsKMqZ7AMPHNwy8qPL77vltM6jyEEGIiyEIXQoiWIIUuhBAtQQpdCCFaQi0f\nupn9C/AHYGvgaeAUYBS4FtjD3eeb2UHAm9Lv93T3KwcyYyGEEIX0tNDNbHHgeOCxdOhIYH933wyY\nBmxnZusDmwMbATsAxwxmukIIIcqo43L5MnAccGf6eQPg4vT6POD1wKbABe4+6u53ADPMbKTfkxVC\nCFFOpcvFzHYC5rj7L8zsM+nwNHcfTa/nAcsCzwPuy/1pdnxO1fjLLbc0M2YsVvr7kZGZlZOvI3vT\nAMce5HgTkR2WeSxqssMyj2GQHZZ5DIPssMyjiWwvH/ouwKiZvR54JXAa8C+5388EHgAeSq+7j1cy\nd+6jlb+fM2deryEmJNvvsUdGZtYeb1CywzKPRU12WOYxDLLDMo9hkB2WeZTJlin5SpeLu7/O3Td3\n9y2APwEfBM4zsy2SyLbApcBlwDZmNt3MVgemu/u9tWYshBCiL0ykUvRTwAlmtgRwA3CWuz9jZpcC\nVxAPiT36OEchhBA1qK3Qk5WesXnB72cDsxd6RkIIISaECouEEKIlSKELIURLkEIXQoiWIIUuhBAt\nQQpdCCFaghS6EEK0BCl0IYRoCVLoQgjRErSn6AS54+rPdV7njq/+qgMnfzJCCIEsdCGEaA1S6EII\n0RKk0IUQoiVIoQshREuQQhdCiJYghS6EEC1BCl0IIVqCFLoQQrQEKXQhhGgJUuhCCNESpNCFEKIl\nSKELIURLkEIXQoiWIIUuhBAtQQpdCCFaghS6EEK0BCl0IYRoCdqxaBLY76qbC48ftuFakzwTIUSb\nkYUuhBAtQQpdCCFaghS6EEK0BCl0IYRoCVLoQgjREqTQhRCiJUihCyFES+iZh25miwEnAAY8A+wM\nTANOAUaBa4E93H2+mR0EvAl4GtjT3a8c0LyFEEJ0UcdCfwuAu78WOBA4Mv3b3903I5T7dma2PrA5\nsBGwA3DMQGYshBCikJ4K3d1/DHw4/bgGcA+wAXBxOnYe8HpgU+ACdx919zuAGWY20v8pCyGEKKJW\n6b+7P21mpwJvB94JvNndR9Ov5wHLAs8D7sv9WXZ8Ttm4yy23NDNmLFb6viMjM+tMr1L2pgGNfceA\nxl2Y8Qbx/s8G2WGZxzDIDss8hkF2WObRRLZ2Lxd3/08z+zTwO+A5uV/NBB4AHkqvu4+XMnfuo5Xv\nOWfOvLrTayQ7yLH7KTsyMrPReE3kJTtc8xgG2WGZxzDIDss8ymTLlHxPl4uZfcDMPpN+fBSYD/ze\nzLZIx7YFLgUuA7Yxs+lmtjow3d3vrTVrIYQQC00dC/2HwP+Y2SXA4sCewA3ACWa2RHp9lrs/Y2aX\nAlcQD4o9BjRnIYQQBfRU6O7+CPDugl9tXiA7G5i90LMSQgjRGBUWCSFES5BCF0KIlqAdi4aQXQ6/\ncNyxk/fdaqFlhRDtRha6EEK0BCl0IYRoCVLoQgjREqTQhRCiJUihCyFES5BCF0KIliCFLoQQLUF5\n6M8i9rhwn3HHjtnqiCmYiRBiEMhCF0KIliCFLoQQLUEuF1HITbN26rzOHV/7xFMmeypCiJrIQhdC\niJYghS6EEC1BCl0IIVqCfOhiofnm4RcVHt993y0mdR5CPNuRhS6EEC1BCl0IIVqCFLoQQrQE+dDF\npHLH1Z/rvM4dX/1VB07+ZIRoGbLQhRCiJUihCyFES5BCF0KIliCFLoQQLUEKXQghWoIUuhBCtAQp\ndCGEaAnKQxdDzX5X3Tzu2GEbrjUFMxFi+JGFLoQQLUEKXQghWoIUuhBCtAQpdCGEaAmVQVEzWxw4\nGXghsCRwKHA9cAowClwL7OHu883sIOBNwNPAnu5+5eCmLYQQopteFvr7gfvcfTNgW+AbwJHA/unY\nNGA7M1sf2BzYCNgBOGZwUxZCCFFEr7TFM4Gzcj8/DWwAXJx+Pg94A+DABe4+CtxhZjPMbMTd51QN\nvtxySzNjxmKlvx8Zmdljer1lbxrQ2HcUyPVj3GGRXdTPW5PxBvH+i6rssMxjGGSHZR5NZCsVurs/\nDGBmMwnFvj/w5aS4AeYBywLPA+7L/Wl2vFKhz537aOXk5syZV/n7icoOcuw2yw7LPHrJjozMbDRe\nE/k2yw7LPIZBdljmUSZbpuR7BkXNbDXg18C33f0MYH7u1zOBB4CH0uvu40IIISaJSoVuZisBFwCf\ndveT0+GrzWyL9Hpb4FLgMmAbM5tuZqsD09393gHNWQghRAG9fOj7AcsBB5jZAenYJ4CjzWwJ4Abg\nLHd/xswuBa4gHhJ7DGrCQgghiunlQ/8EocC72bxAdjYwuy+zEkII0RgVFgkhREuQQhdCiJYghS6E\nEC1B/dBFa9jl8AvHHTt5360WWlaIRQUpdCF6sMeF+xQeP2arIyZ5JkJUI5eLEEK0BCl0IYRoCVLo\nQgjREqTQhRCiJUihCyFES5BCF0KIliCFLoQQLUF56EL0kZtm7dR5nTu+9omnTPZUxLMQWehCCNES\nZKELMUV88/CLCo/vvu8WkzoP0R5koQshREuQQhdCiJYghS6EEC1BCl0IIVqCgqJCLALccfXnOq9z\nx1d/1YGTPxkxtMhCF0KIliCFLoQQLUEKXQghWoJ86EK0kP2uunncscM2XKtQdmH3Yq2SF5OLFLoQ\nYiA02YtVPXD6g1wuQgjREqTQhRCiJUihCyFES5APXQixSKEuleVIoQshWk1WZftsqLCVQhdCiMTC\npnvC1KZwSqELIcSAaaL8m6R7dqOgqBBCtIRaFrqZbQR80d23MLMXA6cAo8C1wB7uPt/MDgLeBDwN\n7OnuVw5ozkIIIQroaaGb2T7AicBS6dCRwP7uvhkwDdjOzNYHNgc2AnYAjhnMdIUQQpRRx+XyF2D7\n3M8bABen1+cBrwc2BS5w91F3vwOYYWYjfZ2pEEKISnq6XNz9bDN7Ye7QNHcfTa/nAcsCzwPuy8lk\nx+dUjb3cckszY8Zipb8fGZnZa3o9ZW8qPLrwY99RINePcYdFVudtYvLPxvPWj7EHdd6g+NwN87lY\nGNmJZLnMz72eCTwAPJRedx+vZO7cRyt/P2fOvNqTaiI7yLHbLDss8xgG2WGZxzDIDss8hkF2suZR\nptwnkuVytZltkV5vC1wKXAZsY2bTzWx1YLq73zuBsYUQQkyQiVjonwJOMLMlgBuAs9z9GTO7FLiC\neEjs0cc5CiGEqEEthe7utwEbp9c3ERkt3TKzgdn9m5oQQogmqLBICCFaghS6EEK0BCl0IYRoCVLo\nQgjREqTQhRCiJUihCyFES5BCF0KIliCFLoQQLUEKXQghWoIUuhBCtAQpdCGEaAlS6EII0RKk0IUQ\noiVIoQshREuQQhdCiJYghS6EEC1BCl0IIVqCFLoQQrQEKXQhhGgJUuhCCNESpNCFEKIlSKELIURL\nkEIXQoiWIIUuhBAtQQpdCCFaghS6EEK0BCl0IYRoCVLoQgjREqTQhRCiJUihCyFES5BCF0KIliCF\nLoQQLUEKXQghWoIUuhBCtIQZ/RzMzKYDxwKvAJ4AZrn7Lf18DyGEEMX020J/G7CUu28C7At8pc/j\nCyGEKKHfCn1T4HwAd/8t8Oo+jy+EEKKEaaOjo30bzMxOBM529/PSz3cAL3L3p/v2JkIIIQrpt4X+\nEDAzP76UuRBCTA79VuiXAW8EMLONgWv6PL4QQogS+prlAvwI2NrMLgemATv3eXwhhBAl9NWHLoQQ\nYupQYZEQQrQEKXQhhGgJUuhCCNESpNCFEKIlDJ1CN7PVGsovPih5M1u5ydgV47yvH+P0m6bnbqrp\n9X2Y2XQzW8zMNjOzJSZrXm0h9WJq+je175G6shOZR50xm14bi9r9Af1PW5wQZvZx4DHg+cDOZna+\nu3+yRHY34JPE3KcBTwNrVYxdW97MPgfsDiwBLA3cBLy0RPalwPOA+cBhwGHu/r8l0/gwcHrZHLvG\n3ZhI91w8zXcVd9+mQv5wYD93n29mywInuvu7SmSbnIuVgeWSzKeBr7v7n0pk3wp8LDfuiu6+Xons\nMmncp4jzcpq7314i2+T7+CJwK7AGsD5wD/CfRbKD/HxJPrsu3g783N3nlsjVvoYGeI7fBSwGLAl8\nycyOcPcvV3y2Jt9JE9mm81iZsffIFRWyta6NprqlyTzMbHlgmy7ZL5TIvhh4V5fsR6rmkTEsFvqO\nwKnAtsQX/soK2VnA5sB5hOK7rsfYTeT/A1iVUL7rAv+okD2O6Ci5P/BZ4KAK2SXN7Goz+56ZnWFm\nZ1TIHg1cBCwL3A7cWyFLmsOvzOxtwCXAzytkm5yL04CVCEXzS+CrFbIHEp//b8T3+OcK2dOBDYAv\nEQrnWxWyTb6PTd39eGATd8/+roqBfD4zOw14K3AE8Frg5Ipxm1xDgzrHexOf//3AasBbKmSh2XfS\nRLb2PMzsZOB/gUuB31P93UH9a6ORbmk4j7PS2B8BPgj8e4Xsadm8gX8DVqiaR55hUeijwMrAPe4+\nCixfIXuvu98FzHT3i3rINpW/z92fSLK3EFZFGU8RX/gSqRFZ1Wrn08CewDeB49O/Mh5w9+8CD7n7\nbHorptnA34EfAMe5+6kVsk3OxQziAfF8d/8eYT2VcV9mmbj7KcQNWcZywE+Bf3X3wwmLrGrcut/H\nYmb2GuC2tKQeqZCFwX2+F7r7d4B13X03wgIvo8k1NKhz/Hj6f152ritks3nU/U6ayDaZxzqE4fcL\n4kHxeIUs1L82muqWRvNI14MDWxPfURmPJuv97+6+E2F41GJYFPqviZvr62b2VeDsCtkHkzU6amYf\nofeN20T+72a2C/CImX2B6ptxFDgDONfM3g08UiH7R+JL/CDxtK2yVEbTUnxpMzPgBRWyABcD9wEv\nBP7DzKqssSbnYgngSOASM9uSamXzhJm9DljczLYhHs5V434K+KOZvQR4boVsk+/jNODrwJcJ6/io\nCtlsHgP5fOl6uN7MVqTaumpyDQ3qHP+VsC5PNrODgN9VyEKz76SJbJN5zEuG3zLufi/xeauoe200\n1S2N5mFmSwHLEN971XcyzcxeAMxM7rNeD5YFDIVCd/fPuvsa7n45sI+7H1IhPotwRewLrE346Kpo\nIv8RYgn138CdwHsqZN8DnOTuRwFzesieTPjw1gbuBk6qkP0k8dQ/mrjZj6uQBfiiu+/l7ne6+3bA\ntRWyTc7FToQ18UXiwn5/hezuhL/vUMJne0CF7KeAfwE+D2wJfLRC9iPAr+h8HzuUCbr7scAbiGv6\ns+5edY5h4T7fgRWyRwDvBL4AfJxwp5TxHuJaOJre19BAznGyANdz958Dx7t7r/upyT3SLVv1/eXn\ncVyPefzBzPYG7jSz79EjFujux7r7Ru5+nbvvWXFtdN8fu1WN23AexxCr9AsIt9mNFbIHE/GXbxMP\nuvN6zGMBUxoUNbNvuPvHzOwK4qmVHcfd/71L9tXu/ntgk3RohDg5hU9FM3tD16ERYmk0Tt7MPlww\nxBPAZsANXbJvThfd9l1/uxblvsoV3P1kM3u/u19uZtNK5HD361J03YCd3L2wwVluHqt2zX/csq/h\nucjO878BtxB+vweAFxMPpSLuJizR5xIKraqfxN+BbxDW2s8q5CCsmQ8T1ug5wJNlgmb2DkJ5zgB+\nYGaj7n5oxdgrEw+/jdL8V6X88x3h7lmm0juqJuzuPwR+mH6sUvyQGtkBH0j/r0bHf9rNmoSLZkXC\nwnzKzFZ1979nArmf7wVOJJT6L6smkFaDx5nZ84HTzezadF11yy1GuKW+Ryjxaek9zgG2Khl+GeL8\nLgU8SOyPcH3XuPu7+6Fm9l3CMs6O4+7vLRrU3fczs+cS1/q29FhVmNlfGXtNPuTuRXG66cR3sFYa\ns9LV0XAet7v72Wk+ZwKvqhj3EjO7nvjO13H3+6vmkWeqs1wyS3xnIsuliv9HLMl2JL6caXS+pAsK\n5HcsGWe0QL5s+VqkmLIldPffVDbFMbN10v+rAs9UyB1C3CBXAp8wsx+5+5cq5tHtkimaR5NzkT/P\nvWQzziUeDg/kZLcvkf1++v104qFxMxH8KeJkwjrZnLBkT0qvi/gksDGxwcqh6TNUKfTMApxGrIhu\nI9x+RSxlZi8nsjTmA7h74cPFzO6i8/mWA25193VLxs2OTyMSAe6nXKEfSnzXfyCUwZNpXifkro9P\npn/H07lHSK/LlO7RxP13AnF+z6M4sL4LsF+ag6exnwF+UzIuwE8Iy/xvuXl0kz3Ue61EF5D84TsQ\nDwoIZVq10lsn/T+NCBYXZoER1/cNQJaVNErEpsrm0R3wfrOZ/Q04JstsMrPNgJcAe5nZkUluOpGx\n9LKScT8K7EUYHC8xs0NSXKYnU6rQ3f2e9PJEdy+7qTPZL6aX/w28yt1/aWYfA8o+aK00nzT2wQBm\ntnoN2Szo+EzeAkw+wjI+DvwPcQOfRbWrY1vgNSkNcTHgCiJboWwea+esx7I5F3a9tIK84Nx5nusl\nqaMFLOXuZYq2e/xshUWyCqsCxLVXNsB8d38iWeajZlblj8bdFzywUqCs9MYlVks/yf08CryoZNwF\n59TM1iCC1mVz+ExOdhrVGUqPAi9398fNbEkizrQ98RD6Uhov+76OdPcFq5/kny/F3W9J522Omc0r\nkTkBOMHMdnH3qsydPNPdvcqVhbtn2TpXE26klxAPziq366mEq6wwHbTgPZ7I/XhZxb36YHL91OU5\nwF+ILJeNgQ2Bf6b5vTXJzCUegkvSMQLnA/tUjLsr4X563MyWJuJkw6/QczySgqFOxwIqc198l44S\nuJ/4oG8ukHPGWwSZVV94M1LDejSzDxG+tnXNLFsyTycs1M9QzAu7FNm7iQu4iL8TEf4HCZ/pPSVy\nGUs2sB4PJiyZnnnBxOd7vrs/UPL7PJekQN0C95S731Hj7x4klpWl1F3ZAJdapIOuambHAVfVeP+M\nGZRfE7h7oSXVC3e/PZt/ETa2wGVl4porY8TdH0/jPmFmK7r7k5YrwjGzNxOpkjuaWXa9TQe2o/yB\ndX8KAC5jZjvQWWWV8bs0dp36i/8zs42AP5HuxbJrk1iNXUykOG4OnEJHKXZzc8r0qUVS4JkuWCXN\nvYhfWOSiL3ALuXvZqg3iO8kMg1+Y2QXufoCZLfgbd78WuDatpO7MzamqaOkeIgcewnNxX4XsGIZF\noV+e/q+TnrOMu58F4O5nmNmuRULuPubmsEjsn5ui0oXUtB6/QwR69iOCThAXyD+7BbtusCwm0OsG\nWwW4ycz+TFgrT1r0lx8XV8jehprWI2H9r0rkyx4JHFsiR3rv+8xsThpz1N1XKZFdCfgaY10uhXm2\nuXjJNMKX/6uKOXSvbKqCe/uZ2X8QD8ob8xZqyTwy18g04j74WoXsr+kyDty90IWR+YLTj6tQ/UDO\njI5pxI1b5FrL+LGZ/YZwxW0I/NTMdmdsEPzPhCvusTQ2xLX5vYpxP0Rcy/cSPu5dKmQhXCOfIAJ3\nnyViJmUKfXPG5pNXXZsruPvX0+s/mdk7K+ZwdgpC5hXv5yrk8wHIP5P2PS5gM8KSzlabo5S74QCe\nZ2bruPuN6cE908yyWFI3bzGzT9EpWnqKCLwWMZ04B5cT7rXFk7FSGlfIGAqF7u4Hm9nrCQvld4Tl\nWMaTZrY18FvgNVRbbVikeh1LBHTONLPba2RAQIn1mJZvt6Un+avp+PH+jfFffv4Gu5H4InvdYGX+\nvTJ2cfcF1qiZbVEhe1+y7mamZXZpXrC7r9FgDlbhJ+4mn+nweM7tVjSHa+kEwSutGutUzJ5vZs83\nszO9pGI28T53v7DmnLNsh8wH+4oK2bwv+HHCl19It9HRg8OJB/e6wMnufq2ZjeTfz93/BpxqZt9O\nh6YT5+/67sFyfNzd981+SNZs2UoTunLnzaxUh7h71Xnq5jlm9gJ3v9vMVqK6LuCjROC5cjWRC/Df\n1fWrjSiOBz3X3V/fYM4fIwLJKxNxgj2IgPHnC2R3JR4U+wNnEhkvZeT/vlaFecZQKHQzO4ywHNcl\ngj2foTyQN4vIJz2KWOL38pUfCryO8DkeRmyTV6jQC6zHqgyBs4gsgnzAZ4xCz91g5xH+z1+Z2R5E\nAK77vfPLwjG4+34F8o2DLYzNCz6cirzgLPuBaMdwOlCY/ZC4xqJlwdX0XlqPECmDS6f3wd0LrcL0\n0NyLTgl0VSl2VjF7NGE9HlkilzEbqKXQ3d1zP96YzmH3XIsypSCU/7e6ZAuzu9J7lVUQ/iHN98T0\noMPd55TIfoEepe4lrsPFiHNdpdB75s5P8PPtT/i3HyYs3MKVd+L+XKyniqYB/muT2yl/HZcal+5+\npZltRZznW939Ycof4Pe6+13JmLrIoi3CGKyTubYO489bVX3JAoZCoROlua8zs1+7+6lpKVlIsiz3\nIdLo/o/qIh2IYNn9KejzeFnQJ1HbegReUHFxdnM6HffNXIr9/lV5qUVMJNjyEeLBeSahVEvzgqmf\n/QDxwHxT7ueqpfU3ibTFuyveO2MWsAX1rJrZRDDqTMLqrKqYhUiR+xFj4zbjHpwwTlmvTHEVY/Yd\n5LNLysgCflXnv5tXEqX0ByXL/DvA95IS6WZTd/90up+2NLMil0ht12EX7yFWxucR301RHvpEPt9T\nxIpmaSJOVrV6udfMjicK9jLFW6TwvpriFHUTJF7B2NVXVXZQ01TZOkVLTTLXChkWhT7Doopq1CKz\noyqt72NE0v3yROBkLcIqLeOWZP2uYGb7EoUDZSxJ+DHXBq4zs095SVMjwlJbJR/oqKDb7z+rWyBT\nQGkJuyG5xjxFA04w2PImYEN3P9DM3kQos9LleJ3shyT38vTePeMURA5wL2Wb0dOqyXExYcWuAXzT\nzF7l7mVWM1T3WOkmnw30ODAua8Q7mVLfIKzowkZfSTYzFFZibPodlMQJPLKeziNu7lnAfxGN7E4t\nUGY9S91zrsOPErn+LyVcnd8sm3eiZ+587vM9l67mY5Tff4cQPuyziFV16UqaqI+AjuIru94aJUak\nh9+yjLW4q2iSKjuLMEL3JfrWFBUtXWFmaxMPtAkxLAr9q8TNOEL40Kua3OxAfPEXuvtRZtYrm2E3\n4mT+BniY6qXcacRy/XIiu+UUotKuiE2BO1LQEKqDht1+/7IoO4RvcAngX4kl8J1Uf8FNgi0HE1Ye\nhGV1HuW55bWzH+rEKaxT3PSgme1HfN+ZdVU2hyal2F9093PS6+0sOnhWcTqxSlmNaD0xrsLWOmms\n/9NjrDw/B/azyMr5NnC6uz9UIls7/c7MjiCC6RcTn/VKiwyXPzC+oO1UotR9F3q3QTieiBf9kvDx\nnki0qCijSe58kwBqtpKmxkr6UOIBtFSFzJgYhUVa6AgRRyo0GBta3Nmc66bK1ilaylbx3Q+hUcJ9\n1JOhUOjufqaZ/YpUjejuVWk6WZpW9qGfKBJKSibjejqW6MaUR64fcfeszPYcMyvNw3b3MqVZROb3\nPzrNo2oJuKy7b25mJxJWWGWlH82CLU+5+z8B3P1BM6sKKHdnP3yoQrZOnCLzYz5IXNSZL7yqYGlX\nIjCdWTVV+ftXmNmOdFY2y1TIQiibO4keO78nlNIbu2S+n/5fgXCzXEMoknsI3/Q43P184PzkFjkK\n+LJFZeBBBau9Jul3NwEbZFajpZRSM3t7gew/gde6+9NUXw8Aa7l7dq/82FJGVRneLHe+dgCVZivp\nJoVsWaLAycS1t5yZ7eruRfdV0+K0JqmyPYuW3H3LNN9/EAp/DlEZ/LiZ3Qx8tGTeCxgKhW5mbyH8\ntUuln3H37psr4wxCIa9hZucCPy6Ry27+NYkv/yoiBehhwvdXxN/MbH8i+LQB0RDpDTDeirRmfcu3\nA3atCGKYoHGYAAAdgklEQVTleSr9v4y7P2a9m/E3cUtcmS7AK4iVwrhc+LTky8i7JVYkrLEiesYp\nPBU3WTSsqlMYBnCWu2eW/acq5CCW6jcBLyeyih7tIb+mu88ys83c/WdJiXTPeZM05x8BH3T3eRbN\nkkpXTGa2LmH5v4Ww/Dcl7rOziQdjnibpd58hruGsEOeHwFbufluB7IbAAclIOsndbyiQyVjKzJZ2\n90fN7DlUZ5c0zZ1v0nwsv5J+hOqVdO1CtsShRFzhTjP7V+LcFSnGpsVpTVJlmxQtXQLMdnc3szWJ\nFhKHEPfK8Ct0wnr9CDWWnu7+jRTkeRlxEgt7nXhK+Dezc4Dt3P3p5J8/p0g+MUo8AF5EKOl76LQa\n6LYijyZcQ+8kLLcqxfsIYf3cRViu51f4mX9kZgcCfzaz3wJly/WMJm6JjxMPl7WBH5RcgPnc++45\nlgWImlhXdQvDAB4ws+0YG7isyjrYzaIcexbV+cMQcZsVifM2k2o32KruPi+9xyNmVuZag3BZfIu4\nIRe0szCzIrdNrfS7xKPAMWZ2gLv/morAq7vvm9xa2wKHWnTuOwH4TrLa8xxFXGvXEllTs3vMo0nu\n/HuIqudzk5Vc2sgrzatu+X/TQrZnsjiTu//DzMra3F5qUUdQaXHb+L42FxJxiwu9pD6BZkVLq7pH\nZpW7/8XM1kjxrO7vbhzDotCv8+g/3BMzezWdtLdtrSLtLZEPaM0gUg0LcfedzexlpPLjqsAWqW+5\nmb3B3Web2cUV4x5HNEB6KeFLPD4pnq95VyWmux+TvU4Po1uopjvYUuWWmEkUOr0UWNnMLvOuxj+5\nZd9SRE/vq9MDo+pB2CROUaswLDHCWJdBr6yDuu1JIaXJEdfHb6l2TVyQvt/fEyubb1fInpcP+prZ\nF9z9M/nvNUfd9DuI5fc7CMNgRTqVhONIrpA3EL7wNYh4wQjx8BhTfenup1sEW18E/LWHu7Np7vyT\nwJYWqbo3EVlp/aB2IVviITP7L+Ih/zpKVpo5i/uPwA1enqbb3dcmm8OlFXNoUrR0l0Va8eXE57rb\nIgZX2pwuY1gU+k8sclbzT9wyJd0k7Q3CYrouZ4GUdsBLX/p7iaDF3mb2Ay/fBqt233KLqtMdiBvs\nASJQNIMoFNm8S/bfiQDjSkRK5iyidLp7zNodFHM0Ka/+DlHFeTVh0b+bODdlc7iVTrfCLSj3izcp\nDNuW+g+V7vakVU2jcPeL4yPYCOG2qqog/mz6rl9KbOc2brcgm1hed930O4Bp7j7XIjvpbMK1VMbN\nhHI52t0vy83xJbnX+YrW/OeorEa02ApvDzquxqrt+Jpcb01oUsgG0Rp5fyI983pKqmHN7IXEyv85\nwPpmtn6RC8wn1temSdHSB4nMo22JYP1swl1cVpuzgGFR6B8nIuB1lp5N0t4gHhIbEsn6t3o0oi/j\nvcBmyT2zOPGELFPoTfqWX0UoyPd4FBsBYGZFLTy/DrzX3a9Pq4VvUWx9NOmgmNGkvPpf08oCdz/C\novy9H3NoUhjW86GSYymP3XmwqBItdFVZQRl/Oj6unN/MZrn7iTa26OuVZvYeH5+zns/rPpROVXBV\nXnfd9DtIDwV3f9iipcQnKmTXL/r8PrZJW+3uhl0cSATrdyNiBFVKqsn11oQmhWwAx3qPBnaJ7xIB\n0brGYpO+NtdaBO3zD+9C96FHz56juw6X7pmaZ1gU+t3u/v0qAZtY2hvAwSmKf2WNeUzLfIzu/pSZ\nPVUm6NG3/G9EIPeNVN+M53rKU+4a47MFsg+4+/Xp99eaWWFwz0s6KPagSXk1Zra2u99ksWltkeyu\n6eFXaxd1aFwYVuehkrFgI+4yZZ7I8n8PIgLqlxErhSI/fvbw7Vn05Z287pOBt7n70WZ2OvHwKmvE\nViv9LvEPi2yZrBvhXt0C1ulPg6W+4qS8a+9KqU0rlKx2YMzmxYRVXcZ97n6Fme3m7qeYWdV12Oh6\na0CTQjao3/740aL7tIImaZmvYOyqaknC/dlXhkWhP2Zm5zP2idttAU0k7Q0aVAQCvzGzs4jl6mbE\nzV6IxWbAmxKriqxYoTCVDVjb6ncu/KdFymKWaTPdUqVi0XLcxjaZWp7q/tsHAJeb2YNEwUeV//oT\nwPfTjXgnxb750wiLOQuUrUAEtudTcoNZw8KwGg+VjCXN7GrGfs/jrPks2GRmK7l7ljb2o+Ru65b9\nRfaSCO5lSvorFfM4mojxQJzvUwgFVEST9LtvEe7GSwiX1kl05Sb72Na9y2QBXK8ufsuyg9YjiqZ6\nZQc12Qovu94eIuI3VddbbTwVsjVgbSoa2Fkns+seM3svY43Fqr5STdIyv0+s6rMHZ6mxuDAMi0Lv\ntXNNPu1tlrufmB233gUktSsC3X3v5KPMGiCdWyFu7l5lFeRp0rkwswbXIjJcLiZumrI+L7X7bxNV\nnC+yaL16r5lVpX6tTwQYnyBSFs+gq49KTmHuRPj97yJy4W+rGLdJYVidh0rGpyt+V0jye19JuLSq\nFFkTJf10boV1q5lVZc80Sb9byt1/ml7/2CpqJCyypJYlUj2PMrPfVwVfvVl20O6E+/JQIpWuNCbl\nkZq6NuFS+kdVnKIJBX78FaqUfObjt/JK5nxm1650HjyVQXiapWXOYmy9SJXLbMIMi0I/nfClZsvJ\nceXHyf/0ViJqnp3k6YRl0e1vajR27j3eQqc0/nwze7rCnXOlmVlm8VXhNToXWmf7sHE5zj2shLxc\nYf9tK2jkZVFlWNXIq/sCrMoCyZdtf57qsu1ahWGJng+VHNfQzHXwPkLhbU88RKv2xmyipG+3aDaX\n5fpXuZSapN/NMLP13P0aM1uPahffdu6+QRrvXWZ2GVGRWog1yw6qvR2fmW1PrGbmEq1md/cehTE1\n6fbjb91jHpWVzD6xzC5okJZJs3qRCTMsCv14YtlZVX58PmEBrkDniTqf2DFkYcfOaFIa/yBwlUV3\nuEI/ZUa6AU8mGmPdTbS87fardm8fBh1XTlWqXp3+2xNp5NXkAmxStl23MAyaPVQauQ6SX/ciIh3Q\n3b3KumqipHcmFM0biYyKqp13mqTffRw42SIH/h9EzKCM+Wa2hMcGGItTvRn8MYQ/vlZ2EA224yNW\nMxu5+z/TKutn9K58rkMTPz7U77jaJAgPzdIym9SLTJhhUeh1yo+XSYql2x/Yy6JoUtrcpDR+S2B5\nH1+oUcTRwCx3/7NFZssxdAVEPG0f5tEgaEXghcAtNfzu+WyFxwj/3xh8Yo28mlyATQqLfkkEjl4W\nU/Oqm6CRVdPEdZDmuxahwP7TzF7n7mXVqJmS3pawpqvKwZ8m0l7/SDyQ30F5ZWnt9LtkAGxYR5a4\nJq41s2sI90iVu+XsBZOpyA7KUemP7uK+3P10T/Kl94Mmfnyo33G1SRAemqVlNqkXmTDDotDrlB/n\nLVgYu1F0lZ8rP/bSJWNn5EvjNyRuyjJuppMr3ovpnnKX3f1PVlHxlayNTxOKYx0zOygXvCvieeTc\nREQf8LJVRZNGXk0uwCZl2yd57B9bVY6e0ciqaeg6eJ27vzb93VFEXnwZT9EJYk2n2t3RpLlaz/Q7\nG79j/VOEW+mJsoeBu59kZj8lyvL/4hXFQhadP/cicq+z9M2q2NDOHptG1OEhM/sFofQ2IGo2Dktz\nLEtMqENtP34iMzhW7GVwWCcIvya9s3Jqp2V6VBpnq/JebSwmzLAo9J7lx10W7AhRon+Td1U6FvA1\n4mRfl8Y+qEL2C0TRwdJEBkuVj3BTIk0ty2uvCnQ+ZZE7fCmx9KvyG+8OvDJZEssQ2S5VCr2Jm6h2\nI68mF6A3K9tusn9sk4dKU9fB4mY23d3n0zEMyvgW4Ra5gN5uuybN1eqk362T5ncMcLxHl8VXUbEd\nX87FtxpRdVjk4svYnXAP1c293tuiAOc7RCuBqhXkz+kUjtUxfOpS24+fyAyOS6muZN6T6LKYGWpF\nLW7zDCotc8IMhUL3BuXHFptf7EWkC73EzA5x96oGTw8TQa+ZwB3EjVi2BdyphI9tD6JA5EhK2ue6\n+4srP9RYPkTkIx9O+FWrLNj76FiDj9G72KqJm2hSAjM9eD3hw8xaMDynTLDhQ6Wp6+D7xA45vyW2\nJKuqg2jktkv/12mutksva9fTjvVmtqa7X5mOXW3WSTQvoKeLL8e9Xt7zv2g+O5jZcoRv+Uwz+ydw\nghe37tjBO83V+kkTPz7Eiia/8i/LGPsd0RIYM1u9IkCdMZC0zIVhKBS6Rarg7ozdlqzMjfJhYju3\nx5ML5WKqO/Z9Kf1NnRzwGYTvdT93/55F8//uue7v7odaQem0l5RMpxumcq/Q3HgjwB+SolmfUOpV\n9OygmKPbhbFij7H7hnVK4x8mfNEQ7oteW57VHb+R68Ddv5LcAesQG1JcVzF8k46ETZqrNbF2HzCz\nQ4g0y02oTg3t6eLLXB/AEuk85CsYe7lDVgJWJ66f64F3mdkHfXy7jgcsUgzzirdWxlYPmmyMDuHy\nupFYvb6W6G///nGDRgr0Y8S2izub2fmZZ6AIH1Ba5sIwFAqd8IPtRb1l3z10GhM9Rli0VVznqSqu\nBksQVvmlZrYlxecny5mvXTptUdm6D5F5UZYR0z3eKPV2Lsk6KBrlHRQzjiEeEvsS/XDqbJbdLya6\n5VldGrkObGyTtzdadZO3rzHWJVjqtvPxzdVurpBtYu1+kNglaFtCOR1Q/ulqufi86/9amNnviOyO\n48h1lEwPhW5GGFvR2iveVQt3L0u1LWMF72yE/RMzK2uitSPhUjufqOAtq/oEBpqWOWGGRaHf30Dp\nTid84pcTDWsWTxZqmYXcpPHXTkRO60mEkhz3FCeyB5YgCgPeQyjorC1v2cX6bqJfemkqnXdKsZ9H\n3KxZ3nxV2htEyfr63smdf8LLc+e/DOyU3C7/TUTlq6oe+0ZyHdxGdbrdwtDIdUCzJm93EW6ZUpeg\nRXvcMgutqhtoXWs33xu+Fz1dfN7Z8nBpok5jbcKNeXy3bBdnE/77vYGDMyXmBXsBeMrvzujhfqqN\nxWYP+VXSU0TcZB93L0pkuM7MXuvul6X4wu0WGV7Tulw1o0TGzD0e/dCX7zGVQaVlTpgpVejW2Xz3\nSTP7FmNLbssCZZ/PvT69xtvUbvzl7jfTsajKApHdrTOzJkxVrTNvo7frJKNph7omQdEmBTKLBAvh\nOmjS5C3rB1QVgM/iMrsTTd0uIzKlXlMx998Rq7YTgQNz/vIia7dJb/hPuHuliy/HGYTFfz4V7ogc\nOxLFND2VWHLrdZe7N9npq4wLiaD+pYT7aVaa99FEskI3mwHbWPRmylJ1b2K8q+bXhMt1xxS4P5tq\nBpWWOWGm2kLP8kd/l/6v03VuXNWlu5ftaQg1Gn81wSfWOnMJIkUt24xjtMzfTvMOdU2Cok0KZBYV\nMpfBQ3SCyIdRsvKwiTV569kPyFPfF4uNxY9Ihy8zsyqL7f3JiBj7gYp3vhphbLl4VXOnda1+76C6\n7oiMJkpsV6LvTJ3CsCas7e6/Sq8vstj043/NrMwV9p+9gs+woE3y/sRqaR9379VvJZ+W+Wr6l5Y5\nYaZUoXtnp/QXE7nU37Vo7F7ln26ySS3Ua/w1EZq0zqy7iQE0T4VqEhTNVzH2KpBZJMi5Dq4k3EnX\nJ6V0ChEP6WYiTd7Oo8bDIvFci9YUVxFVn+PcDMkFWNYVsaxStElzp6x30L3E9VmVUlvXHZHRJLd8\nUFlVT1rs/pNtAPGEmW1AuT6rFXy2zt6jDwHPt/K9RzOywOwoQ2IcTbWFnnEq0X4SogPduE5yGd5s\nk1qo0fhrgjRpnXk19f3i+1O/IyKMDYqe6Z0GTuPw6LP8tR7jLarUcif5xPY23Z56DwsIl9whRF/7\nsh4xO9T4PN3Ubu7kNXoH5ajrjsjIZ5f0UmKDyqp6L3HPvZXYAOIDhDFTGKtoEHyuu/doxrg+UT3S\nJwfOsCh0spPr7pdYNI4qxJptUrvAghsATVpnNvGLr+S5jog15vGB9P9dhFXxwR4uqLbS1J3UZG/T\n2rEHd7/RzA6ms43huKBrFry1aFOQ5ymLHvvHuHv3/rq1rV2r1zsom8tL09+UdSLslm9yPw0kq8rd\n77PYV/ju+NHvI1ZRVdQJPtfdezSjSZ+oSWFYFPoDKUCa3YxVzZ3yvbfvIiL5U0GT1plN/OIfBk6v\nqcyhuQuqrTR1JzXZ27T2w8Iil3lH6m1j+ByiudylwMZEEPWfxIq1+4HfpA1C7cIi69GJcCEZSFaV\nNevDkw8+nwAckFnRBcHnWnuP5mhScDYpDItC34lYQr2dij3/crL53ttTVW6bb525JdWtM5v4xZe0\nzkYNWe/00o5vE3BBtZIJuJOa7G3a5GGxI/W3MRxx98yn/wszu8DdDzCzosZiTdog1O4dRP1OhBNh\nUFlVTfrwQFjOexIZcodaKjorCD7X2ns0R5OCs0lhqtMWsx7gyxFKOuupsTxQZqE26b3dd8zszR67\ngW+ffs5SL9cien4U0V0iXJWL3Wijhi4X1Cr0cEGJBWR7mx5N3Lyle5s2fFjU3saQKEZZJ7lp1gVm\nmtkKFDQW82bNnZr0DqrbiXAiDCqrqkkfHug8jHvVHDwMfJtYOY0S93TV1pU9e1BNNlNtoRf1AM8o\nK9Jp0nt7EKyQ/u/VsnMBKVJe1y/e3T3xq72GJ+3aAvydZhk1zzrMbEZSuHcQBV91FEIT8tsYbkrF\nNobEBiOnm9nKRGHMHsRK7/MVf1OHJr2DmrQ+bsqgsqqa9OGB+kVn2ZaAc+lcF2VbAjbqQTVZTLVC\nf3vyC05LP2eR9qpgxCAvwJ7kgkJre4+dxPPpablj2Thl6WlFhUJFhSYZexABp1sIy25Sz8ciSPc+\nqNC5eetuKVjFIYQiXxc4xd1Ld73x6Jz4BqJz6K3pYV+3Ne04cqu1u4gdmeo8rE5J8/0N8N90rr2F\nZoBZVUcR98Q6xOr870VC1rzorNaWgFZSFWzV7SMmhalW6GWtQfvVe3uQLGm9O75NJD2tSaEQRF+R\njdx9jpm9gNgBaOMJvO+zgiwe4e6Dck2d49Hvvdf2ZaRg+iGEFb2emc326s6hvcgeUnlF3uthdSSd\nlMyvUL1f6pSSru/nEQ/lDxA7BE0n6geKKnKb9qupuyVg46rgyWKqC4vKWoOO2xcz9zdNem8PEiMC\nkCNEZsIzjL9pSv2yRPuAIppssgEwz93nwIJt1aqybZ71FK2aMipWTU2438w+wdiq0rKCpb2ADdz9\nYTObSZS0T1ihT/AhtSi1g9iYyL83wk2btd0oXMFOIGW51paAPrGq4Elhqi30jCatQYeFgwjr5kbC\naihaVWSWQWY11aHWJhu55eQMM/s5sWJ5DdUBMDGxVVMT7iPSR1+Zfq6qQJ3v7g9DBD1r5D3XwqKV\n8J6kdtRp/DILfZFpB+HuPybSA9/o7ucO4C1qbwmY6FkVPNkMi0J/H5GOWKc16LBwAOObFI25cb1T\nlv5zInXqZ+7ey4VSd5ONouXkTwrkRI5cUc+qRMA5q/Lbq+rvGoy/s0WP7DWBa4gt6Mr4S3JzZHnP\nvTY8r8vuxE5IdTpJLortIB42s/8g3C1fJ3LLz+jDuD23BOziQ8DniBjWDVSnLk8KQ6HQPXZcP6an\n4HDRpEnR3oTVfZCZXUBsqFDWJ7vnJhvpPQdVAfts4QSihe4lRAOp0nYTTbBoI/B2IvX2FCL17WMl\n4rsQbrmtCT96o5TVCmq3Eh5g4HKQHEEYgVnB1A+IIr+Fpc6WgAtw9xvosXHNZDMUCn0RpXaTIne/\nEdjHYi/Uo4ie6pcAn/HxXeDqbLIhFp6lvNP35sdmVrozTUN2IOokLnT3o8zsqgrZZYh0xTnp5+2p\n3j+2kglkdSyqPEba6CbFjZbsx6Du/vIm8mZ2Fx136vJEplITl03fkbKYOLWbFJnZtoRLaR0i6LUn\nkZ55LvCKLvGd6L3Jhlh4ZpjZeu5+jUXvk37lomd9iLLxqmIaFxCWeT4IN2GFTsNWwosw84BfAcea\n2R5ETcGEMbNvuPvHigLmVYFyd19Qi2Jma6DCokWXhi6P9wPHeteuTBZNnLrHrbPJhlh4/gs4ycxW\nIfzc/Up/PYNw46xhZucSaaRlPOip+2M/8OathBdV3geslj7fy4j41MKQdT/dmfob0YzB3W+vys6b\nLKaNjk75vqatx8yeT+x2n886eDY20JpyzOyvjM3RXlDM1q/lcirjfxlwo7tfUyH3KaKW4vrsmLsX\n9XFp+v6X5y1LM7vI3bdY2HGHBTP7PyLF80R3v7aP4/4m1RDUlc9vFL8K8IC7v61f85kIstAnhx8S\nFZxZ1oGeolNHWTFbYfC5KTZ28+lte1QPbkbsPJRVJ44S1v3CssikIk6QVxIVrQeluNR3gO9lKaAL\nwSMWW8/lawjK+jPB2HqYx1mIKt9+IYU+OUzr59JaTJyKYjar/svaNNl8+rnu/vo+vW+eRTEVsTbu\nPj/1UBklqsb/C9jZzE7toYB7kbW/Xamm/B+JzKSVicrg+4gWHFOGFPoAyfXWuNViu7p81sGU7mwi\nBlbM1mTz6WvNbEfGXhdlGz/XZhFNRayNmR0BvA24CPhiWmVNJ/aHnbBCd/eDzexNwEvjR+9V13Ey\n0WtpcyKJ4SQ6q60pQQp9sOR7a2xNdZsAMbn0tZjNJrb59CuIrqJrAn8lro2yLqOiw83E9oGPWPSc\nz6z2ty/MoDZ+44zN3H3vij9Zwd1PNrP3u/vlFvsRTClS6AMk661hZtvTu02AmEQGUMw2kc2nv0m4\nQ34JrAf8Tx/n02amEedtL+AcM/u2u3/b3W9byHGbbpxBltmSKo97VYEPHCn0yaFnmwCxaOOdzadn\nufuCNDqLLenK2AtYv6s517cHO9NWsBudpllvIgLJ/ThvTTfO+DjxEF6X2HCnL4H1hUEKfXJo0iZA\nLIIkX/hbgS1TwyaIIqP1iF2RihhIc65nAc+kOEG2K1S/ssYabZyRUiY3ATCz1dz9b32ax4SRQp8c\narcJEIss5xMbS6xAtHaFSH2rarg1qOZcbecnqWDqSmB94Kc95Gvh7l9J9+k6wEm9ctzT6usx4PlE\nls357t6vFhITQgp9cqjdJkAsmrj7XOAiM7u161dV91i+OdcNxAbQogfufmjqYGrAaZ42xF5YzGw1\nItVzKeAlZra9u3+u4k92JLJazicyY/63H/NYGKTQJwF1RnxW8X3C9zqd2LD7ZqKv/TjSZi2LWpfR\nKSNloXS7V15hZu/p00r3TKJHTF3XySiRg36Pu4+a2fJ9mMNCIYUuRB9x902y16nlw/EV4qIZNw54\n/Hnuvn8D+V8TxUjvTFkxZw1mWvWRQhdicDxI5JiLPpBrPjaDqCFYjVCq/erncq2Z7cDYDS6qCr3+\nQNQzHAssS2TfTClS6EL0kVwL1mlEIdmU7zPZQo4jOmRuTfRPOY3wfS8sr2R8O+uqQq+hS0ee3ltE\nCNGAHYhg2Q7Aa91dRWT9Z013P5DokPkzwjruB2sT2UbrEo3TNjSzm81s6xL5MenIRB/6KUUKXYj+\n8jTweaKnyFvNbKMpnk8bmWFmKwKjqSBrfp/GvQR4adq4Yh2iS+q2dPqld/OQmf3CzPYzs7NJ6ci5\nnaMmHSl0IfrLt4imTUsQCuKoqZ1OK/ks0W9lA6JF8LiNYibIqu7uAO7+F2ANd7+FeEgX8RNiQ5N/\nELnwxxP9m7xEfuDIhy5Ef1nK3S80s/3d3VX9ORCMsMpnADOJh2g/mt3dZWaHE5kr/w7cndwthZ1R\nhzEdWRa6EP3lCTPbBljMzDYmNj4Q/WU3whWyPOH37tfGzB8kgq3bErnoOwEP02m8NvTIQheiv3wY\n+DKwIrA36qw5CO5199v7PWjqD9Pdd+eKfr/PINGeokL0gdxmJuPQZib9IRds3IRwg+Q3BlFPJGSh\nC9Evss1MMlYA5hK+Xm1m0h+863/RhSx0IfqImW1OVA4uRvQGuc3dT5raWYlnCwqKCtFfDiGKUu4k\n8tGnfNMD8exBCl2I/jLf3e+HBUG2eVM8H/EsQgpdiP5yS2rzuoKZ7Qv0PRtDiDKk0IXoL7sRSvw3\nwCPArlM7HfFsQkFRIYRoCbLQhRCiJUihCyFES5BCF0KIliCFLoQQLeH/A7IXYKSUxZ0VAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111a2e320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "classes = fauna['genus'].value_counts()\n",
    "classes[0:30].plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Basic filtering on various fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>family</th>\n",
       "      <th>genus</th>\n",
       "      <th>kingdom</th>\n",
       "      <th>order</th>\n",
       "      <th>page-id</th>\n",
       "      <th>phylum</th>\n",
       "      <th>species</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1317</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ailuridae</td>\n",
       "      <td>ailurusfcuvier1825</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>26441</td>\n",
       "      <td>chordata</td>\n",
       "      <td>afulgens</td>\n",
       "      <td>The red panda (Ailurus fulgens), also called t...</td>\n",
       "      <td>Red panda</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         class     family               genus   kingdom      order  page-id  \\\n",
       "1317  mammalia  ailuridae  ailurusfcuvier1825  animalia  carnivora    26441   \n",
       "\n",
       "        phylum   species                                               text  \\\n",
       "1317  chordata  afulgens  The red panda (Ailurus fulgens), also called t...   \n",
       "\n",
       "          title  \n",
       "1317  Red panda  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fauna[fauna['title']=='Red panda']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing similar and disimilar species"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>family</th>\n",
       "      <th>genus</th>\n",
       "      <th>kingdom</th>\n",
       "      <th>order</th>\n",
       "      <th>page-id</th>\n",
       "      <th>phylum</th>\n",
       "      <th>species</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ursidae</td>\n",
       "      <td>ailuropodamilneedwards1870</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>339548</td>\n",
       "      <td>chordata</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ailuropoda is the only extant genus in the urs...</td>\n",
       "      <td>Ailuropoda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ursidae</td>\n",
       "      <td>NaN</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>5478888</td>\n",
       "      <td>chordata</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ailuropodinae is a subfamily of Ursidae that c...</td>\n",
       "      <td>Ailuropodinae</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ursidae</td>\n",
       "      <td>kretzoiarctosabellaetal2012</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>46316720</td>\n",
       "      <td>chordata</td>\n",
       "      <td>kretzoiarctosbeatrixabellaetal2011</td>\n",
       "      <td>Kretzoiarctos beatrix is an extinct bear from ...</td>\n",
       "      <td>Kretzoiarctos beatrix</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ursidae</td>\n",
       "      <td>melursusmeyer1793</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>399740</td>\n",
       "      <td>chordata</td>\n",
       "      <td>mursinus</td>\n",
       "      <td>The sloth bear (Melursus ursinus), also known ...</td>\n",
       "      <td>Sloth bear</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332</th>\n",
       "      <td>mammalia</td>\n",
       "      <td>ursidae</td>\n",
       "      <td>ursus</td>\n",
       "      <td>animalia</td>\n",
       "      <td>carnivora</td>\n",
       "      <td>51054</td>\n",
       "      <td>chordata</td>\n",
       "      <td>uamericanus</td>\n",
       "      <td>The American black bear (Ursus americanus) is ...</td>\n",
       "      <td>American black bear</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         class   family                        genus   kingdom      order  \\\n",
       "187   mammalia  ursidae   ailuropodamilneedwards1870  animalia  carnivora   \n",
       "190   mammalia  ursidae                          NaN  animalia  carnivora   \n",
       "250   mammalia  ursidae  kretzoiarctosabellaetal2012  animalia  carnivora   \n",
       "407   mammalia  ursidae            melursusmeyer1793  animalia  carnivora   \n",
       "1332  mammalia  ursidae                        ursus  animalia  carnivora   \n",
       "\n",
       "       page-id    phylum                             species  \\\n",
       "187     339548  chordata                                 NaN   \n",
       "190    5478888  chordata                                 NaN   \n",
       "250   46316720  chordata  kretzoiarctosbeatrixabellaetal2011   \n",
       "407     399740  chordata                            mursinus   \n",
       "1332     51054  chordata                         uamericanus   \n",
       "\n",
       "                                                   text                  title  \n",
       "187   Ailuropoda is the only extant genus in the urs...             Ailuropoda  \n",
       "190   Ailuropodinae is a subfamily of Ursidae that c...          Ailuropodinae  \n",
       "250   Kretzoiarctos beatrix is an extinct bear from ...  Kretzoiarctos beatrix  \n",
       "407   The sloth bear (Melursus ursinus), also known ...             Sloth bear  \n",
       "1332  The American black bear (Ursus americanus) is ...    American black bear  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bears = fauna[fauna['family'] == 'ursidae']\n",
    "bears.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>family</th>\n",
       "      <th>genus</th>\n",
       "      <th>kingdom</th>\n",
       "      <th>order</th>\n",
       "      <th>page-id</th>\n",
       "      <th>phylum</th>\n",
       "      <th>species</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>chondrichthyes</td>\n",
       "      <td>lamnidae</td>\n",
       "      <td>carcharodonasmith1838</td>\n",
       "      <td>animalia</td>\n",
       "      <td>lamniformes</td>\n",
       "      <td>43619</td>\n",
       "      <td>chordata</td>\n",
       "      <td>ccarcharias</td>\n",
       "      <td>The great white shark (Carcharodon carcharias)...</td>\n",
       "      <td>Great white shark</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2144</th>\n",
       "      <td>chondrichthyes</td>\n",
       "      <td>lamnidae</td>\n",
       "      <td>isurus</td>\n",
       "      <td>animalia</td>\n",
       "      <td>lamniformes</td>\n",
       "      <td>766132</td>\n",
       "      <td>chordata</td>\n",
       "      <td>ioxyrinchus</td>\n",
       "      <td>The shortfin mako shark (Isurus oxyrinchus), a...</td>\n",
       "      <td>Shortfin mako shark</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3881</th>\n",
       "      <td>chondrichthyes</td>\n",
       "      <td>lamnidae</td>\n",
       "      <td>lamnacuvier1816</td>\n",
       "      <td>animalia</td>\n",
       "      <td>lamniformes</td>\n",
       "      <td>8874336</td>\n",
       "      <td>chordata</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Lamna is a genus of mackerel sharks in the fam...</td>\n",
       "      <td>Lamna</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4420</th>\n",
       "      <td>chondrichthyes</td>\n",
       "      <td>lamnidae</td>\n",
       "      <td>isurus</td>\n",
       "      <td>animalia</td>\n",
       "      <td>lamniformes</td>\n",
       "      <td>5640856</td>\n",
       "      <td>chordata</td>\n",
       "      <td>ipaucus</td>\n",
       "      <td>The longfin mako shark (Isurus paucus) is a sp...</td>\n",
       "      <td>Longfin mako shark</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5269</th>\n",
       "      <td>chondrichthyes</td>\n",
       "      <td>lamnidae</td>\n",
       "      <td>isurus</td>\n",
       "      <td>animalia</td>\n",
       "      <td>lamniformes</td>\n",
       "      <td>47090510</td>\n",
       "      <td>chordata</td>\n",
       "      <td>iplanus</td>\n",
       "      <td>Isurus planus, also known as the hook-tooth ma...</td>\n",
       "      <td>Isurus planus</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               class    family                  genus   kingdom        order  \\\n",
       "134   chondrichthyes  lamnidae  carcharodonasmith1838  animalia  lamniformes   \n",
       "2144  chondrichthyes  lamnidae                 isurus  animalia  lamniformes   \n",
       "3881  chondrichthyes  lamnidae        lamnacuvier1816  animalia  lamniformes   \n",
       "4420  chondrichthyes  lamnidae                 isurus  animalia  lamniformes   \n",
       "5269  chondrichthyes  lamnidae                 isurus  animalia  lamniformes   \n",
       "\n",
       "       page-id    phylum      species  \\\n",
       "134      43619  chordata  ccarcharias   \n",
       "2144    766132  chordata  ioxyrinchus   \n",
       "3881   8874336  chordata          NaN   \n",
       "4420   5640856  chordata      ipaucus   \n",
       "5269  47090510  chordata      iplanus   \n",
       "\n",
       "                                                   text                title  \n",
       "134   The great white shark (Carcharodon carcharias)...    Great white shark  \n",
       "2144  The shortfin mako shark (Isurus oxyrinchus), a...  Shortfin mako shark  \n",
       "3881  Lamna is a genus of mackerel sharks in the fam...                Lamna  \n",
       "4420  The longfin mako shark (Isurus paucus) is a sp...   Longfin mako shark  \n",
       "5269  Isurus planus, also known as the hook-tooth ma...        Isurus planus  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sharks = fauna[fauna['family'] == 'lamnidae']\n",
    "sharks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>family</th>\n",
       "      <th>genus</th>\n",
       "      <th>kingdom</th>\n",
       "      <th>order</th>\n",
       "      <th>page-id</th>\n",
       "      <th>phylum</th>\n",
       "      <th>species</th>\n",
       "      <th>text</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>insecta</td>\n",
       "      <td>tenebrionidae</td>\n",
       "      <td>eleodeseschscholtz1829</td>\n",
       "      <td>animalia</td>\n",
       "      <td>coleoptera</td>\n",
       "      <td>17247312</td>\n",
       "      <td>arthropoda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pinacate beetles, also known as stink beetles,...</td>\n",
       "      <td>Pinacate beetle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1139</th>\n",
       "      <td>insecta</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>animalia</td>\n",
       "      <td>coleoptera</td>\n",
       "      <td>339399</td>\n",
       "      <td>arthropoda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A weevil is a type of beetle from the Curculio...</td>\n",
       "      <td>Weevil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1572</th>\n",
       "      <td>insecta</td>\n",
       "      <td>carabidaelatreille1802</td>\n",
       "      <td>NaN</td>\n",
       "      <td>animalia</td>\n",
       "      <td>coleoptera</td>\n",
       "      <td>417048</td>\n",
       "      <td>arthropoda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ground beetles are a large, cosmopolitan famil...</td>\n",
       "      <td>Ground beetle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1579</th>\n",
       "      <td>insecta</td>\n",
       "      <td>lutrochidaekasapcrowson1975</td>\n",
       "      <td>lutrochuserichson1847</td>\n",
       "      <td>animalia</td>\n",
       "      <td>coleoptera</td>\n",
       "      <td>855394</td>\n",
       "      <td>arthropoda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Lutrochidae is a family of water beetles somet...</td>\n",
       "      <td>Lutrochidae</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1611</th>\n",
       "      <td>insecta</td>\n",
       "      <td>geotrupidaelatreille1802</td>\n",
       "      <td>NaN</td>\n",
       "      <td>animalia</td>\n",
       "      <td>coleoptera</td>\n",
       "      <td>2139411</td>\n",
       "      <td>arthropoda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Geotrupidae (from Greek geos, earth, and trype...</td>\n",
       "      <td>Geotrupidae</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        class                       family                   genus   kingdom  \\\n",
       "799   insecta                tenebrionidae  eleodeseschscholtz1829  animalia   \n",
       "1139  insecta                          NaN                     NaN  animalia   \n",
       "1572  insecta       carabidaelatreille1802                     NaN  animalia   \n",
       "1579  insecta  lutrochidaekasapcrowson1975   lutrochuserichson1847  animalia   \n",
       "1611  insecta     geotrupidaelatreille1802                     NaN  animalia   \n",
       "\n",
       "           order   page-id      phylum species  \\\n",
       "799   coleoptera  17247312  arthropoda     NaN   \n",
       "1139  coleoptera    339399  arthropoda     NaN   \n",
       "1572  coleoptera    417048  arthropoda     NaN   \n",
       "1579  coleoptera    855394  arthropoda     NaN   \n",
       "1611  coleoptera   2139411  arthropoda     NaN   \n",
       "\n",
       "                                                   text            title  \n",
       "799   Pinacate beetles, also known as stink beetles,...  Pinacate beetle  \n",
       "1139  A weevil is a type of beetle from the Curculio...           Weevil  \n",
       "1572  Ground beetles are a large, cosmopolitan famil...    Ground beetle  \n",
       "1579  Lutrochidae is a family of water beetles somet...      Lutrochidae  \n",
       "1611  Geotrupidae (from Greek geos, earth, and trype...      Geotrupidae  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beetles = fauna[fauna['order'] == 'coleoptera']\n",
    "beetles.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a TFIDF Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jeremy/Development/lab/ml/ml/tfidf.py:63: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  self.term_frequency = tf.multiply(csr_matrix(1/np.sqrt(tf.multiply(tf).sum(1))))\n"
     ]
    }
   ],
   "source": [
    "tfidf = TFIDFModel()\n",
    "tfidf.create(fauna, 'text', True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the term count over iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1175cb668>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD3CAYAAADyvkg2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl03Gd97/H3aLesXRrLsi3JspbHllfZjpfExA7Z40Da\nQIFSUkpJOVwCpznlnra3DTe393BOb9skt0AbSENJeimUQELYkxiIY4yzOE4sr9Jjy5YlW5K1S6N9\nmZn7x4wVmcTapZn5zed1Ts6Z+c0z0vcrOfPRb3sel9/vR0REBCAm1AWIiEj4UCiIiMgYhYKIiIxR\nKIiIyBiFgoiIjIkLdQGz0draM6tLpzIzk+ns7J+rcsKO0/sD9egETu8Pwq9HtzvVda3XonpPIS4u\nNtQlzCun9wfq0Qmc3h9EVo9RHQoiInI1hYKIiIxRKIiIyBiFgoiIjFEoiIjIGIWCiIiMUSiIiMiY\niL55TUQkWvj9fjp7hjhzsYvzTR7WFWWzoTh7zr+PQkFEJMz4/X7augepbfJQ39xLfXMPdc099PSP\njI3pGxhVKIiIOFF33zB1l3uob+6hpqGbs5e6GBjyXjUmJz2JMpNBYW4qqwszKcpLnZdaFAoiIgto\neMRLY3sf9c29VNV1cuFyD80dV8+LlJu5iHVFqRTlpVGYm0J+biopi+IXpL4JQ8EYEw98C1gJJAJf\nBi4BPwXOBod93Vr7jDHmYWAvMAo8aK09bIwpAZ4G/MBJ4AFrrW86Y+ewVxGRBTcwNMqRqmYOn2ik\nuj4QAuNXQV6UGMvaoiyKl6VRmJtK0bI0MlISQ1bvZHsKnwDarbX3GWOygaPA/wYes9Y+emWQMWYz\nsBvYDuQDzwHXAY8BD1lrXzHGfAO4xxhTN9WxwPNz16qIyPwbHB6l7nIPJ2s7qK7rpLapB18wBWJj\nXKzKS6NgaSor3CmU5WewLDsZl+uak5YuuMlC4QfAs+OejwJbAGOMuYfA3sKDwC5gn7XWD9QbY+KM\nMe7g2APB974A3AbYaYxVKIhIWOvqHaLmUje2votzjd1cbOnF6wuEQIzLRVFeKhWrcylwJ1O6PIPE\nhPCeMXXCULDW9gIYY1IJhMNDBA4jfdNa+5Yx5m+Bh4EuoH3cW3uAdMAV/PAfvy1tGmMnlJmZPOsp\nad3u+TlZEy6c3h+oRyeIpP4aW3s5XdvByfNtnD7fQVN739hrcbExrFqeTnlRNmtXZbOxNIfkpIU5\nFzBXJj3RbIzJJ/AX++PW2u8aYzKstV3Bl58Hvgb8GBj/W00lEBS+99jmmcbYCc120Qq3O5XW1p5Z\nfY1w5vT+QD06QST019Day9GzbRyraeNco2ds+6LEODYUZ1O6Ip3SFRmsWpZGXOw79wT39QzS1zMY\ndj1OFMKTnWjOBfYBn7fW/jq4+SVjzBestYeBm4G3gEPAPxpjHgFWADHW2jZjzFFjzB5r7SvAncB+\noGYaY0VEFpzX56PmUjenL3RyrKaN+pZeAFwuWF2QweYyN2X5GaxwpxATEz7nA+bCZHsKfwNkAl8y\nxnwpuO0vgH82xgwDl4HPWGs9xpiDwGsEps54IDj2i8CTxpgEoAp41lrrnerYOelQRGSKTtV2cKCy\nger6LnoHAjeKxbhcbCrJoaI0h4oy94JdGhoqLr9/Vssch9Rs12gOt126ueb0/kA9OkEo+/P6fJxr\n8HD2UhfnGjxU1rQBkJ6SQEWpmw2rsinLT5/1eYFw+x1OtEazbl4Tkaji6R/mTH0XZy918/rpy1dN\nHZGblczdOwu5ft3SsLpMdCEpFETE8Zo7+nn57QbsxU4aWvvGLhl1uWBDcTa71udRlJdGdnpSiCsN\nPYWCiDhOS9cAx2vasBe7uNDkod0zBIALKFqWxqaSHEpXpFOUl0ZCfHjfN7DQFAoi4gj1zT28UdVM\n5dk2mtrfuVw9LTmejcXZFC1L445tBQqBSSgURCQiDQ6Pcqq2k6q6DuzFLhpaAzeRJcTHsKE4m40l\nOaxflUV2WlLUnh+YCYWCiESE4REv5xq6OXmhg7MXu6lt8oydG4iPi2FNYSbv25BHRZmbRO0NzJhC\nQUTCWmNbH9/fX8PJ8x1jE8sBFOSmsH5VNhuLcyhcmkp8nFYXngsKBREJKz6/n9omD0fPtHH8XBuX\ngoeFlmQsYn1xNuWFmZiCTJKT9PE1H/RTFZGQa+se4OzFbqrrOzl+vp3u3mEgMNX0ulVZXGeWcMP6\nPMdNKRGOFAoisuD8fj/1zb0csS2crO2g7vI7d/umLIrnhvVL2VzqprwoS+cHFphCQUQWxMDQKNX1\nnRyraef4uTa6xu0NrF2ZydqibErz0ylamqY9ghBSKIjIvBgcHqWqrpPTtZ2cruu46t6BlEXxXL9u\nKRuKs7lpWyF9PYMhrFTGUyiIyJzx9A1zuKqZ1083U9vkGVuLODE+ltUFGZSsSGdDcQ6r8t7ZG0hO\nilcohBGFgojM2NCwl6q6Tk5d6KDmUjf1zT34CUwnUbwiHZOfwbqiLIqXp1+1+IyEL4WCiEzZwNAo\nZy52UVXXSVVdYHK58YvSl+VnUFHmZvuaJaSnJIa4WpkJhYKITKh3YITj59p49eRlzlzsYtT7zl3E\nq5anYfIzWLsyi+LlacTPcs10CT2Fgoi8y6jXx7Gadg4eb7zqTuL8JSlsLMmmLD8Dk5+hEHAghYKI\njOnwDHKgspFXKhvGFp9ZlrOY7eW5bFu9hNys5BBXKPNNoSAiADS19/H3//k2vQMjxMfFsGt9Hjdu\nXEbx8jTNMhpFFAoiUay7d4hDJy9zpLqFC8G7igtzU/nixzY5foF6eW8KBZEoMjzi5dVTl6mu6+Ts\npW46ewIrksXGuFhTmMlNFcupKMshNkaXj0YrhYKIw7V0DXDqfDvHzrVTc6mb/qFRANJTEli/Kpu1\nKzO5YUMei5O0ZyAKBRFH6h8c4ddvN3DwWCNt3e/cLZydlsj165by/i0ryM1cpHMF8i4KBRGH6Ood\noqquk5PnO3j7bCtDw15iY1xsLnNjCjKoKMkhJ2NRqMuUMKdQEIlQfYMjVNd1UV337gnnstISuWXL\nCm6qWE5WWlIIq5RIo1AQiRBDI17OXuzidHCKifrLgXmGILBY/bpVWawpzGRNYSYFuanE6NCQzIBC\nQSSMeX0+Xj/ZxIuHajl+vp2RUR8QuFqoND9jLARWLUvThHMyJxQKImFmYGiUk7UdHKlu4VRtx9jV\nQrlZyWwuzWHNykxKV2RoRTKZFwoFkTBQVdfJqyeaONfoobmjf+ywUE56Eru3rGBzcTbFy9NDWqNE\nB4WCSIj1Dozw2DOVeH1+khJiWV2YSfHydLYaN/lLUliyJI3W1p7Jv5DIHFAoiCwwv99PY3s/J8+3\nc6S6hfONHvzA0qxkvnz/dq1PLCE1YSgYY+KBbwErgUTgy8Bp4GnAD5wEHrDW+owxDwN7gVHgQWvt\nYWNMyWzHzmm3IiHS3TtEdX0XZy52cexcGx2ewPQSLheUrEhn3apsdq3PUyBIyE22p/AJoN1ae58x\nJhs4ClQCD1lrXzHGfAO4xxhTB+wGtgP5wHPAdcBjsxkLPD/H/YosGJ/Pz7FzbRyobOT4ufax7YsS\n49hq3JQXZbGpJIcMrVAmYWSyUPgB8Oy456PAFuBA8PkLwG2ABfZZa/1AvTEmzhjjnoOxE4ZCZmYy\ncbNc5MPtTp3V+8Od0/uD8Ouxu3eInx48z/63L9HSEbihbM3KLK4rz2VjqZuiZdNfoSzcepxrTu8P\nIqfHCUPBWtsLYIxJJRAODwGPBD/QAXqAdCANaB/31ivbXbMcO6HOzv7JhkzI7U519Ak8p/cH4dVj\nh2eQ105d5mev1jE04sXlgi1lbu7aWcjKpalj8wx1TfPfbTj1OB+c3h+EX48TBdSkJ5qNMfkE/mJ/\n3Fr7XWPMP457ORXoAjzBx7+73TfLsSJhraG1l8qaNo6ebeN8owcInCeoKM3h03vLSU7StRwSWSY7\n0ZwL7AM+b639dXDzUWPMHmvtK8CdwH6gBvhHY8wjwAogxlrbZoyZ7ViRsOLz+TlzsYs3bQunL3TS\nHDw85ALKV2ay1Sxh6+olWqBGItZkf8b8DZAJfMkY86Xgtj8HvmqMSQCqgGettV5jzEHgNSAGeCA4\n9ovAkzMdOycdiszS0IiXs5e6qKrr5M2qlrGpqBPjY9lUksN1a5ZQXphJuk4YiwO4/H7/5KPCVGtr\nz6yKD7fjfHPN6f3B/PU4MurjWE0bb51p5VhNG4PDXgDiYmPYUZ7LzrW5lBVkLMgKZU7/PTq9Pwi/\nHt3u1Gte+6wDniJBV+YcOlbTRuXZtrE5h7LTkti9aRmrCzJZXZipOYfE0RQKEvWa2vs4eKyJXx65\niNcX2PlMTY7njm0FbCtfQmFuqlYok6ihUJCo1OEZ5NDJy7xV3UJ9Sy8AcbEu9lQs5/q1SylalqrF\n6yUqKRQkanT3DXOkuoXKs62cruvE7w+sS7CxOJuNJTnsWJtLUoL+l5Dopv8DxNF8fj9nL3bx2+NN\nvHrqMleuqyhcmsrOtUu5ft1SXT4qMo5CQRzpyknj5w6co6VzAIDM1ERu31bAVuPWusUi16BQEEep\nbfLw00MXOF3XwfBI4Cb5itIcbt6ygtUFmZqFVGQSCgVxhCPVLbzwRj21TYGpJnLSk9ixdilbytwU\nLo2MichEwoFCQSJa38AIT/zkFG+cbgZgbVEWuzcuY+vqJSGuTCQyKRQkIrV2DXDweCP7jzbSNzBC\nblYyn7i1jLVFWaEuTSSiKRQkovh8fva9eZHv768BYFFiLHftKOSeXUXEx+m+ApHZUihIRBgZ9XK4\nqoUX36inoa2P2BgXd1+/kj+6q5xez0CoyxNxDIWChLX27kFeqWzgQGUjvQMjuFyw1bj5g5tKcGcs\nYlFiHL2hLlLEQRQKEpYutvTyo4Pnqaxpw++HxUlx3LmjgJs2LScnY1GoyxNxLIWChA2fz8/x8+28\n/PYlTp3vwA8U5KZw85YVbF+TS4JmJxWZdwoFCbmBoVF+9dYl9r99ia7eYQCK8tK4e2chm0pzNEOp\nyAJSKEjI+Hx+Dh5v5PmDtXj6holxudi1IY+bKpZTlJcW6vJEopJCQUKi7nIPT/2iivqWXuLjYrh9\nWz53bCvQkpYiIaZQkAXV3j3IS4fr2X+0Aa/PT0VpDn90a5kmqBMJEwoFWRA1l7r5yau1YyeQ3RlJ\nfOzmUipK3aEuTUTGUSjIvPL0D/ObykZ++JvzABQvS+OG9Xlcv26priYSCUMKBZkX3b1D/Pz1Og4e\na2JoxEtCXAx/eEspuzctD3VpIjIBhYLMqeERL79++xI/f7WO/qFRUhbFc/f1hbxvwzLSFieEujwR\nmYRCQebMmYtd/MeL1TS195MQF8NHbirhlq0riIvVRHUikUKhILPWPzjC9/fX8JtjTbiA3ZuW8aHd\nxVr7WCQCKRRkxjz9wxyobOSF1+sYHPay3L2YT96xmpLl6aEuTURmSKEg0+b3+3mlspH/+tVZRr0+\nEuNj2buzkA/eoDUNRCKdQkGmxdM/zL88d4Kahm7i42L4gz3F7N60jOQkHSoScQKFgkxZTUM3T/z4\nJO2eIcpXZnLfbYbcrORQlyUic0ihIJPq7h3ixcP17HvzIvjh1q35fOzmEs1eKuJAUwoFY8x24B+s\ntXuMMZuBnwJngy9/3Vr7jDHmYWAvMAo8aK09bIwpAZ4G/MBJ4AFrrW86Y+eqUZm+3oERfnroAq9U\nNjAy6iMrLZE/u7scU5AZ6tJEZJ5MGgrGmL8E7gP6gps2A49Zax8dN2YzsBvYDuQDzwHXAY8BD1lr\nXzHGfAO4xxhTN9WxwPNz0qVMW3VdJ0/85BTdfcNkpyVy145Cdm3IIz5OU1OIONlU9hTOAfcC3w4+\n3wIYY8w9BPYWHgR2AfustX6g3hgTZ4xxB8ceCL7vBeA2wE5jrEJhgQ0Ne3n6xWreON0MwF07Crln\nl64qEokWk4aCtfY5Y8zKcZsOA9+01r5ljPlb4GGgC2gfN6YHSAdcwQ//8dvSpjF2QpmZycTN8i9X\ntzt1Vu8Pd9Ppz9Z18JVnKrnY3EPh0lQ+9+GNlBdlz2N1c8Ppv0Nwfo9O7w8ip8eZnGh+3lrbdeUx\n8DXgx8D4jlMJBIXvPbZ5pjF2Qp2d/dOt/SpudyqtrT2z+hrhbKr9tXQN8LNDF/jtiSYAbqpYzkff\nX0JCfGzY/3yc/jsE5/fo9P4g/HqcKKBmckzgJWPMtuDjm4G3gEPA7caYGGNMARBjrW0Djhpj9gTH\n3gkcnOZYmUd+f2A5zL/9t9f57YkmlrsX81cfr+C+242mtRaJUjPZU/hvwL8YY4aBy8BnrLUeY8xB\n4DUCQfNAcOwXgSeNMQlAFfCstdY71bEzbUomNzA0ylO/qOKIbSU+LoZP3FbGrg15xMbo3IFINHP5\n/f7JR4Wp1taeWRUfbrt0c+1a/Y2Mevm7p4/Q2NZH6Yp07r+7HHfGohBUOHtO/x2C83t0en8Qfj26\n3anXvMlIN69FofqWXhrb+thUksPnfn+dprYWkTH6NIgy3X3DPPNyDQDFy9MUCCJyFe0pRAmf38+R\n6ha++8szePpHyMtOZvua3FCXJSJhRqEQJZ76RRWHTlwmNsbF3dev5IM3rNRegoi8i0LB4UZGfXzn\nl2c4dOIyK9wpPHDvOnIzNbOpiLw3hYKD9fYP88j3jnL2UjcFS1L4woc2kJ2eFOqyRCSMKRQcqr17\nkH/+98M0tPaydfUS7t+7RjekicikFAoOVN/cw9eeO0G7Z5Bbt+bz0feXEBOjtQ9EZHIKBYepu9zD\n33/nLYZHfHz8NsPNFcu0GI6ITJkuP3GQuss9/N/vVzI84uMzHyjnD29frUAQkWnRnoJDnLnYxVee\nPc7g0Ch/dGsZO9YuDXVJIhKBFAoOcLGlNxAIw6Pc/4FydioQRGSGFAoR7lRtB4//6CQDQ6N8eu8a\nBYKIzIpCIYLtP9rAd/adISYG/uzucnauUyCIyOwoFCKQ1+fje7+q4ddvXyJlUTyfv3c9ZfkZoS5L\nRBxAoRBhBodH+befnKaypo287GQe/IONEbsWgoiEH4VCBOkbHOGf/uso9c29rC7I4PP3biA5Sb9C\nEZk7+kSJED39w/zzD45R39zLrvV5/PEdRrOcisicUyhEgDMXu/j6j07S3TfMzrVL+ZO7VhOjm9JE\nZB4oFMKY3+/nlaMNPLO/Bq/Xz703ruLOHQUKBBGZNwqFMDXq9fHtlywHjzexOCmOz95TzqaSnFCX\nJSIOp1AIQ36/n2/9vIrXTzdTuDSVL9y7nqw0rYMgIvNPoRCGfv5aHa+fbqYoL5X//rEKFiXq1yQi\nC0OXr4SZo2da+eFvzpO2OIHPfHCtAkFEFpRCIYz4fH5+8UYdAPfvXaO1lEVkwenP0DDh6R/mqZ9X\nca7BQ1FeGuVFWaEuSUSikEIhDDS09vLI9yrp7htmdUEGn/v99brsVERCQqEQYjUN3Xz12eP0Dozw\n4T3F3LGtQOspi0jIKBRC6Eh1C0/+7DRer59P3mHYvWl5qEsSkSinUAgRW9/J1390koT4WB748Do2\nFOvGNBEJvSmFgjFmO/AP1to9xpgS4GnAD5wEHrDW+owxDwN7gVHgQWvt4bkYO3etho/OniGe+kU1\nfuAvPrqR0hVaC0FEwsOkl6QaY/4S+CZw5Zbax4CHrLXvA1zAPcaYzcBuYDvwMeBf52Ls7NsLPy1d\nA/yf77xFS9cAd+4oUCCISFiZyn0K54B7xz3fAhwIPn4BuAXYBeyz1vqttfVAnDHGPQdjHcXTN8yj\n3ztKa9cgd+4o4MO7i0NdkojIVSY9fGStfc4Ys3LcJpe11h983AOkA2lA+7gxV7bPduyEMjOTiYuL\nnWzYhNzu1Fm9f6qGR7z80/cqae0a5CO3lHHfnWsW5PsuVH+hpB4jn9P7g8jpcSYnmscf508FugBP\n8PHvbp/t2Al1dvZPp+53cbtTaW3tmdXXmAqf388TPz5F1YUOtq1Zwu1bli/I912o/kJJPUY+p/cH\n4dfjRAE1k2kujhpj9gQf3wkcBA4BtxtjYowxBUCMtbZtDsY6wnOvnOPN6hZKV6Tz6b1rcOnGNBEJ\nUzPZU/gi8KQxJgGoAp611nqNMQeB1wgEzQNzMXamTYWTo2daeeGNepZmJfOFD20gfpaHu0RE5pPL\n7/dPPipMtbb2zKr4+d6lG/X6+Pv/fIvaph7+6uMVmILMefte7yXcdlnng3qMfE7vD8KvR7c79ZqH\nKzRL6jz64YHz1Db1sG5VFmX5uvRURMKfQmGe1F3uYd+bF8lOS+Jzv7dO5xFEJCIoFOZBZ88Q//r8\nCXx+P398hyEpQbOJiEhkUCjMsd6BER59ppK27kE+eMNK1q/KDnVJIiJTplCYY99+ydLY1sf7Ny/n\nnl1FoS5HRGRaFApzaGBolKNn28hKS+Tjt5TpPIKIRByFwhx65uWzjHp93LhhmRbKEZGIpFCYI8dq\n2jh4rInczEXsvb4w1OWIiMyIQmEO9A6M8PSL1cTGuvj03nJiY/RjFZHIpE+vWRr1+nj8+RN09w7z\ngRuKKFkx6eSuIiJhS6EwSz85VEt1fRcVpTns3anDRiIS2RQKs1B3uYcX37hIekoCf/aBcmJ0tZGI\nRDiFwgx19gzxlWeP4fX6+OTtq3XXsog4gkJhBka9Pr767HG6eof5yPtL2FSaE+qSRETmhEJhBn50\nsJa65h62rVnCbdflh7ocEZE5o1CYpoa2Pl46XE/64gQ+cZvRXcsi4igKhWnw+nx882en8foCs5+m\nLIoPdUkiInNKoTANR8+0UXe5hx3luVSUukNdjojInFMoTJHf7+elw/UAfOCGlaEtRkRknigUpujs\npW7ONXrYVJJDXvbiUJcjIjIvFApT4Pf7+cH+GgDu2F4Q4mpEROaPQmEKqus6OdfooaI0h7L8jFCX\nIyIybxQKU/Dy0QYA7tyhuY1ExNkUCpPo6R+m8mwbK9wpFC9LC3U5IiLzSqEwiZffbsDr87NrQ55u\nVBMRx1MoTMDn83PweCNJCbHcuDEv1OWIiMw7hcIETtZ20OEZYnt5rmZBFZGooFCYwP63LwGwe9Oy\nEFciIrIwFArX0NU7xInzHSx3L2blUp1gFpHooFC4hjerW/D5/dy4UXsJIhI9FArX8GZ1Cy5g2+ol\noS5FRGTBzPjsqTHmKNAdfFoLPAF8BRgF9llr/84YEwM8DmwEhoD7rbU1xpgdUx070/pmo6m9j5pL\n3awpzCQ9JTEUJYiIhMSMQsEYkwRgrd0zblsl8CHgPPBzY8xmYCWQZK3dGQyCR4F7gG9MY+yCe+3U\nZQDep8tQRSTKzHRPYSOQbIzZF/wa/wtItNaeAzDGvATcDOQBLwJYa183xmw1xqRNdeyMu5qlU7Wd\nuFywsVhrL4tIdJlpKPQDjwDfBEqBF4Cuca/3AKuANN45xATgDW7zTGWsMSbOWjt6rSIyM5OJi4ud\nYQsBbnfqVc8vtfRQ2+ShosxNwYrMWX3tcPC7/TmReox8Tu8PIqfHmYbCGaDGWusHzhhjuoGsca+n\nEgiJ5ODjK2IIBELqVMZOFAgAnZ39Myw/wO1OpbW156ptPz1wDgicYP7d1yLNe/XnNOox8jm9Pwi/\nHicKqJleffSnBI75Y4xZRuADvc8YU2yMcQG3AweBQ8BdwXE7gBPWWg8wPJWxM6xtxnw+P6+ebGJR\nYhwVpTp0JCLRZ6Z7Cv8OPG2M+S3gJxASPuA7QCyBK4reMMa8CdxqjHkVcAGfCr7/s9MYu2BOX+ig\nq3eYPZuWkRA/u8NSIiKRaEahYK0dBj7+Hi/t+J1xPgIB8Lvvf32qYxfSb080AXDDel11JCLRSTev\nBfUNjvD2mTbyspNZpXUTRCRKKRSCjlS3MOr1ccN6rZsgItFLoRB04nwHAJvL3CGuREQkdBQKgM/v\np+ZSF5mpieRmLgp1OSIiIaNQAGoudePpH2F1QaYOHYlIVFMoAJU1bQBsL9eMqCIS3RQKBPYUYlwu\nTH7kT2shIjIbUR8KA0Oj1DZ5KFyaQmKCblgTkegW9aFw5mIXXp+f8pVZkw8WEXG4qA+FqrpOAIWC\niAgKBeqbAzMXFuVFxrS2IiLzKapDwev1cb7Rw3L3YpISZrwyqYiIY0R1KFzu6Gd41MfKXO0liIhA\nlIdCQ2svAEuykkNciYhIeIjqUKi+EJjvqFB7CiIiQJSHwoWmwFLROsksIhIQ1aHQ0tHPosQ4UpMT\nQl2KiEhYiNpQ8Pn9NLX3k5OeFOpSRETCRtSGQk/fMMMjXpZkaKpsEZErojYULnf0A7BE6yeIiIyJ\n2lBo6RwAIFeXo4qIjInaUOjuGwYgIyUxxJWIiISP6A2F3iuhoCuPRESuiNpQ6OobAiBdewoiImOi\nNhQ6PEPExcaQlhwf6lJERMJG1IZCT/8w6SkJuFyuUJciIhI2ojgURkhfrENHIiLjRWUojHp9DI14\nSdGhIxGRq0RlKAwMjQKweJFCQURkvKgMhb7BQCikKBRERK4SVmtQGmNigMeBjcAQcL+1tmauv0/f\nwAgAaYt1j4KIyHjhtqfwe0CStXYn8NfAo/PxTYZGvAAkJYZVJoqIhFy4hcIu4EUAa+3rwNb5+CYj\noz4AkhJi5+PLi4hErHD7UzkN6B733GuMibPWjr7X4MzMZOLipv/BXpEQx7byFraszsXtdvaqa07v\nD9SjEzi9P4icHsMtFDzA+J9czLUCAaCzs3/G3+izHyzH7U6ltbVnxl8j3Dm9P1CPTuD0/iD8epwo\noMLt8NEh4C4AY8wO4ERoyxERiS7htqfwPHCrMeZVwAV8KsT1iIhElbAKBWutD/hsqOsQEYlW4Xb4\nSEREQkihICIiYxQKIiIyRqEgIiJjFAoiIjLG5ff7Q12DiIiECe0piIjIGIWCiIiMUSiIiMgYhYKI\niIxRKIiIyBiFgoiIjFEoiIjImLCaJXUhGGNigMeBjcAQcL+1tia0VU2dMWY78A/W2j3GmBLgacAP\nnAQesNb6jDEPA3uBUeBBa+3h6Yxd8KYAY0w88C1gJZAIfBk4PdWaw70/AGNMLPAkYAAvganhXTio\nxyuMMUtG30EAAAADFklEQVSAt4BbCdT1NA7q0RhzlHdWiawFngC+QqC+fdbav7vWZ01wrZgpjV3Q\npoKicU/h94Aka+1O4K+BR0Ncz5QZY/4S+CaQFNz0GPCQtfZ9BD5c7jHGbAZ2A9uBjwH/OoOxofAJ\noD1Y353Av+Cs/gA+AGCtvQH4nwRqdlqPVwL+CWAguMlRPRpjkgCstXuC/30K+AbwcQLrzG8P1nyt\nz5rpjF1w0RgKu4AXAay1rwNbQ1vOtJwD7h33fAtwIPj4BeAWAv3ts9b6rbX1QJwxxj3NsaHwA+BL\n456P4qz+sNb+CPhM8Gkh0IzDegx6hMAHX2PwudN63AgkG2P2GWNeNsbcCCRaa89Za/3AS8DNvMdn\njTEmbapjF7yroGgMhTTe2e0D8BpjIuIwmrX2OWBk3CZX8B8WQA+Qzrv7u7J9OmMXnLW211rbY4xJ\nBZ4FHsJB/V1hrR01xvwH8DUCfTqqR2PMnwCt1tqXxm12VI9AP4Hgu53AomBPBbddca26vcFtnqmM\nDdXnUjSGggcYv2p1jLV2NFTFzJJv3ONUoIt393dl+3TGhoQxJh/YD3zbWvtdHNbfFdbaTwJlBM4v\nLBr3khN6/FMCS+q+AmwC/h+wZNzrTujxDPCfwT2XMwQ+zLPGvX6tumPeY9s1x4bqcykaQ+EQcBdA\n8ITPidCWMytHjTF7go/vBA4S6O92Y0yMMaaAwD+utmmOXXDGmFxgH/BX1tpvBTc7pj8AY8x9xpj/\nEXzaT+AD8IiTerTW3mit3W2t3QNUAn8MvOCkHgkE36MAxphlQDLQZ4wpNsa4COxBXKn7qs8aa60H\nGJ7K2IVt6R0Rcdhkjj1P4C+ZVwmcyPpUiOuZjS8CTxpjEoAq4FlrrdcYcxB4jUDoPzCDsaHwN0Am\n8CVjzJVzC38OfNUh/QH8EHjKGPMbIB54kECtTvkdXouT/p0C/DvwtDHmtwSukvpTAgH/HSCWwPmP\nN4wxb/LenzWfncbYBaeps0VEZEw0Hj4SEZFrUCiIiMgYhYKIiIxRKIiIyBiFgoiIjFEoiIjIGIWC\niIiM+f+qNooUmRnACgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1175cbef0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "term_count_series = pd.Series(tfidf.term_count_over_iterations)\n",
    "term_count_series.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top words for Asian black bear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bears         1.340566\n",
      "bear          0.684123\n",
      "black         0.514237\n",
      "asian         0.292425\n",
      "to            0.140059\n",
      "will          0.125510\n",
      "and           0.124383\n",
      "have          0.105714\n",
      "asiatic       0.098804\n",
      "ursus         0.097197\n",
      "their         0.096326\n",
      "they          0.095895\n",
      "are           0.095319\n",
      "thibetanus    0.090546\n",
      "as            0.087520\n",
      "than          0.086124\n",
      "tiger         0.085674\n",
      "hunting       0.085575\n",
      "india         0.084920\n",
      "cubs          0.083660\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "abb_idx = fauna[fauna['title']=='Asian black bear'].index[0]\n",
    "print(tfidf.get_sorted_terms_for_document(abb_idx).head(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top words for Sun bear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sun           1.032571\n",
      "bear          0.956376\n",
      "bears         0.470372\n",
      "bile          0.286178\n",
      "malayanus     0.238481\n",
      "helarctos     0.225075\n",
      "honey         0.202648\n",
      "malayan       0.184175\n",
      "claws         0.176605\n",
      "and           0.169571\n",
      "kalimantan    0.157548\n",
      "are           0.149519\n",
      "mound         0.145788\n",
      "their         0.141885\n",
      "southeast     0.135953\n",
      "they          0.134890\n",
      "termites      0.129149\n",
      "for           0.117258\n",
      "sabah         0.115186\n",
      "paws          0.112623\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "sb_idx = fauna[fauna['title']=='Sun bear'].index[0]\n",
    "print(tfidf.get_sorted_terms_for_document(sb_idx).head(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computer the Euclidean distance between the Asian black bear and the Sun bear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.898834915880105"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "asian_black_bear = tfidf[abb_idx].toarray()\n",
    "sun_bear = tfidf[sb_idx].toarray()\n",
    "np.linalg.norm(asian_black_bear - sun_bear)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top words for Flatfish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flatfishes        0.612517\n",
      "flatfish          0.460255\n",
      "side              0.248945\n",
      "families          0.223232\n",
      "soleidae          0.182676\n",
      "pleuronectidae    0.180269\n",
      "fish              0.173355\n",
      "bottom            0.168934\n",
      "head              0.148232\n",
      "hybrids           0.138811\n",
      "right             0.138650\n",
      "eyes              0.135971\n",
      "turbots           0.131364\n",
      "amphistium        0.131364\n",
      "dextral           0.128280\n",
      "achiridae         0.128280\n",
      "cynoglossidae     0.128280\n",
      "sinistral         0.125760\n",
      "platichthys       0.123630\n",
      "some              0.122857\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "ff_idx = fauna[fauna['title'] == 'Flatfish'].index[0]\n",
    "print(tfidf.get_sorted_terms_for_document(ff_idx).head(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top words for Hagfish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hagfish          2.021367\n",
      "slime            0.521616\n",
      "lampreys         0.281539\n",
      "myxine           0.275481\n",
      "vertebrates      0.240058\n",
      "craniata         0.197047\n",
      "duct             0.196514\n",
      "gill             0.164350\n",
      "to               0.163254\n",
      "knot             0.151613\n",
      "as               0.150045\n",
      "cyclostomata     0.141909\n",
      "gnathostomata    0.137740\n",
      "mucus            0.117616\n",
      "skin             0.112077\n",
      "their            0.110834\n",
      "opening          0.106651\n",
      "have             0.105548\n",
      "and              0.098852\n",
      "mesonephric      0.098523\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "hf_idx = fauna[fauna['title'] == 'Hagfish'].index[0]\n",
    "print(tfidf.get_sorted_terms_for_document(hf_idx).head(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.7082676133675863"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Euclidian distance between the Flatfish and the Hagfish\n",
    "flatfish = tfidf[ff_idx].toarray()\n",
    "hagfish = tfidf[hf_idx].toarray()\n",
    "np.linalg.norm(hagfish - flatfish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.8988349158801048"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Euclidian distance between the Sub bear and the Hagfish\n",
    "np.linalg.norm(sun_bear - asian_black_bear)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}