NSF_Bokeh.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualizing high-dimensional data with Bokeh and PCA/t-SNE\n",
    "\n",
    "Using the toy digits dataset available from scikit-learn, this demonstrates how to make use of PCA & t-SNE to project high dimensional data (in our case, NSF abstracts) down to two dimensions, allowing us to reasonably visualize and explore how the data is distributed (**assuming the projections actually worked the way we expected them to**). This also uses Bokeh, a library that lets us interact with the visualizations from within the notebook. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import bokeh\n",
    "from bokeh.io import output_notebook, show\n",
    "from bokeh.plotting import figure, ColumnDataSource\n",
    "from bokeh.models import HoverTool, BoxSelectTool, BoxZoomTool, WheelZoomTool\n",
    "import numpy as np\n",
    "import gensim\n",
    "import random\n",
    "from gensim.models.doc2vec import Doc2Vec, FAST_VERSION, LabeledSentence\n",
    "import csv\n",
    "import pickle\n",
    "import nltk\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "import matplotlib.pyplot as plt\n",
    "from numpy.linalg import norm\n",
    "import re\n",
    "from matplotlib import colors\n",
    "import six\n",
    "\n",
    "assert gensim.models.doc2vec.FAST_VERSION > -1, \"this will be painfully slow otherwise\"\n",
    "\n",
    "output_notebook(bokeh.resources.INLINE)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "text_fpath = '/Users/Bartley/Desktop/NSF_abstracts_subset.csv'\n",
    "\n",
    "extra_abbreviations = ['dr', 'vs', 'mr', 'mrs', 'prof', 'inc', 'i.e', 'e.g', 'ph.d']\n",
    "sentence_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "sentence_tokenizer._params.abbrev_types.update(extra_abbreviations)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "min_len = 2\n",
    "patt = r'[,\\.-_]+$'\n",
    "size = 100\n",
    "min_count = 50\n",
    "num_abs = 500\n",
    "\n",
    "docs = []\n",
    "div_dir = []\n",
    "abs_uid_map = {}\n",
    "directorates = []\n",
    "texts = []\n",
    "uids = []\n",
    "for iter_ in xrange(1):\n",
    "    with open(text_fpath, 'r') as fp:\n",
    "        csvReader = csv.reader(fp)\n",
    "        csvReader.next()\n",
    "        abs_ix = 0\n",
    "        uid = 0\n",
    "        rand_abs = []\n",
    "        for row in csvReader:\n",
    "            rand_abs.append(row[0])\n",
    "        random.shuffle(rand_abs)\n",
    "\n",
    "        fp.seek(0)\n",
    "        for row in csvReader:\n",
    "            abs_ = row[2].decode('latin-1', 'backslashreplace').encode('ascii', 'backslashreplace')\n",
    "            abs_id = row[0]\n",
    "            if abs_id not in rand_abs:\n",
    "                continue\n",
    "            sents = nltk.sent_tokenize(abs_)\n",
    "            sent_ix = 0\n",
    "            texts.append(abs_)\n",
    "            docs.append(LabeledSentence(words=[re.sub(patt, '', x) if len(re.sub(patt, '', x)) >= min_len else '' for x in abs_.lower().split()], tags = ['%d' % (uid)]))\n",
    " \n",
    "            uids.append(uid)\n",
    "\n",
    "            uid += 1\n",
    "            abs_ix += 1 \n",
    "            div_dir.append(row[14:16])\n",
    "            directorates.append(row[14])\n",
    "            if abs_ix > num_abs:\n",
    "                break\n",
    "\n",
    "\n",
    "directorates_uniq = list(set(directorates))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Doc2Vec(docs, size=size, window=5, min_count=min_count, dm=0,dm_concat=0, workers=4, dbow_words=1, sample=1e-5)\n",
    "\n",
    "print len(model.vocab)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for epoch in xrange(0,15):\n",
    "    random.shuffle(docs)\n",
    "    model.train(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "docvecs = model.docvecs[uids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "docvecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using PCA & t-SNE to change the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "raw_X = docvecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca = PCA(n_components=25)\n",
    "pca_X = pca.fit_transform(raw_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pca_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tsne = TSNE()\n",
    "tsne_X = tsne.fit_transform(pca_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tsne_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Bokeh to plot the low-dimensional data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "colormap = np.array([\n",
    "    \"#1f77b4\", \"#aec7e8\", \"#ff7f0e\", \"#ffbb78\", \"#2ca02c\", \n",
    "    \"#98df8a\", \"#d62728\", \"#ff9896\", \"#9467bd\", \"#c5b0d5\", \n",
    "    \"#8c564b\", \"#c49c94\", \"#e377c2\", \"#f7b6d2\", \"#7f7f7f\", \n",
    "    \"#c7c7c7\", \"#bcbd22\", \"#dbdb8d\", \"#17becf\", \"#9edae5\",\n",
    "    \"#ffff00\", \"ff00ff\", \"#ff0000\", \"00ff00\", \"0000ff\",\n",
    "    \"#00ffff\", \"00ccff\", \"00ffcc\", \"ff00cc\"\n",
    "])\n",
    "\n",
    "\n",
    "# create a new plot with default tools, using figure\n",
    "#hover = HoverTool(tooltips = {\"label\": \"@label\", \"text\": \"@text\"})\n",
    "hover = HoverTool(tooltips=\"\"\"\n",
    "        <div>\n",
    "            <span style=\"font-size: 17px; font-weight: bold; width:400px; display:block;\">@label</span>\n",
    "            <span style=\"font-size: 15px; color: #966; width:400px; display:block;\">[$index]</span>\n",
    "        </div>\n",
    "        <div>\n",
    "            <span style=\"font-size: 15px;\">Text</span>\n",
    "            <span style=\"font-size: 8px; color: #696; display:block; width:400px; word-break: break-all;\">(@text)</span>\n",
    "        </div>\n",
    "\"\"\")\n",
    "\n",
    "\n",
    "p = figure(plot_width=1000, plot_height=1000, tools=[hover, BoxZoomTool(), WheelZoomTool()])\n",
    "\n",
    "# add a circle renderer with a size, color, and alpha\n",
    "#p.circle([1, 2, 3, 4, 5], [6, 7, 2, 4, 5], size=15, line_color=\"navy\", fill_color=\"orange\", fill_alpha=0.5)\n",
    "\n",
    "p.scatter(x=tsne_X[:,0], y=tsne_X[:,1], \n",
    "          color=colormap[[directorates_uniq.index(x) for x in directorates]],\n",
    "          source=ColumnDataSource(data={\n",
    "                        \"label\": div_dir,\n",
    "                        \"text\": [text[:500] for text in texts]\n",
    "                    }))\n",
    "\n",
    "show(p) # show the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}