VCC-Note-Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('blamed_commit', 714L), ('fixing_commit', 1137L), ('other_commit', 349558L)]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import psycopg2\n",
    "conn = psycopg2.connect(\"dbname=postgres host=localhost port=55432 user=postgres\")\n",
    "cur = conn.cursor()\n",
    "cur.execute(\"SELECT type, COUNT(id) FROM export.commits GROUP BY type\")\n",
    "cur.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:26,820 - __main__ - INFO - Started loading data\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "2017-04-14 20:09:27,182 - __main__ - INFO - Data loaded #840\n",
      "Start extracting lines...\n",
      "Completed extracting lines including #0 invalid patches.\n",
      "{0: 0.49202795774039909}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEZCAYAAACNebLAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FdX9//HXJyAKyBZF2UJwhYoWVFRciVgVrNXWVtkE\na9WiLda6tEi1ElrXX221KlaxfFGqCC6tYLUuVVLcUBB3wQUVkKCIiGLAGJLP748zSS6XSXIJubk3\n8H4+HveROzPnznxmcu98Zs45M2PujoiISLKcTAcgIiLZSQlCRERiKUGIiEgsJQgREYmlBCEiIrGU\nIEREJJYSxDbKzN40s6PqKJNnZl+ZmTVWXOlmZh+a2cDo/Xgz+0emY9oSiesj0tCUILKMmX1kZuui\nHfMKM5tiZq0aejnuvq+7z6mjzDJ3b+tpuFgm2jl/G63najN71sz6N/RyUqALgRKYWQ8zKzeziUnj\n882swsxyksZPMbM/JAx3MrO/m1mxmX1pZm9H/+uWmxlHvpk9bWYl0TyOSeEz25nZQjNbmjT+MDN7\nMfquvWpmh29OLNsyJYjs48D33b0tcADQD7g8ruBWcGQ/PVrPnYEi4P7MhiPAKGA1MMTMtkuaVmsy\nNbMOwAvA9sAh7t4OOBZoB+yxmXHcC7wM5BK+/w+Y2U51fOa3wKcxMc0Crovi+BPwsJm128x4tklK\nENnJANx9BfAfYF8AM5ttZldGR9slwG5m1tbMJkdHbMvM7I+JicPMzomOwL6KqpX6RuMTq1oOMrN5\n0RHfCjO7Phq/0VGjmXU2s5lm9rmZvWtmZycsZ7yZzTCzu6JlvWFmB6Sysu5eAdwDdEncCZjZiWb2\nipl9Ea3zfgnTupnZg2a20sw+M7ObovG7m9lTZrYqmna3mbWt1z+hhuVHy/g8YVt2iZZ1VDT804Rt\n/r6Z/TxhngOi/9NvzOxTM1tuZieb2WAzeyeKe1zSdr3fzKZH85tvZt+tIV4zs0ujZX4Wfab9Zq72\nKMIOuQz4wWZ+9mLgK3cf6e7LANx9ubtf5O5vpjoTM9sL2B8odPdSd/8n8Drw41o+sxswHLgmadJh\nwCfu/k8P7gE+A07ZrDXbRilBZDEzywNOABYkjD4dOBtoAywF7gJKgd0JP6pjo+mY2anAFcDp0ZH6\nScDnMYv6K3BjdMS3B3BfwrTEo8YZ0TI7AacCV5tZQcL0HwDTCEdqDwMbVVPUsp4tgDOi2L6Ixu0P\nTAbOIRxF3g7MiqoRcoB/Ax8C3YGuwPTK2QFXRzF+B+gGFKYSR1JMNS7f3T8gHK3ebaHqZAowJaHK\n7lPghGibnwncUJlMIp2AFkAXYDxwBzCC8P87Cvi9meUnlD+JsO07EI6sHzKzZjFh/yoqe2Q07y+A\nWxPW6TUzG1rLOh9J9ba8n/A/2RzHAP+srUAUw+ro9UXS31uiYr2BD9y9JOGjr0Xja3ITMA74JoU4\njeigS+rg7npl0Yuw0/uKcJr/IXAzsH00bTbhqKqy7C6EH8T2CeOGAk9F7x8Dzq9lOQOj90WEHdVO\nSWXygXLCgUQe4aiyVcL0q4H/i96PB55ImPYdoKSW9RxPSGyrgQ2Eo7qjEqbfCkxI+swiws6vP2En\nnJPC9jwZeLmG9R4PTK3hczUuP2H4IcKR7avAdrXE8K/K/wMwACgBLBreEagA+iWUnw+clBDj8wnT\nDCgGDo9Zn7eBoxPKdga+TWU7ReXvAB6M3veP/j87J38Xkj4zBfhD9P5d4OcN8Bs4PXGdo3FXVn7X\nYsr/CHgkYfsuTZiWSzjwOA1oTkh65cDfGuL3urW/dAaRnU5291x3383dz3f30oRpyxLe5wPbASsq\nj8SA24CO0fQ8YHEKyzsL6AksstCY9/2YMp2B1e6+LmHcEsIRZ6VPEt6vA3YwsxwzG25ma6MqkkcS\nysxw91xConuT0N6SuG4XJx5tEs4GukTrtcRD1dRGzGwXM7vXzD42szXA3YQ2js1V2/Ir/Z1wVHuz\nu5clxDDYzF6IqqG+AAYnxfC5R3svYH30d2XC9PWExFGp6n8efe7jpDgSY/5XZcyEhFEG7FrXyprZ\nDoSzwmnRcuZGyx0eFdkQ/U1ul9guWgaEHXHnupaVgq+B5GrBdsDa5IIWOnBcRzh7gqh6tpK7rwZ+\nCFxC+H4eBzxJ2IZSByWI7FRb43Nilc8ywhnETlFC6eDu7d39uwnT62wcdPfF7j7c3TsC/4/QIJjc\n66QYyDWz1gnjugPLU5j/NHdv46FH1CbJJ/oRjwYKzaxyZ7YMuCpar8p129HdZ0TTultSj5rI1YQj\n8t7u3p5wNFqfxvzalk+0HW4kVEMVVtb1R9VlDxC2Y0d370BoR9qSDgV5lW/MzAiJKm67LwUGJ8Xc\n2kNbVl1+RNgp32qhHWoFIQlVVjOtICSCHkmf241woADw32g+NbLQDvZV0qvy4KGyOuwtYPek71qf\naHyyvQiJ8Zko5gcJbVnFZtYdwN2fcfeD3X1nQhvLd4CXat8cAkoQTZq7fwI8QajjbhM1Uu5u1dc3\n/B24xKLGYjPbI2rX2IiZjTCzyiPcLwlJqPLovLLB/GPgeeAaM9s+aig9C6jtOoKUd4ru/i6hSmxs\nNOoO4FwzOziKsbWZnRDtNF4i7LCuNbNWUTyHRZ9rQzgCXWtmXYHfpBpDktqWD6HO+yV3/znwKKGN\nAkLbQgtglbtXmNlgwlHrljjQzH4YtTtcSDgoeDGm3O2EdqHuUcwdzeykFJdxBiHZ7UfYGfcBjgD6\nmlnv6GztQeAqM8s1s+ZmNoyws/1PNI+/AG0tdFSojKGrmf3ZzPaFqu7VbZNelQcPv4jKvEeothsf\n/W9PIbQZPBgT9xuEBNo3ivlswplCH6IzLzPrG8XbFvgzoQrqyRS3yzZNCSL71NaVMG7aKMIO6W1C\nff79hEZQ3P0B4Cpgmpl9RagLz42Z1yDgrajMDcCQhGqtxHLDCEeMxYQf6+/dfXY91yXO9cA5Zraz\nu79MaCC+JaoueZfoaDbaWf2AcPS4lLAjOC2axwTgQGANoaE8eaeSUky1LT/a6R4H/CIqfhGwv5kN\nc/evCdUd90efGwrMrGtxdQzPBIYQGp1HAKe4e3lM2b9GZZ8wsy8JCf3gyonR0fuw5IWbWRdgIHCD\nu69MeC0g7PwrzyJ+SfiOvU5oA/oFoTH+MwB3/4LQa6gMeDGK4UnC/+L9OrZBsqHAQdE6XwX82N0/\nj+I9Ivqu4u4ViTFH8VW4+2cJ1Xi/BVYRznR2pY6zHKlm1dtQRLKNmY0H9nD3UZmORbY9OoMQEZFY\nShAiIhJLVUwiIhJLZxAiIhKreaYDSJWZ6VRHRKQe3L1e1+E0qTOITF92ni2v8ePHZzyGbHlpW2hb\naFvU/toSTSpBiIhI41GCEBGRWEoQTVBBQUGmQ8ga2hbVtC2qaVs0jCbTzdXMvKnEKiKSLcwMz8ZG\nagtPOvvUzF6vpcxNZvaehWfF9q2pnIiINK50VzFNAY6vaWJ0p8s93H0vwu2eb0tzPCIikqK0Jgh3\nf5boEZI1OBmYGpV9EWiX8DwAERHJoExfKNeVjZ+Qtjwa92lc4Seb8B3c+/aFjh3rLiciki0ynSA2\ny+jRhVXvO3QoIDe3IGOxbI4PP4QhQ+CqqzIdiYhs7YqKiigqKmqQeaW9F5OZ5QMPe/VjMBOn3QbM\n9urHOC4CBrj7JmcQTbkX0zXXwFdfhb8iIo0pa3sxRYyaHz05i/BENMysP7AmLjmIiEjjS2sVk5lN\nAwqAncxsKTCe8HhMd/dJ7v5o9Jzf94ES4Mx0xiMiIqlLa4Jw9+EplBmTzhhERKR+dKsNERGJpQQh\nIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSI\niMRSghARkVhKECIiEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIi\nEksJQkREYilBiIhILCUIERGJpQQhIiKxlCBERCSWEoSIiMRSghARkVhKECIiEksJQkREYilBiIhI\nLCUIERGJlfYEYWaDzGyRmb1rZmNjprc1s1lm9qqZvWFmP013TCIiUre0JggzywFuAY4HegPDzKxX\nUrFfAm+5e1/gaODPZtY8nXGJiEjd0n0GcTDwnrsvcfcyYDpwclIZB9pE79sAn7v7hjTHJSIidUh3\ngugKLEsY/jgal+gWYB8zKwZeAy5Ic0wiIpKCbGikPh54xd27APsDE81sxwzHJJF58+BnP4MFCzId\niYg0tnTX9S8HuicMd4vGJToTuAbA3Reb2YdAL2B+8swKCwur3hcUFFBQUNCw0QoA5eUwaxb85S+w\ndCm0aAHHHAMHHJDpyESkLkVFRRQVFTXIvMzdG2RGsTM3awa8AxwDrABeAoa5+8KEMhOBle4+wcx2\nJSSGPu6+Omlens5Y0+maa+Crr8LfbPb11zBlCtx4I3TsCBdfDD/6EZxxBpxwAowYkekIRWRzmRnu\nbvX5bFrPINy93MzGAE8QqrMmu/tCMxsdJvsk4ErgTjN7PfrYb5OTg6TXsmVw883wf/8HRx8Nd98N\nhx6a6ahEJNPS3p3U3R8DeiaNuz3h/QpCO4Q0snnz4IYb4LHH4Kc/DcO77ZbpqEQkW+h6g21McvvC\nBRfA3/4G7dplOjIRyTZKENuImtoXmusbICI10O5hK6f2BRGpr2y4DkLSYN48GD4c+vSBDRvC8P33\nKzmISOp0BrEVUfuCiDQkJYitgNoXRCQdtAtp4l56CYYOhf33V/uCiDQsJYgmyj1cw3DttXDbbXDK\nKZmOSES2NkoQTdCqVeHCtlWrwhlEjx6ZjkhEtkbqxdTEPPNMuGnePvuE90oOIpIuOoNoIsrLw83+\nbrklNEgPHpzpiERka6cE0QR88gmcfjqUlcHLL0PX5EcuiYikgaqYstyTT4YqpcMPh6eeUnIQkcaj\nM4gstWEDjB8Pd94Zuq8OHJjpiERkW6MEkYWWLYNhw6B1a3jlFdhll0xHJCLbIlUxZZmHH4Z+/eDE\nE+E//1FyEJHM0RlElvj2Wxg7Fv75T/jXv+CwwzIdkYhs65QgssDixeF2GV26hCql3NxMRyQioiqm\njLvvvnD/pJEj4aGHlBxEJHvoDCJD1q+HCy+E//4XHn00tDuIiGQTnUFkwKJFcMghsGYNLFig5CAi\n2UkJopHddRcceSScfz7cey+0bZvpiERE4qmKqZF8/TWMGgXz58PTT8N++2U6IhGR2ukMopHcckt4\nwtu8eUoOItI06AyiEQweDHvtBT/5SaYjERFJnRJEI+jbN7xERJoSVTGJiEgsJQgREYmlBCEiIrGU\nIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERipZwgzKyrmR1mZkdVvlL83CAzW2Rm75rZ2BrKFJjZ\nK2b2ppnNTjUmERFJn5SupDaz64AhwNtAeTTagTl1fC4HuAU4BigG5pnZTHdflFCmHTAROM7dl5vZ\nzpu9FiIi0uBSvdXGD4Ge7l66mfM/GHjP3ZcAmNl04GRgUUKZ4cCD7r4cwN1XbeYyREQkDVKtYvoA\n2K4e8+8KLEsY/jgal2hvINfMZpvZPDMbWY/liIhIA0v1DGId8KqZPQVUnUW4+68aKIYDgIFAa+AF\nM3vB3d9PLlhYWFj1vqCggIKCggZYvIjI1qOoqIiioqIGmVeqCWJW9Npcy4HuCcPdonGJPgZWufs3\nwDdmNgfoA9SaIEREZFPJB88TJkyo97xSShDufpeZtSBUBwG84+5lKXx0HrCnmeUDK4ChwLCkMjOB\nm82sGbA9cAjwl1TiEhGR9Em1F1MBcBfwEWBAnpmd4e619mJy93IzGwM8QWjvmOzuC81sdJjsk9x9\nkZk9DrxO6CE1yd3frvcaiYhIg0i1iunPhG6o7wCY2d7AvcCBdX3Q3R8DeiaNuz1p+Hrg+hRjERGR\nRpBqL6btKpMDgLu/S/16NYmISBOR6hnEfDP7O3B3NDwCmJ+ekEREJBukmiDOA34JVHZrfQa4NS0R\niYhIVki1F1MpoWeReheJiGwjak0QZnafu59mZm8Q7r20EXf/btoiExGRjKrrDOKC6O+J6Q5ERESy\nS629mNx9RfR2FbAsuune9oQrnYvTHJuIiGRQqt1c5wA7mFlXwkVvI4E70xWUiIhkXqoJwtx9HXAK\ncKu7nwr0Tl9YIlKpvLzuMiLpkHKCMLNDCdc/PBKNa5aekES2bRUVMH8+FBZCv37Qrp2ShGRGqgni\n18A44F/u/paZ7Q7o0aAiDeTrr+Ghh+Dss6FrVxg5Moy7/npYvx58kz6EIumX6nUQ/wP+lzD8AdUX\nzYlIPXz4Ifz73+H1/PPQvz+ceCJceinsuWd1ObPMxSjbtrqug7jR3X9tZg8Tfx3ESWmLTGQrs2ED\nvPBCdVJYtQpOOAF+/nO4/35o2zbTEYpsrK4ziH9Ef3WnVZF6WL0aHn88JITHHoP8/HCWMGVKaF/I\nSbWSVyQDak0Q7v5y9HY+sN7dKwASHu4jIgncYeHC6rOEV1+FgoKQFK67Drp1y3SEIqlL9WZ9TwHf\nA76OhlsSroc4LB1BiTQlpaXwv/9VJ4Xy8pAQxo0LyaFly0xHKFI/qSaIHdy9Mjng7l+bWas0xSTS\nIN59FyZNghkz4JlnoEePhpv3t9/CE0/A9OkhKfTuHZLCzJmw775qWJatQ6o1oCVmdkDlgJkdCKxP\nT0gi9fftt3DffTBwIBx5JDRvDtttB2vWbPm8N2wISeGss6Bz51BldOihsGgRPPdcOGPYbz8lB9l6\npHoG8WvgfjMrJjyTuhMwJG1RiWymDz6AO+4Ijb/77AOjR8OPfgQtWoTG4foqL4dnnw1nCg8+CLvt\nBkOGhIvY8vIaLHyRrJTqdRDzzKwX1c+Wfsfdy9IXlkjdyspC9c5tt8GCBTBqVGgL6Nmz7s/Wxh1e\nfDEkhfvug112gaFDYe5c2H33holdpClIKUFE7Q0XAfnufo6Z7WVmPd393+kNT2RTS5fC3/8OkyeH\nHfbo0aHuf4cd6j9Pd3jlldBeMWNGaFgeOhSefhp69Wq42EWaklSrmKYALwOHRsPLgfsBJQhpFOXl\n8J//wO23h6uOR4wI7QG9t/CWkW+9Fc4UZswIyxg6FGbNUluCCKSeIPZw9yFmNgzA3deZ6ecj6Vdc\nHM4U7rgDunQJZwszZkCrLehD9957YR7Tp8NXX8Fpp8G0aXDggUoKIolSTRDfmllLotttmNkeQGna\nopJtWkUFPPlkOFsoKgqNwrNmQd++9Z/nkiXhjGPGjJB0Tj01zP/QQ3U1s0hNUk0Q44HHgDwzuwc4\nHPhpuoKSbdOnn4ZeSHfcEW5xfe65cNdd0KbNls23RYvQNfXHPw53Rz3qKGimm9WL1KnOBBFVJS0i\nPCyoP6Gb6wXuvirNsck2wD1cxDZxYjjC//GPQ9VPv34NV93z739Dhw7heggRSV2dCcLd3cwedff9\nqH5YkMgWcQ83sbvySli5Ei64IFz13K5dwy9rl10afp5bA/dw/UhxcbioUCRZqlVMC8zsIHefl9Zo\nZKtXURHaE668Er75Bi6/PLQHqMqncXz8McyeHbrvPv10eBhReTl8/nmmI5NslGqCOAQ43cw+AkoI\n1Uzu7t9NV2CydSkvhwcegKuuClU9l18OJ5+sBuJ0W7kyNPQ//XRIDJ9/DkcfHV5jx0LHjrDXXpmO\nUrJVqgni+LRGIVutsrLQhfTqq2GnncL9iwYNUnfSdFmzBubMqT5DWLIkNMoPHAjnnReu70hMyqtX\nZy5WyX51PVFuB+BcYE/gDWCyu29ojMCkaSsthTvvhGuvDVc733ZbuPW1EkPDKikJ94qqPENYuDB0\n3R04MPQGO/DAcMNCkfqo66tzF1AGPAMMBvYBLkh3UNJ0rVsXdkx/+hP06QP33AOH6akhDaa0NNwT\nqvIM4ZVX4IADQkK4/no45BDYXo/ykgZSV4LYJ+q9hJlNBl5Kf0jSFK1dC7feCjfcEBLCzJnh6FUa\nxrXXhhsRzp0b7lY7cCD8/vdw+OHQunWmo2t8n30G8+bB/PnQqVN4rrc0vLoSRNUdW919Q33urmFm\ng4AbCc+emOzu19VQ7iDgeWCIu/9zsxckGfHFF3DTTXDLLXDssfDf/4YH5kjDOemk0Lj8q1+Fhv50\ndAVOlxUrwu1MtuQOu2vWwMsvVyeEefPgyy/DAchOO4XvnBJEetSVIPqY2VfRewNaRsOVvZja1vZh\nM8sBbgGOAYqBeWY2090XxZS7Fni8HusgGfDZZ+Fs4fbbww7suedg770zHdXW6Z9N5HBpwwZ4/XV4\n4YVwQ8Xnnw/fk/z8cFPEVJSUhGqzxGRQXAz77x8unjzllNDhYc89Q2P7nDmhR5ykR60Jwt23tHf6\nwcB77r4EwMymAycTrsxOdD7wAHDQFi5P0qy4ONR133lnuEfSyy837KM8pelYvTokg8qEMG8edO8e\nqhiPOSZUgVVUhJ16nNJSeO216kQwfz4sXhzOQA86CL73vfCUvl691NCeKene7F2BZQnDHxOSRhUz\n6wL80N2PNrONpkn2WLIEfvGLcBuMM86AN96Arl0zHZU0loqK8GjV55+vTgjLl8PBB4eE8JvfQP/+\n4ZYmiRZFh4IbNoSziMQzg4ULw1lnv37hs2PGhG64LVo0/vpJvGzIyzcCYxOG1REyyzRrBtdcE37A\nixbp1hXbgrVr4aWXqquK5s6F3NyQDA47LLSH7Ltv3VfAN2sWzgratw+PaD3ooJAQzjgj9HLbktu2\nS/qlO0EsB7onDHeLxiXqB0yPbgq4MzDYzMrcfVbyzAoLC6veFxQUUFBQ0NDxSozrroMbbww7CNn6\nuIedeGLbwfvvh3r/Qw8Nz+C4807YddfNn/eee4b59ewJbWttsZSGUlRURFFRUYPMy9y9QWYUO3Oz\nZsA7hEbqFYRussPcfWEN5acAD8f1YjIzT2esItuitWvD0X2nTqHL7KGHhjOE/fdvGlU9lY3Uc+Zk\nOpLsZWa4e71qZtJ6BuHu5WY2BniC6m6uC81sdJjsk5I/ks54RGRjbdqErspt2ugqd9lU2tsg3P0x\noGfSuNtrKPuzdMcjIhtT1Y/URPfSFBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIERE\nJJYShIiIxFKCEBGRWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGR\nWEoQIiISSwlCRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISSwlCRERi\nKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiISK+0JwswGmdkiM3vXzMbGTB9u\nZq9Fr2fNbL90xyQiInVLa4IwsxzgFuB4oDcwzMx6JRX7ADjK3fsAVwJ3pDMmERFJTbrPIA4G3nP3\nJe5eBkwHTk4s4O5z3f3LaHAu0DXNMYmISArSnSC6AssShj+m9gRwNvCftEYkIiIpaZ7pACqZ2dHA\nmcARNZUpLCysel9QUEBBQUHa4xIRaUqKioooKipqkHmZuzfIjGJnbtYfKHT3QdHwpYC7+3VJ5b4L\nPAgMcvfFNczL0xmriDQ9c+bA5ZeHvxLPzHB3q89n013FNA/Y08zyzawFMBSYlVjAzLoTksPImpKD\niIg0vrRWMbl7uZmNAZ4gJKPJ7r7QzEaHyT4J+D2QC9xqZgaUufvB6YxLRETqlvY2CHd/DOiZNO72\nhPfnAOekOw4REdk8upJaRERiKUGIiEgsJQgREYmlBCEiIrGUIEREJJYShIiIxFKCEBGRWEoQIiIS\nSwlCRERiZc3dXOurR48eLFmyJNNhiNRLfn4+H330UabDEInV5BPEkiVL0F1epakKtx8TyU6qYhIR\nkVhKECIiEksJQkREYilBiIhILCWIrcS+++7LnDqeu7hs2TLatm27VTXq77bbbjz99NMATJgwgZEj\nR2Y4IpHUlJbCp5+Gv9mqyfdiynY9evRg5cqVNG/enNatWzNo0CAmTpxIq1atGnQ5b775Zp1l8vLy\n+Oqrrxp0uZUmTJjAVVddxQ477EDz5s3ZZ599uP766+nfv39allcT9QqSVLnD+vXwxRe1v444Ak47\nbdPPl5bCmjX1f5WXQ/PmMHo03HBD469/KpQg0szMeOSRRzj66KNZsWIFxx13HFdeeSVXX331JmXd\nvUnv4IYOHcrUqVOpqKjgiiuu4NRTT2XZsmWZDqtBlZeX06xZs0yHIQnKymDZsvid+5o1te/8c3Kg\nQ4fwat+++n3l68sv4cILYcqU+B18+/a1v/Lyap7WsiX87W+QwrFd5rh7k3iFUDdV0/hs0aNHD3/q\nqaeqhn/zm9/4D37wA3d3Lygo8Msuu8wPP/xwb9WqlS9evNi//PJL/9nPfuadO3f2bt26+eWXX+4V\nFRVVn580aZJ/5zvf8TZt2njv3r39lVde2WQ5L730kvfr18/btm3rnTp18osvvtjd3T/66CM3My8v\nL3d39+LiYj/ppJM8NzfX99prL7/jjjuqllNYWOinnXaajxo1ytu0aeP77ruvv/zyyzWuZ2FhoY8c\nObJq+O233/acnBxftWpV1biHH37Y+/bt6+3bt/fDDz/cX3/99appy5Yt81NOOcU7duzoO++8s59/\n/vnu7r548WIfOHCg77TTTt6xY0cfMWKEf/nll7HbNzmGZA899JD37dvX27Zt63vuuac//vjjsf+j\nwsJCP/300zfaZpMnT/bu3bv7gAEDfPDgwT5x4sSN5t2nTx//17/+5e7uCxcu9GOPPdZzc3O9V69e\nft9999UYU7Z/f7Pda6+5t23r3rWre+/e7kcc4f6DH7iPGuV+wQXuhYXuf/2r+9Sp7g8/7P7ss+5v\nveVeXOy+fn3d81+zxv3ee90fecT9uefCZ5cvdy8pcU/4WdbbxInu55235fOpTfQdq99+t74fbOzX\n1pAgli5d6r179/bx48e7e0gQ+fn5vnDhQi8vL/eysjL/4Q9/6Oedd56vX7/eP/vsMz/kkEN80qRJ\n7u5+333YCb9iAAAMgklEQVT3ebdu3ap21IsXL/alS5duspxDDz3U7777bnd3Lykp8RdffNHdw84u\nJyenKkEceeSRPmbMGP/222/91Vdf9Y4dO/rs2bPdPewkW7Zs6Y899phXVFT4uHHjvH///jWuZ+LO\nubS01MeOHesdO3asWtaCBQt8l1128Xnz5nlFRYVPnTrVe/To4d9++62Xl5d7nz59/OKLL/b169d7\naWmpP/fcc+7u/v777/t///tfLysr81WrVvmAAQP8wgsvjN2+tSWIF1980du1a1dVtri42N95551N\n5pE8n8oEccYZZ/j69ev9m2++8alTp/rhhx9eVf6tt97yDh06eFlZmZeUlHheXp7fddddXlFRUbVd\nFy5cGBtXtn9/Jb2UILIgQYTaxi171VePHj28TZs23qFDB+/Ro4ePGTPGv/nmG3cPCaIyWbi7f/rp\np7799ttXTXd3v/fee33gwIHu7n788cf7TTfdVONyKndyAwYM8MLCwo2O3t03ThBLly715s2be0lJ\nSdX0cePG+ZlnnunuYSd57LHHVk17++23vVWrVjWuZ2Fhobdo0cI7dOjgzZo185133tn/97//VU0/\n77zz/IorrtjoMz179vQ5c+b4Cy+84LvssktVMqnNQw895AcccEDseteWIEaPHu0XXXRR7LS6EkRO\nTo5/9NFHVdPXrl3rO+64Y1Vyvuyyy/yss85yd/cZM2b4UUcdtcmy//CHP8QuWwli25btCWKb6MXU\nECliS8ycOZPVq1fz4YcfcvPNN7P99ttXTcvLy6t6v2TJEsrKyujcuTO5ubl06NCBc889l88++wwI\nvZD22GOPOpc3efJk3nnnHXr16sUhhxzCI488skmZFStWkJubu1FjeX5+PsuXL68a7tSpU9X7Vq1a\n8c0331BRUcG0adNo06YNbdu25fvf/35VmSFDhrB69WpWrlzJvvvuy/z58zdatz//+c/k5uZWrdvH\nH39McXExy5YtIz8/n5ycTb+OK1euZNiwYXTr1o327dtz+umns2rVqjq3QbJUt11NunXrVvV+xx13\n5IQTTmD69OkA3HvvvZx++ulAWM+5c+dutJ7Tpk3jk08+qfeyRTJFjdSNwGvJMImN0nl5eeywww58\n/vnnsY3VeXl5LF68uM7l7bHHHkybNg2ABx98kJ/85CesXr16ozJdunRh9erVlJSU0Lp1awCWLl1K\n165d65z/8OHDGT58eI3Tc3Nzuf322+nXrx8jRoxg1113JS8vj8suu4xx48ZtUn7u3LksXbqUioqK\nTZLE7373O3Jycnjrrbdo164dM2fO5Pzzz68zxmS1bbvWrVuzbt26quG4nXny/2PYsGFMmDCBI488\nktLSUgoKCqqWU1BQwOOPP77ZMYpkm23iDKKp6NSpE8cddxwXXngha9euxd354IMPqq5vOPvss7n+\n+utZsGABAIsXL47tJXTPPfdUHWW3a9cOM6va8VYmq27dunHYYYcxbtw4SktLef3115k8eXKt1xHU\nluiS7b333gwaNIjrrrsOgHPOOYfbbruNl156CYCSkhIeffRRSkpKOPjgg+ncuTOXXnop69ato7S0\nlOeffx6AtWvXsuOOO9KmTRuWL1/On/70p5RjSHTWWWcxZcoUZs+ejbtTXFzMO++8A0Dfvn2ZPn06\nGzZsYP78+TzwwAN1rvcJJ5zAkiVLuOKKKxgyZEjV+BNPPJF3332Xu+++mw0bNlBWVsb8+fNZtGhR\nveIWySSdQaRZbd1W46ZNnTqVsWPHss8++/D111+z++67M3bsWICqM4Hhw4dTXFxMjx49+Mc//kFe\nXt5G83rssce46KKLWL9+Pfn5+cyYMaOqWiux3L333svo0aPp0qULubm5/PGPf+Too4+u17rEueSS\nSzjmmGP43e9+x4EHHsgdd9zBmDFjeP/992nZsiVHHHEEAwYMICcnh4cffpjzzz+f7t27k5OTw/Dh\nwznssMMYP348o0aNon379uy5556MHDmSGxI6jaca00EHHcSUKVP49a9/zYcffkinTp2YOHEiPXv2\n5I9//CPDhg0jNzeXAQMGMGLEiI3OuOKW0aJFC0455RSmTJnCNddcUzV+xx135IknnuDCCy/koosu\nwt3p06cPf/nLXzZr28m2o/KCuXXrwnUZ69Zt/H5L/24J25yjwkwyM4+L1cw268hWJJvo+7tte+AB\nOPNMaNUqXBeRyt/NKduyJXTubLh7vS6wUoIQySB9fyXdou9YvRKE2iBERCSWEoSIiMRSghARkVhK\nECIiEksJQkREYjX56yDy8/Ob9C2yZduWn5+f6RBEapT2bq5mNgi4kXC2Mtndr4spcxMwGCgBfuru\nr8aUie3mKiIiNcvabq5mlgPcAhwP9AaGmVmvpDKDgT3cfS9gNHBbOmPaGhQVFWU6hKyhbVFN26Ka\ntkXDSHcbxMHAe+6+xN3LgOnAyUllTgamArj7i0A7M9s1zXE1afryV9O2qKZtUU3bomGkO0F0BRLv\nJvdxNK62MstjyoiISCNTLyYREYmV1kZqM+sPFLr7oGj4UsLTja5LKHMbMNvdZ0TDi4AB7v5p0rzU\nQi0iUg/1baROdzfXecCeZpYPrACGAsOSyswCfgnMiBLKmuTkAPVfQRERqZ+0Jgh3LzezMcATVHdz\nXWhmo8Nkn+Tuj5rZCWb2PqGb65npjElERFLTZG73LSIijSvrGqnNbJCZLTKzd81sbA1lbjKz98zs\nVTPr29gxNpa6toWZDTez16LXs2a2XybibAypfC+icgeZWZmZndKY8TWmFH8jBWb2ipm9aWazGzvG\nxpLCb6Stmc2K9hVvmNlPMxBm2pnZZDP71Mxer6XM5u833T1rXoSE9T6QD2wHvAr0SiozGHgken8I\nMDfTcWdwW/QH2kXvB23L2yKh3FPAv4FTMh13Br8X7YC3gK7R8M6ZjjuD22IccE3ldgA+B5pnOvY0\nbIsjgL7A6zVMr9d+M9vOIHRhXbU6t4W7z3X3L6PBuWy914+k8r0AOB94AFjZmME1slS2xXDgQXdf\nDuDuqxo5xsaSyrZwoE30vg3wubtvaMQYG4W7Pwt8UUuReu03sy1B6MK6aqlsi0RnA/9Ja0SZU+e2\nMLMuwA/d/W/A1tzjLZXvxd5ArpnNNrN5Zjay0aJrXKlsi1uAfcysGHgNuKCRYss29dpvNvm7uQqY\n2dGE3l9HZDqWDLoRSKyD3pqTRF2aAwcAA4HWwAtm9oK7v5/ZsDLieOAVdx9oZnsAT5rZd93960wH\n1hRkW4JYDnRPGO4WjUsuk1dHma1BKtsCM/suMAkY5O61nWI2Zalsi37AdAv3ft8ZGGxmZe4+q5Fi\nbCypbIuPgVXu/g3wjZnNAfoQ6uu3JqlsizOBawDcfbGZfQj0AuY3SoTZo177zWyrYqq6sM7MWhAu\nrEv+gc8CRkHVldqxF9ZtBercFmbWHXgQGOnuizMQY2Opc1u4++7RazdCO8QvtsLkAKn9RmYCR5hZ\nMzNrRWiUXNjIcTaGVLbFEuB7AFGd+97AB40aZeMxaj5zrtd+M6vOIFwX1lVJZVsAvwdygVujI+cy\ndz84c1GnR4rbYqOPNHqQjSTF38giM3sceB0oBya5+9sZDDstUvxeXAncmdD987fuvjpDIaeNmU0D\nCoCdzGwpMB5owRbuN3WhnIiIxMq2KiYREckSShAiIhJLCUJERGIpQYiISCwlCBERiaUEISIisZQg\nRCJmVm5mC6LbQs80s7YNPP8zzOym6P14M7uoIecv0tCUIESqlbj7Ae6+H+HOmL/MdEAimaQEIRLv\nBRLudmlml5jZS9HDVsYnjB8VPbDpFTO7Kxp3opnNNbOXzewJM+uYgfhFtlhW3WpDJMMMwMyaAccA\nf4+GjwX2cveDo1uazDKzI4DVwO+AQ939CzNrH83nGXfvH332LMJdZi9p3FUR2XJKECLVWprZAsKd\nLt8GnozGHwccG00zwi2094r+3l95F113XxOVzzOz+4DOhCedfdh4qyDScFTFJFJtnbsfQLiFtFHd\nBmGEx1Ye4O77u/ve7j6llvncDNzk7t8FzgV2SGvUImmiBCFSzQCi5yhcAFxiZjnA48DPzKw1hKfX\nRe0KTwOnmlluNL5DNJ+2QHH0/oxGjF+kQamKSaRa1a2N3f1VM3sNGObu95jZdwhPZgNYC5zu7m+b\n2VXA/8xsA/AK8DNgAvCAma0mJJEejbweIg1Ct/sWEZFYqmISEZFYShAiIhJLCUJERGIpQYiISCwl\nCBERiaUEISIisZQgREQklhKEiIjE+v/PH9TpSQad3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10d3666d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import sys\n",
    "from logging import getLogger, Formatter, StreamHandler, DEBUG\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn import cross_validation\n",
    "import re\n",
    "import math\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "from scipy import sparse\n",
    "import enum\n",
    "from datetime import datetime\n",
    "from vccf.line_extractor import LineExtractor\n",
    "from vccf.logger import Logger\n",
    "from unidiff.errors import UnidiffParseError\n",
    "\n",
    "class Colum(enum.IntEnum):\n",
    "    id = 1\n",
    "    type = 3\n",
    "    additions = 13\n",
    "    deletions = 14\n",
    "    past_different_authors = 18\n",
    "    future_different_authors = 19\n",
    "    author_contributions_percent = 20\n",
    "    patch = 21\n",
    "    hunk_count = 23\n",
    "    files_changed = 25\n",
    "    \n",
    "def fetch(filename, key = None):\n",
    "    npz = np.load(filename)\n",
    "    key = npz.files[0] if key is None else key\n",
    "    data = npz[key]\n",
    "    npz.close()\n",
    "    return data\n",
    "\n",
    "\n",
    "logger = Logger.create(name=__name__)\n",
    "logger.info('Started loading data')\n",
    "data = fetch('var/vcc_sample_40x800.npz')\n",
    "# data = fetch('var/vcc_data.npz')\n",
    "logger.info('Data loaded #%d' % len(data))\n",
    "\n",
    "patches = data[:, Colum.patch]\n",
    "labels = data[:, Colum.type]\n",
    "\n",
    "# Note: Be sure that unicode(patch, 'utf-8')\n",
    "print 'Start extracting lines...'\n",
    "cleansed_paches = [u''] * len(patches)\n",
    "invalid_paches = []\n",
    "\n",
    "matcher = r'(\\d+)'\n",
    "for index, patch in enumerate(patches):\n",
    "    try:\n",
    "#         patch = re.sub(matcher, logarithm, patch)\n",
    "        cleansed_paches[index] = u' '.join(LineExtractor.extract_lines(patch.splitlines()))\n",
    "    except UnidiffParseError as e:\n",
    "        # @todo Recover 445 patches at total in vcc_data.npz\n",
    "        invalid_paches.append((index, patch, e))\n",
    "\n",
    "print 'Completed extracting lines including #%d invalid patches.' % len(invalid_paches)\n",
    "\n",
    "# @TODO: stop_words=[]\n",
    "vectorizer = CountVectorizer(min_df=2, stop_words=None)\n",
    "vectorized = vectorizer.fit_transform(cleansed_paches) # csr_matrix\n",
    "# feature_names = vectorizer.get_feature_names()\n",
    "X = vectorized\n",
    "\n",
    "# Now X is sparse csr_matrix looks like:\n",
    "# [[0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  ..., \n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]\n",
    "#  [0 0 0 ..., 0 0 0]]\n",
    "\n",
    "# Bind metrics from Git metadata\n",
    "target_metrics = map(lambda n:n-1, [\n",
    "    Colum.additions,\n",
    "    Colum.deletions,\n",
    "    Colum.past_different_authors,\n",
    "    Colum.future_different_authors,\n",
    "    Colum.author_contributions_percent,\n",
    "    Colum.hunk_count,\n",
    "    Colum.files_changed,\n",
    "])\n",
    "metrics_data = np.array([row[target_metrics] for row in data])\n",
    "\n",
    "X2 = sparse.hstack((X, metrics_data.astype(float)))\n",
    "y = is_vcc = (labels == 'blamed_commit')\n",
    "\n",
    "# Split into training and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X2, y, test_size=.5, random_state=0)\n",
    "\n",
    "# Run classifier\n",
    "weight = {0:1, 1:.01}\n",
    "classifier = LinearSVC(C=1.0, class_weight=weight)\n",
    "y_score = classifier.fit(X_train, y_train).decision_function(X_test)\n",
    "\n",
    "# Compute Precision-Recall and plot curve\n",
    "precision = dict()\n",
    "recall = dict()\n",
    "average_precision = dict()\n",
    "precision[0], recall[0], _ = precision_recall_curve(y_test, y_score)\n",
    "average_precision[0] = average_precision_score(y_test, y_score)\n",
    "\n",
    "print average_precision\n",
    "\n",
    "# Plot Precision-Recall curve\n",
    "plt.clf()\n",
    "plt.plot(recall[0], precision[0], label='Precision-Recall curve')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.title('Precision-Recall example: AUC={0:0.2f}'.format(average_precision[0]))\n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.savefig(\"figure_%s\" % datetime.now().strftime('%s'))\n",
    "\n",
    "!osascript -e 'display notification \"Done\" with title \"VCC-Note\" sound name \"Purr\"'\n",
    "# !open ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}