notebooks/CF Test.ipynb

{"nbformat_minor": 0, "cells": [{"source": "##A notebook to explore the Wikipedia edit history and how a recommender system may work on this data", "cell_type": "markdown", "metadata": {}}, {"execution_count": 168, "cell_type": "code", "source": "from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating\nimport pandas as pd\nimport numpy as np\nimport sys\nimport itertools\nfrom math import sqrt\nfrom operator import add\n%matplotlib inline\nimport matplotlib.pyplot as plt\nimport matplotlib.mlab as mlab", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 13, "cell_type": "code", "source": "data_path = \"wiki-edits.json.gz\"", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 14, "cell_type": "code", "source": "#read in the data\nwiki_edits = sqlCtx.read.json(data_path)", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 188, "cell_type": "code", "source": "#quick view of the data we are working with\nwiki_edits.take(5)", "outputs": [{"execution_count": 188, "output_type": "execute_result", "data": {"text/plain": "[Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=u'*', minor=False, parent_id=None, redirect_target=u'Computer accessibility', revision_id=233192, timestamp=u'2001-01-21T02:12:21Z', user_id=99, user_name=u'RoseParks'),\n Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=u'Automated conversion', minor=True, parent_id=233192, redirect_target=u'Computer accessibility', revision_id=862220, timestamp=u'2002-02-25T15:43:11Z', user_id=0, user_name=u'Conversion script'),\n Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=u'Fixing redirect', minor=True, parent_id=862220, redirect_target=u'Computer accessibility', revision_id=15898945, timestamp=u'2003-04-25T22:18:38Z', user_id=7543, user_name=u'Ams80'),\n Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=u'fix double redirect', minor=True, parent_id=15898945, redirect_target=u'Computer accessibility', revision_id=56681914, timestamp=u'2006-06-03T16:55:41Z', user_id=516514, user_name=u'Nzd'),\n Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=u'cat rd', minor=False, parent_id=56681914, redirect_target=u'Computer accessibility', revision_id=74466685, timestamp=u'2006-09-08T04:16:04Z', user_id=750223, user_name=u'Rory096')]"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 16, "cell_type": "code", "source": "wiki_edits.printSchema()", "outputs": [{"output_type": "stream", "name": "stdout", "text": "root\n |-- article_id: long (nullable = true)\n |-- article_namespace: long (nullable = true)\n |-- article_title: string (nullable = true)\n |-- comment: string (nullable = true)\n |-- minor: boolean (nullable = true)\n |-- parent_id: long (nullable = true)\n |-- redirect_target: string (nullable = true)\n |-- revision_id: long (nullable = true)\n |-- timestamp: string (nullable = true)\n |-- user_id: long (nullable = true)\n |-- user_name: string (nullable = true)\n\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 17, "cell_type": "code", "source": "wiki_edits.registerTempTable('wiki')", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 21, "cell_type": "code", "source": "#simple query to check that the IP addresses are those without a user_id\nsqlCtx.sql(\"select * from wiki where user_id is null\").take(10)", "outputs": [{"execution_count": 21, "output_type": "execute_result", "data": {"text/plain": "[Row(article_id=10, article_namespace=0, article_title=u'AccessibleComputing', comment=None, minor=False, parent_id=133452289, redirect_target=u'Computer accessibility', revision_id=381200179, timestamp=u'2010-08-26T22:23:51Z', user_id=None, user_name=u'76.28.186.133'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=18201, redirect_target=None, revision_id=19746, timestamp=u'2002-02-25T15:43:11Z', user_id=None, user_name=u'140.232.153.45'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=19746, redirect_target=None, revision_id=19749, timestamp=u'2002-02-27T17:34:09Z', user_id=None, user_name=u'24.188.31.147'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=19749, redirect_target=None, revision_id=20514, timestamp=u'2002-02-27T17:36:41Z', user_id=None, user_name=u'24.188.31.147'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'term in bold', minor=False, parent_id=20514, redirect_target=None, revision_id=42733, timestamp=u'2002-03-01T00:13:17Z', user_id=None, user_name=u'213.253.39.175'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=42733, redirect_target=None, revision_id=42738, timestamp=u'2002-04-02T09:51:25Z', user_id=None, user_name=u'206.82.16.35'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=42738, redirect_target=None, revision_id=42740, timestamp=u'2002-04-02T09:53:06Z', user_id=None, user_name=u'206.82.16.35'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*', minor=False, parent_id=42740, redirect_target=None, revision_id=42743, timestamp=u'2002-04-02T09:54:12Z', user_id=None, user_name=u'206.82.16.35'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'more about the individualist anarchists way of thinking', minor=False, parent_id=67475, redirect_target=None, revision_id=61039, timestamp=u'2002-04-26T07:02:43Z', user_id=None, user_name=u'80.65.225.191'),\n Row(article_id=12, article_namespace=0, article_title=u'Anarchism', comment=u'*link politics, government', minor=False, parent_id=103355, redirect_target=None, revision_id=117294, timestamp=u'2002-06-23T15:17:20Z', user_id=None, user_name=u'195.149.37.162')]"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 20, "cell_type": "code", "source": "#total number of revisions in our file\nwiki_edits.count()", "outputs": [{"execution_count": 20, "output_type": "execute_result", "data": {"text/plain": "1846836"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"source": "###View the number of articles and revisions a user is making", "cell_type": "markdown", "metadata": {}}, {"execution_count": 38, "cell_type": "code", "source": "users = sqlCtx.sql(\"select user_id, count(1) as rev_count, count(distinct(article_id)) as article_num \\\nfrom wiki where user_id is not null group by user_id\")", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 39, "cell_type": "code", "source": "users_local = users.collect()", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 40, "cell_type": "code", "source": "users_local[:10]", "outputs": [{"execution_count": 40, "output_type": "execute_result", "data": {"text/plain": "[Row(user_id=2796431, rev_count=1, article_num=1),\n Row(user_id=24532431, rev_count=1, article_num=1),\n Row(user_id=7348231, rev_count=61, article_num=13),\n Row(user_id=10296231, rev_count=1, article_num=1),\n Row(user_id=509431, rev_count=2, article_num=1),\n Row(user_id=3446431, rev_count=1, article_num=1),\n Row(user_id=9041631, rev_count=2, article_num=2),\n Row(user_id=234831, rev_count=1, article_num=1),\n Row(user_id=3996831, rev_count=1, article_num=1),\n Row(user_id=1911631, rev_count=39, article_num=7)]"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 42, "cell_type": "code", "source": "users_local_df = pd.DataFrame(users_local, columns=[\"user_id\", \"rev_count\", \"article_num\"])", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 43, "cell_type": "code", "source": "users_local_df.head()", "outputs": [{"execution_count": 43, "output_type": "execute_result", "data": {"text/plain": "    user_id  rev_count  article_num\n0   2796431          1            1\n1  24532431          1            1\n2   7348231         61           13\n3  10296231          1            1\n4    509431          2            1", "text/html": "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>rev_count</th>\n      <th>article_num</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>  2796431</td>\n      <td>  1</td>\n      <td>  1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td> 24532431</td>\n      <td>  1</td>\n      <td>  1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>  7348231</td>\n      <td> 61</td>\n      <td> 13</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td> 10296231</td>\n      <td>  1</td>\n      <td>  1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>   509431</td>\n      <td>  2</td>\n      <td>  1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 44, "cell_type": "code", "source": "print max(users_local_df.rev_count), min(users_local_df.rev_count), np.mean(users_local_df.rev_count)\nprint max(users_local_df.article_num), min(users_local_df.article_num), np.mean(users_local_df.rev_count)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "12424 1 9.33026352853\n1755 1 3.47698820068\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 73, "cell_type": "code", "source": "#count the number of articles represented (even those outside of namespace=0)\nsqlCtx.sql(\"select distinct(article_id) from wiki\").count()", "outputs": [{"execution_count": 73, "output_type": "execute_result", "data": {"text/plain": "1764"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 167, "cell_type": "code", "source": "#count the number of users represented\nsqlCtx.sql(\"select distinct(user_id) from wiki where user_id is not null\").count()", "outputs": [{"execution_count": 167, "output_type": "execute_result", "data": {"text/plain": "127804"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"source": "###Prepare training and test data, knowing the MLLib takes in (user, page, rating)\n####We will try differeng ways of representing rating and is one of the biggest levers we have", "cell_type": "markdown", "metadata": {}}, {"execution_count": 183, "cell_type": "code", "source": "article_raw = sqlCtx.sql(\"select user_id, article_id, count(1) as article_count from wiki \\\nwhere user_id is not null and article_namespace=0 group by user_id, article_id\")", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 184, "cell_type": "code", "source": "#Some users look at a lot of data, this is where the 5 edit ceiling was developed in the data version above\n#the code below did not use the .map(lambda (u, a, a_c): (u, a, min(a_c, 5))) code\narticle_user_local = article_raw.collect()", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 185, "cell_type": "code", "source": "article_user_local[:10]", "outputs": [{"execution_count": 185, "output_type": "execute_result", "data": {"text/plain": "[Row(user_id=1601135, article_id=2752, article_count=2),\n Row(user_id=24420788, article_id=791, article_count=1),\n Row(user_id=2377922, article_id=633, article_count=1),\n Row(user_id=7691690, article_id=2017, article_count=1),\n Row(user_id=1395162, article_id=2753, article_count=1),\n Row(user_id=82432, article_id=2563, article_count=9),\n Row(user_id=239201, article_id=2310, article_count=1),\n Row(user_id=433328, article_id=1811, article_count=1),\n Row(user_id=10286712, article_id=803, article_count=1),\n Row(user_id=555420, article_id=1807, article_count=1)]"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 186, "cell_type": "code", "source": "article_user_df = pd.DataFrame(article_user_local, columns=[\"user_id\", \"article\", \"art_count\"])", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 187, "cell_type": "code", "source": "print min(article_user_df.art_count), max(article_user_df.art_count), np.mean(article_user_df.art_count)", "outputs": [{"output_type": "stream", "name": "stdout", "text": "1 1663 2.6445193024\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 140, "cell_type": "code", "source": "len(article_user_df)", "outputs": [{"execution_count": 140, "output_type": "execute_result", "data": {"text/plain": "413484"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 149, "cell_type": "code", "source": "len(article_user_df[article_user_df.art_count>5])", "outputs": [{"execution_count": 149, "output_type": "execute_result", "data": {"text/plain": "24154"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 150, "cell_type": "code", "source": "#very small proportion of the users edit a particular article more than 5 times\n24154/413484.0*100", "outputs": [{"execution_count": 150, "output_type": "execute_result", "data": {"text/plain": "5.841580327171063"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 146, "cell_type": "code", "source": "c = article_user_df[article_user_df.art_count<5]", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 127, "cell_type": "code", "source": "np.mean(article_user_df.art_count)", "outputs": [{"execution_count": 127, "output_type": "execute_result", "data": {"text/plain": "2.5951233905060413"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 148, "cell_type": "code", "source": "plt.hist(c.art_count)", "outputs": [{"execution_count": 148, "output_type": "execute_result", "data": {"text/plain": "(array([ 282993.,   63291.,   23954.,   12109.,       0.,    6983.,\n           4469.,    3093.,    2297.,    1703.]),\n array([ 1. ,  1.8,  2.6,  3.4,  4.2,  5. ,  5.8,  6.6,  7.4,  8.2,  9. ]),\n <a list of 10 Patch objects>)"}, "metadata": {}}, {"output_type": "display_data", "data": {"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEACAYAAACpoOGTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFr1JREFUeJzt3X+wX3Wd3/HnSyKIyhKDXX4GZNrYmtYOiCVOrWtcEGNt\nATsO4FTJdDI762SrdjvtCvsHCWWGlc4o4u7AWOVHYJWFSkV0EROBu9XpQJAFjQaWYE2bXCTYYGCt\nA0Pqu398P3f8Gj+5N+THPTfwfMx8537O+5zP+b5PEr6v7znn+72kqpAkaVevGLoBSdLcZEBIkroM\nCElSlwEhSeoyICRJXQaEJKlr2oBI8qok9yd5OMnGJH/S6guSrEvyWJK1SeaPzbk4yaYkjyY5a6x+\nWpINbd1VY/XDktzS6vclOWls3fL2HI8luXD/HrokaTrTBkRVPQe8q6pOAf4x8K4k/wy4CFhXVW8E\n7m7LJFkMnA8sBpYBVydJ2901wIqqWgQsSrKs1VcA21v9SuCKtq8FwCXA6e2xajyIJEkH1oyXmKrq\nF214KHAI8DPgbGBNq68Bzm3jc4Cbq+qFqtoMPA4sSXIscERVrW/b3Tg2Z3xftwFntPF7gLVVtaOq\ndgDrGIWOJGkWzBgQSV6R5GFgG3BvVf0QOLqqtrVNtgFHt/FxwNax6VuB4zv1yVan/dwCUFU7gWeS\nHDXNviRJs2DeTBtU1S+BU5IcCXwzybt2WV9J/H0dkvQSM2NATKmqZ5L8JXAasC3JMVX1ZLt89FTb\nbBJYODbtBEbv/CfbeNf61JwTgSeSzAOOrKrtSSaBpWNzFgL37NqX4SRJe6eqMtMGu30Arwfmt/Hh\nwH9ndI/gPwOfaPWLgE+28WLgYUb3K04GfgSkrbsfWAIEuBNY1uorgWva+ALgL9p4AfA/gfnA66bG\nnR5rumMY4gGsHrqHg6GnudqXPdnTy6GvPXntnOkM4lhgTZJXMLpfcVNV3Z3kIeDWJCuAzcB57dk2\nJrkV2AjsBFZW66QFwQ0taO6sqrta/VrgpiSbgO0tJKiqp5NcBjzQtru0RjerJUmzYNqAqKoNwFs6\n9aeBM3cz53Lg8k79QeDNnfrztIDprLseuH66HiVJB4bfpD4wJoZuoGNi6AZ2Y2LoBjomhm6gY2Lo\nBjomhm6gY2LoBnZjYugG9kZ+dQXo4JSkaqYbLZKkX7Mnr52eQUiSugwISVKXASFJ6jIgJEldBoQk\nqWuPf9WG9kyS3wbeNNDTP15VkwM9t6SXGANi/1sKR9wIf++52X3a/3UYPP1HwJ/O7vNKeqkyIA6I\n33kevn7k7D7nR56Dz83uU0p6SfMehCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIgJEld\nBoQkqcuAkCR1GRCSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdU0bEEkWJrk3yQ+T\n/CDJx1p9dZKtSR5qj/eOzbk4yaYkjyY5a6x+WpINbd1VY/XDktzS6vclOWls3fIkj7XHhfv30CVJ\n05np/0n9AvCHVfVwktcCDyZZBxTw6ar69PjGSRYD5wOLgeOBbyVZVFUFXAOsqKr1Se5Msqyq7gJW\nANuralGS84ErgAuSLAAuAU5ru38wyR1VtWP/HLokaTrTnkFU1ZNV9XAb/xx4hNELP0A6U84Bbq6q\nF6pqM/A4sCTJscARVbW+bXcjcG4bnw2saePbgDPa+D3A2qra0UJhHbDsRR6fJGkv7fE9iCRvAE4F\n7muljyb5XpJrk8xvteOArWPTtjIKlF3rk/wqaI4HtgBU1U7gmSRHTbMvSdIs2KOAaJeXvgx8vJ1J\nXAOcDJwC/AT41AHrUJI0iJnuQZDklYwu/fx5Vd0OUFVPja3/AvC1tjgJLBybfgKjd/6TbbxrfWrO\nicATSeYBR1bV9iSTwNKxOQuBe3bT4+qxxYmqmpjpuCTp5STJUn79NXVG0wZEkgDXAhur6jNj9WOr\n6idt8f3Ahja+A/hSkk8zuhy0CFhfVZXk2SRLgPXAh4HPjs1ZzujS1QeAu1t9LXB5u3wV4N3AJ3p9\nVtXqPT5iSXoZam+cJ6aWk6yaac5MZxBvBz4EfD/JQ632x8AHk5zC6NNMPwZ+vzWwMcmtwEZgJ7Cy\nfYIJYCVwA3A4cGf7BBOMAuimJJuA7cAFbV9PJ7kMeKBtd6mfYJKk2TNtQFTVd+jfp/jGNHMuBy7v\n1B8E3typPw+ct5t9XQ9cP12PkqQDw29SS5K6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNC\nktRlQEiSugwISVKXASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJ\nXQaEJKnLgJAkdRkQkqQuA0KS1GVASJK6DAhJUpcBIUnqMiAkSV0GhCSpa9qASLIwyb1JfpjkB0k+\n1uoLkqxL8liStUnmj825OMmmJI8mOWusflqSDW3dVWP1w5Lc0ur3JTlpbN3y9hyPJblw/x66JGk6\nM51BvAD8YVX9Q+BtwB8keRNwEbCuqt4I3N2WSbIYOB9YDCwDrk6Stq9rgBVVtQhYlGRZq68Atrf6\nlcAVbV8LgEuA09tj1XgQSZIOrGkDoqqerKqH2/jnwCPA8cDZwJq22Rrg3DY+B7i5ql6oqs3A48CS\nJMcCR1TV+rbdjWNzxvd1G3BGG78HWFtVO6pqB7COUehIkmbBHt+DSPIG4FTgfuDoqtrWVm0Djm7j\n44CtY9O2MgqUXeuTrU77uQWgqnYCzyQ5app9SZJmwbw92SjJaxm9u/94Vf3tr64aQVVVkjpA/e2R\nJKvHFieqamKgViRpTkqyFFj6YubMGBBJXskoHG6qqttbeVuSY6rqyXb56KlWnwQWjk0/gdE7/8k2\n3rU+NedE4Ikk84Ajq2p7ksldDmYhcE+vx6paPdNxSNLLWXvjPDG1nGTVTHNm+hRTgGuBjVX1mbFV\ndwDL23g5cPtY/YIkhyY5GVgErK+qJ4Fnkyxp+/ww8NXOvj7A6KY3wFrgrCTzk7wOeDfwzZkOSJK0\nf8x0BvF24EPA95M81GoXA58Ebk2yAtgMnAdQVRuT3ApsBHYCK6tq6vLTSuAG4HDgzqq6q9WvBW5K\nsgnYDlzQ9vV0ksuAB9p2l7ab1ZKkWTBtQFTVd9j9WcaZu5lzOXB5p/4g8OZO/XlawHTWXQ9cP12P\nkqQDw29SS5K6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIg\nJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQkqQuA0KS\n1GVASJK6DAhJUpcBIUnqMiAkSV0zBkSS65JsS7JhrLY6ydYkD7XHe8fWXZxkU5JHk5w1Vj8tyYa2\n7qqx+mFJbmn1+5KcNLZueZLH2uPC/XPIkqQ9sSdnENcDy3apFfDpqjq1Pb4BkGQxcD6wuM25Okna\nnGuAFVW1CFiUZGqfK4DtrX4lcEXb1wLgEuD09liVZP5eHqck6UWaMSCq6tvAzzqr0qmdA9xcVS9U\n1WbgcWBJkmOBI6pqfdvuRuDcNj4bWNPGtwFntPF7gLVVtaOqdgDr+M2gkiQdIPtyD+KjSb6X5Nqx\nd/bHAVvHttkKHN+pT7Y67ecWgKraCTyT5Khp9iVJmgXz9nLeNcB/auPLgE8xulQ0iCSrxxYnqmpi\noFYkaU5KshRY+mLm7FVAVNVTY0/6BeBrbXESWDi26QmM3vlPtvGu9ak5JwJPJJkHHFlV25NM8usH\nsxC4Zzf9rN6b45Ckl4v2xnliajnJqpnm7NUlpnZPYcr7galPON0BXJDk0CQnA4uA9VX1JPBskiXt\npvWHga+OzVnexh8A7m7jtcBZSeYneR3wbuCbe9OvJOnFm/EMIsnNwDuB1yfZAqwCliY5hdGnmX4M\n/D5AVW1MciuwEdgJrKyqartaCdwAHA7cWVV3tfq1wE1JNgHbgQvavp5OchnwQNvu0nazWpI0C/Kr\n1++DU5Kqqt4nqgaR5Dx43+fh6781u8/8kefgc39UVX86u88r6WC0J6+dfpNaktRlQEiSugwISVKX\nASFJ6jIgJEldBoQkqcuAkCR1GRCSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQ\nkqQuA0KS1GVASJK6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ\n6poxIJJcl2Rbkg1jtQVJ1iV5LMnaJPPH1l2cZFOSR5OcNVY/LcmGtu6qsfphSW5p9fuSnDS2bnl7\njseSXLh/DlmStCf25AziemDZLrWLgHVV9Ubg7rZMksXA+cDiNufqJGlzrgFWVNUiYFGSqX2uALa3\n+pXAFW1fC4BLgNPbY9V4EEmSDqwZA6Kqvg38bJfy2cCaNl4DnNvG5wA3V9ULVbUZeBxYkuRY4Iiq\nWt+2u3Fszvi+bgPOaOP3AGurakdV7QDW8ZtBJUk6QPb2HsTRVbWtjbcBR7fxccDWse22Asd36pOt\nTvu5BaCqdgLPJDlqmn1JkmbBvH3dQVVVktofzeytJKvHFieqamKgViRpTkqyFFj6YubsbUBsS3JM\nVT3ZLh891eqTwMKx7U5g9M5/so13rU/NORF4Isk84Miq2p5kkl8/mIXAPb1mqmr1Xh6HJL0stDfO\nE1PLSVbNNGdvLzHdASxv4+XA7WP1C5IcmuRkYBGwvqqeBJ5NsqTdtP4w8NXOvj7A6KY3wFrgrCTz\nk7wOeDfwzb3sV5L0Is14BpHkZuCdwOuTbGH0yaJPArcmWQFsBs4DqKqNSW4FNgI7gZVVNXX5aSVw\nA3A4cGdV3dXq1wI3JdkEbAcuaPt6OsllwANtu0vbzWpJ0iyYMSCq6oO7WXXmbra/HLi8U38QeHOn\n/jwtYDrrrmf0MVtJ0izzm9SSpC4DQpLUZUBIkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQkqQu\nA0KS1GVASJK6DAhJUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIg\nJEldBoQkqWve0A1ov/psks8O8cRVlSGeV9KBY0C85NQAz2k2SC9FXmKSJHUZEJKkLgNCktS1TwGR\nZHOS7yd5KMn6VluQZF2Sx5KsTTJ/bPuLk2xK8miSs8bqpyXZ0NZdNVY/LMktrX5fkpP2pV9J0p7b\n1zOIApZW1alVdXqrXQSsq6o3Ane3ZZIsBs4HFgPLgKuTTN3dvAZYUVWLgEVJlrX6CmB7q18JXLGP\n/UqS9tD+uMS060dYzgbWtPEa4Nw2Pge4uapeqKrNwOPAkiTHAkdU1fq23Y1jc8b3dRtwxn7oV5K0\nB/bHGcS3knw3ye+12tFVta2NtwFHt/FxwNaxuVuB4zv1yVan/dwCUFU7gWeSLNjHniVJe2Bfvwfx\n9qr6SZK/A6xL8uj4yqqqJEN8MF+StI/2KSCq6ift50+TfAU4HdiW5JiqerJdPnqqbT4JLBybfgKj\nM4fJNt61PjXnROCJJPOAI6vq6V37SLJ6bHGiqib25bgk6aUmyVJg6YuZs9cBkeTVwCFV9bdJXgOc\nBVwK3AEsZ3RDeTlwe5tyB/ClJJ9mdOloEbC+nWU8m2QJsB74MPDZsTnLgfuADzC66f0bqmr13h6H\nJL0ctDfOE1PLSVbNNGdfziCOBr7SPog0D/hiVa1N8l3g1iQrgM3Aea25jUluBTYCO4GVVTV1+Wkl\ncANwOHBnVd3V6tcCNyXZBGwHLtiHfiVJL0J+9Rp9cEpSc+kXxSU5D973efj6b83uM3/kOfjcq4b6\nXUxz6e9A0sz25LXTb1JLkroMCElSlwEhSeoyICRJXQaEJKnLgJAkdRkQkqQuA0KS1GVASJK6DAhJ\nUpcBIUnqMiAkSV0GhCSpy4CQJHUZEJKkrn39f1JLwOh3y8/2c/r/oJAOLANC+8ls54PZIB1oXmKS\nJHUZEJKkLgNCktRlQEiSugwISVKXASFJ6jIgJEldfg9CB60hvpwHfkFPLx8GhA5iQ+SD2aCXDy8x\nSZK6DAhJUtecD4gky5I8mmRTkk8M3Y80hCQ11GPoY9dw5nRAJDkE+DNgGbAY+GCSNw3b1Z74P4cM\n3cFvmhi6gd2YGLqBjomhG9iNGuAxTDjt7Z9QkqV7O/dAmqt9zWROBwRwOvB4VW2uqheAvwDOGbin\nPbB9Dt78nxi6gd2YGLqBjomhG5hjdhceq6ZZN0woAffO0bOlpQdw3wfMXA+I44EtY8tbW03SS9re\nhsu+hNaBO1sCVh2Ml/Hm4DvdXzNn/+Cmt/UQ+N1nZvc5Hzlsdp9Peik6UC85q9ujJ4N9p2cmqZqT\nfQGQ5G3A6qpa1pYvBn5ZVVeMbTN3D0CS5rCZvvQ51wNiHvA3wBnAE8B64INV9cigjUnSy8CcvsRU\nVTuT/Fvgm8AhwLWGgyTNjjl9BiFJGs5c/xTTbmUOfoEuyXVJtiXZMHQvU5IsTHJvkh8m+UGSj82B\nnl6V5P4kDyfZmORPhu5pSpJDkjyU5GtD9zIlyeYk3299rR+6H4Ak85N8Ockj7e/wbQP38/fbn8/U\n45k58m/94vbf3oYkX0oy+IdJkny89fODJB+fdtuD8Qwioy/Q/Q1wJjAJPMAcuDeR5B3Az4Ebq+rN\nQ/YyJckxwDFV9XCS1wIPAufOgT+rV1fVL9p9pu8A/6GqvjNkT62vfw+cBhxRVWcP3Q9Akh8Dp1XV\n00P3MiXJGuCvquq69nf4mqqa5U/u9SV5BaPXhdOrastM2x/APt4A3AO8qaqeT3ILcGdVrRmwp38E\n3Az8E+AF4C7gI1X1o972B+sZxJz8Al1VfRv42dB9jKuqJ6vq4Tb+OfAIcNywXUFV/aIND2V0f2nw\nF78kJwD/HPgCc+/Xts6ZfpIcCbyjqq6D0b3CuRIOzZnAj4YMh+ZZRi/Cr24h+mpGwTWkfwDcX1XP\nVdX/A/4K+Fe72/hgDQi/QLcX2juaU4H7h+1k9C4vycPANuDeqto4dE/AlcB/BH45dCO7KOBbSb6b\n5PeGbgY4GfhpkuuT/HWSzyd59dBNjbkA+NLQTbQzvk8B/5vRpzB3VNW3hu2KHwDvSLKg/Z29Dzhh\ndxsfrAFx8F0XG1i7vPRl4OPtTGJQVfXLqjqF0T/O38nAv6smyb8Anqqqh5hD79abt1fVqcB7gT9o\nlzKHNA94C3B1Vb0F+L/ARcO2NJLkUOBfAv91DvTyd4F/B7yB0Vn7a5P86yF7qqpHgSuAtcA3gIeY\n5g3RwRoQk8DCseWFjM4i1JHklcBtwJ9X1e1D9zOuXZr4S+CtA7fyT4Gz2/X+m4HfTXLjwD0BUFU/\naT9/CnyF0SXWIW0FtlbVA235y4wCYy54L/Bg+7Ma2luB/1FV26tqJ/DfGP07G1RVXVdVb62qdwI7\nGN3P7TpYA+K7wKIkb2jvGM4H7hi4pzkpSYBrgY1V9Zmh+wFI8vok89v4cODdjN7JDKaq/riqFlbV\nyYwuUdxTVRcO2ROMbuYnOaKNXwOcBQz6KbmqehLYkuSNrXQm8MMBWxr3QUYBPxc8CrwtyeHtv8Mz\ngcEvpSb57fbzROD9THM5bk5/UW535uoX6JLcDLwTOCrJFuCSqrp+4LbeDnwI+H6SqRfhi6vqrgF7\nOhZY0z5t8grgpqq6e8B+eubKZcyjga+MXl+YB3yxqtYO2xIAHwW+2N6g/Qj4NwP3MxWgZwJz4T4N\nVfW9dhb6XUaXcf4a+C/DdgXAl5McxegG+sqqenZ3Gx6UH3OVJB14B+slJknSAWZASJK6DAhJUpcB\nIUnqMiAkSV0GhCSpy4CQJHUZEJKkrv8Pk6mw9NM9svIAAAAASUVORK5CYII=\n", "text/plain": "<matplotlib.figure.Figure at 0x7f0020864510>"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"source": "###Now that the general anatomy of the data is known we can split the data into training and testing", "cell_type": "markdown", "metadata": {}}, {"execution_count": 169, "cell_type": "code", "source": "article_user_train = sqlCtx.sql(\"select user_id, article_id, count(1) as article_count from wiki \\\nwhere user_id is not null and article_namespace=0 and timestamp not like '%9Z' group by user_id, article_id\")\\\n.map(lambda (u, a, a_c): (u, a, min(a_c, 5)))", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 170, "cell_type": "code", "source": "article_user_test = sqlCtx.sql(\"select user_id, article_id, count(1) as article_count from wiki \\\nwhere user_id is not null and article_namespace=0 and timestamp like '%9Z' group by user_id, article_id\")\\\n.map(lambda (u, a, a_c): (u, a, min(a_c, 5)))", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 171, "cell_type": "code", "source": "print article_user_train.count()\nprint article_user_test.count()", "outputs": [{"output_type": "stream", "name": "stdout", "text": "402133\n74324\n"}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 87, "cell_type": "code", "source": "#checking to see that the split on timestamp is OK\n77077/444373.0*100", "outputs": [{"execution_count": 87, "output_type": "execute_result", "data": {"text/plain": "17.345113226951234"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"source": "###Now that we have some training and test data we can train and test a model", "cell_type": "markdown", "metadata": {}}, {"execution_count": 162, "cell_type": "code", "source": "model = ALS.train(article_user_train, rank = 10, iterations = 5)", "outputs": [], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 96, "cell_type": "code", "source": "##from MovieLensALS.py\ndef computeRmse(model, data):\n    \"\"\"\n    Compute RMSE (Root Mean Squared Error).\n    \"\"\"\n    predictions = model.predictAll(data.map(lambda x: (x[0], x[1])))\n    predictionsAndRatings = predictions.map(lambda x: ((x[0], x[1]), x[2])) \\\n      .join(data.map(lambda x: ((x[0], x[1]), x[2]))) \\\n      .values()\n    n = data.count()\n    return sqrt(predictionsAndRatings.map(lambda x: (x[0] - x[1]) ** 2).reduce(add) / float(n))", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 90, "cell_type": "code", "source": "#using the edit count as the \"rating\"\ncomputeRmse(model, article_user_test)", "outputs": [{"execution_count": 90, "output_type": "execute_result", "data": {"text/plain": "24.31148553449491"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 163, "cell_type": "code", "source": "#using the edit count as the \"rating\" with a ceiling of 5 edits\ncomputeRmse(model, article_user_test)", "outputs": [{"execution_count": 163, "output_type": "execute_result", "data": {"text/plain": "1.4861927877363568"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 97, "cell_type": "code", "source": "#using a bollean edited, not edited\n#this may be a bit more accurate to if a user will interact with an article or not\ncomputeRmse(model, article_user_test)", "outputs": [{"execution_count": 97, "output_type": "execute_result", "data": {"text/plain": "0.0767297258759174"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 173, "cell_type": "code", "source": "predictions = model.predictAll(article_user_test.map(lambda x: (x[0], x[1])))", "outputs": [], "metadata": {"collapsed": true, "trusted": true}}, {"execution_count": 174, "cell_type": "code", "source": "#View of what the preditions look like\npredictions.take(10)", "outputs": [{"execution_count": 174, "output_type": "execute_result", "data": {"text/plain": "[Rating(user=2164608, product=2100, rating=1.913035174886199),\n Rating(user=110590, product=2100, rating=0.2837415583329478),\n Rating(user=624973, product=2100, rating=5.029465109553854),\n Rating(user=241822, product=2100, rating=0.387948899782942),\n Rating(user=796794, product=2100, rating=1.175344818469397),\n Rating(user=2068487, product=2100, rating=4.135496433629301),\n Rating(user=84632, product=2100, rating=3.9837808506369643),\n Rating(user=18797588, product=2100, rating=1.9758460898082797),\n Rating(user=91088, product=2100, rating=3.5840368351676393),\n Rating(user=231030, product=2100, rating=1.34340893408447)]"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 175, "cell_type": "code", "source": "predictions.count()", "outputs": [{"execution_count": 175, "output_type": "execute_result", "data": {"text/plain": "67853"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 176, "cell_type": "code", "source": "article_user_test.count()", "outputs": [{"execution_count": 176, "output_type": "execute_result", "data": {"text/plain": "74324"}, "metadata": {}}], "metadata": {"collapsed": false, "trusted": true}}, {"execution_count": 182, "cell_type": "code", "source": "coverage = predictions.count()/float(article_user_test.count())*100\nprint coverage", "outputs": [{"output_type": "stream", "name": "stdout", "text": "91.2935256445\n"}], "metadata": {"collapsed": false, "trusted": true}}], "nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 2", "name": "python2", "language": "python"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "version": "2.7.10", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython2", "codemirror_mode": {"version": 2, "name": "ipython"}}}}