Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from numpy import array\n",
    "from math import sqrt\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "% matplotlib inline\n",
    "\n",
    "from pyspark import SparkContext\n",
    "from pyspark.mllib.clustering import KMeans, KMeansModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sc = SparkContext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['userId\\tartistId\\tartist\\tplays\\tgender\\tage\\tcountry\\tsignupDate',\n",
       " '03b9c645bc4f578ea1dcb6a975e7ea71fab79da2\\ta16371b9-7d36-497a-a9d4-42b0a0440c5e\\tslowdive\\t0.08359133126934984\\tm\\t28.0\\tGermany\\t2006-09-16']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load and parse the data\n",
    "data = sc.textFile('./lastfm-dataset-360k-small/merged-subset2.csv')\n",
    "data.take(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How the data should look like - [array([ 0.,  0.,  0.]), array([ 0.1,  0.1,  0.1])]\n",
    "\n",
    "### Make a list from every line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = data.map(lambda x: x.split('\\t'))\n",
    "header = data.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of uncleaned data - 10000\n"
     ]
    }
   ],
   "source": [
    "#Remove header\n",
    "data2 = data.filter(lambda line: line!=header)\n",
    "# x[3] --> plays/day; x[4] --> gender; x[5] --> age; x[6] --> Country\n",
    "print (\"length of uncleaned data -\",data2.count())\n",
    "# data2.map(lambda x : len(x[1])).collect()\n",
    "# data2 = data2.filter(lambda x : len(x[1]) == 36) #Clean data - remove artists without artistId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['03b9c645bc4f578ea1dcb6a975e7ea71fab79da2',\n",
       " 'a16371b9-7d36-497a-a9d4-42b0a0440c5e',\n",
       " 'slowdive',\n",
       " '0.08359133126934984',\n",
       " 'm',\n",
       " '28.0',\n",
       " 'Germany',\n",
       " '2006-09-16']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7492"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data2.map(lambda x: x[4]).filter(lambda x: len(x) != 1).count()\n",
    "# data2.map(lambda x: x[5]).filter(lambda x: len(x) < 1).count()\n",
    "# data2.map(lambda x: x[6]).filter(lambda x: len(x) < 1).count()\n",
    "\n",
    "data3 = data2.filter(lambda x: (len(x[4]) == 1) & (len(x[5]) > 1))\n",
    "data3.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "genderdict = {'m':0,'f':1}\n",
    "\n",
    "countrylist = ['United States', 'Germany', 'United Kingdom', 'Poland', 'Russian Federation', 'Brazil', 'Sweden', 'Spain', 'Finland', 'Netherlands', 'Italy', 'France', 'Canada', 'Australia', 'Turkey', 'Norway', 'Czech Republic', 'Ukraine', 'Japan', 'Belgium', 'Mexico', 'Argentina', 'Switzerland', 'Austria', 'Romania', 'Portugal', 'Bulgaria', 'Chile', 'Denmark', 'Colombia', 'Greece', 'Hungary', 'Latvia', 'Slovakia', 'Croatia', 'Serbia', 'Lithuania', 'Estonia', 'Ireland', 'New Zealand', 'Belarus', 'Israel', 'India', 'Venezuela', 'Indonesia', 'Singapore', 'Slovenia', 'Korea, Republic of', 'China', 'South Africa', 'Malaysia', 'Philippines', 'Peru', 'Thailand', 'Moldova', 'Costa Rica', 'Iceland', 'Taiwan', 'Paraguay', 'Bosnia and Herzegovina', 'Antarctica', 'Puerto Rico', 'Georgia', 'Macedonia', 'Uruguay', 'Honduras', 'Barbados', 'Kazakhstan', 'Andorra', 'Saudi Arabia', 'United States Minor Outlying Islands', 'Djibouti', 'Cocos (Keeling) Islands', 'Tunisia', 'Egypt', 'Bolivia', 'Panama', 'Brunei Darussalam', 'Iran, Islamic Republic of', 'Dominican Republic', 'El Salvador', 'Haiti', 'Ecuador', 'Guatemala', 'Morocco', 'Pakistan', 'Burkina Faso', 'Azerbaijan', 'Cambodia', 'Hong Kong', 'Viet Nam', 'United Arab Emirates', 'Jamaica', 'Faroe Islands', 'Somalia', 'Guinea-Bissau', 'Micronesia, Federated States of', 'Tuvalu', \"Cote D'Ivoire\", 'Libyan Arab Jamahiriya', 'Nicaragua', 'Kyrgyzstan', 'Malta', 'Bermuda', 'Luxembourg', 'Kuwait', 'Cyprus', 'Heard Island and Mcdonald Islands', 'Christmas Island', 'Cuba', 'Niue', 'Aruba', 'Vanuatu', 'Dominica', 'Holy See (Vatican City State)', 'Uzbekistan', 'Bhutan', 'Montenegro', 'Reunion', 'Fiji', 'Netherlands Antilles', 'Lebanon', 'Liechtenstein']\n",
    "countrydict = dict(zip(countrylist, [i for i in range(len(countrylist))]))\n",
    "# print(countrydict)\n",
    "\n",
    "# Define a mapper functions\n",
    "def mapr1(key):\n",
    "    \"\"\" Maps numbers to categories (gender)\"\"\"\n",
    "    return genderdict[key]\n",
    "\n",
    "def mapr2(key):\n",
    "    \"\"\" Maps numbers to categories (country)\"\"\"\n",
    "    return countrydict[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7492"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data4 = data3.map(lambda x: [mapr1(x[4]), float(x[5]), mapr2(x[6])])\n",
    "\n",
    "data4.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Evaluate clustering by computing Within Set Sum of Squared Errors\n",
    "def error(point):\n",
    "    center = clusters.centers[clusters.predict(point)]\n",
    "    return sqrt(sum([x**2 for x in (point - center)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Build the model (cluster the data)\n",
    "kdict = dict.fromkeys(range(2,50)) #initialize dictionary with user_id keys\n",
    "for k in kdict.keys():\n",
    "    clusters = KMeans.train(data4, k, maxIterations=100, initializationMode=\"kmeans||\")\n",
    "#     print (k)\n",
    "    WSSSE = data4.map(lambda point: error(point)).reduce(lambda x, y: x + y) #Within Cluster Sum of Squares\n",
    "    kdict[k] = WSSSE\n",
    "# print(\"Within Set Sum of Squared Error = \" + str(WSSSE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x844a860>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VdW5//HPczIyJBAwCSQkBAERiIoQhtrWijigbcVa\nRZSqbX3V3qu2em97qbR2uuqvrZ1utdXq7aCWOtVbq9ViRVBLrQzBiUHDJFMEEmaCkPH5/XH2CYcM\nkECSk+R8369XXmfvdfY+Z+2t5Mlaz1prm7sjIiISLRTrCoiISOej4CAiIo0oOIiISCMKDiIi0oiC\ng4iINKLgICIijSg4iIhIIwoOIiLSiIKDiIg0khjrChyvk046yQsKCmJdDRGRLmXZsmU73D3zWMd1\n2eBQUFBAcXFxrKshItKlmNnGlhynbiUREWlEwUFERBpRcBARkUYUHEREpBEFBxERaSTugkPZvkNM\nf+B1yvYfinVVREQ6rbgLDvfMX8PSDbu456U1sa6KiEin1WXnObTWiNvnUllTV78/Z/Em5izeREpi\niJI7L4phzUREOp9jthzMbISZvRX1s8/MbjWzfmY2z8zWBK8ZUefMNrO1ZlZiZhdGlY8zs+XBe/eY\nmQXlKWb2RFC+2MwK2vpCF86azCVjckhKMABSEkNMG5PDwm9MbuuvEhHp8o4ZHNy9xN3HuPsYYBzw\nIfA0cBsw392HA/ODfcxsFDADGA1MBe4zs4Tg4+4HvgQMD36mBuXXA7vdfRjwc+BHbXN5h2Wlp5KW\nkkhNrQNQVVNHWkoiWWmpbf1VIiJdXmtzDlOAde6+EZgGPByUPwxcGmxPAx5390p3fx9YC0wws4FA\nursvcncHHmlwTuSzngKmRFoVbWlHRSVXT8wnJTHEKdm9Ka+obOuvEBHpFlqbc5gBPBZsZ7v71mB7\nG5AdbOcCi6LO2RKUVQfbDcsj52wGcPcaM9sL9Ad2RH+5md0A3ACQn5/fyqrDA9cUAbC+/AAVlTX1\n+yIicqQWtxzMLBm4BPhTw/eCloC3Yb2a5O4PunuRuxdlZh5zUcFmjRucwaqt+/iwqqYNayci0n20\nplvpIuANd98e7G8PuooIXsuC8lIgL+q8QUFZabDdsPyIc8wsEegD7GxF3Vpl7OC+1NY572zZ215f\nISLSpbUmOFzF4S4lgGeB64Lt64BnospnBCOQhhBOPC8JuqD2mdmkIJ9wbYNzIp91ObAgaI20izPz\nwgOrlm3c3V5fISLSpbUo52BmvYDzgS9HFf8QeNLMrgc2AtMB3H2lmT0JrAJqgJvcvTY450bgIaAH\nMDf4Afgt8AczWwvsIpzbaDcZvZIZmtmLNxQcRESa1KLg4O4HCCeIo8t2Eh691NTxdwF3NVFeDBQ2\nUX4IuKIldWkr4wZn8OKq7bg77TAwSkSkS4u75TMixg3OYM+H1azfcSDWVRER6XTiOjiA8g4iIk2J\n2+Bw8km9SU9NVN5BRKQJcRscQiFj7OAM3tik4CAi0lDcBgeAcfkZrN5ewd6D1bGuiohIpxLfwSHI\nO7yp1oOIyBHiOjickdeXkKG8g4hIA3EdHHqlJDJyYDrL1HIQETlCXAcHCHctvbVpD7V17b5uoIhI\nlxH3wWFsfgYHqmop2bY/1lUREek04j441E+GU9eSiEi9uA8OgzJ6kJmWoqS0iEiUuA8OZsa4/Awt\noyEiEiXugwOEu5Y27fqQsv2HYl0VEZFOQcEBGBvkHd7YuCfGNRER6RwUHIDC3HSSE0JaZ0lEJKDg\nAKQkJlCYm66ktIhIQMEhMG5wBm9v2cMVv/6Xcg8iEvcUHALjBmdQXesUb9jNPS+tiXV1RERiqkXP\nkO7uRtw+l8qaOgAcmLN4E3MWbyIlMUTJnRfFtnIiIjGglgOwcNZkLhmTU7+fmhRi2pgcFn5jcgxr\nJSISOwoOQFZ6Kmkp4UaUAZU1daSlJJKVlhrbiomIxIiCQ2BHRSWn5/bBgelFeZRXVMa6SiIiMaPg\nEHjgmiJuPX84AJePG8QD1xTFuEYiIrGj4BClMKcPACtK98a4JiIisaXgECUrPZXMtBRWlO6LdVVE\nRGJKwaGBwpx0Vn6gloOIxLcWBQcz62tmT5nZe2b2rpl9xMz6mdk8M1sTvGZEHT/bzNaaWYmZXRhV\nPs7Mlgfv3WNmFpSnmNkTQfliMyto6wttqcLcPqwpq+BQdW2sqiAiEnMtbTn8AnjB3U8FzgDeBW4D\n5rv7cGB+sI+ZjQJmAKOBqcB9ZpYQfM79wJeA4cHP1KD8emC3uw8Dfg786ASv67iNzulDbZ3znh4b\nKiJx7JjBwcz6AGcDvwVw9yp33wNMAx4ODnsYuDTYngY87u6V7v4+sBaYYGYDgXR3X+TuDjzS4JzI\nZz0FTIm0KjpaYW46oKS0iMS3lrQchgDlwO/N7E0z+42Z9QKy3X1rcMw2IDvYzgU2R52/JSjLDbYb\nlh9xjrvXAHuB/g0rYmY3mFmxmRWXl5e35PpaLbdvD/r2TFLeQUTiWkuCQyIwFrjf3c8EDhB0IUUE\nLQFv++odyd0fdPcidy/KzMxsl+8wMwpz+rBcLQcRiWMtCQ5bgC3uvjjYf4pwsNgedBURvJYF75cC\neVHnDwrKSoPthuVHnGNmiUAfYGdrL6atjM5Np2TbfqqCxfhEROLNMYODu28DNpvZiKBoCrAKeBa4\nLii7Dngm2H4WmBGMQBpCOPG8JOiC2mdmk4J8wrUNzol81uXAgqA1EhOFOX2ornVWb1dSWkTiU0uX\n7P4K8EczSwbWA18gHFieNLPrgY3AdAB3X2lmTxIOIDXATe4eGRd6I/AQ0AOYG/xAONn9BzNbC+wi\nPNopZgpzwzOlV36wt35bRCSetCg4uPtbQFOLDU1p5vi7gLuaKC8GCpsoPwRc0ZK6dITB/XrSOyWR\nFaX7uHJ8rGsjItLxNEO6CaGQMSonnRUasSQicUrBoRmFOX14d+s+amqVlBaR+KPg0IzC3HQOVdex\nfseBWFdFRKTDKTg0I5KI1kxpEYlHCg7NGJrZm9SkkJbvFpG4pODQjISQMWqgktIiEp8UHI6iMLcP\nqz7YR11dzObjiYjEhILDURTm9KGisoaNuz6MdVVERDqUgsNRjNby3SISpxQcjmJ4VhrJCSHlHUQk\n7ig4HEVyYogRA9JYqRFLIhJnFByOoTA3neWle4nhIrEiIh1OweEYRuf0Ye/BarbsPhjrqoiIdBgF\nh2OIXr5bRCReKDgcw6kD0ggZ/PdfV1G2/1CsqyMi0iEUHI4hNSmB9NQkPth7iHteWhPr6oiIdIiW\nPgkuLo24fS6VUc+RnrN4E3MWbyIlMUTJnRfFsGYiIu1LLYejWDhrMpeMySEpwQBISjCmjclh4Tcm\nx7hmIiLtS8HhKLLSU0lLSaSmzjGgutZJTgyRlZYa66qJiLQrBYdj2FFRycyJg/nFjDEALHl/V4xr\nJCLS/pRzOIYHrimq3359/U7+VLyFtWUVDMvqHcNaiYi0L7UcWuFrF4ygR3ICdzy3SjOmRaRbU3Bo\nhZN6p3DLlOG8urqcBe+Vxbo6IiLtRsGhla47q4Chmb2447lVVNbUxro6IiLtQsGhlZISQnzn06PZ\nsPNDfv/ahlhXR0SkXSg4HIdPnJLJeSOzuHf+GlZ9sJfpD7yupTVEpFtpUXAwsw1mttzM3jKz4qCs\nn5nNM7M1wWtG1PGzzWytmZWY2YVR5eOCz1lrZveYmQXlKWb2RFC+2MwK2vYy2963PjmKqto6bn70\nTZZu2KWlNUSkW2lNy2Gyu49x98jYztuA+e4+HJgf7GNmo4AZwGhgKnCfmSUE59wPfAkYHvxMDcqv\nB3a7+zDg58CPjv+SOsbU//kH1bXO+h0HcA8vrVFw2/OMuH1urKsmInLCTqRbaRrwcLD9MHBpVPnj\n7l7p7u8Da4EJZjYQSHf3RR4eB/pIg3Min/UUMCXSquisFs6azMWnDajfT00KaWkNEek2WhocHHjJ\nzJaZ2Q1BWba7bw22twHZwXYusDnq3C1BWW6w3bD8iHPcvQbYC/RvxXV0uKz0VDJ6JtfvV9bUkZaS\nqKU1RKRbaGlw+Ji7jwEuAm4ys7Oj3wxaAu0+K8zMbjCzYjMrLi8vb++vO6YdFZV8dGg4hl04egDl\nFZUxrpGISNtoUXBw99LgtQx4GpgAbA+6igheI7PCSoG8qNMHBWWlwXbD8iPOMbNEoA+ws4l6POju\nRe5elJmZ2ZKqt6sHriniVzPHkhAyhmf1PmKpDRGRruyYwcHMeplZWmQbuABYATwLXBccdh3wTLD9\nLDAjGIE0hHDieUnQBbXPzCYF+YRrG5wT+azLgQXeRdan6NszmXGDM5i3anusqyIi0mZasvBeNvB0\nkB9OBB519xfMbCnwpJldD2wEpgO4+0ozexJYBdQAN7l7ZCrxjcBDQA9gbvAD8FvgD2a2FthFeLRT\nl3H+yGzu+tu7bNn9IYMyesa6OiIiJ8y6yB/ojRQVFXlxcXGsqwHA+vIKzv3pq3z/ktFcd1ZBrKsj\nItIsM1sWNSWhWZoh3QZOzuzNyZm9eOlddS2JSPeg4NBGzhuZzaL1O9l/qDrWVREROWEKDm3kvJHZ\nVNc6C9fsiHVVREROmIJDGxmb35e+PZN4SaOWRKQbUHBoI4kJIc4dkcXLJWXU1NbFujoiIidEwaEN\nTRmZze4Pq3lj055YV0VE5IQoOLShs085iaQEY75GLYlIF6fg0IbSUpOYdHJ/5ik4iEgXp+DQxs4b\nmc368gOsL6+IdVVERI6bgkMbmzIyC4D575Yd40gRkc5LwaGNDcroyakD0jRbWkS6NAWHdnDeyGyK\nN+5mz4dVsa6KiMhxUXBoB+eNyqa2zvns/f+ibP+hWFdHRKTVFBzawem5fUhNCrGu/AD3vLQm1tUR\nEWm1ljzPQVphxO1zqaw5PEN6zuJNzFm8iZTEECV3XhTDmomItJxaDm1s4azJXDImh5TE8K1NDBnT\nxuSw8BuTY1wzEZGWU3BoY1npqaSlJFJVW0fIoKbOSU4IkZWWGuuqiYi0mIJDO9hRUcnMiYO5/3Nj\nAVi6YVeMayQi0jrKObSDB645/AS+z5yZy9wVWynbf0itBxHpMtRyaGdfnTKc6lrn16+sj3VVRERa\nTMGhnQ05qReXnZnLnMUb2bZXcx5EpGtQcOgAX50ynLo6575X1sa6KiIiLaLg0AHy+vVk+vg8Hl+y\nmdI9B2NdHRGRY1Jw6CA3Tx4GwC8XaMa0iHR+Cg4dJKdvD66akMefirfwxsbdTH/gda27JCKdloJD\nB7px8jASQsbX//Q2Szfs0rpLItJpmbvHug7HpaioyIuLi2NdjVZpuO5ShNZdEpGOYmbL3L3oWMe1\nuOVgZglm9qaZPRfs9zOzeWa2JnjNiDp2tpmtNbMSM7swqnycmS0P3rvHzCwoTzGzJ4LyxWZW0JqL\n7SoWzprM1NHZ9fupSSGtuyQinVJrupVuAd6N2r8NmO/uw4H5wT5mNgqYAYwGpgL3mVlCcM79wJeA\n4cHP1KD8emC3uw8Dfg786LiuppPLSk+lf++U+v3K6jrSUhI1c1pEOp0WBQczGwR8EvhNVPE04OFg\n+2Hg0qjyx9290t3fB9YCE8xsIJDu7os83Jf1SINzIp/1FDAl0qrobnZUVDK9aBC9khPI6ZtKeUVl\nrKskItJIS9dW+h9gFpAWVZbt7luD7W1ApL8kF1gUddyWoKw62G5YHjlnM4C715jZXqA/sKOF9esy\nIusuFZzUi7tfKOEXM86McY1ERBo7ZsvBzD4FlLn7suaOCVoC7Z7ZNrMbzKzYzIrLy8vb++va1efP\nKiAzLYW7Xyihqw4KEJHuqyXdSh8FLjGzDcDjwLlmNgfYHnQVEbyWBceXAnlR5w8KykqD7YblR5xj\nZolAH2Bnw4q4+4PuXuTuRZmZmS26wM6qZ3IiXzl3GEs27OLV1V070IlI93PM4ODus919kLsXEE40\nL3D3zwHPAtcFh10HPBNsPwvMCEYgDSGceF4SdEHtM7NJQT7h2gbnRD7r8uA7uv2f0zPG55PXrwc/\n/nsJdXVHv9yyfYc0cU5EOsyJTIL7IXC+ma0Bzgv2cfeVwJPAKuAF4CZ3rw3OuZFwUnstsA6YG5T/\nFuhvZmuB/yQY+dTdJSeG+I/zTmHlB/uYu2LbUY+9Z/4aTZwTkQ6jSXAxVlvnXPSLf1BT57x469kk\nJhwZrzVxTkTaUptPgpP2kRAyvnbBCNaXH+DPb5Qe8d6W3R8yZWT2EWWaOCciHUGPCe0ELhiVzRl5\nffnpiyU8WbyZH1x2Gk8s3cwjr28Eg5ED0nh3234AKms0cU5E2p9aDp2AmTHrwhFs319J8cbdXPyL\nhfzutfeZNiaHl79+Dvn9e3LBqHALYtKQ/po4JyLtTi2HTqBhXqE6GLn07Nsf8OMrzuCBa4pwd877\n2atU1tTywDWTYlVVEYkTajl0AgtnTeaSMTmkJob/czSVVzAzZozP541Ne1i9fX+sqioicULBoRPI\nSk8lLSWRyto6UhJDzeYVLhubS1KC8diSTTGqqYjECwWHTmJHRSUzJw7m6Rs/ysyJg5vMK/TvncIF\nowfw9JulHKqubeJTRETahnIOnURkQT6AOy8tbPa4q8bn8/w7W/n7ym1MG5Pb7HEiIidCLYcu5qyh\n/cnr10NdSyLSrhQcuphQKJyYXrR+F+/vOBDr6ohIN6Xg0AVdMW4QCSHj8aVqPYhI+1Bw6IKy0lM5\n99Qs/m/ZFqqaWHdJROREKTh0UVdNyGNHRRXz390e66qISDek4NBFfeKULAb2SeWxpZtjXRUR6YYU\nHLqohJBxRVEe/1hdzrRf/lMPARKRNqXg0IVNLwo/dfXtLXv1ECARaVOaBNdFNVysb87iTcxZvImk\nBGP1nRdhZpTtO8TNj73JL68+U0t8i0irqOXQRTVcrC9k4fLqWufjd7/Mnc+t4tvPrNCjRUXkuKjl\n0EU1XKyvqraOy8flMr6gH7P/vJzf/PP9+mMjrQo9WlREWkothy6s4WJ9+w/VcOX4fBbNnsLFhQPq\nWxPJCaZHi4pIq6jl0IU1t1hfVnoqGb2SccCAqlqnrs6VdxCRFlPLoZuKtCp+/4XxpCSGmP9eGXsP\nVse6WiLSRSg4dFMPXFPEnZcWcs6ILB754gSqauq49fE3qQ0eQSoicjQKDnFg4sn9+e4lo3m5pJyf\nvFgS6+qISBegnEOc+NzEfFZ9sJf7X1lHbt8ePPv2B5r/ICLNUsshTpgZ37+kkKLBGXz3mRUsfV/z\nH0SkeWo5xJHTvvf3JmdVa/6DiDR0zJaDmaWa2RIze9vMVprZ94PyfmY2z8zWBK8ZUefMNrO1ZlZi\nZhdGlY8zs+XBe/eYmQXlKWb2RFC+2MwK2v5SJTKrOiXx8H/2BIOrxuex+0AVAGX7DjH9gde1kJ9I\nnGtJt1IlcK67nwGMAaaa2STgNmC+uw8H5gf7mNkoYAYwGpgK3GdmCcFn3Q98CRge/EwNyq8Hdrv7\nMODnwI/a4Nqkgcis6qpgVrUBgzJ68vCijXz87pf56Ysl/PjvJVpyQ0SOHRw8rCLYTQp+HJgGPByU\nPwxcGmxPAx5390p3fx9YC0wws4FAursvcncHHmlwTuSzngKmRFoV0raOmFU9aTCnDkzjhVvO5sOq\nGu5dsJY/LduCe7jLqeC25xlx+9xYV1lEYqBFOYfgL/9lwDDgV+6+2Myy3X1rcMg2IDvYzgUWRZ2+\nJSirDrYblkfO2Qzg7jVmthfoD+xo9RXJUTU3q3rR7CnMeuodXl1djgMpiSGmFg7gW58cGYNaikis\ntWi0krvXuvsYYBDhVkBhg/edcGuiXZnZDWZWbGbF5eXl7f11cSUrPZXcjB71+5U1dfRMTtBQV5E4\n1aqhrO6+B3iZcK5ge9BVRPBaFhxWCuRFnTYoKCsNthuWH3GOmSUCfYCdTXz/g+5e5O5FmZmZram6\ntMCOikpmThrMNy8+FYCFa9RwE4lXLRmtlGlmfYPtHsD5wHvAs8B1wWHXAc8E288CM4IRSEMIJ56X\nBF1Q+8xsUpBPuLbBOZHPuhxYELRGpANFlty44eyh3HjOULbsPsijizcd9RyNbhLpnlrSchgIvGxm\n7wBLgXnu/hzwQ+B8M1sDnBfs4+4rgSeBVcALwE3uXht81o3AbwgnqdcBkWznb4H+ZrYW+E+CkU8S\nO1+7YARnn5LJd59dwbKNu5s97p75azS6SaQbsq76B3pRUZEXFxfHuhrd2p4Pq7jkl69xqLqWh74w\nnu/9dVX9khsNH1MaoQl1Ip2bmS1z96JjHaflM6RZfXsm8+C149h/qIbrfreEpRt28Z2/rODe+WsY\nNTCdhmONx+T15R//pQcKiXQHCg5yVNN++RoHq2spr6jCHV5YuZ2fzlvNW5v3cOrANIzwk+YA3tq8\nh5sefYPV2/cDykeIdGUKDnJUkSU3EoNnjiYlGBcVDmDxt6aQ368nMycN5i83fYzPTcynMCeddeUV\nXPyLhdz9wnv8bN5q5SNEuigtvCdHFVlyo9adlMQQVbV19O+VTFZa6pET6j5zGgC7DlRRdOc87ntl\nXf17WuBPpOtRy0GO6YglNyYOpryistlj+/VKZtHsKZw1tH99TiIhZFx82gAWfkP5CJGuQi0HOabm\nltxoTlZ6KkNO6sXr63eSYEZtnfNKSTkrP9hH1oiOm3Fdtu8QNz/2ph5qJHIc1HKQdhFpbfz15o8x\ntXAABnzh90v5+p/eZu32ig5JVGsOhsjx0zwH6RCVNbXcO38t97+6jqQEo7K6jpkT8+tzFW1JczBE\nmqd5DtKppCQm8L8L11Nb5xyqrsNpv2XB//Ff53BKdu/6/YSQMW1MjnIeIq2g4CAdJjIsNjXp8P92\nZnD1hHw+rKoBTnxuhLszZ/EmVm8PP4IkZFBbF24dK+8g0nIKDtJhIsNiK2uCJ9EZFPTrxe//tYFz\nf/Iqz779wQnlCdydn7xYwr0L1pLbtwefm5jPI1+cQILBkvd3tcMViXRfGq0kHSqSqL56Qj6PLtlE\n+f5D/PiK07ni16/z1cferD+u4dyIo408Cr/3BiMHpPPw6xu5akIed116GqFg4t4NnxjKr19dx7tb\n9zFyYHqHXq9IV6WEtHQKW/cc5N/mLOPtLXvry1ITQ4wfkkHR4P68tXk3r6wu56rxefy/y04/4tzb\nn17OnGBp8ZkT87ljWmF9YADY+2E1H797AeML+vHbz4/vmAsS6aRampBWcJBO41tPL+fRJZtIDBk1\ntU5B/15s2HmgyUcMRn71N/VeU6OS7ntlLXe/UMJT//YRigr6tXXVRboMjVaSLifS5fTMTR9j5qTB\nnDKgN4u/OYWLTxtAckL4f9XEkDHkpF6cPyqbCUP60afH4Z7R1KRQs6OSvnDWEDLTUrj7hRK66h9E\nIh1JOQfpNJqbiZ3RM5nqurr6tZ0+OrR//fyISGsjOSFEZU0daSmJTY5K6pGcwFfPHca3n1nJK6vL\nmTwiq/0vSKQLU8tBOr2jre3UmnWfrhyfT36/nvz4hRLq6tR6EDka5RwkrvzlzVJufeIt7ri0kL++\n/YHWXZK4o5yDSBMuOSOHUwek8aO57zY5n+Jok/D08CKJJwoOEldGfucF3tu2n4rKWtwPL+Ex7Jt/\n41/rdnDn86tYumEXv2hiEl5zE/QUNKQ7UreSxJWyfYe48/l3eX751vplNY5XcmKI1XdexO1PL+eP\nSzYxc0L7LCQo0pZa2q2k0UoSV7LSU0lLTaQu6sl2Vxblcfm4QfWPNa2udRJDxuD+PRmT15cPq2t5\na9Metu87RHQ8qaqpo+C25+v39cQ76U7UrSRxp+EIp90fVlFU0I8hJ/Wipi4cNGrd+cjJ/fnp9DHc\nP3Mc556ahUP9mlCTR2Ty5bNPZkCflPrP1eqv0p2o5SBxp7n5FE2t+3S092ZfPJKKyhoeXbKJEOHV\nX9dsryCzdwoiXZ1yDiIn4Mt/KCYzLZWrxufxH0++xertFfz7OUOZdeEIzKzZ8/QIU4kV5RxEOkB0\nK+SFW87m28+s4P5X1gEcNUBEj3xSEls6I+UcRNpIKGTcMa2QmRPzuf+Vdfz47yVs33vwiGGuI26f\nS8FtzzNn8aYjhtJGPw2vLYfGapitHK9jBgczyzOzl81slZmtNLNbgvJ+ZjbPzNYErxlR58w2s7Vm\nVmJmF0aVjzOz5cF791jwZ5WZpZjZE0H5YjMraPtLFWl/kQBx9cR87ntlHV98qJil7+/iK4++yayn\n3ia3b+MupL49krhyfB7/XLODQ9W1x/XAo+aCwIk8PEni2zFzDmY2EBjo7m+YWRqwDLgU+Dywy91/\naGa3ARnu/g0zGwU8BkwAcoCXgFPcvdbMlgBfBRYDfwPucfe5ZnYjcLq7/5uZzQA+4+5XHq1eyjlI\nZzbi9rlU1tQ1Kg8ZjM5JZ0XpPhITjOpaJ6t3CrsPVlFd2/S/xZY88Cgy1+LKojyumpDPZ+//FzVN\nzOPQMFtpt+c5mNkzwC+Dn3PcfWsQQF5x9xFmNhvA3X8QHP934HvABuBldz81KL8qOP/LkWPc/XUz\nSwS2AZl+lMopOEhnFplsN3fFVqprw8NjpxYO4FufHMm3/7KCzLTUI0Y+/Wz6GF5ctY1fLljL+vIj\nn2ExamAaHxl6Emu272fhmh1MPjWLS87IYUdFJT+Y+16zk/nMAA8/88KAC0Znc8elhUqAx7l2SUgH\n3T1nEv7LP9vdtwZvbQOyg+1cYFHUaVuCsupgu2F55JzNAO5eY2Z7gf7AjgbffwNwA0B+fn5rqi7S\noSKT7SLzJqpqDy8n3txQ2s+cOYjiDbtZv+MAyQkhqmvqGJ2Tzsqt+1i1dX/9cQveK2PBe2UAJIbC\nT8yrrKnDCT/vYmx+X2ZNPZU/v1HKY0s3kRQKt1BeXV3O1j2HFBykRVqckDaz3sD/Abe6+77o94K/\n8Nt9TKy7P+juRe5elJmZ2d5fJ3JCWrOceMNz/nLjR5k5aTC5GT1YPHsKnz59IMmJ4X+uyQkhppya\nxfyvfYI1d13MZ8cNAqN+8t4p2WkUFfRj54HDD0/69OkDAZj+wOs8984HQPN5Ci0+KNDCloOZJREO\nDH909z9MTaSHAAAJbUlEQVQHxdvNbGBUt1JZUF4K5EWdPigoKw22G5ZHn7Ml6FbqA+w8jusR6TSa\nayEczznpPZKorj38wKOBfVIZmtkbaH7yXvRn3Xv1WHZWVPLlPyzj5kffZF3ZAcr3H2o0nPZgVS13\nPLeKpe/v4jvPrOQr5w4jKSFEQshICoW4Z8FqDcGNEy1JSBvwMOHk861R5T8GdkYlpPu5+ywzGw08\nyuGE9HxgeDMJ6Xvd/W9mdhNwWlRC+jJ3n360einnIPEkMtkuOgBE//JvqcqaWkZ++wXa6llH7ZHg\n1gTB9tVmCWkz+xiwEFgORIZffJPwL/gngXxgIzDd3XcF53wL+CJQQ7gbam5QXgQ8BPQA5gJfcXc3\ns1TgD4TzGbuAGe6+/mj1UnAQOT7b9x7k8w8t5d2oPEb/XsmMHdyXzbsOsq68gupaJznBGJPfl0vH\nDKKmro6n3yjlndK99Qnw0wf14cFrihjQ5/h+gTcXBLTKbftqs4f9uPs/3d3c/XR3HxP8/M3dd7r7\nFHcf7u7nRQJDcM5d7j7U3UdEAkNQXuzuhcF7N0dGI7n7IXe/wt2HufuEYwUGETl+2X16MDY/A7PD\nCwleVDiA/712POMGZ9Qn0avrnFOy0rh6Yj7XfqSAUTnp1LmTnBD+tfHOlr186ZFi3t6857jyFw3n\nYHT0BEE5Oi2fIRKHmstTtHjxwcUbWV66l617D3Hpfa8xLLM3a8squPO5VXztghGEzDCDn75YwtL3\nd3Hnc6u4+dzhVFbXcdn9rx0xpyOy1HlT+vdK5vNnFbB170EG9umhZUc6kBbeE5Hjdsrtc6lqYrJf\na/XtmcTHh51E6Z6DvLlpT/0EwT49Etl7sKbZ804k5xGvuQ09Q1pE2t0/Z03mkjE5pATDbJMSwvMs\n/uP84YzJ60tSgtWXjy/I4KdXnMEjX5zAeSOzMMLDcs3gU6cN5N6rx5KZlsLMSeHht5+bNJhJJ/dn\n/tc+wb+fM5Q+PZLqvzdkcNbQ/iycFX52xvF0N2lpkaNTt5KIHLes9FTSUhKpihpmO2pgOrdMOYWy\nfZW8vWVPffmI7LTwnAzgj4s3MnPS0YffRg/l/cbUU9l3sJpHF28iFDJq65x/rdvJjP9dxLWTBrNq\n675mu5sathAaLm3Smif4xVNrQ8FBRE7I8eQvjmcOyI6KyvqA8odFG1hRuo8VpXv53l9X1R8T+UUf\neb43HG4h/Hzeaj55Wg6Xj8vlmbe2UlF5uLvq5MxefP/To3F3zKzZIHA8OY+uGlCUcxCRLqts3yG+\n/tTbvLZmB82sW9ikAekpbN9XWZ/bCBEepz8iO43LxuZSsn0/T79ZypVFedxy3nA+cffLVDXxBdGt\nja4yNLfdFt7rLBQcRATgW08v59Elm0hOCFFVU8cnRmQycUh/3tmyh3+t21Gf0E4IGeMHZ/DDz57O\nD+a+e8Skwg/2HOTcU7P49l9WtHgdoKy0FK7/2BA+c2YuWempjYJAcyvztiSgtCcFBxGJC0ebPX5E\n4KitO+Zf72X7DjH7z8t5dXU5NXVOYsgozO3DlePzeGnVdha8V1bf2uidkkBFZW2r6ppgMCYvg0lD\n+zE2P4MXVmzjqTe2NKrX0YLGiQYUPSZUROLC0fIXR8t7NCUrPZUBfVKp9cOr6RbmpHPVhHxeKSlr\nlET/z/NH8PSbW3h8yWb2HKwGwsujZ6enMrUwm9XbK3h93U6SEkJU19YxLKs3lbW1/OrldUd8byRX\nkhAyfjb9DOau2MbSDbv4yd9L+MFlp5MQOvy42Y6a66GWg4hIlONZx6q5Fkpzn7Vx5wG++eflLH5/\nFzV1TsjCw3oPHceckdbO9VC3kohIB2mrgPKls0/me8+u5LW1O6iqDXdrDcvqzfghGRysqmXx+l2U\n7jlInUNqUogLR4cfINWa7iV1K4mIdJDjHprboMtrcP9e5PTtQXXUQ6KKBmdwx7Rw91EkoKQED3iK\nPECqPSg4iIjEQHMBpcXrW7Ugh3Ii1K0kIhJHtLaSiIgcNwUHERFpRMFBREQaUXAQEZFGFBxERKQR\nBQcREWmkyw5lNbNyYCNwErAjxtWJNd0D3QPQPYj364eW3YPB7p55rA/qssEhwsyKWzJmtzvTPdA9\nAN2DeL9+aNt7oG4lERFpRMFBREQa6Q7B4cFYV6AT0D3QPQDdg3i/fmjDe9Dlcw4iItL2ukPLQURE\n2liXDg5mNtXMSsxsrZndFuv6dAQz+52ZlZnZiqiyfmY2z8zWBK8ZsaxjezKzPDN72cxWmdlKM7sl\nKI+ne5BqZkvM7O3gHnw/KI+bewBgZglm9qaZPRfsx9X1A5jZBjNbbmZvmVlxUNYm96HLBgczSwB+\nBVwEjAKuMrNRsa1Vh3gImNqg7DZgvrsPB+YH+91VDfA1dx8FTAJuCv67x9M9qATOdfczgDHAVDOb\nRHzdA4BbgHej9uPt+iMmu/uYqCGsbXIfumxwACYAa919vbtXAY8D02Jcp3bn7v8AdjUongY8HGw/\nDFzaoZXqQO6+1d3fCLb3E/7lkEt83QN394pgNyn4ceLoHpjZIOCTwG+iiuPm+o+hTe5DVw4OucDm\nqP0tQVk8ynb3rcH2NiA7lpXpKGZWAJwJLCbO7kHQpfIWUAbMc/d4uwf/A8wC6qLK4un6Ixx4ycyW\nmdkNQVmb3Ac9JrSbcXc3s24/BM3MegP/B9zq7vvMrP69eLgH7l4LjDGzvsDTZlbY4P1uew/M7FNA\nmbsvM7NzmjqmO19/Ax9z91IzywLmmdl70W+eyH3oyi2HUiAvan9QUBaPtpvZQIDgtSzG9WlXZpZE\nODD80d3/HBTH1T2IcPc9wMuE81Dxcg8+ClxiZhsIdyefa2ZziJ/rr+fupcFrGfA04e72NrkPXTk4\nLAWGm9kQM0sGZgDPxrhOsfIscF2wfR3wTAzr0q4s3ET4LfCuu/8s6q14ugeZQYsBM+sBnA+8R5zc\nA3ef7e6D3L2A8L/7Be7+OeLk+iPMrJeZpUW2gQuAFbTRfejSk+DM7GLCfY8JwO/c/a4YV6ndmdlj\nwDmEV1/cDnwX+AvwJJBPeKXa6e7eMGndLZjZx4CFwHIO9zd/k3DeIV7uwemEE40JhP/Ae9Ld/9vM\n+hMn9yAi6Fb6urt/Kt6u38xOJtxagHCK4FF3v6ut7kOXDg4iItI+unK3koiItBMFBxERaUTBQURE\nGlFwEBGRRhQcRESkEQUHERFpRMFBREQaUXAQEZFG/j8KrY+nnjmYMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x8466860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(list(kdict.keys()),list(kdict.values()), '*-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34283.92645787765\n"
     ]
    }
   ],
   "source": [
    "k = 10 # number of clusters\n",
    "clusters = KMeans.train(data4, kbest, maxIterations=100, initializationMode=\"kmeans||\")\n",
    "WSSSE = data4.map(lambda point: error(point)).reduce(lambda x, y: x + y) #Within Cluster Sum of Squares\n",
    "print(WSSSE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['userId',\n",
       "  'artistId',\n",
       "  'artist',\n",
       "  'plays',\n",
       "  'gender',\n",
       "  'age',\n",
       "  'country',\n",
       "  'signupDate'],\n",
       " ['03b9c645bc4f578ea1dcb6a975e7ea71fab79da2',\n",
       "  'a16371b9-7d36-497a-a9d4-42b0a0440c5e',\n",
       "  'slowdive',\n",
       "  '0.08359133126934984',\n",
       "  'm',\n",
       "  '28.0',\n",
       "  'Germany',\n",
       "  '2006-09-16']]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters.predict([ 0,  28,  67])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# output = parsedData.map(lambda i: [i, clusters.predict(i)])\n",
    "output = data3.map(lambda x: [x, clusters.predict([mapr1(x[4]), float(x[5]), mapr2(x[6])])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# output.take(10)\n",
    "a = np.empty((5,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  4.61986920e+281,   7.10004881e-154,   2.19529496e-152,\n",
       "         1.14974967e+171,   2.05132425e+161,   2.02439631e-258,\n",
       "         1.32108724e+272,   3.41903149e+228,   3.44568507e+228,\n",
       "         3.30804647e-027])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a[0,:] = [5,2,4,1,3,8,9,7,0,6]\n",
    "# a.sort()\n",
    "a[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for j in range(5):\n",
    "    for i in range(10):\n",
    "        clusters = KMeans.train(data4, kbest, maxIterations=100, initializationMode=\"kmeans||\")\n",
    "        output = data4.map(lambda i: [i, clusters.predict(i)])\n",
    "        a[j,i] = output.filter(lambda x: x[1] == i).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   33.,    35.,   173.,   325.,   389.,   410.,   420.,   551.,\n",
       "          764.,  1446.],\n",
       "       [   29.,    55.,    57.,   324.,   387.,   683.,  1271.,  1345.,\n",
       "         2543.,  2710.],\n",
       "       [   12.,   134.,   183.,   381.,   464.,   749.,   781.,  1275.,\n",
       "         1297.,  2102.],\n",
       "       [  397.,   427.,   444.,   488.,   631.,  1264.,  1305.,  1345.,\n",
       "         1719.,  2849.],\n",
       "       [   34.,    36.,    74.,   127.,   263.,   433.,   445.,   459.,\n",
       "          741.,  3126.]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.sort()\n",
    "a #Visualize the cluster sizes formed through each iteration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#save the model\n",
    "clusters.save(sc, \"./model/clustering1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine top 20 artists for every cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(k):\n",
    "    summation = output.filter(lambda x: x[1] == 0).map(lambda x: (x[0][1], float(x[0][3]))).reduceByKey(lambda x,y : x+y)\n",
    "    count = output.filter(lambda x: x[1] == 0).map(lambda x: (x[0][1], 1)).reduceByKey(lambda x,y : x+y)\n",
    "    average = summation.join(count).map(lambda x: (x[0], x[1][0]/x[1][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('d7ac3cc7-0d65-476b-9b81-d3e872672a2f', 0.9721869110940131),\n",
       " ('592b9a1b-b900-4c1a-a165-bd7e7ce87d23', 0.20020534984733152)]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('d7ac3cc7-0d65-476b-9b81-d3e872672a2f', (0.9721869110940131, 2)),\n",
       " ('592b9a1b-b900-4c1a-a165-bd7e7ce87d23', (0.20020534984733152, 2))]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('c3cceeed-3332-4cf0-8c4c-bbde425147b6', 5.796116504854369),\n",
       " ('b7ffd2af-418f-4be2-bdd1-22f8b48613da', 4.2894736842105265),\n",
       " ('ebfc1398-8d96-47e3-82c3-f782abcdb13d', 3.7221324717285946),\n",
       " ('42689657-cbec-4f66-a9ed-80f939ea23ed', 3.4745762711864407),\n",
       " ('55b92a4f-34eb-4d89-8f84-75812c804ab4', 2.4459567654123298),\n",
       " ('487bfd74-71bf-46dd-b89c-80b7a0f06f2f', 2.3873873873873874),\n",
       " ('f82bcf78-5b69-4622-a5ef-73800768d9ac', 2.3535353535353534),\n",
       " ('1c3919b2-43ca-4a4a-935d-9d50135ec0ef', 2.2062326322925747),\n",
       " ('06aaeca4-2008-4e48-a2bf-119f3d363b4f', 1.9978213507625273),\n",
       " ('cb9b28ec-799f-44d8-a0b2-2eee3e8eb5b1', 1.839907192575406),\n",
       " ('7bbfd77c-1102-4831-9ba8-246fb67460b3', 1.82996803343686),\n",
       " ('067102ea-9519-4622-9077-57ca4164cfbb', 1.6213201575921978),\n",
       " ('125948ec-7f91-4d1a-8b83-accbf50fae3d', 1.597139451728248),\n",
       " ('17167af8-c1da-45cc-bba2-9d23f068b7a3', 1.5411431861131246),\n",
       " ('8c538f11-c141-4588-8ecb-931083524186', 1.5105189679065973),\n",
       " ('674a7e8c-9682-419a-8e05-2358e28b5359', 1.466878313908609),\n",
       " ('820efdc9-3ad8-4893-8172-c9f62ee53a59', 1.463855421686747),\n",
       " ('9390a27f-d63d-43ac-a771-a0e0794fee61', 1.3893939393939394),\n",
       " ('d87e52c5-bb8d-4da8-b941-9f4928627dc8', 1.3546168958742633),\n",
       " ('08f1e9cc-9f22-46f2-9a7b-c6e53e2aa210', 1.347024308466052)]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average.map(lambda x: (x[1],x[0])).sortByKey(0, 1).map(lambda x: (x[1],x[0])).take(20)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}