spark-basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# In this notebook, we'll explore a few methods in Spark Core\n",
    "### Creating RDDs, collect(), map(), join(), reduceByKey(), aggregateByKey(), and filter()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## start with an example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We'll start with an online-dating dataset, described here: https://sites.google.com/a/insightdatascience.com/spark-lab/s3-data/dating-profiles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Here we create an RDD from a csv stored in s3, and use the collect() action, which returns an array\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'10001,Tony',\n",
       " u'10002,Mike',\n",
       " u'10003,Pat',\n",
       " u'10004,Chris',\n",
       " u'10005,Paco',\n",
       " u'10006,Eddie',\n",
       " u'90001,Lisa',\n",
       " u'90002,Cindy',\n",
       " u'90003,Paula',\n",
       " u'90004,Leslie',\n",
       " u'90005,Allman',\n",
       " u'90006,Kimberly']"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#a csv of id, users\n",
    "rawUsersRDD = sc.textFile(\"s3n://insight-spark-after-dark/users-sm.csv\")\n",
    "rawUsersRDD.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The collect action causes data to flow across the network from the worker nodes to the master (where you are running the jupyter notebook, or your data analysis)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'10001,M',\n",
       " u'10002,M',\n",
       " u'10003,M',\n",
       " u'10004,M',\n",
       " u'10005,M',\n",
       " u'10006,M',\n",
       " u'90001,F',\n",
       " u'90002,F',\n",
       " u'90003,F',\n",
       " u'90004,F',\n",
       " u'90005,F',\n",
       " u'90006,F']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#a csv of id, genders\n",
    "rawGendersRDD = sc.textFile(\"s3n://insight-spark-after-dark/gender-sm.csv\")\n",
    "rawGendersRDD.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rec_tup(record):\n",
    "    tokens = record.split(\",\")\n",
    "    return (int(tokens[0]), str(tokens[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Moving toward a join action, we can use the map() method to create Key/Value pairs from the tuple"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(10001, 'Tony'),\n",
       " (10002, 'Mike'),\n",
       " (10003, 'Pat'),\n",
       " (10004, 'Chris'),\n",
       " (10005, 'Paco'),\n",
       " (10006, 'Eddie'),\n",
       " (90001, 'Lisa'),\n",
       " (90002, 'Cindy'),\n",
       " (90003, 'Paula'),\n",
       " (90004, 'Leslie'),\n",
       " (90005, 'Allman'),\n",
       " (90006, 'Kimberly')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usersRDD = rawUsersRDD.map(rec_tup)\n",
    "usersRDD.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(10001, 'M'),\n",
       " (10002, 'M'),\n",
       " (10003, 'M'),\n",
       " (10004, 'M'),\n",
       " (10005, 'M'),\n",
       " (10006, 'M'),\n",
       " (90001, 'F'),\n",
       " (90002, 'F'),\n",
       " (90003, 'F'),\n",
       " (90004, 'F'),\n",
       " (90005, 'F'),\n",
       " (90006, 'F')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gendersRDD = rawGendersRDD.map(rec_tup)\n",
    "gendersRDD.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now that we have two RDDs with Key/Value pairs, use the join method to join the RDDs based on the Key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(10004, ('Chris', 'M')),\n",
       " (90004, ('Leslie', 'F')),\n",
       " (90005, ('Allman', 'F')),\n",
       " (90001, ('Lisa', 'F')),\n",
       " (10005, ('Paco', 'M')),\n",
       " (10001, ('Tony', 'M')),\n",
       " (90006, ('Kimberly', 'F')),\n",
       " (10002, ('Mike', 'M')),\n",
       " (90002, ('Cindy', 'F')),\n",
       " (10006, ('Eddie', 'M')),\n",
       " (10003, ('Pat', 'M')),\n",
       " (90003, ('Paula', 'F'))]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usersWithGenderJoinedRDD = usersRDD.join(gendersRDD)\n",
    "usersWithGenderJoinedRDD.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Next Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are some further questions to get to grips with Spark basics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 1: There exists another zipped csv, s3n://insight-spark-after-dark/gender.csv.gz. Create an RDD called genders_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'1,F', u'2,F', u'3,U', u'4,F', u'5,F']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genders_raw = sc.textFile(\"s3n://insight-spark-after-dark/gender.csv.gz\")\n",
    "genders_raw.take(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 2: Parse each of the record  from the genderRDDD so that we have a tuple of (gender(string), id(int)) and show \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 'F'), (2, 'F'), (3, 'U'), (4, 'F'), (5, 'F')]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genders = genders_raw.map(rec_tup)\n",
    "genders.take(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 3: Calculate the distribution of Males, Females, and Unknowns and show the results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nahiyan's solution. key=gender, value=True, then will sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "genderBool = lambda x: (x[1], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('U', 83164), ('M', 76441), ('F', 61365)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(g[0], sum(g[1]))\n",
    " for g in genders.map(genderBool)\\\n",
    "                .groupByKey().collect()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nahiyan used reduceByKey - smart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F': 61365, 'M': 76441, 'U': 83164}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts = genders.map(genderBool)\\\n",
    "        .reduceByKey(lambda a,b: a+b)\\\n",
    "        .collect()\n",
    "\n",
    "d_counts = {k:v for k,v in counts}\n",
    "d_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 4: Calculate the sum of all the IDs for each gender\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F': 4325473098, 'M': 6982853010, 'U': 13105654827}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swap = lambda x: (x[1], x[0])\n",
    "\n",
    "sums = genders.map(swap).reduceByKey(lambda a,b: a+b).collect()\n",
    "d_sums = {k:v for k,v in sums}\n",
    "d_sums"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 5: Calculate the average of all the IDs for each gender\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F': 70487, 'M': 91349, 'U': 157588}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgs = {}\n",
    "for gender, count in d_counts.items():\n",
    "    avgs[gender] = d_sums[gender] / count\n",
    "\n",
    "avgs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Task 6: Only return records from parsed_rdd which have ids that are a multiple of 5 and show the first 5 results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(5, 'F'),\n",
       " (10, 'M'),\n",
       " (15, 'U'),\n",
       " (20, 'M'),\n",
       " (25, 'F'),\n",
       " (30, 'M'),\n",
       " (35, 'F'),\n",
       " (40, 'F'),\n",
       " (45, 'M'),\n",
       " (50, 'F'),\n",
       " (55, 'F'),\n",
       " (60, 'F'),\n",
       " (65, 'F'),\n",
       " (70, 'M'),\n",
       " (75, 'F'),\n",
       " (80, 'M'),\n",
       " (85, 'M'),\n",
       " (90, 'F'),\n",
       " (95, 'M'),\n",
       " (100, 'F'),\n",
       " (105, 'F'),\n",
       " (110, 'F'),\n",
       " (115, 'F'),\n",
       " (120, 'M'),\n",
       " (125, 'U'),\n",
       " (130, 'M'),\n",
       " (135, 'F'),\n",
       " (140, 'F'),\n",
       " (145, 'F'),\n",
       " (150, 'M'),\n",
       " (155, 'U'),\n",
       " (160, 'F'),\n",
       " (165, 'F'),\n",
       " (170, 'M'),\n",
       " (175, 'M'),\n",
       " (180, 'F'),\n",
       " (185, 'F'),\n",
       " (190, 'F'),\n",
       " (195, 'F'),\n",
       " (200, 'M'),\n",
       " (205, 'M'),\n",
       " (210, 'F'),\n",
       " (215, 'F'),\n",
       " (220, 'F'),\n",
       " (225, 'M'),\n",
       " (230, 'M'),\n",
       " (235, 'U'),\n",
       " (240, 'M'),\n",
       " (245, 'M'),\n",
       " (250, 'M'),\n",
       " (255, 'M'),\n",
       " (260, 'F'),\n",
       " (265, 'M'),\n",
       " (270, 'M'),\n",
       " (275, 'F'),\n",
       " (280, 'F'),\n",
       " (285, 'M'),\n",
       " (290, 'F'),\n",
       " (295, 'M'),\n",
       " (300, 'M'),\n",
       " (305, 'U'),\n",
       " (310, 'F'),\n",
       " (315, 'M'),\n",
       " (320, 'F'),\n",
       " (325, 'M'),\n",
       " (330, 'F'),\n",
       " (335, 'M'),\n",
       " (340, 'F'),\n",
       " (345, 'M'),\n",
       " (350, 'U'),\n",
       " (355, 'M'),\n",
       " (360, 'M'),\n",
       " (365, 'M'),\n",
       " (370, 'F'),\n",
       " (375, 'M'),\n",
       " (380, 'F'),\n",
       " (385, 'F'),\n",
       " (390, 'M'),\n",
       " (395, 'U'),\n",
       " (400, 'M'),\n",
       " (405, 'M'),\n",
       " (410, 'M'),\n",
       " (415, 'M'),\n",
       " (420, 'M'),\n",
       " (425, 'M'),\n",
       " (430, 'M'),\n",
       " (435, 'M'),\n",
       " (440, 'M'),\n",
       " (445, 'F'),\n",
       " (450, 'F'),\n",
       " (455, 'F'),\n",
       " (460, 'F'),\n",
       " (465, 'F'),\n",
       " (470, 'M'),\n",
       " (475, 'F'),\n",
       " (480, 'F'),\n",
       " (485, 'F'),\n",
       " (490, 'F'),\n",
       " (495, 'F'),\n",
       " (500, 'M'),\n",
       " (505, 'F'),\n",
       " (510, 'M'),\n",
       " (515, 'U'),\n",
       " (520, 'F'),\n",
       " (525, 'M'),\n",
       " (530, 'M'),\n",
       " (535, 'F'),\n",
       " (540, 'U'),\n",
       " (545, 'M'),\n",
       " (550, 'F'),\n",
       " (555, 'M'),\n",
       " (560, 'M'),\n",
       " (565, 'F'),\n",
       " (570, 'F'),\n",
       " (575, 'M'),\n",
       " (580, 'F'),\n",
       " (585, 'M'),\n",
       " (590, 'F'),\n",
       " (595, 'M'),\n",
       " (600, 'M'),\n",
       " (605, 'M'),\n",
       " (610, 'M'),\n",
       " (615, 'M'),\n",
       " (620, 'M'),\n",
       " (625, 'U'),\n",
       " (630, 'F'),\n",
       " (635, 'M'),\n",
       " (640, 'M'),\n",
       " (645, 'U'),\n",
       " (650, 'F'),\n",
       " (655, 'M'),\n",
       " (660, 'M'),\n",
       " (665, 'F'),\n",
       " (670, 'M'),\n",
       " (675, 'U'),\n",
       " (680, 'M'),\n",
       " (685, 'U'),\n",
       " (690, 'M'),\n",
       " (695, 'M'),\n",
       " (700, 'F'),\n",
       " (705, 'F'),\n",
       " (710, 'U'),\n",
       " (715, 'F'),\n",
       " (720, 'F'),\n",
       " (725, 'U'),\n",
       " (730, 'F'),\n",
       " (735, 'M'),\n",
       " (740, 'U'),\n",
       " (745, 'F'),\n",
       " (750, 'M'),\n",
       " (755, 'M'),\n",
       " (760, 'M'),\n",
       " (765, 'F'),\n",
       " (770, 'F'),\n",
       " (775, 'M'),\n",
       " (780, 'F'),\n",
       " (785, 'F'),\n",
       " (790, 'M'),\n",
       " (795, 'M'),\n",
       " (800, 'U'),\n",
       " (805, 'F'),\n",
       " (810, 'M'),\n",
       " (815, 'U'),\n",
       " (820, 'M'),\n",
       " (825, 'F'),\n",
       " (830, 'F'),\n",
       " (835, 'M'),\n",
       " (840, 'M'),\n",
       " (845, 'M'),\n",
       " (850, 'F'),\n",
       " (855, 'F'),\n",
       " (860, 'M'),\n",
       " (865, 'F'),\n",
       " (870, 'M'),\n",
       " (875, 'M'),\n",
       " (880, 'F'),\n",
       " (885, 'M'),\n",
       " (890, 'F'),\n",
       " (895, 'F'),\n",
       " (900, 'M'),\n",
       " (905, 'M'),\n",
       " (910, 'F'),\n",
       " (915, 'F'),\n",
       " (920, 'M'),\n",
       " (925, 'F'),\n",
       " (930, 'F'),\n",
       " (935, 'F'),\n",
       " (940, 'F'),\n",
       " (945, 'F'),\n",
       " (950, 'F'),\n",
       " (955, 'M'),\n",
       " (960, 'M'),\n",
       " (965, 'F'),\n",
       " (970, 'M'),\n",
       " (975, 'F'),\n",
       " (980, 'U'),\n",
       " (985, 'M'),\n",
       " (990, 'M'),\n",
       " (995, 'U'),\n",
       " (1000, 'M'),\n",
       " (1005, 'M'),\n",
       " (1010, 'F'),\n",
       " (1015, 'F'),\n",
       " (1020, 'F'),\n",
       " (1025, 'U'),\n",
       " (1030, 'M'),\n",
       " (1035, 'F'),\n",
       " (1040, 'F'),\n",
       " (1045, 'F'),\n",
       " (1050, 'M'),\n",
       " (1055, 'M'),\n",
       " (1060, 'U'),\n",
       " (1065, 'F'),\n",
       " (1070, 'F'),\n",
       " (1075, 'U'),\n",
       " (1080, 'F'),\n",
       " (1085, 'F'),\n",
       " (1090, 'F'),\n",
       " (1095, 'M'),\n",
       " (1100, 'F'),\n",
       " (1105, 'F'),\n",
       " (1110, 'M'),\n",
       " (1115, 'M'),\n",
       " (1120, 'M'),\n",
       " (1125, 'F'),\n",
       " (1130, 'M'),\n",
       " (1135, 'F'),\n",
       " (1140, 'M'),\n",
       " (1145, 'F'),\n",
       " (1150, 'U'),\n",
       " (1155, 'M'),\n",
       " (1160, 'F'),\n",
       " (1165, 'U'),\n",
       " (1170, 'F'),\n",
       " (1175, 'F'),\n",
       " (1180, 'F'),\n",
       " (1185, 'F'),\n",
       " (1190, 'F'),\n",
       " (1195, 'M'),\n",
       " (1200, 'F'),\n",
       " (1205, 'F'),\n",
       " (1210, 'M'),\n",
       " (1215, 'M'),\n",
       " (1220, 'M'),\n",
       " (1225, 'F'),\n",
       " (1230, 'F'),\n",
       " (1235, 'M'),\n",
       " (1240, 'F'),\n",
       " (1245, 'F'),\n",
       " (1250, 'M'),\n",
       " (1255, 'U'),\n",
       " (1260, 'M'),\n",
       " (1265, 'M'),\n",
       " (1270, 'F'),\n",
       " (1275, 'F'),\n",
       " (1280, 'U'),\n",
       " (1285, 'M'),\n",
       " (1290, 'F'),\n",
       " (1295, 'F'),\n",
       " (1300, 'F'),\n",
       " (1305, 'F'),\n",
       " (1310, 'F'),\n",
       " (1315, 'F'),\n",
       " (1320, 'F'),\n",
       " (1325, 'M'),\n",
       " (1330, 'F'),\n",
       " (1335, 'M'),\n",
       " (1340, 'U'),\n",
       " (1345, 'F'),\n",
       " (1350, 'F'),\n",
       " (1355, 'M'),\n",
       " (1360, 'F'),\n",
       " (1365, 'F'),\n",
       " (1370, 'M'),\n",
       " (1375, 'M'),\n",
       " (1380, 'M'),\n",
       " (1385, 'M'),\n",
       " (1390, 'F'),\n",
       " (1395, 'M'),\n",
       " (1400, 'M'),\n",
       " (1405, 'M'),\n",
       " (1410, 'M'),\n",
       " (1415, 'M'),\n",
       " (1420, 'M'),\n",
       " (1425, 'F'),\n",
       " (1430, 'U'),\n",
       " (1435, 'M'),\n",
       " (1440, 'F'),\n",
       " (1445, 'M'),\n",
       " (1450, 'U'),\n",
       " (1455, 'F'),\n",
       " (1460, 'M'),\n",
       " (1465, 'F'),\n",
       " (1470, 'F'),\n",
       " (1475, 'M'),\n",
       " (1480, 'F'),\n",
       " (1485, 'F'),\n",
       " (1490, 'F'),\n",
       " (1495, 'F'),\n",
       " (1500, 'M'),\n",
       " (1505, 'U'),\n",
       " (1510, 'M'),\n",
       " (1515, 'M'),\n",
       " (1520, 'F'),\n",
       " (1525, 'F'),\n",
       " (1530, 'U'),\n",
       " (1535, 'F'),\n",
       " (1540, 'F'),\n",
       " (1545, 'M'),\n",
       " (1550, 'M'),\n",
       " (1555, 'M'),\n",
       " (1560, 'F'),\n",
       " (1565, 'M'),\n",
       " (1570, 'M'),\n",
       " (1575, 'M'),\n",
       " (1580, 'F'),\n",
       " (1585, 'F'),\n",
       " (1590, 'M'),\n",
       " (1595, 'M'),\n",
       " (1600, 'F'),\n",
       " (1605, 'F'),\n",
       " (1610, 'F'),\n",
       " (1615, 'M'),\n",
       " (1620, 'M'),\n",
       " (1625, 'F'),\n",
       " (1630, 'F'),\n",
       " (1635, 'F'),\n",
       " (1640, 'F'),\n",
       " (1645, 'F'),\n",
       " (1650, 'M'),\n",
       " (1655, 'F'),\n",
       " (1660, 'M'),\n",
       " (1665, 'M'),\n",
       " (1670, 'F'),\n",
       " (1675, 'M'),\n",
       " (1680, 'F'),\n",
       " (1685, 'F'),\n",
       " (1690, 'F'),\n",
       " (1695, 'M'),\n",
       " (1700, 'M'),\n",
       " (1705, 'M'),\n",
       " (1710, 'M'),\n",
       " (1715, 'U'),\n",
       " (1720, 'F'),\n",
       " (1725, 'M'),\n",
       " (1730, 'F'),\n",
       " (1735, 'U'),\n",
       " (1740, 'F'),\n",
       " (1745, 'F'),\n",
       " (1750, 'F'),\n",
       " (1755, 'F'),\n",
       " (1760, 'M'),\n",
       " (1765, 'U'),\n",
       " (1770, 'M'),\n",
       " (1775, 'M'),\n",
       " (1780, 'U'),\n",
       " (1785, 'U'),\n",
       " (1790, 'M'),\n",
       " (1795, 'M'),\n",
       " (1800, 'M'),\n",
       " (1805, 'F'),\n",
       " (1810, 'M'),\n",
       " (1815, 'M'),\n",
       " (1820, 'F'),\n",
       " (1825, 'M'),\n",
       " (1830, 'F'),\n",
       " (1835, 'F'),\n",
       " (1840, 'F'),\n",
       " (1845, 'M'),\n",
       " (1850, 'F'),\n",
       " (1855, 'M'),\n",
       " (1860, 'M'),\n",
       " (1865, 'F'),\n",
       " (1870, 'M'),\n",
       " (1875, 'F'),\n",
       " (1880, 'F'),\n",
       " (1885, 'M'),\n",
       " (1890, 'M'),\n",
       " (1895, 'F'),\n",
       " (1900, 'M'),\n",
       " (1905, 'F'),\n",
       " (1910, 'M'),\n",
       " (1915, 'U'),\n",
       " (1920, 'M'),\n",
       " (1925, 'M'),\n",
       " (1930, 'M'),\n",
       " (1935, 'F'),\n",
       " (1940, 'F'),\n",
       " (1945, 'U'),\n",
       " (1950, 'M'),\n",
       " (1955, 'M'),\n",
       " (1960, 'F'),\n",
       " (1965, 'M'),\n",
       " (1970, 'F'),\n",
       " (1975, 'U'),\n",
       " (1980, 'M'),\n",
       " (1985, 'M'),\n",
       " (1990, 'U'),\n",
       " (1995, 'F'),\n",
       " (2000, 'M'),\n",
       " (2005, 'F'),\n",
       " (2010, 'M'),\n",
       " (2015, 'M'),\n",
       " (2020, 'F'),\n",
       " (2025, 'F'),\n",
       " (2030, 'M'),\n",
       " (2035, 'M'),\n",
       " (2040, 'F'),\n",
       " (2045, 'M'),\n",
       " (2050, 'F'),\n",
       " (2055, 'M'),\n",
       " (2060, 'F'),\n",
       " (2065, 'M'),\n",
       " (2070, 'U'),\n",
       " (2075, 'M'),\n",
       " (2080, 'U'),\n",
       " (2085, 'M'),\n",
       " (2090, 'M'),\n",
       " (2095, 'U'),\n",
       " (2100, 'F'),\n",
       " (2105, 'M'),\n",
       " (2110, 'F'),\n",
       " (2115, 'F'),\n",
       " (2120, 'M'),\n",
       " (2125, 'F'),\n",
       " (2130, 'M'),\n",
       " (2135, 'F'),\n",
       " (2140, 'F'),\n",
       " (2145, 'M'),\n",
       " (2150, 'M'),\n",
       " (2155, 'M'),\n",
       " (2160, 'M'),\n",
       " (2165, 'M'),\n",
       " (2170, 'M'),\n",
       " (2175, 'F'),\n",
       " (2180, 'F'),\n",
       " (2185, 'U'),\n",
       " (2190, 'F'),\n",
       " (2195, 'M'),\n",
       " (2200, 'M'),\n",
       " (2205, 'F'),\n",
       " (2210, 'F'),\n",
       " (2215, 'M'),\n",
       " (2220, 'M'),\n",
       " (2225, 'M'),\n",
       " (2230, 'M'),\n",
       " (2235, 'M'),\n",
       " (2240, 'M'),\n",
       " (2245, 'M'),\n",
       " (2250, 'F'),\n",
       " (2255, 'M'),\n",
       " (2260, 'M'),\n",
       " (2265, 'F'),\n",
       " (2270, 'U'),\n",
       " (2275, 'M'),\n",
       " (2280, 'M'),\n",
       " (2285, 'M'),\n",
       " (2290, 'F'),\n",
       " (2295, 'M'),\n",
       " (2300, 'M'),\n",
       " (2305, 'M'),\n",
       " (2310, 'M'),\n",
       " (2315, 'F'),\n",
       " (2320, 'F'),\n",
       " (2325, 'F'),\n",
       " (2330, 'M'),\n",
       " (2335, 'M'),\n",
       " (2340, 'M'),\n",
       " (2345, 'M'),\n",
       " (2350, 'F'),\n",
       " (2355, 'F'),\n",
       " (2360, 'F'),\n",
       " (2365, 'U'),\n",
       " (2370, 'M'),\n",
       " (2375, 'M'),\n",
       " (2380, 'M'),\n",
       " (2385, 'M'),\n",
       " (2390, 'M'),\n",
       " (2395, 'M'),\n",
       " (2400, 'M'),\n",
       " (2405, 'F'),\n",
       " (2410, 'F'),\n",
       " (2415, 'F'),\n",
       " (2420, 'F'),\n",
       " (2425, 'U'),\n",
       " (2430, 'M'),\n",
       " (2435, 'M'),\n",
       " (2440, 'M'),\n",
       " (2445, 'F'),\n",
       " (2450, 'F'),\n",
       " (2455, 'F'),\n",
       " (2460, 'F'),\n",
       " (2465, 'M'),\n",
       " (2470, 'M'),\n",
       " (2475, 'F'),\n",
       " (2480, 'M'),\n",
       " (2485, 'F'),\n",
       " (2490, 'M'),\n",
       " (2495, 'F'),\n",
       " (2500, 'F'),\n",
       " (2505, 'M'),\n",
       " (2510, 'U'),\n",
       " (2515, 'M'),\n",
       " (2520, 'F'),\n",
       " (2525, 'F'),\n",
       " (2530, 'F'),\n",
       " (2535, 'M'),\n",
       " (2540, 'F'),\n",
       " (2545, 'M'),\n",
       " (2550, 'U'),\n",
       " (2555, 'M'),\n",
       " (2560, 'F'),\n",
       " (2565, 'M'),\n",
       " (2570, 'F'),\n",
       " (2575, 'F'),\n",
       " (2580, 'F'),\n",
       " (2585, 'F'),\n",
       " (2590, 'F'),\n",
       " (2595, 'M'),\n",
       " (2600, 'F'),\n",
       " (2605, 'U'),\n",
       " (2610, 'M'),\n",
       " (2615, 'F'),\n",
       " (2620, 'F'),\n",
       " (2625, 'U'),\n",
       " (2630, 'M'),\n",
       " (2635, 'F'),\n",
       " (2640, 'F'),\n",
       " (2645, 'F'),\n",
       " (2650, 'M'),\n",
       " (2655, 'U'),\n",
       " (2660, 'M'),\n",
       " (2665, 'F'),\n",
       " (2670, 'M'),\n",
       " (2675, 'M'),\n",
       " (2680, 'U'),\n",
       " (2685, 'F'),\n",
       " (2690, 'F'),\n",
       " (2695, 'M'),\n",
       " (2700, 'F'),\n",
       " (2705, 'U'),\n",
       " (2710, 'F'),\n",
       " (2715, 'F'),\n",
       " (2720, 'F'),\n",
       " (2725, 'F'),\n",
       " (2730, 'M'),\n",
       " (2735, 'M'),\n",
       " (2740, 'M'),\n",
       " (2745, 'M'),\n",
       " (2750, 'M'),\n",
       " (2755, 'M'),\n",
       " (2760, 'F'),\n",
       " (2765, 'F'),\n",
       " (2770, 'M'),\n",
       " (2775, 'M'),\n",
       " (2780, 'M'),\n",
       " (2785, 'F'),\n",
       " (2790, 'F'),\n",
       " (2795, 'M'),\n",
       " (2800, 'F'),\n",
       " (2805, 'M'),\n",
       " (2810, 'M'),\n",
       " (2815, 'M'),\n",
       " (2820, 'F'),\n",
       " (2825, 'M'),\n",
       " (2830, 'M'),\n",
       " (2835, 'F'),\n",
       " (2840, 'F'),\n",
       " (2845, 'F'),\n",
       " (2850, 'F'),\n",
       " (2855, 'F'),\n",
       " (2860, 'F'),\n",
       " (2865, 'F'),\n",
       " (2870, 'F'),\n",
       " (2875, 'F'),\n",
       " (2880, 'M'),\n",
       " (2885, 'F'),\n",
       " (2890, 'M'),\n",
       " (2895, 'M'),\n",
       " (2900, 'F'),\n",
       " (2905, 'F'),\n",
       " (2910, 'M'),\n",
       " (2915, 'M'),\n",
       " (2920, 'U'),\n",
       " (2925, 'M'),\n",
       " (2930, 'F'),\n",
       " (2935, 'F'),\n",
       " (2940, 'F'),\n",
       " (2945, 'F'),\n",
       " (2950, 'F'),\n",
       " (2955, 'M'),\n",
       " (2960, 'F'),\n",
       " (2965, 'M'),\n",
       " (2970, 'M'),\n",
       " (2975, 'U'),\n",
       " (2980, 'F'),\n",
       " (2985, 'F'),\n",
       " (2990, 'M'),\n",
       " (2995, 'F'),\n",
       " (3000, 'F'),\n",
       " (3005, 'M'),\n",
       " (3010, 'M'),\n",
       " (3015, 'M'),\n",
       " (3020, 'F'),\n",
       " (3025, 'M'),\n",
       " (3030, 'F'),\n",
       " (3035, 'F'),\n",
       " (3040, 'U'),\n",
       " (3045, 'F'),\n",
       " (3050, 'U'),\n",
       " (3055, 'F'),\n",
       " (3060, 'F'),\n",
       " (3065, 'M'),\n",
       " (3070, 'M'),\n",
       " (3075, 'U'),\n",
       " (3080, 'F'),\n",
       " (3085, 'M'),\n",
       " (3090, 'M'),\n",
       " (3095, 'M'),\n",
       " (3100, 'F'),\n",
       " (3105, 'M'),\n",
       " (3110, 'U'),\n",
       " (3115, 'F'),\n",
       " (3120, 'M'),\n",
       " (3125, 'M'),\n",
       " (3130, 'M'),\n",
       " (3135, 'M'),\n",
       " (3140, 'F'),\n",
       " (3145, 'F'),\n",
       " (3150, 'M'),\n",
       " (3155, 'M'),\n",
       " (3160, 'F'),\n",
       " (3165, 'F'),\n",
       " (3170, 'F'),\n",
       " (3175, 'F'),\n",
       " (3180, 'F'),\n",
       " (3185, 'U'),\n",
       " (3190, 'F'),\n",
       " (3195, 'F'),\n",
       " (3200, 'M'),\n",
       " (3205, 'M'),\n",
       " (3210, 'M'),\n",
       " (3215, 'M'),\n",
       " (3220, 'F'),\n",
       " (3225, 'F'),\n",
       " (3230, 'M'),\n",
       " (3235, 'M'),\n",
       " (3240, 'M'),\n",
       " (3245, 'F'),\n",
       " (3250, 'U'),\n",
       " (3255, 'M'),\n",
       " (3260, 'F'),\n",
       " (3265, 'F'),\n",
       " (3270, 'F'),\n",
       " (3275, 'F'),\n",
       " (3280, 'F'),\n",
       " (3285, 'M'),\n",
       " (3290, 'M'),\n",
       " (3295, 'M'),\n",
       " (3300, 'F'),\n",
       " (3305, 'M'),\n",
       " (3310, 'U'),\n",
       " (3315, 'F'),\n",
       " (3320, 'M'),\n",
       " (3325, 'F'),\n",
       " (3330, 'M'),\n",
       " (3335, 'F'),\n",
       " (3340, 'F'),\n",
       " (3345, 'F'),\n",
       " (3350, 'M'),\n",
       " (3355, 'F'),\n",
       " (3360, 'U'),\n",
       " (3365, 'U'),\n",
       " (3370, 'F'),\n",
       " (3375, 'F'),\n",
       " (3380, 'M'),\n",
       " (3385, 'M'),\n",
       " (3390, 'F'),\n",
       " (3395, 'F'),\n",
       " (3400, 'F'),\n",
       " (3405, 'F'),\n",
       " (3410, 'F'),\n",
       " (3415, 'M'),\n",
       " (3420, 'F'),\n",
       " (3425, 'M'),\n",
       " (3430, 'M'),\n",
       " (3435, 'F'),\n",
       " (3440, 'F'),\n",
       " (3445, 'F'),\n",
       " (3450, 'M'),\n",
       " (3455, 'F'),\n",
       " (3460, 'M'),\n",
       " (3465, 'U'),\n",
       " (3470, 'F'),\n",
       " (3475, 'F'),\n",
       " (3480, 'M'),\n",
       " (3485, 'F'),\n",
       " (3490, 'M'),\n",
       " (3495, 'U'),\n",
       " (3500, 'F'),\n",
       " (3505, 'F'),\n",
       " (3510, 'F'),\n",
       " (3515, 'F'),\n",
       " (3520, 'F'),\n",
       " (3525, 'M'),\n",
       " (3530, 'F'),\n",
       " (3535, 'F'),\n",
       " (3540, 'M'),\n",
       " (3545, 'M'),\n",
       " (3550, 'M'),\n",
       " (3555, 'U'),\n",
       " (3560, 'F'),\n",
       " (3565, 'F'),\n",
       " (3570, 'M'),\n",
       " (3575, 'M'),\n",
       " (3580, 'F'),\n",
       " (3585, 'F'),\n",
       " (3590, 'F'),\n",
       " (3595, 'F'),\n",
       " (3600, 'M'),\n",
       " (3605, 'M'),\n",
       " (3610, 'M'),\n",
       " (3615, 'F'),\n",
       " (3620, 'M'),\n",
       " (3625, 'M'),\n",
       " (3630, 'F'),\n",
       " (3635, 'F'),\n",
       " (3640, 'M'),\n",
       " (3645, 'M'),\n",
       " (3650, 'M'),\n",
       " (3655, 'U'),\n",
       " (3660, 'M'),\n",
       " (3665, 'U'),\n",
       " (3670, 'F'),\n",
       " (3675, 'M'),\n",
       " (3680, 'M'),\n",
       " (3685, 'F'),\n",
       " (3690, 'M'),\n",
       " (3695, 'F'),\n",
       " (3700, 'F'),\n",
       " (3705, 'M'),\n",
       " (3710, 'M'),\n",
       " (3715, 'F'),\n",
       " (3720, 'F'),\n",
       " (3725, 'M'),\n",
       " (3730, 'F'),\n",
       " (3735, 'M'),\n",
       " (3740, 'M'),\n",
       " (3745, 'F'),\n",
       " (3750, 'F'),\n",
       " (3755, 'M'),\n",
       " (3760, 'F'),\n",
       " (3765, 'F'),\n",
       " (3770, 'M'),\n",
       " (3775, 'F'),\n",
       " (3780, 'F'),\n",
       " (3785, 'F'),\n",
       " (3790, 'M'),\n",
       " (3795, 'F'),\n",
       " (3800, 'M'),\n",
       " (3805, 'F'),\n",
       " (3810, 'F'),\n",
       " (3815, 'F'),\n",
       " (3820, 'M'),\n",
       " (3825, 'F'),\n",
       " (3830, 'M'),\n",
       " (3835, 'M'),\n",
       " (3840, 'U'),\n",
       " (3845, 'M'),\n",
       " (3850, 'M'),\n",
       " (3855, 'M'),\n",
       " (3860, 'M'),\n",
       " (3865, 'U'),\n",
       " (3870, 'F'),\n",
       " (3875, 'F'),\n",
       " (3880, 'F'),\n",
       " (3885, 'U'),\n",
       " (3890, 'F'),\n",
       " (3895, 'M'),\n",
       " (3900, 'F'),\n",
       " (3905, 'M'),\n",
       " (3910, 'F'),\n",
       " (3915, 'F'),\n",
       " (3920, 'F'),\n",
       " (3925, 'F'),\n",
       " (3930, 'F'),\n",
       " (3935, 'M'),\n",
       " (3940, 'M'),\n",
       " (3945, 'M'),\n",
       " (3950, 'M'),\n",
       " (3955, 'M'),\n",
       " (3960, 'M'),\n",
       " (3965, 'M'),\n",
       " (3970, 'M'),\n",
       " (3975, 'F'),\n",
       " (3980, 'M'),\n",
       " (3985, 'M'),\n",
       " (3990, 'M'),\n",
       " (3995, 'M'),\n",
       " (4000, 'M'),\n",
       " (4005, 'F'),\n",
       " (4010, 'M'),\n",
       " (4015, 'M'),\n",
       " (4020, 'M'),\n",
       " (4025, 'M'),\n",
       " (4030, 'U'),\n",
       " (4035, 'F'),\n",
       " (4040, 'F'),\n",
       " (4045, 'M'),\n",
       " (4050, 'M'),\n",
       " (4055, 'F'),\n",
       " (4060, 'F'),\n",
       " (4065, 'M'),\n",
       " (4070, 'U'),\n",
       " (4075, 'M'),\n",
       " (4080, 'M'),\n",
       " (4085, 'U'),\n",
       " (4090, 'F'),\n",
       " (4095, 'M'),\n",
       " (4100, 'U'),\n",
       " (4105, 'U'),\n",
       " (4110, 'M'),\n",
       " (4115, 'F'),\n",
       " (4120, 'F'),\n",
       " (4125, 'M'),\n",
       " (4130, 'M'),\n",
       " (4135, 'F'),\n",
       " (4140, 'M'),\n",
       " (4145, 'F'),\n",
       " (4150, 'M'),\n",
       " (4155, 'F'),\n",
       " (4160, 'F'),\n",
       " (4165, 'F'),\n",
       " (4170, 'F'),\n",
       " (4175, 'M'),\n",
       " (4180, 'M'),\n",
       " (4185, 'M'),\n",
       " (4190, 'M'),\n",
       " (4195, 'F'),\n",
       " (4200, 'U'),\n",
       " (4205, 'F'),\n",
       " (4210, 'F'),\n",
       " (4215, 'U'),\n",
       " (4220, 'F'),\n",
       " (4225, 'M'),\n",
       " (4230, 'M'),\n",
       " (4235, 'M'),\n",
       " (4240, 'F'),\n",
       " (4245, 'M'),\n",
       " (4250, 'F'),\n",
       " (4255, 'M'),\n",
       " (4260, 'F'),\n",
       " (4265, 'M'),\n",
       " (4270, 'M'),\n",
       " (4275, 'M'),\n",
       " (4280, 'F'),\n",
       " (4285, 'F'),\n",
       " (4290, 'U'),\n",
       " (4295, 'F'),\n",
       " (4300, 'M'),\n",
       " (4305, 'M'),\n",
       " (4310, 'M'),\n",
       " (4315, 'F'),\n",
       " (4320, 'M'),\n",
       " (4325, 'F'),\n",
       " (4330, 'F'),\n",
       " (4335, 'M'),\n",
       " (4340, 'M'),\n",
       " (4345, 'F'),\n",
       " (4350, 'F'),\n",
       " (4355, 'M'),\n",
       " (4360, 'U'),\n",
       " (4365, 'F'),\n",
       " (4370, 'F'),\n",
       " (4375, 'U'),\n",
       " (4380, 'F'),\n",
       " (4385, 'M'),\n",
       " (4390, 'F'),\n",
       " (4395, 'F'),\n",
       " (4400, 'F'),\n",
       " (4405, 'F'),\n",
       " (4410, 'M'),\n",
       " (4415, 'F'),\n",
       " (4420, 'F'),\n",
       " (4425, 'M'),\n",
       " (4430, 'M'),\n",
       " (4435, 'M'),\n",
       " (4440, 'F'),\n",
       " (4445, 'F'),\n",
       " (4450, 'M'),\n",
       " (4455, 'F'),\n",
       " (4460, 'M'),\n",
       " (4465, 'F'),\n",
       " (4470, 'F'),\n",
       " (4475, 'F'),\n",
       " (4480, 'U'),\n",
       " (4485, 'M'),\n",
       " (4490, 'F'),\n",
       " (4495, 'M'),\n",
       " (4500, 'M'),\n",
       " (4505, 'F'),\n",
       " (4510, 'M'),\n",
       " (4515, 'U'),\n",
       " (4520, 'F'),\n",
       " (4525, 'F'),\n",
       " (4530, 'M'),\n",
       " (4535, 'F'),\n",
       " (4540, 'F'),\n",
       " (4545, 'F'),\n",
       " (4550, 'M'),\n",
       " (4555, 'F'),\n",
       " (4560, 'M'),\n",
       " (4565, 'F'),\n",
       " (4570, 'F'),\n",
       " (4575, 'U'),\n",
       " (4580, 'U'),\n",
       " (4585, 'M'),\n",
       " (4590, 'M'),\n",
       " (4595, 'F'),\n",
       " (4600, 'M'),\n",
       " (4605, 'M'),\n",
       " (4610, 'F'),\n",
       " (4615, 'F'),\n",
       " (4620, 'F'),\n",
       " (4625, 'F'),\n",
       " (4630, 'M'),\n",
       " (4635, 'U'),\n",
       " (4640, 'U'),\n",
       " (4645, 'F'),\n",
       " (4650, 'M'),\n",
       " (4655, 'U'),\n",
       " (4660, 'F'),\n",
       " (4665, 'F'),\n",
       " (4670, 'M'),\n",
       " (4675, 'U'),\n",
       " (4680, 'M'),\n",
       " (4685, 'F'),\n",
       " (4690, 'M'),\n",
       " (4695, 'F'),\n",
       " (4700, 'F'),\n",
       " (4705, 'F'),\n",
       " (4710, 'F'),\n",
       " (4715, 'M'),\n",
       " (4720, 'F'),\n",
       " (4725, 'M'),\n",
       " (4730, 'F'),\n",
       " (4735, 'M'),\n",
       " (4740, 'M'),\n",
       " (4745, 'M'),\n",
       " (4750, 'F'),\n",
       " (4755, 'U'),\n",
       " (4760, 'M'),\n",
       " (4765, 'F'),\n",
       " (4770, 'M'),\n",
       " (4775, 'F'),\n",
       " (4780, 'F'),\n",
       " (4785, 'M'),\n",
       " (4790, 'F'),\n",
       " (4795, 'M'),\n",
       " (4800, 'F'),\n",
       " (4805, 'U'),\n",
       " (4810, 'M'),\n",
       " (4815, 'F'),\n",
       " (4820, 'F'),\n",
       " (4825, 'F'),\n",
       " (4830, 'F'),\n",
       " (4835, 'F'),\n",
       " (4840, 'F'),\n",
       " (4845, 'F'),\n",
       " (4850, 'M'),\n",
       " (4855, 'M'),\n",
       " (4860, 'F'),\n",
       " (4865, 'M'),\n",
       " (4870, 'F'),\n",
       " (4875, 'M'),\n",
       " (4880, 'U'),\n",
       " (4885, 'M'),\n",
       " (4890, 'F'),\n",
       " (4895, 'F'),\n",
       " (4900, 'F'),\n",
       " (4905, 'M'),\n",
       " (4910, 'F'),\n",
       " (4915, 'M'),\n",
       " (4920, 'M'),\n",
       " (4925, 'F'),\n",
       " (4930, 'M'),\n",
       " (4935, 'F'),\n",
       " (4940, 'F'),\n",
       " (4945, 'F'),\n",
       " (4950, 'M'),\n",
       " (4955, 'M'),\n",
       " (4960, 'F'),\n",
       " (4965, 'M'),\n",
       " (4970, 'M'),\n",
       " (4975, 'F'),\n",
       " (4980, 'M'),\n",
       " (4985, 'U'),\n",
       " (4990, 'U'),\n",
       " (4995, 'M'),\n",
       " (5000, 'M'),\n",
       " ...]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genders.filter(lambda x: (x[0] % 5)==0).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "what happens if I forgot that columns were swapped?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 48.0 failed 4 times, most recent failure: Lost task 0.3 in stage 48.0 (TID 64, 172.31.33.144): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/worker.py\", line 111, in main\n    process()\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/worker.py\", line 106, in process\n    serializer.dump_stream(func(split_index, iterator), outfile)\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/serializers.py\", line 263, in dump_stream\n    vs = list(itertools.islice(iterator, batch))\n  File \"<ipython-input-54-49e2b6ef7599>\", line 1, in <lambda>\nTypeError: not all arguments converted during string formatting\n\n\tat org.apache.spark.api.python.PythonRDD$$anon$1.read(PythonRDD.scala:138)\n\tat org.apache.spark.api.python.PythonRDD$$anon$1.<init>(PythonRDD.scala:179)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:97)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:70)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1273)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1264)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1263)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1263)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:730)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1457)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1418)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-54-49e2b6ef7599>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mgenders\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfilter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/usr/local/spark/python/pyspark/rdd.py\u001b[0m in \u001b[0;36mcollect\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    755\u001b[0m         \"\"\"\n\u001b[0;32m    756\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mSCCallSiteSync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mcss\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 757\u001b[1;33m             \u001b[0mport\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mPythonRDD\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcollectAndServe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jrdd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    758\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_load_from_socket\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mport\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jrdd_deserializer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    759\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m    536\u001b[0m         \u001b[0manswer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    537\u001b[0m         return_value = get_return_value(answer, self.gateway_client,\n\u001b[1;32m--> 538\u001b[1;33m                 self.target_id, self.name)\n\u001b[0m\u001b[0;32m    539\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m    298\u001b[0m                 raise Py4JJavaError(\n\u001b[0;32m    299\u001b[0m                     \u001b[1;34m'An error occurred while calling {0}{1}{2}.\\n'\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 300\u001b[1;33m                     format(target_id, '.', name), value)\n\u001b[0m\u001b[0;32m    301\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    302\u001b[0m                 raise Py4JError(\n",
      "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 48.0 failed 4 times, most recent failure: Lost task 0.3 in stage 48.0 (TID 64, 172.31.33.144): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/worker.py\", line 111, in main\n    process()\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/worker.py\", line 106, in process\n    serializer.dump_stream(func(split_index, iterator), outfile)\n  File \"/usr/local/spark/python/lib/pyspark.zip/pyspark/serializers.py\", line 263, in dump_stream\n    vs = list(itertools.islice(iterator, batch))\n  File \"<ipython-input-54-49e2b6ef7599>\", line 1, in <lambda>\nTypeError: not all arguments converted during string formatting\n\n\tat org.apache.spark.api.python.PythonRDD$$anon$1.read(PythonRDD.scala:138)\n\tat org.apache.spark.api.python.PythonRDD$$anon$1.<init>(PythonRDD.scala:179)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:97)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:277)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:244)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:63)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:70)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1273)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1264)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1263)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1263)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:730)\n\tat scala.Option.foreach(Option.scala:236)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:730)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1457)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1418)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n"
     ]
    }
   ],
   "source": [
    "genders.filter(lambda x: (x[1] % 5)==0).collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}