Sparkify.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sparkify Project Workspace\n",
    "This workspace contains a tiny subset (128MB) of the full dataset available (12GB). Feel free to use this workspace to build your project, or to explore a smaller subset with Spark before deploying your cluster on the cloud. Instructions for setting up your Spark cluster is included in the last lesson of the Extracurricular Spark Course content.\n",
    "\n",
    "You can follow the steps below to guide your data analysis and model building portion of this project."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import re\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import StringType, LongType\n",
    "from pyspark.sql.functions import col, sum, countDistinct, udf, max, min, isnull, datediff\n",
    "from pyspark.ml.feature import VectorAssembler, StringIndexer, OneHotEncoderEstimator\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "\n",
    "from pyspark.ml.classification import DecisionTreeClassifier, RandomForestClassifier, GBTClassifier\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a Spark session\n",
    "spark = SparkSession.builder.appName(\"sparkify\").getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and Clean Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"medium-sparkify-event-data.json\"\n",
    "df = spark.read.json(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "543705"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- artist: string (nullable = true)\n",
      " |-- auth: string (nullable = true)\n",
      " |-- firstName: string (nullable = true)\n",
      " |-- gender: string (nullable = true)\n",
      " |-- itemInSession: long (nullable = true)\n",
      " |-- lastName: string (nullable = true)\n",
      " |-- length: double (nullable = true)\n",
      " |-- level: string (nullable = true)\n",
      " |-- location: string (nullable = true)\n",
      " |-- method: string (nullable = true)\n",
      " |-- page: string (nullable = true)\n",
      " |-- registration: long (nullable = true)\n",
      " |-- sessionId: long (nullable = true)\n",
      " |-- song: string (nullable = true)\n",
      " |-- status: long (nullable = true)\n",
      " |-- ts: long (nullable = true)\n",
      " |-- userAgent: string (nullable = true)\n",
      " |-- userId: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's look at the variables to get a good understanding of the data and missing values if any"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Stackoverflow](https://stackoverflow.com/a/44413456/4411757)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>artist</th>\n",
       "      <td>110828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>auth</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>firstName</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>itemInSession</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lastName</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>length</th>\n",
       "      <td>110828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>page</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>registration</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sessionId</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>song</th>\n",
       "      <td>110828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>status</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ts</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userAgent</th>\n",
       "      <td>15700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userId</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0\n",
       "artist         110828\n",
       "auth                0\n",
       "firstName       15700\n",
       "gender          15700\n",
       "itemInSession       0\n",
       "lastName        15700\n",
       "length         110828\n",
       "level               0\n",
       "location        15700\n",
       "method              0\n",
       "page                0\n",
       "registration    15700\n",
       "sessionId           0\n",
       "song           110828\n",
       "status              0\n",
       "ts                  0\n",
       "userAgent       15700\n",
       "userId              0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.select(*(sum(col(c).isNull().cast(\"int\")).alias(c) for c in df.columns)).toPandas().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* From the above counts, we can observe that if a song isn't present (is null), there obviously won't be an artist associated. Similarly for length of the song.\n",
    "* The same missing counts for firstName, lastName and gender would indicate that the corresponding info has not been disclosed by the users. Mostly users who are not logged in."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploration of empty User Ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|userId|\n",
      "+------+\n",
      "|      |\n",
      "|      |\n",
      "|      |\n",
      "|      |\n",
      "|      |\n",
      "+------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(isnull(col(\"firstName\"))).select(\"userId\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+\n",
      "|      auth|\n",
      "+----------+\n",
      "|Logged Out|\n",
      "|     Guest|\n",
      "+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(df.userId == \"\").select(\"auth\").distinct().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Empty users correspond to either logged out users or guests who don't have an account yet. We can eliminate these records for our purpose."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.filter(df.userId != \"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UserId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 448 distinct users in the dataset\n"
     ]
    }
   ],
   "source": [
    "print(\"There are {} distinct users in the dataset\".format(df.select(\"UserId\").distinct().count()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NextSong</td>\n",
       "      <td>432877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Thumbs Up</td>\n",
       "      <td>23826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Home</td>\n",
       "      <td>19089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Add to Playlist</td>\n",
       "      <td>12349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Add Friend</td>\n",
       "      <td>8087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Roll Advert</td>\n",
       "      <td>7773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Logout</td>\n",
       "      <td>5990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Thumbs Down</td>\n",
       "      <td>4911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Downgrade</td>\n",
       "      <td>3811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Settings</td>\n",
       "      <td>2964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Help</td>\n",
       "      <td>2644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>About</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Upgrade</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Save Settings</td>\n",
       "      <td>585</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Error</td>\n",
       "      <td>503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Submit Upgrade</td>\n",
       "      <td>287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Submit Downgrade</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Cancel</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Cancellation Confirmation</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         page   count\n",
       "0                    NextSong  432877\n",
       "1                   Thumbs Up   23826\n",
       "2                        Home   19089\n",
       "3             Add to Playlist   12349\n",
       "4                  Add Friend    8087\n",
       "5                 Roll Advert    7773\n",
       "6                      Logout    5990\n",
       "7                 Thumbs Down    4911\n",
       "8                   Downgrade    3811\n",
       "9                    Settings    2964\n",
       "10                       Help    2644\n",
       "11                      About    1026\n",
       "12                    Upgrade     968\n",
       "13              Save Settings     585\n",
       "14                      Error     503\n",
       "15             Submit Upgrade     287\n",
       "16           Submit Downgrade     117\n",
       "17                     Cancel      99\n",
       "18  Cancellation Confirmation      99"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupBy(\"page\").count().sort(col(\"count\").desc()).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* There's a high chance that number of Thumbs down could translate into cancellation of subscription.\n",
    "* Cancellation of subscription could be a metric for churned users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+\n",
      "|UserId|\n",
      "+------+\n",
      "+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_register = df.filter(col(\"page\")==\"Register\").select(\"UserId\")\n",
    "df_register.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Artist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 21248 distinct artists in the dataset\n",
      "Their frequency in the dataset is as follows - \n",
      "+--------------------+-----+\n",
      "|              artist|count|\n",
      "+--------------------+-----+\n",
      "|                null|95128|\n",
      "|       Kings Of Leon| 3497|\n",
      "|            Coldplay| 3439|\n",
      "|Florence + The Ma...| 2314|\n",
      "|                Muse| 2194|\n",
      "|       Dwight Yoakam| 2187|\n",
      "|      The Black Keys| 2160|\n",
      "|            BjÃÂ¶rk| 2150|\n",
      "|       Justin Bieber| 2096|\n",
      "|        Jack Johnson| 2049|\n",
      "+--------------------+-----+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"There are {} distinct artists in the dataset\".format(df.select(\"artist\").distinct().count()))\n",
    "print(\"Their frequency in the dataset is as follows - \")\n",
    "df.groupBy(\"artist\").count().sort(col(\"count\").desc()).show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Auth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------+\n",
      "|     auth| count|\n",
      "+---------+------+\n",
      "|Logged In|527906|\n",
      "|Cancelled|    99|\n",
      "+---------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy(\"auth\").count().sort(col(\"count\").desc()).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can probably remove Guest and Logged Out entries since they won't contribute towards churn anyways. The other three auth are what we care about."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Guest and Logged out entries have been removed when we filtered out empty users."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+------+\n",
      "|level| count|\n",
      "+-----+------+\n",
      "| paid|418044|\n",
      "| free|109961|\n",
      "+-----+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy(\"level\").count().sort(col(\"count\").desc()).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------+\n",
      "|level|count_user|\n",
      "+-----+----------+\n",
      "| free|       370|\n",
      "| paid|       321|\n",
      "+-----+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy(\"level\").agg(countDistinct(\"UserId\").alias(\"count_user\")).sort(col(\"count_user\").desc()).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* There are almost as many free users as there are paid users in the dataset. It'll be interesting to see a trend in churn possibility in paid vs free users.\n",
    "* Free users are generally shown ads in between which should either motivate them to upgrade or cancel their subscription if not satisfied."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### itemInSession and sessionId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserId</th>\n",
       "      <th>itemInSession</th>\n",
       "      <th>sessionId</th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>293</td>\n",
       "      <td>20</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>293</td>\n",
       "      <td>21</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>293</td>\n",
       "      <td>22</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>293</td>\n",
       "      <td>23</td>\n",
       "      <td>292</td>\n",
       "      <td>Thumbs Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>293</td>\n",
       "      <td>24</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>293</td>\n",
       "      <td>25</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>293</td>\n",
       "      <td>26</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>293</td>\n",
       "      <td>27</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>293</td>\n",
       "      <td>28</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>293</td>\n",
       "      <td>29</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>293</td>\n",
       "      <td>30</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>293</td>\n",
       "      <td>31</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>293</td>\n",
       "      <td>32</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>293</td>\n",
       "      <td>33</td>\n",
       "      <td>292</td>\n",
       "      <td>Logout</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>293</td>\n",
       "      <td>39</td>\n",
       "      <td>292</td>\n",
       "      <td>Home</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>293</td>\n",
       "      <td>40</td>\n",
       "      <td>292</td>\n",
       "      <td>Add Friend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>293</td>\n",
       "      <td>41</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>293</td>\n",
       "      <td>42</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>293</td>\n",
       "      <td>43</td>\n",
       "      <td>292</td>\n",
       "      <td>Add to Playlist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>293</td>\n",
       "      <td>44</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>293</td>\n",
       "      <td>45</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>293</td>\n",
       "      <td>46</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>293</td>\n",
       "      <td>47</td>\n",
       "      <td>292</td>\n",
       "      <td>Roll Advert</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>293</td>\n",
       "      <td>48</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>293</td>\n",
       "      <td>49</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>293</td>\n",
       "      <td>50</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>293</td>\n",
       "      <td>51</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>293</td>\n",
       "      <td>52</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>293</td>\n",
       "      <td>53</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>293</td>\n",
       "      <td>54</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>293</td>\n",
       "      <td>55</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>293</td>\n",
       "      <td>56</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>293</td>\n",
       "      <td>57</td>\n",
       "      <td>292</td>\n",
       "      <td>Add to Playlist</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>293</td>\n",
       "      <td>58</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>293</td>\n",
       "      <td>59</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>293</td>\n",
       "      <td>60</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>293</td>\n",
       "      <td>61</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>293</td>\n",
       "      <td>62</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>293</td>\n",
       "      <td>63</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>293</td>\n",
       "      <td>64</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>293</td>\n",
       "      <td>65</td>\n",
       "      <td>292</td>\n",
       "      <td>Roll Advert</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>293</td>\n",
       "      <td>66</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>293</td>\n",
       "      <td>67</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>293</td>\n",
       "      <td>68</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>293</td>\n",
       "      <td>69</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>293</td>\n",
       "      <td>70</td>\n",
       "      <td>292</td>\n",
       "      <td>Settings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>293</td>\n",
       "      <td>71</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>293</td>\n",
       "      <td>72</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>293</td>\n",
       "      <td>73</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>293</td>\n",
       "      <td>74</td>\n",
       "      <td>292</td>\n",
       "      <td>NextSong</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserId  itemInSession  sessionId             page\n",
       "0     293             20        292         NextSong\n",
       "1     293             21        292         NextSong\n",
       "2     293             22        292         NextSong\n",
       "3     293             23        292        Thumbs Up\n",
       "4     293             24        292         NextSong\n",
       "5     293             25        292         NextSong\n",
       "6     293             26        292         NextSong\n",
       "7     293             27        292         NextSong\n",
       "8     293             28        292         NextSong\n",
       "9     293             29        292         NextSong\n",
       "10    293             30        292         NextSong\n",
       "11    293             31        292         NextSong\n",
       "12    293             32        292         NextSong\n",
       "13    293             33        292           Logout\n",
       "14    293             39        292             Home\n",
       "15    293             40        292       Add Friend\n",
       "16    293             41        292         NextSong\n",
       "17    293             42        292         NextSong\n",
       "18    293             43        292  Add to Playlist\n",
       "19    293             44        292         NextSong\n",
       "20    293             45        292         NextSong\n",
       "21    293             46        292         NextSong\n",
       "22    293             47        292      Roll Advert\n",
       "23    293             48        292         NextSong\n",
       "24    293             49        292         NextSong\n",
       "25    293             50        292         NextSong\n",
       "26    293             51        292         NextSong\n",
       "27    293             52        292         NextSong\n",
       "28    293             53        292         NextSong\n",
       "29    293             54        292         NextSong\n",
       "30    293             55        292         NextSong\n",
       "31    293             56        292         NextSong\n",
       "32    293             57        292  Add to Playlist\n",
       "33    293             58        292         NextSong\n",
       "34    293             59        292         NextSong\n",
       "35    293             60        292         NextSong\n",
       "36    293             61        292         NextSong\n",
       "37    293             62        292         NextSong\n",
       "38    293             63        292         NextSong\n",
       "39    293             64        292         NextSong\n",
       "40    293             65        292      Roll Advert\n",
       "41    293             66        292         NextSong\n",
       "42    293             67        292         NextSong\n",
       "43    293             68        292         NextSong\n",
       "44    293             69        292         NextSong\n",
       "45    293             70        292         Settings\n",
       "46    293             71        292         NextSong\n",
       "47    293             72        292         NextSong\n",
       "48    293             73        292         NextSong\n",
       "49    293             74        292         NextSong"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Random user id\n",
    "user_id = df.sample(False, 0.5).limit(1).select('UserId').collect()[0]['UserId']\n",
    "# Results filtered on a particular user based on ascending order of timestamp \n",
    "df.filter(col(\"UserId\") == user_id).select([\"UserId\", \"itemInSession\", \"sessionId\", \"page\"]).sort(col(\"ts\")).toPandas().head(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 'itemInSession' just seems to be a serial numbering of the actions taken in a session. Doesn't seem useful unless some sequential analysis on the corresponding action in 'page' is done.\n",
    "* Thought distinct 'sessionId' can be useful to count the number of times user comes back to the application to interact. Churned users should probably have less number of sessions as against non-churned users."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### userAgent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Writing an UDF function](https://gist.github.com/zoltanctoth/2deccd69e3d1cde1dd78)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "device_udf = udf(lambda x: str(re.findall(r'\\((.*?)\\)', x)[0].split(\";\")[0].split()[0]) if x is not None else None, StringType())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.withColumn(\"device\", device_udf(df.userAgent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_device = df.groupBy(\"device\").agg(countDistinct(\"UserId\").alias(\"count_user\")).sort(col(\"count_user\").desc()).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAF3CAYAAAARh7eaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAH0xJREFUeJzt3Xm4ZHV95/H3RxqXCAJKhyEIaUPQxC3t2C6BqMQlLomCxo1BBDU2ZtToPJrEiDGEjInjOkZHDCoiahARNWiIxhBlUwMNQrMoioAKaaEDKrgEBb7zx/ldKDq36aLpqvrde9+v56nnnvM7S33r3Fvnfup3Tp2TqkKSJEn9uNOsC5AkSdKtGdAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6owBTZIkqTMGNEmSpM4Y0CRJkjqzbNYF3FE77rhjrVixYtZlSJIkbdJZZ531H1W1fFPzLfiAtmLFCtasWTPrMiRJkjYpybfHmc9DnJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ1ZNusCpuGhf3z0rEtYEM568/NnXYIkScIeNEmSpO4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzkw0oCXZNckXklyY5IIkr2jt90zy+STfbD93aO1J8rdJLk6yNsl/n2R9kiRJPZp0D9oNwKuq6v7AI4GXJrk/8BrgpKraAzipjQM8GdijPVYDh0+4PkmSpO5MNKBV1bqqOrsNXwd8DdgF2Af4YJvtg8C+bXgf4OgafAXYPsnOk6xRkiSpN1M7By3JCuAhwL8BO1XVujbpe8BObXgX4Lsji13e2iRJkpaMqQS0JNsAxwOvrKprR6dVVQF1O9e3OsmaJGvWr1+/BSuVJEmavYkHtCRbM4Szj1TVJ1rzlXOHLtvPq1r7FcCuI4vfu7XdSlUdUVWrqmrV8uXLJ1e8JEnSDEz6W5wB3g98rareNjLpBODANnwg8A8j7c9v3+Z8JPDDkUOhkiRJS8KyCa9/L+AA4Lwk57S21wJvBD6W5EXAt4Fnt2knAk8BLgZ+ArxgwvVJkiR1Z6IBrapOA7KRyY+bZ/4CXjrJmiRJknrnnQQkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6owBTZIkqTMGNEmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6owBTZIkqTMGNEmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzkw0oCU5MslVSc4faTs2yTntcVmSc1r7iiQ/HZn2nknWJkmS1KtlE17/UcC7gKPnGqrqOXPDSd4K/HBk/m9V1coJ1yRJktS1iQa0qjolyYr5piUJ8GzgsZOsQZIkaaGZ5TlojwKurKpvjrTdJ8lXk5yc5FGzKkySJGmWJn2I87bsBxwzMr4O2K2qrk7yUOBTSR5QVdduuGCS1cBqgN12220qxUqSJE3LTHrQkiwDngEcO9dWVddX1dVt+CzgW8B951u+qo6oqlVVtWr58uXTKFmSJGlqZnWI8/HA16vq8rmGJMuTbNWGfwXYA7hkRvVJkiTNzEQPcSY5Btgb2DHJ5cBfVNX7gedy68ObAI8GDkvyc+Am4CVVdc0k69PkfOewB826hAVht9efN+sSJEkdmvS3OPfbSPtB87QdDxw/yXokSZIWAu8kIEmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHVmogEtyZFJrkpy/kjboUmuSHJOezxlZNqfJbk4yUVJnjjJ2iRJkno16R60o4AnzdP+9qpa2R4nAiS5P/Bc4AFtmXcn2WrC9UmSJHVnogGtqk4Brhlz9n2Aj1bV9VV1KXAx8PCJFSdJktSpWZ2D9rIka9sh0B1a2y7Ad0fmuby1SZIkLSmzCGiHA7sDK4F1wFtv7wqSrE6yJsma9evXb+n6JEmSZmrqAa2qrqyqG6vqJuC93HIY8wpg15FZ793a5lvHEVW1qqpWLV++fLIFS5IkTdnUA1qSnUdGnw7MfcPzBOC5Se6S5D7AHsAZ065PkiRp1pZNcuVJjgH2BnZMcjnwF8DeSVYCBVwGHAxQVRck+RhwIXAD8NKqunGS9UmSJPVoogGtqvabp/n9tzH/G4A3TK4iSZKk/nknAUmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6owBTZIkqTMGNEmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6owBTZIkqTMGNEmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzEw1oSY5MclWS80fa3pzk60nWJvlkku1b+4okP01yTnu8Z5K1SZIk9eo2A1qSrZJ8/Q6s/yjgSRu0fR54YFU9GPgG8Gcj075VVSvb4yV34HklSZIWrNsMaFV1I3BRkt02Z+VVdQpwzQZt/1xVN7TRrwD33px1S5IkLVbLxphnB+CCJGcAP55rrKqnbYHnfyFw7Mj4fZJ8FbgWeF1VnboFnkOSJGlBGSeg/fkknjjJIcANwEda0zpgt6q6OslDgU8leUBVXTvPsquB1QC77bZZnXuSJEnd2uSXBKrqZOAyYOs2fCZw9h150iQHAb8H7F9V1Z7n+qq6ug2fBXwLuO9GajqiqlZV1arly5ffkVIkSZK6s8mAluTFwMeBv2tNuwCf2twnTPIk4E+Ap1XVT0balyfZqg3/CrAHcMnmPo8kSdJCNc5lNl4K7MVwXhhV9U3gF8dZeZJjgC8D90tyeZIXAe8CtgU+v8HlNB4NrE1yDkMgfElVXTPviiVJkhaxcc5Bu76qfpYEgCTLgBpn5VW13zzN79/IvMcDx4+zXkmSpMVsnB60k5O8FrhbkicAxwGfnmxZkiRJS9c4Ae01wHrgPOBg4ETgdZMsSpIkaSnb5CHOqroJeC/w3iT3BO49981LSZIkbXnjfIvzi0nu0cLZWQxB7e2TL02SJGlpGucQ53btYrHPAI6uqkcAj5tsWZIkSUvXOAFtWZKdgWcDn5lwPZIkSUveOAHtMOBzwMVVdWa7iOw3J1uWJEnS0jXOlwSOY7i0xtz4JcDvT7IoSZKkpWyTAS3JB5jnwrRV9cKJVCRJkrTEjXMngdHzzu4KPB3498mUI0mSpHEOcd7q9kvt/pqnTawiSZKkJW6cLwlsaA/GvFm6JEmSbr9xzkG7jlufg/Y94E8nVpEkSdISN84hzm1va3qSB1TVBVuuJEmSpKVtcw5xbuhDW2AdkiRJarZEQMsWWIckSZKaLRHQ/ss10iRJkrT5tkRAkyRJ0ha0JQLaz7bAOiRJktRsMqAlOem22qrqkVu6KEmSpKVso5fZSHJX4BeAHZPswC1fBrgHsMsUapMkSVqSbus6aAcDrwR+CTiLWwLatcC7JlyXJEnSkrXRgFZV7wDekeTlVfXOKdYkSZK0pI1zJ4F3JtkTWDE6f1UdPcG6JEmSlqxx7sX5IWB34BzgxtZcgAFNkiRpAjYZ0IBVwP2rygvSSpIkTcE410E7H/hvky5EkiRJg3F60HYELkxyBnD9XGNVPW1iVUmSJC1h4wS0Qzd35UmOBH4PuKqqHtja7gkcy/Clg8uAZ1fV95MEeAfwFOAnwEFVdfbmPrckSdJCtclDnFV18nyPMdd/FPCkDdpeA5xUVXsAJ7VxgCcDe7THauDwMZ9DkiRpURnnVk/XJbm2Pf4zyY1Jrh1n5VV1CnDNBs37AB9swx8E9h1pP7oGXwG2T7LzeC9DkiRp8RjnOmjbzg23w5D7AHfk/ps7VdW6Nvw9YKc2vAvw3ZH5Lm9t65AkSVpCxvkW581a79angCduiSdvl+643ZfvSLI6yZoka9avX78lSpEkSerGOBeqfcbI6J0Yrov2n3fgOa9MsnNVrWuHMK9q7VcAu47Md+/W9l9U1RHAEQCrVq3y+mySJGlRGacH7akjjycC1zEc5txcJwAHtuEDgX8YaX9+Bo8EfjhyKFSSJGnJGOcctBds7sqTHAPsDeyY5HLgL4A3Ah9L8iLg28Cz2+wnMlxi42KGy2xs9vNKkiQtZOMc4rw38E5gr9Z0KvCKqrp8U8tW1X4bmfS4eeYt4KWbWqckSdJiN84hzg8wHH78pfb4dGuTJEnSBIwT0JZX1Qeq6ob2OApYPuG6JEmSlqxxAtrVSZ6XZKv2eB5w9aQLkyRJWqrGCWgvZDiR/3sMF419JnDQBGuSJEla0sa5WfphwIFV9X24+Wbnb2EIbpIkSdrCxulBe/BcOAOoqmuAh0yuJEmSpKVtnIB2pyQ7zI20HrRxet4kSZK0GcYJWm8FvpzkuDb+LOANkytJkiRpaRvnTgJHJ1kDPLY1PaOqLpxsWZIkSUvXWIcqWyAzlEmSJE3BOOegSZIkaYoMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdWTaLJ01yP+DYkaZfAV4PbA+8GFjf2l9bVSdOuTxJkqSZmklAq6qLgJUASbYCrgA+CbwAeHtVvWUWdUmSJPWgh0OcjwO+VVXfnnUhkiRJPeghoD0XOGZk/GVJ1iY5MskOsypKkiRpVmYa0JLcGXgacFxrOhzYneHw5zrgrRtZbnWSNUnWrF+/fr5ZJEmSFqxZ96A9GTi7qq4EqKorq+rGqroJeC/w8PkWqqojqmpVVa1avnz5FMuVJEmavFkHtP0YObyZZOeRaU8Hzp96RZIkSTM2k29xAiS5O/AE4OCR5jclWQkUcNkG0yRJkpaEmQW0qvoxcK8N2g6YUTmSJEndmPUhTkmSJG3AgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1ZtmsnjjJZcB1wI3ADVW1Ksk9gWOBFcBlwLOr6vuzqlGSJGkWZt2D9ttVtbKqVrXx1wAnVdUewEltXJIkaUmZdUDb0D7AB9vwB4F9Z1iLJEnSTMwyoBXwz0nOSrK6te1UVeva8PeAneZbMMnqJGuSrFm/fv00apUkSZqamZ2DBvxWVV2R5BeBzyf5+ujEqqokNd+CVXUEcATAqlWr5p1HkiRpoZpZD1pVXdF+XgV8Eng4cGWSnQHaz6tmVZ8kSdKszCSgJbl7km3nhoHfAc4HTgAObLMdCPzDLOqTJEmapVkd4twJ+GSSuRr+vqo+m+RM4GNJXgR8G3j2jOqTJEmamZkEtKq6BPiNedqvBh43/YokSZL60dtlNiRJkpY8A5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHXGgCZJktQZA5okSVJnDGiSJEmdMaBJkiR1ZiYBLcmuSb6Q5MIkFyR5RWs/NMkVSc5pj6fMoj5JkqRZWjaj570BeFVVnZ1kW+CsJJ9v095eVW+ZUV2SJEkzN5OAVlXrgHVt+LokXwN2mUUt0mKx1zv3mnUJC8LpLz991iVI0ibN/By0JCuAhwD/1ppelmRtkiOT7DCzwiRJkmZkpgEtyTbA8cArq+pa4HBgd2AlQw/bWzey3Ooka5KsWb9+/dTqlSRJmoaZBbQkWzOEs49U1ScAqurKqrqxqm4C3gs8fL5lq+qIqlpVVauWL18+vaIlSZKmYFbf4gzwfuBrVfW2kfadR2Z7OnD+tGuTJEmatVl9i3Mv4ADgvCTntLbXAvslWQkUcBlw8GzKkyRJmp1ZfYvzNCDzTDpx2rVIkiT1Zubf4pQkSdKtGdAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzszqMhuStOCd/OjHzLqEBeExp5w86xKkBcceNEmSpM4Y0CRJkjpjQJMkSeqMAU2SJKkzBjRJkqTOGNAkSZI642U2JEkLxrte9elZl7AgvOytT511CbqD7EGTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOmNAkyRJ6ozf4pQkSRv1huc9c9YlLAiHfPjjW3R99qBJkiR1xoAmSZLUGQOaJElSZwxokiRJnTGgSZIkdcaAJkmS1BkDmiRJUmcMaJIkSZ0xoEmSJHWmu4CW5ElJLkpycZLXzLoeSZKkaesqoCXZCvh/wJOB+wP7Jbn/bKuSJEmarq4CGvBw4OKquqSqfgZ8FNhnxjVJkiRNVW8BbRfguyPjl7c2SZKkJSNVNesabpbkmcCTquoP2vgBwCOq6mUbzLcaWN1G7wdcNNVCt4wdgf+YdRFLjNt8+tzm0+c2nz63+fQt5G3+y1W1fFMzLZtGJbfDFcCuI+P3bm23UlVHAEdMq6hJSLKmqlbNuo6lxG0+fW7z6XObT5/bfPqWwjbv7RDnmcAeSe6T5M7Ac4ETZlyTJEnSVHXVg1ZVNyR5GfA5YCvgyKq6YMZlSZIkTVVXAQ2gqk4ETpx1HVOwoA/RLlBu8+lzm0+f23z63ObTt+i3eVdfEpAkSVJ/56BJkiQteQa0TUjy9iSvHBn/XJL3jYy/Nclrk3z8dq73oCTv2pK1LnRJKsmHR8aXJVmf5DObub73bepOFEn23dy7VSQ5NMmrN2fZxSDJrkkuTXLPNr5DG1+R5LNJfrDh7y7Jy9pt3CrJjrOpvG9JvtR+rkjy0yTnJLkwyXuS3CnJ3pv7ntB0bbh/SXJYkse34cvmew8s9f0KbPo9cDvWs6C3pQFt004H9gRofxg7Ag8Ymb4n8K9V9cwZ1LbY/Bh4YJK7tfEnMM9lVsZVVX9QVRduYrZ9GW4rptupqr4LHA68sTW9ETiiqi4D3gwcMM9ipwOPB749jRoXoqrac2T0W1W1Engww9/pvrOpSpvpVvuXqnp9Vf3LDOtZEHwPDAxom/Yl4Dfb8AOA84HrWm/BXYBfB65Jcj7c3DP2idaD8M0kb5pbUZIXJPlGkjOAvUbaVyT51yRrk5yUZLckW7XeiCTZPsmNSR7d5j8lyR5JHtM+WZyT5KtJtp3SNpmkE4HfbcP7AcfMTUjy8CRfbq/1S0nu19q3SvKWJOe3bfjy1v7FJKva8I+SvCHJuUm+kmSnJHsCTwPe3Lbh7klWtulrk3wyyQ5t+T9qn+DWJvnoSL33b89zSZI/mvzm6c7bgUe2XubfAt4CUFUnAddtOHNVfbUFOG1Ekh9t2FZVNzDsi361NW2T5ONJvp7kI0nSln1ce3+cl+TIto+a6635yyRnt2m/1trv3uY7oy23qG6tl+T57T17bpIPzbevbfMdleTw9t6/pPVSHpnka0mOGlnfjzIcVbmgLb+8tb84yZnteY5P8gsb2b8cleGC7HP+pP0+zkjyq2ygLfPZJGclOXXu97bYbeo9kGSbtv3n/p73GVn2kPZ/9jSGC9kvXFXlYxMP4FJgN+Bg4CXAXwFPYQhZpwIrgPPbvAcBlwDbAXdl6CnYFdgZ+A6wHLgzQ0/Cu9oynwYObMMvBD7Vhj/LEAp/j+EacYcAdwEuHVlurza8DbBs1tvqDm7nHzF8Svp423bnAHsDn2nT7zH3Ghl6YY5vw3/Ylpmbds/284vAqjZcwFPb8JuA17Xho4BnjtSwFnhMGz4M+L9t+N+Bu7Th7dvPQxl2GHdh6Fm9Gth61ttxBr+3J7bt+4QN2m/+3c2zzGXAjrOuvccH8KP2c3S/8gttH/Dktl1/yHAh7zsBX2YIx3dluFXefdsyRwOvHNneL2/D/xN4Xxv+a+B5bXh74BvA3We9DbbQdnxAez07tvF73sa+9iiGez+H4f7P1wIPatv3LGBlm6+A/dvw67llH36vkef93yPbesP9y83j7XdySBt+Prfs5w4FXt2GTwL2aMOPYDhaM/NtO4Xf3abeA8uAe7T2HYGL2+/uocB5bd57tPZXz/r1bO7DHrTxfInhUOaeDDvDL4+Mnz7P/CdV1Q+r6j+BC4FfZnhzfbGq1tdwI/hjR+b/TeDv2/CHGHa2MIS/R7fH37T2hzH8kdKe+22t52b7Gj5hLGhVtZbhTbkf//VyK9sBx2XorXw7txxqfjzwd3Ovv6qumWfVPwPmzts5qz3HrSTZjmE7ntyaPsiw7WEIbh9J8jxgdDv/Y1VdX1X/AVwF7DTeK11UngysAx4460IWod2TnMPwXv/Hqvqn1n5GVV1eVTcxfJBZwdBbcGlVfaPNM/r3C/CJ9nP07/93gNe05/giQ8jbbTIvZeoeCxzX3ptz+4WN7WsBPl3Df/zzgCur6ry2fS/glu11E7fsuz88svwDWw/XecD+3Po0mNtyzMjP3xydkGQbhv8xx7Xfz98xfNBfauZ7DwT46yRrgX9huGf3TsCjgE9W1U+q6loW+IXuu7sOWqfmzkN7EMMhzu8Cr2L4lPWBeea/fmT4RjZ/O5/C0Dv0Swyf1v6Y4dPzqQBV9cYk/8jQm3d6kidW1dc387l6cgLDobK9gXuNtP8V8IWqenqSFQz/UMb187bzhc37nfwuwz+7pwKHJHlQa99Sv+sFKclKhnMFHwmcluSjVbVuxmUtJnPn32xoc/7u5pYZnT/A71fVQryf8ZY2t31u4tbb9yY2vn3n9ilHAftW1blJDmLYd42jNjIMQ+/dDzby+19K5nsP7M9wNOqhVfXzJJcxfLhYVOxBG8+XGA4zXlNVN7ZPYtszfOL50pjr+DfgMUnulWRr4FkbrP+5bXh/WgADzmAIhje13rhzGA6zngLD+QntU97/YehVWyznJxwJ/GVVnbdB+3bc8qWBg0baPw8cnGQZQNq3Csd0HbAtQFX9EPh+kke1aQcAJ2f4csiuVfUF4E9bHdvcjudYlNp5T4czHEb7DsMXA94y26qWtIuAFSPnMh0AnHwb88Nw15aXj5zD9pAJ1jdt/wo8K8m94Ob9wsb2teO6EzB3Dtn/AE5rw9sC69q+ff+R+W/ev2zEc0Z+fnl0QusBujTJs1r9SfIbt7PexWo74KoWzn6b4SgVDP8b901ytwznZD91ZhVuAQa08ZzHcJz7Kxu0/XCu+3xTWq/CoQxvwtOBr41MfjnwgtZdewDwirbM9Qy9dXPPeyrDm30uuLwy7cR44OfAP7EItEM3fzvPpDcBf5Pkq9z6E+37GM7vW5vkXIYd57g+CvxxO0F6d+BAhpN61wIrGc5D2wr4cDt88VXgb6vqB7f7hS0+Lwa+U1Wfb+PvBn49w5dXTgWOAx6X5PIkT4Sbv2xxOcP5U2szcska3THtQ9wLGA6JncfQ8/OeTSz2V8DWDL+LC9r4olDDbQLfwPAh61zgbWxkX3s7/Bh4eDvN4rEM+weAP2f4EH46MHoUY8P9y4Z2aLW8Avhf80zfH3hRq/8ChvPjBB8BVrW/8+fTtnlVnc1wCPpchv+HZ250DQuAdxKQJGkMSX5UVUu+91zTYQ+aJElSZ+xBkyRJ6ow9aJIkSZ0xoEmSJHXGgCZJktQZA5qkJSHJoUlevRnLHZbk8ZOoSZI2Zkld9VySbq+qev2sa5C09NiDJmnRSnJIkm8kOY3hXpUk2T3JZ5Oc1e6f+GtJtkvy7XbXCJLcPcl3k2yd5Kgkz2ztD0vypSTnJjkjybZJtkry5iRnJlmb5OAZvmRJi4Q9aJIWpSQPZbitz0qGfd3ZDDcKPwJ4SVV9M8kjgHdX1WPbDZkfA3yB4dZun2u3kplb350ZrlL+nKo6M8k9gJ8CL2K4q8jDktyF4b64/1xVl071BUtaVAxokharRwGfrKqfACQ5geGGynsy3A5pbr67tJ/HMtwT8QsMwe7dG6zvfsC6qjoTbr5XIkl+B3jwXC8bw30C9wAMaJI2mwFN0lJyJ+AHVbVynmknAH/dbqr9UIabbY8jwMur6nNbqEZJ8hw0SYvWKcC+Se6WZFvgqcBPgEuTPAsgg98AqKofMdxc+R3AZ6rqxg3WdxGwc5KHtWW3TbIM+Bzwh0m2bu33TXL3Kbw+SYuYPWiSFqWqOjvJscC5wFUM4Qtgf+DwJK8DtgY+2uaB4TDnccDe86zvZ0meA7wzyd0Yzj97PPA+YAVwdobjpuuBfSf0siQtEd6LU5IkqTMe4pQkSeqMAU2SJKkzBjRJkqTOGNAkSZI6Y0CTJEnqjAFNkiSpMwY0SZKkzhjQJEmSOvP/AY8h+Q6u3oxOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,6))\n",
    "\n",
    "g = sns.barplot(ax=ax, x=\"device\", y=\"count_user\", data=df_device)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can check whether the interface platform contributes in churned users with this information."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ts (timestamp) and registration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's view registration and ts for a user 293"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------+---------------+\n",
      "| registration|           ts|           page|\n",
      "+-------------+-------------+---------------+\n",
      "|1532063507000|1538352011000|       NextSong|\n",
      "|1532063507000|1538352608000|       NextSong|\n",
      "|1532063507000|1538352788000|       NextSong|\n",
      "|1532063507000|1538352789000|      Thumbs Up|\n",
      "|1532063507000|1538353056000|       NextSong|\n",
      "|1532063507000|1538353288000|       NextSong|\n",
      "|1532063507000|1538353527000|       NextSong|\n",
      "|1532063507000|1538353716000|       NextSong|\n",
      "|1532063507000|1538354011000|       NextSong|\n",
      "|1532063507000|1538354251000|       NextSong|\n",
      "|1532063507000|1538354431000|       NextSong|\n",
      "|1532063507000|1538354770000|       NextSong|\n",
      "|1532063507000|1538355003000|       NextSong|\n",
      "|1532063507000|1538355004000|         Logout|\n",
      "|1532063507000|1538355201000|           Home|\n",
      "|1532063507000|1538355202000|     Add Friend|\n",
      "|1532063507000|1538355235000|       NextSong|\n",
      "|1532063507000|1538355444000|       NextSong|\n",
      "|1532063507000|1538355470000|Add to Playlist|\n",
      "|1532063507000|1538355611000|       NextSong|\n",
      "+-------------+-------------+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(col(\"UserId\") == 293).select([\"registration\", \"ts\", \"page\"]).sort(col(\"ts\")).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**\"Unix timestamp no pretty. Me want normal datetime.\"** - Probably a person alive before 1st Jan 1970"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_dt_udf = udf(lambda x: x//1000, LongType())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_dt_udf = udf(lambda x: x//1000, LongType())\n",
    "df = df.withColumn(\"registration_dt\", ts_dt_udf(df.registration).cast(\"timestamp\"))\n",
    "df = df.withColumn(\"timestamp_dt\", ts_dt_udf(df.ts).cast(\"timestamp\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+-------------------+---------------+\n",
      "|    registration_dt|       timestamp_dt|           page|\n",
      "+-------------------+-------------------+---------------+\n",
      "|2018-07-20 10:41:47|2018-10-01 05:30:11|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:40:08|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:43:08|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:43:09|      Thumbs Up|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:47:36|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:51:28|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:55:27|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 05:58:36|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:03:31|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:07:31|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:10:31|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:16:10|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:20:03|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:20:04|         Logout|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:23:21|           Home|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:23:22|     Add Friend|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:23:55|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:27:24|       NextSong|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:27:50|Add to Playlist|\n",
      "|2018-07-20 10:41:47|2018-10-01 06:30:11|       NextSong|\n",
      "+-------------------+-------------------+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(col(\"UserId\") == 293).select([\"registration_dt\", \"timestamp_dt\", \"page\"]).sort(col(\"timestamp_dt\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+-------------------+\n",
      "|  min(timestamp_dt)|  max(timestamp_dt)|\n",
      "+-------------------+-------------------+\n",
      "|2018-10-01 05:30:11|2018-12-01 05:31:06|\n",
      "+-------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Timeframe of the dataset we have at hand\n",
    "df.agg(min(\"timestamp_dt\"), max(\"timestamp_dt\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+\n",
      "|min(registration_dt)|max(registration_dt)|\n",
      "+--------------------+--------------------+\n",
      "| 2017-11-05 09:26:33| 2018-11-24 21:07:54|\n",
      "+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Timeframe of registrations\n",
    "df.agg(min(\"registration_dt\"), max(\"registration_dt\")).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation scatter plot\n",
    "Let's see if the variables in the dataset are derivatives of each other and whether we can eliminate some."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('artist', 'string'),\n",
       " ('auth', 'string'),\n",
       " ('firstName', 'string'),\n",
       " ('gender', 'string'),\n",
       " ('itemInSession', 'bigint'),\n",
       " ('lastName', 'string'),\n",
       " ('length', 'double'),\n",
       " ('level', 'string'),\n",
       " ('location', 'string'),\n",
       " ('method', 'string'),\n",
       " ('page', 'string'),\n",
       " ('registration', 'bigint'),\n",
       " ('sessionId', 'bigint'),\n",
       " ('song', 'string'),\n",
       " ('status', 'bigint'),\n",
       " ('ts', 'bigint'),\n",
       " ('userAgent', 'string'),\n",
       " ('userId', 'string'),\n",
       " ('device', 'string'),\n",
       " ('registration_dt', 'timestamp'),\n",
       " ('timestamp_dt', 'timestamp')]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col0 {\n",
       "            background-color:  #023858;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col1 {\n",
       "            background-color:  #fdf5fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col2 {\n",
       "            background-color:  #e4e1ef;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col3 {\n",
       "            background-color:  #fef6fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col4 {\n",
       "            background-color:  #f5eff6;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col0 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col1 {\n",
       "            background-color:  #023858;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col2 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col3 {\n",
       "            background-color:  #fdf5fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col4 {\n",
       "            background-color:  #fcf4fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col0 {\n",
       "            background-color:  #e7e3f0;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col1 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col2 {\n",
       "            background-color:  #023858;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col3 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col4 {\n",
       "            background-color:  #1c7fb8;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col0 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col1 {\n",
       "            background-color:  #fcf4fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col2 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col3 {\n",
       "            background-color:  #023858;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col4 {\n",
       "            background-color:  #fff7fb;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col0 {\n",
       "            background-color:  #f5eef6;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col1 {\n",
       "            background-color:  #faf2f8;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col2 {\n",
       "            background-color:  #1b7eb7;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col3 {\n",
       "            background-color:  #fdf5fa;\n",
       "        }    #T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col4 {\n",
       "            background-color:  #023858;\n",
       "        }</style>  \n",
       "<table id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6\" > \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >itemInSession</th> \n",
       "        <th class=\"col_heading level0 col1\" >registration</th> \n",
       "        <th class=\"col_heading level0 col2\" >sessionId</th> \n",
       "        <th class=\"col_heading level0 col3\" >status</th> \n",
       "        <th class=\"col_heading level0 col4\" >ts</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6level0_row0\" class=\"row_heading level0 row0\" >itemInSession</th> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col0\" class=\"data row0 col0\" >1</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col1\" class=\"data row0 col1\" >-0.00120451</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col2\" class=\"data row0 col2\" >0.147675</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col3\" class=\"data row0 col3\" >-0.00137426</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row0_col4\" class=\"data row0 col4\" >0.0667605</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6level0_row1\" class=\"row_heading level0 row1\" >registration</th> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col0\" class=\"data row1 col0\" >-0.00120451</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col1\" class=\"data row1 col1\" >1</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col2\" class=\"data row1 col2\" >-0.0163056</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col3\" class=\"data row1 col3\" >0.00400609</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row1_col4\" class=\"data row1 col4\" >0.0203365</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6level0_row2\" class=\"row_heading level0 row2\" >sessionId</th> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col0\" class=\"data row2 col0\" >0.147675</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col1\" class=\"data row2 col1\" >-0.0163056</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col2\" class=\"data row2 col2\" >1</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col3\" class=\"data row2 col3\" >-0.0129708</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row2_col4\" class=\"data row2 col4\" >0.689274</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6level0_row3\" class=\"row_heading level0 row3\" >status</th> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col0\" class=\"data row3 col0\" >-0.00137426</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col1\" class=\"data row3 col1\" >0.00400609</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col2\" class=\"data row3 col2\" >-0.0129708</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col3\" class=\"data row3 col3\" >1</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row3_col4\" class=\"data row3 col4\" >0.000471797</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6level0_row4\" class=\"row_heading level0 row4\" >ts</th> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col0\" class=\"data row4 col0\" >0.0667605</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col1\" class=\"data row4 col1\" >0.0203365</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col2\" class=\"data row4 col2\" >0.689274</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col3\" class=\"data row4 col3\" >0.000471797</td> \n",
       "        <td id=\"T_8681772a_4e45_11e9_a763_e0d55e2acff6row4_col4\" class=\"data row4 col4\" >1</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f7ccc484a58>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numeric_features = [t[0] for t in df.dtypes if t[1] == 'bigint']\n",
    "\n",
    "sampled_data = df.select(numeric_features).sample(False, 0.10).toPandas()\n",
    "sampled_data\n",
    "# axs = pd.scatter_matrix(sampled_data, figsize=(12, 12));\n",
    "corr = sampled_data.corr()\n",
    "corr.style.background_gradient()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "'ts' and 'sessionId' seem to have a correlation between them."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Churn\n",
    "\n",
    "Once you've done some preliminary analysis, create a column `Churn` to use as the label for your model. I suggest using the `Cancellation Confirmation` events to define your churn, which happen for both paid and free users. As a bonus task, you can also look into the `Downgrade` events."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "cancelled_users = df.filter(col(\"page\") == \"Cancellation Confirmation\").select(\"userId\").toPandas().userId.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.withColumn(\"churned\", df.userId.isin(cancelled_users))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------------------+\n",
      "|churned|count(DISTINCT userId)|\n",
      "+-------+----------------------+\n",
      "|   true|                    99|\n",
      "|  false|                   349|\n",
      "+-------+----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.groupBy(\"churned\").agg(countDistinct(\"userId\")).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explore Data\n",
    "Once you've defined churn, perform some exploratory data analysis to observe the behavior for users who stayed vs users who churned. You can start by exploring aggregates on these two groups of users, observing how much of a specific action they experienced per a certain time unit or number of songs played."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Platform used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_device = df.groupBy(\"device\", \"churned\").agg(countDistinct(\"UserId\").alias(\"count_user\")).sort(col(\"count_user\").desc()).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmgAAAF3CAYAAAARh7eaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XucnvOd//HXJwmiqEMyrIo0YbFCI42hDj1YlLarTktLlWhZdlu02y3a2l+rlm27umt7dChKWr86tVot25agTtVIiIQEyY9gbEhK6SlU5PP747ombskkmSRz39c1c7+ej8c85rq+1+H+3NfMXPO+v9cpMhNJkiTVx6CqC5AkSdLrGdAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqZkjVBayp4cOH56hRo6ouQ5IkaaWmTp3628zsWNl8/T6gjRo1iilTplRdhiRJ0kpFxBO9mc9DnJIkSTVjQJMkSaoZA5okSVLN9Ptz0CQNXK+88gpdXV289NJLVZfSMkOHDmXEiBGstdZaVZciqUIGNEm11dXVxQYbbMCoUaOIiKrLabrM5LnnnqOrq4vRo0dXXY6kCnmIU1JtvfTSSwwbNqwtwhlARDBs2LC26jGU1DMDmqRaa5dw1q3d3q+knhnQJA1Ixx57LNdee23b1yCpfzKgSVIPXn311apLkNTGDGiSBoSJEycyduxYdtppJ44++mgAbr/9dvbYYw+22mqrJT1Zt912GwcccMCS5U466SQuu+wyoHgyyemnn8748eO55ppr2GuvvTj99NPZdddd2XbbbbnjjjuAIrydeuqp7LLLLowdO5YLL7wQKE7yP+mkk9huu+3Yd999mT9/fgu3gKSBxKs4JfV7Dz30EGeffTZ33303w4cP5/nnn+dTn/oU8+bN48477+Thhx/mwAMP5LDDDlvpuoYNG8Z9990HwAUXXMCiRYuYPHkyN954I1/84he5+eabueSSS9hwww259957efnll9lzzz3Zb7/9uP/++3nkkUeYOXMmzz77LGPGjOGjH/1os9++pAHIgCap37vllls4/PDDGT58OACbbLIJAAcffDCDBg1izJgxPPvss71a1wc/+MHXjR966KEA7LzzzsydOxeAX/7yl0yfPn1Jr9yLL77I7Nmzuf322znyyCMZPHgwb3rTm9h777374u1JakMGNEkD1jrrrLNkODMBGDJkCIsXL17SvvQtLdZbb70e1zF48GAWLVq0ZF3f+MY32H///V8374033th3xUtqa20R0HY+dWLVJSxj6rnHVF2CNGDsvffeHHLIIXzqU59i2LBhPP/888ud981vfjMzZ87k5ZdfZuHChUyaNIm3v/3tq/R6+++/P+effz577703a621Fo8++ihbbLEF73znO7nwwguZMGEC8+fP59Zbb+VDH/rQmr49SW2oLQKapIFthx124IwzzuBd73oXgwcP5q1vfety591yyy35wAc+wI477sjo0aNXOO/yHH/88cydO5fx48eTmXR0dPDjH/+YQw45hFtuuYUxY8YwcuRIdt999zV5W5LaWHR3+zdl5RGXAgcA8zNzx4b2k4GPA68CN2TmaWX7Z4HjyvZTMvMXK3uNzs7OnDJlygrnsQdN6p9mzZrF9ttvX3UZLdeu71tqBxExNTM7VzZfs3vQLgO+CSxJSBHxt8BBwE6Z+XJEbFq2jwGOAHYA3gTcHBHbZqY3I5IkSW2lqfdBy8zbgaVPBvkn4MuZ+XI5T/eNgg4CrszMlzPzcWAOsGsz65MkSaqjKm5Uuy3wjoj4TUT8KiJ2Kdu3AJ5qmK+rbJMkSWorVVwkMATYBNgN2AW4OiK2WpUVRMQJwAkAI0eO7PMCJUmSqlRFD1oX8KMsTAYWA8OBp4EtG+YbUbYtIzMvyszOzOzs6OhoesGSJEmtVEVA+zHwtwARsS2wNvBb4HrgiIhYJyJGA9sAkyuoT5IkqVJNDWgR8QPg18B2EdEVEccBlwJbRcSDwJXAhLI37SHgamAm8HPg417BKalqgwcPZty4cUu+uh/31JO5c+ey4447Lne6JPVWU89By8wjlzPpw8uZ/xzgnOZVJKk/6+t7GvbmfoTrrrsu06ZN69PXlaSVqeIQpyT1a3PnzuUd73gH48ePZ/z48dx9993LzPPQQw+x6667Mm7cOMaOHcvs2bMB+P73v7+k/cQTT+TVVz1QIGlZBjRJWoGFCxcuObx5yCGHALDpppty0003cd9993HVVVdxyimnLLPcBRdcwCc+8QmmTZvGlClTGDFiBLNmzeKqq67irrvuYtq0aQwePJgrrrii1W9JUj/gszglaQV6OsT5yiuvcNJJJy0JWY8++ugyy+2+++6cc845dHV1ceihh7LNNtswadIkpk6dyi67FLd/XLhwIZtuumlL3oek/sWAJkmr6LzzzmOzzTbjgQceYPHixQwdOnSZeT70oQ/xtre9jRtuuIH3ve99XHjhhWQmEyZM4Etf+lIFVUvqTzzEKUmr6MUXX2TzzTdn0KBBfO973+vxPLLHHnuMrbbailNOOYWDDjqI6dOns88++3Dttdcyf37xhLvnn3+eJ554otXlS+oHDGiStIo+9rGPcfnll7PTTjvx8MMPs9566y0zz9VXX82OO+7IuHHjePDBBznmmGMYM2YMZ599Nvvttx9jx47l3e9+N/PmzavgHUiqu8jMqmtYI52dnTllypQVztPXl+b3hd5c3i+1u1mzZrH99ttXXUbLtev7ltpBREzNzM6VzWcPmiRJUs0Y0CRJkmrGgCZJklQzBjRJkqSaMaBJkiTVjAFNkiSpZnySgCT14LnnnmOfffYB4JlnnmHw4MF0dHQAMHnyZNZee+0qy5M0wBnQJPUbT571lj5d38jPz1jutGHDhi15BueZZ57J+uuvz6c//enXzZOZZCaDBnkwQlLfcq8iSatgzpw5jBkzhqOOOooddtiBp556io022mjJ9CuvvJLjjz8egGeffZZDDz2Uzs5Odt11V+65556qypbUz9iDJkmr6OGHH2bixIl0dnayaNGi5c53yimncNppp7Hbbrsxd+5cDjjgAB588MEWViqpvzKgSdIq2nrrrensXOmTWrj55pt55JFHloz/7ne/Y+HChay77rrNLE/SAGBAk6RV1Phw9EGDBtH4TOOXXnppyXBmekGBpNXiOWiStAYGDRrExhtvzOzZs1m8eDHXXXfdkmn77rsv3/rWt5aMd190IEkrY0CTpDX0la98hf3335899tiDESNGLGn/1re+xV133cXYsWMZM2YM3/nOdyqsUlJ/Eo1d8/1RZ2dnTpkyZYXz7HzqxBZV03tTzz2m6hKk2ps1axbbb7991WW0XLu+b6kdRMTUzFzpSaz2oEmSJNWMAU2SJKlmDGiSJEk1Y0CTVGv9/TzZVdVu71dSzwxokmpr6NChPPfcc20TWjKT5557jqFDh1ZdiqSKeaNaSbU1YsQIurq6WLBgQdWltMzQoUNfd6sOSe3JgCapttZaay1Gjx5ddRmS1HJNPcQZEZdGxPyIWObpwBHxLxGRETG8HI+I+HpEzImI6RExvpm1SZIk1VWzz0G7DHjP0o0RsSWwH/BkQ/N7gW3KrxOA85tcmyRJUi01NaBl5u3A8z1MOg84DWg88/cgYGIW7gE2iojNm1mfJElSHbX8Ks6IOAh4OjMfWGrSFsBTDeNdZZskSVJbaelFAhHxBuBzFIc312Q9J1AcBmXkyJF9UJkkSVJ9tLoHbWtgNPBARMwFRgD3RcRfAU8DWzbMO6JsW0ZmXpSZnZnZ2dHR0eSSJUmSWqulAS0zZ2Tmppk5KjNHURzGHJ+ZzwDXA8eUV3PuBryYmfNaWZ8kSVIdNPs2Gz8Afg1sFxFdEXHcCma/EXgMmAN8B/hYM2uTJEmqq6aeg5aZR65k+qiG4QQ+3sx61Do7nzqx6hKWMfXcY6ouQZKkXvFZnJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUM00NaBFxaUTMj4gHG9rOjYiHI2J6RFwXERs1TPtsRMyJiEciYv9m1iZJklRXze5Buwx4z1JtNwE7ZuZY4FHgswARMQY4AtihXObbETG4yfVJkiTVTlMDWmbeDjy/VNsvM3NROXoPMKIcPgi4MjNfzszHgTnArs2sT5IkqY6qPgfto8D/lMNbAE81TOsq2yRJktpKZQEtIs4AFgFXrMayJ0TElIiYsmDBgr4vTpIkqUKVBLSIOBY4ADgqM7NsfhrYsmG2EWXbMjLzoszszMzOjo6OptYqSZLUai0PaBHxHuA04MDM/HPDpOuBIyJinYgYDWwDTG51fZIkSVUb0syVR8QPgL2A4RHRBXyB4qrNdYCbIgLgnsz8x8x8KCKuBmZSHPr8eGa+2sz6JEmS6qipAS0zj+yh+ZIVzH8OcE7zKpIkSaq/qq/ilCRJ0lIMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqpqkBLSIujYj5EfFgQ9smEXFTRMwuv29ctkdEfD0i5kTE9IgY38zaJEmS6qrZPWiXAe9Zqu0zwKTM3AaYVI4DvBfYpvw6ATi/ybVJkiTVUlMDWmbeDjy/VPNBwOXl8OXAwQ3tE7NwD7BRRGzezPokSZLqqIpz0DbLzHnl8DPAZuXwFsBTDfN1lW2SJEltpdKLBDIzgVzV5SLihIiYEhFTFixY0ITKJEmSqlNFQHu2+9Bl+X1+2f40sGXDfCPKtmVk5kWZ2ZmZnR0dHU0tVpIkqdVWGNAiYnBEPNzHr3k9MKEcngD8pKH9mPJqzt2AFxsOhUqSJLWNISuamJmvRsQjETEyM59c1ZVHxA+AvYDhEdEFfAH4MnB1RBwHPAF8oJz9RuB9wBzgz8BHVvX1JEmSBoIVBrTSxsBDETEZ+FN3Y2YeuLIFM/PI5Uzap4d5E/h4L+qRJEka0HoT0P5P06uQJEnSEisNaJn5q4h4M7BNZt4cEW8ABje/NEmSpPa00qs4I+IfgGuBC8umLYAfN7MoSZKkdtab22x8HNgT+D1AZs4GNm1mUZIkSe2sNwHt5cz8S/dIRAxhNW4uK0mSpN7pTUD7VUR8Dlg3It4NXAP8tLllSZIkta/eBLTPAAuAGcCJFPcr+9dmFiVJktTOenMV52LgO8B3ImITYER5zzJJkiQ1QW+u4rwtIt5YhrOpFEHtvOaXJkmS1J56c4hzw8z8PXAoMDEz30YPTwKQJElS3+hNQBsSEZtTPDPzZ02uR5Ikqe31JqCdBfwCmJOZ90bEVsDs5pYlSZLUvnpzkcA1FLfW6B5/DPj7ZhYlSZLUzlYa0CLiu/RwY9rM/GhTKpIkSWpzKw1ovP68s6HAIcD/NqccSZIk9eYQ5w8bxyPiB8CdTatIkiSpzfXmIoGlbYMPS5ckSWqa3pyD9gdefw7aM8DpTatIkiSpzfXmEOcGK5oeETtk5kN9V5IkSVJ7W51DnEv7Xh+sQ5IkSaW+CGjRB+uQJElSqS8C2jL3SJMkSdLq64uAJkmSpD7UFwHtL32wDkmSJJVWGtAiYtKK2jJzt74uSpIkqZ0t9zYbETEUeAMwPCI25rWLAd4IbNGC2iRJktrSiu6DdiLwSeBNwFReC2i/B77Z5LokSZLa1nIDWmZ+DfhaRJycmd9oYU2SJEltrTdPEvhGROwBjGqcPzMnNrEuSZKkttWbZ3F+D9gamAa8WjYnsEYBLSL+GTi+XNcM4CPA5sCVwDCKw6pHZ6ZXiUqSpLay0oAGdAJjMrPPbkgbEVsAp5TrXRgRVwNHAO8DzsvMKyPiAuA44Py+el1JkqT+oDf3QXsQ+KsmvPYQYN2IGEJxteg8YG/g2nL65cDBTXhdSZKkWutND9pwYGZETAZe7m7MzANX90Uz8+mI+CrwJLAQ+CXFIc0XMnNROVsX3s5DkiS1od4EtDP7+kXL+6odBIwGXgCuAd6zCsufAJwAMHLkyL4uT5IkqVK9uYrzV0143X2BxzNzAUBE/AjYE9goIoaUvWgjgKeXU9NFwEUAnZ2dPqxdkiQNKL151NMfIuL35ddLEfFqRPx+DV/3SWC3iHhDRASwDzATuBU4rJxnAvCTNXwdSZKkfqc3PWgbdA+XYeogYI2ev5mZv4mIa4H7gEXA/RQ9YjcAV0bE2WXbJWvyOpIkSf1Rb85BW6K81caPI+ILwGfW5IUz8wvAF5ZqfgzYdU3WK0mS1N/15ka1hzaMDqK4L9pLTatIkiSpzfWmB+39DcOLgLkUhzklSZLUBL05B+0jrShEkiRJhd5cxTkiIq6LiPnl1w8jYkQripMkSWpHvXnU03eB64E3lV8/LdskSZLUBL0JaB2Z+d3MXFR+XQZ0NLkuSZKkttWbgPZcRHw4IgaXXx8Gnmt2YZIkSe2qNwHto8AHgGeAeRR3+j+2iTVJkiS1td7cZuMsYEJm/g4gIjYBvkoR3CRJktTHetODNrY7nAFk5vPAW5tXkiRJUnvrTUAbFBEbd4+UPWir9IgoSZIk9V5vgtZ/Ar+OiGvK8cOBc5pXkiRJUnvrzZMEJkbEFGDvsunQzJzZ3LIkSZLaV68OVZaBzFAmSZLUAr05B02SJEktZECTJEmqGQOaJElSzRjQJEmSasaAJkmSVDMGNEmSpJoxoEmSJNWMAU2SJKlmDGiSJEk1Y0CTJEmqGQOaJElSzRjQJEmSasaAJkmSVDMGNEmSpJoxoEmSJNWMAU2SJKlmKgtoEbFRRFwbEQ9HxKyI2D0iNomImyJidvl946rqkyRJqkqVPWhfA36emX8D7ATMAj4DTMrMbYBJ5bgkSVJbqSSgRcSGwDuBSwAy8y+Z+QJwEHB5OdvlwMFV1CdJklSlqnrQRgMLgO9GxP0RcXFErAdslpnzynmeATbraeGIOCEipkTElAULFrSoZEmSpNaoKqANAcYD52fmW4E/sdThzMxMIHtaODMvyszOzOzs6OhoerGSJEmtVFVA6wK6MvM35fi1FIHt2YjYHKD8Pr+i+iRJkipTSUDLzGeApyJiu7JpH2AmcD0woWybAPykgvIkSZIqNaTC1z4ZuCIi1gYeAz5CERivjojjgCeAD1RYnyRJUiUqC2iZOQ3o7GHSPq2uRZIkqU58koAkSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1cyQqgtoV0+e9ZaqS1jGyM/PqLoESZKEPWiSJEm1Y0CTJEmqGQOaJElSzRjQJEmSasaAJkmSVDMGNEmSpJoxoEmSJNVMpQEtIgZHxP0R8bNyfHRE/CYi5kTEVRGxdpX1SZIkVaHqHrRPALMaxr8CnJeZfw38DjiukqokSZIqVFlAi4gRwN8BF5fjAewNXFvOcjlwcDXVSZIkVafKHrT/Bk4DFpfjw4AXMnNROd4FbFFFYZIkSVWqJKBFxAHA/MycuprLnxARUyJiyoIFC/q4OkmSpGpV1YO2J3BgRMwFrqQ4tPk1YKOI6H6A+wjg6Z4WzsyLMrMzMzs7OjpaUa8kSVLLVBLQMvOzmTkiM0cBRwC3ZOZRwK3AYeVsE4CfVFGfJElSlaq+inNppwOfiog5FOekXVJxPZIkSS03ZOWzNFdm3gbcVg4/BuxaZT2SJElVq1sPmiRJUtszoEmSJNWMAU2SJKlmDGiSJEk1U/lFAlKrPHnWW6ouYRkjPz+j6hIkSTVkD5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaoZA5okSVLNGNAkSZJqxoAmSZJUMwY0SZKkmjGgSZIk1YwBTZIkqWYMaJIkSTVjQJMkSaqZSgJaRGwZEbdGxMyIeCgiPlG2bxIRN0XE7PL7xlXUJ0mSVKWqetAWAf+SmWOA3YCPR8QY4DPApMzcBphUjkuSJLWVIVW8aGbOA+aVw3+IiFnAFsBBwF7lbJcDtwGnV1Ci1O/sfOrEqktYxtRzj6m6BEnqlyo/By0iRgFvBX4DbFaGN4BngM0qKkuSJKkylQa0iFgf+CHwycz8feO0zEwgl7PcCRExJSKmLFiwoAWVSpIktU5lAS0i1qIIZ1dk5o/K5mcjYvNy+ubA/J6WzcyLMrMzMzs7OjpaU7AkSVKLVHUVZwCXALMy878aJl0PTCiHJwA/aXVtkiRJVavkIgFgT+BoYEZETCvbPgd8Gbg6Io4DngA+UFF9kiRJlanqKs47gVjO5H1aWYskSVLdVH4VpyRJkl7PgCZJklQzBjRJkqSaMaBJkiTVjAFNkiSpZgxokiRJNWNAkyRJqhkDmiRJUs0Y0CRJkmrGgCZJklQzBjRJkqSaMaBJkiTVTCUPS5ckNceTZ72l6hKWMfLzM/psXTufOrHP1tVXpp57TNUlaACyB02SJKlmDGiSJEk1Y0CTJEmqGc9BkyRJy+V5f9WwB02SJKlmDGiSJEk14yFOSVpNdTz0c90GVVcgqS/YgyZJklQzBjRJkqSaMaBJkiTVjOegSWqagf7YIUlqFnvQJEmSasaAJkmSVDMGNEmSpJoxoEmSJNWMAU2SJKlmancVZ0S8B/gaMBi4ODO/XHFJkiQtl1crqxlq1YMWEYOBbwHvBcYAR0bEmGqrkiRJaq1aBTRgV2BOZj6WmX8BrgQOqrgmSZKklqpbQNsCeKphvKtskyRJahuRmVXXsEREHAa8JzOPL8ePBt6WmSctNd8JwAnl6HbAIy0ttG8MB35bdRFtxm3eem7z1nObt57bvPX68zZ/c2Z2rGymul0k8DSwZcP4iLLtdTLzIuCiVhXVDBExJTM7q66jnbjNW89t3npu89Zzm7deO2zzuh3ivBfYJiJGR8TawBHA9RXXJEmS1FK16kHLzEURcRLwC4rbbFyamQ9VXJYkSVJL1SqgAWTmjcCNVdfRAv36EG0/5TZvPbd567nNW89t3noDfpvX6iIBSZIk1e8cNEmSpLZnQFuJiDgvIj7ZMP6LiLi4Yfw/I+JzEXHtKq732Ij4Zl/W2t9FREbE9xvGh0TEgoj42Wqu7+KVPYkiIg5e3adVRMSZEfHp1Vl2IIiILSPi8YjYpBzfuBwfFRE/j4gXlv7ZRcRJETGn/FkPr6byeouIu8vvoyJiYURMi4iZEXFBRAyKiL1W929CrbX0/iUizoqIfcvhuT39DbhfWfHv/yqsp99vRwPayt0F7AFQ/nIMB3ZomL4HcEtmHlZBbQPNn4AdI2Ldcvzd9HCbld7KzOMzc+ZKZjuY4rFiWkWZ+RRwPtD9vNwvAxdl5lzgXODoHha7C9gXeKIVNfZHmblHw+j/y8xxwFiK39ODq6lKq+l1+5fM/Hxm3lxhPbXn7/9rDGgrdzewezm8A/Ag8Ieyt2AdYHvg+Yh4EJb0jP2o7EGYHRH/0b2iiPhIRDwaEZOBPRvaR0XELRExPSImRcTIiBhc9kZERGwUEa9GxDvL+W+PiG0i4l3lp4tpEXF/RGzQom3STDcCf1cOHwn8oHtCROwaEb8u3+vdEbFd2T44Ir4aEQ+W2/Dksv22iOgsh/8YEedExAMRcU9EbBYRewAHAueW23DriBhXTp8eEddFxMbl8qeUn+KmR8SVDfWOKV/nsYg4pfmbp3bOA3Yre5nfDnwVIDMnAX9YeubMvL8McFqOiPjj0m2ZuYhiX/TXZdP6EXFtRDwcEVdERJTL7lP+fcyIiEvLfVR3b80XI+K+ctrflO3rlfNNLpcbUI/Wi4hjyr/ZByLiez3ta8v5LouI88u//cfKXspLI2JWRFzWsL4/RnFU5aFy+Y6y/R8i4t7ydX4YEW9Yzv7lsihuyN7ttPLnMTki/pqllMv8PCKmRsQd3T+3gWxlv/8RsX657bt/lw9qWPaM8n/snRQ3se/fMtOvlXwBjwMjgROBfwT+DXgfRci6AxgFPFjOeyzwGLAhMJSip2BLYHPgSaADWJuiJ+Gb5TI/BSaUwx8FflwO/5wiFB5AcY+4M4B1gMcbltuzHF4fGFL1tlrD7fxHik9K15bbbhqwF/Czcvobu98jRS/MD8vhfyqX6Z62Sfn9NqCzHE7g/eXwfwD/Wg5fBhzWUMN04F3l8FnAf5fD/wusUw5vVH4/k2KnsQ5Fz+pzwFpVb8cKfm77l9v33Uu1L/nZ9bDMXGB41bXX8Qv4Y/m9cb/yhnIf8N5yu75IcSPvQcCvKcLxUIpH5W1bLjMR+GTD9j65HP4YcHE5/O/Ah8vhjYBHgfWq3gZ9tB13KN/P8HJ8kxXsay+jePZzUDz/+ffAW8rtOxUYV86XwFHl8Od5bR8+rOF1z27Y1kvvX5aMlz+TM8rhY3htP3cm8OlyeBKwTTn8NoqjNZVv2yb/3Fb2+z8EeGPZPhyYU/7cdgZmlPO+sWz/dNXvZ02+7EHrnbspDmXuQbEz/HXD+F09zD8pM1/MzJeAmcCbKf64bsvMBVk8CP6qhvl3B/5vOfw9ip0tFOHvneXXl8r2XSh+USlf+7/KnpuNsviU0a9l5nSKP8wjWfZ2Kxsyt90xAAAGLUlEQVQC10TRW3kerx1q3he4sPv9Z+bzPaz6L0D3eTtTy9d4nYjYkGI7/qpsupxi20MR3K6IiA8Djdv5hsx8OTN/C8wHNuvdOx1Q3gvMA3asupABaOuImEbxt35DZv5P2T45M7syczHFB5lRFD0Gj2fmo+U8jb+/AD8qvzf+/u8HfKZ8jdsoQt7I5ryVltsbuKb82+zeLyxvXwvw0yz+688Ans3MGeX2fYjXttdiXtt3f79h+R3LHq4ZwFG8/jSYFflBw/fdGydExPoU/2OuKX8+F1J80G8nPf3+B/DvETEduJnied2bAe8ArsvMP2fm7xkAN7mv3X3Qaqr7PLS3UBzifAr4F4pPWd/tYf6XG4ZfZfW38+0UvUNvovi0dirFp+c7ADLzyxFxA0Vv3l0RsX9mPryar1Un11McKtsLGNbQ/m/ArZl5SESMoviH0luvlDtfWL2fyd9R/LN7P3BGRLylbO+rn3W/FBHjKM4V3A24MyKuzMx5FZc1kHSfg7O01fm9616mcf4A/j4z++PzjPta9/ZZzOu372KWv3279ymXAQdn5gMRcSzFvqs3cjnDUPTevbCcn3+76On3/yiKI1E7Z+YrETGX4oPFgGMPWu/cTXGY8fnMfLX8JLYRxSeeu3u5jt8A74qIYRGxFnD4Uus/ohw+ijKAAZMpguHisjduGsVh1tuhOD+h/JT3FYpetYFyfsKlwBczc8ZS7Rvy2kUDxza03wScGBFDAKK8qrCX/gBsAJCZLwK/i4h3lNOOBn4VxcUhW2bmrcDpZR3rr8JrDEjleU/nUxxGe5LiwoCvVltVW3sEGNVwLtPRwK9WMD8UT205ueEctrc2sb5WuwU4PCKGwZL9wvL2tb01COg+h+xDwJ3l8AbAvHLfflTD/Ev2L8vxwYbvv26cUPYCPR4Rh5f1R0TstIr1DkQbAvPLcPa3FEeooPi/eHBErBvF+djvr6zCPmJA650ZFMe671mq7cXu7vOVKXsVzqT4I7wLmNUw+WTgI2WX7dHAJ8plXqboret+3Tso/ti7g8snozwxHngF+B8GgPLQzdd7mPQfwJci4n5e/4n2Yorz+6ZHxAMUO87euhI4tTxBemtgAsVJvdOBcRTnoQ0Gvl8evrgf+HpmvrDKb2zg+Qfgycy8qRz/NrB9FBev3AFcA+wTEV0RsT8sudiii+L8qenRcMsarZnyQ9xHKA6JzaDo+blgJYv9G7AWxc/ioXJ8QMjiMYHnUHzIegD4L5azr10FfwJ2LU+z2Jti/wDwfyg+hN8FNB7FWHr/srSNy1o+AfxzD9OPAo4r63+I4vy4dncF0Fn+jh9Dub0z8z6Kw88PUPwvvHe5a+gnfJKAJEm9EBF/zMy27z1Xa9iDJkmSVDP2oEmSJNWMPWiSJEk1Y0CTJEmqGQOaJElSzRjQJLWFiDgzIj69GsudFRH7NqMmSVqetrrruSStqsz8fNU1SGo/9qBJGrAi4oyIeDQi7qR4ViURsXVE/DwippbPT/ybiNgwIp4onxpBRKwXEU9FxFoRcVlEHFa27xIRd0fEAxExOSI2iIjBEXFuRNwbEdMj4sQK37KkAcIeNEkDUkTsTPFYn3EU+7r7KB4UfhHwj5k5OyLeBnw7M/cuH8r8LuBWike7/aJ8nEz3+tamuFP5BzPz3oh4I7AQOI7iqSK7RMQ6FM/F/WVmPt7SNyxpQDGgSRqo3gFcl5l/BoiI6ykeqrwHxeOQuudbp/x+FcUzEW+lCHbfXmp92wHzMvNeWPKsRCJiP2Bsdy8bxbMCtwEMaJJWmwFNUjsZBLyQmeN6mHY98O/lQ7V3pnjYdm8EcHJm/qKPapQkz0GTNGDdDhwcEetGxAbA+4E/A49HxOEAUdgJIDP/SPGA5a8BP8vMV5da3yPA5hGxS7nsBhExBPgF8E8RsVbZvm1ErNeC9ydpALMHTdKAlJn3RcRVwAPAfIrwBXAUcH5E/CuwFnBlOQ8UhzmvAfbqYX1/iYgPAt+IiHUpzj/bF7gYGAXcF8Vx0wXAwU16W5LahM/ilCRJqhkPcUqSJNWMAU2SJKlmDGiSJEk1Y0CTJEmqGQOaJElSzRjQJEmSasaAJkmSVDMGNEmSpJr5/1KcAMWk6g77AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,6))\n",
    "\n",
    "g = sns.barplot(ax=ax, x=\"device\", y=\"count_user\", hue=\"churned\", data=df_device)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can observe that churned users are more than non churned users for iPhone. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- artist: string (nullable = true)\n",
      " |-- auth: string (nullable = true)\n",
      " |-- firstName: string (nullable = true)\n",
      " |-- gender: string (nullable = true)\n",
      " |-- itemInSession: long (nullable = true)\n",
      " |-- lastName: string (nullable = true)\n",
      " |-- length: double (nullable = true)\n",
      " |-- level: string (nullable = true)\n",
      " |-- location: string (nullable = true)\n",
      " |-- method: string (nullable = true)\n",
      " |-- page: string (nullable = true)\n",
      " |-- registration: long (nullable = true)\n",
      " |-- sessionId: long (nullable = true)\n",
      " |-- song: string (nullable = true)\n",
      " |-- status: long (nullable = true)\n",
      " |-- ts: long (nullable = true)\n",
      " |-- userAgent: string (nullable = true)\n",
      " |-- userId: string (nullable = true)\n",
      " |-- device: string (nullable = true)\n",
      " |-- registration_dt: timestamp (nullable = true)\n",
      " |-- timestamp_dt: timestamp (nullable = true)\n",
      " |-- churned: boolean (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_device = df.select([\"userId\", \"device\"]).distinct()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_device = StringIndexer(inputCol = \"device\", outputCol=\"device_index\").fit(df_device).transform(df_device)\n",
    "df_device = OneHotEncoderEstimator(inputCols=[\"device_index\"], outputCols=[\"device_classVec\"]).fit(df_device).transform(df_device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---------+------------+---------------+\n",
      "|userId|   device|device_index|device_classVec|\n",
      "+------+---------+------------+---------------+\n",
      "|200008|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   114|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   260|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|200025|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   166|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|    92|     iPad|         5.0|      (5,[],[])|\n",
      "|    71|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|100011|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   271|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   300|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|300035|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|   101|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|   294|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|300015|   iPhone|         3.0|  (5,[3],[1.0])|\n",
      "|   257|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|200049|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|   293|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "|   291|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|    97|  Windows|         0.0|  (5,[0],[1.0])|\n",
      "|200043|Macintosh|         1.0|  (5,[1],[1.0])|\n",
      "+------+---------+------------+---------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_device.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Time since registration until last action timestamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ages = df.select([\"userId\", \"churned\", datediff(\"timestamp_dt\", \"registration_dt\")]).groupBy([\"userId\", \"churned\"]).max().select(\"churned\", col(\"max(datediff(timestamp_dt, registration_dt))\").alias(\"age\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>avg(age)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>68.010101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>87.074499</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned   avg(age)\n",
       "0     True  68.010101\n",
       "1    False  87.074499"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ages.groupBy(\"churned\").mean().toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This shows that churned users used the sparkify app for less periods of times on an average than non-churned users."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Number of events/actions transpired in given users' timelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_number_events = df.groupBy([\"userId\", \"churned\"]).count().select(\"churned\", col(\"count\").alias(\"number_events\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>avg(number_events)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>1046.929293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>1215.928367</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned  avg(number_events)\n",
       "0     True         1046.929293\n",
       "1    False         1215.928367"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_number_events.groupBy(\"churned\").mean().toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On an average, churned users have less number of interactions than non-churned users. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Number of thumbs down/thumbs up given by churned vs non-churned users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_thumbs_down = df.filter(col(\"page\") == \"Thumbs Down\").groupBy([\"churned\", \"userId\"]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>avg(count)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>12.988764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>12.516667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned  avg(count)\n",
       "0     True   12.988764\n",
       "1    False   12.516667"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_thumbs_down.groupBy(\"churned\").mean().toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "On an average, people who thumbs down more often indicate signs of churning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "users_thumbs_down = df.filter(col(\"page\") == \"Thumbs Up\").groupBy([\"churned\", \"userId\"]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>avg(count)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>43.877551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>57.260997</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned  avg(count)\n",
       "0     True   43.877551\n",
       "1    False   57.260997"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users_thumbs_down.groupBy(\"churned\").mean().toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Number of sessions per user"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look at one user's session shenanigans."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----+\n",
      "|sessionId|count|\n",
      "+---------+-----+\n",
      "|     4101|  433|\n",
      "|     2687|  296|\n",
      "|     2063|  204|\n",
      "|      443|  204|\n",
      "|     4211|  197|\n",
      "|     4480|  138|\n",
      "|     1361|   80|\n",
      "|     3437|   73|\n",
      "|     3360|   58|\n",
      "|     4774|   39|\n",
      "|     2943|   37|\n",
      "|     4252|   13|\n",
      "+---------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.filter(col(\"userId\")==209).groupBy(\"sessionId\").count().sort(col(\"count\").desc()).show(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_number_sessions = df.groupBy([\"userId\", \"churned\"]).agg(countDistinct(\"sessionId\")).select(\"churned\", col(\"count(DISTINCT sessionId)\").alias(\"number of sessions\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|churned|number of sessions|\n",
      "+-------+------------------+\n",
      "|  false|                13|\n",
      "|  false|                 3|\n",
      "|  false|                12|\n",
      "|  false|                 5|\n",
      "|   true|                 2|\n",
      "|  false|                 2|\n",
      "|   true|                 3|\n",
      "|  false|                13|\n",
      "|  false|                 6|\n",
      "|   true|                 2|\n",
      "|   true|                20|\n",
      "|  false|                37|\n",
      "|   true|                26|\n",
      "|   true|                 6|\n",
      "|  false|                11|\n",
      "|  false|                 2|\n",
      "|   true|                22|\n",
      "|  false|                16|\n",
      "|  false|                18|\n",
      "|  false|                 3|\n",
      "+-------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "user_number_sessions.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>avg(number of sessions)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>12.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>13.991404</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned  avg(number of sessions)\n",
       "0     True                12.090909\n",
       "1    False                13.991404"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_number_sessions.groupBy(\"churned\").mean().toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Page counts distribution among users (Features done, visualization and exploration necessary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = df.select('page').distinct().sort('page')\n",
    "pages_list = [r.page for r in pages.collect()]\n",
    "page_counts = df.groupby('userId').pivot('page', pages_list).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_page = df.groupBy([\"churned\", \"page\"]).count().toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_page[\"log_count\"] = np.log(df_page[\"count\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>churned</th>\n",
       "      <th>page</th>\n",
       "      <th>count</th>\n",
       "      <th>log_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>Save Settings</td>\n",
       "      <td>481</td>\n",
       "      <td>6.175867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>Logout</td>\n",
       "      <td>1184</td>\n",
       "      <td>7.076654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>Submit Downgrade</td>\n",
       "      <td>88</td>\n",
       "      <td>4.477337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>NextSong</td>\n",
       "      <td>348518</td>\n",
       "      <td>12.761445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>Save Settings</td>\n",
       "      <td>104</td>\n",
       "      <td>4.644391</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   churned              page   count  log_count\n",
       "0    False     Save Settings     481   6.175867\n",
       "1     True            Logout    1184   7.076654\n",
       "2    False  Submit Downgrade      88   4.477337\n",
       "3    False          NextSong  348518  12.761445\n",
       "4     True     Save Settings     104   4.644391"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_page.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_page = df_page[~df_page.page.isin(['Cancellation Confirmation', 'Cancel'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16]),\n",
       " <a list of 17 Text xticklabel objects>)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm8AAAGlCAYAAACsrRHHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XecHVX9//HXZ9OBQCAktJBGT0ILm0DoTQIY6VKliAgCERSpFjoCIk16hwDSRZQmXUCkJPQOaoDwlSIoqD9AQj6/Pz5ncmcvG7J7607yfj4e+9g7c8s5d+7Mmc9pM+buiIiIiEgxtDQ7AyIiIiLScQreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiDdm52Belp44YV96NChzc6GiIiIyGxNmTLlH+4+YHavm6ODt6FDhzJ58uRmZ0NERERktszsjY68Tt2mIiIiIgWi4E1ERESkQBS8iYiIiBTIHD3mTURERLq2zz//nGnTpvHpp582OysN07t3bwYNGkSPHj0qer+CNxEREWmaadOm0bdvX4YOHYqZNTs7defufPDBB0ybNo1hw4ZV9BnqNhUREZGm+fTTT+nfv/9cEbgBmBn9+/evqqVRwZuIiIg01dwSuGWq/b4K3kREREQKRMGbiIiIFNYee+zBjTfeOFflQcGbiIiIzLW++OKLZmeh0xS8iYiISGFMmjSJlVZaiZVXXpldd90VgAcffJA111yT4cOHz2wBe+CBB5gwYcLM902cOJHLL78ciNtnHnbYYYwePZobbriB9ddfn8MOO4yxY8ey7LLL8tBDDwER2B1yyCGMGTOGlVZaiQsuuACIGaMTJ05kueWWY+ONN+a9995r4BbQpUJERESkIF544QWOP/54HnnkERZeeGE+/PBDDjroIP7+97/z8MMP8/LLL7PFFluw3Xbbzfaz+vfvz5NPPgnA+eefz/Tp03n88ce5/fbbOeaYY7jnnnu45JJLWGCBBXjiiSf47LPPWGuttdhkk0146qmneOWVV3jxxRd59913GTFiBHvuuWe9v/5MCt5ERESkEO677z6++c1vsvDCCwOw0EILAbDVVlvR0tLCiBEjePfddzv0WTvssEOb5W222QaA1VZbjalTpwJw11138eyzz85szfvoo4947bXXePDBB9lpp53o1q0biy++OBtuuGEtvl6HKXgTERGRQuvVq9fMx+4OQPfu3ZkxY8bM9eXXVZt33nnb/Yxu3boxffr0mZ911llnMX78+Davvf3222uX+QooeJOGW+2QSZ1+z5RTdqtDTkREpEg23HBDtt56aw466CD69+/Phx9+OMvXDhkyhBdffJHPPvuMTz75hHvvvZe11167U+mNHz+e8847jw033JAePXrw6quvssQSS7DuuutywQUXsPvuu/Pee+9x//33s/POO1f79TpMwZuIiIgUwsiRI/nJT37CeuutR7du3Vh11VVn+doll1yS7bffnlGjRjFs2LCvfO2s7LXXXkydOpXRo0fj7gwYMIDf/va3bL311tx3332MGDGCwYMHM27cuGq+VqdZ1rw4J2ptbfXJkyc3OxtSRi1vIiKSeemll1hhhRWanY2Ga+97m9kUd2+d3Xt1qRARERGRAlHwJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRAFbyIiIiIFouu8iYiISJdRyeWkvkpHLjXVrVs3VlxxxZnLv/3tbxk6dGi7r506dSoTJkzg+eefr1UWO03Bm4iIiMzV+vTpw9NPP93sbHSYuk1FREREykydOpV11lmH0aNHM3r0aB555JEvveaFF15g7NixrLLKKqy00kq89tprAFx11VUz1++zzz588cUXNc2bgjcRERGZq33yySesssoqrLLKKmy99dYADBw4kLvvvpsnn3yS6667jgMOOOBL7zv//PM58MADefrpp5k8eTKDBg3ipZde4rrrruNPf/oTTz/9NN26dePqq6+uaX7VbSoiIiJztfa6TT///HMmTpw4MwB79dVXv/S+cePGccIJJzBt2jS22WYblllmGe69916mTJnCmDFjgAgMBw4cWNP8KngTERERKXP66aezyCKL8MwzzzBjxgx69+79pdfsvPPOrL766tx2221svvnmXHDBBbg7u+++OyeeeGLd8tbluk3N7FIze8/Mns+tO8XMXjazZ83sZjPr18w8ioiIyJzto48+YrHFFqOlpYUrr7yy3XFrf/3rXxk+fDgHHHAAW265Jc8++ywbbbQRN954I++99x4AH374IW+88UZN89YVW94uB84G8nOF7waOcPfpZnYycARwWBPyJiIiInXUkUt7NMJ+++3Htttuy6RJk9h0002Zd955v/Sa66+/niuvvJIePXqw6KKL8uMf/5iFFlqI448/nk022YQZM2bQo0cPzjnnHIYMGVKzvJm71+zDasXMhgK3uvuodp7bGtjO3XeZ3ee0trb65MmTa59BqUol1/DpKgeziIjU1ksvvcQKK6zQ7Gw0XHvf28ymuHvr7N7b5bpNO2BP4I5ZPWlme5vZZDOb/P777zcwWyIiIiL1V6jgzcx+AkwHZjnn1t0vdPdWd28dMGBA4zInIiIi0gBdccxbu8xsD2ACsJF3xb5eERERkQYoRPBmZpsChwLrufv/a3Z+RERERJqly3Wbmtk1wJ+B5cxsmpl9h5h92he428yeNrPzm5pJERERkSbpci1v7r5TO6svaXhGRERERLqgLhe8iYiIyNzrzWNXrOnnDT7yua98/oMPPmCjjTYC4J133qFbt25kEx4ff/xxevbsWdP81IKCNxEREZlr9e/ff+Z9TY8++mjmm28+Dj744DavcXfcnZaWrjHarGvkQkRERKQLef311xkxYgS77LILI0eO5K233qJfv9LdOa+99lr22msvAN5991222WYbWltbGTt2LI8++mhd86aWNxEREZF2vPzyy0yaNInW1lamT58+y9cdcMABHHrooayxxhpMnTqVCRMm8Pzzz8/y9dVS8CYiIiLSjqWWWorW1tnerYp77rmHV155ZebyP//5Tz755BP69OlTl3wpeBMRERFpR/5m9C0tLeTvEfDpp5/OfOzuDZ3coDFvIiIiIrPR0tLCggsuyGuvvcaMGTO4+eabZz638cYbc84558xcziZA1Ita3kRERKTLmN2lPZrp5JNPZvz48QwcOJDVVluNzz77DIBzzjmHfffdl8suu4zp06ezwQYbtAnmas3m5NuEtra2+uTJk5udDSmz2iGTOv2eKafsVoeciIhIs7300kussMIKzc5Gw7X3vc1sirvPdpCduk1FRERECkTdplIIlVxxuys3vYuIiFRKLW8iIiLSVHPyEK72VPt9FbyJiIhI0/Tu3ZsPPvhgrgng3J0PPviA3r17V/wZ6jYVERGRphk0aBDTpk3j/fffb3ZWGqZ3794MGjSo4vcreBMREZGm6dGjB8OGDWt2NgpF3aYiIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEAVvIiIiIgWi4E1ERESkQBS8iYiIiBSIgjcRERGRAlHwJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRAFbyIiIiIFouBNREREpEAUvImIiIgUiII3ERERkQJR8CYiIiJSIAreRERERApEwZuIiIhIgSh4ExERESmQLhm8mdmlZvaemT2fW7eQmd1tZq+l/ws2M48iIiIizdAlgzfgcmDTsnWHA/e6+zLAvWlZREREZK7SJYM3d38Q+LBs9ZbAFenxFcBWDc2UiIiISBfQJYO3WVjE3f+eHr8DLNLMzIiIiIg0Q5GCt5nc3QFv7zkz29vMJpvZ5Pfff7/BORMRERGpryIFb++a2WIA6f977b3I3S9091Z3bx0wYEBDMygiIiJSb0UK3n4H7J4e7w7c0sS8iIiIiDRF92ZnoD1mdg2wPrCwmU0DjgJOAq43s+8AbwDbNy+HIiJSS6sdMqmi9005Zbca50Sk6+uSwZu77zSLpzZqaEZEREREupgidZuKiIiIzPUUvImIiIgUiII3ERERkQJR8CYiIiJSIF1ywoKIiEijVTLjVbNdpRnU8iYiIiJSIAreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiC6w4KINI2uaC8i0nlqeRMREREpEAVvIiIiIgWi4E1ERESkQBS8iYiIiBSIgjcRERGRAlHwJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRDdYUGkSXR3ARERqYRa3kREREQKRMGbiIiISIEoeBMREREpEI15k7mSxpuJiEhRqeVNREREpEAUvImIiIgUiII3ERERkQLRmLcG0jgrERERqZZa3kREREQKRMGbiIiISIGo21RERArrzWNX7PR7Bh/5XB1yItI4ankTERERKRAFbyIiIiIFom5TERGRClXSbQtdu+tWV0bo+tTyJiIiIlIghQrezOyHZvaCmT1vZteYWe9m50lERESkkQoTvJnZEsABQKu7jwK6ATs2N1ciIiIijVWY4C3pDvQxs+7APMD/NTk/IiIiIg1VmODN3d8Gfgm8Cfwd+Mjd72purkREREQaqzCzTc1sQWBLYBjwL+AGM/uWu19V9rq9gb0BBg8e3PB8ioiISPF15Vm3hWl5AzYG/ubu77v758BvgDXLX+TuF7p7q7u3DhgwoOGZFBEREamnIgVvbwJrmNk8ZmbARsBLTc6TiIiISEMVJnhz98eAG4EngeeIvF/Y1EyJiIiINFhhxrwBuPtRwFHNzoeIiIhIsxQqeBMRmRN15YHRItL1KHgTkUKp5F6SXfk+kiIinaXgTaSDFDSIiEhXoOBNREREuhQNJfhqdZ1tambDOrJORERERDqm3pcKuamddTfWOU0RERGROVZduk3NbHlgJLCAmW2Te2p+oHc90hQRERGZG9RrzNtywASgH/CN3Pp/A9+tU5oiIiIic7y6BG/ufgtwi5mNc/c/1yMNERERkblRvWebvm5mPwaG5tNy9z3rnK6IiIjIHKnewdstwEPAPcAXdU5LREREZI5X7+BtHnc/rM5piMw1dKFgERGp96VCbjWzzeuchoiIiMhco97B24FEAPeJmX1sZv82s4/rnKaIiIjIHKuu3abu3reeny8iIiIyt6lr8GZm67a33t0frGe6IiIiInOqek9YOCT3uDcwFpgCbFjndEVERETmSPXuNs3fXQEzWxI4o55pzmk0u1BERETy6t3yVm4asEKD0xQREZE6UkNDY9V7zNtZgKfFFmAV4Ml6pikiIiIyJ6t3y9vk3OPpwDXu/qc6pykiIiIyx6r3mLcrzKwnsGxa9Uo90xMRERGZ09W723R94ApgKmDAkma2uy4VIiIiIlKZenebngps4u6vAJjZssA1wGp1TldERERkjlTv22P1yAI3AHd/FehR5zRFRERE5lh1n7BgZhcDV6XlXWg7iUFEpKlWO2RSp98z5ZTd6pATEZGOqXfwti+wP3BAWn4IOLfOaYqIiIg0XKOud1fv4K07cKa7nwZgZt2AXnVOU0RERGSOVe8xb/cCfXLLfYB76pymiIiIyByr3i1vvd39P9mCu//HzOapc5oiIiKFpDGY0hH1bnn7r5mNzhbMbDXgkzqnKSIiIjLHqnfL2w+AG8zs/4iL9C4K7FDnNEVERETmWPW+PdYTZrY8sFxa9Yq7f549b2Zfc/e765kHERERkTlJvVveSMHa87N4+mRAwZuISCc16pIEItL11HvM2+xYk9MXERERKZRmB2/e5PRFRERECqXZwZuIiIiIdELdx7zNxtQmpy8iIiJzgLlpHGhdgzcz26ad1R8Bz7n7e+7e3vMiIiIiMgv1bnn7DjAOuD8trw9MAYaZ2bHufmWd05+pkqtWg65cLSIiIl1Lvce8dQdWcPdt3X1bYAQxSWF14LDOfpiZ9TOzG83sZTN7yczG1Ti/IiIiIl1avVvelnT3d3PL76V1H5rZ57N601c4E7jT3bczs56A7pMqIiIic5V6B28PmNmtwA1pebu0bl7gX535IDNbAFgX2APA3f8H/K92WRURERHp+uodvO0PbAOsnZavAG5ydwc26ORnDQPeBy4zs5WJsXMHuvt/8y8ys72BvQEGDx5cRdZF5myVjAPVGFARkear65i3FKQ9DNwH3As8mNZVojswGjjP3VcF/gsc3k6aF7p7q7u3DhgwoMKkRERERLqmugZvZrY98DjRXbo98JiZbVfhx00Dprn7Y2n5RiKYExEREZlr1Lvb9CfAGHd/D8DMBgD3EIFXp7j7O2b2lpkt5+6vABsBL9Y0tyIiIiJdXL2Dt5YscEs+oLrWvu8DV6eZpn8Fvl1N5kRERESKpt7B251m9gfgmrS8A3B7pR/m7k8DrbXImIiIiEgR1TV4c/dDzGxbYK206kJ3v7meaYqIiIjMyep+Y3p3vwm4qd7piIiIiMwN6hK8mdm/idtgfekp4goi89cjXREREZE5XV2CN3fvW4/PFREREZnb1fvG9CIiIiJSQwreRERERApEwZuIiIhIgdR9tqmIiIjUz5vHrtjp9ww+8rk65EQaRS1vIiIiIgWiljcREWG1QyZ1+j1TTtmtDjkRkdlRy5uIiIhIgSh4ExERESkQBW8iIiIiBaIxbyIinaTZfSLSTGp5ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEAVvIiIiIgWi4E1ERESkQBS8iYiIiBSIgjcRERGRAune7Ax0dW8eu2Kn3zP4yOfqkBMRERERtbyJiIiIFIpa3kSkwyppiQa1RouI1JJa3kREREQKRC1vc5nVDpnU6fdMOWW3OuREREREKqGWNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEM02FRGRiugONCLNoZY3ERERkQIpXPBmZt3M7Ckzu7XZeRERERFptMIFb8CBwEvNzoSIiIhIMxQqeDOzQcDXgYubnRcRERGRZihU8AacARwKzGh2RkRERESaoTDBm5lNAN5z9ymzed3eZjbZzCa///77DcqdiIiISGMUJngD1gK2MLOpwLXAhmZ2VfmL3P1Cd29199YBAwY0Oo8iIiIidVWY4M3dj3D3Qe4+FNgRuM/dv9XkbImIiIg0VGGCNxEREREp6B0W3P0B4IEmZ0NERESk4dTyJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRAFbyIiIiIFouBNREREpEAUvImIiIgUiII3ERERkQJR8CYiIiJSIAreRERERAqkkLfHksZ689gVO/2ewUc+V4eciIiIiFreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEAVvIiIiIgWi4E1ERESkQBS8iYiIiBSIgjcRERGRAlHwJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRAFbyIiIiIFouBNREREpEAUvImIiIgUiII3ERERkQJR8CYiIiJSIAreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAChO8mdmSZna/mb1oZi+Y2YHNzpOIiIhIo3VvdgY6YTrwI3d/0sz6AlPM7G53f7HZGRMRERFplMK0vLn73939yfT438BLwBLNzZWIiIhIYxUmeMszs6HAqsBjzc2JiIiISGMVLngzs/mAm4AfuPvH7Ty/t5lNNrPJ77//fuMzKCIiIlJHhQrezKwHEbhd7e6/ae817n6hu7e6e+uAAQMam0ERERGROitM8GZmBlwCvOTupzU7PyIiIiLNUJjgDVgL2BXY0MyeTn+bNztTIiIiIo1UmEuFuPvDgDU7HyIiIiLNVKSWNxEREZG5noI3ERERkQJR8CYiIiJSIAreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEAVvIiIiIgWi4E1ERESkQBS8iYiIiBSIgjcRERGRAlHwJiIiIlIgCt5ERERECkTBm4iIiEiBKHgTERERKRAFbyIiIiIFouBNREREpEAUvImIiIgUiII3ERERkQJR8CYiIiJSIAreRERERApEwZuIiIhIgSh4ExERESkQBW8iIiIiBaLgTURERKRAFLyJiIiIFIiCNxEREZECUfAmIiIiUiAK3kREREQKRMGbiIiISIEoeBMREREpEAVvIiIiIgWi4E1ERESkQAoVvJnZpmb2ipm9bmaHNzs/IiIiIo1WmODNzLoB5wCbASOAncxsRHNzJSIiItJYhQnegLHA6+7+V3f/H3AtsGWT8yQiIiLSUObuzc5Dh5jZdsCm7r5XWt4VWN3dJ5a9bm9g77S4HPBKlUkvDPyjys8ocvrKg/LQldJXHpSHrpaHZqevPMxZeRji7gNm96LuVSTQJbn7hcCFtfo8M5vs7q21+ryipa88KA9dKX3lQXnoanlodvrKw9yZhyJ1m74NLJlbHpTWiYiIiMw1ihS8PQEsY2bDzKwnsCPwuybnSURERKShCtNt6u7TzWwi8AegG3Cpu7/QgKRr1gVb0PRBecgoD81PH5SHjPIQmp2HZqcPykNmrslDYSYsiIiIiEixuk1FRERE5noK3kREREQKRMGbYGbW7Dx0BWY2X7PzAGBmq5jZCk3OQ9P2CTPrbWa9m5V+yoPpuBApMTPFC13IXP1jpFtuNV3+oGj0CcPMunkXGfjYzMLBzIYAx6fHy5vZIk3Mx77AZmbWlAlFZtaS7RNN2B8XAr4GLGtm3c1smUamn/IwGBib2wZN2S+bXT6l+LVb+boG56HpZXSzt0FXSDeVCTOakX5ZXpoes2T7Q7Mrd03fEM1iZubuX6TH65jZek3Kx5LEZU8wsx4NStPS/xZ3/8LM+pjZKWa2t5kt34g8lOWnG0BWODQyaMkKA3d/A1jfzJ4DzgOGNioP7eTjz8AiwLhG5iHj7jPMbH4zOxn4eiPSzJ0g/wksD/wKeBUY1agCO1cY9wV2NbOfm9kjQMMDSIBc+TSs0Wmn8tFT+TDAzL5pZvMCDTlh5cqEbBssmCu3GnrSzOVhTL5i0yi5bdGUSnbaD2aY2TJmdoaZbduMIKrZAWSujP4irWpqxWKuC96yA9/d3cwWM7M7gGNo8GVTcvl4CzjYzM4DHgfG1zndlYjgJDtJrw7cDrxH7A93mtni9cxDuVzhONbMjgQ2aWDaWcC4F3ErtaWA7dz9sUYWULl8HAHsAmwAbGRm89c77ex75v5vDpwMfOLut9Y57axWnxWIfYB/EwHcBe5+c6MK7NzJ8TVgRWBX4Ax3r/YWex1SHpSY2XgzmwLsZ2bzNCgPW0JpW5jZD4AHgPWAnwPfqHP6g1P6WZmwuZn9FTgfuCCft0Yxs5XN7G7iPPEzM9u9gWm35LbFj8zsUDPbsAHpWu5xNzPbDzgV+B/wI2AXa8Awk3w+0vlqoRRA7mlmg+qdfspD/yz9tLybmf0Z+KmZ7d+IPLRnrgne8rWX3A6xK/C4u2/o7vc2Mj9lBdDTKS97u/ud9UgvnSQN+BQYaGY7pqf6EcHc74nuqmeBT+qRh6/I25JmdhdwFLAHsI6ZzfbebhWmVX6C3MDMzgbWd/cdgGuBM9PTdatZmVlLO3nZFVgb2BK4EVga2KheecjykRVKuSBpPLAmcGv2mjqku5CZ9c3V6lcws2uJoPFy4MfAvGY2utZpl+Wje+5xi5ntDEwkTlD3Ah+aWa965iGlbfkywcxGAkcDP3L3Q9z9/zUgD32ASWa2SVpeAhjs7iOBW4AtgP71avkysx2Aw8xsRTPrZWbnANsRPRN7A+NSJaturW/25S7SHikPRwHfIipV69czmE4tneeZ2RLp2FjczG4kWoAfB35vZnVplc83bqTlpYF1gCOBy9z9UOAcYAywWj3ykMvLOsBBueX1gSuB6cBo4Ndmtmid8/BD4EBLw2jMbDywGVHBfhv4kZmNqGceZmWuCd5ytZdNgEMtxtb8negeOcnMTjWze8zs6+l1NS8c8i0cqTbznVQYnQA8BIxNz/esddrpJOnAZ8CTwPYpnZHAEUSt9jfuvoW7/7NRhWOyC/Csu3+dOGkvSB2CFmt/fN+BROD2rbT8A2BzMxvp7p+b2YJ1yEeLu89IFYl+uRPBcOCtdKI+E3iOOGEtVus8ZHKB01VmdphFS+zRROvTEmbWvdYtXyko+C4wf1peEZgE3AYc6u6fAn8kui/Xyr2v5sMK3H16brE30Is4Kb2Y8rA10RpbF/mTZSoXDkotCvMDjwCrmtkuZnaima1ndRhSYKXuoE+AE4ED0lMzgNXN7A/A4cA+7n4pULPApaw8eJGoOK7u7p8BiwNDgL+4+0cpXwenvNal9S13nsgmDLUAqxPDB35P3Onnu3UOpmcAXxBjXyH2yQeIls/xwOvAx/VIuKwCsT1RNr8K3EMELRAVy8+AsfUsm4jbYY6ymMDVSlTs/uHuB7v7ROANYNuU15qer3JlzX0pH6uk5THAncA3gX2AH7v7i7VMu6PmmuDNYrzCvcTJ+ghgA3e/EjiLqM3cCPwW+CHUp3DInQS7p0KiH7FT/JcoHH9oZou6+/9qnTaAmf2MuEPFEkQtbifgemA+4DvufnV63cXAzjVOOztJZYXjqlaaFNCXUivXDcBbwAZmNrSWeUjjd/padD9snU6ExxAtkUul13wMnA5camanAafWOmhIAVNvMzufCFguSE89B/zLzIank9dLwDZES1xN5As5CysT+/4U4EPgGuL3uINoZRhew7RHm9mh7v62u58M9LXool8ceBe4C1jLolvqY+KEsayZXW5mVwNV17It1+KZvn93M3vGzMakE/KfgGnA/sDVxLGxZqpofWs1m3ZUAAAgAElEQVTWn1yZXAvHvsApxAn7IOAx4E2ifJiHOIHsTB0CydwYxzOJ1t71zez7xPjDN4Cp7r6Ru99lMaFmgtVgNnCqKNyYy8dzxDGwspktR7R2tQALpwrP/cA7qUWmJtI+kJ8wtqaZPQZcadFV2EL0RmwBbJtaQaeb2dctdafVwb+J3pixFmOQhwK7ATcD/3H3ld39BUvdltUGLmXff8FUiRtJBG1ru/v/EWNQFzWztVPZdCcRyIyqJu128pIP5v9IBKrbuvtkokz6PO2DAFcQle2an6/d/fP0cAxxvtwmnQf+BlwK9ATGuPu1ZjbEooWyoebI4K28dSct/wi4JrXuXAhsYmbLuvvp7v4bopAaBfyhVlG8lXWNpRP2yUTLBsBlgAM7uPszRO3qSDP7tpmdVat007qFgJWAce6+L1GL2QH4D3AJcKKZXWpmT6Y83V5p+u3JnaRWM7NHiZal29IB8RzwsUVr1xdEwbU6sG4t82AxVuQhYnzjdsBJxIn6IuDYXF6PI7pP/wlMzB3IlabbUra8FhEg/gPYEFg+nSz/SgwI/2F66aJEa8S0Wu2Tud/hu8QJYQRwbToOLgIuJsYXXU60gG5gVXYR5fI+CBhhZhun/fEnwNfd/Q/EWLfziIDx28RJ/U4imP8cOM1jfGhV+ci1eA4E+qaWt1spdZX/heguHUNs/zOBFYhWn2nVpJ/loZ39YVviO19L7BcbEyfNM9199/S7XEQEcG/XIA/tlfsHEUH0vsBhwKHEb3I7sIiZ7WNmBxIn0EGpdbRarxOtu9vm1t1JlElbufuzROB0JLC2xXi8bsDzNUg7awHPuu1HpmByHPB9opt2E+K3+CMxHnZNMxthZrcRx07NK/hmtgXwMtH6u3DKxwMprZPc/YT0uh8Tx0/VgUtZy/qSwE+J/fAZIljayd0fBx4kWsxJw4zOdPe7q0k7Y7nJABYT6OZ397eJlq8lLLpMryZax9dOb3sCeNbMFqhB+uXny4XN7PfE/nA90fK6LVGpugWYlsqRbYheg7q1zs+Su88xf0C3suWlgHnS418DW6bHCxInh+8DCxBNw38BjqphXiz3eCiwcHq8GfA7YPm0vFXKy4pELf9kYudYrwbpjiJqBxDdMH8Hlk3LixE1iB/n8vhdYLX2Pqua34MISHoQJ6hbgK3T+luJbtLlgeOA36TtMCm97uQa/RZLpf97A5sC8xIB4q+IVo1FiRmem9fqu+e+d0s722NvomDcIS2PIbqylyJaPq4hgsw7gKFV5qGlbJ9YFTiECIp6ES1M9+ReO1/aFgsCexI12941+h0GEEHQGcRJeKf0GyxXlsd+6RjpP6t9u4o89CIC1IfTcfe1tP41YOf0eJm0DS7I3lOj75/fF+bLPT6d6IojbZd9cr/JKCKQfpJogahlHgbkHp8KbJNb/j1wanr8NaJyczWwXC22RS6d7YFHy9ZNIILmIUQw8SdirNP1wPgap9+DKPf+SpQ/HwMLpecOJ7oqlyECucvSfjGxBum2KRtyeTkj+47A5ilPaxLnjd8QrU33pbJhxUr3gbLjrQ9xftwh5eEAosv+yHT8n5xeN5Ios9es4fYfVbb8PaLH4XTgl2ndAcDZKW8T07FwJhFQH12DPHRrZ90gIljNbiE6keilW5YY3vSntC3uAzap5T7Z4Xw3I9GafwkYWba8CRGM3Q5MSutOJSYF9E/LJxM1qlHEyTtfkLVUkZf8QTEPUWN+Frg/FRJLEDWmM3OvezHtGP2BHtXugMBCRGD2aCrwvp/yciQxey573WXAC0Qtv813qHIb5E8Q3SkF0NsRXdRZ0LJ02jYrEifs49Nv9k1gDeDqarZDWh5DnKwHEYHiS6lgyp+ouhGtXZ1ObxZ5mL9seSkiGDkV2CutO41o7Zg/LZ9NDAiGCDCGVpmHecqWlySCsV8SNfs9cs+9TcywJW33K2uwDcpPTLsTMxVXIrqJtyOC6F8Q45jmI4K7s9LxsO9XfV4V+dgVuCg9/mHKy3pEC8v/Ea0d30n7zG7l+1ON9o+fEsH7D4gT57eAP+eeX5cov7ZK+dm1Bmnmy6XlieD9ZiJ47532i+Nyr/kGMfZq5fLtSNnJv4Ppt7sd07F3E3BEls+Unz8BG6V1hxAVuvk7k2YH9oUFiHLyVmCJtO4mooULoqy+nAgoeqR1FZXPX5GnRdPvnQUJjwF7psf9iFbQq9J2Woio8Hy9vd+1grRbU9rd0v/jiDJ4C2KoxuHEUIY7iXK8B6kRokbffXPi/DQqLW+Yjrt+xDl8BjFsY1UiqN2JOI9dQwx3qSh4nUVesoaTLYlzxSJpnxubnu9L9BD9OPdbLVf2GVVXLjvzV/huUzNbk2hS720xnukYooDewd03B9YzswnETrIesK/FAOlhxE472t3fcff3LSYRmFcwQDvX7Ou51dsRNeyViANjA6I78E6g1cx2tZhhOJU0s81TF90sujbaS7d8LNmixNiYKe6+BjGW6HvEgXIjMZ7lB2Z2OLHDHuruD+c/z0Ml22DplJeZU6qJFq7zzOwkd7+RGMc02MwWdvfXiaDmZHf/l7v/lBjj9W/ipPpIR7sL0ziNsR7N7t2tNAvpbaLm+h+i2+sZYnzfbyzG+ZxCFBC/cvddOvud28nHqsD3c+NRRhDd0pOIbsEL07r7gcGUBuQfDaxoZoPd/TN3n1ph+t3M7DjguDR+xSzGUx1BnEAPJoKj3hazCyGC+z3M7Hqiy/SR9FkVd9W2s/8MIIKQZ4kT1CbESfpOStuhF3FMbuju583m82bJzJYzs29n7zOz4VYa6L828EF6fDkRJIx393uI3+gyovw40d0neekSJp1mMRnl5qxbx8xWt5g16MQwjpWIMbi/BT41s2xm3VJEy+C3gY89xubOarLP7PIws1wys54WYxxPI1rRfk6cEL9LBNETzGwni0HoyxGtTAPS+7NjeuZkmw6mX14+DWxn/SnE/rdEKns+JYLXbOzvWUTX9QSrcMKGxWSl7Du0mtniHpMg7ib2wyXTS48GNjWz5T267R4jAuzsagUVD6Fop7v8J0SX6HeA08ysH1HB+3rK77+IiQutRGXmQ3e/xt1vy32n2f4O5fuNlYbvTCJ6nZ4jtvdVRGPGEUTr40nEvrkyMcTgc3f/R6Xfv528/JWovGfd5ssR5fPxxHjHb7p79pqniPNpL6KCvxip27qj58pcHtobtvBn4CNiP7vE3d8lxqKvbmb93P3fxFjsFUldpJ4uIWTNug5fIyPFWv0RtYAF0+NuRG1ts7T8S+KAWCYtb0UMuIVohbksPT+WGN9xdA3yk6+VbgD8JD3ej9T1kpZ/RIzdgQimbiFqekOqSTMtr0MEgAcQhc0ixOSEK4kZZBcRNZqViRrVr4nLAFRdayBOiH8kmrIHpnWrEl1/KxE12NeIpv81iMJ4q/S6nkRta4G0/DXgOlJ3b0f2hfT/6PT9exInpQty+8DFRKvTQCKIfpBo/XyKGP/Ypwb7QNYlOi/RMrF0Wl6fuNTBN4gm9kvT8/MAPyNOmkum11bVPQfslX6HKyi1MPcC3geuyr1uK2KixNK5df3S79O/wrQttw1a0v53MqlVnOhyOZc4Ua6Qnts/ve9kYiZjm+1ZyT6Z0p6QfuNxRIA2JW2TNYnj7rzcfrorcEV63IPUZVbl79CS2xYDKZVVmxKtCSul5ax7cD3iuLybOIn8njhR/Yqo8deiu3gnogxYk2jhX5toBb8sbauliGPvDErdVvNUm24u/c2AycTM+i99H6LcvpsYUnA/0TLYN7cd1ye1jnUizQWA7XPLA4hy78m0P5yX1l9Cmv2cln8B3Fur754dH7nHw9K2PjEtb5uO0YlEt9yv0367M1G52TnbhypI99D0W+5LjHeG6I6+k9R1n47LrLVxZSKY+wewaLY/12gbLAPcX7ZuC6IFeGliXNkXwO6551dN++ZAYpIhRNl5HtHwUs3vsAnRWNCatsmotF1mEL0/Q4nzyO1EJe/8zu6D9fxregYq2Pi9UsGza1oeQxSC1xJdH0sSzarjKZ3Y/wickx73TD/+zkT30QY1ytcI4vo3F6UffzHihH0SpfFtw4mWqHnT8gK593e4K6L8YCKavP9bttNvSer+Sjvmh0Q3Ue+v+qwKvvf3iSBox7TcJ/3fhrZdw1sQrYEQNbuzqLJrsCwfSxO1x42J8X1HEU3+ixHdU9n+0pc4iR4CtNYo7R65xyumz/91KhQ2J1oSrwTWyLYRMaZsTPpNKiqYy/IwMO13o3LrssBpZ2Jwb77guowYx1T1CZq2Xfbz5x4fTwSr3yICtttzz32TGMOzTP44qHSfJALAbYiKXW8iMP4z8P30/FFEMLQfMXzgkrT+FOCUGu6HK1Aa0zk/EZy+Q2ls3bXAubl98YdES1g2ziobn/lLUlDZyfTLx1kuQpzorqA03nXetD9mebqNmLSSvWeJ3ONKfovy8mls+i3W/6rXEuX4SVTRLVi2j3+dqAhm3a9bAJenxwsSQcrO6Ti9DFgrPTcA2K2S9GeVl7S8HBEMZAPv50375D1EV/p1xOziBYgW2evz26wzeUnH131EEPzN9P3eIoKzlYlWtiHZ700E7FmFc9t0vCxQzfefRb6eJi4FA+kSUcT5+ZS07kmiUrc4UeGYQhqLWvY53SvdJ4nu5x8RjRvrpGNmw5TW6LTfvFO2LTeudJ+s11/TM9CJjd8/93h1ovXsb0QLSjZL6hwiODuQKJSzoGkEcTJtScs/JLoQl6owL+XjqnoStcbDiWj9RqJVrYUIUi4mTuqnpr8eZe/vUAEJXxpkeiSlwv4JSgOMjWh1+Fs6UH+eDuIxlaQ7q7yk730u7QxiJgLKB/MHGVF7GZYOkPGV5iXbDikPfYnC8HqihvR/lIL2s4lC8XFygWSN9sfFgZ/llvsSwdBNROvrYZRq1s8A38q97lrgsDocIxcTXQ2k/f1uSgX0o+Rat4gT1o1U0dJECkZzy4cTMwFPyH3fcUQ37DeIbpJV0vrBRIGZ36crKhSJlqQXiZbeX6V1qxBB0+659A4iAvnF0/5yF1HRW6CSdMvy0DP970W0Mp9JjCddiBjT94f0/HLECWvVtLx+em02sWg00fJ0Jp0/QbUJooF10uO7iUC5d1pemegO7EWcuC9L++SQsmOrw+USUdZtQXSvQQTRWUvWTkTr/w5EgH0UubJ3VulQwXhDYPHc40WJyR+npfwcR66nhWjxfCY9Pp9ova9qXN2svlPa597I9s/cvvCH3PIbRJm1aDvbtzOB20iiIrdR2frjiIBuEBGwr5P7va8jhivQmbQq2C7D0nF5C9HCvAoxBvNcotxclmgpvC09v3qV6eWPiez8/z2iTN4l99y3gOPT43Fp+x0/u9+1mX9Nz0AHf4D5iS6hDYmg5WDS7KDca1qJAu9bxAnyeqIJ/kstC3SyUPyKfGUF1ZB8XtK694hxPH2Jk9otRA143hqk+yOiqf894Na0bj3ixJl1B81LtC49SnSF9KzRdx5IrpuXqFGfRBTKv0jf8dr0m2VXzF+O6Dq8lSoG/Kbvs1LZulbgjtzyw5S6ABYmTo4fELWqWhXMPyMCwuPS8tBU+N1OaWDzukSX7Dopj1kX+VPAsXU6TuYlLqvxLNFdki+4NiJq11V3C6bPWyvtWyuk5SOJArg/0bL6CjA8Pbcx0Qr3IbBuHb53LyIQ2y5t4xOJVu79gRtyrzuONKOcKEdq0UXajaipZ60WS6R94f2y42QK8O30+Cfkuo9oO/N0ALnJUxXkx4jy5ibi5Nc/lQ13AcNyr7uWqNC9S27yShXpDiBasjZJv8MbaTv8nCgDTyXGWH2faGk6l1l00VN5EL848K+ydfsSQf1aRAvgtNxzi1Ga0DYqv31quG9+F5iQHp9HmllLaQLCq8Qwhj2JlqAtaFuh6VSwQCkYu4NS+dQn9/ybRJftNkTF5Wfpt/lzLY6HDubxLODG/P6fttPFpOCbtq2/nQpeZ5HmuLTP706UkxcSFfte6fmJRK/ZmcQ4v12Afo3YHhV/p2ZnYDYbPIuUjRjU+SYRoIwhaqg/J24pBVGz2omoRS6RDoL1yj6vJjPHiCDyfqLpORvf9jK5WZtEy8eU3PIC5d+rA+m0N518U6KVbQhxLaD3KU0tP5s0jiP3+gU7m+5s8jSBaL3YkQiiV0sHxYvEDKAjiRadm4lxVEenguROyma1djLdscRtUW4kjZdL678NnJ1bHkFcn21obt14YJEafPdeRLfHNMrGyRGtOn+gNHOqL1F5OJdSl9gapOC6jsfMt4GbZvHczeS61iv47BWJbseRRJfKT0ktmumYW4AogO8kTgzlFZr7KM2kq9U4mqyMOJGoRPQlTkiXEIHEVUTg2IMIZg6u4bbOTpS7EMHRHSmNbHhEvvtvAhG4z0t02f2aqAhZ/ntU8t1zy/3Tb3wOUYl9AjggPXcJEVRn45zmI8ajLjKrz+vI96dtd9T+RAXtorSvDCRaIbcue99GREtgTSrRZZ99EaUg+UJinN2F6bg1oqy6lKjUTSLN8K5DPrZKn/8YqTeAqEw+SWl8dneii/J+Khz73E662RjBwcRwjSFpOWt1PZC43ilEpfLnRI9B1Y0KncjjvMR5q3du3SpEb9maZa+t6pydjsWniPLo3nR8zk8MKzqTFCMQZdf66TgZk3t/l2lp+9J3a3YGZrHBv7TBiKDgeeCYtNyLOFFdQOnkuCZRu964Rvnolv+fHi9OdP9tRQQKf0yF1oSUv0WJpuHjiFrVrrn3dqYros31mCgV8nuSux4dUbPMmv6XSgVFeetU1TWX3GcNJ8ZOvE7pxJC1QPbMvW4qsFh6XPUgz/QdniPGAl2Q2w9GES1Ni+Ve+0ZWQNV6vyQGXl9H1JxHEieCVYmxllcShXFWmxtNnEx2qnVeZpPHaZRaglqJQGHmGNAKPrMPEYROTtv+ubQdxhIn4Y3T63YCLk6PxxEF9Ka5z/kucGSdvve2pOtvpTx+TASRJxEnsVuoUYtn+XYkWnX+TttJITsQJ438eMh7yF2Oo9rfOfc467IdCjyRW78jUaFbkeiOepx2rtFFhZNDcu/vl3t8Q9ruWZfpFsBz6fEqxNi7p4Fv1Gk/mJeYpfoscHhatwjRdbobMd75e0SQe1SN0iwfRjM4HSubEgHb09m+R/Qg3Vf2+jaXqerob8EszieUzlunU+quz3oETqVt13FTghOiYntp2bqqridZ/jukdbtTOle0EmMMf5a283Hp8ZdagKnhObNu27DZGZjNj7EFcfLbgagpjk6FcDZuZlmi5vBbosa3GbkuiCrTXiY7yGjbjL1xfqcjmt7fJcZ/nUiMv3qTuCfiYeT61SvZGYnxQ28QXZKrpgIh3004H/D/KNU2qxojUJZ+ec1+L6LGchxxgmw3KCO6ha6ibPZkewdXR7dD+n8MEbwOJk6E38utv5YonE8hWh62q8H3nyf3OAue+xMtOa8TXQ175F7zPaKLOts/u1GHrpgO5HscUcM/jmh9+U4Vn2WpAPwX6SRDqYu0H1FxyWryx5HGARI1/N8AP0jLI4jW6S3r9J23JrqFnycCiIXT7/RnchMCqkyjfLLPnkRtfXmiNf63ROUmC97vI64LdVjaHvOV7VOdbekqDxL6E61KZxGB9BLpuNw8Pd+PqGhmYy8PrbZ8aCcPh6Vj4SfEWKo10rE5PPeaP6VtNIayWcV12hfatD4Tra7fTGXE6LSuJhdezu8b6fc1ojfiUkpjb5cnZm8OSctPUHYdw0r2h9z7liHNJE3L+fPVu5Ral4YTl2basN6/QQfy3JLKlKXK1lcyw7x8Ysh2lGbw/zxXPs1DxAgPEBWd9YhzRfn12rpsa1ubfDY7A2ljLU3UCFbL/bCHEWOIVk07+wnpdUdTGpS8ENHicQxlTfM1ytczlLoks5rLMkQtO9/kex2pRpsKzO5Eof5U/qDqZNobE7XUw1OaBwA3p+fuJLpAliQC2zuA5+v4+2Tf/Xqiu7aFGL9xAKU7RyxJ1GKeIALYioIWYixGNpi7/MKgp+V+j98TQfLFRJC0BdFN8Utq0D1OtHZeQ2o1o23r62giKBiff44ovG8guhZrcleCKvJ/P1HLrPoklX7zc4AD0/LSRHfQPEQr86/T9l8l/QavEt1AC+c+YyHitkr1/M7PkAL6tNyP2rT6bkQEYhOIFv/hxNjKq9IxcFd63UXAT3PvG050zdxE7oKiVNZFunp2/KflTVM5cHgqC24iKg9HEN1gWW/EtSlfIzubZjt5yAcF2azqo4mg7ARKV8Q/l7jUxRpEa+wDfPmi0TW/+HF++xKtzyvl1g0jWnuG1ODz1yLXmk5cbuatdBwcQ7T0TaZtj8nTpEo/UZGpxeWJWojz5kvEWMI2F0dP/79LKbh+FjikXtu9gvxXNYSELwdtGxKTcx5O/1uJ3pHfUQra1yIm0mUVmqp/h6Ztvyb/eC1E7fgZ4sS/XO65g1OBtQMxbiI7ia6cCq1biRNl3a67QkTnr+WWswPi8pT2gsRlS+6lNFGgLzGw/hHS1Zk7kE7+NlJLEy1cyxAXDdw+PTeQ6Jbbg2h5OpEIlK6ldEulwTX4zvm8ZNci+yVRq59CCsqI2YOXEbXMpVOefkC6/ED2+3Yi3ayL9ReUXSIhVwDuQ8xUejwdkIOIwcjnpt+iFkFbvgDcgwjMe+afS9tlv/R7ZOuyfWMsNbwKebW/Y60+i6ix3kwUkFcQLZwtRIvzNsR4vwXTa1fNb08a0P1AVJjOIHXT1mhf6E0ErY8Srbq9UzrfIAIXIyoqDxPjaFYhgrxdidlyK9M26K+ma7J/2u+3SvvflaThEun5b5PuFUuc0O8jytXsTjKbVLJdyvOcjvXfEZWXV4D90vo1iC7arYihI89TmgW+fhP2/3GU3Xarhp99KFEx+no6Di4jeoEGE8NFVkvrLyGCh7XSa96h7TmuMzNI2+seXTul8aXeJtoG2k8RFY2qL0nUFf/SsTeeuLpCNmP2RKJisSoR2D5NTNS4LZUTF5Mqk40on+ryvZu80VciaozlU6MXIwKkV4haY3bB0exCrmOI6eb5wfh1+QHSwXFwepyfFXYBpQs+7lj2nk4PjqdUU16RCEyNCEqya0K1EFOpH6EUQA0kWnvOIgbyVzwAmLZBSx9KY1ayaxNdQTT9r5t73eEp3X9Sdr289gqbr0h7HaK1Jgteb6B0Ed98vpYiAtbdcuuGUptWhfLLtyxPtP7eTTuD7FM+ryB3T8p67H+N/Puq70B0Q55OXE+w/NZVSxCtojUdbFxB/n9Vvh9W+XlLAbfllrNA/WiiteNRcpeLSc9tlo7b48vWd7aLdAjp1kW5dTsCj6THWSvDeml5JNGisETu+eWI4PoW0jXMqtwe6xETMw5Ly2cB16fHvYkA9zKixfN75C5VQRNOkERZuVINPmcx0qzq3H5xPFFJ/zNRgc6u3bk7ETh3Jyq+d6TXjEr750adSNdS2idQOj+MIwKS3kSl6TqifP5ZSndzSt32WYWyqb0ANf5NyysT3yAq0tnFhbPrOo5I22WXtLwPEbBtTFSwL2v2d6l6WzT5h9iH0s2fRxC12Y2JmmV2Iddsp90sFUIDyz6jrv3TxADYDyldfLY7UbP+UgtLpQVUKnjfJgZdL0sMsh5KNL//lVKT7/zpuax1YUmi1n90Fd9v6bLl7MbAZ5AubJzWr0BcmuQuoja/cS7v8+ZeV8mYhW5EDT27v+FeRFBf3uK1QlqfzZyq1SVfdiG6F7KZeNsS3S4HEeOY7iC1alJqmeyR9oPrqfH4mWb/0XYAdf46fWOImutyablX7rmmB6+1zgMRlN5HDIHYhKjBH0qcoF+jbQvjfpRq/bVo9duDdAu1/G9BtH4elJZ/SrpPa1p+gNK1LXukvD4J/LCTaZePdd2TmAw2MB0Lp6f1fYhKV9aql13LsjyIb9ag+FpdXeBAYHLZut1I96hOZcDQ3HNPUJow1J+oeE8kei4WqyD9h4gu3yOJC5FfmfbLnsRFhvcjWveOJRo7GjZztIG/Zfk+uV7a/3YlXW0gHTM3UZpAtw9Rwc6G4fQhKkAvUBqLW8hWN/cmBW+UusEGEbXXR4hmzRvSznk90WV6GtEdcwPtt3A1ZMOnneBsYuzLX9IOkm/161Ah0c4OOJo0XZ/oZjiT0q2LsquCHwM8NKu0qHCCBlGr24WonWQX+m3vxsDZVfrXT7/LfERhfh1tx/11uKBsrzAnguHniNm8/YiW1wPLX0+M36j4kiNlaWaB2I5EC2t2E+LTKV0lf2miln1yO++fnxpdP68Zf+XHD9H19SAx0WDmhJPc8TofMcZrVpciKWxBOIvv05M4ab5BdD2dSgRUFxGVrVuJk/hdxCD9IfltUe32IFr2f0DbIHkcEQQsSLQ2PEN0Yz5KVKryLXWbUcUdNHK/+yGkWazEYPALKJ0Q9yU33raa9LrqH1GB/wNpUlha158omycSY7PPI7pL9yKG9WTHzvzE5IWr6OBwivLykeiefZhcaxERvB1R9rqsslnYcVwd3D47ED0AxxKB8WvEmMb5iYstZy3D/dPxmwVz44gJDMs0+zvUZDs0aGO3N4U3O3EOJWaKLUspUHiWNFuQGHRY9czBKvOfzYx5nk40e5d/Ru5xj/SZBxDj5boTwcOpxKDoV2l72YFryc2Wq+akkAr9rEt09VTg75uW90uF0dnEDLFtcu9bPeWxZtdmIgLCTSnNDDo1K6DSdvhd7rmsFa5vHX7ftYju36zl72RKg4t7ESfoP1IK7goxG+mr9kWixTS7iG4vIlC5M33XHkQAl937Mb/vjiTGgVU9vrIof0QXeh9K9yjdk5hZuxsxtqaqS8Fk5UFuOSsb1yKCwuxeqJZeewFR4eiWjtlLSJO9yn+vTuQh62LrRgTpx9F2fNZDxLU2exLdeEflnruW3GSUasqnrvpHdM9NoW0gvaunXF4AABZPSURBVCUxHniD9NwlpIaHsvd2aKxZ+XYjWpeyXobT0vGZ3W90JNE7MC/RW/UwMe656utZdpU/2o6/no8YrrB02kcnEcOqxqd9NWuN3pSoyHxpwtyctl+2UGdmZu7+RXq8UPnz7j7V3W9291fd/YW0+mHiB8PdJ7v7jen93eqd3/a4+wyidWqUu99roUPbzsyy7zHDzAaa2TnEyW8DYmzMU0Rt4E7AiVbI94GxZjYqvXdHd/8wlx+v5HuY2VLE3QZOM7Pe7v4Y0eI5ysyGE7M3zyBq2Wu5+2/MbGUzG0J00d7t7tNzn9fRbdC3bHkeMzuTUkvj78xsZEp7RTNby91vJQag/jR95/+l//+u5Lu3kydL+biYqMH9E5hoZusTJ6MBZraOu39GXDfKiO7UbH8osmzG5PfN7DiiO3AlYkzjHe7+OdE1uLmZrZD23ey3fsnd93f3N5uS8yZw95fd/RN3/2datRHwrrtPcvcj3P0aqKx8Svv9Rln5kNL7Iv3/E9HFs7OZLejhc6Ky91Z63aPAJ8TgdcyspaP7Z9r/R6e0pptZD3f/wt3/Q4zR2s7M5kkvP5JogXNirN1KZva19N4d3X1abntVVD51cbcSFyI/Klvh7rcQwcOTROvbp8R9OB/Ll425/WaW0u/m6fFCZnYFMX7wDDNbh+gN6ENMiiCdKydTukvD99x9J3d/txZftitw9y/MrDcx1v0/afVewPZES+Z5ROvyikSPDcQs+APd/W/Z5+TOwXPWflmPiJAv1yDGEjv/JUQBMF/+dURhtHTudfdRwdiARvzRwZYnoraev0DngsR4kSOJptzriFauFiKYu5o0y5LosvxufjtSuyvS/4kIDn9GjG+bh+gWzC5o+TRRm89uDPyl7upOpJVdp+4RYvZwdv2zpYhAMHvdCUQXxMIpT/ek9a3AtjX63icTLbz5W8UMB/6YWz4t5WM40VX+AtEM/zQVtrh21b/0/T4ldckTs6QfB0bkXnMR7VyPqpb7YxH+Uvk0LB2vjxFd+vlxgZ2+eTpxaZVsyMSDxNic10ljpyiN8xxEDBs5hhgXfARtx7Z1J1rhNulkHiYSvQlvUWpdfiaVBYukY/SBdAy2pP3jOeAX6f1fo+1Fuef4/YHoHXqLGP/XkziX3ZCeW4zoMRpP54aQ5Mv4/kRwsg+lOwftT7S09k1l6H3p+QuJ1tA5bnxbbnvslY6J64neoD5Ej8xDROC2JjGz+bfETN7CDmGpaPs04AdYlBjDtS7R9fYy7Vw0lAhW/gjs1eyNUuX33YqoEf2GaEo/kAiQlqft7bLWSwfl0mkbnUqMMav6Jtmzyd8gYrzO8FTY7E+0bh1FjMEbQQ1uDJwKsXfS525CBKhnpUJvRDogs1tJLUsElUPSieQxajA7LpeX7MblDwJn5daPKsvHCum3yy7PskoqKOt6S6s6/97zEZdfWT0tZ91jw4iA9jRKs7mPIbqP1yTGU91Pbpbd3PxHBFoXkrvsBZVPUJonHf/difGdd5C7F2z55xOV2kPTMXkBZd1wnckHMa71EWLy1wrEJS3uT/vC+kRl5YSUtx+n9JZM+T2XGIdX89taFeWPGOoyiWjxvJy23cWblf82X/E55WOXt07l0yXEXXKyIRwj0/G7H9FlPpmoVB1IFfeJ7kp/RGywczrGsgmKo9N+OSyVYY+Q7lZEnLM+JN0fOO2ryzf7ezR8u9XwByhvbduTaMpflQgGtiSCmWNpO6g269fu2976ovwRAdhNRBCyetqhtiYGuh5IBE03kS5lQMxky1880MjdU629bVrDvF5OzNLpl/L2BqXr32StolXdGDhtgxm55c1JgRNRq7+A3IWViRbXtdPjmgawtL1x+Q3ECas/cRHNS8ry8TAR0C1Vyzw0aZ/MxoocBVyYW5+16gxP2+Kw3HOHEd01j5Eug6K/drdrJfcBzZdvfdJJaB+iK+x2osKTjXH60vFG23GvnU0/a8F7mWi9WS+tH0kMpdghLa9DBPTbpzyeRLTAPU9uXN3c/JfK9qXzy1V+3uZED0d+7PGDuee3TGXSskQgv2g16XWVP2K8+6T03c8mGm/uTs+tk57LzkdfJwVrafkEogKyWK48m+Nbf/N/NRvz5mnr5QxOO9p7RJP8XsRtW4706MtePb0vG9/x77S+Jb++QJYnxh/8yGMs2RfufjMxrXt5Yme8h9gWuPvbRFCRjeVyd38i/4HtbNNa2Z8I1D5z9zOJGvZCRPfLoFz+MLNuKW+dykvaBjea2aVp1cfA8DR25y9EMLWDmZ2Wxp3NQ7SO4e4fVf0NkzSW5DOi4tBKXMz0Y2Kg8XSiANjWzM40s3OJLuUzUh4LK401dY+xIq8AS5rZptnT6f/fiK7Spc1sDTPLbhg+kbjcw0UNz3gXl41N8k6Oe0zv+cLM+qXlT4ibYS9DjDW8igjk5s9en9Kz3Gd8mI237Wz6RFn8JBGs3QRsbGbDPMZOXUqpXHqICPDWJrpwDyduaTXK3aekPDVl7HFX4e7T3f11mLk/TJ/de7LXli2fbmb7ufvtRPm0cvr8c4HuZrZveunTRMD9hbs/7u7v1Oq7NIuZDSUug3W/u49294nEWNK+ZnYsMaSjN1HJxt1vA/qZ2RrpI05y9y3d/e/ZsVDBMVFoVQVv+Z3RzPqb2RG5p/9HBAdvE4XDe+7+rpmNNLPbgHXMrGf5Zxb1B3D3B4jWio3NbEgu2PkdUYPtQxSa/czsJjP7C3E9oJeakNf/EmNnzkvLVxMHzqru/nLZa6sJor9HBEZnk8YJATeZ2c/c/SZi3N17wMvuvqHnJmXUSm5/mgxMS8HMEqSbqBNXRD+aaH142923dvdHap2PRjCzVc3sa2bW3d3dzMaa2StEt/X8wB5m1jMFEd3SPvowceK4jghs/+Hu/81e07xv0zV1pnxq52T9Q+AFM/uFmY0j9r9exGzVX6eX7WJm55vZYSm9NpWmSgLH9L7fuPuJ6Xh+itgf1klP/xxYyMy2ScuPAl9QGhz/Ssp/t7RctIp13XTmt/CYlNLdzHqkVXcDB6Tz4FVErL5Zeu5I4Odm1s/d33D384teocxz96n/v717j7GrquI4/v3RmvQBimKCgii1ClYESYBAFaWAVCNKiPFRRWq1ykNQNKXFFkV8axQfgGAqItYWwTT4AKPF8BKxNlWg0AI+UAqGIkWhothKZfnH2mfmzLS0wzh37pwzv0/SdO5jzpyZe+856+y91tpk2s5tAJImliB4Ntl+Zi2l16akV0t6GzmdvKZ8/6Pl+0bvMWogw3P9/9F3+L/KodmFPAnMJw9IRwArymMTySHRxeRVxGmD+bkj5R9PXtq/LznSdgyZrF8N534RWFq+HkfmWnV1Wo5+CwPTm18z1I1OTyevoqrtH0wmPl8A7DyMv+/WFi7/fPn6pcO1Hx3+HT9C5mVNKbfPoHcqZhoZLGzRO6/cftFw7edo+Ad9EtGnki0M5pGrHpxIXshNIBdMP49MJZhEnrQXAbt1eJ9OKcelqg3J8fRdbquVSyl14X1QP1c+m0xPeWXteHgZvUUgc8kp62qqsGrN0qoWF7W/x57AH2u3qzW0l5bPwdPJQO7HZP71Id3e55H07ymNvJWyXaK39ceHyTYPJwCbyGT9Z5WTxP3A7ZIOjBzpmV7enK+InKobcKuJkWQ7pf23kxVbh5EJlNVV2b/JqywiYmNErI6IuyXtUJ8WGU5l3/aKcjUX5VMTQ3xVHRFfAv5KnryInE49lsx/e2Qof9Z29uMH5Gjw+RHxloh4iEwEPi0i7hiu/eiwxeSF09Ry++XkARLywuoG4ChJu0Xf9h9EbRpo+Ha3vSIiJO0laQmZ83seGRD9jpyqX0eOTP+QHOWaSR4zPxMRMyPi/qE+NpR9qra5rPz/qtIi5LvAKklTypT7w9B32tYGRtJ0Se+FnnYXL5f09HLMuZ0c9d+5PH0BOTuxB5n3uCtZMEREfCsi/lMdm9smcvTtBkmnl7uqEcl1wOMR8Y+IuJAscHxjRPza78deA+3TdaSka4EjJY2T9ExJC8n8idPJ/lfzyYXU55H9whaTCZYPAUTEpsj56cdqw++NmCIteSbHSKp6YS0ofXh+Vebu6we5ReQo5OTSr+eTZOByY//tRsQT3fxgRsSDw/SjziBPWNXPvTsihnW6WNJYsnrpnnJ7TET8PSLuH8796BRJJ5FNOp8HzJG0Nzkt/jJJu0dOM2wig7nTYOufv6Z8JkeSrZ1QlD0aF5MjC4eQqSO7lWmwB8gR39eSFdaXAT8rx8jqwniHThwbahdpfyRz4A4gR4IoQeOd9Z/b1sChEyS9QNKl5OzLp8t035fJAP0bkl5BNpSdQvmbkxe2D5MNsdcACyLi6i7sfrd8kDyfjo+Ix8p9u1I7X0bEeujNv+7CPo5I2wzeSqD2dbKy4xJyNYDHyYTvb5LLccwmcyP2IDvyR0ScSb5hn0PmevUx1KM7w2A8GZjeQZ4A/0UuJTS9XD30XNWWgOhq8gS5ghwqf12LRneesoi4DDhd0pgujjRuJt/vm8rtpr0HAZB0uKRJ/e6bDLyz/DuK7I30evIKdjWwSNLRZE7buWTbFhsCko6gNG8ut/eVNJF8De4mUyQgG1LvSlbNQaaPrCCnuH8dEcvr2+1kEF37DF5NjsD9pvaYR16fonJc+zRZ/PPniNiVnPa7kMznnUR+Dt9CphAtBN4t6RjyHHolWW1PRKztwq/QNdGbf/0lSYdK+i15rr15K89t5DG7U7StQLacFM6NiKPLbdUjX0kfI+epz5J0Lrlw8eciYpWyg//GDu9/x5QD3A61K+HxZPuTzWRfpm+QB+QbImJj/W9TDoCnkOXeq8p9Y/zm666mvwbKFUpWk4HBooi4qNy/N1k9fEJE3KdcJWIeOQV3k6RTyKmYK0sgvcVn2QZH0sfJdiwrySmw9WRe6/HkxetcMkXgekkzyKnS90TEn8p05eNd2nUbAqXA4MtkRfdDwA8j4qqSUnMLmc/2tTIS+06yaOr88l6YQQ4EnBRDtHJME5Xz5d/JY9v8yIpn247tXWVtBMZLmiZpOvB+SR+X9PoS3BxGLs0BJUGfPGBRBW5NrQYpI4iDKe0fW6ZDzytBrMoUSGODhrZowWvwX/KK9HvAbEmzyv2byc7vL4SeyufdgZMl7RERX4+I4xy4dcTl5LHvOODMiDiKPAm9m3y9rgNmQc8I9G/L/VSBW7fzeLr98xtuHTAzIo4FlgOHS9q/zMCcRQZoRMRqMpjbX9KR5b3wjvK5HLWBG/RZfvLQiLixOmd2e79Guu39gdaT+RjfITtwv5AsL59NHpzOAeZLuok8WcyLiGX1DTTphNn/DaPBlfbX1/6sem05j8j+L+W9tIHMj9mRXI/0UEnzI4tO7gRmSppRpkgfJJurPlDbRtVD0YHbIKhva6Qxkk4l2xlcR1aLVikiXyCrSseSo/S7S3o7QETM6T811u3Xo9s/v8ki4tbo7c95I7n2a1UE8i1gvaS55fFqrdq/lu99bIsNjlIR8Tfo01fU58ztGLutByMXA18o6Rdk35VxEfGwspJmn4i4WNJGck2xnqCtiVf2ZZ+fKF9PJUfZnka2PJlGFmYcTy5h8uoypTyn3DeBTFLto2l/A2uEK8g8qd9IWkX2gppA5tf8nmyGPYG8kLql/o0+IA5OlQLS7++3EzkVvQu5asxhZCPknSPiHkn/IVsbfEXSAko/q7K9wTTZtREuIn5fcrYOJgsSrifbsXxf0qKI+AtZeWxPokmDPd02oKHJiLgrIv4dpXycXIvsvvLYdVXgpt4q0sYFLREjr7TfrFL7TE0kp14uJ9+PHyD7C36WnK47LiJeGxG3lOkHvycHSX2r7MdLmizpC+XhDeRxYho5I/EjskXLGZIOIFdVWQUQESsjYlP1Wjhwa5/a52wZ+d6YKmmXkr/1MWCTP4s2lAbaKmSspEmSTpW0glxAfUn/5zUlat7ah0gjtLTfrJ+rgNeQK5bsExGXkEnx343sxP4YDH5ZM9t6lX3JeV1HVgkeWf6ua8j+jXPLSXoNGczNA86JiGvr2/Vr0V7l4l+Rq8QsJ6fNq+WuLoqIR/z621Da5rRpJSI2S9oJ2I9cxPp6aOz06BFkI+Gl5fa+5BprWyvtv4Is7V9C39L+K/tv11fTNkw2kAHFT6EnSPsD8If6k5pyITVC7U6u63kI9F7sRfao/Cg5TXpNuX0XMKMcV5YAv4yIa6oNNfEYaYNTe51/DtwfEbdt6/lm/48BBW8A5Y14AvQczNTQgOVVwI4l+bh/af/XgLmSpkWW9l8CnCRpeWRp/6fCpf3WfZOBcSUwcJA29Hqq7MnliV6sbP3wq4hYKOmtkj4UEV8l2yOtIXOAryUvAnvy2hy4jT7lM+nAzTpqm33etvoNDU+2lfQSMgidDCyMiJ9IupCszruKTDY9MCJmleefQ/a6W1vbhq+mrWskPatMz1gHKBcKnwWcSfafupacAnuUzG1bAXyCzGtbB7wvIu7tys6a2aj0lIO3JqkHmqWY4mRyyulw4HPA2RGxVLnE1efJkbcHyQa8F0fE97qw22YD4ouIzioXev2r7KdExBxJzyB7U91ce36jL2zNrDla2QhP0jjYIg+tKu2fQ46w/YxaaT+5aPkhkT2zFpD5btX2Wvl3smZz4NZZT1Jlv7Y8tqEK3NSwtZrNrPlaFZS4tN/MhsqTVNlvMRrvvEMzG26tmDYtI23nAAcAFwDfj1xvdAJwD7kiwjXl9oeA50fEScp1CV8H3AtcERGXd+c3MLORSNJ+wKnApU2usjezdmlL8DaZLCo4utyuLxJ/AvCuiHhluf0m4GwyiLsXeIFL+81sexpeZW9mLTLgViEjnEv7zaxjasUIPj6YWde1ZeTNpf1mZmY2KrQieKu4tN/MzMzarlXVpi7tNzMzs7ZrVfDm0n4zMzNru7YULAAQEZsl7QTsB5zh0n4zMzNrm1blvPXn0n4zMzNrm1aNvNW5tN/MzMzaqNUjb2ZmZmZt06qCBTMzM7O2c/BmZmZm1iAO3szMzMwaxMGbmZmZWYM4eDMzMzNrEAdvZmaApD0l3SVpiaQ7JS2VNEHSWZJWSlotaWHpH4mkgyTdJulWSV+UtLrcP6bcXlkeP7G7v5mZtY2DNzOzXnsDF0TEFOAfwPuB8yPioIh4GTAeeEN57reBEyNif6C+5N5sYENEHAQcBLxP0qRh+w3MrPUcvJmZ9bovIm4qXy8GDgUOl7RC0u3AEcA+knYGdoqI5eW5l9a2MR2YKelWYAWwC/Di4dl9MxsNWrvCgpnZIPTvWh7ABcCBEXGfpLOBcdvZhoAPRMSyDuyfmZlH3szMap4vaWr5+h3AL8vXD0naEXgzQEQ8Ajwq6eDy+IzaNpYBJ0t6GoCkvSRN7Pyum9lo4ZE3M7NevwNOkXQxcAdwIfBMYDXwALCy9tzZwDclPQHcAGwo918E7AncXIob1gPHDsvem9mo4LVNzczIalPgqlKYMJDn7xgR/yxffwR4bkSc1rk9NDNLHnkzMxucoyXNJ4+ja4FZ3d0dMxstPPJmZmZm1iAuWDAzMzNrEAdvZmZmZg3i4M3MzMysQRy8mZmZmTWIgzczMzOzBnHwZmZmZtYg/wOnvK99RL8I6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "sns.barplot(x='page',\n",
    "            y='log_count',\n",
    "            hue='churned',\n",
    "            data=df_page)\n",
    "plt.xticks(rotation=30, ha=\"right\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visually the difference between churned and non churned users is less for 'Roll Advert' page which is understandable. It seems that free users are majority of churned users."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering\n",
    "Once you've familiarized yourself with the data, build out the features you find promising to train your model on. To work with the full dataset, you can follow the following steps.\n",
    "- Write a script to extract the necessary features from the smaller subset of data\n",
    "- Ensure that your script is scalable, using the best practices discussed in Lesson 3\n",
    "- Try your script on the full data set, debugging your script if necessary\n",
    "\n",
    "If you are working in the classroom workspace, you can just extract features based on the small subset of data contained here. Be sure to transfer over this work to the larger dataset when you work on your Spark cluster."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: We will append all the relevant columns corresponding to each user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = df.select(\"userId\").distinct()\n",
    "col_names = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Session counts for each user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------------+\n",
      "|userId|session_count|\n",
      "+------+-------------+\n",
      "|200002|            5|\n",
      "|   296|            5|\n",
      "|100010|            2|\n",
      "|   125|            3|\n",
      "|     7|           15|\n",
      "|   124|           17|\n",
      "|    51|            5|\n",
      "|200037|            7|\n",
      "|   205|            5|\n",
      "|   169|            8|\n",
      "|   272|            2|\n",
      "|   282|           29|\n",
      "|    15|           18|\n",
      "|    54|            8|\n",
      "|   234|           26|\n",
      "|   232|           23|\n",
      "|200043|           12|\n",
      "|   155|            4|\n",
      "|100014|            4|\n",
      "|   154|            4|\n",
      "+------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "session_counts = df.groupby('userId').agg(countDistinct('sessionId').alias('session_count'))\n",
    "session_counts.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = feature_df.join(session_counts, on=\"userId\")\n",
    "col_names.append(\"session_count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+-------------+\n",
      "|userId|session_count|\n",
      "+------+-------------+\n",
      "|100010|            2|\n",
      "|200002|            5|\n",
      "|   296|            5|\n",
      "|   125|            3|\n",
      "|   124|           17|\n",
      "|    51|            5|\n",
      "|     7|           15|\n",
      "|200037|            7|\n",
      "|   169|            8|\n",
      "|   205|            5|\n",
      "|   272|            2|\n",
      "|    15|           18|\n",
      "|   232|           23|\n",
      "|   234|           26|\n",
      "|   282|           29|\n",
      "|    54|            8|\n",
      "|   155|            4|\n",
      "|200043|           12|\n",
      "|100014|            4|\n",
      "|   132|            2|\n",
      "+------+-------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "feature_df.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counts of each event action (page) per user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = df.select('page').distinct().sort('page')\n",
    "pages_list = [r.page for r in pages.collect()]\n",
    "page_counts = df.groupby('userId').pivot('page', pages_list).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop the \"Cancel\" page column\n",
    "page_counts = page_counts.drop(\"Cancel\")\n",
    "\n",
    "# Fill NaNs with 0 - This will inherently transform \"Cancellation Confirmation\" into \"churn\"\n",
    "page_counts = page_counts.fillna(value=0)\n",
    "\n",
    "# Rename Cancellation Confirmation column to churn\n",
    "page_counts = page_counts.withColumnRenamed(\"Cancellation Confirmation\", \"Churn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>About</th>\n",
       "      <th>Add Friend</th>\n",
       "      <th>Add to Playlist</th>\n",
       "      <th>Churn</th>\n",
       "      <th>Downgrade</th>\n",
       "      <th>Error</th>\n",
       "      <th>Help</th>\n",
       "      <th>Home</th>\n",
       "      <th>Logout</th>\n",
       "      <th>NextSong</th>\n",
       "      <th>Roll Advert</th>\n",
       "      <th>Save Settings</th>\n",
       "      <th>Settings</th>\n",
       "      <th>Submit Downgrade</th>\n",
       "      <th>Submit Upgrade</th>\n",
       "      <th>Thumbs Down</th>\n",
       "      <th>Thumbs Up</th>\n",
       "      <th>Upgrade</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>200002</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>11</td>\n",
       "      <td>310</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100010</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>96</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>296</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>112</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>6</td>\n",
       "      <td>266</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  About  Add Friend  Add to Playlist  Churn  Downgrade  Error  Help  \\\n",
       "0  200002      1           2                6      1          3      0     1   \n",
       "1  100010      0           3                1      1          0      0     0   \n",
       "2     296      0           2                3      1          1      0     2   \n",
       "3     125      0           3                2      0          0      0     2   \n",
       "4      51      0           3                8      1          4      2     0   \n",
       "\n",
       "   Home  Logout  NextSong  Roll Advert  Save Settings  Settings  \\\n",
       "0    23      11       310           11              0         2   \n",
       "1     3       1        96           22              0         0   \n",
       "2     7       3       112            9              0         1   \n",
       "3     2       0        62            6              0         3   \n",
       "4    17       6       266            1              0         2   \n",
       "\n",
       "   Submit Downgrade  Submit Upgrade  Thumbs Down  Thumbs Up  Upgrade  \n",
       "0                 0               1            5         15        2  \n",
       "1                 0               0            3          4        2  \n",
       "2                 0               1            0          8        1  \n",
       "3                 0               0            1          3        0  \n",
       "4                 0               0            1         16        0  "
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page_counts.toPandas().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Join these feature columns to our feature dataframe\n",
    "feature_df = feature_df.join(page_counts, on=\"userId\")\n",
    "col_names.extend(list(page_counts.toPandas().columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['session_count',\n",
       " 'userId',\n",
       " 'About',\n",
       " 'Add Friend',\n",
       " 'Add to Playlist',\n",
       " 'Churn',\n",
       " 'Downgrade',\n",
       " 'Error',\n",
       " 'Help',\n",
       " 'Home',\n",
       " 'Logout',\n",
       " 'NextSong',\n",
       " 'Roll Advert',\n",
       " 'Save Settings',\n",
       " 'Settings',\n",
       " 'Submit Downgrade',\n",
       " 'Submit Upgrade',\n",
       " 'Thumbs Down',\n",
       " 'Thumbs Up',\n",
       " 'Upgrade']"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time since registration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ages = df.select([\"userId\", datediff(\"timestamp_dt\", \"registration_dt\")]).groupBy(\"userId\").max().select(\"userId\", col(\"max(datediff(timestamp_dt, registration_dt))\").alias(\"age\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_df = feature_df.join(user_ages, on='userId')\n",
    "col_names.append(\"age\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_number_events = df.groupBy(\"userId\").count().select(\"userId\", col(\"count\").alias(\"num_events\"))\n",
    "feature_df = feature_df.join(user_number_events, on=\"userId\")\n",
    "col_names.append(\"num_events\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine everything into a function (Use this function below instead executing individual cells above for feature engineering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_features(df):\n",
    "    \"\"\"\n",
    "    Create a vector Assembler of the features.\n",
    "    \"\"\"\n",
    "    feature_df = df.select(\"userId\").distinct()\n",
    "    col_names = []\n",
    "    \n",
    "    ts_dt_udf = udf(lambda x: x//1000, LongType())\n",
    "    df = df.withColumn(\"registration_dt\", ts_dt_udf(df.registration).cast(\"timestamp\"))\n",
    "    df = df.withColumn(\"timestamp_dt\", ts_dt_udf(df.ts).cast(\"timestamp\"))\n",
    "    \n",
    "    # Session Counts\n",
    "    session_counts = df.groupby('userId').agg(countDistinct('sessionId').alias('session_count'))\n",
    "    \n",
    "    feature_df = feature_df.join(session_counts, on=\"userId\")\n",
    "    col_names.append(\"session_count\")\n",
    "\n",
    "    \n",
    "    # Page Counts\n",
    "    pages = df.select('page').distinct().sort('page')\n",
    "    pages_list = [r.page for r in pages.collect()]\n",
    "    page_counts = df.groupby('userId').pivot('page', pages_list).count()\n",
    "    \n",
    "    # Drop the \"Cancel\" page column\n",
    "    # Fill NaNs with 0 - This will inherently transform \"Cancellation Confirmation\" column into \"label\" \n",
    "    # with 1 as churned and 0 as non churned\n",
    "    page_counts = page_counts.drop(\"Cancel\")\n",
    "    page_counts = page_counts.fillna(value=0)\n",
    "    page_counts = page_counts.withColumnRenamed(\"Cancellation Confirmation\", \"label\")\n",
    "    \n",
    "    # Join these feature columns to our feature dataframe\n",
    "    feature_df = feature_df.join(page_counts, on=\"userId\")\n",
    "    \n",
    "    # Normalize by session counts\n",
    "    cut_columns = {'userId', 'session_count', 'label'}\n",
    "    remaining_cols = sorted(list(set(feature_df.columns) - cut_columns))\n",
    "    for column in remaining_cols:\n",
    "        feature_df = feature_df.withColumn(column, col(column) / feature_df.session_count)\n",
    "    col_names.extend(remaining_cols)\n",
    "    \n",
    "    \n",
    "    #col_names.extend(list(page_counts.toPandas().columns))\n",
    "\n",
    "    \n",
    "    # Time since registration\n",
    "    user_ages = df.select([\"userId\", datediff(\"timestamp_dt\", \"registration_dt\")]).groupBy(\"userId\").max().select(\"userId\", col(\"max(datediff(timestamp_dt, registration_dt))\").alias(\"age\"))    \n",
    "    feature_df = feature_df.join(user_ages, on=\"userId\")\n",
    "    col_names.append(\"age\")\n",
    "    \n",
    "    # Total number of events \n",
    "    user_number_events = df.groupBy(\"userId\").count().select(\"userId\", col(\"count\").alias(\"num_events\"))\n",
    "    feature_df = feature_df.join(user_number_events, on=\"userId\")\n",
    "    col_names.append(\"num_events\")\n",
    "    \n",
    "    # Include device categorical variable\n",
    "    device_udf = udf(lambda x: str(re.findall(r'\\((.*?)\\)', x)[0].split(\";\")[0].split()[0]) if x is not None else None, StringType())\n",
    "    df = df.withColumn(\"device\", device_udf(df.userAgent))\n",
    "\n",
    "    df_device = df.select([\"userId\", \"device\"]).distinct()\n",
    "    df_device = StringIndexer(inputCol = \"device\", outputCol=\"device_index\").fit(df_device).transform(df_device)\n",
    "    df_device = OneHotEncoderEstimator(inputCols=[\"device_index\"], outputCols=[\"device_classVec\"]).fit(df_device).transform(df_device)\n",
    "    feature_df = feature_df.join(df_device.select(\"userId\", \"device_classVec\"), on=\"userId\")\n",
    "    col_names.append(\"device_classVec\")\n",
    "    # Remove user_id string from input columns\n",
    "    # col_names = list(set(col_names)-{'userId', 'sessionCount', 'label'})\n",
    "    print(col_names)\n",
    "    # Assemble the vector\n",
    "    assembler = VectorAssembler(inputCols=col_names, outputCol='features')\n",
    "    \n",
    "    return assembler.transform(feature_df), col_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['session_count', 'About', 'Add Friend', 'Add to Playlist', 'Downgrade', 'Error', 'Help', 'Home', 'Logout', 'NextSong', 'Roll Advert', 'Save Settings', 'Settings', 'Submit Downgrade', 'Submit Upgrade', 'Thumbs Down', 'Thumbs Up', 'Upgrade', 'age', 'num_events', 'device_classVec']\n"
     ]
    }
   ],
   "source": [
    "# Extract features\n",
    "feature_df, col_names = extract_features(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- userId: string (nullable = true)\n",
      " |-- session_count: long (nullable = false)\n",
      " |-- About: double (nullable = true)\n",
      " |-- Add Friend: double (nullable = true)\n",
      " |-- Add to Playlist: double (nullable = true)\n",
      " |-- label: long (nullable = true)\n",
      " |-- Downgrade: double (nullable = true)\n",
      " |-- Error: double (nullable = true)\n",
      " |-- Help: double (nullable = true)\n",
      " |-- Home: double (nullable = true)\n",
      " |-- Logout: double (nullable = true)\n",
      " |-- NextSong: double (nullable = true)\n",
      " |-- Roll Advert: double (nullable = true)\n",
      " |-- Save Settings: double (nullable = true)\n",
      " |-- Settings: double (nullable = true)\n",
      " |-- Submit Downgrade: double (nullable = true)\n",
      " |-- Submit Upgrade: double (nullable = true)\n",
      " |-- Thumbs Down: double (nullable = true)\n",
      " |-- Thumbs Up: double (nullable = true)\n",
      " |-- Upgrade: double (nullable = true)\n",
      " |-- age: integer (nullable = true)\n",
      " |-- num_events: long (nullable = false)\n",
      " |-- device_classVec: vector (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "feature_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(userId='100010', session_count=2, About=0.0, Add Friend=1.5, Add to Playlist=0.5, label=1, Downgrade=0.0, Error=0.0, Help=0.0, Home=1.5, Logout=0.5, NextSong=48.0, Roll Advert=11.0, Save Settings=0.0, Settings=0.0, Submit Downgrade=0.0, Submit Upgrade=0.0, Thumbs Down=1.5, Thumbs Up=2.0, Upgrade=1.0, age=14, num_events=137, device_classVec=SparseVector(5, {3: 1.0}), features=SparseVector(25, {0: 2.0, 2: 1.5, 3: 0.5, 7: 1.5, 8: 0.5, 9: 48.0, 10: 11.0, 15: 1.5, 16: 2.0, 17: 1.0, 18: 14.0, 19: 137.0, 23: 1.0}))]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_df.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling\n",
    "Split the full dataset into train, test, and validation sets. Test out several of the machine learning methods you learned. Evaluate the accuracy of the various models, tuning parameters as necessary. Determine your winning model based on test accuracy and report results on the validation set. Since the churned users are a fairly small subset, I suggest using F1 score as the metric to optimize."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|label|count|\n",
      "+-----+-----+\n",
      "|    0|  349|\n",
      "|    1|   99|\n",
      "+-----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "feature_df.groupBy(\"label\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = feature_df.randomSplit([0.8, 0.2], 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|label|count|\n",
      "+-----+-----+\n",
      "|    0|  275|\n",
      "|    1|   87|\n",
      "+-----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train.groupBy(\"label\").count().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----+\n",
      "|label|count|\n",
      "+-----+-----+\n",
      "|    0|   74|\n",
      "|    1|   12|\n",
      "+-----+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test.groupBy(\"label\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(res):\n",
    "    \"\"\"Given a results vector returns accuracy and f1-score\"\"\"\n",
    "    total = res.count()\n",
    "    \n",
    "    tp = res.where((res.label==1) & (res.prediction==1)).count()\n",
    "    tn = res.where((res.label==0) & (res.prediction==0)).count()\n",
    "    \n",
    "    fp = res.where((res.label==0) & (res.prediction==1)).count()\n",
    "    fn = res.where((res.label==1) & (res.prediction==0)).count()\n",
    "        \n",
    "    accuracy = (1.0*tp + tn) / total\n",
    "    precision = 1.0*tp / (tp + fp)\n",
    "    recall = 1.0*tp / (tp + fn)\n",
    "    f1 = 2.0 * (precision * recall) / (precision + recall)\n",
    "    \n",
    "    print('Accuracy: ', round(accuracy, 2))\n",
    "    print('Precision: ', round(precision, 2))\n",
    "    print('Recall: ', round(recall, 2))\n",
    "    print('F1-Score: ', round(f1, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr =  LogisticRegression(labelCol=\"label\", featuresCol=\"features\", maxIter=10, regParam=0.0, elasticNetParam=0)\n",
    "model = lr.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = model.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.84\n",
      "Precision:  0.43\n",
      "Recall:  0.5\n",
      "F1-Score:  0.46\n"
     ]
    }
   ],
   "source": [
    "get_metrics(test_predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine tuning Logistic Regression using CrossValidation\n",
    "\n",
    "**Note**: Even if you see empty outputs, I did get the output of this crossvalidation run and then decided to put in some maxIterations in the parameter grid and from then on no matter what I wasn't able to get any results. Tried debugging it a lot but to no avail. \n",
    "\n",
    "Anyways the result was less than good. The F1 score went up to just 0.48. So I guess not that of a big deal. I will be trying to debug this further.\n",
    "\n",
    "I am just keeping this here in hopes of showing off what I attempted than to just scrap it off. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [],
   "source": [
    "paramGrid = ParamGridBuilder()\\\n",
    "    .addGrid(lr.regParam, [0.1, 0.01]) \\\n",
    "    .addGrid(lr.fitIntercept, [False, True])\\\n",
    "    .addGrid(lr.elasticNetParam, [0.0, 0.5, 1.0])\\\n",
    "    .build()\n",
    "\n",
    "crossval = CrossValidator(estimator=lr,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=BinaryClassificationEvaluator(),\n",
    "                          numFolds=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    679\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 680\u001b[0;31m                 \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_items\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopleft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    681\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: pop from an empty deque",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-383-1e406a59d567>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcvModel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcrossval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.virtualenvs/text/lib/python3.5/site-packages/pyspark/ml/base.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, dataset, params)\u001b[0m\n\u001b[1;32m    130\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m             raise ValueError(\"Params must be either a param map or a list/tuple of param maps, \"\n",
      "\u001b[0;32m~/.virtualenvs/text/lib/python3.5/site-packages/pyspark/ml/tuning.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m             \u001b[0mtasks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_parallelFitTasks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meva\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollectSubModelsParam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubModel\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimap_unordered\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m                 \u001b[0mmetrics\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmetric\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnFolds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcollectSubModelsParam\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    682\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_index\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_length\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    683\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 684\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    685\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    686\u001b[0m                     \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_items\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopleft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    294\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "cvModel = crossval.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = cvModel.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_metrics(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Elastic Net Parameter:{}\".format(cvModel.bestModel._java_obj.getElasticNetParam()))\n",
    "print(\"Fit Intercept Parameter:{}\".format(cvModel.bestModel._java_obj.getFitIntercept()))\n",
    "print(\"Regularization Parameter:{}\".format(cvModel.bestModel._java_obj.getRegParam()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.86\n",
      "Precision:  0.5\n",
      "Recall:  0.33\n",
      "F1-Score:  0.4\n"
     ]
    }
   ],
   "source": [
    "decision_tree_model = DecisionTreeClassifier(labelCol=\"label\", featuresCol=\"features\")\n",
    "dt_model = decision_tree_model.fit(train)\n",
    "results_dt = dt_model.transform(test)\n",
    "get_metrics(results_dt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.86\n",
      "Precision:  0.5\n",
      "Recall:  0.33\n",
      "F1-Score:  0.4\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestClassifier(labelCol=\"label\", featuresCol=\"features\")\n",
    "rf_model = rf.fit(train)\n",
    "results_rf = rf_model.transform(test)\n",
    "get_metrics(results_dt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosted Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.85\n",
      "Precision:  0.44\n",
      "Recall:  0.33\n",
      "F1-Score:  0.38\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import GBTClassifier\n",
    "\n",
    "gbt = GBTClassifier(labelCol=\"label\", featuresCol=\"features\", maxIter=10)\n",
    "gbt_model = gbt.fit(train)\n",
    "results_gbt = gbt_model.transform(test)\n",
    "get_metrics(results_gbt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_paramGrid = ParamGridBuilder() \\\n",
    "                .addGrid(rf.maxDepth, [3,7]) \\\n",
    "                .addGrid(rf.minInstancesPerNode, [3, 5]) \\\n",
    "                .addGrid(rf.numTrees, [10, 20, 30, 40]) \\\n",
    "                .build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_crossval = CrossValidator(estimator = rf,\n",
    "                             estimatorParamMaps=rf_paramGrid,\n",
    "                             evaluator = BinaryClassificationEvaluator(),\n",
    "                             numFolds=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_cross_model = rf_crossval.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_rf_cross = rf_cross_model.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7078496917695946,\n",
       " 0.6311182916158642,\n",
       " 0.7030986688146883,\n",
       " 0.6169236129794381,\n",
       " 0.6981606744956262,\n",
       " 0.6948514272543397,\n",
       " 0.6847484890567417,\n",
       " 0.6761847235148204,\n",
       " 0.690249622591856,\n",
       " 0.7071814025697523,\n",
       " 0.6785053432504891,\n",
       " 0.6973630462708132]"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_cross_model.avgMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.87\n",
      "Precision:  0.6\n",
      "Recall:  0.25\n",
      "F1-Score:  0.35\n"
     ]
    }
   ],
   "source": [
    "get_metrics(results_rf_cross)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Steps\n",
    "Clean up your code, adding comments and renaming variables to make the code easier to read and maintain. Refer to the Spark Project Overview page and Data Scientist Capstone Project Rubric to make sure you are including all components of the capstone project and meet all expectations. Remember, this includes thorough documentation in a README file in a Github repository, as well as a web app or blog post."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}