APPeal - Clustering-BisectingKMeans.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "rapid-indonesian",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "\n",
    "from pyspark.sql.functions import col,isnan, when, count\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import avg\n",
    "from pyspark.sql.functions import udf\n",
    "from scipy.spatial import distance\n",
    "\n",
    "import pyspark.sql.functions as F\n",
    "\n",
    "\n",
    "import pyspark\n",
    "\n",
    "from pyspark.sql.types import FloatType\n",
    "from pyspark.sql.types import BooleanType\n",
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.sql.types import DateType\n",
    "from pyspark.sql.functions import regexp_replace\n",
    "from pyspark.sql import SQLContext\n",
    "\n",
    "from pyspark.ml.clustering import KMeans\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "married-threshold",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sc = pyspark.SparkContext()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "spectacular-girlfriend",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder \\\n",
    "    .master(\"local\") \\\n",
    "    .appName(\"AppName\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "#data = spark.read.option(\"header\",True).option('inferSchema', True).csv(\"Google-Playstore.csv\", sep =',')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "advance-saying",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = spark.read.option(\"header\",True).option('inferSchema', True).json(\"cleaned_v3.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "muslim-implement",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "detected-priest",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Ad Supported: boolean (nullable = true)\n",
      " |-- Age: long (nullable = true)\n",
      " |-- App Id: string (nullable = true)\n",
      " |-- App Name: string (nullable = true)\n",
      " |-- Category: string (nullable = true)\n",
      " |-- Developer Id: string (nullable = true)\n",
      " |-- Editors Choice: boolean (nullable = true)\n",
      " |-- In App Purchases: boolean (nullable = true)\n",
      " |-- Installs: double (nullable = true)\n",
      " |-- Installs_stdev_from_Category: double (nullable = true)\n",
      " |-- Minimum Android: double (nullable = true)\n",
      " |-- Price: double (nullable = true)\n",
      " |-- Price_stdev_from_Category: double (nullable = true)\n",
      " |-- Quarter of Release: string (nullable = true)\n",
      " |-- Rating: double (nullable = true)\n",
      " |-- Rating_Reviews_Ratio: double (nullable = true)\n",
      " |-- Rating_stdev_from_Category: double (nullable = true)\n",
      " |-- Reviews_Installs_Ratio: double (nullable = true)\n",
      " |-- Size: string (nullable = true)\n",
      " |-- Size_stdev_from_Category: double (nullable = true)\n",
      " |-- Year of Last Update: string (nullable = true)\n",
      " |-- Year of Release: string (nullable = true)\n",
      " |-- nReviews: double (nullable = true)\n",
      " |-- nReviews_stdev_from_Category: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fifteen-andrews",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.withColumn(\"Year of Last Update\", data[\"Year of Last Update\"].cast(IntegerType()).alias(\"Year of Last Update\"))\n",
    "data = data.withColumn(\"Year of Release\", data[\"Year of Release\"].cast(IntegerType()).alias(\"Year of Release\"))\n",
    "data = data.withColumn(\"Size\", data[\"Size\"].cast(FloatType()).alias(\"Size\"))\n",
    "for col in data.columns:\n",
    "    newcol = '_'.join(col.split(' '))\n",
    "    data = data.withColumnRenamed(col, newcol) \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "inclusive-amendment",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "designing-developer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Ad_Supported', 'boolean'),\n",
       " ('Age', 'bigint'),\n",
       " ('App_Id', 'string'),\n",
       " ('App_Name', 'string'),\n",
       " ('Category', 'string'),\n",
       " ('Developer_Id', 'string'),\n",
       " ('Editors_Choice', 'boolean'),\n",
       " ('In_App_Purchases', 'boolean'),\n",
       " ('Installs', 'double'),\n",
       " ('Installs_stdev_from_Category', 'double'),\n",
       " ('Minimum_Android', 'double'),\n",
       " ('Price', 'double'),\n",
       " ('Price_stdev_from_Category', 'double'),\n",
       " ('Quarter_of_Release', 'string'),\n",
       " ('Rating', 'double'),\n",
       " ('Rating_Reviews_Ratio', 'double'),\n",
       " ('Rating_stdev_from_Category', 'double'),\n",
       " ('Reviews_Installs_Ratio', 'double'),\n",
       " ('Size', 'float'),\n",
       " ('Size_stdev_from_Category', 'double'),\n",
       " ('Year_of_Last_Update', 'int'),\n",
       " ('Year_of_Release', 'int'),\n",
       " ('nReviews', 'double'),\n",
       " ('nReviews_stdev_from_Category', 'double')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "micro-genetics",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_cluster = [\"Reviews_Installs_Ratio\", \"Size\", \"Rating\", \"Price\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "roman-worcester",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_other = [\"Ad_Supported\", \"Category\", \"Developer_Id\", \"Editors_Choice\", \"In_App_Purchases\",\n",
    "                       \"Minimum_Android\", \"Year_of_Release\", \"Year_of_Last_Update\"]\n",
    "features_numeric = [\"Rating\", \"Installs\", \"nReviews\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "monthly-consumer",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "\n",
    "assemble=VectorAssembler(inputCols=features_cluster, outputCol='features_cluster')\n",
    "\n",
    "assembled_data=assemble.transform(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "published-composition",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import MinMaxScaler\n",
    "\n",
    "scale=MinMaxScaler(inputCol='features_cluster',outputCol='standardized')\n",
    "\n",
    "data_scale=scale.fit(assembled_data)\n",
    "\n",
    "data_scale_output=data_scale.transform(assembled_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "transsexual-willow",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Silhouette Score: 0.9602294005572355\n",
      "Silhouette Score: 0.8444086848149641\n",
      "Silhouette Score: 0.8271196428644819\n",
      "Silhouette Score: 0.7572141305572114\n",
      "Silhouette Score: 0.668457692155185\n",
      "Silhouette Score: 0.7515404715356496\n",
      "Silhouette Score: 0.7453495419387886\n",
      "Silhouette Score: 0.6052770798603161\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.clustering import BisectingKMeans\n",
    "from pyspark.ml.evaluation import ClusteringEvaluator\n",
    "\n",
    "silhouette_score=[]\n",
    "evaluator = ClusteringEvaluator(predictionCol='prediction', featuresCol='standardized', \n",
    "                                metricName='silhouette', distanceMeasure='squaredEuclidean')\n",
    "\n",
    "for i in range(2,10):\n",
    "    bkm = BisectingKMeans(featuresCol='standardized', k=i)\n",
    "    bkm = bkm.fit(data_scale_output)\n",
    "    output=bkm.transform(data_scale_output)\n",
    "    score=evaluator.evaluate(output)\n",
    "    silhouette_score.append(score)\n",
    "    print(\"Silhouette Score:\",score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "veterinary-cathedral",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'silhouette')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAFzCAYAAADSXxtkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU5dnH8e89WQiENRDWAAkQVhGEACruuOCKS0Vx16rFFbtYrfVt1da2b61v3cWtal1rq9aNui8UV8Imu4awBQQCyL6G3O8fM2CMAQbIyZlMfp/rypXMnHMy91xifvM85z7PMXdHREREkksk7AJERESk+ingRUREkpACXkREJAkp4EVERJKQAl5ERCQJKeBFRESSUGrYBVSnFi1aeG5ubthliIiI1IgJEyYsd/fsqrYlVcDn5uZSWFgYdhkiIiI1wszm72ybpuhFRESSkAJeREQkCSngRUREkpACXkREJAkp4EVERJKQAl5ERCQJKeBFRESSkAJeREQkCSngRUREkpACXkREJAkp4EVERJKQAn4nvixZRdGytWGXISIislcU8FXYtHUbFz8+nl+9NBV3D7scERGRPaaAr0JGWgrXH9eN8fO+5d+TF4VdjoiIyB5TwO/E8IL29GnflNvfmMWaTVvDLkdERGSPKOB3IhIxfjesFyvWb+aud74OuxwREZE9ooDfhf1zmjJiYAee/HQes5asCbscERGRuCngd+P6Y7vRKCOV3/x7uhruRESk1gg04M1sqJnNNrMiM7uxiu3NzOxlM/vSzL4ws/0qbJtnZlPNbLKZFQZZ5640y0znhqHd+WLeSl6ZvDisMkRERPZIYAFvZinA/cDxQE9ghJn1rLTbTcBkd98fuAC4u9L2I929r7sXBFVnPM4qaE+fnCbcPmYma9VwJyIitUCQI/iBQJG7F7v7FuB5YFilfXoC7wG4+ywg18xaBVjTXolEjNuG7cfydZu561013ImISOILMuDbAQsrPC6JPVfRFOB0ADMbCHQEcmLbHHjbzCaY2eU7exEzu9zMCs2ssLS0tNqKr6xP+6acPaADT3wyj9lLtMKdiIgktiAD3qp4rnKX2p+AZmY2GbgGmASUxbYNdvd+RKf4rzKzw6p6EXd/2N0L3L0gOzu7mkqv2i+Pizbc/c8r09RwJyIiCS3IgC8B2ld4nAN8r0vN3de4+8Xu3pfoOfhsYG5s2+LY92XAy0Sn/EPVLDOdXx7XnS/mruTVKWq4ExGRxBVkwI8H8s0sz8zSgbOBVyvuYGZNY9sALgXGuvsaM8s0s0axfTKBY4FpAdYat7MGtGf/nCb8/g013ImISOIKLODdvQy4GngLmAm84O7TzWykmY2M7dYDmG5ms4hOxY+KPd8KGGdmU4AvgDfc/c2gat0TKRUa7u5Ww52IiCSo1CB/ubuPAcZUem50hZ8/BfKrOK4Y6BNkbfuib/umnD2gPY9/Mo8zC9rTrXWjsEsSERH5Hq1kt5euP657dIU7NdyJiEgCUsDvpazMdH5xbDc+V8OdiIgkIAX8PhgxsAO92zXhdjXciYhIglHA74Now10vlq3dzD3vqeFOREQShwJ+Hx3QoVm04e7jeXy1VCvciYhIYlDAV4NfDu1OZj013ImISOJQwFeDrMx0fnFcNz4rXslrX34TdjkiIiIK+OpyzsAO7NeuMbe/MYN1m8t2f4CIiEiAFPDVZPsKd0vXqOFORETCp4CvRv06NOOsgvb8bdxcvlbDnYiIhEgBX81+ObQbDdJT+M0r09VwJyIioVHAV7PmDetx/XHd+LR4Ba+r4U5EREKigA/AOYM60qttY36vhjsREQmJAj4AFRvu7lXDnYiIhEABH5D+HZsxvCCHx8bNpWiZGu5ERKRmKeADdMPQ7mq4ExGRUCjgA9S8YT1+cVw3PpmzgjemquFORERqjgI+YOdub7h7fSbr1XAnIiI1RAEfsO0Nd0vWbOKe99VwJyIiNUMBXwP6d2zGmf1zeOy/argTEZGaoYCvITccH224++2rargTEZHgKeBrSIuG9fj5sd34uGgFY6YuCbscERFJcgr4GnTuoA70bNOY370+Qw13IiISKAV8DUpNifC7U3uxZM0m7n2/KOxyREQkiSnga1j/jln8qH8Oj40rpmjZurDLERGRJKWAD8GNx3cnIy2FW9RwJyIiAVHAh6BFw3r8/JiujCtazn+mqeFORESqnwI+JOcd2JEeargTEZGAKOBDkpoS4XfDevHN6k3c94Ea7kREpHop4ENUkJvFGf1yePS/xcwpVcOdiIhUHwV8yNRwJyIiQVDAhyy7UT1+dkxX/vv1ct5Uw52IiFSTQAPezIaa2WwzKzKzG6vY3szMXjazL83sCzPbL95jk8n5B3ake+tG/O71GWzYooY7ERHZd4EFvJmlAPcDxwM9gRFm1rPSbjcBk919f+AC4O49ODZpRFe424/Fqzdxn1a4ExGRahDkCH4gUOTuxe6+BXgeGFZpn57AewDuPgvINbNWcR6bVAbkZnF6v3Y88t9iitVwJyIi+yjIgG8HLKzwuCT2XEVTgNMBzGwg0BHIifPYpPOr43uQkapbyoqIyL4LMuCtiucqp9afgGZmNhm4BpgElMV5bPRFzC43s0IzKywtLd2XekOX3ageP4013L01XQ13IiKy94IM+BKgfYXHOcDiiju4+xp3v9jd+xI9B58NzI3n2Aq/42F3L3D3guzs7OqsPxQXHBRtuLvtNTXciYjI3gsy4McD+WaWZ2bpwNnAqxV3MLOmsW0AlwJj3X1NPMcmq9SUCLcNizbc3a8V7kREZC8FFvDuXgZcDbwFzARecPfpZjbSzEbGdusBTDezWUQ75kft6tigak00A/OyOP2Adjwydq4a7kREZK9YMjVzFRQUeGFhYdhlVItlazcx5C8fcUDHZjx58QDMqmpLEBGRuszMJrh7QVXbtJJdgmrZKIPrjunK2K9KeWv60rDLERGRWkYBn8AuPOi7Fe42btkWdjkiIlKLKOATWGpKhFtP6cWiVRvVcCciIntEAZ/gBnVqzmkHtOPhscXMXb4+7HJERKSWUMDXAr86vjvpqRHdUlZEROKmgK8FWjbO4Lqj8/noq1LenqGGOxER2T0FfC1x4cG5dGsVXeFODXciIrI7CvhaIi0lwm3Dog13D3yohjsREdk1BXwtMqhTc07t25aHPipmnhruRERkFxTwtcxNJ/SINty9poY7ERHZOQV8LbO94e7D2aW8o4Y7ERHZCQV8LXThwbl0bdWQW9VwJyIiO6GAr4XSYreUXbRqIw+q4U5ERKqggK+lDuzUnGF92zJ6rBruRETkhxTwtdhNJ/QgLWLcqoY7ERGpRAFfi7VqnMF1R3flg9mlvDtzWdjliIhIAlHA13IXDc4lv2VDbn1tOpu2quFORESiFPC13PaGu5JvN/LAh3PCLkdERBKEAj4JHNS5OSf3acvoj+Ywf4Ua7kRERAGfNH69o+FuRtiliIhIAlDAJ4nWTTIYdXQ+789axrta4U5EpM5TwCeRiwfnkd+yIbeo4U5EpM5TwCeRtJQItw7rRcm3G3lQDXciInWaAj7JHNy5BSft34YHP5rDghUbwi5HRERCooBPQjef2JPU2Ap3IiJSNyngk1DrJhmMGpLPe2q4ExGpsxTwSeriwXl0admQW19Xw52ISF2kgE9S6akRbjulFwtXbmT0R2q4ExGpaxTwSezgLi04cf82PPihGu5EROoaBXySu/nEHqREjNteV8OdiEhdooBPcm2a1OfaIfm8O3MZ781Uw52ISF2hgK8DLhmcR+fsTG59bYYa7kRE6ggFfB2Qnhq9peyClRt46KPisMsREZEaEGjAm9lQM5ttZkVmdmMV25uY2WtmNsXMppvZxRW2zTOzqWY22cwKg6yzLhjcpQUn9m7DAx8WsXClGu5ERJJdYAFvZinA/cDxQE9ghJn1rLTbVcAMd+8DHAHcaWbpFbYf6e593b0gqDrrkptPijbc6ZayIiLJL8gR/ECgyN2L3X0L8DwwrNI+DjQyMwMaAiuBsgBrqtPaNKnPNUfl8+7Mpbw/Sw13IiLJLMiAbwcsrPC4JPZcRfcBPYDFwFRglLuXx7Y58LaZTTCzywOss0758SF5dMrO5JZX1XAnIpLMggx4q+I5r/T4OGAy0BboC9xnZo1j2wa7ez+iU/xXmdlhVb6I2eVmVmhmhaWlpdVUevKKrnAXbbh7eKwa7kREklWQAV8CtK/wOIfoSL2ii4GXPKoImAt0B3D3xbHvy4CXiU75/4C7P+zuBe5ekJ2dXc1vITkdkt+CE3q35v4P1HAnIpKsggz48UC+meXFGufOBl6ttM8CYAiAmbUCugHFZpZpZo1iz2cCxwLTAqy1zrn5xJ5EzLjtdTXciYgko8AC3t3LgKuBt4CZwAvuPt3MRprZyNhuvwMONrOpwHvADe6+HGgFjDOzKcAXwBvu/mZQtdZFbZvW55ohXXhnxlIe+LCIucvX4175DIqIiNRWlkx/1AsKCrywUJfMx2tLWTnDH/qUyQtXAdCyUT0O7NQ89pVFXotMohc4iIhIIjKzCTu7lDy1pouRxJGeGuHlKw+mePl6PitewWfFK/m0eAWvTom2SmTvCPwsDuzUnE4KfBGRWkMBX8eZGZ2zG9I5uyHnDuqIuzN3+Xo+K14ZC/0VvFYh8AflZe0Y5XfOVuCLiCQqBbx8j5nRKbshnbIbcs6gDrg781Zs2BH2nxWv4PUvvwGgRcN6DIqN7g/qlEXn7IYKfBGRBKGAl10yM/JaZJLXIpMRA6OBP/97gb+SN3YEfjqD8r6b0u/SUoEvIhIWBbzsETMjt0UmuS0yObtC4H8+N3YOf84K3pgaDfzmmekc2Kn5jlF+vgJfRKTGKOBln1QM/LMGRAN/wcoNfB47h/9p8fcDf1CnrNgoPxr4kYgCX0QkCAp4qVZmRsfmmXRsnsnwAe1xdxau3Bidzp+7gs/mrGDM1CUAZGWm72jaG9Qpi64tGynwRUSqiQJeAmVmdGjegA7NG+wI/JJvN/Jp8Yodo/z/TIsGfrMGaTvO4Q/q1JxurRT4IiJ7SwEvNcrMaJ/VgPZZDRheEL1VwcKVG3Y07H0+dwVvTo8GftMGad+7LE+BLyISPwW8hG574J9ZIfA/nxsd3X8+dwVvTY/eu75pgzQG5n4X+N1bK/BFRHZGAS8JZ3vg/6h/DgAl337XtPf53JW8PSMa+E3qpzFwxwg/ix6tGyvwRURiFPCS8HKaNSCnfwPOiAX+olUb+bzCdfjvxAK/cUYqAytch9+jTWNSFPgiUkfpZjNS6y1etTF6Hf6clXw2dwXzV0TvcR8N/O+m9Hu20QhfRJLLrm42o4CXpPPN6o07pvQ/K17BvFjgH9OzFfef04/01MDukiwiUqMU8FKnLVm9iRcnlnDHW7MV8iKSVHYV8PorJ0mvdZMMrjqyC7ee0ot3Zizl6mcnsnVbedhliYgESgEvdcaFB+dyy8k9eXvGUq55dpJCXkSSmgJe6pSLBufxm5N68ub0JVz7nEJeRJKXAl7qnEsOyePmE3vwn2lLuO75yQp5EUlKug5e6qRLD+0EwO/fmAkGd5/Vl9QUfd4VkeShgJc669JDO1Huzh/GzCJixl+H91HIi0jSUMBLnXb5YZ0pd/jTf2ZhwP8p5EUkSSjgpc4beXhnyt3585uziRjcObyvlrgVkVpPAS8CXHlEF9zhjrdmEzHjjjP7KORFpFZTwIvEXHVkF9ydv7z9FRjc8SOFvIjUXgp4kQquPiqfcof/e+crImb8+Yz9dYMaEamVFPAilVw7JJ9yd+5692sM+F+FvIjUQgp4kSpcd3RXyh3uee9rImb88fTeCnkRqVXiDngzqw90cPfZAdYjkjB+enQ+7s697xdhBn84TSEvIrVHXAFvZicDfwHSgTwz6wvc5u6nBFmcSJjMjJ8d05Vyd+7/YA5mxu2n7qeQF5FaId4R/C3AQOBDAHefbGa5gVQkkkDMjF8c241yhwc/nEPE4Pen7oeZQl5EElu8AV/m7qv1R03qIjPjl8d1wx1GfzSHiBm3DeulkBeRhBZvwE8zs3OAFDPLB64FPgmuLJHEYmbcMLQb7s5DY4uJGNxyikJeRBJXvItuXwP0AjYDzwKrgVG7O8jMhprZbDMrMrMbq9jexMxeM7MpZjbdzC6O91iRmmZm3Hh8dy47NI8nP53Pra/NwN3DLktEpErxjuBPdPdfA7/e/oSZnQn8c2cHmFkKcD9wDFACjDezV919RoXdrgJmuPvJZpYNzDazZ4BtcRwrUuPMjJtO6EG5w2Pj5mIGvzmpp0byIpJw4h3B/yrO5yoaCBS5e7G7bwGeB4ZV2seBRhb969gQWAmUxXmsSCjMjJtP7MHFg3N5/ON5/O71mRrJi0jC2eUI3syOB04A2pnZPRU2NSYaxLvSDlhY4XEJMKjSPvcBrwKLgUbAWe5ebmbxHLu9xsuBywE6dOiwm5JEqoeZ8ZuTeuIOf/t4LhGDX5/YQyN5EUkYu5uiXwwUAqcAEyo8vxb46W6OreovXeVhznHAZOAooDPwjpn9N85jo0+6Pww8DFBQUKBhlNQYM+O3J/fE3Xl03FwiEeNXx3dXyItIQthlwLv7FGCKmbVy9ycrbjOzUcDduzi8BGhf4XEO0Q8MFV0M/Mmj85tFZjYX6B7nsSKhMzNuOaUX5Q4Pjy3GDG4cqpAXkfDFew7+7Cqeu2g3x4wH8s0sz8zSY7/j1Ur7LACGAJhZK6AbUBznsSIJwWLXxZ93YAce+qiYP781W+fkRSR0uzsHPwI4h+jytBUDthGwYlfHunuZmV0NvAWkAH9z9+lmNjK2fTTwO+AJM5tKdFr+BndfHnvtHxy7N29QpCaYGbedst/3Vrz7xbHdNJIXkdDs7hz8J8A3QAvgzgrPrwW+3N0vd/cxwJhKz42u8PNi4Nh4jxVJZJGI8fth++GxtesjsbXsFfIiEobdnYOfD8wHDjKzjkC+u78bu7NcfaJBLyIxkYhx+6m9KS8ndhe6aMiLiNS0eO8mdxnRS9GyiHa75wCjiZ0/F5HvRCLR+8eXu8fuJx+9v7yISE2KdyW7q4guPvM5gLt/bWYtA6tKpJaLRIz/PWN/HLjr3a8xjFFH54ddlojUIfEG/GZ337L9XKKZpbKT69JFJGpHyDv89d2viBhcM0QhLyI1I96A/8jMbgLqm9kxwJXAa8GVJZIcUiLGn3+0P+7One98RSRiXHVkl7DLEpE6IN6AvxH4MTAV+AnR7vZHgypKJJmkRIw7zuxDuTt3vDUbM7jyCIW8iAQrroB393LgkdiXiOyhlIhx5/C+OPDnN2cTMWPk4Z3DLktEkli8XfRzqeKcu7t3qvaKRJJUSsS488w+lDv86T+zMOAnCnkRCUi8U/QFFX7OAM4kesmciOyB1JQIfx0ena7/439mETHjssP0OVlEql+8U/SVl6W9y8zGAb+p/pJEkltqSoS7z+oLDrePmYkZXHqoQl5Eqle8U/T9KjyMEB3RNwqkIpE6IDUlwl1n98Vxfv/GTCJmXHJIXthliUgSiXeKvuI69GXAPGB4tVcjUoekpUS4++wDKC+fxG2vzyBicNFghbyIVI94p+iPDLoQkbooLSXCveccwNXPTuSW12YQiRgXHJQbdlkikgTiuh+8mTUxs/8zs8LY151m1iTo4kTqgrSUCPeO6McxPVvxm1em89Sn88IuSUSSQFwBD/yN6J3jhse+1gCPB1WUSF2Tnhrh/nP6cXSPlvzPK9N5+rP5YZckIrVcvAHf2d1/6+7Fsa9bAbX9ilSj9NQI95/bjyHdW3Lzv6fxzOcKeRHZe/EG/EYzO2T7AzMbDGwMpiSRuqteagoPnNePo7q35NcvT+O5LxaEXZKI1FLxdtGPBP4eO+9uwErgoqCKEqnL6qWm8OB5/Rj51AR+9dJUIgZnDegQdlkiUsvE20U/BehjZo1jj9cEWpVIHRcN+f785KkJ3PjSVAxj+ID2YZclIrVIvAvd1APOAHKB1O33hXf32wKrTKSOy0hL4aHz+3PZ3wu54aUvMYMzCxTyIhKfeM/BvwIMI7rIzfoKXyISoIy0FB65oIBDurTgly9+yYsTSsIuSURqiXjPwee4+9BAKxGRKm0P+UufLOQX/5qCGZzeLyfsskQkwcU7gv/EzHoHWomI7NT2kD+oU3N+/s8pvDxJI3kR2bVdjuDNbCrR+8CnAhebWTGwmWgnvbv7/sGXKCIA9dNTeOzCAVzyxHh+/sIUImYM69su7LJEJEHtbor+pBqpQkTiUj89hccuKuCSJ8bz039Mxsw4pU/bsMsSkQS0uyn6tbv5EpEa1iA9lb9dNICC3Cyue34Sr01ZHHZJIpKAdjeCn0B0it6q2OZouVqRUDRIT+XxiwZw8ePjue4fk4mYceL+bcIuS0QSyC4D3t11c2qRBJVZL5XHLx7ARY9/wbXPTyJicHxvhbyIRO1yit7Muse+96vqq2ZKFJGdiYb8QPq2b8o1z03izWnfhF2SiCSI3U3R/wy4HLizwnNe4eejqr0iEdkjDeul8sTFA7jwb19w9bOTePiCCEd1bxV2WSISsl2O4N398tiPDwLD3P1I4ANgNfCLgGsTkTg1ykjjyUsG0qNNY0Y9N5ni0nVhlyQiIYt3oZub3X1N7JaxxwBPEA19EUkQjTLSePC8fqSmGCOfnsCGLWVhlyQiIYo34LfFvp8IjHb3V4D0YEoSkb2V06wB947oR9Gyddz44lTcffcHiUhSijfgF5nZQ8BwYEzs7nK7PdbMhprZbDMrMrMbq9h+vZlNjn1NM7NtZpYV2zbPzKbGthXuyZsSqcsOyW/Bz4/txqtTFvP4x/PCLkdEQhLvzWaGA0OBv7j7KjNrA1y/qwPMLAW4n+iUfgkw3sxedfcZ2/dx9zuAO2L7nwz81N1XVvg1R7r78rjfjYgAcMXhnZm8cBV/GDOT/do1YWBeVtglyR5asnoTFz8xnpXrN9OiYb0KX+nR743Sv/d8VmY6KZGqliyRuiqugHf3DcBLFR5/A+zuepyBQJG7FwOY2fNEbzk7Yyf7jwCei6ceEdm1SMS4c3gfTrl3HFc9O5E3rjmElo0zwi5L4rRxyzYu+3shC1as54TebVixfgvL123m66VrWb5uC1u2lf/gGDPIavDD8G8e+0CQvf3DQKN0mmfWIz013glcqa3iHcHvjXbAwgqPS4BBVe1oZg2IzhBcXeFpB942MwcecveHd3Ls5UQv5aNDhw7VULZIcmickcbo8/tz2v2fcNWzE3n2sgNJS9Ef9URXXu78/J+TmbZ4NY9eUMCQHt+/5NHdWbu5jOVrN7N8XTT4o1+xn9dGH09asIrl6zazYcu2Kl+nSf20SuG//cNBPZpnptOi0XcfCuqnp9TEW5dqFmTA72x526qcDHxcaXp+sLsvNrOWwDtmNsvdx/7gF0aD/2GAgoICdRSJVNC9dWP+dEZvRj0/mT+OmcVvTu4ZdkmyG3e99zVjpi7h1yf0+EG4A5gZjTPSaJyRRqfs3f++DVvKWLFuC6U7wr/ih4Lo45lL1rB87WbWbKr6yovM9JTvgj/2ISD6wSA9NksQ+4DQqB6N6qViplMFiSDIgC8B2ld4nAPs7K4YZ1Npet7dF8e+LzOzl4lO+f8g4EVk14b1bcfkhav428dz6duhqe4+l8BembyIe977muEFOVx6aPWsFN4gPZUGWam0z2qw2303l21jRewDwI4PBes2s3ztdx8K5q/YwIT537JywxaqukgjPTWyY0agecWZgR0fDL573LR+GhH1DQQmyIAfD+SbWR6wiGiIn1N5JzNrAhwOnFfhuUwg4u5rYz8fC9wWYK0iSe2mE3owtWQ1N/zrS7q1akS31o3CLkkqmbTgW67/15cMzMvi96f2DmUUXC81hbZN69O2af3d7lu2rZyVG7Z8L/y3fzgojc0MLFm9iWmLVrNi/Ra2lf/w00BqxMjKTKd1kwx+e3JP+ndUM2h1siCvkzWzE4C7gBTgb+5+u5mNBHD30bF9LgKGuvvZFY7rBLwce5gKPOvut+/u9QoKCrywUFfUiVRl2ZpNnHjvOBrWS+WVqwfTOCMt7JIkZvGqjQy7/2My0iK8ctUhZGUm1zIj5eXO6o1bvxf+y9duZsX66OzAB7OX0aJhPd649hBN7+8hM5vg7gVVbkumhTAU8CK79sXclYx45DOO6t6Sh87rr+nRBLBhSxk/evBTFq7cwEtXHkx+q7o3u/LihBJ+/s8pPHx+f47t1TrscmqVXQW8WmpF6pCBeVncdEIP3pmxlNFj54RdTp1XXu787B9TmLVkDfecc0CdDHeAYX3b0rF5A+5+72utvliNFPAidcwlg3M5uU9b/vLWbMZ9rXWkwvR/73zFm9OX8OsTe3Jkt5ZhlxOa1JQIVx3ZhemL1/D+rGVhl5M0FPAidYyZ8afTe9OlZUOufX4Si1ZtDLukOunfkxZx3wdFjBjYnksG54ZdTuhOO6Ad7bPqaxRfjRTwInVQZr1URp/Xny1l5Vz59AQ2ba16MRQJxoT53/LLF7/kwE5Z3HrKfmosA9JSIlx9ZBe+LFnNh7NLwy4nKSjgReqoTtkN+cuZfZhSsppbX9vZCtJS3Uq+3cBPniqkTZMMHjy3v5aMreD0fjnkNKvPXRrFVwv9yxKpw4bu15orjujMc18s4IXxC3d/gOyT9ZvLuPTJQjaXlfPYhQNolmSXw+2rtNi5+CkLV/HRVxrF7ysFvEgd9/NjujK4S3NufmUa0xatDrucpFVe7lz3j8l8vWwd95/Tjy4tG4ZdUkI6o18O7ZrqXHx1UMCL1HGpKRHuOfsAWmSmM/LpCXy7fkvYJSWlO96ezTszlvI/J/bgsK5xLCJfR6WnRrjiiM5MWrCKcUW6ymNfKOBFhOYN6/HAef1ZtmYzo/4xucplRWXvvTihhAc/nMO5gzpw4cG5YZeT8M4syKFNkwzuflej+H2hgBcRAPq2b8otp/Ri7Fel3P3uV2GXkzQK563kVy9N5eDOzbnllF7qmI9DvdQUrjyiM4Xzv+WTOSvCLqfWUsCLyA4jBrbnzP453PN+Ee/NXBp2ObXewpUb+MlTE2jXrMkr6Q4AABjQSURBVD4PnNuPtBT9yY3X8AHtad1Yo/h9oX9tIrKDmfG7U/ejV9vGXPePycxbvj7skmqtdbGO+a3bynn0wgKaNlDH/J6ol5rCFUd05ot5K/m0WKP4vaGAF5HvyUhLYfR5/YmYMfLpCWzcokVw9tS2cmfUc5MoKl3HA+f2p3O2Oub3xlkD2tOyUT3ufvfrsEuplRTwIvID7bMacPfZfZm9dC03vTxVU6R76M9vzuK9Wcu45eSeHJLfIuxyaq2MtBRGHt6Zz+eu5DON4veYAl5EqnREt5b89OiuvDxpEU99Nj/scmqNFwoX8tDYYi44qCPnH5Qbdjm13jmDOpDdqB73vKdR/J5SwIvITl19ZBeO6t6S216bwYT5K8MuJ+F9MXclv355Kod0acFvTuoZdjlJISMthZ8c1olP5qxg/Dz9G9wTCngR2alIxPjr8L60bVqfK5+ZSOnazWGXlLAWrIiuMd8+qwH3n9OPVHXMV5tzB3WkRUOdi99T+hcoIrvUpEEao8/rz+qNW7n62YmUbSsPu6SEs3bTVn785HjKHR67cABNGqSFXVJSqZ8eHcWPK1qumaQ9oIAXkd3q2bYxfzy9N5/PXcmf35oddjkJZVu5c+1zk5i7fD0PntePvBaZYZeUlM49sAPNM9O5S6P4uCngRSQupx2QwwUHdeThscWMmfpN2OUkjD+OmckHs0u5dVgvDu6sjvmgNEhP5bLDOvHfr5czccG3YZdTKyjgRSRuN5/YkwM6NOX6f06haNnasMsJ3fNfLODRcXO56OBczh3UMexykt75B3YkKzNdHfVxUsCLSNzSUyM8cG4/6qencPlTE1i7aWvYJYXm0zkruPnf0zisazY3n9gj7HLqhMx6qVx6aB4fzi5l8sJVYZeT8BTwIrJH2jSpz70j+jF/xQZ++a8v6+QiOPNXrOeKZyaQ2yKT+845QB3zNeiCg3Jp2iBNo/g46F+liOyxgzo354ah3fjPtCU88t/isMupUWs2beWSJ8YD8NiFBTTOUMd8TWpYL5XLDu3E+7OW8WWJRvG7ooAXkb1y2aGdOKF3a/70n1l8Mmd52OXUiLJt5Vz97CTmr9jA6PP607G5OubDcMFBHWlSX6P43VHAi8heMTP+/KM+5LXI5JpnJ/HN6o1hlxS428fMZOxXpdx+2n4c2Kl52OXUWY0y0vjxIXm8O3MZ0xatDruchKWAF5G91rBeKg+d359NW7dxxdMT2VyWvHeee+bz+Tz+8Tx+fEgeZw3oEHY5dd5Fg3NpnJGqUfwuKOBFZJ90admIO87sw+SFq/j96zPDLicQnxQt57evTOfIbtncdII65hNB44w0Ljkkj7dnLGXG4jVhl5OQFPAiss9O6N2Gyw/rxFOfzefFCSVhl1Ot5i5fzxXPTCSvRSb3jDiAlIiFXZLEXDw4j0Yaxe+UAl5EqsUvj+vGgZ2yuOnlqUxfnBznRVdv2MqPnxhPSsR47MIBNFLHfEJpUj+Niwfn8eb0Jcz8RqP4yhTwIlItUlMi3DuiH80apHPF0xNZvaF2L4JTtq2cq56dyMJvox3zHZo3CLskqcIlg3NpWC+Ve9/XKL4yBbyIVJvsRvW4/9x+fLN6I9f9YxLl5bV3EZzfvT6DcUXL+cNpvRmYlxV2ObITTRukc9HBuYyZuoTZS7R8ckWBBryZDTWz2WZWZGY3VrH9ejObHPuaZmbbzCwrnmNFJDH179iM35zUkw9ml3Lv+0Vhl7NXnvp0Hk9+Op/LD+vEmQXtwy5HduPHh+SRmZ6iUXwlgQW8maUA9wPHAz2BEWbWs+I+7n6Hu/d1977Ar4CP3H1lPMeKSOI678COnH5AO+567ys+mL0s7HL2yLivl3PLazMY0r0lNwztHnY5EodmmelceHAub0z9hq+XahS/XZAj+IFAkbsXu/sW4Hlg2C72HwE8t5fHikgCMTNuP6033Vs35rrnJ7Nw5YawS4rLnNJ1XPnMBLpkN+RudczXKpce2on6aSm1dtYoCEEGfDtgYYXHJbHnfsDMGgBDgRf39FgRSUz101MYfV4/3J2RT09g09bEXgRn1YYtXPpkIWkpER69sICG9VLDLkn2QFZmOhcclMtrXy6maNm6sMtJCEEGfFUffXfWcXMy8LG7r9zTY83scjMrNLPC0tLSvShTRILSsXkmd53dl+mL13Dzv6cl7J3ntm4r58pnJrLo2408dH5/2mepY742uuzQPDJSU7hP5+KBYAO+BKjYnZIDLN7Jvmfz3fT8Hh3r7g+7e4G7F2RnZ+9DuSIShKO6t+LaIfn8a0IJz36xIOxyfsDdueXV6XwyZwV/PL03BbnqmK+tmjesx/kHdeTVKYspLtUoPsiAHw/km1memaUTDfFXK+9kZk2Aw4FX9vRYEakdRg3J5/Cu2dzy6nQmLfg27HK+5++fzueZzxdwxRGdOaN/TtjlyD667NBOpKdGuO8DnYsPLODdvQy4GngLmAm84O7TzWykmY2ssOtpwNvuvn53xwZVq4gEKyVi3H12X1o1zuDKZyayYt3msEsCYOxXpdz62nSO6dmK64/tFnY5Ug2yG9XjvEEdeWXyYuYtX7/7A5KYJeo5sb1RUFDghYWFYZchIjsxbdFqznjwE/p3bMbfLxlIakp4a20VLVvLaQ98Qk6zBvxr5EFkqqkuaSxbu4lD//cDTu7Tlr+c2SfscgJlZhPcvaCqbVrJTkRqzH7tmvD7U/fjkzkruPOdr0Kr49v1W/jxk4XUS412zCvck0vLRhmcO6gjL09axPwVdXcUr4AXkRp1ZkF7zhnUgQc/nMOb05bU+OtvKStn5NMT+Gb1Jh46v4B2TevXeA0SvJ8c3omUiHF/HT4Xr4AXkRr325N70ienCb/45xTm1GC3s7vz21en8fnclfz5jP3p37FZjb221KxWjTM4Z2AHXpq4qNYstFTdFPAiUuPqpabwwHn9SU+NMPKpCazfXFYjr/v4x/N47ouFXH1kF049QGtnJbuRh3cmYsYDH9bNUbwCXkRC0a5pfe4dcQBzStdxw4tfBr4Izgezl/H7N2ZwXK9W/OyYroG+liSG1k0yOHtge/5ZWELJt3VvFK+AF5HQDO7SguuP687rX37D3z6eF9jrfLV0Ldc8O4nurRvz17P6EtEa83XGFUdsH8XPCbuUGqeAF5FQjTy8E8f1asUfxszki7krd3/AHlq5fgs/fnI89dNTePTCAhqkq2O+LmnTpD7DB+Twz8KFLFq1MexyapQCXkRCZWbccWYfOmY14MpnJrJ0zaZq+91bysoZ+dQElq3ZzCMXFNBWHfN10hVHdAHgwTp2Ll4BLyKha5yRxujz+7N+cxlXPjORLWXl+/w73Z2b/z2VL+at5I4z+9C3fdNqqFRqo3ZN6/Oj/u15YXwJ36yuO6N4BbyIJISurRrxvz/anwnzv+UPY2bu8+97bNxcXigs4doh+ZzSp201VCi12ZVHdKbcndF16Fy8Al5EEsYpfdpyyeA8nvhkHq9MXrTXv+e9mUu5fcxMTuzdhuuG5FdjhVJbtc9qwI/65/Dc+IXVehookSngRSSh/OqE7gzMzeLGF6cya8maPT5+9pK1XPvcJPZr24S/nNlHHfOyw1VHdqG83HmwjoziFfAiklDSUiLcd+4BNMpIZeRTE1izaWvcxy5ft5kfPzmezHqpPHJBAfXTUwKsVGqb9lkNOL1fO577YgHL6sAoXgEvIgmnZaMM7j+3HyXfbuRn/5hCefnuF8HZXLaNkU9NoHTtZh69sIDWTTJqoFKpba46sgtl5c7oj4rDLiVwCngRSUgDcrP49Yk9eHfmUh78aNdTqu7OTS9No3D+t9w5vA/756hjXqrWsXkmp/ZtxzOfz2fZ2uQexSvgRSRhXXRwLqf0actf3p7N2K9Kd7rfw2OLeXFiCT89uisn7a+Oedm1q4/qwtZt5TwyNrlH8Qp4EUlYZsafzuhN15aNGPX8pCrXE39nxlL+9OYsTu7TlmuHdAmhSqlt8lpER/FPfTaf5es2h11OYBTwIpLQGqSnMvr8/pRtc658ZiKbtm7bsW3mN2sY9fwk9m/XhDt+tD9m6piX+Fx9VBe2lCX3KF4BLyIJL69FJncO78OXJau55dXpAJSu3cylTxbSOCONhy8oICNNHfMSv07ZDTmlT1v+/ul8ViTpKF4BLyK1wrG9WnPVkZ15fvxC/v7pPH7yVCEr1kc75ls1Vse87Lmrj+rCprJtPPLfuWGXEggFvIjUGj87phuHdGnBb16ZzsQFq/jr8L7s165J2GVJLdWlZSNO2r8tf/90HivXbwm7nGqngBeRWiMlYtwz4gD6tm/KzSf24PjebcIuSWq5a4/qwsat23hsXPKdi1fAi0itkpWZzr+vGsylh3YKuxRJAvmtGnFC7zY8+cl8Vm1IrlG8Al5EROq0a4/KZ93mMh4bl1zn4hXwIiJSp3Vr3YgTerfmiY/nsXpD/Pc+SHQKeBERqfOuOSqftZvLeOzj5BnFK+BFRKTO69GmMcf1asXjH89l9cbkGMUr4EVERIBrh+SzdlMZT3w8L+xSqoUCXkREBOjVtgnH9GzFY+OKWbOp9o/iFfAiIiIxo4bks2ZTGU8mwSheAS8iIhKzX7smHN2jJY+Om8vaWj6KV8CLiIhUcO2QfFZv3MrfP50fdin7RAEvIiJSwf45TTmyWzaP/LeYdZvLwi5nrwUa8GY21Mxmm1mRmd24k32OMLPJZjbdzD6q8Pw8M5sa21YYZJ0iIiIVjTq6K6s2bOWpWjyKDyzgzSwFuB84HugJjDCznpX2aQo8AJzi7r2AMyv9miPdva+7FwRVp4iISGV92zfl8K7RUfz6WjqKD3IEPxAocvdid98CPA8Mq7TPOcBL7r4AwN2XBViPiIhI3EYdnc/K9Vt4+rPaOYoPMuDbAQsrPC6JPVdRV6CZmX1oZhPM7IIK2xx4O/b85QHWKSIi8gP9OjTj0PwWPDy2mA1bat8oPsiAtyqe80qPU4H+wInAccD/mFnX2LbB7t6P6BT/VWZ2WJUvYna5mRWaWWFpaWk1lS4iIhK9Ln7F+i0889mCsEvZY0EGfAnQvsLjHGBxFfu86e7r3X05MBboA+Dui2PflwEvE53y/wF3f9jdC9y9IDs7u5rfgoiI1GUFuVkM7tKch8bOYeOWbWGXs0eCDPjxQL6Z5ZlZOnA28GqlfV4BDjWzVDNrAAwCZppZppk1AjCzTOBYYFqAtYqIiFRp1JCuLF+3hWe/qF2j+MAC3t3LgKuBt4CZwAvuPt3MRprZyNg+M4E3gS+BL4BH3X0a0AoYZ2ZTYs+/4e5vBlWriIjIzgzMy+KgTs0Z/dEcNm2tPaN4c698Wrz2Kigo8MJCXTIvIiLV67PiFZz98Gf89uSeXDw4L+xydjCzCTu7lFwr2YmIiOzGgZ2aMygvq1aN4hXwIiIicRg1JJ+lazbzj/ELd79zAlDAi4iIxOGgzs0ZkNuMBz+cw+ayxB/FK+BFRETiYGaMGtKVJWs28UJhSdjl7JYCXkREJE6DuzSnf8dmPPhBUcKP4hXwIiIicYqO4vNZvHoT/5qQ2KN4BbyIiMgeODS/BQd0aMoDH8xhS1l52OXslAJeRERkD5gZ1w7JZ9Gqjbw4MXFH8Qp4ERGRPXRE12z65DTh/g+K2LotMUfxCngREZE9ZGaMOjqfkm838vLERWGXUyUFvIiIyF44sltL9s9pwn0JOopXwIuIiOwFM+Pao/JZsHID/56UeKN4BbyIiMheGtKjJb3aNua+D4ooS7BRvAJeRERkL23vqJ+/YgOvTF4cdjnfo4AXERHZB8f2bEWPNok3ilfAi4iI7IPo6nZdmLt8Pa9/+U3Y5eyggBcREdlHx/ZsTffWjbjn/a/ZVu5hlwMo4EVERPZZJBI9F19cup7Xv0yMc/EKeBERkWowtFdrurZqyL3vFyXEKF4BLyIiUg0iEeOao/IpWraOMVPDPxevgBcREakmJ/RuQ5eWDbn3/a8pD3kUr4AXERGpJikR45qjuvDV0nX8Z9qSUGtRwIuIiFSjk/ZvS+fszNBH8Qp4ERGRapQSOxc/a8la3p4R3iheAS8iIlLNTtq/DXktMrn7vaLQRvEKeBERkWqWmhLh6iO7MPObNbwzc2koNSjgRUREAjCsb1tymzfgnve+xr3mR/EKeBERkQCkpkS46sguTF+8hndnLqvx11fAi4iIBOS0A9rRISucUbwCXkREJCDbz8VPXbSaD2bX7CheAS8iIhKg0/q1I6dZfe5+t2ZH8Qp4ERGRAKXFzsVPKVnNh1+V1tjrKuBFREQCdka/HAbmZrG1rLzGXjPQgDezoWY228yKzOzGnexzhJlNNrPpZvbRnhwrIiJSG6SnRnhh5EEc26t1jb1malC/2MxSgPuBY4ASYLyZveruMyrs0xR4ABjq7gvMrGW8x4qIiMjOBTmCHwgUuXuxu28BngeGVdrnHOAld18A4O7L9uBYERER2YkgA74dsLDC45LYcxV1BZqZ2YdmNsHMLtiDYwEws8vNrNDMCktLa655QUREJJEFNkUPWBXPVb4+IBXoDwwB6gOfmtlncR4bfdL9YeBhgIKCgvDuyyciIpJAggz4EqB9hcc5wOIq9lnu7uuB9WY2FugT57EiIiKyE0FO0Y8H8s0sz8zSgbOBVyvt8wpwqJmlmlkDYBAwM85jRUREZCcCG8G7e5mZXQ28BaQAf3P36WY2MrZ9tLvPNLM3gS+BcuBRd58GUNWxQdUqIiKSbCyMW9gFpaCgwAsLC8MuQ0REpEaY2QR3L6hqm1ayExERSUIKeBERkSSkgBcREUlCCngREZEkpIAXERFJQknVRW9mpcD8avyVLYDl1fj7EpXeZ3LR+0wuep/JpbrfZ0d3z65qQ1IFfHUzs8KdXX6QTPQ+k4veZ3LR+0wuNfk+NUUvIiKShBTwIiIiSUgBv2sPh11ADdH7TC56n8lF7zO51Nj71Dl4ERGRJKQRvIiISBJSwFdiZu3N7AMzm2lm081sVNg1BcHMMszsCzObEnuft4ZdU5DMLMXMJpnZ62HXEhQzm2dmU81sspkl7V2XzKypmf3LzGbF/j89KOyaqpuZdYv9d9z+tcbMrgu7riCY2U9jf4OmmdlzZpYRdk1BMLNRsfc4vab+W2qKvhIzawO0cfeJZtYImACc6u4zQi6tWpmZAZnuvs7M0oBxwCh3/yzk0gJhZj8DCoDG7n5S2PUEwczmAQXuntTXEpvZk8B/3f1RM0sHGrj7qrDrCoqZpQCLgEHuXp3rfITOzNoR/dvT0903mtkLwBh3fyLcyqqXme0HPA8MBLYAbwJXuPvXQb6uRvCVuPs37j4x9vNaYCbQLtyqqp9HrYs9TIt9JeWnPTPLAU4EHg27Ftk3ZtYYOAx4DMDdtyRzuMcMAeYkW7hXkArUN7NUoAGwOOR6gtAD+MzdN7h7GfARcFrQL6qA3wUzywUOAD4Pt5JgxKatJwPLgHfcPSnfJ3AX8EugPOxCAubA22Y2wcwuD7uYgHQCSoHHY6dcHjWzzLCLCtjZwHNhFxEEd18E/AVYAHwDrHb3t8OtKhDTgMPMrLmZNQBOANoH/aIK+J0ws4bAi8B17r4m7HqC4O7b3L0vkAMMjE0jJRUzOwlY5u4Twq6lBgx2937A8cBVZnZY2AUFIBXoBzzo7gcA64Ebwy0pOLFTEKcA/wy7liCYWTNgGJAHtAUyzey8cKuqfu4+E/hf4B2i0/NTgLKgX1cBX4XYOekXgWfc/aWw6wlabIrzQ2BoyKUEYTBwSuz89PPAUWb2dLglBcPdF8e+LwNeJnq+L9mUACUVZpv+RTTwk9XxwER3Xxp2IQE5Gpjr7qXuvhV4CTg45JoC4e6PuXs/dz8MWAkEev4dFPA/EGs+ewyY6e7/F3Y9QTGzbDNrGvu5PtH/0WaFW1X1c/dfuXuOu+cSnep8392TboRgZpmxplBiU9bHEp0WTCruvgRYaGbdYk8NAZKqAbaSESTp9HzMAuBAM2sQ+9s7hGjfU9Ixs5ax7x2A06mB/66pQb9ALTQYOB+YGjs/DXCTu48JsaYgtAGejHXoRoAX3D1pLyGrA1oBL0f/RpIKPOvub4ZbUmCuAZ6JTV8XAxeHXE8gYudqjwF+EnYtQXH3z83sX8BEolPWk0jeFe1eNLPmwFbgKnf/NugX1GVyIiIiSUhT9CIiIklIAS8iIpKEFPAiIiJJSAEvIiKShBTwIiIiSUgBLyJ7zcxyzSzprrcXSQYKeBERkSSkgBeRamFmnWI3gBkQdi0iooAXkWoQWzr2ReBidx8fdj0ioqVqRWTfZQOvAGe4+/SwixGRKI3gRWRfrQYWEr2Pg4gkCI3gRWRfbQFOBd4ys3Xu/mzYBYmIAl5EqoG7rzezk4B3zGy9u78Sdk0idZ3uJiciIpKEdA5eREQkCSngRUREkpACXkREJAkp4EVERJKQAl5ERCQJKeBFRESSkAJeREQkCSngRUREktD/AzMIl1tYP13IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Visualizing the silhouette scores in a plot\n",
    "import matplotlib.pyplot as plt\n",
    "fig, ax = plt.subplots(1,1, figsize =(8,6))\n",
    "ax.plot(range(2,10),silhouette_score)\n",
    "ax.set_xlabel('k')\n",
    "ax.set_ylabel('silhouette')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "answering-child",
   "metadata": {},
   "source": [
    "## Bisecting Kmeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "sought-wales",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bkm = BisectingKMeans(featuresCol='standardized', k=3, seed=42)\n",
    "    \n",
    "bkm = bkm.fit(data_scale_output)\n",
    "    \n",
    "output= bkm.transform(data_scale_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "happy-dylan",
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_clusters = output.select(\"prediction\").distinct().rdd.map(lambda x: x[0]).collect()\n",
    "dd = {}\n",
    "for j in dist_clusters:\n",
    "    dd[j] = output.filter(output[\"prediction\"] == j).select(features_cluster+features_other+features_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "quantitative-bidder",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------- Statistiche cluster n°: 1 --------\n",
      "+-------+----------------------+\n",
      "|summary|Reviews_Installs_Ratio|\n",
      "+-------+----------------------+\n",
      "|count  |410427                |\n",
      "|mean   |0.11109354884040736   |\n",
      "|stddev |36.84925364123248     |\n",
      "|min    |1.2E-7                |\n",
      "|max    |21921.0               |\n",
      "+-------+----------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|Size              |\n",
      "+-------+------------------+\n",
      "|count  |410427            |\n",
      "|mean   |20.252285161583515|\n",
      "|stddev |24.598761694335344|\n",
      "|min    |9.765625E-4       |\n",
      "|max    |1126.4            |\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|Rating            |\n",
      "+-------+------------------+\n",
      "|count  |410427            |\n",
      "|mean   |3.4847814593092963|\n",
      "|stddev |0.4731042861559817|\n",
      "|min    |2.1               |\n",
      "|max    |4.0               |\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+-------------------+\n",
      "|summary|Price              |\n",
      "+-------+-------------------+\n",
      "|count  |410427             |\n",
      "|mean   |0.08384295484458759|\n",
      "|stddev |1.9010751494465001 |\n",
      "|min    |0.0                |\n",
      "|max    |399.99             |\n",
      "+-------+-------------------+\n",
      "\n",
      "----------- Statistiche cluster n°: 2 --------\n",
      "+-------+----------------------+\n",
      "|summary|Reviews_Installs_Ratio|\n",
      "+-------+----------------------+\n",
      "|count  |683742                |\n",
      "|mean   |0.6558914293919688    |\n",
      "|stddev |479.6900710217384     |\n",
      "|min    |1.4E-6                |\n",
      "|max    |396647.0              |\n",
      "+-------+----------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|Size              |\n",
      "+-------+------------------+\n",
      "|count  |683742            |\n",
      "|mean   |20.042391907571513|\n",
      "|stddev |24.88408941234997 |\n",
      "|min    |9.765625E-4       |\n",
      "|max    |1536.0            |\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|Rating            |\n",
      "+-------+------------------+\n",
      "|count  |683742            |\n",
      "|mean   |4.540873019355319 |\n",
      "|stddev |0.2909363811498064|\n",
      "|min    |4.1               |\n",
      "|max    |5.0               |\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+-------------------+\n",
      "|summary|Price              |\n",
      "+-------+-------------------+\n",
      "|count  |683742             |\n",
      "|mean   |0.09327306431226699|\n",
      "|stddev |1.9292513850603292 |\n",
      "|min    |0.0                |\n",
      "|max    |399.99             |\n",
      "+-------+-------------------+\n",
      "\n",
      "----------- Statistiche cluster n°: 0 --------\n",
      "+-------+----------------------+\n",
      "|summary|Reviews_Installs_Ratio|\n",
      "+-------+----------------------+\n",
      "|count  |995683                |\n",
      "|mean   |4.987050032791568E-4  |\n",
      "|stddev |0.09915073405380409   |\n",
      "|min    |0.0                   |\n",
      "|max    |95.0                  |\n",
      "+-------+----------------------+\n",
      "\n",
      "+-------+------------------+\n",
      "|summary|Size              |\n",
      "+-------+------------------+\n",
      "|count  |995683            |\n",
      "|mean   |17.725139069710345|\n",
      "|stddev |22.31116231246103 |\n",
      "|min    |9.765625E-4       |\n",
      "|max    |1126.4            |\n",
      "+-------+------------------+\n",
      "\n",
      "+-------+-------------------+\n",
      "|summary|Rating             |\n",
      "+-------+-------------------+\n",
      "|count  |995683             |\n",
      "|mean   |0.02611694685959294|\n",
      "|stddev |0.21215987059062014|\n",
      "|min    |0.0                |\n",
      "|max    |2.0                |\n",
      "+-------+-------------------+\n",
      "\n",
      "+-------+-------------------+\n",
      "|summary|Price              |\n",
      "+-------+-------------------+\n",
      "|count  |995683             |\n",
      "|mean   |0.09794875108744243|\n",
      "|stddev |2.6450119929385685 |\n",
      "|min    |0.0                |\n",
      "|max    |400.0              |\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def basic_statistics(df):\n",
    "    for i in df.dtypes:\n",
    "        if(i[1] == 'int' or i[1] == 'double' or i[1] == 'float' or i[1] == 'bigint'):\n",
    "            df.describe(i[0]).show(truncate=False)\n",
    "\n",
    "for i in dist_clusters:\n",
    "    print(\"----------- Statistiche cluster n°: \" + str(i) + \" --------\")\n",
    "    basic_statistics(dd[i].select(features_cluster))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "concerned-place",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------- Info categorici cluster n°: 1 --------\n",
      "<<-- feature:Ad_Supported\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|Ad_Supported|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|        true|                                                          57.30958245924367|\n",
      "|       false|                                                         42.690417540756336|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Category\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|        Category|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|   Music & Audio|                                                          4.377392325553631|\n",
      "|       Education|                                                          8.957500359381815|\n",
      "|          Trivia|                                                         0.6376286160510882|\n",
      "| Auto & Vehicles|                                                         1.0165023256267252|\n",
      "|   Entertainment|                                                          7.802849227755485|\n",
      "|       Adventure|                                                         1.3973252247050023|\n",
      "|          Arcade|                                                          2.085145470449069|\n",
      "|          Sports|                                                           2.34706780986631|\n",
      "|  Travel & Local|                                                         2.8818766796531414|\n",
      "|    Food & Drink|                                                         1.6034520146091753|\n",
      "|    Role Playing|                                                         0.7155961961566857|\n",
      "|         Finance|                                                          3.637431260613946|\n",
      "| Personalization|                                                          3.356260674858136|\n",
      "|          Racing|                                                           0.87762257356363|\n",
      "|           Tools|                                                          9.137556739688177|\n",
      "|     Educational|                                                         1.0783890923355433|\n",
      "|          Comics|                                                        0.19930462664493317|\n",
      "|          Social|                                                         1.4830895628211593|\n",
      "|Libraries & Demo|                                                        0.22245125198878243|\n",
      "|        Shopping|                                                          2.407492684448148|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Developer_Id\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|        Developer_Id|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|       Lotuz Studio.|                                                       4.872973756599834E-4|\n",
      "|   Rubega Developers|                                                       4.872973756599834E-4|\n",
      "|            Point.56|                                                       2.436486878299917E-4|\n",
      "|  avauniquecomputing|                                                       2.436486878299917E-4|\n",
      "|        MJ Acoustics|                                                       2.436486878299917E-4|\n",
      "|      Fabrice PASCAL|                                                       4.872973756599834E-4|\n",
      "|Chord Electronics...|                                                       2.436486878299917E-4|\n",
      "|  Eagle War App Zone|                                                       9.745947513199668E-4|\n",
      "|Apps FM Pro - Rad...|                                                       9.745947513199668E-4|\n",
      "|          Musik_Q id|                                                       0.001705540814809942|\n",
      "|                MFEQ|                                                       2.436486878299917E-4|\n",
      "|           Arab Unit|                                                       0.001218243439149...|\n",
      "|   Wondrous Apps LLP|                                                       7.309460634899751E-4|\n",
      "|           Ulduzsoft|                                                       4.872973756599834E-4|\n",
      "|KID Media Enterta...|                                                       2.436486878299917E-4|\n",
      "|  Professional amera|                                                       2.436486878299917E-4|\n",
      "|FLEX STATION Co.,...|                                                       2.436486878299917E-4|\n",
      "|          Brave Bird|                                                       0.002680135566129...|\n",
      "|      Rigolas Studio|                                                       7.309460634899751E-4|\n",
      "|             Judo id|                                                       2.436486878299917E-4|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Editors_Choice\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|Editors_Choice|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|          true|                                                       0.015106218645459486|\n",
      "|         false|                                                          99.98489378135454|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:In_App_Purchases\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|In_App_Purchases|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|            true|                                                         12.362978069181608|\n",
      "|           false|                                                          87.63702193081839|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Minimum_Android\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Minimum_Android|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|            8.0|                                                         0.5533261700619112|\n",
      "|            0.0|                                                        0.34793032622122816|\n",
      "|            7.0|                                                         1.4251011751176215|\n",
      "|            2.3|                                                          5.755712952607894|\n",
      "|            1.0|                                                        0.01729905683592941|\n",
      "|            3.1|                                                        0.10915461214783628|\n",
      "|            4.1|                                                         25.653283044244162|\n",
      "|            2.2|                                                         1.6558364824926235|\n",
      "|            4.0|                                                         15.498249384177942|\n",
      "|            5.1|                                                         2.3590065955699795|\n",
      "|            4.2|                                                          4.924627278419792|\n",
      "|            4.3|                                                          2.234745764776684|\n",
      "|            3.2|                                                        0.17250327098363413|\n",
      "|            1.6|                                                         0.4963123771096931|\n",
      "|            3.0|                                                          1.058897197309144|\n",
      "|            2.0|                                                        0.19272611207352344|\n",
      "|            1.1|                                                       0.012182434391499585|\n",
      "|            1.5|                                                        0.15715340365034464|\n",
      "|            4.4|                                                         17.191364115908552|\n",
      "|            6.0|                                                         3.3077745859799674|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<<-- feature:Year_of_Release\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Release|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|           2018|                                                         15.428078562082902|\n",
      "|           2015|                                                         7.7563123283799555|\n",
      "|           2013|                                                         3.9005718434703374|\n",
      "|           2014|                                                          5.943078793549157|\n",
      "|           2019|                                                         18.279986453132956|\n",
      "|           2020|                                                         17.646987162150637|\n",
      "|           2012|                                                         2.4474510692522666|\n",
      "|           2016|                                                          9.612184383581003|\n",
      "|           2010|                                                         0.4595214252473643|\n",
      "|           2011|                                                         1.4626230730434402|\n",
      "|           2017|                                                         13.096360619549882|\n",
      "|           2021|                                                          3.966844286560095|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Year_of_Last_Update\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Last_Update|((CAST(count(1) AS DOUBLE) / CAST(410427 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|               2018|                                                          9.897496997029922|\n",
      "|               2015|                                                         2.6594254276643596|\n",
      "|               2013|                                                         0.6242279382204386|\n",
      "|               2014|                                                           1.46408496517042|\n",
      "|               2019|                                                         16.873646226978245|\n",
      "|               2020|                                                         29.219081590636094|\n",
      "|               2012|                                                        0.21562908872954262|\n",
      "|               2009|                                                       4.872973756599834E-4|\n",
      "|               2016|                                                          3.882054543195258|\n",
      "|               2010|                                                       0.024852166158659154|\n",
      "|               2011|                                                        0.09502298825369676|\n",
      "|               2017|                                                          6.570717813399216|\n",
      "|               2021|                                                         28.473272957188488|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "\n",
      "----------- Info categorici cluster n°: 2 --------\n",
      "<<-- feature:Ad_Supported\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|Ad_Supported|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|        true|                                                          58.11797432364839|\n",
      "|       false|                                                          41.88202567635161|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Category\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|        Category|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|   Music & Audio|                                                          8.098376288132073|\n",
      "|       Education|                                                         12.156485925977927|\n",
      "|          Trivia|                                                         0.5850159855618055|\n",
      "| Auto & Vehicles|                                                         0.6005189091791934|\n",
      "|   Entertainment|                                                          5.979009626438042|\n",
      "|       Adventure|                                                         1.0735043335059131|\n",
      "|          Arcade|                                                          2.687417183674544|\n",
      "|          Sports|                                                           2.04463086953851|\n",
      "|  Travel & Local|                                                          2.449169423554499|\n",
      "|    Food & Drink|                                                         1.9619973615779052|\n",
      "|    Role Playing|                                                         0.4058548399835026|\n",
      "|         Finance|                                                         2.9208385619137043|\n",
      "| Personalization|                                                          5.661638454270763|\n",
      "|          Racing|                                                        0.45192484884649475|\n",
      "|           Tools|                                                          5.825735438220849|\n",
      "|     Educational|                                                         1.0182203228703224|\n",
      "|          Comics|                                                        0.10954424329644807|\n",
      "|          Social|                                                         2.0626201110945357|\n",
      "|Libraries & Demo|                                                         0.2075344208780505|\n",
      "|        Shopping|                                                         2.8601431534116672|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Developer_Id\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|        Developer_Id|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|  Professional amera|                                                       5.850159855618055E-4|\n",
      "|           Arab Unit|                                                       0.007020191826741666|\n",
      "|Proven Digital We...|                                                       0.002778825931418576|\n",
      "|      Rigolas Studio|                                                       0.004533873888103992|\n",
      "|           Netcandle|                                                       2.925079927809027...|\n",
      "|     Shotfine Studio|                                                       0.001901301953075868|\n",
      "|Ramadan Radio Wol...|                                                       1.462539963904513...|\n",
      "|            ForTunes|                                                       1.462539963904513...|\n",
      "|      ALG Movil Apps|                                                       2.925079927809027...|\n",
      "|Apps FM Pro - Rad...|                                                       0.003071333924199479|\n",
      "|      FreeAppsForAll|                                                       0.002340063942247222|\n",
      "|         Adeel Rezan|                                                       0.004095111898932639|\n",
      "|    Juan Pablo Sanso|                                                       7.312699819522568E-4|\n",
      "|          Musik_Q id|                                                       0.001755047956685...|\n",
      "|      Fabrice PASCAL|                                                       2.925079927809027...|\n",
      "|      Nada Salsabila|                                                       4.387619891713541E-4|\n",
      "|       The Encounter|                                                       1.462539963904513...|\n",
      "|            Napplics|                                                       1.462539963904513...|\n",
      "|    Real Voice Media|                                                       1.462539963904513...|\n",
      "|       application29|                                                       0.001023777974733...|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<<-- feature:Editors_Choice\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|Editors_Choice|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|          true|                                                        0.07020191826741666|\n",
      "|         false|                                                          99.92979808173259|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:In_App_Purchases\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|In_App_Purchases|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|            true|                                                         11.555528254809563|\n",
      "|           false|                                                          88.44447174519044|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Minimum_Android\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Minimum_Android|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|            8.0|                                                        0.47093786837725343|\n",
      "|            0.0|                                                         0.2897291668494842|\n",
      "|            7.0|                                                         1.2201970918855358|\n",
      "|            2.3|                                                          4.408241705204595|\n",
      "|            1.0|                                                        0.01184657370762656|\n",
      "|            3.1|                                                        0.09755141559243106|\n",
      "|            4.1|                                                         28.097440262555175|\n",
      "|            2.2|                                                         1.2321899195895527|\n",
      "|            4.0|                                                         15.227088580195453|\n",
      "|            5.1|                                                          2.269715769983415|\n",
      "|            4.2|                                                          6.203070748908214|\n",
      "|            4.3|                                                         1.7123417897394047|\n",
      "|            3.2|                                                        0.12972729479833037|\n",
      "|            1.6|                                                        0.45016980088980935|\n",
      "|            3.0|                                                         0.9061897616352366|\n",
      "|            2.0|                                                        0.17038590579487584|\n",
      "|            1.1|                                                       0.006288921844789409|\n",
      "|            1.5|                                                        0.09945271754550694|\n",
      "|            4.4|                                                         16.407358331066398|\n",
      "|            6.0|                                                         3.3110442242834286|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Year_of_Release\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Release|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|           2018|                                                         15.202956670791028|\n",
      "|           2015|                                                          6.941214668690822|\n",
      "|           2013|                                                         2.4623322832296393|\n",
      "|           2014|                                                          4.365389284262192|\n",
      "|           2019|                                                          18.44657780273846|\n",
      "|           2020|                                                         22.226512339449673|\n",
      "|           2012|                                                         1.5052461308505256|\n",
      "|           2016|                                                          9.313161982151161|\n",
      "|           2010|                                                        0.24512169795039648|\n",
      "|           2011|                                                         0.7984005662954741|\n",
      "|           2017|                                                          13.13902027372898|\n",
      "|           2021|                                                         5.3540662998616435|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Year_of_Last_Update\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Last_Update|((CAST(count(1) AS DOUBLE) / CAST(683742 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|               2018|                                                          9.122300516861623|\n",
      "|               2015|                                                         2.1623653366328233|\n",
      "|               2013|                                                         0.3992734101459322|\n",
      "|               2014|                                                         0.9611812642780463|\n",
      "|               2019|                                                         15.890642961818932|\n",
      "|               2020|                                                          30.69125488853983|\n",
      "|               2012|                                                        0.11831948307987517|\n",
      "|               2016|                                                         3.2671680253662934|\n",
      "|               2010|                                                       0.007166445823132117|\n",
      "|               2011|                                                        0.03934232502903142|\n",
      "|               2017|                                                          5.888039640683182|\n",
      "|               2021|                                                         31.452945701741303|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "\n",
      "----------- Info categorici cluster n°: 0 --------\n",
      "<<-- feature:Ad_Supported\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|Ad_Supported|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "|        true|                                                          41.45847624193644|\n",
      "|       false|                                                          58.54152375806356|\n",
      "+------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Category\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|        Category|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|   Music & Audio|                                                          6.916860084986888|\n",
      "|       Education|                                                         10.774614008675453|\n",
      "|          Trivia|                                                         0.4345760648720527|\n",
      "| Auto & Vehicles|                                                         0.8106997909977373|\n",
      "|   Entertainment|                                                          5.049599119398443|\n",
      "|       Adventure|                                                         0.6477965376530481|\n",
      "|          Arcade|                                                         2.0939395369811478|\n",
      "|          Sports|                                                         1.9171764507378353|\n",
      "|  Travel & Local|                                                         3.3490578828803947|\n",
      "|    Food & Drink|                                                         4.6507774060619695|\n",
      "|    Role Playing|                                                        0.11961638392942331|\n",
      "|         Finance|                                                          2.499088565336558|\n",
      "| Personalization|                                                         2.9304507559132773|\n",
      "|          Racing|                                                        0.25369520218784497|\n",
      "|           Tools|                                                            5.5193269343757|\n",
      "|     Educational|                                                          0.822149218174861|\n",
      "|          Comics|                                                        0.07120740235597073|\n",
      "|          Social|                                                          1.677541948592072|\n",
      "|Libraries & Demo|                                                        0.22788377425345213|\n",
      "|        Shopping|                                                          3.779616604883281|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<<-- feature:Developer_Id\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|        Developer_Id|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "|             Judo id|                                                       1.004335717291547...|\n",
      "|  Professional amera|                                                       0.001707370719395631|\n",
      "|Appgoo Apps - Rad...|                                                       0.019383679343726867|\n",
      "|         KERISH APPS|                                                       0.001004335717291...|\n",
      "|      Nada Salsabila|                                                       7.030350021040834E-4|\n",
      "|         Oualidosdev|                                                       0.004519510727811964|\n",
      "|          河豚沒了刺|                                                       1.004335717291547...|\n",
      "|            gururoga|                                                       2.008671434583095...|\n",
      "|Apps FM Pro - Rad...|                                                       0.002510839293228869|\n",
      "|        designer2018|                                                       1.004335717291547...|\n",
      "|              jahana|                                                       2.008671434583095...|\n",
      "|Electrifying Vent...|                                                       0.001807804291124...|\n",
      "|denis huaman - in...|                                                       1.004335717291547...|\n",
      "|Rocky Mountain Hi...|                                                       1.004335717291547...|\n",
      "|    Sistema Nossa FM|                                                       1.004335717291547...|\n",
      "|   Maxim Troschinsky|                                                       3.013007151874642...|\n",
      "|          Musik_Q id|                                                       1.004335717291547...|\n",
      "|         Adeel Rezan|                                                       2.008671434583095...|\n",
      "|      Wanky Massenat|                                                       5.021678586457738E-4|\n",
      "|    William Rinehart|                                                       1.004335717291547...|\n",
      "+--------------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Editors_Choice\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|Editors_Choice|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "|         false|                                                                      100.0|\n",
      "+--------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:In_App_Purchases\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|In_App_Purchases|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "|            true|                                                         3.5758368878448263|\n",
      "|           false|                                                          96.42416311215517|\n",
      "+----------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Minimum_Android\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Minimum_Android|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|            8.0|                                                         0.7173970028613524|\n",
      "|            0.0|                                                         0.4147906512414092|\n",
      "|            7.0|                                                         1.7218331537246292|\n",
      "|            2.3|                                                          2.846287422804246|\n",
      "|            1.0|                                                       0.013357665039977583|\n",
      "|            3.1|                                                        0.07542561236859523|\n",
      "|            4.1|                                                         26.020430197161144|\n",
      "|            2.2|                                                          0.760583438704889|\n",
      "|            4.0|                                                         14.570500852178855|\n",
      "|            5.1|                                                         2.9460179595312965|\n",
      "|            4.2|                                                          4.688640862603861|\n",
      "|            4.3|                                                         1.7261517973089828|\n",
      "|            3.2|                                                        0.07934252166603227|\n",
      "|            1.6|                                                         0.3093354009257967|\n",
      "|            3.0|                                                         0.5747813310059526|\n",
      "|            2.0|                                                        0.11308820176702827|\n",
      "|            1.1|                                                       0.005423412873374357|\n",
      "|            1.5|                                                         0.0621683809003468|\n",
      "|            4.4|                                                         18.586035917053923|\n",
      "|            6.0|                                                          4.675986232565988|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "<<-- feature:Year_of_Release\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Release|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "|           2018|                                                          14.85633479732003|\n",
      "|           2015|                                                         2.7644340618449847|\n",
      "|           2013|                                                         0.5766895688688066|\n",
      "|           2014|                                                         1.2017881193110658|\n",
      "|           2019|                                                         24.896678963083634|\n",
      "|           2020|                                                         28.632406097121272|\n",
      "|           2012|                                                         0.2703671750948846|\n",
      "|           2016|                                                          5.287727117968269|\n",
      "|           2010|                                                       0.019785413630643487|\n",
      "|           2011|                                                        0.10274354387892531|\n",
      "|           2017|                                                         10.090159217341261|\n",
      "|           2021|                                                         11.300885924536223|\n",
      "+---------------+---------------------------------------------------------------------------+\n",
      "\n",
      "<<-- feature:Year_of_Last_Update\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|Year_of_Last_Update|((CAST(count(1) AS DOUBLE) / CAST(995683 AS DOUBLE)) * CAST(100 AS DOUBLE))|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "|               2018|                                                         10.152628898956797|\n",
      "|               2015|                                                         1.0475221531350842|\n",
      "|               2013|                                                        0.12534109751798514|\n",
      "|               2014|                                                         0.3359502974340227|\n",
      "|               2019|                                                         21.595829194633232|\n",
      "|               2020|                                                           32.9251378199688|\n",
      "|               2012|                                                        0.05001591872111907|\n",
      "|               2016|                                                         2.3540624877596583|\n",
      "|               2010|                                                       0.002912573580145488|\n",
      "|               2011|                                                        0.01817847648297701|\n",
      "|               2017|                                                          5.730337868578654|\n",
      "|               2021|                                                          25.66208321323152|\n",
      "+-------------------+---------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in dist_clusters:\n",
    "    print(\"----------- Info categorici cluster n°: \" + str(i) + \" --------\")\n",
    "    for j in features_other:\n",
    "        dd[i].createOrReplaceTempView(\"tmp\")\n",
    "        tot_j = dd[i].select(j).count()\n",
    "        query = spark.sql(\"SELECT \"+str(j)+\", count(*)/\"+str(tot_j)+\" * 100 FROM tmp GROUP BY \"+ str(j) + \";\")\n",
    "        print(\"<<-- feature:\"+ str(j))\n",
    "        query.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "curious-training",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "israeli-basin",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "civil-python",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "utility-encounter",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}