2_main_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import itertools\n",
    "import re\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "# for tests of statistical significance\n",
    "import numpy as np\n",
    "import copy\n",
    "import random\n",
    "import statsmodels.stats.api as sms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/ios_signatures.json', encoding='utf-8') as fh:\n",
    "    signatures = json.load(fh)\n",
    "    \n",
    "with open('data_processed/ios_trackers.json', encoding='utf-8') as fh:\n",
    "    trackers = json.load(fh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.187897024406553"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apps = {}\n",
    "for app, v in trackers.items():\n",
    "    entry = {}\n",
    "    \n",
    "    for signature in signatures:\n",
    "        if signature['id'] in v:\n",
    "            entry[signature['name']] = True\n",
    "\n",
    "    apps[app] = entry\n",
    "\n",
    "apps = pd.DataFrame.from_dict(apps, orient='index').fillna(False)\n",
    "nontrackers = [\n",
    "    \"Twitter Account Kit\",\n",
    "    \"Google Firebase\",\n",
    "    \"Get device information\",\n",
    "    \"AdID access\",\n",
    "    \"Google Firebase Messaging\",\n",
    "    \"Google Consent API\",\n",
    "    \"YouTube\",\n",
    "    \"Google Sign-In\",\n",
    "    \"Google Maps\",\n",
    "    \"Amazon AWS\",\n",
    "    \"iAd\"\n",
    "]\n",
    "\n",
    "included_trackers = [\n",
    "    \"Alipay\",\n",
    "    \"Tencent Login\",\n",
    "    \"Google AdMob\",\n",
    "    \"Facebook\",\n",
    "    \"Facebook Ads\",\n",
    "    \"Facebook Analytics\",\n",
    "    \"Google CrashLytics\",\n",
    "    \"Google Analytics\",\n",
    "    \"Inmobi\",\n",
    "    \"Unity3d Ads\",\n",
    "    \"Moat\",\n",
    "    \"Twitter MoPub\",\n",
    "    \"AppLovin (MAX and SparkLabs)\",\n",
    "    \"ChartBoost\",\n",
    "    \"AdColony\",\n",
    "    \"Amazon Analytics (Amazon insights)\",\n",
    "    \"HockeyApp\",\n",
    "    \"Tapjoy\",\n",
    "    \"Branch\",\n",
    "    \"Microsoft Visual Studio App Center\",            # iOS\n",
    "    \"Microsoft Visual Studio App Center Crashes\",    # Android\n",
    "    \"Microsoft Visual Studio App Center Analytics\",  # Android\n",
    "    \"Adobe Experience Cloud\",\n",
    "    \"MixPanel\",\n",
    "    \"Adjust\",\n",
    "    \"Amplitude\",\n",
    "    \"Heyzap (bought by Fyber)\",\n",
    "    \"Amazon Advertisement\",\n",
    "    \"Vungle\",\n",
    "    \"AppsFlyer\",\n",
    "    \"WeChat\",          # iOS\n",
    "    \"Baidu\",           # iOS + Android\n",
    "    #\"Baidu APPX\",\n",
    "    #\"Baidu Location\",\n",
    "    \"Baidu Map\",\n",
    "    \"Baidu Mobile Ads\",\n",
    "    \"Baidu Mobile Stat\",\n",
    "    #\"Baidu Navigation\",\n",
    "    #\"Umeng+\",           # iOS + Android\n",
    "    #\"Umeng Analytics\",\n",
    "    #\"Umeng Feedback\",\n",
    "    \"JiGuang Aurora Mobile JPush\",\n",
    "    \"Tencent\",\n",
    "    \"Tencent Ads\",\n",
    "    \"Tencent MTA\",\n",
    "    \"Bugly\",\n",
    "    \"Tencent Map LBS\",\n",
    "    \"WeChat Location\",\n",
    "    \"ironSource\",\n",
    "    \"Startapp\",\n",
    "    \"Google Tag Manager\",\n",
    "    \"Pollfish\",\n",
    "    \"Nexage\",\n",
    "    \"Flurry\",\n",
    "    \"Verizon Ads\",\n",
    "    \"Revmob\",\n",
    "    \"New Relic\",\n",
    "    \"Supersonic Ads\",\n",
    "    \"Appodeal\",\n",
    "    \"Fyber\",\n",
    "    \"Smaato\",\n",
    "    \"Urbanairship\",\n",
    "    \"MobFox\",\n",
    "    \"Localytics\",\n",
    "    \"Appcelerator Analytics\",\n",
    "    \"AdBuddiz\",\n",
    "    \"Radius Networks\",\n",
    "    \"ComScore\",\n",
    "    \"Soomla\",\n",
    "    \"BugSense\",\n",
    "    \"Yandex Ad\",\n",
    "    \"Mail.ru\",\n",
    "    \"Quantcast\",\n",
    "    \"VKontakte SDK\",\n",
    "    \"Batch\",\n",
    "    \"Tapdaq\",\n",
    "    \"Fyber SponsorPay\",\n",
    "    \"Ooyala\",\n",
    "    \"Google Firebase Analytics\",\n",
    "    \"AdTech Mobile SDK\",\n",
    "    \"PlayHaven\",\n",
    "    \"WeiboSDK\",\n",
    "    #\"Weibo\",\n",
    "    \"AppMetrica\",\n",
    "    \"Mintegral\",\n",
    "    \"Ogury Presage\",\n",
    "    \"Appnext\",\n",
    "    \"myTarget\",\n",
    "    \"CleverTap\",\n",
    "    \"Sensors Analytics\",\n",
    "    \"Braze (formerly Appboy)\",\n",
    "    \"Bugsnag\",\n",
    "    \"Kochava\",\n",
    "    \"Pangle\",\n",
    "    \"SKAdNetwork\",\n",
    "    \"Google Play Services\",\n",
    "    \"Umeng Analytics\",\n",
    "    \"Umeng Social\",\n",
    "    \"Mob.com\",\n",
    "    \"Alibaba Cloud Utils\",\n",
    "    #\"Alibaba Cloud Push\",\n",
    "    #\"Alibaba Crash Reporting\",\n",
    "    \"Alibaba AutoNavi\",\n",
    "    \"Alibaba Analytics\",\n",
    "    \"Yueying\"\n",
    "]\n",
    "\n",
    "apps_nontrackers = apps[[col for col in apps if col not in included_trackers]]\n",
    "apps_trackers = apps[[col for col in apps if col in included_trackers]]\n",
    "apps_trackers = apps_trackers.rename({'WeiboSDK': 'Weibo'}, axis = 1)\n",
    "\n",
    "apps_trackers[\"Tencent\"] = apps_trackers[\"Tencent Ads\"] | apps_trackers[\"Tencent MTA\"]\n",
    "apps_trackers = apps_trackers.drop(['Tencent Ads', 'Tencent MTA'], axis = 1)\n",
    "\n",
    "#apps_nontrackers.mean() * 100\n",
    "apps_trackers.sum().sum() / len(apps) # mean trackers per app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SKAdNetwork                     69.600468\n",
       "Google Firebase Analytics       53.853226\n",
       "Google CrashLytics              31.812103\n",
       "Facebook                        25.534938\n",
       "Google AdMob                    22.826814\n",
       "Google Analytics                12.704781\n",
       "Unity3d Ads                      7.288532\n",
       "Google Tag Manager               7.263457\n",
       "Flurry                           5.240722\n",
       "Moat                             5.040120\n",
       "Twitter MoPub                    4.714142\n",
       "AppsFlyer                        4.672350\n",
       "Umeng+                           4.170846\n",
       "AppLovin (MAX and SparkLabs)     4.103979\n",
       "Amplitude                        3.560682\n",
       "Vungle                           3.485456\n",
       "ChartBoost                       3.443664\n",
       "Branch                           3.326647\n",
       "AdColony                         3.243063\n",
       "Supersonic Ads                   2.674691\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "apps_trackers.mean().sort_values(ascending=False)[0:20] * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "android = pd.read_csv(\"data/appsAndroid.csv\", index_col = 0, converters={'permissions':ast.literal_eval, 'trackersettings':ast.literal_eval})\n",
    "android = android[android.index.isin(pd.read_csv('data_processed/android_all_apps.csv').app)]\n",
    "ios = pd.read_csv(\"data/appsIos.csv\", index_col = 0, converters={'permissions':ast.literal_eval, 'trackersettings':ast.literal_eval})\n",
    "ios = ios[ios.index.isin(pd.read_csv('data_processed/ios_all_apps.csv').app)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "mlb = MultiLabelBinarizer()\n",
    "\n",
    "def expand(df,col):\n",
    "    return df.join(pd.DataFrame(mlb.fit_transform(df.get(col)),\n",
    "                              columns=mlb.classes_,\n",
    "                              index=df.index).astype(bool))\n",
    "\n",
    "#ios = expand(ios, 'trackers')\n",
    "ios = expand(ios, 'trackersettings')\n",
    "android = expand(android, 'trackersettings')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86.1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "android['AdID'].mean().round(3)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Get device information       99.9\n",
       "Google Firebase              40.8\n",
       "AdID access                  42.7\n",
       "Google Sign-In                8.1\n",
       "YouTube                      52.6\n",
       "iAd                          12.9\n",
       "Google Firebase Messaging    22.2\n",
       "Amazon AWS                    3.2\n",
       "WeChat                        5.0\n",
       "WeiboSDK                      2.2\n",
       "Tencent Login                 2.9\n",
       "Twitter Account Kit           2.8\n",
       "Google Maps                   8.5\n",
       "Tencent Map LBS               0.4\n",
       "Alipay                        1.8\n",
       "Parse                         1.3\n",
       "Google Consent API            0.8\n",
       "Sentry                        1.8\n",
       "Kakao                         0.4\n",
       "Sensors Analytics             0.2\n",
       "dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(apps_nontrackers.mean() * 100).round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9946332737030411"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(apps_nontrackers['AdID access'] & apps_trackers['AppsFlyer']).sum() / apps_trackers['AppsFlyer'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "android.permission.INTERNET                                    98.7\n",
       "android.permission.ACCESS_NETWORK_STATE                        94.5\n",
       "android.permission.WAKE_LOCK                                   70.0\n",
       "android.permission.WRITE_EXTERNAL_STORAGE                      63.3\n",
       "android.permission.READ_EXTERNAL_STORAGE                       41.4\n",
       "                                                               ... \n",
       "android.permission.SET_ANIMATION_SCALE                          0.0\n",
       "android.permission.SET_ALWAYS_FINISH                            0.0\n",
       "android.permission.REQUEST_COMPANION_USE_DATA_IN_BACKGROUND     0.0\n",
       "android.permission.REQUEST_COMPANION_RUN_IN_BACKGROUND          0.0\n",
       "android.permission.SIGNAL_PERSISTENT_PROCESSES                  0.0\n",
       "Name: permissions, Length: 125, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ios.permissions.explode().value_counts() / len(ios) * 100)[0:20].round(1).to_csv('data_processed/ios_top_permissions.csv')\n",
    "system_permissions = [\"android.permission.ACCEPT_HANDOVER\",\"android.permission.ACCESS_BACKGROUND_LOCATION\",\"android.permission.ACCESS_CHECKIN_PROPERTIES\",\"android.permission.ACCESS_COARSE_LOCATION\",\"android.permission.ACCESS_FINE_LOCATION\",\"android.permission.ACCESS_LOCATION_EXTRA_COMMANDS\",\"android.permission.ACCESS_MEDIA_LOCATION\",\"android.permission.ACCESS_NETWORK_STATE\",\"android.permission.ACCESS_NOTIFICATION_POLICY\",\"android.permission.ACCESS_WIFI_STATE\",\"android.permission.ACCOUNT_MANAGER\",\"android.permission.ACTIVITY_RECOGNITION\",\"com.android.voicemail.permission.ADD_VOICEMAIL\",\"android.permission.ANSWER_PHONE_CALLS\",\"android.permission.BATTERY_STATS\",\"android.permission.BIND_ACCESSIBILITY_SERVICE\",\"android.permission.BIND_APPWIDGET\",\"android.permission.BIND_AUTOFILL_SERVICE\",\"android.permission.BIND_CALL_REDIRECTION_SERVICE\",\"android.permission.BIND_CARRIER_MESSAGING_CLIENT_SERVICE\",\"android.permission.BIND_CARRIER_MESSAGING_SERVICE\",\"android.permission.BIND_CARRIER_SERVICES\",\"android.permission.BIND_CHOOSER_TARGET_SERVICE\",\"android.permission.BIND_COMPANION_DEVICE_SERVICE\",\"android.permission.BIND_CONDITION_PROVIDER_SERVICE\",\"android.permission.BIND_CONTROLS\",\"android.permission.BIND_DEVICE_ADMIN\",\"android.permission.BIND_DREAM_SERVICE\",\"android.permission.BIND_INCALL_SERVICE\",\"android.permission.BIND_INPUT_METHOD\",\"android.permission.BIND_MIDI_DEVICE_SERVICE\",\"android.permission.BIND_NFC_SERVICE\",\"android.permission.BIND_NOTIFICATION_LISTENER_SERVICE\",\"android.permission.BIND_PRINT_SERVICE\",\"android.permission.BIND_QUICK_ACCESS_WALLET_SERVICE\",\"android.permission.BIND_QUICK_SETTINGS_TILE\",\"android.permission.BIND_REMOTEVIEWS\",\"android.permission.BIND_SCREENING_SERVICE\",\"android.permission.BIND_TELECOM_CONNECTION_SERVICE\",\"android.permission.BIND_TEXT_SERVICE\",\"android.permission.BIND_TV_INPUT\",\"android.permission.BIND_VISUAL_VOICEMAIL_SERVICE\",\"android.permission.BIND_VOICE_INTERACTION\",\"android.permission.BIND_VPN_SERVICE\",\"android.permission.BIND_VR_LISTENER_SERVICE\",\"android.permission.BIND_WALLPAPER\",\"android.permission.BLUETOOTH\",\"android.permission.BLUETOOTH_ADMIN\",\"android.permission.BLUETOOTH_CONNECT\",\"android.permission.BLUETOOTH_PRIVILEGED\",\"android.permission.BLUETOOTH_SCAN\",\"android.permission.BODY_SENSORS\",\"android.permission.BROADCAST_PACKAGE_REMOVED\",\"android.permission.BROADCAST_SMS\",\"android.permission.BROADCAST_STICKY\",\"android.permission.BROADCAST_WAP_PUSH\",\"android.permission.CALL_COMPANION_APP\",\"android.permission.CALL_PHONE\",\"android.permission.CALL_PRIVILEGED\",\"android.permission.CAMERA\",\"android.permission.CAPTURE_AUDIO_OUTPUT\",\"android.permission.CHANGE_COMPONENT_ENABLED_STATE\",\"android.permission.CHANGE_CONFIGURATION\",\"android.permission.CHANGE_NETWORK_STATE\",\"android.permission.CHANGE_WIFI_MULTICAST_STATE\",\"android.permission.CHANGE_WIFI_STATE\",\"android.permission.CLEAR_APP_CACHE\",\"android.permission.CONTROL_LOCATION_UPDATES\",\"android.permission.DELETE_CACHE_FILES\",\"android.permission.DELETE_PACKAGES\",\"android.permission.DIAGNOSTIC\",\"android.permission.DISABLE_KEYGUARD\",\"android.permission.DUMP\",\"android.permission.EXPAND_STATUS_BAR\",\"android.permission.FACTORY_TEST\",\"android.permission.FOREGROUND_SERVICE\",\"android.permission.GET_ACCOUNTS\",\"android.permission.GET_ACCOUNTS_PRIVILEGED\",\"android.permission.GET_PACKAGE_SIZE\",\"android.permission.GET_TASKS\",\"android.permission.GLOBAL_SEARCH\",\"android.permission.HIDE_OVERLAY_WINDOWS\",\"android.permission.HIGH_SAMPLING_RATE_SENSORS\",\"android.permission.INSTALL_LOCATION_PROVIDER\",\"android.permission.INSTALL_PACKAGES\",\"com.android.launcher.permission.INSTALL_SHORTCUT\",\"android.permission.INSTANT_APP_FOREGROUND_SERVICE\",\"android.permission.INTERACT_ACROSS_PROFILES\",\"android.permission.INTERNET\",\"android.permission.KILL_BACKGROUND_PROCESSES\",\"android.permission.LOADER_USAGE_STATS\",\"android.permission.LOCATION_HARDWARE\",\"android.permission.MANAGE_DOCUMENTS\",\"android.permission.MANAGE_EXTERNAL_STORAGE\",\"android.permission.MANAGE_MEDIA\",\"android.permission.MANAGE_ONGOING_CALLS\",\"android.permission.MANAGE_OWN_CALLS\",\"android.permission.MASTER_CLEAR\",\"android.permission.MEDIA_CONTENT_CONTROL\",\"android.permission.MODIFY_AUDIO_SETTINGS\",\"android.permission.MODIFY_PHONE_STATE\",\"android.permission.MOUNT_FORMAT_FILESYSTEMS\",\"android.permission.MOUNT_UNMOUNT_FILESYSTEMS\",\"android.permission.NFC\",\"android.permission.NFC_PREFERRED_PAYMENT_INFO\",\"android.permission.NFC_TRANSACTION_EVENT\",\"android.permission.PACKAGE_USAGE_STATS\",\"android.permission.PERSISTENT_ACTIVITY\",\"android.permission.PROCESS_OUTGOING_CALLS\",\"android.permission.QUERY_ALL_PACKAGES\",\"android.permission.READ_CALENDAR\",\"android.permission.READ_CALL_LOG\",\"android.permission.READ_CONTACTS\",\"android.permission.READ_EXTERNAL_STORAGE\",\"android.permission.READ_INPUT_STATE\",\"android.permission.READ_LOGS\",\"android.permission.READ_PHONE_NUMBERS\",\"android.permission.READ_PHONE_STATE\",\"android.permission.READ_PRECISE_PHONE_STATE\",\"android.permission.READ_SMS\",\"android.permission.READ_SYNC_SETTINGS\",\"android.permission.READ_SYNC_STATS\",\"com.android.voicemail.permission.READ_VOICEMAIL\",\"android.permission.REBOOT\",\"android.permission.RECEIVE_BOOT_COMPLETED\",\"android.permission.RECEIVE_MMS\",\"android.permission.RECEIVE_SMS\",\"android.permission.RECEIVE_WAP_PUSH\",\"android.permission.RECORD_AUDIO\",\"android.permission.REORDER_TASKS\",\"android.permission.REQUEST_COMPANION_PROFILE_WATCH\",\"android.permission.REQUEST_COMPANION_RUN_IN_BACKGROUND\",\"android.permission.REQUEST_COMPANION_USE_DATA_IN_BACKGROUND\",\"android.permission.REQUEST_DELETE_PACKAGES\",\"android.permission.REQUEST_IGNORE_BATTERY_OPTIMIZATIONS\",\"android.permission.REQUEST_INSTALL_PACKAGES\",\"android.permission.REQUEST_OBSERVE_COMPANION_DEVICE_PRESENCE\",\"android.permission.REQUEST_PASSWORD_COMPLEXITY\",\"android.permission.RESTART_PACKAGES\",\"android.permission.SCHEDULE_EXACT_ALARM\",\"android.permission.SEND_RESPOND_VIA_MESSAGE\",\"android.permission.SEND_SMS\",\"com.android.alarm.permission.SET_ALARM\",\"android.permission.SET_ALWAYS_FINISH\",\"android.permission.SET_ANIMATION_SCALE\",\"android.permission.SET_DEBUG_APP\",\"android.permission.SET_PREFERRED_APPLICATIONS\",\"android.permission.SET_PROCESS_LIMIT\",\"android.permission.SET_TIME\",\"android.permission.SET_TIME_ZONE\",\"android.permission.SET_WALLPAPER\",\"android.permission.SET_WALLPAPER_HINTS\",\"android.permission.SIGNAL_PERSISTENT_PROCESSES\",\"android.permission.SMS_FINANCIAL_TRANSACTIONS\",\"android.permission.START_FOREGROUND_SERVICES_FROM_BACKGROUND\",\"android.permission.START_VIEW_PERMISSION_USAGE\",\"android.permission.STATUS_BAR\",\"android.permission.SYSTEM_ALERT_WINDOW\",\"android.permission.TRANSMIT_IR\",\"com.android.launcher.permission.UNINSTALL_SHORTCUT\",\"android.permission.UPDATE_DEVICE_STATS\",\"android.permission.UPDATE_PACKAGES_WITHOUT_USER_ACTION\",\"android.permission.USE_BIOMETRIC\",\"android.permission.USE_FINGERPRINT\",\"android.permission.USE_FULL_SCREEN_INTENT\",\"android.permission.USE_ICC_AUTH_WITH_DEVICE_IDENTIFIER\",\"android.permission.USE_SIP\",\"android.permission.VIBRATE\",\"android.permission.WAKE_LOCK\",\"android.permission.WRITE_APN_SETTINGS\",\"android.permission.WRITE_CALENDAR\",\"android.permission.WRITE_CALL_LOG\",\"android.permission.WRITE_CONTACTS\",\"android.permission.WRITE_EXTERNAL_STORAGE\",\"android.permission.WRITE_GSERVICES\",\"android.permission.WRITE_SECURE_SETTINGS\",\"android.permission.WRITE_SETTINGS\",\"android.permission.WRITE_SYNC_SETTINGS\",\"com.android.voicemail.permission.WRITE_VOICEMAIL\"]\n",
    "permissions_counts = android.permissions.explode().value_counts()\n",
    "permissions_counts = (permissions_counts[permissions_counts.index.isin(system_permissions)]/ len(android) * 100).round(1)\n",
    "permissions_counts.to_csv('data_processed/android_top_permissions.csv')\n",
    "permissions_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            Android   iOS\n",
      "Motion          1.4   8.0\n",
      "Contacts        6.4  16.1\n",
      "Bluetooth       8.4  17.2\n",
      "Calendar        3.2  25.2\n",
      "Microphone      9.6  26.2\n",
      "Location       28.0  49.2\n",
      "Camera         21.2  56.3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANoAAAFKCAYAAAB7DEEsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAabUlEQVR4nO3de3RV5Z3/8fdHpAIGpOUiVsCwLGopRTSB31SRAUZpq9bWseON4q2V1fnhbfpzHNvO6kBtq72sTqtWKZ0pU5VGZ5yq2Dq0lYBy0aqRmyCWgjhS71oqqFjE7++P8wQPIckJJOc5QT+vtVg559nPfvZ3Z/HJs/fOztmKCMysvPapdAFm7wUOmlkGDppZBg6aWQYOmlkGDppZBvtWuoDOrm/fvlFdXV3pMmwv0dDQ8FJE9Gva7qCVUF1dzSOPPFLpMmwvIemp5tp96GiWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloF/YV1Cw+YtqH5hpcuwTiQmHLfb63hGM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8sge9AkDZB0q6R1khok3SPpsNx1mOWUNWiSBNwBLIiIQyOiBvgycGCGbfuX81YxuWe08cC2iJjR2BARy4GlkuZJelTSSkmfBpBULWmNpP+Q9HtJsyUdL2mxpLWSRqd++0v6qaSHJC0tWv88SXMk1QPzJFU1tx2zcsv9U3440NBM+1bg1Ih4VVJf4EFJc9KyDwF/B1wAPAycDYwBTgG+AnwG+CpQHxEXSOoNPCTp3rT+0cCIiHglzWq7bCeaPIBA0hRgCsCgXlWsnXtbB+1+5ez3nesrXcJ7Wmc5nBLwLUljgbeBg3nncPLJiFgJIGkVMC8iQtJKoDr1mQicIuny9L4bMDi9/m1EvFJiO88VFxMRM4GZADUD+vspINZuuYO2CvhsM+2TgH5ATURsk7SBQlgA3izq93bR+7d5p34Bp0XEE8WDSvo/wGtt3I5Z2eQ+R6sH9kuHZgBIGgEcAryQ/vOPT+93x6+Bi9PFFiQd1UK/A9q5HbM9kjVo6VzoVOD4dHl/FXA1cA9Qmw4HzwHW7ObQVwFdgRVpzKta6De7ndsx2yPygwhbVzOgfyw55/RKl9FuvhiSh6SGiKht2u47Q8wycNDMMnDQzDJw0MwycNDMMugsd4Z0Who42FfsrN08o5ll4KCZZeCgmWXgoJll4KCZZeCgmWXgoJll4KCZZeCgmWXgoJll4KCZZeCgmWXgoJll4KCZZeCgmWXgoJll4KCZZeCgmWXgoJll4KCZZeAP5ymhYfMWVL+w0mVYB4kJx1Vku57RzDJw0MwycNDMMnDQzDJw0MwycNDMMih70CRtKdO4vSX936L3H5R0ezm2ZdZee/OM1hvYEbSIeCYiPlu5csxaVpGgSRop6UFJKyTdIen9qf1Dku6VtFzSo5IOlVQlaV56v1LSp9Mw1wCHSlom6buSqiU9lsbpJmlW6r9U0vjUfp6kX0iaK2mtpO9UYv/tvadSd4bcBFwcEfdJ+jrwL8BlwGzgmoi4Q1I3Cj8I/gKcGhGvSuoLPChpDnAlMDwiRgJIqi4afyoQEfFRSUcAv5F0WFo2EjgKeBN4QtJ1EfF0eXfX3uuyB03SAUDviLgvNf0M+C9JPYGDI+IOgIjYmvp3Bb4laSzwNnAwcGCJzYwBrkvjrJH0FNAYtHkR8ec09mrgEGCnoEmaAkwBGNSrirVzb2vHHr87+Blx7bM3nKNNAvoBNWn2eh7o1o7x3ix6vZ1mfthExMyIqI2I2n7du7djU2YF2YOWZpM/SWq8u3MycF9EbAY2SvoMgKT9JPUADgBeiIht6VzrkLTeZqBnC5tZSCGgpEPGwcAT5dgfs7bIcejYQ9LGovffB84FZqQgrQfOT8smAz9O523bgL+jcN52t6SVwCPAGoCIeFnS4nQB5H+AHxVt4wbgxrTOW8B5EfGmpLLtpFlrFBGVrqFTqxnQP5acc3qly6g4n6O1jaSGiKht2r43nKOZ7fUcNLMMHDSzDBw0swwcNLMMHDSzDPwpWCVo4GBf2rZ284xmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBP5ynhIbNW1D9wkqXkV1MOK50J2szz2hmGThoZhk4aGYZOGhmGThoZhk4aGYZlAyapJB0S9H7fSW9KOmX6f0pkq4sZ5GStpRzfLNya8vv0V4DhkvqHhFvACcAf2xcGBFzgDlt2ZgKT2tXRLy9J8Wa7a3aeuh4D3BSen0WUNe4QNJ5kq5Prw+UdIek5enfMZKqJT0h6SbgMWCQpO9KekzSSklnpHXHSbpf0q9S/xmS9inazjfTmA9KOjC1VUuql7RC0jxJg1P7f0i6VtISSeslfbZonH+U9HBaZ/qef+vM2q6tQbsVOFNSN2AE8LsW+l0L3BcRRwJHA6tS+1Dghoj4CFALjASOBI4HvivpoNRvNHAxMAw4FPjb1L4/8GAa937gwtR+HfCziBgBzE7bb3QQMAY4GbgGQNLEVMvoVEONpLFt/B6Y7bE23YIVESskVVOYze5ppesE4Jy0znbgz5LeDzwVEQ+mPmOAurT8eUn3AaOAV4GHImI9gKS61Pd24C/AL9P6DRQOXwE+xjthvBn4TlEtd6ZD1NWNMyAwMf1bmt5XUQje/cU7IWkKMAVgUK8q1s69rZVd3nN+7tp7x+7c6zgH+B4wDuizm9t5rY39ooX32yKi8fV22lb3m0WvVfT16oj4catFRMwEZgLUDOjftCaz3bY7l/d/CkyPiJWt9JkH/D2ApC6SDmimz0LgjLS8HzAWeCgtGy1pSDo3OwNYVKKmJcCZ6fWkNHZrfg1cIKkq1XiwpP4l1jFrtzYHLSI2RsS1JbpdCoyXtJLCId6wZvrcAawAlgP1wBUR8Vxa9jBwPfA48GTq25qLgfMlrQAmp+23tg+/AX4OPJBqvB3oWWIbZu2md47IKkvSOODyiDi5wqXspGZA/1hyzullGdvnaO8+khoiorZpu+8MMcug0/zhZ0QsABZUuAyzsvCMZpaBg2aWgYNmlkGnOUfrrDRwsK8OWrt5RjPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAn4JVQsPmLai+1ENq3j1iwnGVLuFdyTOaWQYOmlkGDppZBg6aWQYOmlkGDppZBmULmqQBkm6VtE5Sg6R7JB3WSv8tHbTdakmPdcRYZh2lLEGTJAoPel8QEYdGRA3wZeDAcmyvPST5d4lWduWa0cYD2yJiRmNDRCwHlkqaJ+lRSSslfbq5lSX9o6SHJa2QND21VUt6XNJPJK2S9BtJ3dOyGknLJS0HphaNUy1pYdreo5KOSe3jUvscYHWZvgdmO5Trp/lwoKGZ9q3AqRHxqqS+wIOS5kRENHaQNBEYCowGBMyRNBb439R+VkRcKOk/gdOAW4BZwEURcb+k7xZt7wXghIjYKmkoUAfUpmVHA8Mj4smmRUqaAkwBGNSrirVzb2txR/3sNGuL3IdNAr6VgvM2cDCFw8nnivpMTP+WpvdVFAL2v8CTEbEstTcA1ZJ6A70j4v7UfjPwyfS6K3C9pJHAdqD4HPGh5kIGEBEzgZkANQP6R3N9zHZHuYK2CvhsM+2TgH5ATURsk7QB6Nakj4CrI+LHOzVK1cCbRU3bge4l6vgH4HngSAqHyVuLlr1WYl2zDlOuc7R6YL90CAaApBHAIcALKWTj0/umfg1cIKkqrXewpP4tbSgiNgGbJI1JTZOKFh8APBsRbwOTgS7t2CezPVaWoKVzrlOB49Pl/VXA1cA9QK2klcA5wJpm1v0N8HPggdTvdqBniU2eD/xI0jIKM2KjG4Bz00WSI/AsZhWiousQ1oyaAf1jyTmnt7jcF0OsmKSGiKht2u47Q8wycNDMMnDQzDJw0MwycNDMMvANtSVo4GBfWbR284xmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmloGDZpaBP5ynhIbNW1D9wkqX0aqYcFylS7ASPKOZZeCgmWXgoJll4KCZZeCgmWXgoJll0GFBk7Rd0jJJyyU9KumY1F4t6bF2jHuZpB7tWP8rRa/bVYvZnurIGe2NiBgZEUcCX6bwzOqOcBmwx0EDvlK6i1l5levQsRfwp6aNks6TdH3R+19KGpdeT5T0QJoN/0tSlaRLgA8C8yXNT/3OkrRS0mOSvl001i7tkq4BuqeZdnbq2kXSTyStkvQbSd3L9D0w26Ejg9b4H3oN8G/AVW1dUVJf4J+B4yPiaOAR4EsRcS3wDDA+IsZL+iDwbWACMBIYJekzLbVHxJW8M9NOSpsbCvwoIj4CbAJOa++Om5XSkbdgvRERIwEkfQy4SdLwNq77V8AwYLEkgPcBDzTTbxSwICJeTNuZDYwFooX2O5sZ48mIWJZeNwDVTTtImgJMARjUq4q1c29r426U5metvTeV5V7HiHggzVL9mix6i51n0W7pq4DfRsRZ5ainiTeLXm8Hdjl0jIiZwEyAmgH9I0NN9i5XlnM0SUcAXYCXmyzaAIyUtI+kQcDo1P4gcKykD6X195d0WFq2GeiZXj8E/LWkvpK6AGcB97XSDrBNUtcO30mz3dCRM1p3ScvSawHnRsT2dCjYaDHwJLAaeBx4FCAiXpR0HlAnab/U95+B31OYWeZKeiadp10JzE/b+FVE3AXQUntaf4WkR4GvduD+mrWZInxk1JqaAf1jyTmnd9h4Pkd7d5PUEBG1Tdt9Z4hZBg6aWQYOmlkGDppZBg6aWQYOmlkG/hSsEjRwsC/JW7t5RjPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8BBM8vAQTPLwEEzy8AfzlNCw+YtqH5hpcvYRUw4rtIl2G7wjGaWgYNmloGDZpaBg2aWgYNmloGDZpZBWYMmaYCkWyWtk9Qg6Z6ih8DvzjiXSeqxhzWMlHTinqxr1lHKFjQVnhJ/B7AgIg6NiBrgy8CBezDcZcAeBQ0YCThoVlHlnNHGA9siYkZjQ0QsBxZJ+q6kxyStlHQGgKRxkhZIul3SGkmzVXAJ8EFgvqT5qe+Nkh6RtErS9MbxJY2StETSckkPSToA+DpwhqRlks6Q9Nfp9TJJSyX1LOP3wAwo750hw4GGZtr/lsIscyTQF3hY0v1p2VHAR4BngMXAsRFxraQvAeMj4qXU76sR8YqkLsA8SSOANcBtwBkR8bCkXsDrwNeA2oi4CEDS3cDUiFgsqQrY2uF7btZEJW7BGgPURcR24HlJ9wGjgFeBhyJiI4CkZUA1sKiZMU6XNIVC/QcBw4AAno2IhwEi4tU0TtN1FwPflzQb+EXj9oqlsacADOpVxdq5t+3Rjvq5ataonIeOq4Ca3VznzaLX22nmB4GkIcDlwN9ExAjgV0C3tm4gIq4BvgB0BxZLOqKZPjMjojYiavt1776bu2C2q3IGrR7YL80OAKRDvE0Uzpm6SOoHjAUeKjHWZqDxXKoX8BrwZ0kHAp9M7U8AB0kalbbVU9K+TdZF0qERsTIivg08DOwSNLOOVrZDx4gISacCP5D0TxTOhTZQuIJYBSyncLh3RUQ819zMUmQmMFfSMxExXtJSCudkT1M4FCQi/pIurFwnqTvwBnA8MB+4Mh2KXg2MkTQeeJvCrPs/HbvnZrtSRFS6hk6tZkD/WHLO6Xu0rs/R3nskNUREbdN23xliloGDZpaBg2aWgYNmloGDZpaBP5ynBA0c7KuH1m6e0cwycNDMMnDQzDJw0MwycNDMMnDQzDJw0MwycNDMMnDQzDJw0MwycNDMMnDQzDJw0MwycNDMMnDQzDLw36PZTrZt28bGjRvZutWflN6abt26MXDgQLp27dqm/g6a7WTjxo307NmT6urq5j5O3YCI4OWXX2bjxo0MGTKkTev40NF2snXrVvr06eOQtUISffr02a1Z30GzXThkpe3u98hBs07pzjvvRBJr1qzZrfUWLFjAySefvFvrzJgxg5tuummX9g0bNjB8+PDdGqslPkezVr15xUUdOl5bP+iorq6OMWPGUFdXx/Tp00uvUMJbb73Fvvs2/9/9i1/8YrvHL8VBK6Fh8xZUv7BDxooJx3XIOO92W7ZsYdGiRcyfP59PfepTTJ8+nQULFjBt2jT69u3LY489Rk1NDbfccguSmDt3Lpdddhk9evRgzJgxO8aZNm0a69atY/369QwePJirr76aCy64gJdeeol+/foxa9YsBg8ezLRp06iqquLyyy+noaGBCy64AICJEyd22D750NE6nbvuuotPfOITHHbYYfTp04eGhsKDY5cuXcoPfvADVq9ezfr161m8eDFbt27lwgsv5O6776ahoYHnnntup7FWr17NvffeS11dHRdffDHnnnsuK1asYNKkSVxyySW7bPv888/nuuuuY/ny5R26Tw6adTp1dXWceeaZAJx55pnU1dUBMHr0aAYOHMg+++zDyJEj2bBhA2vWrGHIkCEMHToUSXzuc5/baaxTTjmF7ulhkg888ABnn302AJMnT2bRop0fJrtp0yY2bdrE2LFjd/TpKD50tE7llVdeob6+npUrVyKJ7du3I4mTTjqJ/fbbb0e/Ll268NZbb5Ucb//99y9nuW3mGc06ldtvv53Jkyfz1FNPsWHDBp5++mmGDBnCwoXNnycfccQRbNiwgXXr1gHsmP2ac8wxx3DrrbcCMHv2bI47budz5t69e9O7d+8dM93s2bM7YpcAB806mbq6Ok499dSd2k477bQWA9StWzdmzpzJSSedxNFHH03//v1bHPu6665j1qxZjBgxgptvvpkf/vCHu/SZNWsWU6dOZeTIkXTkQzr9xM8SdPgRwY0/6ZCx9oarjo8//jgf/vCHK13GXqG571WnfeKnpJB0S9H7fSW9KOmXJdYbJ+mYovdflHROOWs121Od4WLIa8BwSd0j4g3gBOCPbVhvHLAFWAIQETPKVqFZO1V8RkvuAU5Kr88CdhyQS/qApDslrZD0oKQRkqqBLwL/IGmZpOMkTZN0eVpnZOq7QtIdkt6f2hdI+rakhyT9XlLnP5azd4XOMKMB3Ap8LR0ujgB+CjSGYDqwNCI+I2kCcFNEjJQ0A9gSEd8DkPQ3RePdBFwcEfdJ+jrwL8Bladm+ETFa0omp/fimxUiaAkwBGNSrirVzbwPafvuQWVOdYkaLiBVANYXZ7J4mi8cAN6d+9UAfSb1aGkvSAUDviLgvNf0MGFvU5Rfpa0PaZnP1zIyI2oio7Zd+2WnWHp1lRgOYA3yPwrlXnzJu5830dTuda//tXaxTzGjJT4HpEbGySftCYBIUrjQCL0XEq8BmoGfTQSLiz8Cfis6/JgP3Ne1nndcxx+y4mMyqVauYMGEChx9+OEOHDuWqq67a8fut559/npNPPpkjjzySYcOGceKJJ1aq5JI6zU/0iNgIXNvMomnATyWtAF4Hzk3tdwO3S/o0cHGTdc4FZkjqAawHzi9L0e8BHfWXC43a8rvEJUuWAPDGG29wyimncOONNzJx4kRef/11TjvtNG644QamTp3K1772NU444QQuvfRSAFasWNGhtXakigctIqqaaVsALEivXwE+00yf31O4cNJoYdGyZcBfNbPOuKLXL9HCOZpVVlVVFVu2bOHnP/85xx577I4/V+nRowfXX38948aNY+rUqTz77LM7/SnLiBEjWhqy4jrToaPZTlatWkVNTc1ObYceeihbtmzh1VdfZerUqXz+859n/PjxfPOb3+SZZ56pUKWlOWi21/r4xz/O+vXrufDCC1mzZg1HHXUUL774YqXLapaDZp3WsGHDdvzRZ6P169dTVVVFr16F3/B84AMf4Oyzz+bmm29m1KhR3H///ZUotSQHzTqtSZMmsWjRIu69916gcHHkkksu4YorrgCgvr6e119/HYDNmzezbt06Bg8eXLF6W+OgWafVvXt37rrrLr7xjW9w+OGH89GPfpRRo0Zx0UWFDwxqaGigtraWESNG8LGPfYwvfOELjBo1qsJVN89/JlNCbW1tPPLII5UuIxv/mUzb7VV/JmP2XuCgmWXgoJll4KDZLnzeXtrufo8cNNtJt27dePnllx22VjQ+tqlbt25tXqfi9zpa5zJw4EA2btzYae+w6CwaH0TYVg6a7aRr165tfrietZ0PHc0ycNDMMnDQzDLwLVglSNoMPFHpOvZQX+ClShfRDntj/YdERL+mjb4YUtoTzd27tjeQ9MjeWjvs/fUX86GjWQYOmlkGDlppMytdQDvszbXD3l//Dr4YYpaBZzSzDBy0Vkj6hKQnJP1B0pWVrqc1kgZJmi9ptaRVki5N7R+Q9FtJa9PX91e61pZI6iJpaeOz8SQNkfS79P2/TdL7Kl3jnnLQWiCpC/Aj4JPAMOAsScMqW1Wr3gL+X0QMo/DhsVNTvVcC8yJiKDAvve+sLgUeL3r/beBfI+JDwJ+Az1ekqg7goLVsNPCHiFgfEX+h8GipT1e4phZFxLMR8Wh6vZnCf9iDKdT8s9TtZzTzqc+dgaSBFJ6R92/pvYAJwO2pS6etvS0ctJYdDDxd9H5jauv00oMajwJ+BxwYEc+mRc8BB1aqrhJ+AFwBvJ3e9wE2RcRb6f1e8/1vjoP2LiOpCvhv4LL01J0donCJudNdZpZ0MvBCRDSU7LyX8i1YLfsjMKjo/UDa9mztipHUlULIZkdE4wMXn5d0UEQ8K+kg4IXKVdiiY4FT0lNYuwG9gB8CvSXtm2a1Tv/9b41ntJY9DAxNV77eB5xJ4WGJnVI6p/l34PGI+H7Rojm886irc4G7ctdWSkR8OSIGRkQ1he9zfURMAuYDn03dOmXtbeWgtSD9FL0I+DWFCwv/GRGrKltVq46l8NDFCZKWpX8nAtcAJ0haS+F53ddUssjd9E/AlyT9gcI5279XuJ495jtDzDLwjGaWgYNmloGDZpaBg2aWgYNmloGDZpaBg2aWgYNmlsH/B8rxcuDL/ByqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 180x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cross_platforms_perms = ['Motion', 'Contacts', 'Bluetooth', 'Calendar', 'Microphone', 'Location', 'Camera']\n",
    "colors = ['#f8756b', '#00bdc2']\n",
    "data = pd.DataFrame(\n",
    "    {\n",
    "        'Android': android[cross_platforms_perms].mean() * 100,\n",
    "        'iOS': ios[cross_platforms_perms].mean() * 100\n",
    "    }\n",
    ")\n",
    "ax = data.plot.barh(color=colors, figsize=(2.5,5.5));\n",
    "ax.figure.savefig('figures/cross-platform-permissions.pdf', bbox_inches='tight');\n",
    "print(data.round(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.0\n",
      "94.5\n",
      "97.6\n",
      "47.099999999999994\n"
     ]
    }
   ],
   "source": [
    "print(android[android.Calendar].permissions.apply(lambda x: \"android.permission.READ_CALENDAR\" in x).mean().round(3) * 100)\n",
    "print(android[android.Calendar].permissions.apply(lambda x: \"android.permission.WRITE_CALENDAR\" in x).mean().round(3) * 100)\n",
    "print(android[android.Contacts].permissions.apply(lambda x: \"android.permission.READ_CONTACTS\" in x).mean().round(3) * 100)\n",
    "print(android[android.Contacts].permissions.apply(lambda x: \"android.permission.WRITE_CONTACTS\" in x).mean().round(3) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def android_trackers(app):\n",
    "    for t in json.loads(app[\"exodus_analysis\"])['trackers']:\n",
    "        app[t['name']] = True\n",
    "    \n",
    "    return app\n",
    "\n",
    "android_trackers = android[[\"exodus_analysis\"]].apply(android_trackers, axis = 1).fillna(False).drop([\"exodus_analysis\"], axis = 1)\n",
    "android_trackers.to_csv('data_processed/android_trackers.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5m/w3frhhfs56sd392wfgn8yzz00000gn/T/ipykernel_46539/2643412632.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  apps_trackers_android['Facebook'] = apps_trackers_android['Facebook Ads'] | apps_trackers_android['Facebook Analytics']\n",
      "/var/folders/5m/w3frhhfs56sd392wfgn8yzz00000gn/T/ipykernel_46539/2643412632.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  apps_trackers_android[\"Microsoft Visual Studio App Center\"] = apps_trackers_android[\"Microsoft Visual Studio App Center Crashes\"] | apps_trackers_android[\"Microsoft Visual Studio App Center Analytics\"]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4.01515661839003"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "android_trackers = pd.read_csv('data_processed/android_trackers.json', index_col=\"app\")\n",
    "apps_trackers_android = android_trackers[[col for col in android_trackers if col in included_trackers]]\n",
    "apps_trackers_android['Facebook'] = apps_trackers_android['Facebook Ads'] | apps_trackers_android['Facebook Analytics']\n",
    "apps_trackers_android[\"Microsoft Visual Studio App Center\"] = apps_trackers_android[\"Microsoft Visual Studio App Center Crashes\"] | apps_trackers_android[\"Microsoft Visual Studio App Center Analytics\"]\n",
    "apps_trackers_android = apps_trackers_android.drop(['Facebook Ads',\n",
    "                                                    'Facebook Analytics',\n",
    "                                                    \"Microsoft Visual Studio App Center Crashes\",\n",
    "                                                    \"Microsoft Visual Studio App Center Analytics\"], axis = 1)\n",
    "\n",
    "apps_trackers_android = apps_trackers_android[apps_trackers_android.index.isin(android.index)]\n",
    "\n",
    "apps_trackers_android.mean().sort_values(ascending=False)[0:20] * 100\n",
    "apps_trackers_android.sum().sum() / len(android_trackers) # trackers mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86.1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "android['AdID'].mean().round(3) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Google AdMob\n",
      "100.0\n",
      "99.6\n",
      "Unity3d Ads\n",
      "99.9\n",
      "99.6\n",
      "Moat\n",
      "100.0\n",
      "100.0\n",
      "Inmobi\n",
      "94.7\n",
      "100.0\n"
     ]
    }
   ],
   "source": [
    "ad_trackers = ['Google AdMob', 'Unity3d Ads', 'Moat', 'Inmobi']\n",
    "for ad_tracker in ad_trackers:\n",
    "    print(ad_tracker)\n",
    "    print(((apps_nontrackers['AdID access'] & apps_trackers[ad_tracker]).sum() / apps_trackers[ad_tracker].sum() * 100).round(1))\n",
    "    print(((android['AdID'] & apps_trackers_android[ad_tracker]).sum() / apps_trackers_android[ad_tracker].sum() * 100).round(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Google AdMob\n",
      "53.4\n",
      "71.3\n",
      "Unity3d Ads\n",
      "17.0\n",
      "14.1\n",
      "Moat\n",
      "11.8\n",
      "8.7\n",
      "Inmobi\n",
      "4.9\n",
      "8.6\n"
     ]
    }
   ],
   "source": [
    "ad_trackers = ['Google AdMob', 'Unity3d Ads', 'Moat', 'Inmobi']\n",
    "for ad_tracker in ad_trackers:\n",
    "    print(ad_tracker)\n",
    "    print(((apps_nontrackers['AdID access'] & apps_trackers[ad_tracker]).sum() / apps_nontrackers['AdID access'].sum() * 100).round(1))\n",
    "    print(((android['AdID'] & apps_trackers_android[ad_tracker]).sum() / android['AdID'].sum() * 100).round(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['FB_Events_Delayed',\n",
       " 'Firebase_Analytics_Disabled',\n",
       " 'Firebase_Messaging_Delayed',\n",
       " 'FB_ADID_Disabled',\n",
       " 'Firebase_IDFV_Disabled',\n",
       " 'GAds_Init_Delayed',\n",
       " 'Firebase_Analytics_Deactivated',\n",
       " 'FB_Init_Delayed',\n",
       " 'Firebase_Analytics_Ads_Disabled']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(android['trackersettings'].explode().dropna().unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7507507507507507\n",
      "2.2336140607835957\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5m/w3frhhfs56sd392wfgn8yzz00000gn/T/ipykernel_46539/888906426.py:2: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  print(len(apps_trackers[apps_trackers['Google AdMob'] & ios['GAds_Init_Delayed']]) / sum(apps_trackers['Google AdMob']) * 100)\n"
     ]
    }
   ],
   "source": [
    "print(len(apps_trackers_android[apps_trackers_android['Google AdMob'] & android['GAds_Init_Delayed']]) / sum(apps_trackers_android['Google AdMob']) * 100)\n",
    "print(len(apps_trackers[apps_trackers['Google AdMob'] & ios['GAds_Init_Delayed']]) / sum(apps_trackers['Google AdMob']) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FB_Events_Delayed\n",
      "2.285027059530968\n",
      "4.615384615384616\n",
      "FB_Init_Delayed\n",
      "0.39085989176187613\n",
      "0.8837970540098199\n",
      "FB_ADID_Disabled\n",
      "0.9621166566446182\n",
      "3.044189852700491\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5m/w3frhhfs56sd392wfgn8yzz00000gn/T/ipykernel_46539/3932271214.py:6: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  print(len(apps_trackers[apps_trackers['Facebook'] & ios[setting]]) / sum(apps_trackers['Facebook']) * 100)\n"
     ]
    }
   ],
   "source": [
    "settings = ['FB_Events_Delayed','FB_Init_Delayed','FB_ADID_Disabled']\n",
    "\n",
    "for setting in settings:\n",
    "    print(setting)\n",
    "    print(len(apps_trackers_android[apps_trackers_android['Facebook'] & android[setting]]) / sum(apps_trackers_android['Facebook']) * 100)\n",
    "    print(len(apps_trackers[apps_trackers['Facebook'] & ios[setting]]) / sum(apps_trackers['Facebook']) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Firebase_Analytics_Deactivated\n",
      "0.5491585473870682\n",
      "0.4119214926096438\n",
      "Firebase_IDFV_Disabled\n",
      "1.204605845881311\n",
      "0.0726920281075842\n",
      "Firebase_Messaging_Delayed\n",
      "1.2223206377325067\n",
      "0.48461352071722796\n",
      "Firebase_Analytics_Disabled\n",
      "1.9840566873339238\n",
      "0.5088441967530893\n",
      "Firebase_Analytics_Ads_Disabled\n",
      "0.01771479185119575\n",
      "0.0484613520717228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/5m/w3frhhfs56sd392wfgn8yzz00000gn/T/ipykernel_46539/1110892005.py:10: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  print(len(apps_trackers[ios['Google Firebase'] & ios[setting]]) / sum(ios['Google Firebase']) * 100)\n"
     ]
    }
   ],
   "source": [
    "settings = ['Firebase_Analytics_Deactivated',\n",
    " 'Firebase_IDFV_Disabled',\n",
    " 'Firebase_Messaging_Delayed',\n",
    " 'Firebase_Analytics_Disabled',\n",
    " 'Firebase_Analytics_Ads_Disabled']\n",
    "\n",
    "for setting in settings:\n",
    "    print(setting)\n",
    "    print(len(apps_trackers_android[android['Google Firebase'] & android[setting]]) / sum(android['Google Firebase']) * 100)\n",
    "    print(len(apps_trackers[ios['Google Firebase'] & ios[setting]]) / sum(ios['Google Firebase']) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PlayHaven                              0.025075\n",
      "AdTech Mobile SDK                      0.033681\n",
      "Fyber SponsorPay                       0.050522\n",
      "BugSense                               0.075226\n",
      "Ooyala                                 0.083584\n",
      "WeChat Location                        0.100301\n",
      "Mail.ru                                0.134725\n",
      "Soomla                                 0.142093\n",
      "Verizon Ads                            0.151566\n",
      "Batch                                  0.159987\n",
      "Quantcast                              0.200602\n",
      "Tapdaq                                 0.202088\n",
      "Appcelerator Analytics                 0.244190\n",
      "AdBuddiz                               0.252610\n",
      "Radius Networks                        0.252610\n",
      "Revmob                                 0.336814\n",
      "Braze (formerly Appboy)                0.367770\n",
      "Appodeal                               0.395756\n",
      "VKontakte SDK                          0.401204\n",
      "CleverTap                              0.496800\n",
      "MobFox                                 0.606265\n",
      "Nexage                                 0.614685\n",
      "Yandex Ad                              0.665207\n",
      "Kochava                                0.693748\n",
      "Adobe Experience Cloud                 0.727182\n",
      "ComScore                               0.760615\n",
      "Localytics                             0.810766\n",
      "Pangle                                 0.844199\n",
      "Amazon Analytics (Amazon insights)     0.844199\n",
      "Heyzap (bought by Fyber)               0.867295\n",
      "Pollfish                               0.875716\n",
      "Urbanairship                           0.952859\n",
      "Mintegral                              1.052543\n",
      "Bugsnag                                1.069876\n",
      "Smaato                                 1.237044\n",
      "myTarget                               1.279892\n",
      "Tencent                                1.313574\n",
      "Fyber                                  1.431458\n",
      "Baidu                                  1.462721\n",
      "AppMetrica                             1.473560\n",
      "New Relic                              1.830491\n",
      "Adjust                                 1.880642\n",
      "HockeyApp                              2.081244\n",
      "Bugly                                  2.265129\n",
      "MixPanel                               2.281846\n",
      "Microsoft Visual Studio App Center     2.315279\n",
      "JiGuang Aurora Mobile JPush            2.315279\n",
      "Tapjoy                                 2.450320\n",
      "Supersonic Ads                         2.674691\n",
      "Amazon Advertisement                   3.081846\n",
      "Branch                                 3.326647\n",
      "Amplitude                              3.560682\n",
      "Umeng+                                 4.170846\n",
      "ironSource                             4.546986\n",
      "Vungle                                 4.648030\n",
      "AppsFlyer                              4.672350\n",
      "Twitter MoPub                          4.714142\n",
      "Startapp                               5.245874\n",
      "ChartBoost                             5.346918\n",
      "AdColony                               5.490064\n",
      "AppLovin (MAX and SparkLabs)           6.896261\n",
      "Inmobi                                 7.426743\n",
      "Moat                                   7.502526\n",
      "Flurry                                10.575951\n",
      "Unity3d Ads                           12.175817\n",
      "Google Tag Manager                    14.937689\n",
      "Google Analytics                      19.349949\n",
      "Facebook                              28.006063\n",
      "Google CrashLytics                    31.812103\n",
      "Google Firebase Analytics             57.645672\n",
      "Google AdMob                          61.687437\n",
      "SKAdNetwork                           69.600468\n",
      "Google Play Services                  87.318963\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "merged = pd.concat([apps_trackers.mean().rename('iOS'), apps_trackers_android.mean().rename('Android')], axis=1)\n",
    "\n",
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "    print((merged * 100).max(axis=1).sort_values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "companies = {\n",
    "    \"Google AdMob\": \"AdMob\",\n",
    "    \"Facebook\": \"Facebook\",\n",
    "    \"Google CrashLytics\": \"Crashlytics\",\n",
    "    \"Google Analytics\": \"Google Analytics\",\n",
    "    \"Inmobi\": \"InMobi\",\n",
    "    \"Unity3d Ads\": \"Unity Technologies\",\n",
    "    \"Moat\": \"Moat\",\n",
    "    \"Twitter MoPub\": \"MoPub\",\n",
    "    \"AppLovin (MAX and SparkLabs)\": \"AppLovin\",\n",
    "    \"ChartBoost\": \"Chartboost\",\n",
    "    \"AdColony\": \"AdColony\",\n",
    "    \"Amazon Analytics (Amazon insights)\": \"Amazon Analytics\",\n",
    "    \"HockeyApp\": \"Bit Stadium\",\n",
    "    \"Tapjoy\": \"Tapjoy\",\n",
    "    \"Branch\": \"Branch\",\n",
    "    \"Microsoft Visual Studio App Center\": \"App Center\",\n",
    "    \"Adobe Experience Cloud\": \"Adobe Experience Cloud\",\n",
    "    \"MixPanel\": \"Mixpanel\",\n",
    "    \"Adjust\": \"Adjust\",\n",
    "    \"Amplitude\": \"Amplitude\",\n",
    "    \"Heyzap (bought by Fyber)\": \"Heyzap\",\n",
    "    \"Amazon Advertisement\": \"Amazon Advertising\",\n",
    "    \"Vungle\": \"Vungle\",\n",
    "    \"AppsFlyer\": \"AppsFlyer\",\n",
    "    \"WeChat\": \"Tencent\",\n",
    "    \"Baidu\": \"Baidu\",\n",
    "    \"Umeng+\": \"Umeng+\",\n",
    "    \"JiGuang Aurora Mobile JPush\": \"JPush\",\n",
    "    \"Tencent MTA\": \"Tencent\",\n",
    "    \"Bugly\": \"Tencent\",\n",
    "    \"Tencent Map LBS\": \"Tencent\",\n",
    "    \"WeChat Location\": \"Tencent\",\n",
    "    \"ironSource\": \"ironSource\",\n",
    "    \"Startapp\": \"StartApp\",\n",
    "    \"Google Tag Manager\": \"Google Tag Manager\",\n",
    "    \"Pollfish\": \"Pollfish\",\n",
    "    \"Nexage\": \"NEXAGE\",\n",
    "    \"Flurry\": \"Flurry\",\n",
    "    \"Verizon Ads\": \"Verizon Media\",\n",
    "    \"Revmob\": \"RevMob\",\n",
    "    \"New Relic\": \"New Relic\",\n",
    "    \"Supersonic Ads\": \"Supersonic Studios\",\n",
    "    \"Appodeal\": \"Appodeal\",\n",
    "    \"Fyber\": \"Fyber\",\n",
    "    \"Smaato\": \"Smaato\",\n",
    "    \"Urbanairship\": \"Airship\",\n",
    "    \"MobFox\": \"Mobfox\",\n",
    "    \"Localytics\": \"Localytics\",\n",
    "    \"Appcelerator Analytics\": \"Appcelerator\",\n",
    "    \"AdBuddiz\": \"AdBuddiz\",\n",
    "    \"Radius Networks\": \"Radius Networks\",\n",
    "    \"ComScore\": \"comScore\",\n",
    "    \"Soomla\": \"Soomla\",\n",
    "    \"BugSense\": \"BugSense\",\n",
    "    \"Yandex Ad\": \"Yandex\",\n",
    "    \"Mail.ru\": \"Mail.ru\",\n",
    "    \"Quantcast\": \"Quantcast\",\n",
    "    \"VKontakte SDK\": \"VKontakte\",\n",
    "    \"Batch\": \"Batch\",\n",
    "    \"Tapdaq\": \"Tapdaq\",\n",
    "    \"Fyber SponsorPay\": \"Fyber\",\n",
    "    \"Ooyala\": \"Ooyala - Flex Media Platform\",\n",
    "    \"Google Firebase Analytics\": \"Firebase\",\n",
    "    \"AdTech Mobile SDK\": \"AdTech\",\n",
    "    \"PlayHaven\": \"PlayHaven\",\n",
    "    \"iAd\": \"Apple\",\n",
    "    \"WeiboSDK\": \"Weibo\",\n",
    "    \"SKAdNetwork\": \"Apple\",\n",
    "    \"CleverTap\": \"CleverTap\",\n",
    "    \"Braze (formerly Appboy)\": \"Braze\",\n",
    "    \"Bugsnag\": \"Bugsnag\",\n",
    "    \"myTarget\": \"My.com\",\n",
    "    \"Tencent\": \"Tencent\",\n",
    "    \"Kochava\": \"Kochava\",\n",
    "    \"Mintegral\": \"Mintegral\",\n",
    "    \"Pangle\": \"Pangle\",\n",
    "    \"AppMetrica\": \"Yandex\",\n",
    "    \"Google Play Services\": \"Google\",\n",
    "    \"Alipay\": \"Alibaba\",\n",
    "    \"Baidu Map\": \"Baidu\",\n",
    "    \"Baidu Mobile Stat\": \"Baidu\",\n",
    "    \"Baidu Mobile Ads\": \"Baidu\",\n",
    "    \"Umeng Analytics\": \"Umeng+\",\n",
    "    \"Umeng Social\": \"Umeng+\",\n",
    "    \"Mob.com\": \"MobTech\",\n",
    "    \"Alibaba Cloud Utils\": \"Alibaba\",\n",
    "    \"Alibaba Cloud Push\": \"Alibaba\",\n",
    "    \"Alibaba Crash Reporting\": \"Alibaba\",\n",
    "    \"Alibaba AutoNavi\": \"Alibaba\",\n",
    "    \"Alibaba Analytics\": \"Alibaba\",\n",
    "    \"Yueying\": \"Alibaba\",\n",
    "    'Tencent Login': 'Tencent',\n",
    "    'Sensors Analytics': 'Sensors Data'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "iosIds = {}\n",
    "for appId,dbId in ios.id.items():\n",
    "    iosIds[appId] = dbId\n",
    "    \n",
    "androidIds = {}\n",
    "for appId,dbId in android.id.items():\n",
    "    androidIds[appId] = dbId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "hosts = []\n",
    "\n",
    "significant_trackers = merged[(merged * 100).max(axis=1) >= 1].index\n",
    "significant_cols_android = [col for col in apps_trackers_android if col in significant_trackers]\n",
    "\n",
    "def host_list_android(app):\n",
    "    appId = app.name\n",
    "    for k,v in app.items():\n",
    "        if v == True:\n",
    "            hosts.append(str(androidIds[appId]) + \";\" + k + \";\" + companies[k])\n",
    "\n",
    "apps_trackers_android[significant_cols_android].apply(host_list_android, axis = 1);\n",
    "with open('data_processed/android_trackers.csv', 'w', encoding='utf-8') as f:\n",
    "    f.write(\"id;hosts;company\\n\" + \"\\n\".join(hosts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "hosts = []\n",
    "\n",
    "significant_cols = [col for col in apps_trackers if col in significant_trackers]\n",
    "\n",
    "def host_list_ios(app):\n",
    "    appId = app.name\n",
    "    for k,v in app.items():\n",
    "        if v == True:\n",
    "            hosts.append(str(iosIds[appId]) + \";\" + k + \";\" + companies[k])\n",
    "    \n",
    "apps_trackers[significant_cols].apply(host_list_ios, axis = 1);\n",
    "with open('data_processed/ios_trackers.csv', 'w', encoding='utf-8') as f:\n",
    "    f.write(\"id;hosts;company\\n\" + \"\\n\".join(hosts))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison of All/Cross-Platform/Kids apps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "android_fuzzy = android[android.index.isin(pd.read_csv('data_processed/android_fuzzy_apps.csv').app)]\n",
    "ios_fuzzy = ios[ios.index.isin(pd.read_csv('data_processed/ios_fuzzy_apps.csv').app)]\n",
    "\n",
    "android_kids = android[~android.family_genre.isnull()]\n",
    "ios_kids = ios[~ios.family_genre.isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "370"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(android_kids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "109"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ios_kids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_total = 10000\n",
    "\n",
    "def conf(a):\n",
    "    return sms.DescrStatsW(a).tconfint_mean()\n",
    "\n",
    "def permTest(df1, df2):\n",
    "    gT = np.abs(np.average(df1[:]) - np.average(df2[:]))\n",
    "    pV = list(df1[:]) + list(df2[:])\n",
    "\n",
    "    #Copy pooled distribution:\n",
    "    pS = np.array(copy.copy(pV))\n",
    "    #Initialize permutation:\n",
    "    pD = []\n",
    "    #Define p (number of permutations):\n",
    "    # Permutation loop:\n",
    "    for i in range(0,p_total):\n",
    "        np.random.shuffle(pS)\n",
    "        # Compute permuted absolute difference of your two sampled distributions and store it in pD:\n",
    "        #pD.append(np.abs(np.average(pS[0:int(len(pS)/2)]) - np.average(pS[int(len(pS)/2):])))\n",
    "        pD.append(np.abs(np.average(pS[0:len(df1)]) - np.average(pS[len(df1)+1:])))\n",
    "\n",
    "    p_val = len(np.where(pD>=gT)[0])/p_total\n",
    "    \n",
    "    if p_val > 0.1:\n",
    "        return (p_val, conf(df1), conf(df2))\n",
    "    else:\n",
    "        return p_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [android,android_fuzzy,android_kids,ios,ios_fuzzy,ios_kids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.0\n",
      "14.3\n",
      "6.9\n",
      "3.7\n",
      "4.0\n",
      "2.7\n"
     ]
    }
   ],
   "source": [
    "for df in datasets:\n",
    "    print(df.permissions.apply(len).mean().round(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8\n",
      "1.2\n",
      "0.2\n",
      "2.0\n",
      "2.1\n",
      "1.4\n"
     ]
    }
   ],
   "source": [
    "for df in datasets:\n",
    "    print(df[cross_platforms_perms].sum(axis=1).mean().round(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28.000000000000004\n",
      "41.099999999999994\n",
      "3.8\n",
      "49.2\n",
      "53.1\n",
      "26.6\n"
     ]
    }
   ],
   "source": [
    "for df in datasets:\n",
    "    print(df['Location'].mean().round(3) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86.1\n",
      "84.39999999999999\n",
      "89.7\n",
      "42.699999999999996\n",
      "49.9\n",
      "50.5\n"
     ]
    }
   ],
   "source": [
    "for df in datasets[0:3]:\n",
    "    print(df['AdID'].mean().round(3) * 100)\n",
    "    \n",
    "for df in datasets[3:6]:\n",
    "    print(df.join(apps_nontrackers[['AdID access']])['AdID access'].mean().round(3) * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "android_adid_shared = pd.read_csv('data_processed/android_ad_id_shared.txt', header=None, names=['app']).app.apply(lambda x: x[12:-4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "ios_adid_shared = pd.read_csv('data_processed/ios_ad_id_shared.txt', header=None, names=['app']).app.apply(lambda x: x[12:-5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.4\n",
      "64.3\n",
      "59.2\n",
      "30.9\n",
      "38.3\n",
      "24.8\n"
     ]
    }
   ],
   "source": [
    "for df in datasets[0:3]:\n",
    "    print(round(df.index.isin(android_adid_shared).sum() / len(df) * 100, 1))\n",
    "    \n",
    "for df in datasets[3:6]:\n",
    "    print(round(df.index.isin(ios_adid_shared).sum() / len(df) * 100, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(android_kids.index).to_csv('data_processed/android_kids_apps.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.Series(ios_kids.index).to_csv('data_processed/ios_kids_apps.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Permutation tests: All v Cross-Platform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_root_companies = pd.read_csv('results/all_root_companies.csv')\n",
    "fuzzy_root_companies = pd.read_csv('results/fuzzy_root_companies.csv')\n",
    "kids_root_companies = pd.read_csv('results/kids_root_companies.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(all_root_companies[all_root_companies.crawl == 'iOS'].num_companies, fuzzy_root_companies[fuzzy_root_companies.crawl == 'iOS'].num_companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(all_root_companies[all_root_companies.crawl == 'Android'].num_companies, fuzzy_root_companies[fuzzy_root_companies.crawl == 'Android'].num_companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0019"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.permissions.apply(len), ios_fuzzy.permissions.apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android.permissions.apply(len), android_fuzzy.permissions.apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0055"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios[cross_platforms_perms].sum(axis=1), ios_fuzzy[cross_platforms_perms].sum(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android[cross_platforms_perms].sum(axis=1), android_fuzzy[cross_platforms_perms].sum(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.join(apps_nontrackers[['AdID access']])['AdID access'], ios_fuzzy.join(apps_nontrackers[['AdID access']])['AdID access'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0672"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android['AdID'], android_fuzzy['AdID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.index.isin(ios_adid_shared), ios_fuzzy.index.isin(ios_adid_shared))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android.index.isin(android_adid_shared), android_fuzzy.index.isin(android_adid_shared))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Permutation tests: All v Kids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4129,\n",
       " (2.552173825649671, 2.639066562180486),\n",
       " (2.052143813283159, 2.755195636258125))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(all_root_companies[all_root_companies.crawl == 'iOS'].num_companies, kids_root_companies[kids_root_companies.crawl == 'iOS'].num_companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0134"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(all_root_companies[all_root_companies.crawl == 'Android'].num_companies, kids_root_companies[kids_root_companies.crawl == 'Android'].num_companies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0011"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.permissions.apply(len), ios_kids.permissions.apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android.permissions.apply(len), android_kids.permissions.apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1739,\n",
       " (0.30114397412728217, 0.3177125955818452),\n",
       " (0.16536998026835048, 0.3300428637683468))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.index.isin(ios_adid_shared), ios_kids.index.isin(ios_adid_shared))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.1469,\n",
       " (0.5447800073730408, 0.562663576325174),\n",
       " (0.5415800158562154, 0.6422037679275683))"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android.index.isin(android_adid_shared), android_kids.index.isin(android_adid_shared))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios['Location'], ios_kids['Location'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(android['Location'], android_kids['Location'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios_kids.index.isin(ios_adid_shared), android_kids.index.isin(android_adid_shared))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios['Location'], android_kids['Location'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Permutation test: Android v iOS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "permTest(ios.index.isin(ios_adid_shared), android.index.isin(android_adid_shared))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Furhter references:\n",
    "- https://42matters.com/sdk-analysis/top-analytics-sdks\n",
    "- https://42matters.com/sdk-analysis/top-ad-network-sdks\n",
    "- https://www.appsight.io/\n",
    "- https://github.com/CocoaPods/Specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}