Talk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import read_csv, DataFrame\n",
    "from requests import get\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "from os.path import isfile\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import geopandas as gpd\n",
    "\n",
    "data_url = \"http://opendata.ecdc.europa.eu/covid19/casedistribution/csv/\"\n",
    "local_file = \"snapshot-{0}.csv\".format(datetime.today().strftime(\"%d-%m-%Y\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First step, retrieve a fresh snapsot of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not isfile(local_file):\n",
    "    data = get(data_url)\n",
    "    with open(local_file, \"wb\") as out_f:\n",
    "        out_f.write(data.content)\n",
    "    print(\"Got new one\")\n",
    "else:\n",
    "    print(\"We have snapshot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we can load this dataset and make a sneak peek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# category and sparse types\n",
    "dtypes_dict = {\"geoId\": \"category\", \"countryterritoryCode\": \"category\", \"popData2018\": \"Int64\"}\n",
    "dparser = lambda val: datetime.strptime(val, \"%d/%m/%Y\")\n",
    "frame = read_csv(local_file, dtype=dtypes_dict, parse_dates=[\"dateRep\"], date_parser=dparser)\n",
    "frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.memory_usage().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can optimise memory consumption, using proper dtypes, for example, country/territory name can be category too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.countriesAndTerritories = frame.countriesAndTerritories.astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.memory_usage().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%script false --no-raise-error\n",
    "\n",
    "# example of manual date parsing, we won't run it\n",
    "from datetime import datetime\n",
    "frame['dateConv'] = frame.apply(lambda x: datetime.strptime(x.dateRep, \"%d/%m/%Y\"), axis=1)\n",
    "frame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataframe has all abovementioned features, for example — filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame[frame.deaths>=100].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frame.countriesAndTerritories.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get a small slice of data to work with"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty = frame[frame.countriesAndTerritories == \"Montenegro\"].copy()\n",
    "monty.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "monty.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty.sort_values(by=[\"dateRep\"], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty['totalCases'] = 0\n",
    "\n",
    "monty.loc[0, 'totalCases'] = monty.loc[0, 'cases']\n",
    "for idx in range(1, len(monty)):\n",
    "    monty.loc[idx, 'totalCases'] = monty.loc[idx, 'cases'] + monty.loc[idx-1, 'totalCases']\n",
    "    \n",
    "monty.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Usage of iteration approaches is discouraged in NumPy/Pandas, instead you need to use builtin methods working on series/frames level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%script false --no-raise-error\n",
    "\n",
    "monty['totalCases'] = monty.cases.cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "monty.plot(x=\"dateRep\", y=\"totalCases\", kind='line')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's make plots a bit more fancy, for that we can use a seaborn package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(\"dateRep\", \"totalCases\", data=monty)\n",
    "plt.plot(\"dateRep\", \"cases\", data=monty)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's try to get a cumulative statistics on per-country level, for that we will use gropung"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped = frame.groupby(\"countriesAndTerritories\")[[\"cases\", \"deaths\"]].sum().copy()\n",
    "grouped.sort_values(by=[\"deaths\"], inplace=True, ascending=False)\n",
    "\n",
    "grouped.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_data = grouped[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure() # Create matplotlib figure\n",
    "\n",
    "ax = fig.add_subplot(111) # Create matplotlib axes\n",
    "ax2 = ax.twinx() # Create another axes that shares the same x-axis as ax.\n",
    "\n",
    "width = 0.4\n",
    "\n",
    "plot_data = grouped[:20]\n",
    "\n",
    "plot_data.cases.plot(kind='bar', color='blue', ax=ax, width=width, position=0)\n",
    "plot_data.deaths.plot(kind='bar', color='red', ax=ax2, width=width, position=1)\n",
    "\n",
    "ax.set_ylabel('Cases')\n",
    "ax2.set_ylabel('Deaths')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_data.plot(kind=\"bar\", secondary_y=\"deaths\", figsize=(12, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped2 = frame.groupby(\"countriesAndTerritories\").agg(\n",
    "    totalCases = (\"cases\", \"sum\"),\n",
    "    totalDeath = (\"deaths\", \"sum\"),\n",
    "    population = (\"popData2018\", \"first\"),\n",
    "    countryCode = (\"countryterritoryCode\", \"first\")\n",
    ").copy()\n",
    "\n",
    "grouped2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped2[\"lethalRate\"] = grouped2.totalDeath / grouped2.population * 1000\n",
    "grouped2.sort_values(by=[\"lethalRate\"], inplace=True, ascending=False)\n",
    "\n",
    "new_plot_data = grouped2[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_plot_data.lethalRate.plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's make better visualisation, building _choropleth_ map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\n",
    "\n",
    "world.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "world.plot(figsize=(16,14))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = world.merge(grouped2, left_on=\"iso_a3\", right_on=\"countryCode\")\n",
    "merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged.plot(column=\"lethalRate\", cmap=\"tab10\", figsize=(16,14), scheme='equal_interval', k=10, legend=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "world[world.iso_a3 == '-99']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fixes = {\n",
    "    'name': ['France', 'Norway', 'Somaliland', 'Kosovo', 'N. Cyprus'],\n",
    "    'iso_a3': ['FRA', 'NOR', 'SOM', 'RKS', 'CYP']\n",
    "}\n",
    "fix_frame = DataFrame(fixes)\n",
    "\n",
    "world = gpd.GeoDataFrame(fix_frame.set_index('name').combine_first(world.set_index('name')).reset_index())\n",
    "\n",
    "world[world.iso_a3 == '-99']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = world.merge(grouped2, left_on=\"iso_a3\", right_on=\"countryCode\")\n",
    "merged.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = merged.plot(column=\"lethalRate\", cmap=\"Reds\", figsize=(16,10), scheme='equal_interval', k=10, legend=True)\n",
    "ax.set_title(\"Scary map\", fontdict={'fontsize': 20}, loc='left')\n",
    "ax.annotate(\"Are we doomed yet?\", xy=(0.1, 0.1), size=12, xycoords='figure fraction')\n",
    "\n",
    "ax.set_axis_off()\n",
    "ax.get_legend().set_bbox_to_anchor((.12, .4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wes McKinney — **Python for Data Analysis**\n",
    "\n",
    "Владимир Савельев — **Статистика и котики**\n",
    "\n",
    "Дарелл Хафф — **Как лгать при помощи статистики**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Thank you for watching, please subscribe to our **YouTube** channel!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import qrcode as qr\n",
    "qr.make('https://www.youtube.com/channel/UCJOEr3_V_F0V3MJPNyYR9jw', box_size=10, border=4, error_correction=qr.constants.ERROR_CORRECT_H)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}