notebooks/02-data-acquisition-and-exploration.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "reproducible-research-in-python.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s9nC5_RwqJcx",
        "colab_type": "text"
      },
      "source": [
        "# Data Acquisition and Exploration\n",
        "\n",
        "This notebook was created as part of a workshop on *Reproducible Research in Python*. \n",
        "\n",
        "- You can access the entire workshop materials at: [Reproducible Research in Python](https://github.com/mickaeltemporao/reproducible-research-in-python).\n",
        "\n",
        "**Learning Objective:** \n",
        "- Get familiar with common data exploration libraries\n",
        "- Learn to acquire and clean data\n",
        "- Learn to explore and visualize \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sofsPeRDqNRO",
        "colab_type": "text"
      },
      "source": [
        "## Acquiring Data\n",
        "With some Python basics we will start combining existing packages to acquire, and explore data.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDX2f5WNcUqY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Install requirred libraries\n",
        "!pip install wikipedia"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PTx-1l9i4MPy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load the required libraries\n",
        "import pandas as pd\n",
        "import wikipedia as wp\n",
        "import matplotlib\n",
        "matplotlib.rcParams['figure.figsize'] = [10, 5]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5CrD5shz5Vwu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Identify Wikipedia Page and acquire the date\n",
        "page_title = \"Opinion polling for the 2019 Canadian federal election\"\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XtBh69Va9lfm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Get the html source\n",
        "html = wp.page(page_title).html().encode(\"UTF-8\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlqQzI8R5Zz3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Extract tables and convert the html tables into pd.DataFrame()\n",
        "df = pd.read_html(html)[0].iloc[2:,:]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6KIE5UUg5cu7",
        "colab_type": "text"
      },
      "source": [
        "## Cleaning Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K9n7kcR4dLTb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Inspect the data\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjUNJ9Ba5iln",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We notice that there seems to be a double header\n",
        "df.columns\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fK6ERcta5jWT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What is the type of columns\n",
        "type(df.columns)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xA2JYlIe5kcC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's use a loop to extract and edit each element of the MultiIndex dataframe\n",
        "columnn_names = []\n",
        "for c in df.columns:\n",
        "    tmp = c[0].lower()\n",
        "    columnn_names.append(tmp.replace(\" \", \"_\"))\n",
        "\n",
        "columnn_names\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QrBVR0gU5lUK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's use regular expressions in a list comprehension this time\n",
        "import re\n",
        "regex = \"[a-z]+\"\n",
        "columnn_names = [\"_\".join(re.findall(regex, i)) for i in columnn_names]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wmGVxVMs5l3h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's edit the columns of our dataset\n",
        "df.columns = columnn_names\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HSoXadEo5mXQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's further rename those columns\n",
        "names_dict = {\n",
        "    \"polling_firm\": \"source\",\n",
        "    \"last_dateof_polling\": \"date\",\n",
        "    \"samplesize\": \"sample_size\",\n",
        "    \"marginof_error\": \"error\",\n",
        "    \"cons\": \"cpc\",\n",
        "    \"liberal\": \"lpc\",\n",
        "    \"green\": \"gpc\",\n",
        "    \"polling_method\": \"method\",\n",
        "}\n",
        "\n",
        "type(names_dict)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Gke_V1T5nRh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Pass the new dictionary as an argument to the .rename method\n",
        "df.rename(columns=names_dict, inplace=True)\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n9V7ryxd5n2h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's check the data types\n",
        "df.dtypes\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkoKzrsj5ppH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# The date field needs to be converted\n",
        "df[['date']] = pd.to_datetime(df.date)\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mJd2dc2Y5qWE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We should also only keep the numeric values for the margins of error\n",
        "regex = \"(\\d+\\.*\\d*)\"\n",
        "df.error = df.error.str.extract(regex)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CY-ZCGVA5q7U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's look again at our dataset\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KS93D1r55rbv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What if we look at a random subsample\n",
        "df.sample(5)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0GIcT7Pq5r4W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's clean the sample\n",
        "regex = r\"\\(.*\\)\"\n",
        "df.sample_size = df.sample_size.str.replace(regex, \"\")\n",
        "df.sample_size = df.sample_size.str.replace(\" |,\", \"\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iU8LAAX5tjx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# How does the data look now?\n",
        "df.sample(5)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7eaf8vO05uuZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What about the data types?\n",
        "df.info()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6HADs6lW5vmB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Which of these variables are still objects?\n",
        "df.select_dtypes(include='object')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-rXgXmeb5yC3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's use a dictionary to recode the data types\n",
        "convert_dict = {\n",
        "    'error': float,\n",
        "    'sample_size': int,\n",
        "    'lead': float\n",
        "}\n",
        "\n",
        "df = df.astype(convert_dict)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nUku2bRB5y4D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's look once again at our data\n",
        "df.sample(5)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i84Qzw2W5zaf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What are the remaining objects?\n",
        "df.select_dtypes(include='object')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S14uJehz5z_8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Keep only necessary variables by creating a variable filter\n",
        "to_keep = [\n",
        "    'source',\n",
        "    'date',\n",
        "    'lpc',\n",
        "    'cpc',\n",
        "    'ndp',\n",
        "    'bq',\n",
        "    'gpc',\n",
        "    'method'\n",
        "]\n",
        "\n",
        "df = df[to_keep]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_jc2nEBV7SaA",
        "colab_type": "text"
      },
      "source": [
        "## Data IO\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5IHjSTOddkIx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Save the cleaned dataframe to a file\n",
        "file_name = \"national_polls_2019.csv\"\n",
        "df.to_csv(file_name, index=False)\n",
        "print(df)\n",
        "\n",
        "df.dtypes\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CWw_gIt07TLE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Read the data back-in from the recorded csv file.\n",
        "\n",
        "# More info on read_csv\n",
        "# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_csv.html\n",
        "df = pd.read_csv(\"national_polls_2019.csv\", parse_dates=['date'])\n",
        "df.dtypes\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UpZQqLUB7Zxr",
        "colab_type": "text"
      },
      "source": [
        "## Data Exploration and Visualization\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eDoOasNpdr6D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's convert this into a time-series dataframe\n",
        "df.set_index('date', inplace=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TcaAaVWD7ch7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Time-series data should be stored in descending order\n",
        "df = df.sort_values(by=['date', 'source'])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDLa0Ws_7c_W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# How does the data look now?\n",
        "df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dIMp9oU07eXZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What about the tail?\n",
        "df.tail()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXfA8snl7ezQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# A time indexed data frame provides much more control over the data\n",
        "df.loc[df.index > '2019-10-15']\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mgCuP_EU7fOG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can look at a single party\n",
        "df.lpc.loc['2019-10-20']\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h96QxTsi7gBd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can focus on a subset of columns\n",
        "parties = [\"lpc\", \"cpc\", \"ndp\", \"bq\", \"gpc\"]\n",
        "df.loc[:, parties]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXkeJ4CE7ggt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can aggregate/resample the data\n",
        "df[parties].resample('D', how='mean').head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KnKK-ba07g-E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can also use pandas to plot\n",
        "df[parties].resample('D', how='mean').plot()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R30Sw-VXb1KC",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "### Anatomy of a Figure\n",
        "![Anatomy of a Figure](https://matplotlib.org/3.1.1/_images/anatomy.png)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8gJjUSkR7hYj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can look at the distributions for each party\n",
        "df[parties].plot(kind='kde')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4_ZrItD7iDm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Or do a simple box-plot\n",
        "df[parties].boxplot()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZykjhTV7iei",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's look at missing values\n",
        "df.isnull().mean()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PvyYBoaa7i72",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can remove missing values\n",
        "df.dropna()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V4IlZuOX7jou",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We just lost half of our dataset...\n",
        "# Maybe we should fill the missing values\n",
        "tmp_df = df.fillna(method='ffill', limit=3).copy()\n",
        "tmp_df.isnull().mean()\n",
        "\n",
        "df = tmp_df\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xu5b5Dtx7kTu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Let's investigate which polling firms have been most active\n",
        "df.source.value_counts()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bLZIK8ik7k-g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Remove the firms that released less than 5 polls\n",
        "tmp_mask = df.source.value_counts() >= 5\n",
        "mask = tmp_mask.index[tmp_mask]\n",
        "\n",
        "df = df[df.source.isin(mask)]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Hn75nR57ll_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Once again we could decide to visualize directly the result\n",
        "df.source.value_counts().plot(kind='barh')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e8f6prY87mPR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Try to do grouped operations and see how did each of these firms portrayed the liberal party\n",
        "df.groupby('source').lpc.describe().sort_values(by='mean')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LFhQ0vrt7mvU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can also look at the means for all the parties\n",
        "df.groupby('source')[parties].mean().sort_values('lpc')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXft9CUK7nZ_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can also apply custom functions by groups\n",
        "z_score = lambda x: (x-x.mean()) / x.std()\n",
        "df.reset_index().groupby('source')[parties].apply(z_score).head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DsuHM2tt7oCX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Most algorithms need you to shape the date in a long format\n",
        "long_df = pd.melt(\n",
        "    df.reset_index(),\n",
        "    id_vars=['date', 'source'],\n",
        "    value_vars=parties,\n",
        "    var_name='party',\n",
        "    value_name='share',\n",
        ")\n",
        "\n",
        "long_df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iAamHM8f7qmT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Seaborn, a statistical data visualization library uses long-format\n",
        "import seaborn as sns\n",
        "sns.set(style=\"whitegrid\", palette=\"muted\")\n",
        "\n",
        "sns.swarmplot(\n",
        "    x=\"party\",\n",
        "    y=\"share\",\n",
        "    hue=\"source\",\n",
        "    data=long_df,\n",
        ")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X1x73YzS7ud-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# What if we need to add the sample size back?\n",
        "new_df = long_df.merge(\n",
        "    df[['method', 'source']].reset_index(),\n",
        "    on=['date', 'source']\n",
        ")\n",
        "\n",
        "new_df.head()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxCX7w2j7vDh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can also expand the dataframe back to a wide format\n",
        "new_df = new_df.pivot_table(\n",
        "    index=['date', 'source', 'method'],\n",
        "    columns='party',\n",
        "    values='share',\n",
        ")\n",
        "\n",
        "new_df.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lueympPcWbKB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "new_df.stack()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4pLtcneZY_A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}