06_nsrr_demographics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demographcis of the NSRR training/testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import pingouin as pg\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "sns.set(style=\"ticks\", font_scale=1.25)\n",
    "\n",
    "from helper_functions import mean_std\n",
    "\n",
    "# Define paths\n",
    "wdir_demo = \"output/demo/\"\n",
    "outdir = \"output/plots/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, let's merge with the main demographics\n",
    "df_demo = pd.read_csv(wdir_demo + \"demo_nsrr_all.csv\")\n",
    "\n",
    "print(df_demo.shape)\n",
    "print(df_demo['set'].value_counts())\n",
    "df_demo.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_training = pd.read_parquet(\"output/features/features_all.parquet\", columns=['age']).reset_index()\n",
    "print(df_training['subj'].nunique(), 'subjects')\n",
    "print(df_training.shape)\n",
    "df_training.head().round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose model\n",
    "model = \"eeg+eog+emg+demo\"\n",
    "feat_files = glob.glob(\"output/cv/%s/cv_loo_nsrr_*.parquet\" % model)\n",
    "\n",
    "df_testing = []\n",
    "for f in feat_files:\n",
    "    df_testing.append(pd.read_parquet(f))\n",
    "    \n",
    "df_testing = pd.concat(df_testing)\n",
    "df_testing['subj'] = df_testing['subj'].astype(str)\n",
    "df_testing['dataset'] = df_testing['dataset'].str.upper()\n",
    "\n",
    "print(df_testing['subj'].nunique(), 'subjects')\n",
    "print(df_testing.shape)\n",
    "df_testing.head().round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep index of subjects included in the training/testing set\n",
    "idx_training = df_training['subj'].unique().tolist()\n",
    "idx_testing = df_testing['subj'].unique().tolist()\n",
    "df_demo = df_demo[df_demo['subj'].isin(idx_training + idx_testing)].reset_index(drop=True)\n",
    "\n",
    "# Add apnea severity\n",
    "df_demo['apnea'] = pd.cut(\n",
    "    df_demo['ahi'], bins=[0, 5, 15, 30, np.inf], right=False, include_lowest=True, \n",
    "    labels=[\"None\", \"Mild\", \"Moderate\", \"Severe\"])\n",
    "\n",
    "df_demo.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional: remove HOMEPAP\n",
    "# df_demo = df_demo[~df_demo['dataset'].isin(['HOMEPAP'])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "********\n",
    "\n",
    "## Descriptive statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_set = df_demo.groupby(\"set\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Continuous variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of nights\n",
    "grp_set['age'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table_desc = grp_set[['age', 'bmi', 'ahi']].agg(mean_std).T\n",
    "\n",
    "for dv in table_desc.index:\n",
    "    tt = df_demo.pairwise_ttests(dv=dv, between=\"set\")\n",
    "    table_desc.loc[dv, 'T'] = tt.loc[0, 'T']\n",
    "    table_desc.loc[dv, 'dof'] = tt.loc[0, 'dof']\n",
    "    table_desc.loc[dv, 'p-unc'] = tt.loc[0, 'p-unc']\n",
    "    table_desc.loc[dv, 'hedges'] = tt.loc[0, 'hedges']\n",
    "\n",
    "table_desc.round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(df_demo.groupby(['set', 'dataset'])['age'].agg(['mean', 'std', 'min', 'max']).round(2))\n",
    "\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8, 4.5), sharey=True, sharex=True)\n",
    "sns.histplot(df_demo[df_demo['set'] == 'training']['age'], \n",
    "             bins=np.linspace(0, 100, 11), stat='probability', alpha=0.95, ax=ax1)\n",
    "sns.histplot(df_demo[df_demo['set'] == 'testing']['age'], \n",
    "             bins=np.linspace(0, 100, 11), stat='probability', alpha=0.95, ax=ax2)\n",
    "\n",
    "ax1.set_xlabel(\"Age (yrs)\")\n",
    "ax2.set_xlabel(\"Age (yrs)\")\n",
    "\n",
    "ax1.set_title(\"Training set\")\n",
    "ax2.set_title(\"Testing set\")\n",
    "\n",
    "plt.xlim(0, 100)\n",
    "plt.xticks([0, 20, 40, 60, 80, 100])\n",
    "\n",
    "sns.despine()\n",
    "\n",
    "plt.savefig(outdir + \"hist_age.png\", dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xticks = np.arange(0, 95, 5)\n",
    "\n",
    "sns.displot(df_demo, x=\"age\", bins=xticks, palette=\"colorblind\", height=5, aspect=1.5)\n",
    "plt.xlim(0, None)\n",
    "plt.xticks(xticks);\n",
    "plt.xlabel(\"Age (yrs)\");\n",
    "plt.ylabel(\"Number of participants\")\n",
    "plt.title(\"Age distribution in the NSRR testing set\", fontweight=\"bold\")\n",
    "plt.savefig(outdir + \"hist_age_testing.png\", dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xticks = np.arange(0, 95, 5)\n",
    "\n",
    "sns.displot(df_demo, x=\"age\", hue=\"dataset\", bins=xticks, palette=\"colorblind\", height=5, aspect=1.5)\n",
    "plt.xlim(0, None)\n",
    "plt.xticks(xticks);\n",
    "plt.xlabel(\"Age (yrs)\");\n",
    "plt.ylabel(\"Number of participants\")\n",
    "plt.title(\"Age distribution in the NSRR testing set\", fontweight=\"bold\")\n",
    "plt.savefig(outdir + \"hist_age_dataset.png\", dpi=300, bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### AHI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_demo.groupby(['set', 'dataset'])['ahi'].agg(['mean', 'std', 'min', 'max']).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def value_counts_chi2(dv, normalize=True):\n",
    "    \"\"\"Show value counts and chi-square test.\n",
    "    \n",
    "    NaN values are droppped by chi2_independence (see pd.crosstab)\"\"\"\n",
    "    if normalize:\n",
    "        display(100 * grp_set[dv].value_counts(normalize=normalize, sort=False).round(5))\n",
    "    else:\n",
    "        display(grp_set[dv].value_counts(sort=False).round(5))\n",
    "    print(\"\")\n",
    "    display(pg.chi2_independence(df_demo, x=dv, y='set', correction=True)[2].iloc[0, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sex and chi2-test\n",
    "value_counts_chi2(\"male\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ethnicity\n",
    "value_counts_chi2(\"ethnicity\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset\n",
    "value_counts_chi2(\"dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grp_set['dataset'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apnea severity\n",
    "value_counts_chi2(\"apnea\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Insomnia\n",
    "value_counts_chi2(\"insomnia\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Narcolepsy\n",
    "value_counts_chi2(\"narcolepsy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Depression\n",
    "value_counts_chi2(\"depression\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Diabetes\n",
    "value_counts_chi2(\"diabete\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hypertension\n",
    "value_counts_chi2(\"hypertension\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 4
}