assemble_dataset_segmented.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf5ff2fe-7bf0-459a-b776-3bbd457e326a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Calibration_1'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Calibration_2'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Calibration_3'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Nasal'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Inhale_1'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Inhale_2'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Inhale_3'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Exhale_1'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Exhale_2'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Exhale_3'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AWARE STUDY ID:</th>\n",
       "      <th>Calculated age (years):</th>\n",
       "      <th>Sex:</th>\n",
       "      <th>Race/ethnicity: (choice=White)</th>\n",
       "      <th>Race/ethnicity: (choice=Black / African American)</th>\n",
       "      <th>Race/ethnicity: (choice=Hispanic / Latino)</th>\n",
       "      <th>Race/ethnicity: (choice=Asian)</th>\n",
       "      <th>Race/ethnicity: (choice=Other)</th>\n",
       "      <th>Height (cm):</th>\n",
       "      <th>Weight (kg):</th>\n",
       "      <th>...</th>\n",
       "      <th>1012790</th>\n",
       "      <th>1012791</th>\n",
       "      <th>1012792</th>\n",
       "      <th>1012793</th>\n",
       "      <th>1012794</th>\n",
       "      <th>1012795</th>\n",
       "      <th>1012796</th>\n",
       "      <th>1012797</th>\n",
       "      <th>1012798</th>\n",
       "      <th>1012799</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Female</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>160.20</td>\n",
       "      <td>75.8</td>\n",
       "      <td>...</td>\n",
       "      <td>41</td>\n",
       "      <td>44</td>\n",
       "      <td>43</td>\n",
       "      <td>43</td>\n",
       "      <td>40</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>38</td>\n",
       "      <td>41</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Female</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>160.20</td>\n",
       "      <td>75.8</td>\n",
       "      <td>...</td>\n",
       "      <td>69</td>\n",
       "      <td>71</td>\n",
       "      <td>70</td>\n",
       "      <td>74</td>\n",
       "      <td>75</td>\n",
       "      <td>76</td>\n",
       "      <td>78</td>\n",
       "      <td>77</td>\n",
       "      <td>77</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Female</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>160.20</td>\n",
       "      <td>75.8</td>\n",
       "      <td>...</td>\n",
       "      <td>43</td>\n",
       "      <td>41</td>\n",
       "      <td>40</td>\n",
       "      <td>37</td>\n",
       "      <td>34</td>\n",
       "      <td>31</td>\n",
       "      <td>30</td>\n",
       "      <td>30</td>\n",
       "      <td>32</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Female</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>160.20</td>\n",
       "      <td>75.8</td>\n",
       "      <td>...</td>\n",
       "      <td>105</td>\n",
       "      <td>92</td>\n",
       "      <td>75</td>\n",
       "      <td>57</td>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>19</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Female</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>160.20</td>\n",
       "      <td>75.8</td>\n",
       "      <td>...</td>\n",
       "      <td>90</td>\n",
       "      <td>77</td>\n",
       "      <td>66</td>\n",
       "      <td>49</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>2</td>\n",
       "      <td>-12</td>\n",
       "      <td>-30</td>\n",
       "      <td>-46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1314</th>\n",
       "      <td>381</td>\n",
       "      <td>26.8</td>\n",
       "      <td>Male</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>186.00</td>\n",
       "      <td>97.0</td>\n",
       "      <td>...</td>\n",
       "      <td>204</td>\n",
       "      <td>197</td>\n",
       "      <td>195</td>\n",
       "      <td>196</td>\n",
       "      <td>200</td>\n",
       "      <td>201</td>\n",
       "      <td>196</td>\n",
       "      <td>189</td>\n",
       "      <td>184</td>\n",
       "      <td>187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>381</td>\n",
       "      <td>26.8</td>\n",
       "      <td>Male</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>186.00</td>\n",
       "      <td>97.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-42</td>\n",
       "      <td>-42</td>\n",
       "      <td>-45</td>\n",
       "      <td>-51</td>\n",
       "      <td>-54</td>\n",
       "      <td>-54</td>\n",
       "      <td>-51</td>\n",
       "      <td>-51</td>\n",
       "      <td>-50</td>\n",
       "      <td>-53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1316</th>\n",
       "      <td>381</td>\n",
       "      <td>26.8</td>\n",
       "      <td>Male</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>186.00</td>\n",
       "      <td>97.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-143</td>\n",
       "      <td>-140</td>\n",
       "      <td>-136</td>\n",
       "      <td>-136</td>\n",
       "      <td>-136</td>\n",
       "      <td>-141</td>\n",
       "      <td>-142</td>\n",
       "      <td>-139</td>\n",
       "      <td>-135</td>\n",
       "      <td>-134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1317</th>\n",
       "      <td>382</td>\n",
       "      <td>41.8</td>\n",
       "      <td>Female</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>162.56</td>\n",
       "      <td>81.0</td>\n",
       "      <td>...</td>\n",
       "      <td>88</td>\n",
       "      <td>88</td>\n",
       "      <td>89</td>\n",
       "      <td>91</td>\n",
       "      <td>91</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "      <td>92</td>\n",
       "      <td>92</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1318</th>\n",
       "      <td>382</td>\n",
       "      <td>41.8</td>\n",
       "      <td>Female</td>\n",
       "      <td>Checked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>Unchecked</td>\n",
       "      <td>162.56</td>\n",
       "      <td>81.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-5</td>\n",
       "      <td>-7</td>\n",
       "      <td>-7</td>\n",
       "      <td>-3</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-3</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1319 rows × 1012846 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     AWARE STUDY ID:  Calculated age (years):    Sex:  \\\n",
       "0                  1                     16.5  Female   \n",
       "1                  1                     16.5  Female   \n",
       "2                  1                     16.5  Female   \n",
       "3                  1                     16.5  Female   \n",
       "4                  1                     16.5  Female   \n",
       "...              ...                      ...     ...   \n",
       "1314             381                     26.8    Male   \n",
       "1315             381                     26.8    Male   \n",
       "1316             381                     26.8    Male   \n",
       "1317             382                     41.8  Female   \n",
       "1318             382                     41.8  Female   \n",
       "\n",
       "     Race/ethnicity: (choice=White)  \\\n",
       "0                         Unchecked   \n",
       "1                         Unchecked   \n",
       "2                         Unchecked   \n",
       "3                         Unchecked   \n",
       "4                         Unchecked   \n",
       "...                             ...   \n",
       "1314                        Checked   \n",
       "1315                        Checked   \n",
       "1316                        Checked   \n",
       "1317                        Checked   \n",
       "1318                        Checked   \n",
       "\n",
       "     Race/ethnicity: (choice=Black / African American)  \\\n",
       "0                                              Checked   \n",
       "1                                              Checked   \n",
       "2                                              Checked   \n",
       "3                                              Checked   \n",
       "4                                              Checked   \n",
       "...                                                ...   \n",
       "1314                                         Unchecked   \n",
       "1315                                         Unchecked   \n",
       "1316                                         Unchecked   \n",
       "1317                                         Unchecked   \n",
       "1318                                         Unchecked   \n",
       "\n",
       "     Race/ethnicity: (choice=Hispanic / Latino)  \\\n",
       "0                                     Unchecked   \n",
       "1                                     Unchecked   \n",
       "2                                     Unchecked   \n",
       "3                                     Unchecked   \n",
       "4                                     Unchecked   \n",
       "...                                         ...   \n",
       "1314                                  Unchecked   \n",
       "1315                                  Unchecked   \n",
       "1316                                  Unchecked   \n",
       "1317                                  Unchecked   \n",
       "1318                                  Unchecked   \n",
       "\n",
       "     Race/ethnicity: (choice=Asian) Race/ethnicity: (choice=Other)  \\\n",
       "0                         Unchecked                      Unchecked   \n",
       "1                         Unchecked                      Unchecked   \n",
       "2                         Unchecked                      Unchecked   \n",
       "3                         Unchecked                      Unchecked   \n",
       "4                         Unchecked                      Unchecked   \n",
       "...                             ...                            ...   \n",
       "1314                      Unchecked                      Unchecked   \n",
       "1315                      Unchecked                      Unchecked   \n",
       "1316                      Unchecked                      Unchecked   \n",
       "1317                      Unchecked                      Unchecked   \n",
       "1318                      Unchecked                      Unchecked   \n",
       "\n",
       "      Height (cm):  Weight (kg):  ... 1012790  1012791  1012792  1012793  \\\n",
       "0           160.20          75.8  ...      41       44       43       43   \n",
       "1           160.20          75.8  ...      69       71       70       74   \n",
       "2           160.20          75.8  ...      43       41       40       37   \n",
       "3           160.20          75.8  ...     105       92       75       57   \n",
       "4           160.20          75.8  ...      90       77       66       49   \n",
       "...            ...           ...  ...     ...      ...      ...      ...   \n",
       "1314        186.00          97.0  ...     204      197      195      196   \n",
       "1315        186.00          97.0  ...     -42      -42      -45      -51   \n",
       "1316        186.00          97.0  ...    -143     -140     -136     -136   \n",
       "1317        162.56          81.0  ...      88       88       89       91   \n",
       "1318        162.56          81.0  ...      -1       -5       -7       -7   \n",
       "\n",
       "      1012794  1012795  1012796  1012797  1012798  1012799  \n",
       "0          40       39       39       38       41       39  \n",
       "1          75       76       78       77       77       78  \n",
       "2          34       31       30       30       32       35  \n",
       "3          40       25       19       14       12        6  \n",
       "4          34       17        2      -12      -30      -46  \n",
       "...       ...      ...      ...      ...      ...      ...  \n",
       "1314      200      201      196      189      184      187  \n",
       "1315      -54      -54      -51      -51      -50      -53  \n",
       "1316     -136     -141     -142     -139     -135     -134  \n",
       "1317       91       92       91       92       92       91  \n",
       "1318       -3       -1        1       -1       -3       -2  \n",
       "\n",
       "[1319 rows x 1012846 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "COLUMNS_EXTENDED = [\n",
    "    'AWARE STUDY ID:',\n",
    "    'Calculated age (years):',\n",
    "    'Sex:',\n",
    "    'Race/ethnicity: (choice=White)',\n",
    "    'Race/ethnicity: (choice=Black / African American)',\n",
    "    'Race/ethnicity: (choice=Hispanic / Latino)',\n",
    "    'Race/ethnicity: (choice=Asian)',\n",
    "    'Race/ethnicity: (choice=Other)',\n",
    "    'Height (cm):',\n",
    "    'Weight (kg):',\n",
    "    'Participant:',\n",
    "    'R5', 'R5-20', 'R20', 'X5', 'AX', 'TV',\n",
    "    'R5 z-score',\n",
    "    'R5-20 z-score',\n",
    "    'R20 z-score',\n",
    "    'X5 z-score',\n",
    "    'AX z-score',\n",
    "    'Baseline FEV1 (liters):',\n",
    "    'Baseline FVC (liters):',\n",
    "    'Baseline FEV1/FVC (raw ratio):',\n",
    "    'Baseline FEF2575 (liters):',\n",
    "    'Predictive equations used:',\n",
    "    'Baseline FEV1 (%pred):',\n",
    "    'Baseline FVC (%pred):',\n",
    "    'Baseline FEV1/FVC (%pred):',\n",
    "    'Baseline FEF2575 (%pred):',\n",
    "    'Pre-BD \\'score\\'',\n",
    "    'Post-BD \\'score\\'',\n",
    "    'Post FEV1 (liters):',\n",
    "    'Post FVC (liters):',\n",
    "    'Post FEV1/FVC (raw ratio):',\n",
    "    'Post FEF2575 (liters):',\n",
    "    'Post FEV1 (%pred):',\n",
    "    'Post FVC (%pred):',\n",
    "    'Post FEV1/FVC (%pred):',\n",
    "    'Post FEF2575 (%pred):',\n",
    "    'BDR (as percent of baseline FEV1)',\n",
    "    'BDR (as percent of predicted FEV1)',\n",
    "    'BDR (liters)',\n",
    "    'BDR (difference in %preds)'\n",
    "]\n",
    "\n",
    "COLUMNS_SIGNALS = [\n",
    "    'Calibration_1',\n",
    "    'Calibration_2',\n",
    "    'Calibration_3',\n",
    "    'Nasal',\n",
    "    'Inhale_1', 'Inhale_2', 'Inhale_3', \n",
    "    'Exhale_1', 'Exhale_2', 'Exhale_3'\n",
    "]\n",
    "\n",
    "df = pd.read_pickle(\"data/aware_segmented.pkl\")\n",
    "\n",
    "arr = []\n",
    "for col_name in COLUMNS_SIGNALS:\n",
    "    display(col_name)\n",
    "    tmp = np.array(df[col_name].to_list())\n",
    "    tmp = tmp.reshape(tmp.shape[0], -1)\n",
    "    arr += [tmp]\n",
    "arr = np.concatenate(arr, axis=1)\n",
    "\n",
    "df = df[COLUMNS_EXTENDED+['Test Number']]\n",
    "df = pd.concat([df, pd.DataFrame(arr)], axis=1)\n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a801063-33ea-4fba-844b-6365bb1ba0cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Dumping data to .csv file ...')\n",
    "for i in tqdm(df.index):\n",
    "    if i == 0:\n",
    "        df.loc[[i]].to_csv('data/aware_segmented.csv', header=None, index=None, mode='w')\n",
    "    else:\n",
    "        df.loc[[i]].to_csv('data/aware_segmented.csv', header=None, index=None, mode='a')\n",
    "print('Complete')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5e8f53c-9182-49c3-8e25-ca30ed9f0aab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}