DataSetGeneration_HDF_Seperated_Sec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a71fcdef",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db67e785",
   "metadata": {},
   "source": [
    "### Processor\n",
    "* VOC_duration : 1min\n",
    "* Wash_Out : 8min\n",
    "\n",
    "### Sensors\n",
    "+ TGS : TGS2600, TGS2602, TGS2603, TGS2610, TGS2620\n",
    "+ MQ : MQ2, MQ3, MQ4, MQ5, MQ6, MQ7, MQ8, MQ9, MQ135\n",
    "\n",
    "### VOC\n",
    "1. DPG ( 10%) + DPG (100%)\n",
    "2. Actetic (10%) + DPG (99%)\n",
    "3. Benzaldehyde (10%) + DPG (99%)\n",
    "4. Keras (10%) + DPG (99%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "257f9c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pymysql\n",
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "from matplotlib import cm\n",
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "import os, sys\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime, timedelta\n",
    "import datetime, time\n",
    "\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor, export_graphviz, _tree\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, learning_curve\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, mean_squared_error, make_scorer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.preprocessing import minmax_scale\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import statsmodels.api as sm\n",
    "from scipy.stats import spearmanr\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Dense, LSTM\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM, Dense\n",
    "from keras import backend as K\n",
    "import keras\n",
    "\n",
    "import graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "spoken-engine",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "plot_style = plt.style.available\n",
    "# plt.style.use('ggplot')\n",
    "# plt.style.use('default')\n",
    "# plt.style.use('fivethirtyeight')\n",
    "plt.style.use('seaborn-paper')\n",
    "# plt.style.use('seaborn-muted')\n",
    "\n",
    "# pd.options.display.float_format = '{:.5f}'.format\n",
    "# pd.set_option('display.max_row', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "\n",
    "pd.set_option('display.expand_frame_repr', False)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "58862e57",
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT_DIR = Path('.')\n",
    "CSV_DIR = Path(PROJECT_DIR, 'csv')\n",
    "DATA_DIR = Path(PROJECT_DIR, 'data')\n",
    "RESULT_DIR = Path(PROJECT_DIR, 'results')\n",
    "NPY_DIR = Path(PROJECT_DIR, 'data', 'npy')\n",
    "HDF_DIR = Path(PROJECT_DIR, 'data', 'hdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1faca046",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cccd92f",
   "metadata": {},
   "source": [
    "### Check of CPU/GPU status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ca3aac7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU\n"
     ]
    }
   ],
   "source": [
    "gpu_devices = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpu_devices:\n",
    "    print('Using GPU')\n",
    "    tf.config.experimental.set_memory_growth(gpu_devices[0], True)\n",
    "else:\n",
    "    print('Using CPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "941d9081",
   "metadata": {},
   "source": [
    "### Data read (1-sec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bd343dc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.io.pytables.HDFStore'>\n",
      "File path: drug_data_1sec.h5\n",
      "/drug/tgs_sensers/1sec_mean            frame        (shape->[394550,6])\n"
     ]
    }
   ],
   "source": [
    "with pd.HDFStore(PROJECT_DIR/'drug_data_1sec.h5') as store:\n",
    "    print(store.info())\n",
    "    df_01 = store['/drug/tgs_sensers/1sec_mean']\n",
    "\n",
    "# df_01 = pd.read_hdf('drug_data_1s.h5', '/drug/tgs_sensors/1sec_mean')\n",
    "# df_01.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "43749f79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "MultiIndex: 394550 entries, ('Acetic_001', Timestamp('2023-05-07 03:44:03')) to ('Keras_725', Timestamp('2023-05-11 17:57:43'))\n",
      "Data columns (total 6 columns):\n",
      " #   Column   Non-Null Count   Dtype  \n",
      "---  ------   --------------   -----  \n",
      " 0   TGS2600  390472 non-null  float64\n",
      " 1   TGS2602  390472 non-null  float64\n",
      " 2   TGS2603  390472 non-null  float64\n",
      " 3   TGS2620  390472 non-null  float64\n",
      " 4   TGS2610  390472 non-null  float64\n",
      " 5   TGS822   390472 non-null  float64\n",
      "dtypes: float64(6)\n",
      "memory usage: 31.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df_01.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8c5f8462",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "MultiIndex: 394550 entries, ('Acetic_001', Timestamp('2023-05-07 03:44:03')) to ('Keras_725', Timestamp('2023-05-11 17:57:43'))\n",
      "Data columns (total 6 columns):\n",
      " #   Column   Non-Null Count   Dtype  \n",
      "---  ------   --------------   -----  \n",
      " 0   TGS2600  390472 non-null  float64\n",
      " 1   TGS2602  390472 non-null  float64\n",
      " 2   TGS2603  390472 non-null  float64\n",
      " 3   TGS2620  390472 non-null  float64\n",
      " 4   TGS2610  390472 non-null  float64\n",
      " 5   TGS822   390472 non-null  float64\n",
      "dtypes: float64(6)\n",
      "memory usage: 31.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df = df_01.copy()\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f7060941",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['TGS2600', 'TGS2602', 'TGS2603', 'TGS2620', 'TGS2610', 'TGS822',\n",
    "           'CYCLE', 'VOC_CYCLE', 'UNIXSTAMP']\n",
    "available_senosors = ['TGS2600', 'TGS2602', 'TGS2603', 'TGS2620', 'TGS2610', 'TGS822']\n",
    "non_available_sensors = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d3b44a64",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = {\n",
    "    'DPG' : 0,\n",
    "    'Acetic' : 1,\n",
    "    'Benzaldehyde' : 2,\n",
    "    'Keras' : 3\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7990603e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[available_senosors]\n",
    "idx = pd.IndexSlice\n",
    "available_sensors_no = len(available_senosors)\n",
    "samples = df.index.get_level_values('VOC_CYCLE').unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "770b34e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% 10/10 [01:07<00:00,  6.79s/it]\n"
     ]
    }
   ],
   "source": [
    "analysis_seconds = [30, 60*1, 60*2, 60*3, 60*4, 60*5, 60*6, 60*7, 60*8, 60*9]\n",
    "\n",
    "for sec in tqdm(analysis_seconds):\n",
    "    analysis_time = int(sec) # seconds\n",
    "    \n",
    "    df_min = pd.DataFrame()\n",
    "    voc_labels = []\n",
    "    for sample in samples:\n",
    "        voc_label = sample.split('_')[0]\n",
    "        df_temp = df.loc[idx[str(sample), :], :]\n",
    "        df_temp = df_temp.fillna(method='ffill').fillna(method='bfill')[:analysis_time]\n",
    "        if len(df_temp) == analysis_time:\n",
    "            df_min = pd.concat([df_min, df_temp])\n",
    "            voc_labels.append(voc_label)\n",
    "        else:\n",
    "            continue\n",
    "\n",
    "    y_voc = pd.DataFrame({'voc': voc_labels})\n",
    "    y_voc_onehot = pd.get_dummies(y_voc['voc']).astype(int)\n",
    "    y_voc_encoded = LabelEncoder().fit_transform(y_voc)\n",
    "    \n",
    "    # Data Saving\n",
    "    df_min.to_hdf(f'{HDF_DIR}/1_sec_data.h5', f'{analysis_time}sec/X_sensor')\n",
    "    y_voc.to_hdf(f'{HDF_DIR}/1_sec_data.h5', f'{analysis_time}sec/y_voc')\n",
    "    y_voc_onehot.to_hdf(f'{HDF_DIR}/1_sec_data.h5', f'{analysis_time}sec/y_voc_onehot')\n",
    "#     y_voc_encoded.to_hdf(f'./data/hdf/1_sec.npy', f'{analysis_time}sec/y_voc_encode')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}