5.TEXT_SIM/5.3.3_Quora_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bG4_-K7Z6aub"
   },
   "source": [
    "# 모듈 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ixJi5iWk6VzH"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 시각화 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 학습 데이터 파일 로드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_IN_PATH = './data_in/'\n",
    "DATA_OUT_PATH = './data_out/'\n",
    "TRAIN_Q1_DATA_FILE = 'train_q1.npy'\n",
    "TRAIN_Q2_DATA_FILE = 'train_q2.npy'\n",
    "TRAIN_LABEL_DATA_FILE = 'train_label.npy'\n",
    "DATA_CONFIGS = 'data_configs.json'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 랜덤 시드 고정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 파일 로드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb'))\n",
    "q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb'))\n",
    "labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))\n",
    "prepro_configs = json.load(open(DATA_IN_PATH + DATA_CONFIGS, 'r'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모델 하이퍼파라메터 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'malstm_similarity'\n",
    "BATCH_SIZE = 128\n",
    "NUM_EPOCHS = 5\n",
    "VALID_SPLIT = 0.1\n",
    "\n",
    "kargs = {\n",
    "    'vocab_size': prepro_configs['vocab_size'],\n",
    "    'embedding_dimension': 100,\n",
    "    'lstm_dimension': 150,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nvXOXlSywPR2"
   },
   "source": [
    "# 모델 선언 및 컴파일"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MaLSTM(tf.keras.Model):\n",
    "    \n",
    "    def __init__(self, **kargs):\n",
    "        super(MaLSTM, self).__init__(name=model_name)\n",
    "        self.embedding = layers.Embedding(input_dim=kargs['vocab_size'],\n",
    "                                     output_dim=kargs['embedding_dimension'])\n",
    "        self.lstm = layers.LSTM(units=kargs['lstm_dimension'])\n",
    "        \n",
    "    def call(self, x):\n",
    "        x1, x2 = x\n",
    "        x1 = self.embedding(x1)\n",
    "        x2 = self.embedding(x2)\n",
    "        x1 = self.lstm(x1)\n",
    "        x2 = self.lstm(x2)\n",
    "        x = tf.exp(-tf.reduce_sum(tf.abs(x1 - x2), axis=1))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MaLSTM(**kargs)\n",
    "\n",
    "model.compile(optimizer=tf.keras.optimizers.Adam(1e-3),\n",
    "              loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "              metrics=[tf.keras.metrics.BinaryAccuracy(name='accuracy')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Callback 선언"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data_out/malstm_similarity -- Folder already exists \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# overfitting을 막기 위한 ealrystop 추가\n",
    "earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=1)\n",
    "# min_delta: the threshold that triggers the termination (acc should at least improve 0.0001)\n",
    "# patience: no improvment epochs (patience = 1, 1번 이상 상승이 없으면 종료)\\\n",
    "\n",
    "checkpoint_path = DATA_OUT_PATH + model_name + '/weights.h5'\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "# Create path if exists\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "\n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모델 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history = model.fit((q1_data, q2_data), labels, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS,\n",
    "                    validation_split=VALID_SPLIT, callbacks=[earlystop_callback, cp_callback])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 결과 플롯"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 테스트 데이터 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GLab41oLQSQV"
   },
   "outputs": [],
   "source": [
    "TEST_Q1_DATA_FILE = 'test_q1.npy'\n",
    "TEST_Q2_DATA_FILE = 'test_q2.npy'\n",
    "TEST_ID_DATA_FILE = 'test_id.npy'\n",
    "\n",
    "test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb'))\n",
    "test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb'))\n",
    "test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb'), allow_pickle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 베스트 모델 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAVE_FILE_NM = 'weights.h5'\n",
    "model.load_weights(os.path.join(DATA_OUT_PATH, model_name, SAVE_FILE_NM))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 베스트 데이터 예측하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict((test_q1_data, test_q2_data), batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output = pd.DataFrame( data={\"test_id\":test_id_data, \"is_duplicate\": list(predictions)} )\n",
    "output.to_csv(DATA_OUT_PATH+\"rnn_predict.csv\", index=False, quoting=3)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Day4_2_Ma_LSTM.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}