aspect-categorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import string\n",
    "import gensim\n",
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "import keras\n",
    "from keras.layers.core import Reshape, Flatten\n",
    "from keras import regularizers\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input, Dense, Activation, Embedding, Flatten, GlobalMaxPool1D, Dropout, Conv1D, LSTM, MaxPooling1D, concatenate\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n",
    "from keras.losses import binary_crossentropy\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "from skmultilearn.problem_transform import BinaryRelevance, ClassifierChain\n",
    "from sklearn.svm import LinearSVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import classification_report, hamming_loss\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_all_label(x_train, y_train):\n",
    "    y_train_set = []\n",
    "    sent = x_train[0]\n",
    "    X_train_set = [sent]\n",
    "    labels = []\n",
    "    for i in range (len(x_train)):\n",
    "        if sent != x_train[i]:\n",
    "            if (len(labels)>0):\n",
    "                y_train_set.append(labels)\n",
    "            sent = x_train[i]\n",
    "            X_train_set.append(sent)\n",
    "            labels = []\n",
    "        labels.append(y_train[i])\n",
    "    return X_train_set, y_train_set\n",
    "\n",
    "def tokenize(msg):\n",
    "    clean = [char for char in msg if char not in string.punctuation]\n",
    "    clean = ''.join(clean)\n",
    "    return clean.lower().split()\n",
    "\n",
    "def find_maxlen(reviews):\n",
    "    longest = 0\n",
    "    strlong = \"test\"\n",
    "    for review in reviews:\n",
    "        if len(tokenize(review)) > longest:\n",
    "            longest = len(tokenize(review))\n",
    "            strlong = review\n",
    "    return longest\n",
    "\n",
    "def train_w2v(corpus, size, min_count):\n",
    "    docs = [tokenize(doc) for doc in corpus]\n",
    "    model = gensim.models.Word2Vec(docs,\n",
    "                                   size=size,\n",
    "                                   window=10,\n",
    "                                   min_count=min_count,\n",
    "                                   workers=10)\n",
    "    model.train(docs, total_examples=len(docs), epochs=10)\n",
    "    return model\n",
    "\n",
    "def tokenize_text(vocab_size, reviews,maxlen):\n",
    "    tokenizer = Tokenizer(num_words=vocab_size, lower=True)\n",
    "    tokenizer.fit_on_texts(reviews)\n",
    "    sequences = tokenizer.texts_to_sequences(reviews)\n",
    "    x = pad_sequences(sequences, maxlen=maxlen)\n",
    "    word_index = tokenizer.word_index\n",
    "    return x, word_index\n",
    "\n",
    "def createEmbeddingMatrix(word_index, vocab_size, dim, word_vectors):\n",
    "    EMBEDDING_DIM=dim\n",
    "    vocabulary_size=min(len(word_index)+1,vocab_size)\n",
    "    embedding_matrix = np.zeros((vocabulary_size, EMBEDDING_DIM))\n",
    "    for word, i in word_index.items():\n",
    "        if i>=vocab_size:\n",
    "            continue\n",
    "        try:\n",
    "            embedding_vector = word_vectors[word]\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "        except KeyError:\n",
    "            embedding_matrix[i]=np.random.normal(0,np.sqrt(0.25),EMBEDDING_DIM)\n",
    "    return embedding_matrix\n",
    "\n",
    "def createCNNModel(filter_sizes, num_filters, embedding_matrix, embedding_dim, vocabulary_size, maxlen, num_classes):\n",
    "    filter_sizes = filter_sizes\n",
    "    num_filters = num_filters\n",
    "    drop = 0.5\n",
    "    \n",
    "    inputs = Input(shape=(maxlen,))\n",
    "    embedding_layer = Embedding(vocabulary_size,\n",
    "                            embedding_dim,\n",
    "                            weights=[embedding_matrix],\n",
    "                            trainable=True)\n",
    "    embedding = embedding_layer(inputs)\n",
    "    \n",
    "    conv_0 = Conv1D(num_filters, filter_sizes[0],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    conv_1 = Conv1D(num_filters, filter_sizes[1],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    conv_2 = Conv1D(num_filters, filter_sizes[2],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    \n",
    "    maxpool_0 = MaxPooling1D(maxlen - filter_sizes[0] + 1, strides=1)(conv_0)\n",
    "    maxpool_1 = MaxPooling1D(maxlen - filter_sizes[1] + 1, strides=1)(conv_1)\n",
    "    maxpool_2 = MaxPooling1D(maxlen - filter_sizes[2] + 1, strides=1)(conv_2)\n",
    "    \n",
    "    merged_tensor = concatenate([maxpool_0, maxpool_1, maxpool_2], axis=1)\n",
    "    flatten = Flatten()(merged_tensor)\n",
    "    \n",
    "    dense1 = Dense(256)(flatten)\n",
    "    dropout = Dropout(drop)(dense1)\n",
    "    \n",
    "    output = Dense(units=num_classes, activation='softmax',kernel_regularizer=regularizers.l2(0.01))(dropout)\n",
    "    \n",
    "    # this creates a model that includes\n",
    "    model = Model(inputs, output)\n",
    "    print(model.summary())\n",
    "    return model\n",
    "\n",
    "def createCNNLSTMModel(filter_sizes, num_filters, embedding_matrix, embedding_dim, vocabulary_size, maxlen, num_classes):\n",
    "    filter_sizes = filter_sizes\n",
    "    num_filters = num_filters\n",
    "    drop = 0.5\n",
    "    \n",
    "    inputs = Input(shape=(maxlen,))\n",
    "    embedding_layer = Embedding(vocabulary_size,\n",
    "                            embedding_dim,\n",
    "                            weights=[embedding_matrix],\n",
    "                            trainable=True)\n",
    "    embedding = embedding_layer(inputs)\n",
    "    \n",
    "    conv_0 = Conv1D(num_filters, filter_sizes[0],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    conv_1 = Conv1D(num_filters, filter_sizes[1],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    conv_2 = Conv1D(num_filters, filter_sizes[2],activation='relu',kernel_regularizer=regularizers.l2(0.01))(embedding)\n",
    "    \n",
    "    maxpool_0 = MaxPooling1D(maxlen - filter_sizes[0] + 1, strides=1)(conv_0)\n",
    "    maxpool_1 = MaxPooling1D(maxlen - filter_sizes[1] + 1, strides=1)(conv_1)\n",
    "    maxpool_2 = MaxPooling1D(maxlen - filter_sizes[2] + 1, strides=1)(conv_2)\n",
    "    \n",
    "    merged_tensor = concatenate([maxpool_0, maxpool_1, maxpool_2], axis=1)\n",
    "    flatten = Flatten()(merged_tensor)\n",
    "    \n",
    "    dense1 = Dense(256)(merged_tensor)\n",
    "    dropout = Dropout(drop)(dense1)\n",
    "    lstm_1 = LSTM(128)(dropout)\n",
    "    output = Dense(units=num_classes, activation='softmax',kernel_regularizer=regularizers.l2(0.01))(lstm_1)\n",
    "    \n",
    "    # this creates a model that includes\n",
    "    model = Model(inputs, output)\n",
    "    print(model.summary())\n",
    "    return model\n",
    "\n",
    "def trainCNN(x_train, y_train, model, epochs, filepath):\n",
    "    adam = Adam(lr=1e-3)\n",
    "    model.compile(loss='binary_crossentropy', metrics=['categorical_accuracy'],\n",
    "              optimizer=adam)\n",
    "    callbacks = [EarlyStopping(patience=4),\n",
    "            ModelCheckpoint(filepath=filepath, save_best_only=True)]\n",
    "    model.fit(x_train, y_train, batch_size=32, epochs=epochs, validation_split=0.1,\n",
    "         callbacks=callbacks)\n",
    "    \n",
    "def get_output_cnn(model, x_train, x_test):\n",
    "    total_layers = len(model.layers)\n",
    "    fl_index = total_layers-1\n",
    "    feature_layer_model = Model(\n",
    "                     inputs=model.input,\n",
    "                     outputs=model.get_layer(index=fl_index).output)\n",
    "    x_train_xg = feature_layer_model.predict(x_train)\n",
    "    x_test_xg = feature_layer_model.predict(x_test)\n",
    "    return x_train_xg, x_test_xg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preproses Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('dataset/priority_3k_labelled.csv', sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>category_sentiment</th>\n",
       "      <th>category</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kotor berdebu. Saya tdk berhenti bersin ketika...</td>\n",
       "      <td>wifi_P1-neg</td>\n",
       "      <td>wifi_P1</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kotor berdebu. Saya tdk berhenti bersin ketika...</td>\n",
       "      <td>kebersihan-neg</td>\n",
       "      <td>kebersihan</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kamar ada semutnya. kamar mandi bermasalah. bu...</td>\n",
       "      <td>kebersihan-neg</td>\n",
       "      <td>kebersihan</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kamar mandi bau, airnya bau</td>\n",
       "      <td>bau_P1-neg</td>\n",
       "      <td>bau_P1</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tak sesuai espektasi, kamar sempit, pintu kama...</td>\n",
       "      <td>service-neg</td>\n",
       "      <td>service</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review category_sentiment  \\\n",
       "0  Kotor berdebu. Saya tdk berhenti bersin ketika...        wifi_P1-neg   \n",
       "1  Kotor berdebu. Saya tdk berhenti bersin ketika...     kebersihan-neg   \n",
       "2  kamar ada semutnya. kamar mandi bermasalah. bu...     kebersihan-neg   \n",
       "3                        Kamar mandi bau, airnya bau         bau_P1-neg   \n",
       "4  tak sesuai espektasi, kamar sempit, pintu kama...        service-neg   \n",
       "\n",
       "     category sentiment  \n",
       "0     wifi_P1       neg  \n",
       "1  kebersihan       neg  \n",
       "2  kebersihan       neg  \n",
       "3      bau_P1       neg  \n",
       "4     service       neg  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>category</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kotor berdebu. Saya tdk berhenti bersin ketika...</td>\n",
       "      <td>wifi_P1</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Kotor berdebu. Saya tdk berhenti bersin ketika...</td>\n",
       "      <td>kebersihan</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>kamar ada semutnya. kamar mandi bermasalah. bu...</td>\n",
       "      <td>kebersihan</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Kamar mandi bau, airnya bau</td>\n",
       "      <td>bau_P1</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tak sesuai espektasi, kamar sempit, pintu kama...</td>\n",
       "      <td>service</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review    category sentiment\n",
       "0  Kotor berdebu. Saya tdk berhenti bersin ketika...     wifi_P1       neg\n",
       "1  Kotor berdebu. Saya tdk berhenti bersin ketika...  kebersihan       neg\n",
       "2  kamar ada semutnya. kamar mandi bermasalah. bu...  kebersihan       neg\n",
       "3                        Kamar mandi bau, airnya bau      bau_P1       neg\n",
       "4  tak sesuai espektasi, kamar sempit, pintu kama...     service       neg"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.drop(columns='category_sentiment')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, y_train = get_all_label(data['review'], data['category'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kotor berdebu. Saya tdk berhenti bersin ketika...</td>\n",
       "      <td>[wifi_P1, kebersihan]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>kamar ada semutnya. kamar mandi bermasalah. bu...</td>\n",
       "      <td>[kebersihan]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kamar mandi bau, airnya bau</td>\n",
       "      <td>[bau_P1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tak sesuai espektasi, kamar sempit, pintu kama...</td>\n",
       "      <td>[service]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>buruk. kasur ada bekas sperma seprai jg air ba...</td>\n",
       "      <td>[linen_P1, wifi_P1]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review                 labels\n",
       "0  Kotor berdebu. Saya tdk berhenti bersin ketika...  [wifi_P1, kebersihan]\n",
       "1  kamar ada semutnya. kamar mandi bermasalah. bu...           [kebersihan]\n",
       "2                        Kamar mandi bau, airnya bau               [bau_P1]\n",
       "3  tak sesuai espektasi, kamar sempit, pintu kama...              [service]\n",
       "4  buruk. kasur ada bekas sperma seprai jg air ba...    [linen_P1, wifi_P1]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train = list(zip(X_train, y_train))\n",
    "df_train = pd.DataFrame(data_train, columns=['review', 'labels'])\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ac_P1', 'air_panas_P1', 'bau_P1', 'general', 'kebersihan',\n",
       "       'linen_P1', 'service', 'sunrise_meal_P1', 'tv_P1', 'wifi_P1'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlb = MultiLabelBinarizer()\n",
    "y = df_train.labels\n",
    "y = mlb.fit_transform(y)\n",
    "mlb.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "find_maxlen(df_train.review)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train w2v model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = train_w2v(df_train.review, 400, 2)\n",
    "model.wv.save_word2vec_format('model/w2v_model.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_vectors = KeyedVectors.load_word2vec_format('model/w2v_model.bin', binary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize & create embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x, word_index = tokenize_text(5000, df_train.review, 180)\n",
    "embedding_matrix = createEmbeddingMatrix(word_index, 5000, 400, word_vectors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build CNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 180)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 180, 400)     2000000     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 180, 128)     51328       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 178, 128)     153728      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 176, 128)     256128      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 1, 128)       0           conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 1, 128)       0           conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1D)  (None, 1, 128)       0           conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 3, 128)       0           max_pooling1d_1[0][0]            \n",
      "                                                                 max_pooling1d_2[0][0]            \n",
      "                                                                 max_pooling1d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 384)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          98560       flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 256)          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 10)           2570        dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 2,562,314\n",
      "Trainable params: 2,562,314\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "cnn_model = createCNNModel([1,3,5], 128, embedding_matrix, 400, 5000, 180, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2073 samples, validate on 231 samples\n",
      "Epoch 1/100\n",
      "2073/2073 [==============================] - 59s 29ms/step - loss: 2.8802 - categorical_accuracy: 0.3131 - val_loss: 1.0112 - val_categorical_accuracy: 0.4589\n",
      "Epoch 2/100\n",
      "2073/2073 [==============================] - 55s 27ms/step - loss: 0.7092 - categorical_accuracy: 0.4626 - val_loss: 0.5443 - val_categorical_accuracy: 0.6277\n",
      "Epoch 3/100\n",
      "2073/2073 [==============================] - 57s 28ms/step - loss: 0.4985 - categorical_accuracy: 0.5374 - val_loss: 0.4668 - val_categorical_accuracy: 0.5628\n",
      "Epoch 4/100\n",
      "2073/2073 [==============================] - 61s 29ms/step - loss: 0.4504 - categorical_accuracy: 0.5813 - val_loss: 0.4296 - val_categorical_accuracy: 0.6450\n",
      "Epoch 5/100\n",
      "2073/2073 [==============================] - 55s 27ms/step - loss: 0.4285 - categorical_accuracy: 0.5986 - val_loss: 0.4149 - val_categorical_accuracy: 0.6970\n",
      "Epoch 6/100\n",
      "2073/2073 [==============================] - 56s 27ms/step - loss: 0.4155 - categorical_accuracy: 0.6083 - val_loss: 0.4101 - val_categorical_accuracy: 0.6840\n",
      "Epoch 7/100\n",
      "2073/2073 [==============================] - 55s 27ms/step - loss: 0.3970 - categorical_accuracy: 0.6271 - val_loss: 0.3810 - val_categorical_accuracy: 0.6883\n",
      "Epoch 8/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3805 - categorical_accuracy: 0.6339 - val_loss: 0.3743 - val_categorical_accuracy: 0.5801\n",
      "Epoch 9/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3624 - categorical_accuracy: 0.6310 - val_loss: 0.3553 - val_categorical_accuracy: 0.5584\n",
      "Epoch 10/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.3466 - categorical_accuracy: 0.6319 - val_loss: 0.3430 - val_categorical_accuracy: 0.5584\n",
      "Epoch 11/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.3481 - categorical_accuracy: 0.6416 - val_loss: 0.3465 - val_categorical_accuracy: 0.5931\n",
      "Epoch 12/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3344 - categorical_accuracy: 0.6387 - val_loss: 0.3323 - val_categorical_accuracy: 0.6797\n",
      "Epoch 13/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3258 - categorical_accuracy: 0.6392 - val_loss: 0.3272 - val_categorical_accuracy: 0.5325\n",
      "Epoch 14/100\n",
      "2073/2073 [==============================] - 56s 27ms/step - loss: 0.3192 - categorical_accuracy: 0.6271 - val_loss: 0.3231 - val_categorical_accuracy: 0.5411\n",
      "Epoch 15/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3153 - categorical_accuracy: 0.6266 - val_loss: 0.3339 - val_categorical_accuracy: 0.5887\n",
      "Epoch 16/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3090 - categorical_accuracy: 0.6281 - val_loss: 0.3197 - val_categorical_accuracy: 0.7100\n",
      "Epoch 17/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.3097 - categorical_accuracy: 0.6358 - val_loss: 0.3245 - val_categorical_accuracy: 0.6364\n",
      "Epoch 18/100\n",
      "2073/2073 [==============================] - 55s 26ms/step - loss: 0.2990 - categorical_accuracy: 0.6488 - val_loss: 0.3132 - val_categorical_accuracy: 0.5974\n",
      "Epoch 19/100\n",
      "2073/2073 [==============================] - 52s 25ms/step - loss: 0.2963 - categorical_accuracy: 0.6334 - val_loss: 0.3163 - val_categorical_accuracy: 0.5974\n",
      "Epoch 20/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.2929 - categorical_accuracy: 0.6372 - val_loss: 0.3214 - val_categorical_accuracy: 0.6364\n",
      "Epoch 21/100\n",
      "2073/2073 [==============================] - 52s 25ms/step - loss: 0.2900 - categorical_accuracy: 0.6329 - val_loss: 0.3076 - val_categorical_accuracy: 0.6494\n",
      "Epoch 22/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.2883 - categorical_accuracy: 0.6247 - val_loss: 0.3015 - val_categorical_accuracy: 0.6840\n",
      "Epoch 23/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.2853 - categorical_accuracy: 0.6189 - val_loss: 0.3097 - val_categorical_accuracy: 0.5541\n",
      "Epoch 24/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.2802 - categorical_accuracy: 0.6454 - val_loss: 0.3072 - val_categorical_accuracy: 0.5152\n",
      "Epoch 25/100\n",
      "2073/2073 [==============================] - 54s 26ms/step - loss: 0.2803 - categorical_accuracy: 0.6281 - val_loss: 0.3083 - val_categorical_accuracy: 0.5368\n",
      "Epoch 26/100\n",
      "2073/2073 [==============================] - 56s 27ms/step - loss: 0.2789 - categorical_accuracy: 0.6329 - val_loss: 0.2998 - val_categorical_accuracy: 0.6190\n",
      "Epoch 27/100\n",
      "2073/2073 [==============================] - 55s 27ms/step - loss: 0.2764 - categorical_accuracy: 0.6315 - val_loss: 0.3079 - val_categorical_accuracy: 0.5801\n",
      "Epoch 28/100\n",
      "2073/2073 [==============================] - 55s 26ms/step - loss: 0.2800 - categorical_accuracy: 0.6343 - val_loss: 0.3054 - val_categorical_accuracy: 0.5628\n",
      "Epoch 29/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.2736 - categorical_accuracy: 0.6310 - val_loss: 0.3032 - val_categorical_accuracy: 0.5238\n",
      "Epoch 30/100\n",
      "2073/2073 [==============================] - 53s 26ms/step - loss: 0.2708 - categorical_accuracy: 0.6440 - val_loss: 0.3045 - val_categorical_accuracy: 0.6320\n"
     ]
    }
   ],
   "source": [
    "trainCNN(x_train, y_train, cnn_model, 100, 'model/model-cnn.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "576/576 [==============================] - 2s 3ms/step\n",
      "loss: 0.2900873389508989\n",
      "categorical_accuracy: 0.6284722222222222\n"
     ]
    }
   ],
   "source": [
    "cnn_model = keras.models.load_model('model/model-cnn.h5')\n",
    "metrics = cnn_model.evaluate(x_test, y_test)\n",
    "print(\"{}: {}\".format(cnn_model.metrics_names[0], metrics[0]))\n",
    "print(\"{}: {}\".format(cnn_model.metrics_names[1], metrics[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train XGBoost model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_xg, x_test_xg = get_output_cnn(cnn_model, x_train, x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = BinaryRelevance(XGBClassifier())\n",
    "clf.fit(x_train_xg, y_train)\n",
    "y_pred = clf.predict(x_test_xg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Result CNN-XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "          ac_P1     0.9593    0.9752    0.9672       121\n",
      "   air_panas_P1     0.9306    0.9437    0.9371        71\n",
      "         bau_P1     0.8864    0.9176    0.9017        85\n",
      "        general     0.4400    0.3729    0.4037        59\n",
      "     kebersihan     0.8672    0.9367    0.9006       237\n",
      "       linen_P1     0.8418    0.8817    0.8613       169\n",
      "        service     0.8639    0.8581    0.8610       148\n",
      "sunrise_meal_P1     0.7826    0.7660    0.7742        47\n",
      "          tv_P1     0.8913    0.8039    0.8454        51\n",
      "        wifi_P1     0.9694    0.9596    0.9645        99\n",
      "\n",
      "      micro avg     0.8658    0.8786    0.8721      1087\n",
      "      macro avg     0.8433    0.8415    0.8417      1087\n",
      "   weighted avg     0.8623    0.8786    0.8698      1087\n",
      "    samples avg     0.9000    0.9038    0.8821      1087\n",
      "\n"
     ]
    }
   ],
   "source": [
    "category = mlb.classes_.tolist()\n",
    "print(classification_report(y_test,y_pred, target_names=category, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04861111111111111\n"
     ]
    }
   ],
   "source": [
    "hammloss = hamming_loss(y_test,y_pred)\n",
    "print(hammloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = ClassifierChain(XGBClassifier(), order=[0,1,9,8,5,4,2,3,7,6])\n",
    "clf.fit(x_train_xg, y_train)\n",
    "y_pred = clf.predict(x_test_xg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = y_pred.todense().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in y_pred:\n",
    "    p[0], p[1], p[9], p[8], p[5], p[4], p[2], p[3], p[7], p[6] =  p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.array(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "          ac_P1     0.9593    0.9752    0.9672       121\n",
      "   air_panas_P1     0.9306    0.9437    0.9371        71\n",
      "         bau_P1     0.8876    0.9294    0.9080        85\n",
      "        general     0.4902    0.4237    0.4545        59\n",
      "     kebersihan     0.8770    0.9325    0.9039       237\n",
      "       linen_P1     0.8466    0.8817    0.8638       169\n",
      "        service     0.8759    0.8581    0.8669       148\n",
      "sunrise_meal_P1     0.7955    0.7447    0.7692        47\n",
      "          tv_P1     0.8913    0.8039    0.8454        51\n",
      "        wifi_P1     0.9794    0.9596    0.9694        99\n",
      "\n",
      "      micro avg     0.8740    0.8804    0.8772      1087\n",
      "      macro avg     0.8533    0.8452    0.8485      1087\n",
      "   weighted avg     0.8711    0.8804    0.8752      1087\n",
      "    samples avg     0.9056    0.9069    0.8869      1087\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred, target_names=category, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.04652777777777778\n"
     ]
    }
   ],
   "source": [
    "hammloss = hamming_loss(y_test,y_pred)\n",
    "print(hammloss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost without CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "          ac_P1     0.8163    0.3306    0.4706       121\n",
      "   air_panas_P1     0.7333    0.1549    0.2558        71\n",
      "         bau_P1     0.4286    0.0353    0.0652        85\n",
      "        general     0.3333    0.0169    0.0323        59\n",
      "     kebersihan     0.5315    0.3207    0.4000       237\n",
      "       linen_P1     0.5493    0.2308    0.3250       169\n",
      "        service     0.6087    0.1892    0.2887       148\n",
      "sunrise_meal_P1     0.0000    0.0000    0.0000        47\n",
      "          tv_P1     0.6667    0.0392    0.0741        51\n",
      "        wifi_P1     1.0000    0.0909    0.1667        99\n",
      "\n",
      "      micro avg     0.6040    0.1923    0.2917      1087\n",
      "      macro avg     0.5668    0.1409    0.2078      1087\n",
      "   weighted avg     0.5969    0.1923    0.2716      1087\n",
      "    samples avg     0.2718    0.1998    0.2166      1087\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Annisa Nurul Azhar\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "C:\\Users\\Annisa Nurul Azhar\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "clf = BinaryRelevance(XGBClassifier())\n",
    "clf.fit(x_train, y_train)\n",
    "y_pred = clf.predict(x_test)\n",
    "category = mlb.classes_.tolist()\n",
    "print(classification_report(y_test, y_pred, target_names=category, digits=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline: CNN-SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "          ac_P1     0.9661    0.9421    0.9540       121\n",
      "   air_panas_P1     0.9394    0.8732    0.9051        71\n",
      "         bau_P1     0.8902    0.8588    0.8743        85\n",
      "        general     0.4615    0.2034    0.2824        59\n",
      "     kebersihan     0.8645    0.9156    0.8893       237\n",
      "       linen_P1     0.8683    0.8580    0.8631       169\n",
      "        service     0.8947    0.8041    0.8470       148\n",
      "sunrise_meal_P1     0.8214    0.4894    0.6133        47\n",
      "          tv_P1     0.8667    0.7647    0.8125        51\n",
      "        wifi_P1     0.9892    0.9293    0.9583        99\n",
      "\n",
      "      micro avg     0.8880    0.8243    0.8550      1087\n",
      "      macro avg     0.8562    0.7639    0.7999      1087\n",
      "   weighted avg     0.8752    0.8243    0.8443      1087\n",
      "    samples avg     0.9093    0.8640    0.8672      1087\n",
      "\n"
     ]
    }
   ],
   "source": [
    "svm_clf = BinaryRelevance(LinearSVC())\n",
    "svm_clf.fit(x_train_xg, y_train)\n",
    "y_pred = svm_clf.predict(x_test_xg)\n",
    "print(classification_report(y_test,y_pred, target_names=category, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.05277777777777778\n"
     ]
    }
   ],
   "source": [
    "hammloss = hamming_loss(y_test,y_pred)\n",
    "print(hammloss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline CNN-LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 180)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 180, 400)     2000000     input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 180, 128)     51328       embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 178, 128)     153728      embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 176, 128)     256128      embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1D)  (None, 1, 128)       0           conv1d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_5 (MaxPooling1D)  (None, 1, 128)       0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_6 (MaxPooling1D)  (None, 1, 128)       0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 3, 128)       0           max_pooling1d_4[0][0]            \n",
      "                                                                 max_pooling1d_5[0][0]            \n",
      "                                                                 max_pooling1d_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 3, 256)       33024       concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 3, 256)       0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 128)          197120      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 10)           1290        lstm_1[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 2,692,618\n",
      "Trainable params: 2,692,618\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "cnn_lstm_model = createCNNLSTMModel([1,3,5], 128, embedding_matrix, 400, 5000, 180, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2073 samples, validate on 231 samples\n",
      "Epoch 1/100\n",
      "2073/2073 [==============================] - 48s 23ms/step - loss: 2.3986 - categorical_accuracy: 0.2325 - val_loss: 0.7976 - val_categorical_accuracy: 0.4805\n",
      "Epoch 2/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.6048 - categorical_accuracy: 0.4182 - val_loss: 0.5075 - val_categorical_accuracy: 0.5281\n",
      "Epoch 3/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.4764 - categorical_accuracy: 0.5263 - val_loss: 0.4493 - val_categorical_accuracy: 0.5844\n",
      "Epoch 4/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.4395 - categorical_accuracy: 0.5499 - val_loss: 0.4115 - val_categorical_accuracy: 0.6364\n",
      "Epoch 5/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.4115 - categorical_accuracy: 0.5697 - val_loss: 0.3886 - val_categorical_accuracy: 0.5844\n",
      "Epoch 6/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3843 - categorical_accuracy: 0.6040 - val_loss: 0.3728 - val_categorical_accuracy: 0.6450\n",
      "Epoch 7/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3689 - categorical_accuracy: 0.6257 - val_loss: 0.3630 - val_categorical_accuracy: 0.5931\n",
      "Epoch 8/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3531 - categorical_accuracy: 0.6397 - val_loss: 0.3571 - val_categorical_accuracy: 0.6970\n",
      "Epoch 9/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.3392 - categorical_accuracy: 0.6372 - val_loss: 0.3461 - val_categorical_accuracy: 0.5844\n",
      "Epoch 10/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.3306 - categorical_accuracy: 0.6368 - val_loss: 0.3435 - val_categorical_accuracy: 0.5368\n",
      "Epoch 11/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.3183 - categorical_accuracy: 0.6315 - val_loss: 0.3342 - val_categorical_accuracy: 0.5455\n",
      "Epoch 12/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3113 - categorical_accuracy: 0.6315 - val_loss: 0.3256 - val_categorical_accuracy: 0.6364\n",
      "Epoch 13/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3041 - categorical_accuracy: 0.6536 - val_loss: 0.3283 - val_categorical_accuracy: 0.5541\n",
      "Epoch 14/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.3089 - categorical_accuracy: 0.6401 - val_loss: 0.3341 - val_categorical_accuracy: 0.5455\n",
      "Epoch 15/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.2956 - categorical_accuracy: 0.6454 - val_loss: 0.3175 - val_categorical_accuracy: 0.6797\n",
      "Epoch 16/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.2934 - categorical_accuracy: 0.6546 - val_loss: 0.3271 - val_categorical_accuracy: 0.5411\n",
      "Epoch 17/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.2925 - categorical_accuracy: 0.6536 - val_loss: 0.3205 - val_categorical_accuracy: 0.6797\n",
      "Epoch 18/100\n",
      "2073/2073 [==============================] - 43s 21ms/step - loss: 0.2886 - categorical_accuracy: 0.6551 - val_loss: 0.3308 - val_categorical_accuracy: 0.7186\n",
      "Epoch 19/100\n",
      "2073/2073 [==============================] - 42s 20ms/step - loss: 0.2885 - categorical_accuracy: 0.6614 - val_loss: 0.3227 - val_categorical_accuracy: 0.5455\n"
     ]
    }
   ],
   "source": [
    "trainCNN(x_train, y_train, cnn_lstm_model, 100, 'model/model-cnn-lstm.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "576/576 [==============================] - 4s 7ms/step\n",
      "loss: 0.322007159392039\n",
      "categorical_accuracy: 0.6701388888888888\n"
     ]
    }
   ],
   "source": [
    "cnn_lstm_model = keras.models.load_model('model/model-cnn-lstm.h5')\n",
    "metrics = cnn_lstm_model.evaluate(x_test, y_test)\n",
    "print(\"{}: {}\".format(cnn_lstm_model.metrics_names[0], metrics[0]))\n",
    "print(\"{}: {}\".format(cnn_lstm_model.metrics_names[1], metrics[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = cnn_lstm_model.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_bool = (y_pred > 0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.array([[0 if x==False else 1 for x in arr] for arr in y_pred_bool])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 1, 0, 0, 0])"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 precision    recall  f1-score   support\n",
      "\n",
      "          ac_P1     0.9435    0.9669    0.9551       121\n",
      "   air_panas_P1     0.9079    0.9718    0.9388        71\n",
      "         bau_P1     0.8444    0.8941    0.8686        85\n",
      "        general     0.0952    0.0339    0.0500        59\n",
      "     kebersihan     0.8129    0.9536    0.8777       237\n",
      "       linen_P1     0.8045    0.8521    0.8276       169\n",
      "        service     0.7975    0.8514    0.8235       148\n",
      "sunrise_meal_P1     0.0000    0.0000    0.0000        47\n",
      "          tv_P1     0.7576    0.9804    0.8547        51\n",
      "        wifi_P1     0.8505    0.9192    0.8835        99\n",
      "\n",
      "      micro avg     0.8191    0.8289    0.8240      1087\n",
      "      macro avg     0.6814    0.7423    0.7079      1087\n",
      "   weighted avg     0.7594    0.8289    0.7910      1087\n",
      "    samples avg     0.8578    0.8627    0.8383      1087\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred, target_names=category, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.06684027777777778\n"
     ]
    }
   ],
   "source": [
    "hammloss = hamming_loss(y_test,y_pred)\n",
    "print(hammloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}