03_AspectSentiment-Notebook - AdaBound.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\run_configuration.py:150: SyntaxWarning: assertion is always true, perhaps remove parentheses?\n",
      "  assert(config.model_size % config.n_heads == 0, f\"number of heads {config.n_heads} is not a valid number of heads for model size {config.model_size}.\")\n"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "import copy\n",
    "import logging\n",
    "\n",
    "from data.data_loader import Dataset\n",
    "from data.germeval2017 import germeval2017_dataset\n",
    "\n",
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import get_default_params, randomize_params, LearningSchedulerType\n",
    "from misc import utils\n",
    "\n",
    "from optimizer import get_default_optimizer\n",
    "from criterion import NllLoss, LossCombiner\n",
    "\n",
    "from models.transformer.encoder import TransformerEncoder\n",
    "from models.jointAspectTagger import JointAspectTagger\n",
    "from models.transformer.train import Trainer\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "    data_root='./data/germeval2017',\n",
    "    data_train='train_v1.4.tsv',    \n",
    "    data_validation='dev_v1.4.tsv',\n",
    "    data_test='test_TIMESTAMP1.tsv',\n",
    "    early_stopping='highest_5_F1'\n",
    ")\n",
    "def load(hp, logger):\n",
    "    dataset = Dataset(\n",
    "        'germeval',\n",
    "        logger,\n",
    "        hp,\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        data_path=PREFERENCES.data_root,\n",
    "        train_file=PREFERENCES.data_train,\n",
    "        valid_file=PREFERENCES.data_validation,\n",
    "        test_file=PREFERENCES.data_test,\n",
    "        file_format='.tsv',\n",
    "        init_token=None,\n",
    "        eos_token=None\n",
    "    )\n",
    "    dataset.load_data(germeval2017_dataset, verbose=True)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_model(dataset, hp, experiment_name):\n",
    "    loss = LossCombiner(4, dataset.class_weights, NllLoss)\n",
    "    transformer = TransformerEncoder(dataset.source_embedding,\n",
    "                                     hyperparameters=hp)\n",
    "    model = JointAspectTagger(transformer, hp, 4, 20, dataset.target_names)\n",
    "    optimizer = get_default_optimizer(model, hp)\n",
    "    trainer = Trainer(\n",
    "                        model,\n",
    "                        loss,\n",
    "                        optimizer,\n",
    "                        hp,\n",
    "                        dataset,\n",
    "                        experiment_name,\n",
    "                        enable_tensorboard=False,\n",
    "                        verbose=True)\n",
    "    return trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiment_name = 'AdaBound'\n",
    "use_cuda = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\AdaBound\\20190226\\0\n"
     ]
    }
   ],
   "source": [
    "# get general logger just for search\n",
    "experiment_name = utils.create_loggers(experiment_name=experiment_name)\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "utils.get_current_git_commit()\n",
    "logger.info('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------------------------+\n",
      "|                     Hyperparameters                      |\n",
      "+-------------------------+--------------------------------+\n",
      "|        Parameter        |             Value              |\n",
      "+-------------------------+--------------------------------+\n",
      "|        batch_size       |               11               |\n",
      "|        model_size       |              300               |\n",
      "|    learning_rate_type   | LearningSchedulerType.AdaBound |\n",
      "|      learning_rate      |               1                |\n",
      "|   learning_rate_warmup  |              4800              |\n",
      "|   learning_rate_factor  |               2                |\n",
      "|     optim_adam_beta1    |              0.9               |\n",
      "|     optim_adam_beta2    |              0.98              |\n",
      "|      early_stopping     |               5                |\n",
      "|         use_cuda        |              True              |\n",
      "|       n_enc_blocks      |               2                |\n",
      "|         n_heads         |               10               |\n",
      "|           d_k           |               30               |\n",
      "|           d_v           |               30               |\n",
      "|       dropout_rate      |             0.069              |\n",
      "|   pointwise_layer_size  |              134               |\n",
      "|    output_layer_type    |   OutputLayerType.LinearSum    |\n",
      "| output_conv_num_filters |              300               |\n",
      "| output_conv_kernel_size |               5                |\n",
      "|    output_conv_stride   |               1                |\n",
      "|   output_conv_padding   |               0                |\n",
      "| log_every_xth_iteration |               -1               |\n",
      "|        num_epochs       |               35               |\n",
      "|      embedding_type     |            fasttext            |\n",
      "|      embedding_name     |               6B               |\n",
      "|      embedding_dim      |              300               |\n",
      "|     clip_comments_to    |              391               |\n",
      "|         language        |               de               |\n",
      "|      use_stop_words     |              True              |\n",
      "|           seed          |               42               |\n",
      "+-------------------------+--------------------------------+\n"
     ]
    }
   ],
   "source": [
    "hp = get_default_params(use_cuda)\n",
    "hp.num_epochs = 35\n",
    "hp.batch_size = 11\n",
    "#hp.learning_rate = 0.00275\n",
    "#hp.learning_rate_warmup = 7628.240\n",
    "#hp.learning_rate_factor = 0.893619\n",
    "#hp.optim_adam_beta1 = 0.8315827\n",
    "#hp.optim_adam_beta1 = 0.937167988175\n",
    "hp.n_enc_blocks = 2\n",
    "hp.n_heads = 10\n",
    "hp.d_k = 30\n",
    "hp.d_v = 30\n",
    "hp.dropout_rate = 0.069\n",
    "hp.pointwise_layer_size = 134\n",
    "hp.clip_comments_to = 391\n",
    "hp.learning_rate_type = LearningSchedulerType.AdaBound\n",
    "logger.info(hp)\n",
    "print(hp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|           comments           | 84428 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 29470 |\n",
      "|            Image             |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|            Design            |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|          Allgemein           |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                              Image                              |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "| negative |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "| positive |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "| neutral  |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------+\n",
      "|                           Toiletten                            |\n",
      "+----------+---------+----------------------+--------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+----------+---------+----------------------+--------------------+\n",
      "|   n/a    |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "| negative |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "| neutral  |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "| positive |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|   Sum    |  21187  |                      |        1.0         |\n",
      "+----------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                               Design                              |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|   n/a    |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "| negative |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "| positive |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "| neutral  |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|   Sum    |  21187  |                      |          1.0          |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                         Barrierefreiheit                        |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "| positive |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "| neutral  |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "| negative |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                     Komfort_und_Ausstattung                     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "| positive |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "| negative |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "| neutral  |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                        Reisen_mit_Kindern                        |\n",
      "+----------+---------+----------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+----------+---------+----------------------+----------------------+\n",
      "|   n/a    |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "| positive |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "| neutral  |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "| negative |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|   Sum    |  21187  |                      |         1.0          |\n",
      "+----------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------+\n",
      "|                           Ticketkauf                          |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|  Label   | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|   n/a    |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "| negative |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "| neutral  |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "| positive |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|   Sum    |  21187  |                    |         1.0         |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------+\n",
      "|                            Zugfahrt                           |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|  Label   | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|   n/a    |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "| positive |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "| negative |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "| neutral  |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|   Sum    |  21187  |                    |         1.0         |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------+\n",
      "|                  Service_und_Kundenbetreuung                  |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|  Label   | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|   n/a    |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "| negative |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "| positive |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "| neutral  |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|   Sum    |  21187  |                    |         1.0         |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------+\n",
      "|                           Atmosphäre                          |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|  Label   | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "|   n/a    |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "| negative |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "| positive |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "| neutral  |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|   Sum    |  21187  |                    |         1.0         |\n",
      "+----------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                          Informationen                          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "| positive |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "| negative |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "| neutral  |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Sicherheit                           |\n",
      "+----------+---------+----------------------+---------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+----------+---------+----------------------+---------------------+\n",
      "|   n/a    |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "| negative |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "| positive |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "| neutral  |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|   Sum    |  21187  |                      |         1.0         |\n",
      "+----------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                               Gepäck                              |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|   n/a    |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "| neutral  |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "| negative |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "| positive |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   Sum    |  21187  |                      |          1.0          |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------+\n",
      "|                          Allgemein                           |\n",
      "+----------+---------+-------------------+---------------------+\n",
      "|  Label   | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+----------+---------+-------------------+---------------------+\n",
      "| neutral  |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|   n/a    |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "| negative |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "| positive |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|   Sum    |  21187  |                   |         1.0         |\n",
      "+----------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                        DB_App_und_Website                       |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "| positive |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "| negative |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "| neutral  |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                     Gastronomisches_Angebot                      |\n",
      "+----------+---------+---------------------+-----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+----------+---------+---------------------+-----------------------+\n",
      "|   n/a    |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "| negative |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "| neutral  |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "| positive |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|   Sum    |  21187  |                     |          1.0          |\n",
      "+----------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                   Auslastung_und_Platzangebot                   |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "| negative |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "| positive |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "| neutral  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                           Connectivity                          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "|   n/a    |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "| positive |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "| negative |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "| neutral  |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|   Sum    |  21187  |                     |         1.0          |\n",
      "+----------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                              QR-Code                              |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "|   n/a    |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "| positive |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "| neutral  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   Sum    |  21187  |                      |          1.0          |\n",
      "+----------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------+\n",
      "|                  Sonstige_Unregelmässigkeiten                  |\n",
      "+----------+---------+---------------------+---------------------+\n",
      "|  Label   | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+----------+---------+---------------------+---------------------+\n",
      "|   n/a    |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "| negative |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "| positive |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "| neutral  |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|   Sum    |  21187  |                     |         1.0         |\n",
      "+----------+---------+---------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - 20 initialized\n",
      "pre_training - DEBUG - Initilize parameters with nn.init.xavier_uniform_\n",
      "pre_training - DEBUG - Tagger initialized\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (src_embeddings): Embedding(84428, 300)\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.1)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=134, bias=True)\n",
      "          (w_2): Linear(in_features=134, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=134, bias=True)\n",
      "          (w_2): Linear(in_features=134, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "  ), weights=((84428, 300), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (134, 300), (134,), (300, 134), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (134, 300), (134,), (300, 134), (300,), (300,), (300,), (300,), (300,)), parameters=26214268\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 26.238.348\n",
      "==================================\n",
      "\n",
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "         Embedding-1             [-1, 391, 300]      25,328,400\n",
      "           Dropout-2             [-1, 391, 300]               0\n",
      "PositionalEncoding2-3             [-1, 391, 300]               0\n",
      "            Linear-4             [-1, 391, 300]          90,000\n",
      "            Linear-5             [-1, 391, 300]          90,000\n",
      "            Linear-6             [-1, 391, 300]          90,000\n",
      "           Dropout-7             [-1, 391, 391]               0\n",
      "ScaledDotProductAttentionLayer-8              [-1, 391, 30]               0\n",
      "            Linear-9             [-1, 391, 300]          90,000\n",
      "          Dropout-10             [-1, 391, 300]               0\n",
      "        LayerNorm-11             [-1, 391, 300]               0\n",
      "MultiHeadedSelfAttentionLayer-12             [-1, 391, 300]               0\n",
      "           Linear-13             [-1, 391, 134]          40,334\n",
      "           Linear-14             [-1, 391, 300]          40,500\n",
      "          Dropout-15             [-1, 391, 300]               0\n",
      "        LayerNorm-16             [-1, 391, 300]               0\n",
      "     EncoderBlock-17             [-1, 391, 300]               0\n",
      "           Linear-18             [-1, 391, 300]          90,000\n",
      "           Linear-19             [-1, 391, 300]          90,000\n",
      "           Linear-20             [-1, 391, 300]          90,000\n",
      "          Dropout-21             [-1, 391, 391]               0\n",
      "ScaledDotProductAttentionLayer-22              [-1, 391, 30]               0\n",
      "           Linear-23             [-1, 391, 300]          90,000\n",
      "          Dropout-24             [-1, 391, 300]               0\n",
      "        LayerNorm-25             [-1, 391, 300]               0\n",
      "MultiHeadedSelfAttentionLayer-26             [-1, 391, 300]               0\n",
      "           Linear-27             [-1, 391, 134]          40,334\n",
      "           Linear-28             [-1, 391, 300]          40,500\n",
      "          Dropout-29             [-1, 391, 300]               0\n",
      "        LayerNorm-30             [-1, 391, 300]               0\n",
      "     EncoderBlock-31             [-1, 391, 300]               0\n",
      "TransformerEncoder-32             [-1, 391, 300]               0\n",
      "           Linear-33               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-34                    [-1, 4]               0\n",
      "           Linear-35               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-36                    [-1, 4]               0\n",
      "           Linear-37               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-38                    [-1, 4]               0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Linear-39               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-40                    [-1, 4]               0\n",
      "           Linear-41               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-42                    [-1, 4]               0\n",
      "           Linear-43               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-44                    [-1, 4]               0\n",
      "           Linear-45               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-46                    [-1, 4]               0\n",
      "           Linear-47               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-48                    [-1, 4]               0\n",
      "           Linear-49               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-50                    [-1, 4]               0\n",
      "           Linear-51               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-52                    [-1, 4]               0\n",
      "           Linear-53               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-54                    [-1, 4]               0\n",
      "           Linear-55               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-56                    [-1, 4]               0\n",
      "           Linear-57               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-58                    [-1, 4]               0\n",
      "           Linear-59               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-60                    [-1, 4]               0\n",
      "           Linear-61               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-62                    [-1, 4]               0\n",
      "           Linear-63               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-64                    [-1, 4]               0\n",
      "           Linear-65               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-66                    [-1, 4]               0\n",
      "           Linear-67               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-68                    [-1, 4]               0\n",
      "           Linear-69               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-70                    [-1, 4]               0\n",
      "           Linear-71               [-1, 391, 4]           1,204\n",
      "CommentWiseSumLogSoftmax-72                    [-1, 4]               0\n",
      "================================================================\n",
      "Total params: 26,234,148\n",
      "Trainable params: 26,234,148\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 26.82\n",
      "Params size (MB): 100.08\n",
      "Estimated Total Size (MB): 126.90\n",
      "----------------------------------------------------------------\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (src_embeddings): Embedding(84428, 300)\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.1)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=134, bias=True)\n",
      "          (w_2): Linear(in_features=134, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.1)\n",
      "          )\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (layer_norm): LayerNorm()\n",
      "          (w_1): Linear(in_features=300, out_features=134, bias=True)\n",
      "          (w_2): Linear(in_features=134, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.069)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "  ), weights=((84428, 300), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (134, 300), (134,), (300, 134), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300,), (300,), (300, 300), (300,), (300,), (134, 300), (134,), (300, 134), (300,), (300,), (300,), (300,), (300,)), parameters=26214268\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 26.238.348\n",
      "==================================\n",
      "\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - DEBUG - train with cuda support\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - INFO - 1550 Iterations per epoch with batch size of 11\n",
      "pre_training - INFO - Total iterations: 54250\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\t1550\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.40m - 6.4m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t3100\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.35m - 12.8m / 224.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t4650\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.43m - 19.3m / 222.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t6200\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.31m - 25.6m / 225.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t7750\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.25m - 31.8m / 221.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                                                       \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t9300\tnan\t\tnan\t\t0.240\t\t0.944\t\t6.19m - 38.0m / 219.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6: 100%|█████████████████████████████████████████████████████████████████████| 1550/1550 [38:01<00:00,  4.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre_training - DEBUG - --- Valid Scores ---\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((nan, 0.2403811203710183, None),\n",
       " (nan, 0.24035293348721692, None),\n",
       " (-1, -1, array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "         [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_logger = logging.getLogger('data_loader')\n",
    "    \n",
    "logger.debug('Load dataset')\n",
    "dataset = load(hp, dataset_logger)\n",
    "logger.debug('dataset loaded')\n",
    "logger.debug('Load model')\n",
    "trainer = load_model(dataset, hp, experiment_name)\n",
    "logger.debug('model loaded')\n",
    "\n",
    "logger.debug('Begin training')\n",
    "model = None\n",
    "result = trainer.train(use_cuda=hp.use_cuda, perform_evaluation=False)\n",
    "model = result['model']\n",
    "\n",
    "# perform evaluation and log results\n",
    "result = trainer.perform_final_evaluation(use_test_set=False, verbose=False)\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}