18_XX_GermEvalExperiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\run_configuration.py:527: SyntaxWarning: assertion is always true, perhaps remove parentheses?\n",
      "  assert(config.model_size % config.n_heads == 0, f'number of heads {config.n_heads} is not a valid number of heads for model size {config.model_size}.')\n"
     ]
    }
   ],
   "source": [
    "from misc.preferences import PREFERENCES\n",
    "from misc.run_configuration import good_germeval_params, default_params, OutputLayerType\n",
    "from misc.experimental_environment import Experiment\n",
    "import time\n",
    "from data.germeval2017 import germeval2017_dataset as dsl\n",
    "import pprint\n",
    "from misc import utils\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "STATUS_FAIL = 'fail'\n",
    "STATUS_OK = 'ok'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "        data_root='./data/data/germeval2017',\n",
    "        data_train='train_v1.4.tsv',    \n",
    "        data_validation='dev_v1.4.tsv',\n",
    "        data_test='test_TIMESTAMP1.tsv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        file_format='csv',\n",
    "        language='de'\n",
    "    )\n",
    "main_experiment_name = 'GermEval-2017_Experiment'\n",
    "use_cuda = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{ 'att_d_k': 60,\n",
      "  'att_d_v': 60,\n",
      "  'batch_size': 26,\n",
      "  'clip_comments_to': 230,\n",
      "  'dropout_rate': 0.3116352148277839,\n",
      "  'early_stopping': 10,\n",
      "  'embedding_dim': 300,\n",
      "  'embedding_name': '6B',\n",
      "  'embedding_type': 'fasttext',\n",
      "  'language': 'de',\n",
      "  'learning_rate_scheduler': { 'noam_learning_rate_factor': 1.120962889284992,\n",
      "                               'noam_learning_rate_warmup': 6706},\n",
      "  'learning_rate_scheduler_type': <LearningSchedulerType.Noam: 1>,\n",
      "  'log_every_xth_iteration': -1,\n",
      "  'model_size': 300,\n",
      "  'num_encoder_blocks': 6,\n",
      "  'num_epochs': 35,\n",
      "  'num_heads': 5,\n",
      "  'optimizer': { 'adam_beta1': 0.8278419040185792,\n",
      "                 'adam_beta2': 0.7523040247084006,\n",
      "                 'adam_eps': 0.001028230097476593,\n",
      "                 'adam_weight_decay': 0.0001,\n",
      "                 'learning_rate': 0.09624249687454951},\n",
      "  'optimizer_type': <OptimizerType.Adam: 1>,\n",
      "  'output_conv_kernel_size': 5,\n",
      "  'output_conv_num_filters': 117,\n",
      "  'output_conv_padding': 0,\n",
      "  'output_conv_stride': 9,\n",
      "  'output_dropout_rate': 0.1410769136750667,\n",
      "  'output_layer_type': <OutputLayerType.LinearSum: 1>,\n",
      "  'pointwise_layer_size': 199,\n",
      "  'task': 'germeval',\n",
      "  'transformer_use_bias': True,\n",
      "  'use_spell_checkers': False,\n",
      "  'use_stop_words': True}\n"
     ]
    }
   ],
   "source": [
    "baseline = good_germeval_params\n",
    "print(pprint.pformat(baseline, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "experiments = [\n",
    "    {\n",
    "        'name': 'Baseline - LM-H',\n",
    "        'description': 'Baseline classification for the GermEval-2017 task using the linear mean head',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'task': 'germeval'\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'Baseline - CNN-H',\n",
    "        'description': 'Baseline classification for the GermEval-2017 task using the cnn head',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'task': 'germeval',\n",
    "            'output_layer_type': OutputLayerType.Convolutions\n",
    "        }\n",
    "    },\n",
    "    {\n",
    "        'name': 'GermEval Multitask',\n",
    "        'description': 'Multitask Classification',\n",
    "        'loss': 1000,\n",
    "        'f1': -1,\n",
    "        'rc': {\n",
    "            'task': 'germeval_multitask'\n",
    "        }\n",
    "    }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current commit: b'eb9da80'\n"
     ]
    }
   ],
   "source": [
    "utils.get_current_git_commit()\n",
    "print('Current commit: ' + utils.get_current_git_commit())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline - LM-H\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment GermEval-2017_Experiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\0\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\1\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 22.103993892669678\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 24.033.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "262f9a8a7d724aa4b06c5e62e9bbe5df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.53\t\t0.38\t\t0.218\t\t0.870\t\t4.80m - 4.8m / 0.0m\n",
      "2\t34k\t0.38\t\t0.33\t\t0.308\t\t0.917\t\t4.34m - 9.2m / 167.9m\n",
      "3\t51k\t0.36\t\t0.33\t\t0.308\t\t0.916\t\t4.36m - 13.6m / 152.6m\n",
      "4\t68k\t0.35\t\t0.36\t\t0.294\t\t0.912\t\t4.33m - 17.9m / 153.1m\n",
      "5\t85k\t0.35\t\t0.35\t\t0.342\t\t0.930\t\t4.33m - 22.3m / 152.3m\n",
      "6\t102k\t0.35\t\t0.37\t\t0.276\t\t0.901\t\t4.36m - 26.6m / 152.2m\n",
      "7\t119k\t0.36\t\t0.36\t\t0.281\t\t0.904\t\t4.34m - 31.0m / 153.1m\n",
      "8\t136k\t0.35\t\t0.37\t\t0.358\t\t0.937\t\t4.22m - 35.2m / 152.4m\n",
      "9\t154k\t0.35\t\t0.38\t\t0.244\t\t0.906\t\t4.20m - 39.5m / 149.3m\n",
      "10\t171k\t0.35\t\t0.38\t\t0.345\t\t0.933\t\t4.21m - 43.7m / 148.8m\n",
      "11\t188k\t0.34\t\t0.39\t\t0.348\t\t0.934\t\t4.21m - 47.9m / 149.0m\n",
      "12\t205k\t0.32\t\t0.40\t\t0.355\t\t0.937\t\t4.22m - 52.2m / 149.1m\n",
      "13\t222k\t0.30\t\t0.37\t\t0.398\t\t0.949\t\t4.20m - 56.4m / 149.3m\n",
      "14\t239k\t0.29\t\t0.38\t\t0.400\t\t0.948\t\t4.22m - 60.6m / 148.9m\n",
      "15\t256k\t0.27\t\t0.40\t\t0.406\t\t0.950\t\t4.21m - 64.9m / 149.2m\n",
      "16\t273k\t0.26\t\t0.49\t\t0.442\t\t0.956\t\t4.21m - 69.1m / 149.1m\n",
      "17\t290k\t0.25\t\t0.46\t\t0.377\t\t0.942\t\t4.21m - 73.3m / 149.1m\n",
      "18\t307k\t0.24\t\t0.52\t\t0.396\t\t0.949\t\t4.22m - 77.6m / 149.1m\n",
      "19\t324k\t0.23\t\t0.49\t\t0.425\t\t0.954\t\t4.21m - 81.8m / 149.4m\n",
      "20\t341k\t0.22\t\t0.52\t\t0.335\t\t0.944\t\t4.22m - 86.0m / 149.2m\n",
      "21\t358k\t0.21\t\t0.55\t\t0.433\t\t0.958\t\t4.21m - 90.3m / 149.3m\n",
      "22\t375k\t0.21\t\t0.65\t\t0.440\t\t0.959\t\t4.20m - 94.5m / 149.2m\n",
      "23\t392k\t0.20\t\t0.68\t\t0.413\t\t0.953\t\t4.20m - 98.7m / 149.1m\n",
      "24\t409k\t0.19\t\t0.61\t\t0.408\t\t0.957\t\t4.20m - 102.9m / 149.1m\n",
      "25\t426k\t0.19\t\t0.70\t\t0.453\t\t0.960\t\t4.23m - 107.1m / 149.2m\n",
      "26\t443k\t0.19\t\t0.72\t\t0.450\t\t0.959\t\t4.22m - 111.4m / 149.5m\n",
      "27\t461k\t0.18\t\t0.57\t\t0.440\t\t0.959\t\t4.21m - 115.6m / 149.4m\n",
      "28\t478k\t0.18\t\t0.61\t\t0.415\t\t0.955\t\t4.20m - 119.8m / 149.3m\n",
      "29\t495k\t0.18\t\t0.60\t\t0.465\t\t0.962\t\t4.22m - 124.1m / 149.2m\n",
      "30\t512k\t0.18\t\t0.58\t\t0.443\t\t0.960\t\t4.20m - 128.3m / 149.4m\n",
      "31\t529k\t0.17\t\t0.60\t\t0.446\t\t0.960\t\t4.20m - 132.5m / 149.3m\n",
      "32\t546k\t0.17\t\t0.63\t\t0.436\t\t0.959\t\t4.23m - 136.8m / 149.4m\n",
      "33\t563k\t0.17\t\t0.64\t\t0.445\t\t0.960\t\t4.20m - 141.0m / 149.5m\n",
      "34\t580k\t0.16\t\t0.53\t\t0.408\t\t0.952\t\t4.23m - 145.2m / 149.4m\n",
      "35\t597k\t0.16\t\t0.62\t\t0.461\t\t0.962\t\t4.20m - 149.4m / 149.5m\n",
      "Training duration was 8977.296337842941\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.11613567158030791\n",
      "VAL f1\t0.4646884272997033 - (0.4646884272997033)\n",
      "(macro) f1\t{'valid': 0.15260158290022222, 'test': 0.11613567158030791}\n",
      "VAL loss\t0.3313939276549461\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [0/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3313939276549461\n",
      " | \\     )|_\tf1: 0.4646884272997033\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\2\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 22.90478253364563\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 24.033.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa6761574f15464c9a7263497cbea3e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.53\t\t0.36\t\t0.300\t\t0.913\t\t4.19m - 4.2m / 0.0m\n",
      "2\t34k\t0.37\t\t0.34\t\t0.293\t\t0.908\t\t4.19m - 8.4m / 146.8m\n",
      "3\t51k\t0.36\t\t0.33\t\t0.348\t\t0.932\t\t4.22m - 12.7m / 146.7m\n",
      "4\t68k\t0.35\t\t0.33\t\t0.277\t\t0.901\t\t4.20m - 16.9m / 147.6m\n",
      "5\t85k\t0.35\t\t0.36\t\t0.298\t\t0.911\t\t4.21m - 21.1m / 146.9m\n",
      "6\t102k\t0.35\t\t0.37\t\t0.357\t\t0.936\t\t4.20m - 25.3m / 147.3m\n",
      "7\t119k\t0.35\t\t0.37\t\t0.239\t\t0.885\t\t4.20m - 29.6m / 147.1m\n",
      "8\t136k\t0.35\t\t0.36\t\t0.373\t\t0.939\t\t4.20m - 33.8m / 147.0m\n",
      "9\t154k\t0.35\t\t0.39\t\t0.245\t\t0.899\t\t4.20m - 38.0m / 147.1m\n",
      "10\t171k\t0.34\t\t0.37\t\t0.358\t\t0.938\t\t4.22m - 42.2m / 147.3m\n",
      "11\t188k\t0.33\t\t0.35\t\t0.307\t\t0.920\t\t4.20m - 46.4m / 147.9m\n",
      "12\t205k\t0.31\t\t0.38\t\t0.337\t\t0.932\t\t4.20m - 50.7m / 147.2m\n",
      "13\t222k\t0.30\t\t0.38\t\t0.405\t\t0.949\t\t4.21m - 54.9m / 147.2m\n",
      "14\t239k\t0.28\t\t0.39\t\t0.335\t\t0.933\t\t4.19m - 59.1m / 147.5m\n",
      "15\t256k\t0.27\t\t0.42\t\t0.399\t\t0.949\t\t4.19m - 63.3m / 147.0m\n",
      "16\t273k\t0.26\t\t0.53\t\t0.413\t\t0.951\t\t4.19m - 67.5m / 147.0m\n",
      "17\t290k\t0.25\t\t0.43\t\t0.389\t\t0.944\t\t4.22m - 71.8m / 147.1m\n",
      "18\t307k\t0.24\t\t0.60\t\t0.378\t\t0.948\t\t4.19m - 76.0m / 147.7m\n",
      "19\t324k\t0.23\t\t0.48\t\t0.374\t\t0.945\t\t4.20m - 80.2m / 147.2m\n",
      "20\t341k\t0.22\t\t0.53\t\t0.410\t\t0.952\t\t4.21m - 84.4m / 147.4m\n",
      "21\t358k\t0.21\t\t0.66\t\t0.453\t\t0.961\t\t4.20m - 88.6m / 147.5m\n",
      "22\t375k\t0.21\t\t0.62\t\t0.427\t\t0.955\t\t4.20m - 92.8m / 147.4m\n",
      "23\t392k\t0.19\t\t0.65\t\t0.409\t\t0.952\t\t4.20m - 97.0m / 147.4m\n",
      "24\t409k\t0.19\t\t0.58\t\t0.430\t\t0.957\t\t4.19m - 101.3m / 147.4m\n",
      "25\t426k\t0.19\t\t0.75\t\t0.453\t\t0.961\t\t4.21m - 105.5m / 147.4m\n",
      "26\t443k\t0.18\t\t0.69\t\t0.442\t\t0.958\t\t4.19m - 109.7m / 147.6m\n",
      "27\t461k\t0.18\t\t0.68\t\t0.443\t\t0.958\t\t4.20m - 113.9m / 147.4m\n",
      "28\t478k\t0.17\t\t0.60\t\t0.398\t\t0.953\t\t4.19m - 118.1m / 147.5m\n",
      "29\t495k\t0.18\t\t0.68\t\t0.454\t\t0.958\t\t4.19m - 122.3m / 147.5m\n",
      "30\t512k\t0.17\t\t0.60\t\t0.441\t\t0.959\t\t4.19m - 126.5m / 147.5m\n",
      "31\t529k\t0.16\t\t0.63\t\t0.401\t\t0.950\t\t4.21m - 130.8m / 147.5m\n",
      "32\t546k\t0.16\t\t0.64\t\t0.426\t\t0.957\t\t4.22m - 135.0m / 147.6m\n",
      "33\t563k\t0.16\t\t0.61\t\t0.429\t\t0.957\t\t4.20m - 139.2m / 147.6m\n",
      "34\t580k\t0.16\t\t0.59\t\t0.405\t\t0.953\t\t4.20m - 143.4m / 147.6m\n",
      "35\t597k\t0.15\t\t0.60\t\t0.414\t\t0.953\t\t4.20m - 147.6m / 147.6m\n",
      "Training duration was 8867.425741910934\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.11438483788701506\n",
      "VAL f1\t0.45393258426966293 - (0.45393258426966293)\n",
      "(macro) f1\t{'valid': 0.14929553351012528, 'test': 0.11438483788701506}\n",
      "VAL loss\t0.32679489710546855\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [1/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.32679489710546855\n",
      " | \\     )|_\tf1: 0.45393258426966293\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\3\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 23.199719190597534\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 24.033.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79d67ac29f82430c8a580622a7b80f52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.53\t\t0.37\t\t0.267\t\t0.896\t\t4.19m - 4.2m / 0.0m\n",
      "2\t34k\t0.37\t\t0.34\t\t0.293\t\t0.912\t\t4.23m - 8.5m / 146.7m\n",
      "3\t51k\t0.35\t\t0.32\t\t0.334\t\t0.928\t\t4.20m - 12.7m / 148.0m\n",
      "4\t68k\t0.35\t\t0.35\t\t0.273\t\t0.903\t\t4.22m - 16.9m / 147.0m\n",
      "5\t85k\t0.35\t\t0.34\t\t0.289\t\t0.908\t\t4.21m - 21.2m / 147.7m\n",
      "6\t102k\t0.35\t\t0.34\t\t0.349\t\t0.932\t\t4.21m - 25.4m / 147.6m\n",
      "7\t119k\t0.35\t\t0.37\t\t0.244\t\t0.887\t\t4.21m - 29.6m / 147.6m\n",
      "8\t136k\t0.35\t\t0.40\t\t0.350\t\t0.934\t\t4.21m - 33.8m / 147.5m\n",
      "9\t154k\t0.35\t\t0.37\t\t0.271\t\t0.912\t\t4.23m - 38.1m / 147.6m\n",
      "10\t171k\t0.34\t\t0.35\t\t0.319\t\t0.922\t\t4.32m - 42.5m / 148.2m\n",
      "11\t188k\t0.34\t\t0.37\t\t0.294\t\t0.914\t\t4.23m - 46.7m / 150.5m\n",
      "12\t205k\t0.31\t\t0.35\t\t0.360\t\t0.938\t\t4.22m - 50.9m / 148.3m\n",
      "13\t222k\t0.29\t\t0.38\t\t0.392\t\t0.948\t\t4.21m - 55.2m / 148.0m\n",
      "14\t239k\t0.28\t\t0.40\t\t0.399\t\t0.948\t\t4.22m - 59.4m / 147.8m\n",
      "15\t256k\t0.27\t\t0.43\t\t0.330\t\t0.936\t\t4.31m - 63.8m / 148.1m\n",
      "16\t273k\t0.26\t\t0.44\t\t0.383\t\t0.945\t\t4.22m - 68.0m / 150.0m\n",
      "17\t290k\t0.24\t\t0.49\t\t0.399\t\t0.948\t\t4.25m - 72.3m / 148.2m\n",
      "18\t307k\t0.23\t\t0.44\t\t0.397\t\t0.948\t\t4.21m - 76.5m / 148.7m\n",
      "19\t324k\t0.22\t\t0.52\t\t0.436\t\t0.957\t\t4.21m - 80.7m / 148.1m\n",
      "20\t341k\t0.22\t\t0.60\t\t0.445\t\t0.960\t\t4.22m - 85.0m / 148.1m\n",
      "21\t358k\t0.21\t\t0.60\t\t0.447\t\t0.959\t\t4.20m - 89.2m / 148.2m\n",
      "22\t375k\t0.21\t\t0.64\t\t0.435\t\t0.958\t\t4.22m - 93.4m / 148.1m\n",
      "23\t392k\t0.20\t\t0.69\t\t0.457\t\t0.961\t\t4.22m - 97.7m / 148.3m\n",
      "24\t409k\t0.19\t\t0.61\t\t0.441\t\t0.959\t\t4.24m - 102.1m / 148.5m\n",
      "25\t426k\t0.19\t\t0.72\t\t0.440\t\t0.958\t\t4.23m - 106.3m / 148.7m\n",
      "26\t443k\t0.18\t\t0.66\t\t0.441\t\t0.959\t\t4.22m - 110.6m / 148.6m\n",
      "27\t461k\t0.18\t\t0.63\t\t0.460\t\t0.961\t\t4.22m - 114.8m / 148.5m\n",
      "28\t478k\t0.17\t\t0.65\t\t0.420\t\t0.956\t\t4.22m - 119.0m / 148.5m\n",
      "29\t495k\t0.17\t\t0.64\t\t0.444\t\t0.958\t\t4.22m - 123.3m / 148.6m\n",
      "30\t512k\t0.17\t\t0.68\t\t0.445\t\t0.960\t\t4.22m - 127.5m / 148.6m\n",
      "31\t529k\t0.17\t\t0.65\t\t0.427\t\t0.958\t\t4.24m - 131.7m / 148.6m\n",
      "32\t546k\t0.16\t\t0.52\t\t0.371\t\t0.946\t\t4.22m - 136.0m / 148.7m\n",
      "33\t563k\t0.16\t\t0.59\t\t0.408\t\t0.954\t\t4.22m - 140.2m / 148.7m\n",
      "34\t580k\t0.15\t\t0.52\t\t0.393\t\t0.949\t\t4.34m - 144.6m / 148.7m\n",
      "35\t597k\t0.15\t\t0.62\t\t0.432\t\t0.958\t\t4.32m - 148.9m / 148.9m\n",
      "Training duration was 8944.200639486313\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.12119197678036638\n",
      "VAL f1\t0.4598265895953757 - (0.4598265895953757)\n",
      "(macro) f1\t{'valid': 0.1600267710070547, 'test': 0.12119197678036638}\n",
      "VAL loss\t0.324681851160445\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [2/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.324681851160445\n",
      " | \\     )|_\tf1: 0.4598265895953757\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\4\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 23.865264415740967\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 24.033.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7163c6db3cb4ba3b15b6837b8b7347d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.52\t\t0.36\t\t0.260\t\t0.893\t\t4.38m - 4.4m / 0.0m\n",
      "2\t34k\t0.38\t\t0.35\t\t0.300\t\t0.914\t\t4.36m - 8.8m / 153.4m\n",
      "3\t51k\t0.36\t\t0.33\t\t0.307\t\t0.917\t\t4.50m - 13.3m / 152.6m\n",
      "4\t68k\t0.35\t\t0.35\t\t0.260\t\t0.894\t\t4.44m - 17.8m / 157.3m\n",
      "5\t85k\t0.35\t\t0.34\t\t0.301\t\t0.913\t\t4.43m - 22.2m / 155.4m\n",
      "6\t102k\t0.35\t\t0.38\t\t0.335\t\t0.926\t\t4.39m - 26.6m / 155.2m\n",
      "7\t119k\t0.36\t\t0.37\t\t0.251\t\t0.891\t\t4.34m - 31.0m / 154.0m\n",
      "8\t136k\t0.35\t\t0.40\t\t0.294\t\t0.917\t\t4.36m - 35.4m / 152.7m\n",
      "9\t154k\t0.35\t\t0.37\t\t0.265\t\t0.908\t\t4.36m - 39.8m / 153.2m\n",
      "10\t171k\t0.35\t\t0.35\t\t0.338\t\t0.932\t\t4.32m - 44.1m / 153.1m\n",
      "11\t188k\t0.34\t\t0.36\t\t0.300\t\t0.913\t\t4.32m - 48.5m / 152.1m\n",
      "12\t205k\t0.33\t\t0.35\t\t0.369\t\t0.939\t\t4.31m - 52.8m / 152.1m\n",
      "13\t222k\t0.31\t\t0.36\t\t0.371\t\t0.941\t\t4.31m - 57.1m / 151.9m\n",
      "14\t239k\t0.29\t\t0.40\t\t0.360\t\t0.938\t\t4.31m - 61.5m / 152.0m\n",
      "15\t256k\t0.28\t\t0.45\t\t0.348\t\t0.935\t\t4.33m - 65.8m / 152.0m\n",
      "16\t273k\t0.27\t\t0.44\t\t0.416\t\t0.951\t\t4.31m - 70.1m / 152.4m\n",
      "17\t290k\t0.25\t\t0.41\t\t0.410\t\t0.950\t\t4.32m - 74.5m / 152.1m\n",
      "18\t307k\t0.24\t\t0.49\t\t0.406\t\t0.951\t\t4.31m - 78.8m / 152.2m\n",
      "19\t324k\t0.24\t\t0.46\t\t0.396\t\t0.948\t\t4.31m - 83.1m / 152.1m\n",
      "20\t341k\t0.23\t\t0.58\t\t0.436\t\t0.959\t\t4.24m - 87.4m / 152.1m\n",
      "21\t358k\t0.22\t\t0.49\t\t0.400\t\t0.949\t\t4.24m - 91.7m / 151.0m\n",
      "22\t375k\t0.22\t\t0.72\t\t0.445\t\t0.959\t\t4.24m - 95.9m / 151.0m\n",
      "23\t392k\t0.21\t\t0.62\t\t0.418\t\t0.955\t\t4.22m - 100.2m / 151.0m\n",
      "24\t409k\t0.20\t\t0.56\t\t0.439\t\t0.957\t\t4.22m - 104.4m / 150.9m\n",
      "25\t426k\t0.19\t\t0.67\t\t0.427\t\t0.956\t\t4.23m - 108.7m / 150.9m\n",
      "26\t443k\t0.19\t\t0.67\t\t0.451\t\t0.959\t\t4.23m - 112.9m / 151.0m\n",
      "27\t461k\t0.18\t\t0.58\t\t0.427\t\t0.955\t\t4.23m - 117.2m / 151.0m\n",
      "28\t478k\t0.18\t\t0.59\t\t0.412\t\t0.956\t\t4.23m - 121.4m / 151.0m\n",
      "29\t495k\t0.18\t\t0.61\t\t0.448\t\t0.960\t\t4.23m - 125.7m / 151.0m\n",
      "30\t512k\t0.17\t\t0.55\t\t0.428\t\t0.957\t\t4.24m - 129.9m / 151.1m\n",
      "31\t529k\t0.17\t\t0.63\t\t0.383\t\t0.951\t\t4.24m - 134.2m / 151.1m\n",
      "32\t546k\t0.17\t\t0.62\t\t0.400\t\t0.957\t\t4.22m - 138.4m / 151.2m\n",
      "33\t563k\t0.17\t\t0.63\t\t0.426\t\t0.957\t\t4.23m - 142.6m / 151.1m\n",
      "34\t580k\t0.17\t\t0.61\t\t0.451\t\t0.960\t\t4.22m - 146.9m / 151.1m\n",
      "35\t597k\t0.16\t\t0.57\t\t0.400\t\t0.953\t\t4.23m - 151.1m / 151.1m\n",
      "Training duration was 9074.165378808975\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.11115530603327275\n",
      "VAL f1\t0.4510822510822511 - (0.4510822510822511)\n",
      "(macro) f1\t{'valid': 0.14239010524723822, 'test': 0.11115530603327275}\n",
      "VAL loss\t0.3256822244101861\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [3/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3256822244101861\n",
      " | \\     )|_\tf1: 0.4510822510822511\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\5\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 20.537307024002075\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 24.033.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cbe8cbf3ab44b03ace8ecd06d3044f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.50\t\t0.36\t\t0.301\t\t0.914\t\t4.21m - 4.2m / 0.0m\n",
      "2\t34k\t0.37\t\t0.35\t\t0.284\t\t0.908\t\t4.22m - 8.5m / 147.4m\n",
      "3\t51k\t0.36\t\t0.33\t\t0.337\t\t0.928\t\t4.22m - 12.7m / 147.9m\n",
      "4\t68k\t0.35\t\t0.35\t\t0.232\t\t0.890\t\t4.22m - 17.0m / 147.8m\n",
      "5\t85k\t0.35\t\t0.34\t\t0.299\t\t0.913\t\t4.22m - 21.2m / 147.7m\n",
      "6\t102k\t0.35\t\t0.34\t\t0.293\t\t0.911\t\t4.23m - 25.4m / 147.9m\n",
      "7\t119k\t0.35\t\t0.39\t\t0.222\t\t0.871\t\t4.33m - 29.8m / 148.1m\n",
      "8\t136k\t0.35\t\t0.38\t\t0.320\t\t0.926\t\t4.25m - 34.0m / 151.1m\n",
      "9\t154k\t0.35\t\t0.37\t\t0.303\t\t0.921\t\t4.40m - 38.5m / 148.8m\n",
      "10\t171k\t0.35\t\t0.36\t\t0.343\t\t0.934\t\t4.24m - 42.7m / 152.9m\n",
      "11\t188k\t0.34\t\t0.36\t\t0.284\t\t0.910\t\t4.33m - 47.1m / 148.7m\n",
      "12\t205k\t0.32\t\t0.34\t\t0.355\t\t0.937\t\t4.34m - 51.4m / 151.1m\n",
      "13\t222k\t0.30\t\t0.38\t\t0.406\t\t0.950\t\t4.32m - 55.8m / 151.3m\n",
      "14\t239k\t0.29\t\t0.39\t\t0.364\t\t0.938\t\t4.34m - 60.2m / 150.8m\n",
      "15\t256k\t0.28\t\t0.39\t\t0.344\t\t0.935\t\t4.47m - 64.6m / 151.4m\n",
      "16\t273k\t0.26\t\t0.46\t\t0.380\t\t0.944\t\t4.41m - 69.1m / 154.0m\n",
      "17\t290k\t0.25\t\t0.42\t\t0.416\t\t0.951\t\t4.77m - 73.9m / 152.8m\n",
      "18\t307k\t0.24\t\t0.49\t\t0.417\t\t0.953\t\t4.50m - 78.4m / 159.7m\n",
      "19\t324k\t0.23\t\t0.53\t\t0.435\t\t0.957\t\t4.46m - 82.9m / 155.0m\n",
      "20\t341k\t0.22\t\t0.48\t\t0.418\t\t0.955\t\t4.53m - 87.5m / 154.2m\n",
      "21\t358k\t0.22\t\t0.62\t\t0.452\t\t0.960\t\t4.42m - 91.9m / 155.5m\n",
      "22\t375k\t0.22\t\t0.70\t\t0.446\t\t0.959\t\t4.38m - 96.3m / 153.8m\n",
      "23\t392k\t0.21\t\t0.62\t\t0.411\t\t0.952\t\t4.38m - 100.7m / 153.2m\n",
      "24\t409k\t0.20\t\t0.52\t\t0.420\t\t0.955\t\t4.36m - 105.1m / 153.3m\n",
      "25\t426k\t0.20\t\t0.66\t\t0.427\t\t0.956\t\t4.32m - 109.4m / 153.1m\n",
      "26\t443k\t0.19\t\t0.60\t\t0.458\t\t0.961\t\t4.34m - 113.8m / 152.7m\n",
      "27\t461k\t0.19\t\t0.56\t\t0.402\t\t0.949\t\t4.35m - 118.3m / 153.0m\n",
      "28\t478k\t0.18\t\t0.67\t\t0.448\t\t0.958\t\t4.32m - 122.6m / 153.1m\n",
      "29\t495k\t0.18\t\t0.63\t\t0.433\t\t0.955\t\t4.32m - 126.9m / 152.9m\n",
      "30\t512k\t0.18\t\t0.57\t\t0.434\t\t0.958\t\t4.31m - 131.3m / 152.9m\n",
      "31\t529k\t0.17\t\t0.66\t\t0.441\t\t0.959\t\t4.32m - 135.6m / 152.8m\n",
      "32\t546k\t0.17\t\t0.56\t\t0.432\t\t0.956\t\t4.32m - 139.9m / 152.9m\n",
      "33\t563k\t0.17\t\t0.61\t\t0.441\t\t0.959\t\t4.32m - 144.3m / 152.9m\n",
      "34\t580k\t0.16\t\t0.58\t\t0.433\t\t0.958\t\t4.35m - 148.6m / 152.9m\n",
      "35\t597k\t0.16\t\t0.59\t\t0.430\t\t0.957\t\t4.35m - 153.0m / 153.0m\n",
      "Training duration was 9186.94121336937\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.11687052618379043\n",
      "VAL f1\t0.4580759046778464 - (0.4580759046778464)\n",
      "(macro) f1\t{'valid': 0.14962657377197283, 'test': 0.11687052618379043}\n",
      "VAL loss\t0.32923449780483677\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [4/5]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.32923449780483677\n",
      " | \\     )|_\tf1: 0.4580759046778464\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "#################################################################################\n",
      "############################## EXPERIMENT COMPLETE ##############################\n",
      "\n",
      "\n",
      "Run [0/5]: 0.38534278959810875\n",
      "Run [1/5]: 0.3865441906653426\n",
      "Run [2/5]: 0.38578680203045684\n",
      "Run [3/5]: 0.37452615617892343\n",
      "Run [4/5]: 0.38577754623599897\n",
      "------------------------------\n",
      "Mean: 0.38359549694176615\n",
      "TEST MICRO F1 Statistics\n",
      "count    5.000000\n",
      "mean     0.383595\n",
      "std      0.005088\n",
      "min      0.374526\n",
      "25%      0.385343\n",
      "50%      0.385778\n",
      "75%      0.385787\n",
      "max      0.386544\n",
      "Name: test_f1, dtype: float64\n",
      "TEST MACRO F1 Statistics\n",
      "count    5.000000\n",
      "mean     0.115948\n",
      "std      0.003667\n",
      "min      0.111155\n",
      "25%      0.114385\n",
      "50%      0.116136\n",
      "75%      0.116871\n",
      "max      0.121192\n",
      "Name: test_f1_macro, dtype: float64\n",
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline - CNN-H\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment GermEval-2017_Experiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\6\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\7\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the cnn head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   output_conv_num_filters    |                        300                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   2095  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 21187 |\n",
      "|           comments           | 70418 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 25733 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   3   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------+\n",
      "|                            Allgemein                            |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy  |     Class Weight    |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "|   neutral   |  12308  | 58.09222636522395 | 0.41907773634776047 |\n",
      "|     n/a     |   6176  | 29.14995044130835 |  0.7085004955869165 |\n",
      "|   negative  |   1922  | 9.071600509746542 |  0.9092839949025345 |\n",
      "|   positive  |   781   |  3.68622268372115 |  0.9631377731627885 |\n",
      "|     Sum     |  21187  |                   |         1.0         |\n",
      "| Head Weight |         |                   |  0.2914995044130835 |\n",
      "+-------------+---------+-------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Atmosphäre                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19854  | 93.70840609807901  | 0.06291593901920989 |\n",
      "|   negative  |   1035  | 4.8850710341246995 |  0.951149289658753  |\n",
      "|   positive  |   133   | 0.6277434275735121 |  0.9937225657242649 |\n",
      "|   neutral   |   165   | 0.7787794402227781 |  0.9922122055977722 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9370840609807901 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20868  |  98.49435974890262  | 0.015056402510973732 |\n",
      "|   negative  |   261   |  1.2318874781705764 |  0.9876811252182942  |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   neutral   |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9849435974890263  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21097  |  99.57521121442394  | 0.004247887855760579 |\n",
      "|   positive  |    20   |  0.0943975079057913 |  0.999056024920942   |\n",
      "|   neutral   |    16   | 0.07551800632463303 |  0.9992448199367536  |\n",
      "|   negative  |    54   |  0.2548732713456365 |  0.9974512672865437  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9957521121442394  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20871  |   98.5085193750885  | 0.014914806249115009 |\n",
      "|   positive  |    69   |  0.3256714022749799 |  0.9967432859772501  |\n",
      "|   negative  |   197   |  0.9298154528720441 |  0.9907018454712796  |\n",
      "|   neutral   |    50   | 0.23599376976447822 |  0.9976400623023552  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.985085193750885   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                         |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20951  |  98.88610940671167  | 0.011138905932883358 |\n",
      "|   positive  |    41   | 0.19351489120687212 |  0.9980648510879313  |\n",
      "|   negative  |   134   |  0.6324633029688016 |  0.993675366970312   |\n",
      "|   neutral   |    61   |  0.2879123991126634 |  0.9971208760088733  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9888610940671166  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21148  |   99.8159248595837   | 0.0018407514041629547 |\n",
      "|   negative  |    18   | 0.08495775711521215  |   0.9991504224288479  |\n",
      "|   positive  |    19   | 0.08967763251050172  |   0.999103223674895   |\n",
      "|   neutral   |    2    | 0.009439750790579128 |   0.9999056024920943  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.998159248595837   |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                       |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |      Class Weight     |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "|     n/a     |  21136  |  99.75928635484024  | 0.0024071364515976246 |\n",
      "|   negative  |    32   | 0.15103601264926605 |   0.9984896398735074  |\n",
      "|   neutral   |    10   | 0.04719875395289565 |   0.9995280124604711  |\n",
      "|   positive  |    9    | 0.04247887855760608 |   0.9995752112144239  |\n",
      "|     Sum     |  21187  |                     |          1.0          |\n",
      "| Head Weight |         |                     |   0.9975928635484024  |\n",
      "+-------------+---------+---------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Gepäck                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21164  |  99.89144286590835   | 0.0010855713409165801 |\n",
      "|   neutral   |    10   | 0.04719875395289565  |   0.9995280124604711  |\n",
      "|   negative  |    12   | 0.056638504743474774 |   0.9994336149525652  |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9989144286590834  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                               Image                                |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  21129  |   99.7262472270732  | 0.002737527729267941 |\n",
      "|   negative  |    34   | 0.16047576343984518 |  0.9983952423656015  |\n",
      "|   positive  |    10   | 0.04719875395289565 |  0.9995280124604711  |\n",
      "|   neutral   |    14   |  0.0660782555340539 |  0.9993392174446595  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9972624722707321  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                           Informationen                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20765  |   98.0082125831878  | 0.019917874168121963 |\n",
      "|   positive  |    51   |  0.2407136451597678 |  0.9975928635484024  |\n",
      "|   negative  |   285   |  1.345164487657526  |  0.9865483551234248  |\n",
      "|   neutral   |    86   | 0.40590928399490256 |  0.995940907160051   |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.980082125831878   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20989  |  99.06546467173266  | 0.009345353282673385 |\n",
      "|   positive  |    48   |  0.2265540189738991 |  0.997734459810261   |\n",
      "|   negative  |   117   |  0.5522254212488791 |  0.9944777457875112  |\n",
      "|   neutral   |    33   | 0.15575588804455562 |  0.9984424411195545  |\n",
      "|     Sum     |  21187  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9906546467173266  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                               QR-Code                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  21185  |  99.99056024920942   | 9.439750790574131e-05 |\n",
      "|   positive  |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|   neutral   |    1    | 0.004719875395289564 |   0.9999528012460471  |\n",
      "|     Sum     |  21187  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9999056024920943  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                         |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  21132  |  99.74040685325907   | 0.002595931467409218 |\n",
      "|   positive  |    8    | 0.037759003162316514 |  0.9996224099683768  |\n",
      "|   neutral   |    17   | 0.08023788171992259  |  0.9991976211828008  |\n",
      "|   negative  |    30   | 0.14159626185868693  |  0.9985840373814131  |\n",
      "|     Sum     |  21187  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974040685325908  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                   Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20589  | 97.17751451361684  | 0.02822485486383164 |\n",
      "|   negative  |   388   | 1.831311653372351  |  0.9816868834662765 |\n",
      "|   positive  |   146   | 0.6891018077122764 |  0.9931089819228772 |\n",
      "|   neutral   |    64   | 0.3020720252985321 |  0.9969792797470147 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9717751451361684 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight    |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "|     n/a     |  20611  |   97.2813517723132   | 0.02718648227686793 |\n",
      "|   negative  |   542   |  2.558172464246944   |  0.9744182753575306 |\n",
      "|   positive  |    22   | 0.10383725869637041  |  0.9989616274130363 |\n",
      "|   neutral   |    12   | 0.056638504743474774 |  0.9994336149525652 |\n",
      "|     Sum     |  21187  |                      |         1.0         |\n",
      "| Head Weight |         |                      |  0.9728135177231321 |\n",
      "+-------------+---------+----------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  19483  |  91.95733232642658  | 0.08042667673573423 |\n",
      "|   negative  |   1582  |  7.4668428753480915 |  0.9253315712465191 |\n",
      "|   positive  |    49   | 0.23127389436918866 |  0.9976872610563081 |\n",
      "|   neutral   |    73   |  0.3445509038561382 |  0.9965544909614387 |\n",
      "|     Sum     |  21187  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9195733232642658 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  20429  |  96.4223344503705  | 0.03577665549629494 |\n",
      "|   negative  |   505   | 2.3835370746212297 |  0.9761646292537877 |\n",
      "|   neutral   |   137   | 0.6466229291546703 |  0.9935337707084533 |\n",
      "|   positive  |   116   | 0.5475055458535895 |  0.9945249445414641 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9642233445037051 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Toiletten                             |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |    Class Weight    |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "|     n/a     |  21131  |  99.73568697786378   | 0.0026431302213622 |\n",
      "|   negative  |    41   | 0.19351489120687212  | 0.9980648510879313 |\n",
      "|   neutral   |    13   | 0.06135838013876433  | 0.9993864161986123 |\n",
      "|   positive  |    2    | 0.009439750790579128 | 0.9999056024920943 |\n",
      "|     Sum     |  21187  |                      |        1.0         |\n",
      "| Head Weight |         |                      | 0.9973568697786378 |\n",
      "+-------------+---------+----------------------+--------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18958  | 89.47939774389955  | 0.10520602256100442 |\n",
      "|   positive  |   371   | 1.7510737716524283 |  0.9824892622834758 |\n",
      "|   negative  |   1706  | 8.052107424363996  |   0.91947892575636  |\n",
      "|   neutral   |   152   | 0.7174210600840137 |  0.9928257893991599 |\n",
      "|     Sum     |  21187  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947939774389956 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                           Test                          |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           |  47 | 414  | 1112 | 1573 |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 365  |  5   | 373  |\n",
      "|          Ticketkauf          |  32 |  50  |  76  | 158  |\n",
      "|          Sicherheit          |  6  | 188  |  3   | 197  |\n",
      "|          Atmosphäre          |  22 | 222  |  28  | 272  |\n",
      "|           Zugfahrt           |  51 | 251  |  23  | 325  |\n",
      "|        Informationen         |  5  |  37  |  37  |  79  |\n",
      "| Auslastung_und_Platzangebot  |  5  |  37  |  2   |  44  |\n",
      "|      DB_App_und_Website      |  3  |  20  |  12  |  35  |\n",
      "| Service_und_Kundenbetreuung  |  16 |  43  |  20  |  79  |\n",
      "|         Connectivity         |  16 |  27  |  8   |  51  |\n",
      "|      Reisen_mit_Kindern      |  0  |  5   |  5   |  10  |\n",
      "|   Komfort_und_Ausstattung    |  9  |  10  |  10  |  29  |\n",
      "|            Design            |  5  |  1   |  0   |  6   |\n",
      "|          Toiletten           |  0  |  5   |  3   |  8   |\n",
      "|       Barrierefreiheit       |  5  |  6   |  2   |  13  |\n",
      "|            Gepäck            |  0  |  1   |  1   |  2   |\n",
      "|   Gastronomisches_Angebot    |  1  |  4   |  1   |  6   |\n",
      "|           QR-Code            |  0  |  0   |  1   |  1   |\n",
      "|            Total             | 226 | 1686 | 1349 | 3261 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 19.5501651763916\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(70418, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (70418, 300)), parameters=24008994\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (1): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (2): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (3): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (4): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (5): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (6): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (7): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (8): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (9): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (10): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (11): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (12): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (13): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (14): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (15): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (16): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (17): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (18): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "    (19): CommentWiseConvLogSoftmax(\n",
      "      (conv): Conv2d(1, 300, kernel_size=(5, 300), stride=(1, 1))\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "      (pooling): MaxPool2d(kernel_size=(226, 1), stride=1, padding=0, dilation=1, ceil_mode=False)\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "    )\n",
      "  ), weights=((300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,), (300, 1, 5, 300), (300,), (4, 300), (4,)), parameters=9030080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 33.039.074\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcfe8dcb55a24022a34db4381a147c58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.48\t\t0.41\t\t0.253\t\t0.893\t\t6.73m - 6.7m / 0.0m\n",
      "2\t34k\t0.44\t\t0.44\t\t0.145\t\t0.843\t\t6.85m - 13.6m / 235.5m\n",
      "3\t51k\t0.50\t\t0.59\t\t0.130\t\t0.738\t\t6.81m - 20.4m / 239.6m\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-4012c1bada68>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[0mtest_params\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mtest_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'rc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0me\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mExperiment\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmain_experiment_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'description'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdsl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mruns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\experimental_environment.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    257\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mruns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_initialize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 259\u001b[1;33m                         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_objective\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    260\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_print_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    261\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\experimental_environment.py\u001b[0m in \u001b[0;36m_objective\u001b[1;34m(self, rc, run)\u001b[0m\n\u001b[0;32m    174\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m                         \u001b[0mtr_start\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 176\u001b[1;33m                         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mperform_evaluation\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    177\u001b[0m                         \u001b[0mtr_end\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m                         \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\trainer\\train.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, use_cuda, perform_evaluation)\u001b[0m\n\u001b[0;32m    334\u001b[0m                                                 \u001b[0mpadding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    335\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 336\u001b[1;33m                                         \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msource_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    337\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_logger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_loss_history\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'loss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcurrent_sample_iteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_logger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'lr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'general'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcurrent_sample_iteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\trainer\\train.py\u001b[0m in \u001b[0;36m_step\u001b[1;34m(self, input, target, source_mask)\u001b[0m\n\u001b[0;32m    196\u001b[0m                 \u001b[1;31m# preform training step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    197\u001b[0m                 \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 198\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    199\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    200\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\optimizer.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m     21\u001b[0m             \u001b[0mp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'lr'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mzero_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\optim\\adam.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, closure)\u001b[0m\n\u001b[0;32m     92\u001b[0m                 \u001b[1;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m                 \u001b[0mexp_avg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 94\u001b[1;33m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     95\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m                     \u001b[1;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for e in [experiments[0]]:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    test_params = {**baseline, **{'num_epochs': 35, 'language': 'de'}}\n",
    "    test_params = {**test_params, **e['rc']}\n",
    "    e = Experiment(main_experiment_name, e['description'], default_params, test_params, dsl, runs=2)\n",
    "    df, e_path = e.run()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Further experiments beyond baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline - CNN-H\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment GermEval-2017_Experiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190502\\7\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190502\\8\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the cnn head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   output_conv_num_filters    |                        300                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "dataset loaded. Duration: 25.536199808120728\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "410e1932ff324ec1a6c6f99e25e4ee53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.48\t\t0.56\t\t0.114\t\t0.752\t\t7.14m - 7.1m / 0.0m\n",
      "2\t34k\t0.43\t\t0.44\t\t0.180\t\t0.877\t\t7.14m - 14.3m / 249.8m\n",
      "3\t51k\t0.49\t\t0.52\t\t0.132\t\t0.765\t\t7.21m - 21.6m / 250.1m\n",
      "4\t68k\t0.47\t\t0.45\t\t0.210\t\t0.861\t\t7.47m - 29.0m / 252.3m\n",
      "5\t85k\t0.48\t\t0.45\t\t0.185\t\t0.838\t\t8.06m - 37.2m / 260.6m\n",
      "6\t102k\t0.49\t\t0.45\t\t0.189\t\t0.842\t\t7.53m - 44.7m / 279.1m\n",
      "7\t119k\t0.46\t\t0.45\t\t0.177\t\t0.854\t\t7.09m - 51.8m / 263.2m\n",
      "8\t136k\t0.46\t\t0.41\t\t0.268\t\t0.897\t\t7.05m - 58.9m / 250.5m\n",
      "9\t154k\t0.44\t\t0.45\t\t0.205\t\t0.856\t\t7.45m - 66.4m / 249.2m\n",
      "10\t171k\t0.43\t\t0.40\t\t0.292\t\t0.915\t\t7.34m - 73.7m / 260.0m\n",
      "11\t188k\t0.42\t\t0.39\t\t0.271\t\t0.901\t\t7.73m - 81.5m / 257.3m\n",
      "12\t205k\t0.39\t\t0.50\t\t0.161\t\t0.798\t\t7.65m - 89.2m / 266.9m\n",
      "13\t222k\t0.37\t\t0.43\t\t0.310\t\t0.920\t\t7.87m - 97.1m / 265.0m\n",
      "14\t239k\t0.36\t\t0.39\t\t0.344\t\t0.933\t\t7.96m - 105.2m / 270.3m\n",
      "15\t256k\t0.35\t\t0.36\t\t0.324\t\t0.923\t\t7.17m - 112.5m / 272.4m\n",
      "16\t273k\t0.34\t\t0.35\t\t0.415\t\t0.954\t\t7.15m - 119.6m / 256.0m\n",
      "17\t290k\t0.33\t\t0.39\t\t0.293\t\t0.913\t\t7.15m - 126.9m / 255.6m\n",
      "18\t307k\t0.33\t\t0.38\t\t0.321\t\t0.925\t\t7.21m - 134.1m / 255.6m\n",
      "19\t324k\t0.32\t\t0.38\t\t0.331\t\t0.928\t\t7.21m - 141.3m / 256.6m\n",
      "20\t341k\t0.32\t\t0.36\t\t0.325\t\t0.927\t\t7.30m - 148.6m / 256.6m\n",
      "21\t358k\t0.32\t\t0.38\t\t0.374\t\t0.941\t\t7.17m - 155.8m / 258.2m\n",
      "22\t375k\t0.32\t\t0.39\t\t0.293\t\t0.912\t\t7.01m - 162.8m / 256.2m\n",
      "23\t392k\t0.31\t\t0.40\t\t0.339\t\t0.933\t\t7.00m - 169.9m / 254.0m\n",
      "24\t409k\t0.31\t\t0.42\t\t0.336\t\t0.931\t\t6.93m - 176.8m / 253.9m\n",
      "25\t426k\t0.31\t\t0.42\t\t0.284\t\t0.916\t\t6.96m - 183.8m / 253.0m\n",
      "26\t443k\t0.30\t\t0.44\t\t0.387\t\t0.944\t\t6.88m - 190.7m / 253.4m\n",
      "Training duration was 11449.181146860123\n",
      "VAL f1\t0.4147002983455384 - (0.4147002983455384)\n",
      "(macro) f1\t{'valid': 0.11801352720697116, 'test': 0.10231300318506942}\n",
      "VAL loss\t0.3527421216537565\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [1/2]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3527421216537565\n",
      " | \\     )|_\tf1: 0.4147002983455384\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190503\\0\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the cnn head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |            OutputLayerType.Convolutions           |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   output_conv_num_filters    |                        300                        |\n",
      "|   output_conv_kernel_size    |                         5                         |\n",
      "|      output_conv_stride      |                         1                         |\n",
      "|     output_conv_padding      |                         0                         |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "dataset loaded. Duration: 22.60133981704712\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "902e063a45004c71bb582c8905cbfee8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-c1f06e773dbb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[0mtest_params\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mtest_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'rc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0me\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mExperiment\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmain_experiment_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'description'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdefault_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdsl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mruns\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0me_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\experimental_environment.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    257\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mruns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_initialize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 259\u001b[1;33m                         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_objective\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    260\u001b[0m                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_print_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    261\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\misc\\experimental_environment.py\u001b[0m in \u001b[0;36m_objective\u001b[1;34m(self, rc, run)\u001b[0m\n\u001b[0;32m    174\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m                         \u001b[0mtr_start\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 176\u001b[1;33m                         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mrc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muse_cuda\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mperform_evaluation\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    177\u001b[0m                         \u001b[0mtr_end\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m                         \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'model'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\trainer\\train.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(self, use_cuda, perform_evaluation)\u001b[0m\n\u001b[0;32m    334\u001b[0m                                                 \u001b[0mpadding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    335\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 336\u001b[1;33m                                         \u001b[0mtrain_loss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msource_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    337\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_logger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_loss_history\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'loss'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcurrent_sample_iteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m                                         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_logger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'lr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'general'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcurrent_sample_iteration\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\trainer\\train.py\u001b[0m in \u001b[0;36m_step\u001b[1;34m(self, input, target, source_mask)\u001b[0m\n\u001b[0;32m    192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    193\u001b[0m                 \u001b[1;31m# Compute loss and gradient\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 194\u001b[1;33m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msource_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    195\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    196\u001b[0m                 \u001b[1;31m# preform training step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\trainer\\train_evaluator.py\u001b[0m in \u001b[0;36mget_loss\u001b[1;34m(self, input, source_mask, target)\u001b[0m\n\u001b[0;32m     97\u001b[0m \t\t\"\"\"\n\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m                 \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msource_mask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    488\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 489\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    490\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\models\\jointAspectTagger.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x, *args)\u001b[0m\n\u001b[0;32m     82\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m                 \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;31m# result will be [batch_size, num_words, model_size]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m                 \u001b[0moutput\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTensor\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    488\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 489\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    490\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\models\\transformer\\encoder.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x, mask)\u001b[0m\n\u001b[0;32m     81\u001b[0m                 \u001b[1;31m# apply the forward pass for each encoding sub layer\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     82\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menc_sub_layer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoder_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 83\u001b[1;33m                         \u001b[0mencoder_output\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menc_sub_layer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mencoder_output\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     84\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     85\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mencoder_output\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    488\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 489\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    490\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\models\\transformer\\encoder.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, x, mask)\u001b[0m\n\u001b[0;32m    146\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    147\u001b[0m                 \u001b[0mfcResult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeed_forward_layer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattentionResult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 148\u001b[1;33m                 \u001b[0mfcResult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayer_norm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfcResult\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mresidual\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    149\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    150\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mfcResult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for e in [experiments[1]]:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    test_params = {**baseline, **{'num_epochs': 35, 'language': 'de'}}\n",
    "    test_params = {**test_params, **e['rc']}\n",
    "    e = Experiment(main_experiment_name, e['description'], default_params, test_params, dsl, runs=2)\n",
    "    df, e_path = e.run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multitask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: GermEval Multitask\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment GermEval-2017_Experiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190502\\1\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190502\\2\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Multitask Classification\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                 germeval_multitask                |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         45                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "dataset loaded. Duration: 16.651029109954834\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d0893de9b8c4a479b9ecffa11486bec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.57\t\t0.44\t\t0.187\t\t0.827\t\t4.27m - 4.3m / 0.0m\n",
      "2\t34k\t0.39\t\t0.34\t\t0.292\t\t0.903\t\t4.41m - 8.7m / 192.4m\n",
      "3\t51k\t0.37\t\t0.36\t\t0.269\t\t0.892\t\t4.82m - 13.6m / 198.3m\n",
      "4\t68k\t0.37\t\t0.35\t\t0.375\t\t0.930\t\t4.73m - 18.3m / 215.9m\n",
      "5\t85k\t0.37\t\t0.39\t\t0.303\t\t0.909\t\t4.57m - 22.9m / 212.3m\n",
      "6\t102k\t0.37\t\t0.37\t\t0.326\t\t0.914\t\t4.43m - 27.4m / 205.6m\n",
      "7\t119k\t0.37\t\t0.38\t\t0.286\t\t0.903\t\t4.63m - 32.0m / 200.1m\n",
      "8\t136k\t0.36\t\t0.38\t\t0.347\t\t0.922\t\t4.33m - 36.4m / 207.8m\n",
      "9\t154k\t0.36\t\t0.37\t\t0.276\t\t0.891\t\t4.27m - 40.6m / 196.4m\n",
      "10\t171k\t0.36\t\t0.40\t\t0.358\t\t0.926\t\t4.28m - 44.9m / 194.5m\n",
      "11\t188k\t0.36\t\t0.39\t\t0.353\t\t0.922\t\t4.35m - 49.3m / 194.9m\n",
      "12\t205k\t0.33\t\t0.39\t\t0.366\t\t0.926\t\t4.43m - 53.7m / 197.1m\n",
      "13\t222k\t0.31\t\t0.48\t\t0.385\t\t0.934\t\t4.35m - 58.1m / 200.0m\n",
      "14\t239k\t0.29\t\t0.41\t\t0.389\t\t0.933\t\t4.39m - 62.5m / 197.2m\n",
      "15\t256k\t0.28\t\t0.43\t\t0.319\t\t0.914\t\t4.33m - 66.9m / 198.6m\n",
      "16\t273k\t0.27\t\t0.62\t\t0.411\t\t0.940\t\t4.32m - 71.2m / 196.8m\n",
      "17\t290k\t0.26\t\t0.48\t\t0.370\t\t0.932\t\t4.33m - 75.6m / 196.5m\n",
      "18\t307k\t0.25\t\t0.50\t\t0.346\t\t0.924\t\t4.37m - 80.0m / 196.8m\n",
      "19\t324k\t0.24\t\t0.60\t\t0.453\t\t0.947\t\t4.39m - 84.4m / 198.0m\n",
      "20\t341k\t0.23\t\t0.51\t\t0.367\t\t0.931\t\t4.47m - 88.9m / 198.6m\n",
      "21\t358k\t0.22\t\t0.57\t\t0.397\t\t0.936\t\t4.49m - 93.4m / 200.6m\n",
      "22\t375k\t0.22\t\t0.64\t\t0.403\t\t0.939\t\t4.44m - 97.8m / 201.0m\n",
      "23\t392k\t0.21\t\t0.66\t\t0.426\t\t0.946\t\t4.39m - 102.2m / 200.0m\n",
      "24\t409k\t0.21\t\t0.68\t\t0.419\t\t0.943\t\t4.44m - 106.7m / 198.9m\n",
      "25\t426k\t0.20\t\t0.66\t\t0.401\t\t0.940\t\t4.49m - 111.2m / 200.0m\n",
      "26\t443k\t0.19\t\t0.88\t\t0.453\t\t0.950\t\t4.50m - 115.7m / 200.9m\n",
      "27\t461k\t0.19\t\t0.87\t\t0.426\t\t0.945\t\t4.54m - 120.3m / 201.1m\n",
      "28\t478k\t0.19\t\t0.76\t\t0.411\t\t0.943\t\t4.40m - 124.7m / 201.9m\n",
      "29\t495k\t0.18\t\t0.83\t\t0.438\t\t0.947\t\t4.80m - 129.5m / 199.5m\n",
      "30\t512k\t0.17\t\t0.81\t\t0.449\t\t0.949\t\t4.64m - 134.2m / 206.4m\n",
      "31\t529k\t0.17\t\t0.74\t\t0.400\t\t0.942\t\t4.55m - 138.7m / 203.8m\n",
      "32\t546k\t0.17\t\t0.75\t\t0.419\t\t0.942\t\t4.73m - 143.5m / 202.4m\n",
      "33\t563k\t0.17\t\t0.74\t\t0.436\t\t0.947\t\t5.06m - 148.5m / 204.9m\n",
      "34\t580k\t0.16\t\t0.69\t\t0.441\t\t0.949\t\t5.06m - 153.6m / 209.3m\n",
      "35\t597k\t0.16\t\t0.71\t\t0.428\t\t0.945\t\t5.06m - 158.7m / 209.3m\n",
      "36\t614k\t0.16\t\t0.66\t\t0.391\t\t0.937\t\t5.89m - 164.6m / 209.3m\n",
      "Training duration was 9885.481767654419\n",
      "VAL f1\t0.45320493529942824 - (0.45320493529942824)\n",
      "(macro) f1\t{'valid': 0.13483647202844085, 'test': 0.12018249883235144}\n",
      "VAL loss\t0.3441517397166459\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [1/2]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3441517397166459\n",
      " | \\     )|_\tf1: 0.45320493529942824\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190502\\3\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Multitask Classification\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                 germeval_multitask                |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         45                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "dataset loaded. Duration: 24.30037498474121\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7ce5eb8a97d44ce84e3d27df672bcaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n",
      "1\t17k\t0.54\t\t0.44\t\t0.172\t\t0.822\t\t4.74m - 4.7m / 0.0m\n",
      "2\t34k\t0.39\t\t0.38\t\t0.314\t\t0.911\t\t4.72m - 9.5m / 213.3m\n",
      "3\t51k\t0.37\t\t0.37\t\t0.282\t\t0.894\t\t5.07m - 14.6m / 212.5m\n",
      "4\t68k\t0.37\t\t0.35\t\t0.399\t\t0.935\t\t5.20m - 19.8m / 227.6m\n",
      "5\t85k\t0.37\t\t0.37\t\t0.351\t\t0.922\t\t5.47m - 25.3m / 233.2m\n",
      "6\t102k\t0.37\t\t0.35\t\t0.368\t\t0.929\t\t5.56m - 30.9m / 244.2m\n",
      "7\t119k\t0.36\t\t0.39\t\t0.285\t\t0.902\t\t4.55m - 35.5m / 247.9m\n",
      "8\t136k\t0.36\t\t0.38\t\t0.343\t\t0.921\t\t4.90m - 40.4m / 208.3m\n",
      "9\t154k\t0.36\t\t0.38\t\t0.295\t\t0.903\t\t4.69m - 45.1m / 221.7m\n",
      "10\t171k\t0.36\t\t0.39\t\t0.369\t\t0.928\t\t4.72m - 49.8m / 213.9m\n",
      "11\t188k\t0.35\t\t0.36\t\t0.344\t\t0.920\t\t4.65m - 54.5m / 215.2m\n",
      "12\t205k\t0.33\t\t0.38\t\t0.348\t\t0.920\t\t5.04m - 59.6m / 212.5m\n",
      "13\t222k\t0.31\t\t0.49\t\t0.384\t\t0.935\t\t4.78m - 64.4m / 226.0m\n",
      "14\t239k\t0.29\t\t0.40\t\t0.333\t\t0.920\t\t4.45m - 68.8m / 217.4m\n",
      "Training duration was 4131.669258356094\n",
      "VAL f1\t0.39925925925925926 - (0.39925925925925926)\n",
      "(macro) f1\t{'valid': 0.12979019107832418, 'test': 0.11552086037102696}\n",
      "VAL loss\t0.3455875503749904\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [2/2]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3455875503749904\n",
      " | \\     )|_\tf1: 0.39925925925925926\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "#################################################################################\n",
      "############################## EXPERIMENT COMPLETE ##############################\n",
      "\n",
      "\n",
      "Run [0/2]: 0.39786666666666665\n",
      "Run [1/2]: 0.346547593116968\n",
      "------------------------------\n",
      "Mean: 0.3722071298918173\n",
      "TEST MICRO F1 Statistics\n",
      "count    2.000000\n",
      "mean     0.372207\n",
      "std      0.036288\n",
      "min      0.346548\n",
      "25%      0.359377\n",
      "50%      0.372207\n",
      "75%      0.385037\n",
      "max      0.397867\n",
      "Name: test_f1, dtype: float64\n",
      "TEST MACRO F1 Statistics\n",
      "count    2.000000\n",
      "mean     0.117852\n",
      "std      0.003296\n",
      "min      0.115521\n",
      "25%      0.116686\n",
      "50%      0.117852\n",
      "75%      0.119017\n",
      "max      0.120182\n",
      "Name: test_f1_macro, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for e in [experiments[-1]]:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    test_params = {**baseline, **{'num_epochs': 45, 'language': 'de'}}\n",
    "    test_params = {**test_params, **e['rc']}\n",
    "    e = Experiment(main_experiment_name, e['description'], default_params, test_params, dsl, runs=2)\n",
    "    df, e_path = e.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p = os.path.join(e_path, 'exp_df.pkl')\n",
    "print('Save dataframe of experiment to ' + p)\n",
    "df.to_pickle(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test set 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PREFERENCES.defaults(\n",
    "        data_root='./data/data/germeval2017',\n",
    "        data_train='train_v1.4.tsv',    \n",
    "        data_validation='dev_v1.4.tsv',\n",
    "        data_test='test_TIMESTAMP2.tsv',\n",
    "        source_index=0,\n",
    "        target_vocab_index=2,\n",
    "        file_format='csv',\n",
    "        language='de'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#########################################################################\n",
      "\n",
      "Experiment Name: Baseline - LM-H\n",
      "\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "Experiment GermEval-2017_Experiment initialized\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\12\n",
      "Log path is  C:\\Users\\felix\\OneDrive\\Studium\\Studium\\6. Semester\\MA\\Project\\ABSA-Transformer\\logs\\GermEval-2017_Experiment\\20190501\\13\n",
      "\n",
      "\n",
      "#########################################################################\n",
      "Name: GermEval-2017_Experiment\n",
      "Description: Baseline classification for the GermEval-2017 task using the linear mean head\n",
      "#########################################################################\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------------------+\n",
      "|                                 Hyperparameters                                  |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|          Parameter           |                       Value                       |\n",
      "+------------------------------+---------------------------------------------------+\n",
      "|            kwargs            | {'batch_size': 26, 'learning_rate_schedu[...]one} |\n",
      "|    use_random_classifier     |                       False                       |\n",
      "|          model_size          |                        300                        |\n",
      "|        early_stopping        |                         10                        |\n",
      "|           use_cuda           |                        True                       |\n",
      "|             task             |                      germeval                     |\n",
      "|          batch_size          |                         26                        |\n",
      "| learning_rate_scheduler_type |             LearningSchedulerType.Noam            |\n",
      "|      output_layer_type       |             OutputLayerType.LinearSum             |\n",
      "|        optimizer_type        |                 OptimizerType.Adam                |\n",
      "|        learning_rate         |                0.09624249687454951                |\n",
      "|  noam_learning_rate_warmup   |                        6706                       |\n",
      "|  noam_learning_rate_factor   |                 1.120962889284992                 |\n",
      "|          adam_beta1          |                 0.8278419040185792                |\n",
      "|          adam_beta2          |                 0.7523040247084006                |\n",
      "|           adam_eps           |                0.001028230097476593               |\n",
      "|      adam_weight_decay       |                       0.0001                      |\n",
      "|         adam_amsgrad         |                       False                       |\n",
      "|           use_bias           |                        True                       |\n",
      "|         n_enc_blocks         |                         6                         |\n",
      "|           n_heads            |                         5                         |\n",
      "|             d_k              |                         60                        |\n",
      "|             d_v              |                         60                        |\n",
      "|         dropout_rate         |                 0.3116352148277839                |\n",
      "|     pointwise_layer_size     |                        199                        |\n",
      "|      last_layer_dropout      |                 0.1410769136750667                |\n",
      "|   log_every_xth_iteration    |                         -1                        |\n",
      "|          num_epochs          |                         35                        |\n",
      "|        embedding_type        |                      fasttext                     |\n",
      "|        embedding_name        |                         6B                        |\n",
      "|        embedding_dim         |                        300                        |\n",
      "|       clip_comments_to       |                        230                        |\n",
      "|      finetune_embedding      |                        True                       |\n",
      "|           language           |                         de                        |\n",
      "|        use_stop_words        |                        True                       |\n",
      "|         use_stemming         |                       False                       |\n",
      "|        harmonize_bahn        |                       False                       |\n",
      "|      use_spell_checkers      |                       False                       |\n",
      "|      replace_url_tokens      |                        True                       |\n",
      "|       use_text_cleaner       |                       False                       |\n",
      "|     contraction_removal      |                       False                       |\n",
      "|    organic_text_cleaning     |                       False                       |\n",
      "|       token_removal_1        |                       False                       |\n",
      "|       token_removal_2        |                       False                       |\n",
      "|       token_removal_3        |                       False                       |\n",
      "|             seed             |                        None                       |\n",
      "+------------------------------+---------------------------------------------------+\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Load train_v: 0it [00:00, ?it/s]\n",
      "Load train_v: 370it [00:00, 3699.30it/s]\n",
      "Load train_v: 787it [00:00, 3828.95it/s]\n",
      "Load train_v: 1302it [00:00, 4128.55it/s]\n",
      "Load train_v: 1686it [00:00, 4024.87it/s]\n",
      "Load train_v: 2151it [00:00, 4194.01it/s]\n",
      "Load train_v: 2621it [00:00, 4322.03it/s]\n",
      "Load train_v: 3158it [00:00, 4590.45it/s]\n",
      "Load train_v: 3595it [00:00, 4466.97it/s]\n",
      "Load train_v: 4044it [00:00, 4407.55it/s]\n",
      "Load train_v: 4525it [00:01, 4373.02it/s]\n",
      "Load train_v: 4989it [00:01, 4437.07it/s]\n",
      "Load train_v: 5473it [00:01, 4537.95it/s]\n",
      "Load train_v: 5974it [00:01, 4657.03it/s]\n",
      "Load train_v: 6456it [00:01, 4691.37it/s]\n",
      "Load train_v: 6925it [00:01, 4387.65it/s]\n",
      "Load train_v: 7389it [00:01, 4409.61it/s]\n",
      "Load train_v: 7833it [00:01, 4217.20it/s]\n",
      "Load train_v: 8260it [00:01, 4207.91it/s]\n",
      "Load train_v: 8762it [00:01, 4410.90it/s]\n",
      "Load train_v: 9374it [00:02, 4803.26it/s]\n",
      "Load train_v: 9868it [00:02, 4759.48it/s]\n",
      "Load train_v: 10372it [00:02, 4839.87it/s]\n",
      "Load train_v: 10863it [00:02, 4720.93it/s]\n",
      "Load train_v: 11341it [00:02, 4696.10it/s]\n",
      "Load train_v: 11815it [00:02, 4469.85it/s]\n",
      "Load train_v: 12405it [00:02, 4820.37it/s]\n",
      "Load train_v: 12899it [00:02, 4597.84it/s]\n",
      "Load train_v: 13369it [00:02, 4456.63it/s]\n",
      "Load train_v: 13823it [00:03, 4339.93it/s]\n",
      "Load train_v: 14277it [00:03, 4398.08it/s]\n",
      "Load train_v: 14722it [00:03, 4336.59it/s]\n",
      "Load train_v: 15201it [00:03, 4462.87it/s]\n",
      "Load train_v: 15651it [00:03, 4319.37it/s]\n",
      "Load train_v: 16088it [00:03, 4321.61it/s]\n",
      "Load train_v: 16634it [00:03, 4575.57it/s]\n",
      "Load train_v: 17141it [00:03, 4713.07it/s]\n",
      "Load train_v: 17618it [00:03, 4567.36it/s]\n",
      "Load train_v: 18093it [00:03, 4607.07it/s]\n",
      "Load train_v: 18573it [00:04, 4663.12it/s]\n",
      "Load train_v: 19042it [00:04, 4615.84it/s]\n",
      "Load train_v: 19536it [00:04, 4695.10it/s]\n",
      "Load train_v: 20008it [00:04, 4661.25it/s]\n",
      "Load train_v: 20476it [00:04, 4597.64it/s]\n",
      "Load train_v: 20937it [00:04, 4403.51it/s]\n",
      "                                          \n",
      "Load dev_v1.: 0it [00:00, ?it/s]\n",
      "Load dev_v1.: 401it [00:00, 3931.58it/s]\n",
      "Load dev_v1.: 836it [00:00, 3981.26it/s]\n",
      "Load dev_v1.: 1292it [00:00, 4127.64it/s]\n",
      "Load dev_v1.: 1739it [00:00, 4200.90it/s]\n",
      "Load dev_v1.: 2081it [00:00, 3905.14it/s]\n",
      "Load dev_v1.: 2517it [00:00, 4031.30it/s]\n",
      "                                         \n",
      "Load test_TI: 0it [00:00, ?it/s]\n",
      "Load test_TI: 557it [00:00, 5514.86it/s]\n",
      "Load test_TI: 948it [00:00, 4891.85it/s]\n",
      "Load test_TI: 1338it [00:00, 4545.03it/s]\n",
      "Load test_TI: 1713it [00:00, 4258.70it/s]\n",
      "                                         "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------+\n",
      "|  GERM EVAL 2017 DATASET |\n",
      "+---------------+---------+\n",
      "|     Split     |   Size  |\n",
      "+---------------+---------+\n",
      "|     train     |  17043  |\n",
      "|   validation  |   2049  |\n",
      "|      test     |   1547  |\n",
      "+---------------+---------+\n",
      "+--------------------------------------+\n",
      "|           Vocabulary Stats           |\n",
      "+------------------------------+-------+\n",
      "|          Vocabulary          |  Size |\n",
      "+------------------------------+-------+\n",
      "|              id              | 20639 |\n",
      "|           comments           | 69247 |\n",
      "|      general_sentiments      |   3   |\n",
      "|      aspect_sentiments       |   4   |\n",
      "|           padding            | 22936 |\n",
      "|          Allgemein           |   4   |\n",
      "|          Atmosphäre          |   4   |\n",
      "| Auslastung_und_Platzangebot  |   4   |\n",
      "|       Barrierefreiheit       |   4   |\n",
      "|         Connectivity         |   4   |\n",
      "|      DB_App_und_Website      |   4   |\n",
      "|            Design            |   4   |\n",
      "|   Gastronomisches_Angebot    |   4   |\n",
      "|            Gepäck            |   4   |\n",
      "|            Image             |   4   |\n",
      "|        Informationen         |   4   |\n",
      "|   Komfort_und_Ausstattung    |   4   |\n",
      "|           QR-Code            |   2   |\n",
      "|      Reisen_mit_Kindern      |   4   |\n",
      "| Service_und_Kundenbetreuung  |   4   |\n",
      "|          Sicherheit          |   4   |\n",
      "| Sonstige_Unregelmässigkeiten |   4   |\n",
      "|          Ticketkauf          |   4   |\n",
      "|          Toiletten           |   4   |\n",
      "|           Zugfahrt           |   4   |\n",
      "+------------------------------+-------+\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Allgemein                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|   neutral   |  12114  | 58.69470420078492  |  0.4130529579921508 |\n",
      "|     n/a     |   6002  | 29.080866321042688 |  0.7091913367895731 |\n",
      "|   negative  |   1747  |  8.46455739134648  |  0.9153544260865352 |\n",
      "|   positive  |   776   | 3.759872086825912  |  0.9624012791317409 |\n",
      "|     Sum     |  20639  |                    |         1.0         |\n",
      "| Head Weight |         |                    | 0.29080866321042687 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                             Atmosphäre                            |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|     n/a     |  19401  | 94.00164736663598  | 0.059983526333640236 |\n",
      "|   negative  |   952   | 4.612626580745191  |  0.9538737341925481  |\n",
      "|   positive  |   127   | 0.6153398904985707 |  0.9938466010950143  |\n",
      "|   neutral   |   159   | 0.7703861621202578 |  0.9922961383787974  |\n",
      "|     Sum     |  20639  |                    |         1.0          |\n",
      "| Head Weight |         |                    |  0.9400164736663598  |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                    Auslastung_und_Platzangebot                     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20335  |  98.52706041959397  | 0.014729395804060297 |\n",
      "|   negative  |   248   |  1.201608605068075  |  0.9879839139493193  |\n",
      "|   positive  |    47   | 0.22772421144435293 |  0.9977227578855564  |\n",
      "|   neutral   |    9    |  0.0436067638935995 |  0.999563932361064   |\n",
      "|     Sum     |  20639  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9852706041959397  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                          Barrierefreiheit                          |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20556  |  99.59784873298125  | 0.004021512670187466 |\n",
      "|   positive  |    18   |  0.087213527787199  |  0.999127864722128   |\n",
      "|   neutral   |    14   |  0.0678327438344881 |  0.9993216725616552  |\n",
      "|   negative  |    51   | 0.24710499539706382 |  0.9975289500460294  |\n",
      "|     Sum     |  20639  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9959784873298125  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                            Connectivity                            |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20286  |  98.28964581617326  | 0.017103541838267344 |\n",
      "|   positive  |    80   | 0.38761567905421773 |  0.9961238432094578  |\n",
      "|   negative  |   224   |  1.0853239013518097 |  0.9891467609864819  |\n",
      "|   neutral   |    49   | 0.23741460342070836 |  0.997625853965793   |\n",
      "|     Sum     |  20639  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9828964581617327  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                         DB_App_und_Website                        |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  20413  |  98.90498570667184  | 0.01095014293328167 |\n",
      "|   positive  |    39   | 0.18896264353893116 |  0.9981103735646107 |\n",
      "|   negative  |   133   |  0.644411066427637  |  0.9935558893357236 |\n",
      "|   neutral   |    54   | 0.26164058336159696 |  0.997383594166384  |\n",
      "|     Sum     |  20639  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9890498570667183 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Design                                |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  20602  |  99.82072774843742   | 0.0017927225156257887 |\n",
      "|   negative  |    18   |  0.087213527787199   |   0.999127864722128   |\n",
      "|   positive  |    17   | 0.08236833179902127  |   0.9991763166820098  |\n",
      "|   neutral   |    2    | 0.009690391976355444 |   0.9999030960802364  |\n",
      "|     Sum     |  20639  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9982072774843742  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                       Gastronomisches_Angebot                        |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  20588  |  99.75289500460293   | 0.0024710499539706277 |\n",
      "|   negative  |    32   |  0.1550462716216871  |   0.9984495372837832  |\n",
      "|   neutral   |    11   | 0.05329715586995494  |   0.9994670284413004  |\n",
      "|   positive  |    8    | 0.038761567905421776 |   0.9996123843209458  |\n",
      "|     Sum     |  20639  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9975289500460294  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                                Gepäck                               |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  20612  |   99.8691797083192   | 0.001308202916807999 |\n",
      "|   neutral   |    10   | 0.048451959881777217 |  0.9995154804011822  |\n",
      "|   negative  |    15   | 0.07267793982266582  |  0.9992732206017734  |\n",
      "|   positive  |    2    | 0.009690391976355444 |  0.9999030960802364  |\n",
      "|     Sum     |  20639  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.998691797083192   |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                                Image                                 |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  20578  |  99.70444304472116   | 0.0029555695527884174 |\n",
      "|   negative  |    37   |  0.1792722515625757  |   0.9982072774843742  |\n",
      "|   positive  |    10   | 0.048451959881777217 |   0.9995154804011822  |\n",
      "|   neutral   |    14   |  0.0678327438344881  |   0.9993216725616552  |\n",
      "|     Sum     |  20639  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9970444304472116  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                           Informationen                           |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|     n/a     |  20240  | 98.06676680071709  | 0.019332331992829133 |\n",
      "|   positive  |    53   | 0.2567953873734193 |  0.9974320461262658  |\n",
      "|   negative  |   282   | 1.3663452686661175 |  0.9863365473133389  |\n",
      "|   neutral   |    64   | 0.3100925432433742 |  0.9968990745675662  |\n",
      "|     Sum     |  20639  |                    |         1.0          |\n",
      "| Head Weight |         |                    |  0.9806676680071709  |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                      Komfort_und_Ausstattung                       |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20454  |  99.10363874218712  | 0.008963612578128832 |\n",
      "|   positive  |    45   |  0.2180338194679975 |   0.99781966180532   |\n",
      "|   negative  |   116   |  0.5620427346286158 |  0.9943795726537138  |\n",
      "|   neutral   |    24   | 0.11628470371626533 |  0.9988371529628374  |\n",
      "|     Sum     |  20639  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.9910363874218712  |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-----------------------------------------------------------------------+\n",
      "|                                QR-Code                                |\n",
      "+-------------+---------+----------------------+------------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight      |\n",
      "+-------------+---------+----------------------+------------------------+\n",
      "|     n/a     |  20638  |  99.99515480401182   | 4.8451959881790074e-05 |\n",
      "|   positive  |    1    | 0.004845195988177722 |   0.9999515480401182   |\n",
      "|     Sum     |  20639  |                      |          1.0           |\n",
      "| Head Weight |         |                      |   0.9999515480401182   |\n",
      "+-------------+---------+----------------------+------------------------+\n",
      "\n",
      "\n",
      "\n",
      "+----------------------------------------------------------------------+\n",
      "|                          Reisen_mit_Kindern                          |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |      Class Weight     |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "|     n/a     |  20589  |  99.75774020059112   | 0.0024225979940888376 |\n",
      "|   positive  |    8    | 0.038761567905421776 |   0.9996123843209458  |\n",
      "|   neutral   |    16   | 0.07752313581084355  |   0.9992247686418916  |\n",
      "|   negative  |    26   | 0.12597509569262075  |   0.9987402490430738  |\n",
      "|     Sum     |  20639  |                      |          1.0          |\n",
      "| Head Weight |         |                      |   0.9975774020059112  |\n",
      "+-------------+---------+----------------------+-----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Service_und_Kundenbetreuung                    |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "|     n/a     |  20077  | 97.27699985464412  | 0.027230001453558805 |\n",
      "|   negative  |   373   | 1.8072581035902904 |  0.9819274189640971  |\n",
      "|   positive  |   137   | 0.6637918503803478 |  0.9933620814961965  |\n",
      "|   neutral   |    52   | 0.2519501913852415 |  0.9974804980861476  |\n",
      "|     Sum     |  20639  |                    |         1.0          |\n",
      "| Head Weight |         |                    |  0.9727699985464412  |\n",
      "+-------------+---------+--------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+--------------------------------------------------------------------+\n",
      "|                             Sicherheit                             |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight     |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "|     n/a     |  20105  |   97.4126653423131  | 0.025873346576869016 |\n",
      "|   negative  |   503   |  2.437133582053394  |  0.975628664179466   |\n",
      "|   positive  |    22   | 0.10659431173990988 |  0.998934056882601   |\n",
      "|   neutral   |    9    |  0.0436067638935995 |  0.999563932361064   |\n",
      "|     Sum     |  20639  |                     |         1.0          |\n",
      "| Head Weight |         |                     |  0.974126653423131   |\n",
      "+-------------+---------+---------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+-------------------------------------------------------------------+\n",
      "|                    Sonstige_Unregelmässigkeiten                   |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "|     n/a     |  18995  |  92.03449779543583  | 0.07965502204564179 |\n",
      "|   negative  |   1524  |  7.384078685982848  |  0.9261592131401715 |\n",
      "|   positive  |    51   | 0.24710499539706382 |  0.9975289500460294 |\n",
      "|   neutral   |    69   |  0.3343185231842628 |  0.9966568147681574 |\n",
      "|     Sum     |  20639  |                     |         1.0         |\n",
      "| Head Weight |         |                     |  0.9203449779543582 |\n",
      "+-------------+---------+---------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                            Ticketkauf                            |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  19928  | 96.55506565240563  | 0.03444934347594364 |\n",
      "|   negative  |   497   | 2.4080624061243276 |  0.9759193759387568 |\n",
      "|   neutral   |   107   | 0.5184359707350162 |  0.9948156402926498 |\n",
      "|   positive  |   107   | 0.5184359707350162 |  0.9948156402926498 |\n",
      "|     Sum     |  20639  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.9655506565240564 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n",
      "\n",
      "\n",
      "+---------------------------------------------------------------------+\n",
      "|                              Toiletten                              |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|    Label    | Samples |    Triv. Accuracy    |     Class Weight     |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "|     n/a     |  20586  |  99.74320461262658   | 0.002567953873734208 |\n",
      "|   negative  |    40   | 0.19380783952710887  |  0.998061921604729   |\n",
      "|   neutral   |    11   | 0.05329715586995494  |  0.9994670284413004  |\n",
      "|   positive  |    2    | 0.009690391976355444 |  0.9999030960802364  |\n",
      "|     Sum     |  20639  |                      |         1.0          |\n",
      "| Head Weight |         |                      |  0.9974320461262658  |\n",
      "+-------------+---------+----------------------+----------------------+\n",
      "\n",
      "\n",
      "\n",
      "+------------------------------------------------------------------+\n",
      "|                             Zugfahrt                             |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|    Label    | Samples |   Triv. Accuracy   |     Class Weight    |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "|     n/a     |  18467  | 89.47623431367799  | 0.10523765686322006 |\n",
      "|   positive  |   372   | 1.8024129076021127 |  0.9819758709239789 |\n",
      "|   negative  |   1662  | 8.052715732351373  |  0.9194728426764862 |\n",
      "|   neutral   |   138   | 0.6686370463685256 |  0.9933136295363147 |\n",
      "|     Sum     |  20639  |                    |         1.0         |\n",
      "| Head Weight |         |                    |  0.8947623431367799 |\n",
      "+-------------+---------+--------------------+---------------------+\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "+-------------------------------------------------------------+\n",
      "|                            Train                            |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|           Category           | POS  |  NEG  |  NEU  |  Sum  |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "|          Allgemein           | 1102 |  2149 | 10043 | 13294 |\n",
      "|           Zugfahrt           | 464  |  1813 |  173  |  2450 |\n",
      "| Service_und_Kundenbetreuung  | 146  |  389  |   50  |  585  |\n",
      "| Sonstige_Unregelmässigkeiten |  66  |  1936 |  107  |  2109 |\n",
      "|          Sicherheit          |  21  |  903  |   15  |  939  |\n",
      "|         Connectivity         |  75  |  240  |   67  |  382  |\n",
      "|        Informationen         |  44  |  282  |   69  |  395  |\n",
      "|          Atmosphäre          | 142  |  1165 |  223  |  1530 |\n",
      "|          Ticketkauf          | 109  |  517  |  119  |  745  |\n",
      "| Auslastung_und_Platzangebot  |  47  |  246  |   9   |  302  |\n",
      "|   Komfort_und_Ausstattung    |  47  |  110  |   24  |  181  |\n",
      "|      DB_App_und_Website      |  36  |  129  |   74  |  239  |\n",
      "|       Barrierefreiheit       |  21  |   54  |   16  |   91  |\n",
      "|            Image             |  8   |   37  |   13  |   58  |\n",
      "|   Gastronomisches_Angebot    |  9   |   35  |   11  |   55  |\n",
      "|          Toiletten           |  2   |   39  |   12  |   53  |\n",
      "|            Gepäck            |  0   |   11  |   11  |   22  |\n",
      "|      Reisen_mit_Kindern      |  11  |   29  |   19  |   59  |\n",
      "|            Design            |  15  |   17  |   4   |   36  |\n",
      "|            Total             | 2365 | 10101 | 11059 | 23525 |\n",
      "+------------------------------+------+-------+-------+-------+\n",
      "\n",
      "+---------------------------------------------------------+\n",
      "|                        Validation                       |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|           Category           | POS | NEG  | NEU  | Sum  |\n",
      "+------------------------------+-----+------+------+------+\n",
      "|          Allgemein           | 139 | 250  | 1207 | 1596 |\n",
      "|          Atmosphäre          |  12 | 152  |  30  | 194  |\n",
      "|          Sicherheit          |  2  | 107  |  2   | 111  |\n",
      "| Sonstige_Unregelmässigkeiten |  3  | 196  |  7   | 206  |\n",
      "|            Gepäck            |  1  |  2   |  0   |  3   |\n",
      "|         Connectivity         |  12 |  23  |  1   |  36  |\n",
      "|           Zugfahrt           |  59 | 187  |  29  | 275  |\n",
      "| Service_und_Kundenbetreuung  |  19 |  31  |  7   |  57  |\n",
      "|          Ticketkauf          |  12 |  59  |  17  |  88  |\n",
      "| Auslastung_und_Platzangebot  |  12 |  34  |  1   |  47  |\n",
      "|          Toiletten           |  0  |  3   |  2   |  5   |\n",
      "|        Informationen         |  3  |  33  |  10  |  46  |\n",
      "|   Komfort_und_Ausstattung    |  8  |  15  |  3   |  26  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3   |  4   |  7   |\n",
      "|            Design            |  2  |  3   |  0   |  5   |\n",
      "|       Barrierefreiheit       |  5  |  13  |  4   |  22  |\n",
      "|      DB_App_und_Website      |  7  |  15  |  5   |  27  |\n",
      "|            Image             |  4  |  1   |  2   |  7   |\n",
      "|      Reisen_mit_Kindern      |  1  |  2   |  1   |  4   |\n",
      "|           QR-Code            |  1  |  0   |  0   |  1   |\n",
      "|            Total             | 302 | 1129 | 1332 | 2763 |\n",
      "+------------------------------+-----+------+------+------+\n",
      "\n",
      "+-------------------------------------------------------+\n",
      "|                          Test                         |\n",
      "+------------------------------+-----+-----+-----+------+\n",
      "|           Category           | POS | NEG | NEU | Sum  |\n",
      "+------------------------------+-----+-----+-----+------+\n",
      "| Auslastung_und_Platzangebot  |  4  |  19 |  0  |  23  |\n",
      "| Sonstige_Unregelmässigkeiten |  4  | 271 |  1  | 276  |\n",
      "|          Allgemein           |  42 |  87 | 913 | 1042 |\n",
      "|          Atmosphäre          |  10 |  81 |  10 | 101  |\n",
      "|          Ticketkauf          |  9  |  41 |  13 |  63  |\n",
      "|      DB_App_und_Website      |  1  |  20 |  5  |  26  |\n",
      "|          Sicherheit          |  5  |  75 |  0  |  80  |\n",
      "|           Zugfahrt           |  40 | 203 |  7  | 250  |\n",
      "| Service_und_Kundenbetreuung  |  4  |  23 |  2  |  29  |\n",
      "|         Connectivity         |  42 |  59 |  7  | 108  |\n",
      "|        Informationen         |  7  |  30 |  2  |  39  |\n",
      "|            Gepäck            |  1  |  4  |  1  |  6   |\n",
      "|   Komfort_und_Ausstattung    |  3  |  8  |  0  |  11  |\n",
      "|   Gastronomisches_Angebot    |  0  |  3  |  1  |  4   |\n",
      "|      Reisen_mit_Kindern      |  0  |  1  |  1  |  2   |\n",
      "|            Image             |  0  |  4  |  0  |  4   |\n",
      "|          Toiletten           |  0  |  12 |  0  |  12  |\n",
      "|       Barrierefreiheit       |  1  |  1  |  0  |  2   |\n",
      "|            Design            |  1  |  1  |  0  |  2   |\n",
      "|            Total             | 174 | 943 | 963 | 2080 |\n",
      "+------------------------------+-----+-----+-----+------+\n",
      "\n",
      "\n",
      "dataset loaded. Duration: 18.104995727539062\n",
      "pre_training - INFO - Classes: ['n/a', 'neutral', 'negative', 'positive']\n",
      "pre_training - INFO - JointAspectTagger (\n",
      "  (encoder): TransformerEncoder(\n",
      "    (positional_encoding): PositionalEncoding2(\n",
      "      (dropout): Dropout(p=0.3116352148277839)\n",
      "    )\n",
      "    (encoder_blocks): ModuleList(\n",
      "      (0): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (1): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (2): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (3): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (4): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "      (5): EncoderBlock(\n",
      "        (self_attention_layer): MultiHeadedSelfAttentionLayer(\n",
      "          (query_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (key_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (value_projections): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (attention_layer): ScaledDotProductAttentionLayer(\n",
      "            (dropout): Dropout(p=0.3116352148277839)\n",
      "          )\n",
      "          (w_0): Linear(in_features=300, out_features=300, bias=False)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (feed_forward_layer): PointWiseFCLayer(\n",
      "          (w_1): Linear(in_features=300, out_features=199, bias=True)\n",
      "          (w_2): Linear(in_features=199, out_features=300, bias=True)\n",
      "          (dropout): Dropout(p=0.3116352148277839)\n",
      "        )\n",
      "        (layer_norm): LayerNorm()\n",
      "      )\n",
      "    )\n",
      "    (layer_norm): LayerNorm()\n",
      "    (src_embeddings): Embedding(69247, 300)\n",
      "  ), weights=((300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300, 300), (300, 300), (300, 300), (300, 300), (199, 300), (199,), (300, 199), (300,), (300,), (300,), (300,), (300,), (69247, 300)), parameters=23657694\n",
      "  (taggers): ModuleList(\n",
      "    (0): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (1): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (2): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (3): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (4): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (5): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (6): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (7): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (8): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (9): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (10): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (11): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (12): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (13): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (14): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (15): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (16): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (17): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (18): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "    (19): CommentWiseSumLogSoftmax(\n",
      "      (output_projection): Linear(in_features=300, out_features=4, bias=True)\n",
      "      (dropout): Dropout(p=0.1410769136750667)\n",
      "    )\n",
      "  ), weights=((4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,), (4, 300), (4,)), parameters=24080\n",
      ")\n",
      "==================================\n",
      "Total Number of parameters: 23.681.774\n",
      "==================================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pre_training - DEBUG - train with cuda support\n",
      "pre_training - INFO - 656 Iterations per epoch with batch size of 26\n",
      "pre_training - INFO - Total iterations: 22960\n",
      "pre_training - INFO - Total number of samples: 596960\n",
      "pre_training - INFO - START training.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81298c1527a547b1878a806fb3606868",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# EP\t# IT\ttr loss\t\tval loss\tf1\t\tacc\t\tduration / total time\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\t17k\t0.53\t\t0.39\t\t0.337\t\t0.931\t\t4.61m - 4.6m / 0.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\t34k\t0.38\t\t0.36\t\t0.255\t\t0.894\t\t4.64m - 9.4m / 161.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\t51k\t0.35\t\t0.36\t\t0.293\t\t0.911\t\t4.61m - 14.0m / 162.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\t68k\t0.35\t\t0.36\t\t0.235\t\t0.889\t\t4.62m - 18.6m / 161.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5\t85k\t0.35\t\t0.36\t\t0.276\t\t0.901\t\t4.62m - 23.2m / 161.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\t102k\t0.35\t\t0.37\t\t0.295\t\t0.913\t\t4.61m - 27.9m / 161.9m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\t119k\t0.35\t\t0.35\t\t0.347\t\t0.930\t\t4.59m - 32.5m / 161.6m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\t136k\t0.35\t\t0.35\t\t0.290\t\t0.911\t\t4.59m - 37.1m / 161.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\t154k\t0.35\t\t0.36\t\t0.297\t\t0.914\t\t4.59m - 41.7m / 161.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\t171k\t0.34\t\t0.45\t\t0.386\t\t0.950\t\t4.58m - 46.3m / 161.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\t188k\t0.34\t\t0.35\t\t0.407\t\t0.953\t\t4.60m - 50.9m / 160.8m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                          \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\t205k\t0.32\t\t0.47\t\t0.395\t\t0.949\t\t4.58m - 55.5m / 161.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\t222k\t0.30\t\t0.38\t\t0.363\t\t0.937\t\t4.59m - 60.1m / 161.0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14\t239k\t0.28\t\t0.43\t\t0.356\t\t0.936\t\t4.59m - 64.7m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\t256k\t0.27\t\t0.40\t\t0.389\t\t0.947\t\t4.59m - 69.3m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16\t273k\t0.26\t\t0.46\t\t0.420\t\t0.952\t\t4.59m - 73.9m / 161.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\t290k\t0.25\t\t0.51\t\t0.441\t\t0.955\t\t4.58m - 78.6m / 161.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\t307k\t0.24\t\t0.44\t\t0.397\t\t0.947\t\t4.60m - 83.3m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\t324k\t0.23\t\t0.46\t\t0.411\t\t0.951\t\t4.59m - 87.8m / 161.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20\t341k\t0.22\t\t0.57\t\t0.416\t\t0.955\t\t4.58m - 92.4m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\t358k\t0.22\t\t0.69\t\t0.406\t\t0.951\t\t4.58m - 97.0m / 161.1m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\t375k\t0.21\t\t0.69\t\t0.431\t\t0.957\t\t4.58m - 101.6m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23\t392k\t0.20\t\t0.65\t\t0.432\t\t0.956\t\t4.58m - 106.2m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\t409k\t0.20\t\t0.58\t\t0.426\t\t0.957\t\t4.59m - 110.8m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25\t426k\t0.19\t\t0.75\t\t0.467\t\t0.962\t\t4.59m - 115.4m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\t443k\t0.19\t\t0.68\t\t0.448\t\t0.959\t\t4.58m - 120.0m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\t461k\t0.18\t\t0.64\t\t0.419\t\t0.954\t\t4.58m - 124.6m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\t478k\t0.18\t\t0.62\t\t0.425\t\t0.956\t\t4.57m - 129.2m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29\t495k\t0.17\t\t0.60\t\t0.421\t\t0.955\t\t4.58m - 133.8m / 161.2m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30\t512k\t0.17\t\t0.58\t\t0.403\t\t0.950\t\t4.58m - 138.4m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\t529k\t0.16\t\t0.50\t\t0.397\t\t0.949\t\t4.58m - 143.0m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\t546k\t0.16\t\t0.59\t\t0.439\t\t0.958\t\t4.58m - 147.6m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "33\t563k\t0.16\t\t0.60\t\t0.412\t\t0.954\t\t4.58m - 152.2m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34\t580k\t0.16\t\t0.55\t\t0.412\t\t0.953\t\t4.58m - 156.8m / 161.3m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                            \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\t597k\t0.15\t\t0.64\t\t0.459\t\t0.962\t\t4.58m - 161.4m / 161.4m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Load train_v: 13626it [2:49:11, 4287.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training duration was 9685.623672962189\n",
      "pre_training - DEBUG - --- Valid Scores ---\n",
      "pre_training - INFO - TEST MACRO mean f1: 0.11126918130285399\n",
      "VAL f1\t0.46694711538461536 - (0.46694711538461536)\n",
      "(macro) f1\t{'valid': 0.14882394133828408, 'test': 0.11126918130285399}\n",
      "VAL loss\t0.3490273736584152\n",
      ".---.\n",
      " /     \\\n",
      " \\.@-@./\t\tExperiment: [0/1]\n",
      " /`\\_/`\\\t\tStatus: ok\n",
      " //  _  \\\\\tLoss: 0.3490273736584152\n",
      " | \\     )|_\tf1: 0.46694711538461536\n",
      " /`\\_`>  <_/ \\\n",
      " \\__/'---'\\__/\n",
      "\n",
      "#################################################################################\n",
      "############################## EXPERIMENT COMPLETE ##############################\n",
      "\n",
      "\n",
      "Run [0/1]: 0.4133184523809524\n",
      "------------------------------\n",
      "Mean: 0.4133184523809524\n",
      "TEST MICRO F1 Statistics\n",
      "count    1.000000\n",
      "mean     0.413318\n",
      "std           NaN\n",
      "min      0.413318\n",
      "25%      0.413318\n",
      "50%      0.413318\n",
      "75%      0.413318\n",
      "max      0.413318\n",
      "Name: test_f1, dtype: float64\n",
      "TEST MACRO F1 Statistics\n",
      "count    1.000000\n",
      "mean     0.111269\n",
      "std           NaN\n",
      "min      0.111269\n",
      "25%      0.111269\n",
      "50%      0.111269\n",
      "75%      0.111269\n",
      "max      0.111269\n",
      "Name: test_f1_macro, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for e in [experiments[0]]:\n",
    "    name = e['name']\n",
    "    print(f'#########################################################################\\n\\nExperiment Name: {name}\\n')\n",
    "    print('#########################################################################\\n\\n')\n",
    "    test_params = {**baseline, **{'num_epochs': 35, 'language': 'de'}}\n",
    "    test_params = {**test_params, **e['rc']}\n",
    "    e = Experiment(main_experiment_name, e['description'], default_params, test_params, dsl, runs=5)\n",
    "    df, e_path = e.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}