train_fasttext_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*In our dev this file was called \"train_best_lstm-2.ipynb\". I modified it slightly for the cantemist-ner repo!*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -U biome "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from biome.text import Pipeline\n",
    "from biome.text.configuration import VocabularyConfiguration, TrainerConfiguration\n",
    "from biome.text.data import DataSource\n",
    "from biome.text.loggers import MlflowLogger\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import allennlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_path = Path(\"data/NER/train_for_bert.json\")\n",
    "dev1_path = Path(\"data/NER/dev1_for_bert.json\")\n",
    "dev2_path = Path(\"data/NER/dev2_for_bert.json\")\n",
    "train_full_path = Path(\"data/NER/train_full.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create train_full data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_full = pd.concat([pd.read_json(train_path, orient=\"records\", lines=True), \n",
    "                        pd.read_json(dev1_path, orient=\"records\", lines=True),\n",
    "                        pd.read_json(dev2_path, orient=\"records\", lines=True)], \n",
    "                       ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_full.to_json(train_full_path, orient=\"records\", lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = DataSource(str(train_full_path), orient=\"records\", lines=True)\n",
    "valid_ds = DataSource(str(dev2_path), orient=\"records\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_org</th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "      <th>file</th>\n",
       "      <th>sentence_offset</th>\n",
       "      <th>entity_text</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-TAC abdominal urgente: En lóbulo hepático izq...</td>\n",
       "      <td>[-, TAC, abdominal, urgente, :, En, lóbulo, he...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco340.txt</td>\n",
       "      <td>976</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Se efectua una RM de control a los 10 días, ap...</td>\n",
       "      <td>[Se, efectua, una, RM, de, control, a, los, 10...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco1175.txt</td>\n",
       "      <td>8309</td>\n",
       "      <td>[]</td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Anti-Gangliósidos IgG e IgM Negativo, IgG séri...</td>\n",
       "      <td>[Anti, -, Gangliósidos, IgG, e, IgM, Negativo,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco570.txt</td>\n",
       "      <td>3696</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Material constituido por necrosis y grupos ais...</td>\n",
       "      <td>[Material, constituido, por, necrosis, y, grup...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco1333.txt</td>\n",
       "      <td>1521</td>\n",
       "      <td>[]</td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>La paciente niega episodios previos similares.</td>\n",
       "      <td>[La, paciente, niega, episodios, previos, simi...</td>\n",
       "      <td>[O, O, O, O, O, O, O]</td>\n",
       "      <td>cc_onco427.txt</td>\n",
       "      <td>1004</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35872</th>\n",
       "      <td>Se identifican numerosas figuras mitóticas, al...</td>\n",
       "      <td>[Se, identifican, numerosas, figuras, mitótica...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O]</td>\n",
       "      <td>cc_onco1488.txt</td>\n",
       "      <td>2894</td>\n",
       "      <td>[]</td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35873</th>\n",
       "      <td>» Dados los hallazgos, se comenta el caso en C...</td>\n",
       "      <td>[», Dados, los, hallazgos, ,, se, comenta, el,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco193.txt</td>\n",
       "      <td>2154</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35874</th>\n",
       "      <td>Se diagnostica de úlcera neurotrófica OI como ...</td>\n",
       "      <td>[Se, diagnostica, de, úlcera, neurotrófica, OI...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O]</td>\n",
       "      <td>cc_onco815.txt</td>\n",
       "      <td>5090</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35875</th>\n",
       "      <td>Todo ello era compatible preliminarmente con u...</td>\n",
       "      <td>[Todo, ello, era, compatible, preliminarmente,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, B-MORFOLOGIA_NEOPLASIA, ...</td>\n",
       "      <td>cc_onco1077.txt</td>\n",
       "      <td>6265</td>\n",
       "      <td>[SE, SE, SE atípico, SE]</td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35876</th>\n",
       "      <td>Entre abril y mayo de 2017 recibió tratamiento...</td>\n",
       "      <td>[Entre, abril, y, mayo, de, 2017, recibió, tra...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>cc_onco457.txt</td>\n",
       "      <td>1070</td>\n",
       "      <td></td>\n",
       "      <td>data/NER_david/train_full.json</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>35877 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text_org  \\\n",
       "0      -TAC abdominal urgente: En lóbulo hepático izq...   \n",
       "1      Se efectua una RM de control a los 10 días, ap...   \n",
       "2      Anti-Gangliósidos IgG e IgM Negativo, IgG séri...   \n",
       "3      Material constituido por necrosis y grupos ais...   \n",
       "4         La paciente niega episodios previos similares.   \n",
       "...                                                  ...   \n",
       "35872  Se identifican numerosas figuras mitóticas, al...   \n",
       "35873  » Dados los hallazgos, se comenta el caso en C...   \n",
       "35874  Se diagnostica de úlcera neurotrófica OI como ...   \n",
       "35875  Todo ello era compatible preliminarmente con u...   \n",
       "35876  Entre abril y mayo de 2017 recibió tratamiento...   \n",
       "\n",
       "                                                    text  \\\n",
       "0      [-, TAC, abdominal, urgente, :, En, lóbulo, he...   \n",
       "1      [Se, efectua, una, RM, de, control, a, los, 10...   \n",
       "2      [Anti, -, Gangliósidos, IgG, e, IgM, Negativo,...   \n",
       "3      [Material, constituido, por, necrosis, y, grup...   \n",
       "4      [La, paciente, niega, episodios, previos, simi...   \n",
       "...                                                  ...   \n",
       "35872  [Se, identifican, numerosas, figuras, mitótica...   \n",
       "35873  [», Dados, los, hallazgos, ,, se, comenta, el,...   \n",
       "35874  [Se, diagnostica, de, úlcera, neurotrófica, OI...   \n",
       "35875  [Todo, ello, era, compatible, preliminarmente,...   \n",
       "35876  [Entre, abril, y, mayo, de, 2017, recibió, tra...   \n",
       "\n",
       "                                                  labels             file  \\\n",
       "0      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   cc_onco340.txt   \n",
       "1      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...  cc_onco1175.txt   \n",
       "2      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   cc_onco570.txt   \n",
       "3      [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...  cc_onco1333.txt   \n",
       "4                                  [O, O, O, O, O, O, O]   cc_onco427.txt   \n",
       "...                                                  ...              ...   \n",
       "35872                  [O, O, O, O, O, O, O, O, O, O, O]  cc_onco1488.txt   \n",
       "35873  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   cc_onco193.txt   \n",
       "35874         [O, O, O, O, O, O, O, O, O, O, O, O, O, O]   cc_onco815.txt   \n",
       "35875  [O, O, O, O, O, O, O, B-MORFOLOGIA_NEOPLASIA, ...  cc_onco1077.txt   \n",
       "35876  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   cc_onco457.txt   \n",
       "\n",
       "       sentence_offset               entity_text  \\\n",
       "0                  976                             \n",
       "1                 8309                        []   \n",
       "2                 3696                             \n",
       "3                 1521                        []   \n",
       "4                 1004                             \n",
       "...                ...                       ...   \n",
       "35872             2894                        []   \n",
       "35873             2154                             \n",
       "35874             5090                             \n",
       "35875             6265  [SE, SE, SE atípico, SE]   \n",
       "35876             1070                             \n",
       "\n",
       "                                 path  \n",
       "0      data/NER_david/train_full.json  \n",
       "1      data/NER_david/train_full.json  \n",
       "2      data/NER_david/train_full.json  \n",
       "3      data/NER_david/train_full.json  \n",
       "4      data/NER_david/train_full.json  \n",
       "...                               ...  \n",
       "35872  data/NER_david/train_full.json  \n",
       "35873  data/NER_david/train_full.json  \n",
       "35874  data/NER_david/train_full.json  \n",
       "35875  data/NER_david/train_full.json  \n",
       "35876  data/NER_david/train_full.json  \n",
       "\n",
       "[35877 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds.to_dataframe().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pl_trained1 = Pipeline.from_pretrained(\"runs/tune/first-experiment/tune_hpo_train_14_hidden_size=128,num_layers=1,type=lstm,dropout=0.16517,hidden_size=128,dropout=0.26896,lr=0.0038931_2020-07-24_22-55-05k2vwybhk/training/model.tar.gz\")\n",
    "#pipeline_config = pl_trained1.config.as_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config = {'name': 'candemist-ner-first-hpo',\n",
    " 'tokenizer': {'lang': 'en',\n",
    "  'max_sequence_length': None,\n",
    "  'max_nr_of_sentences': None,\n",
    "  'text_cleaning': None,\n",
    "  'segment_sentences': False,\n",
    "  'start_tokens': None,\n",
    "  'end_tokens': None},\n",
    " 'features': {'word': {'embedding_dim': 300,\n",
    "   'lowercase_tokens': False,\n",
    "   'trainable': True,\n",
    "   'weights_file': None},\n",
    "  'char': {'embedding_dim': 64,\n",
    "   'lowercase_characters': False,\n",
    "   'encoder': {'bidirectional': True,\n",
    "    'hidden_size': 128,\n",
    "    'num_layers': 1,\n",
    "    'type': 'gru'},\n",
    "   'dropout': 0.16517050992687604},\n",
    "  'transformers': None},\n",
    " 'head': {'dropout': 0.2689579604286324,\n",
    "  'feedforward': {'activations': ['relu'],\n",
    "   'dropout': [0],\n",
    "   'hidden_dims': [32],\n",
    "   'num_layers': 1},\n",
    "  'labels': ['O',\n",
    "   'I-MORFOLOGIA_NEOPLASIA',\n",
    "   'U-MORFOLOGIA_NEOPLASIA',\n",
    "   'B-MORFOLOGIA_NEOPLASIA',\n",
    "   'L-MORFOLOGIA_NEOPLASIA'],\n",
    "  'type': 'biome.text.modules.heads.token_classification.TokenClassification'},\n",
    " 'encoder': {'bidirectional': True,\n",
    "  'hidden_size': 128,\n",
    "  'input_size': 556,\n",
    "  'num_layers': 1,\n",
    "  'type': 'allennlp.modules.seq2seq_encoders.pytorch_seq2seq_wrapper.LstmSeq2SeqEncoder'}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'candemist-ner-first-hpo',\n",
       " 'tokenizer': {'lang': 'en',\n",
       "  'max_sequence_length': None,\n",
       "  'max_nr_of_sentences': None,\n",
       "  'text_cleaning': None,\n",
       "  'segment_sentences': False,\n",
       "  'start_tokens': None,\n",
       "  'end_tokens': None},\n",
       " 'features': {'word': {'embedding_dim': 300,\n",
       "   'lowercase_tokens': False,\n",
       "   'trainable': True,\n",
       "   'weights_file': None},\n",
       "  'char': {'embedding_dim': 64,\n",
       "   'lowercase_characters': False,\n",
       "   'encoder': {'bidirectional': True,\n",
       "    'hidden_size': 128,\n",
       "    'num_layers': 1,\n",
       "    'type': 'gru'},\n",
       "   'dropout': 0.16517050992687604},\n",
       "  'transformers': None},\n",
       " 'head': {'dropout': 0.2689579604286324,\n",
       "  'feedforward': {'activations': ['relu'],\n",
       "   'dropout': [0],\n",
       "   'hidden_dims': [32],\n",
       "   'num_layers': 1},\n",
       "  'labels': ['O',\n",
       "   'I-MORFOLOGIA_NEOPLASIA',\n",
       "   'U-MORFOLOGIA_NEOPLASIA',\n",
       "   'B-MORFOLOGIA_NEOPLASIA',\n",
       "   'L-MORFOLOGIA_NEOPLASIA'],\n",
       "  'type': 'biome.text.modules.heads.token_classification.TokenClassification'},\n",
       " 'encoder': {'bidirectional': True,\n",
       "  'hidden_size': 128,\n",
       "  'input_size': 556,\n",
       "  'num_layers': 1,\n",
       "  'type': 'allennlp.modules.seq2seq_encoders.pytorch_seq2seq_wrapper.LstmSeq2SeqEncoder'}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config[\"features\"][\"word\"][\"weights_file\"] = \"cc.es.300.vec.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config[\"encoder\"][\"hidden_size\"] = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config[\"features\"][\"word\"][\"lowercase_tokens\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config[\"features\"][\"char\"][\"lowercase_characters\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config[\"head\"][\"feedforward\"] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PipelineModel(\n",
       "  (_head): TokenClassification(\n",
       "    (backbone): ModelBackbone(\n",
       "      (embedder): BasicTextFieldEmbedder(\n",
       "        (token_embedder_word): Embedding()\n",
       "        (token_embedder_char): TokenCharactersEncoder(\n",
       "          (_embedding): TimeDistributed(\n",
       "            (_module): Embedding()\n",
       "          )\n",
       "          (_encoder): TimeDistributed(\n",
       "            (_module): GruSeq2VecEncoder(\n",
       "              (_module): GRU(64, 128, batch_first=True, bidirectional=True)\n",
       "            )\n",
       "          )\n",
       "          (_dropout): Dropout(p=0.16517050992687604, inplace=False)\n",
       "        )\n",
       "      )\n",
       "      (encoder): LstmSeq2SeqEncoder(\n",
       "        (_module): LSTM(556, 512, batch_first=True, bidirectional=True)\n",
       "      )\n",
       "    )\n",
       "    (dropout): Dropout(p=0.2689579604286324, inplace=False)\n",
       "    (_label_projection_layer): TimeDistributed(\n",
       "      (_module): Linear(in_features=1024, out_features=5, bias=True)\n",
       "    )\n",
       "    (_crf): ConditionalRandomField()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl = Pipeline.from_config(pipeline_config) ; pl._model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_config = VocabularyConfiguration(\n",
    "    sources=[train_ds], min_count={\"word\": 2}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1568e0ea71574143b9fd41fcae42d125",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c1338a6778b4d1eb80c610446c24b92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2000000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "pl.create_vocabulary(vocab_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define trainer and logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer_dict={\n",
    "    \"optimizer\": {\n",
    "        \"type\": \"adamw\",\n",
    "        \"lr\": 0.0038931174186587806,\n",
    "        \"weight_decay\": 0.01,\n",
    "    },\n",
    "    \"learning_rate_scheduler\": {\n",
    "      \"type\": \"step\",\n",
    "      \"step_size\":2, \n",
    "      \"gamma\":0.1\n",
    "    },\n",
    "    \"batch_size\": 32,\n",
    "    \"num_epochs\": 4,\n",
    "    \"validation_metric\": \"+f1-measure-overall\",\n",
    "    \"patience\":3\n",
    "}\n",
    "trainer_config = TrainerConfiguration(**trainer_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlflow_logger = MlflowLogger(experiment_name=\"cantemist-ner\", run_name=\"best_normal_lstm_full_train\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Execute training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 16:48:27,111 - allennlp.common.params - INFO - config = <biome.text.configuration.PipelineConfiguration object at 0x7f450684d0d0>\n",
      "2020-08-02 16:48:27,372 - allennlp.common.params - INFO - type = single_id\n",
      "2020-08-02 16:48:27,373 - allennlp.common.params - INFO - namespace = word\n",
      "2020-08-02 16:48:27,374 - allennlp.common.params - INFO - lowercase_tokens = True\n",
      "2020-08-02 16:48:27,375 - allennlp.common.params - INFO - start_tokens = None\n",
      "2020-08-02 16:48:27,375 - allennlp.common.params - INFO - end_tokens = None\n",
      "2020-08-02 16:48:27,376 - allennlp.common.params - INFO - feature_name = text\n",
      "2020-08-02 16:48:27,377 - allennlp.common.params - INFO - default_value = THIS IS A REALLY UNLIKELY VALUE THAT HAS TO BE A STRING\n",
      "2020-08-02 16:48:27,378 - allennlp.common.params - INFO - token_min_padding_length = 0\n",
      "2020-08-02 16:48:27,379 - allennlp.common.params - INFO - type = characters\n",
      "2020-08-02 16:48:27,381 - allennlp.common.params - INFO - namespace = char\n",
      "2020-08-02 16:48:27,382 - allennlp.common.params - INFO - character_tokenizer.byte_encoding = None\n",
      "2020-08-02 16:48:27,383 - allennlp.common.params - INFO - character_tokenizer.lowercase_characters = True\n",
      "2020-08-02 16:48:27,384 - allennlp.common.params - INFO - character_tokenizer.start_tokens = None\n",
      "2020-08-02 16:48:27,385 - allennlp.common.params - INFO - character_tokenizer.end_tokens = None\n",
      "2020-08-02 16:48:27,392 - allennlp.common.params - INFO - start_tokens = None\n",
      "2020-08-02 16:48:27,393 - allennlp.common.params - INFO - end_tokens = None\n",
      "2020-08-02 16:48:27,394 - allennlp.common.params - INFO - min_padding_length = 0\n",
      "2020-08-02 16:48:27,394 - allennlp.common.params - INFO - token_min_padding_length = 0\n",
      "2020-08-02 16:48:27,395 - allennlp.common.params - INFO - type = basic\n",
      "2020-08-02 16:48:27,397 - allennlp.common.params - INFO - token_embedders.word.type = embedding\n",
      "2020-08-02 16:48:27,398 - allennlp.common.params - INFO - token_embedders.word.embedding_dim = 300\n",
      "2020-08-02 16:48:27,398 - allennlp.common.params - INFO - token_embedders.word.num_embeddings = None\n",
      "2020-08-02 16:48:27,399 - allennlp.common.params - INFO - token_embedders.word.projection_dim = None\n",
      "2020-08-02 16:48:27,400 - allennlp.common.params - INFO - token_embedders.word.weight = None\n",
      "2020-08-02 16:48:27,401 - allennlp.common.params - INFO - token_embedders.word.padding_index = None\n",
      "2020-08-02 16:48:27,402 - allennlp.common.params - INFO - token_embedders.word.trainable = True\n",
      "2020-08-02 16:48:27,402 - allennlp.common.params - INFO - token_embedders.word.max_norm = None\n",
      "2020-08-02 16:48:27,403 - allennlp.common.params - INFO - token_embedders.word.norm_type = 2.0\n",
      "2020-08-02 16:48:27,404 - allennlp.common.params - INFO - token_embedders.word.scale_grad_by_freq = False\n",
      "2020-08-02 16:48:27,404 - allennlp.common.params - INFO - token_embedders.word.sparse = False\n",
      "2020-08-02 16:48:27,405 - allennlp.common.params - INFO - token_embedders.word.vocab_namespace = word\n",
      "2020-08-02 16:48:27,406 - allennlp.common.params - INFO - token_embedders.word.pretrained_file = cc.es.300.vec.gz\n",
      "2020-08-02 16:48:27,409 - allennlp.modules.token_embedders.embedding - INFO - Reading pretrained embeddings from file\n",
      "2020-08-02 16:48:27,411 - allennlp.modules.token_embedders.embedding - INFO - Recognized a header line in the embedding file with number of tokens: 2000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59a6be89279540a291c0fa7a26d02a12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2000000.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 16:49:11,503 - allennlp.modules.token_embedders.embedding - INFO - Initializing pre-trained embedding layer\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 16:49:11,705 - allennlp.modules.token_embedders.embedding - INFO - Pretrained embeddings were found for 11990 out of 14133 tokens\n",
      "2020-08-02 16:49:11,712 - allennlp.common.params - INFO - token_embedders.char.type = character_encoding\n",
      "2020-08-02 16:49:11,716 - allennlp.common.params - INFO - token_embedders.char.embedding.embedding_dim = 64\n",
      "2020-08-02 16:49:11,716 - allennlp.common.params - INFO - token_embedders.char.embedding.num_embeddings = None\n",
      "2020-08-02 16:49:11,717 - allennlp.common.params - INFO - token_embedders.char.embedding.projection_dim = None\n",
      "2020-08-02 16:49:11,718 - allennlp.common.params - INFO - token_embedders.char.embedding.weight = None\n",
      "2020-08-02 16:49:11,719 - allennlp.common.params - INFO - token_embedders.char.embedding.padding_index = None\n",
      "2020-08-02 16:49:11,720 - allennlp.common.params - INFO - token_embedders.char.embedding.trainable = True\n",
      "2020-08-02 16:49:11,720 - allennlp.common.params - INFO - token_embedders.char.embedding.max_norm = None\n",
      "2020-08-02 16:49:11,721 - allennlp.common.params - INFO - token_embedders.char.embedding.norm_type = 2.0\n",
      "2020-08-02 16:49:11,722 - allennlp.common.params - INFO - token_embedders.char.embedding.scale_grad_by_freq = False\n",
      "2020-08-02 16:49:11,723 - allennlp.common.params - INFO - token_embedders.char.embedding.sparse = False\n",
      "2020-08-02 16:49:11,724 - allennlp.common.params - INFO - token_embedders.char.embedding.vocab_namespace = char\n",
      "2020-08-02 16:49:11,724 - allennlp.common.params - INFO - token_embedders.char.embedding.pretrained_file = None\n",
      "2020-08-02 16:49:11,727 - allennlp.common.params - INFO - token_embedders.char.encoder.type = allennlp.modules.seq2vec_encoders.pytorch_seq2vec_wrapper.GruSeq2VecEncoder\n",
      "2020-08-02 16:49:11,728 - allennlp.common.params - INFO - token_embedders.char.encoder.input_size = 64\n",
      "2020-08-02 16:49:11,729 - allennlp.common.params - INFO - token_embedders.char.encoder.hidden_size = 128\n",
      "2020-08-02 16:49:11,730 - allennlp.common.params - INFO - token_embedders.char.encoder.num_layers = 1\n",
      "2020-08-02 16:49:11,731 - allennlp.common.params - INFO - token_embedders.char.encoder.bias = True\n",
      "2020-08-02 16:49:11,732 - allennlp.common.params - INFO - token_embedders.char.encoder.dropout = 0.0\n",
      "2020-08-02 16:49:11,732 - allennlp.common.params - INFO - token_embedders.char.encoder.bidirectional = True\n",
      "2020-08-02 16:49:11,736 - allennlp.common.params - INFO - token_embedders.char.dropout = 0.16517050992687604\n",
      "2020-08-02 16:49:11,737 - allennlp.common.params - INFO - input_size = 556\n",
      "2020-08-02 16:49:11,738 - allennlp.common.params - INFO - hidden_size = 512\n",
      "2020-08-02 16:49:11,738 - allennlp.common.params - INFO - num_layers = 1\n",
      "2020-08-02 16:49:11,739 - allennlp.common.params - INFO - bias = True\n",
      "2020-08-02 16:49:11,740 - allennlp.common.params - INFO - dropout = 0.0\n",
      "2020-08-02 16:49:11,741 - allennlp.common.params - INFO - bidirectional = True\n",
      "2020-08-02 16:49:11,742 - allennlp.common.params - INFO - stateful = False\n",
      "2020-08-02 16:49:11,779 - allennlp.common.params - INFO - labels = ['O', 'I-MORFOLOGIA_NEOPLASIA', 'U-MORFOLOGIA_NEOPLASIA', 'B-MORFOLOGIA_NEOPLASIA', 'L-MORFOLOGIA_NEOPLASIA']\n",
      "2020-08-02 16:49:11,780 - allennlp.common.params - INFO - label_encoding = BIOUL\n",
      "2020-08-02 16:49:11,780 - allennlp.common.params - INFO - top_k = 1\n",
      "2020-08-02 16:49:11,781 - allennlp.common.params - INFO - dropout = 0.2689579604286324\n",
      "2020-08-02 16:49:11,782 - allennlp.common.params - INFO - feedforward = None\n",
      "2020-08-02 16:49:20,880 - allennlp.common.params - INFO - random_seed = 13370\n",
      "2020-08-02 16:49:20,881 - allennlp.common.params - INFO - numpy_seed = 1337\n",
      "2020-08-02 16:49:20,882 - allennlp.common.params - INFO - pytorch_seed = 133\n",
      "2020-08-02 16:49:20,897 - allennlp.common.checks - INFO - Pytorch version: 1.5.1\n",
      "2020-08-02 16:49:20,958 - allennlp.common.params - INFO - type = gradient_descent\n",
      "2020-08-02 16:49:20,960 - allennlp.common.params - INFO - local_rank = 0\n",
      "2020-08-02 16:49:20,961 - allennlp.common.params - INFO - patience = 3\n",
      "2020-08-02 16:49:20,962 - allennlp.common.params - INFO - validation_metric = +f1-measure-overall\n",
      "2020-08-02 16:49:20,962 - allennlp.common.params - INFO - num_epochs = 4\n",
      "2020-08-02 16:49:20,964 - allennlp.common.params - INFO - cuda_device = -1\n",
      "2020-08-02 16:49:20,965 - allennlp.common.params - INFO - grad_norm = None\n",
      "2020-08-02 16:49:20,966 - allennlp.common.params - INFO - grad_clipping = None\n",
      "2020-08-02 16:49:20,967 - allennlp.common.params - INFO - distributed = None\n",
      "2020-08-02 16:49:20,967 - allennlp.common.params - INFO - world_size = 1\n",
      "2020-08-02 16:49:20,968 - allennlp.common.params - INFO - num_gradient_accumulation_steps = 1\n",
      "2020-08-02 16:49:20,969 - allennlp.common.params - INFO - opt_level = None\n",
      "2020-08-02 16:49:20,970 - allennlp.common.params - INFO - no_grad = None\n",
      "2020-08-02 16:49:20,971 - allennlp.common.params - INFO - momentum_scheduler = None\n",
      "2020-08-02 16:49:20,972 - allennlp.common.params - INFO - moving_average = None\n",
      "2020-08-02 16:49:20,972 - allennlp.common.params - INFO - batch_callbacks = None\n",
      "2020-08-02 16:49:20,974 - allennlp.common.util - INFO - The following parameters are Frozen (without gradient):\n",
      "2020-08-02 16:49:20,974 - allennlp.common.util - INFO - _head._crf._constraint_mask\n",
      "2020-08-02 16:49:20,975 - allennlp.common.util - INFO - The following parameters are Tunable (with gradient):\n",
      "2020-08-02 16:49:20,976 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_word.weight\n",
      "2020-08-02 16:49:20,976 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._embedding._module.weight\n",
      "2020-08-02 16:49:20,977 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_ih_l0\n",
      "2020-08-02 16:49:20,978 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_hh_l0\n",
      "2020-08-02 16:49:20,978 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_ih_l0\n",
      "2020-08-02 16:49:20,979 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_hh_l0\n",
      "2020-08-02 16:49:20,979 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_ih_l0_reverse\n",
      "2020-08-02 16:49:20,980 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_hh_l0_reverse\n",
      "2020-08-02 16:49:20,981 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_ih_l0_reverse\n",
      "2020-08-02 16:49:20,981 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_hh_l0_reverse\n",
      "2020-08-02 16:49:20,982 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_ih_l0\n",
      "2020-08-02 16:49:20,983 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_hh_l0\n",
      "2020-08-02 16:49:20,983 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_ih_l0\n",
      "2020-08-02 16:49:20,984 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_hh_l0\n",
      "2020-08-02 16:49:20,985 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_ih_l0_reverse\n",
      "2020-08-02 16:49:20,985 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_hh_l0_reverse\n",
      "2020-08-02 16:49:20,986 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_ih_l0_reverse\n",
      "2020-08-02 16:49:20,987 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_hh_l0_reverse\n",
      "2020-08-02 16:49:20,987 - allennlp.common.util - INFO - _head._label_projection_layer._module.weight\n",
      "2020-08-02 16:49:20,988 - allennlp.common.util - INFO - _head._label_projection_layer._module.bias\n",
      "2020-08-02 16:49:20,989 - allennlp.common.util - INFO - _head._crf.transitions\n",
      "2020-08-02 16:49:20,989 - allennlp.common.util - INFO - _head._crf.start_transitions\n",
      "2020-08-02 16:49:20,990 - allennlp.common.util - INFO - _head._crf.end_transitions\n",
      "2020-08-02 16:49:20,991 - allennlp.common.params - INFO - optimizer.type = adamw\n",
      "2020-08-02 16:49:20,992 - allennlp.common.params - INFO - optimizer.parameter_groups = None\n",
      "2020-08-02 16:49:20,993 - allennlp.common.params - INFO - optimizer.lr = 0.0038931174186587806\n",
      "2020-08-02 16:49:20,993 - allennlp.common.params - INFO - optimizer.betas = (0.9, 0.999)\n",
      "2020-08-02 16:49:20,994 - allennlp.common.params - INFO - optimizer.eps = 1e-08\n",
      "2020-08-02 16:49:20,995 - allennlp.common.params - INFO - optimizer.weight_decay = 0.01\n",
      "2020-08-02 16:49:20,995 - allennlp.common.params - INFO - optimizer.amsgrad = False\n",
      "2020-08-02 16:49:20,996 - allennlp.training.optimizers - INFO - Number of trainable parameters: 8783556\n",
      "2020-08-02 16:49:20,998 - allennlp.common.params - INFO - learning_rate_scheduler.type = step\n",
      "2020-08-02 16:49:20,998 - allennlp.common.params - INFO - learning_rate_scheduler.step_size = 2\n",
      "2020-08-02 16:49:20,999 - allennlp.common.params - INFO - learning_rate_scheduler.gamma = 0.1\n",
      "2020-08-02 16:49:21,000 - allennlp.common.params - INFO - learning_rate_scheduler.last_epoch = -1\n",
      "2020-08-02 16:49:21,001 - allennlp.common.params - INFO - checkpointer.type = default\n",
      "2020-08-02 16:49:21,002 - allennlp.common.params - INFO - checkpointer.keep_serialized_model_every_num_seconds = None\n",
      "2020-08-02 16:49:21,002 - allennlp.common.params - INFO - checkpointer.num_serialized_models_to_keep = 1\n",
      "2020-08-02 16:49:21,003 - allennlp.common.params - INFO - checkpointer.model_save_interval = None\n",
      "2020-08-02 16:49:21,004 - allennlp.common.params - INFO - tensorboard_writer.summary_interval = 100\n",
      "2020-08-02 16:49:21,004 - allennlp.common.params - INFO - tensorboard_writer.histogram_interval = None\n",
      "2020-08-02 16:49:21,005 - allennlp.common.params - INFO - tensorboard_writer.batch_size_interval = None\n",
      "2020-08-02 16:49:21,006 - allennlp.common.params - INFO - tensorboard_writer.should_log_parameter_statistics = True\n",
      "2020-08-02 16:49:21,006 - allennlp.common.params - INFO - tensorboard_writer.should_log_learning_rate = True\n",
      "2020-08-02 16:49:21,007 - allennlp.common.params - INFO - tensorboard_writer.get_batch_num_total = None\n",
      "2020-08-02 16:49:21,590 - allennlp.training.trainer - INFO - Beginning training.\n",
      "2020-08-02 16:49:21,591 - allennlp.training.trainer - INFO - Epoch 0/3\n",
      "2020-08-02 16:49:21,593 - allennlp.training.trainer - INFO - Worker 0 memory usage MB: 1117.38\n",
      "2020-08-02 16:49:21,644 - allennlp.training.trainer - INFO - Training\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04e3932d47334c1dbc88a711dc6c094f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1122.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 17:17:10,553 - allennlp.training.tensorboard_writer - INFO -                                     Training |  Validation\n",
      "2020-08-02 17:17:10,557 - allennlp.training.tensorboard_writer - INFO - accuracy                        |     0.984  |       N/A\n",
      "2020-08-02 17:17:10,559 - allennlp.training.tensorboard_writer - INFO - accuracy_1                      |     0.984  |       N/A\n",
      "2020-08-02 17:17:10,562 - allennlp.training.tensorboard_writer - INFO - f1-measure-MORFOLOGIA_NEOPLASIA |     0.724  |       N/A\n",
      "2020-08-02 17:17:10,563 - allennlp.training.tensorboard_writer - INFO - f1-measure-overall              |     0.724  |       N/A\n",
      "2020-08-02 17:17:10,566 - allennlp.training.tensorboard_writer - INFO - loss                            |    31.405  |       N/A\n",
      "2020-08-02 17:17:10,567 - allennlp.training.tensorboard_writer - INFO - precision-MORFOLOGIA_NEOPLASIA  |     0.756  |       N/A\n",
      "2020-08-02 17:17:10,569 - allennlp.training.tensorboard_writer - INFO - precision-overall               |     0.756  |       N/A\n",
      "2020-08-02 17:17:10,570 - allennlp.training.tensorboard_writer - INFO - recall-MORFOLOGIA_NEOPLASIA     |     0.695  |       N/A\n",
      "2020-08-02 17:17:10,572 - allennlp.training.tensorboard_writer - INFO - recall-overall                  |     0.695  |       N/A\n",
      "2020-08-02 17:17:10,573 - allennlp.training.tensorboard_writer - INFO - reg_loss                        |     0.000  |       N/A\n",
      "2020-08-02 17:17:10,575 - allennlp.training.tensorboard_writer - INFO - worker_0_memory_MB              |  1117.380  |       N/A\n",
      "2020-08-02 17:17:10,656 - allennlp.training.checkpointer - INFO - Best validation performance so far. Copying weights to 'runs/best_normal_lstm/lstm_full_train/best.th'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 17:17:11,049 - allennlp.training.trainer - INFO - Epoch duration: 0:27:49.458116\n",
      "2020-08-02 17:17:11,051 - allennlp.training.trainer - INFO - Estimated training time remaining: 1:23:28\n",
      "2020-08-02 17:17:11,054 - allennlp.training.trainer - INFO - Epoch 1/3\n",
      "2020-08-02 17:17:11,054 - allennlp.training.trainer - INFO - Worker 0 memory usage MB: 1581.404\n",
      "2020-08-02 17:17:11,108 - allennlp.training.trainer - INFO - Training\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b23d7df19b66486294fe32020433a8a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1122.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 17:44:10,659 - allennlp.training.tensorboard_writer - INFO -                                     Training |  Validation\n",
      "2020-08-02 17:44:10,660 - allennlp.training.tensorboard_writer - INFO - accuracy                        |     0.990  |       N/A\n",
      "2020-08-02 17:44:10,662 - allennlp.training.tensorboard_writer - INFO - accuracy_1                      |     0.990  |       N/A\n",
      "2020-08-02 17:44:10,664 - allennlp.training.tensorboard_writer - INFO - f1-measure-MORFOLOGIA_NEOPLASIA |     0.843  |       N/A\n",
      "2020-08-02 17:44:10,665 - allennlp.training.tensorboard_writer - INFO - f1-measure-overall              |     0.843  |       N/A\n",
      "2020-08-02 17:44:10,666 - allennlp.training.tensorboard_writer - INFO - loss                            |     9.481  |       N/A\n",
      "2020-08-02 17:44:10,668 - allennlp.training.tensorboard_writer - INFO - precision-MORFOLOGIA_NEOPLASIA  |     0.852  |       N/A\n",
      "2020-08-02 17:44:10,670 - allennlp.training.tensorboard_writer - INFO - precision-overall               |     0.852  |       N/A\n",
      "2020-08-02 17:44:10,671 - allennlp.training.tensorboard_writer - INFO - recall-MORFOLOGIA_NEOPLASIA     |     0.835  |       N/A\n",
      "2020-08-02 17:44:10,673 - allennlp.training.tensorboard_writer - INFO - recall-overall                  |     0.835  |       N/A\n",
      "2020-08-02 17:44:10,675 - allennlp.training.tensorboard_writer - INFO - reg_loss                        |     0.000  |       N/A\n",
      "2020-08-02 17:44:10,678 - allennlp.training.tensorboard_writer - INFO - worker_0_memory_MB              |  1581.404  |       N/A\n",
      "2020-08-02 17:44:10,759 - allennlp.training.checkpointer - INFO - Best validation performance so far. Copying weights to 'runs/best_normal_lstm/lstm_full_train/best.th'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 17:44:11,188 - allennlp.training.trainer - INFO - Epoch duration: 0:27:00.133853\n",
      "2020-08-02 17:44:11,189 - allennlp.training.trainer - INFO - Estimated training time remaining: 0:54:49\n",
      "2020-08-02 17:44:11,192 - allennlp.training.trainer - INFO - Epoch 2/3\n",
      "2020-08-02 17:44:11,192 - allennlp.training.trainer - INFO - Worker 0 memory usage MB: 1807.144\n",
      "2020-08-02 17:44:11,253 - allennlp.training.trainer - INFO - Training\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af97e04a55c945f2945cb42fe907419d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1122.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 18:11:20,105 - allennlp.training.tensorboard_writer - INFO -                                     Training |  Validation\n",
      "2020-08-02 18:11:20,107 - allennlp.training.tensorboard_writer - INFO - accuracy                        |     0.994  |       N/A\n",
      "2020-08-02 18:11:20,110 - allennlp.training.tensorboard_writer - INFO - accuracy_1                      |     0.994  |       N/A\n",
      "2020-08-02 18:11:20,113 - allennlp.training.tensorboard_writer - INFO - f1-measure-MORFOLOGIA_NEOPLASIA |     0.907  |       N/A\n",
      "2020-08-02 18:11:20,114 - allennlp.training.tensorboard_writer - INFO - f1-measure-overall              |     0.907  |       N/A\n",
      "2020-08-02 18:11:20,116 - allennlp.training.tensorboard_writer - INFO - loss                            |     5.085  |       N/A\n",
      "2020-08-02 18:11:20,117 - allennlp.training.tensorboard_writer - INFO - precision-MORFOLOGIA_NEOPLASIA  |     0.910  |       N/A\n",
      "2020-08-02 18:11:20,119 - allennlp.training.tensorboard_writer - INFO - precision-overall               |     0.910  |       N/A\n",
      "2020-08-02 18:11:20,120 - allennlp.training.tensorboard_writer - INFO - recall-MORFOLOGIA_NEOPLASIA     |     0.905  |       N/A\n",
      "2020-08-02 18:11:20,122 - allennlp.training.tensorboard_writer - INFO - recall-overall                  |     0.905  |       N/A\n",
      "2020-08-02 18:11:20,123 - allennlp.training.tensorboard_writer - INFO - reg_loss                        |     0.000  |       N/A\n",
      "2020-08-02 18:11:20,125 - allennlp.training.tensorboard_writer - INFO - worker_0_memory_MB              |  1807.144  |       N/A\n",
      "2020-08-02 18:11:20,206 - allennlp.training.checkpointer - INFO - Best validation performance so far. Copying weights to 'runs/best_normal_lstm/lstm_full_train/best.th'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 18:11:20,669 - allennlp.training.trainer - INFO - Epoch duration: 0:27:09.476878\n",
      "2020-08-02 18:11:20,670 - allennlp.training.trainer - INFO - Estimated training time remaining: 0:27:19\n",
      "2020-08-02 18:11:20,673 - allennlp.training.trainer - INFO - Epoch 3/3\n",
      "2020-08-02 18:11:20,674 - allennlp.training.trainer - INFO - Worker 0 memory usage MB: 1807.144\n",
      "2020-08-02 18:11:20,740 - allennlp.training.trainer - INFO - Training\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8726f8f6066348429fe6016f1e330918",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1122.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 18:38:24,587 - allennlp.training.tensorboard_writer - INFO -                                     Training |  Validation\n",
      "2020-08-02 18:38:24,588 - allennlp.training.tensorboard_writer - INFO - accuracy                        |     0.995  |       N/A\n",
      "2020-08-02 18:38:24,590 - allennlp.training.tensorboard_writer - INFO - accuracy_1                      |     0.995  |       N/A\n",
      "2020-08-02 18:38:24,592 - allennlp.training.tensorboard_writer - INFO - f1-measure-MORFOLOGIA_NEOPLASIA |     0.927  |       N/A\n",
      "2020-08-02 18:38:24,594 - allennlp.training.tensorboard_writer - INFO - f1-measure-overall              |     0.927  |       N/A\n",
      "2020-08-02 18:38:24,595 - allennlp.training.tensorboard_writer - INFO - loss                            |     3.938  |       N/A\n",
      "2020-08-02 18:38:24,596 - allennlp.training.tensorboard_writer - INFO - precision-MORFOLOGIA_NEOPLASIA  |     0.928  |       N/A\n",
      "2020-08-02 18:38:24,598 - allennlp.training.tensorboard_writer - INFO - precision-overall               |     0.928  |       N/A\n",
      "2020-08-02 18:38:24,599 - allennlp.training.tensorboard_writer - INFO - recall-MORFOLOGIA_NEOPLASIA     |     0.925  |       N/A\n",
      "2020-08-02 18:38:24,601 - allennlp.training.tensorboard_writer - INFO - recall-overall                  |     0.925  |       N/A\n",
      "2020-08-02 18:38:24,602 - allennlp.training.tensorboard_writer - INFO - reg_loss                        |     0.000  |       N/A\n",
      "2020-08-02 18:38:24,604 - allennlp.training.tensorboard_writer - INFO - worker_0_memory_MB              |  1807.144  |       N/A\n",
      "2020-08-02 18:38:24,690 - allennlp.training.checkpointer - INFO - Best validation performance so far. Copying weights to 'runs/best_normal_lstm/lstm_full_train/best.th'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-08-02 18:38:25,177 - allennlp.training.trainer - INFO - Epoch duration: 0:27:04.504302\n",
      "2020-08-02 18:38:25,182 - allennlp.training.checkpointer - INFO - loading best weights\n",
      "2020-08-02 18:38:25,204 - allennlp.models.archival - INFO - archiving weights and vocabulary to runs/best_normal_lstm/lstm_full_train/model.tar.gz\n",
      "Registered model 'candemist-ner-first-hpo' already exists. Creating a new version of this model...\n",
      "Created version '31' of model 'candemist-ner-first-hpo'.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainingResults(model_path='runs/best_normal_lstm/lstm_full_train/model.tar.gz', metrics={'best_epoch': 3, 'peak_worker_0_memory_MB': 1807.144, 'training_duration': '1:49:03.014038', 'training_start_epoch': 0, 'training_epochs': 3, 'epoch': 3, 'training_accuracy': 0.995463095028384, 'training_accuracy_1': 0.995463095028384, 'training_precision-MORFOLOGIA_NEOPLASIA': 0.9279664501274566, 'training_recall-MORFOLOGIA_NEOPLASIA': 0.9254551418730523, 'training_f1-measure-MORFOLOGIA_NEOPLASIA': 0.9267090946417074, 'training_precision-overall': 0.9279664501274566, 'training_recall-overall': 0.9254551418730523, 'training_f1-measure-overall': 0.9267090946417074, 'training_loss': 3.9377396170468253, 'training_reg_loss': 0.0, 'training_worker_0_memory_MB': 1807.144})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.train(\n",
    "    output=\"runs/best_normal_lstm/lstm_full_train\",\n",
    "    training=train_ds,\n",
    "    #validation=valid_ds,\n",
    "    trainer=trainer_config,\n",
    "    loggers=[mlflow_logger],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}