Fine-tuningDeepAttentions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a3106297-f71d-4de9-ae2b-bbd4f16c74f3",
   "metadata": {},
   "source": [
    "### Installation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b29b61dd-e00d-4668-a632-e31a53541ec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install datasets transformers rouge-score nltk\n",
    "!git clone https://github.com/xmu-xiaoma666/External-Attention-pytorch.git ext_attns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39745406-4976-4762-b166-2fe29fe8d92e",
   "metadata": {},
   "source": [
    "### About dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "828b258b-c091-4f2f-9f91-fda068d293f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: 1020\n",
      "val:   408\n",
      "test:  409\n"
     ]
    }
   ],
   "source": [
    "print(f'train: {len(tr)}')\n",
    "print(f'val:   {len(val)}')\n",
    "print(f'test:  {len(te)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "178f33ca-f212-49c6-a73f-5a120a7dea6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "### 1L Attentions w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a19eef10-d82b-4e73-baf4-efaeeae98c2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(2) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)+x\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)+x\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf2f090a-b1cf-48d3-a28b-b47f87495411",
   "metadata": {},
   "source": [
    "### 1L Attentions  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f911f43-b4ee-4cde-aa6b-3b331fefcd8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:20, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.805763</td>\n",
       "      <td>18.729400</td>\n",
       "      <td>2.232400</td>\n",
       "      <td>15.120100</td>\n",
       "      <td>15.100500</td>\n",
       "      <td>19.225500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.377273</td>\n",
       "      <td>16.439500</td>\n",
       "      <td>2.495900</td>\n",
       "      <td>13.501100</td>\n",
       "      <td>13.508500</td>\n",
       "      <td>19.208300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.615000</td>\n",
       "      <td>6.192240</td>\n",
       "      <td>18.489900</td>\n",
       "      <td>2.970200</td>\n",
       "      <td>14.754100</td>\n",
       "      <td>14.772300</td>\n",
       "      <td>19.938700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.615000</td>\n",
       "      <td>6.077089</td>\n",
       "      <td>18.041300</td>\n",
       "      <td>2.816100</td>\n",
       "      <td>14.566800</td>\n",
       "      <td>14.602400</td>\n",
       "      <td>19.938700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.615000</td>\n",
       "      <td>5.941630</td>\n",
       "      <td>19.622500</td>\n",
       "      <td>3.125100</td>\n",
       "      <td>15.594300</td>\n",
       "      <td>15.623100</td>\n",
       "      <td>19.953400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>5.037400</td>\n",
       "      <td>5.900609</td>\n",
       "      <td>19.821400</td>\n",
       "      <td>3.503700</td>\n",
       "      <td>15.962900</td>\n",
       "      <td>15.972400</td>\n",
       "      <td>19.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>5.037400</td>\n",
       "      <td>5.843113</td>\n",
       "      <td>19.004700</td>\n",
       "      <td>3.237300</td>\n",
       "      <td>15.422800</td>\n",
       "      <td>15.463900</td>\n",
       "      <td>19.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>5.037400</td>\n",
       "      <td>5.823614</td>\n",
       "      <td>18.970100</td>\n",
       "      <td>3.295100</td>\n",
       "      <td>15.309600</td>\n",
       "      <td>15.332400</td>\n",
       "      <td>19.977900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>4.319400</td>\n",
       "      <td>5.769156</td>\n",
       "      <td>18.256700</td>\n",
       "      <td>3.483500</td>\n",
       "      <td>14.742000</td>\n",
       "      <td>14.767200</td>\n",
       "      <td>19.833300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>4.319400</td>\n",
       "      <td>5.745662</td>\n",
       "      <td>18.762500</td>\n",
       "      <td>3.410700</td>\n",
       "      <td>15.070700</td>\n",
       "      <td>15.079200</td>\n",
       "      <td>19.897100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>4.319400</td>\n",
       "      <td>5.724029</td>\n",
       "      <td>18.192800</td>\n",
       "      <td>3.176500</td>\n",
       "      <td>14.655600</td>\n",
       "      <td>14.698700</td>\n",
       "      <td>19.683800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.886300</td>\n",
       "      <td>5.714006</td>\n",
       "      <td>17.858100</td>\n",
       "      <td>3.196600</td>\n",
       "      <td>14.499800</td>\n",
       "      <td>14.528800</td>\n",
       "      <td>19.568600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.886300</td>\n",
       "      <td>5.702322</td>\n",
       "      <td>17.857000</td>\n",
       "      <td>3.455800</td>\n",
       "      <td>14.615500</td>\n",
       "      <td>14.636300</td>\n",
       "      <td>19.674000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.886300</td>\n",
       "      <td>5.701082</td>\n",
       "      <td>17.862900</td>\n",
       "      <td>3.359700</td>\n",
       "      <td>14.543300</td>\n",
       "      <td>14.554200</td>\n",
       "      <td>19.627500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.623600</td>\n",
       "      <td>5.699451</td>\n",
       "      <td>17.901400</td>\n",
       "      <td>3.247600</td>\n",
       "      <td>14.548900</td>\n",
       "      <td>14.571400</td>\n",
       "      <td>19.676500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-170\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2210] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-340\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-510\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-680\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-850\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1020\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1190\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1360\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1530\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1700\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1870\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2040\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2210\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2380\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2550\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2040] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-2550 (score: 5.699451446533203).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 34:02, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.545643</td>\n",
       "      <td>11.527800</td>\n",
       "      <td>1.646700</td>\n",
       "      <td>8.978300</td>\n",
       "      <td>8.970700</td>\n",
       "      <td>12.098000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.171666</td>\n",
       "      <td>18.407100</td>\n",
       "      <td>2.738800</td>\n",
       "      <td>14.823700</td>\n",
       "      <td>14.826700</td>\n",
       "      <td>18.514700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.236800</td>\n",
       "      <td>5.978276</td>\n",
       "      <td>18.721400</td>\n",
       "      <td>3.086300</td>\n",
       "      <td>15.059900</td>\n",
       "      <td>15.046200</td>\n",
       "      <td>19.117600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.236800</td>\n",
       "      <td>5.838930</td>\n",
       "      <td>18.493600</td>\n",
       "      <td>2.978600</td>\n",
       "      <td>14.926400</td>\n",
       "      <td>14.955800</td>\n",
       "      <td>19.558800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.236800</td>\n",
       "      <td>5.763635</td>\n",
       "      <td>18.549100</td>\n",
       "      <td>2.925900</td>\n",
       "      <td>14.674600</td>\n",
       "      <td>14.688700</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.518000</td>\n",
       "      <td>5.645795</td>\n",
       "      <td>18.264800</td>\n",
       "      <td>2.986500</td>\n",
       "      <td>14.824000</td>\n",
       "      <td>14.848600</td>\n",
       "      <td>19.852900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.518000</td>\n",
       "      <td>5.628169</td>\n",
       "      <td>17.177800</td>\n",
       "      <td>2.872700</td>\n",
       "      <td>13.899400</td>\n",
       "      <td>13.934900</td>\n",
       "      <td>19.897100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.518000</td>\n",
       "      <td>5.613062</td>\n",
       "      <td>16.148800</td>\n",
       "      <td>3.013500</td>\n",
       "      <td>13.141800</td>\n",
       "      <td>13.165100</td>\n",
       "      <td>19.835800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.791500</td>\n",
       "      <td>5.533463</td>\n",
       "      <td>15.988700</td>\n",
       "      <td>2.759300</td>\n",
       "      <td>13.122200</td>\n",
       "      <td>13.144100</td>\n",
       "      <td>19.791700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.791500</td>\n",
       "      <td>5.494953</td>\n",
       "      <td>15.786000</td>\n",
       "      <td>2.838000</td>\n",
       "      <td>12.865600</td>\n",
       "      <td>12.890000</td>\n",
       "      <td>19.877500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>3.791500</td>\n",
       "      <td>5.518980</td>\n",
       "      <td>15.270200</td>\n",
       "      <td>2.524300</td>\n",
       "      <td>12.501100</td>\n",
       "      <td>12.542400</td>\n",
       "      <td>19.791700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.352600</td>\n",
       "      <td>5.512968</td>\n",
       "      <td>15.022100</td>\n",
       "      <td>2.567800</td>\n",
       "      <td>12.250800</td>\n",
       "      <td>12.274500</td>\n",
       "      <td>19.774500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.352600</td>\n",
       "      <td>5.502841</td>\n",
       "      <td>15.132300</td>\n",
       "      <td>2.694100</td>\n",
       "      <td>12.664100</td>\n",
       "      <td>12.666400</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.352600</td>\n",
       "      <td>5.496234</td>\n",
       "      <td>15.188000</td>\n",
       "      <td>2.579300</td>\n",
       "      <td>12.400700</td>\n",
       "      <td>12.427800</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.089500</td>\n",
       "      <td>5.505139</td>\n",
       "      <td>14.914700</td>\n",
       "      <td>2.428200</td>\n",
       "      <td>12.230000</td>\n",
       "      <td>12.262000</td>\n",
       "      <td>19.759800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-170\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-340\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-510\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-680\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-850\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1020\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1190\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1360\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1530\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1700\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1870\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2040\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2210\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2380\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2550\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-1700 (score: 5.494953155517578).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:22, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.163243</td>\n",
       "      <td>20.095200</td>\n",
       "      <td>3.422000</td>\n",
       "      <td>16.205200</td>\n",
       "      <td>16.233500</td>\n",
       "      <td>19.458300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.869232</td>\n",
       "      <td>19.689400</td>\n",
       "      <td>3.329400</td>\n",
       "      <td>15.640800</td>\n",
       "      <td>15.684800</td>\n",
       "      <td>19.740200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.552000</td>\n",
       "      <td>5.829080</td>\n",
       "      <td>19.818200</td>\n",
       "      <td>3.546000</td>\n",
       "      <td>15.803300</td>\n",
       "      <td>15.829000</td>\n",
       "      <td>19.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.552000</td>\n",
       "      <td>5.787114</td>\n",
       "      <td>18.778000</td>\n",
       "      <td>3.192600</td>\n",
       "      <td>14.856800</td>\n",
       "      <td>14.905200</td>\n",
       "      <td>19.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.552000</td>\n",
       "      <td>5.801150</td>\n",
       "      <td>20.180600</td>\n",
       "      <td>3.938200</td>\n",
       "      <td>16.090500</td>\n",
       "      <td>16.130800</td>\n",
       "      <td>19.833300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.203600</td>\n",
       "      <td>5.789032</td>\n",
       "      <td>19.662900</td>\n",
       "      <td>3.850900</td>\n",
       "      <td>15.793600</td>\n",
       "      <td>15.827300</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.203600</td>\n",
       "      <td>5.791893</td>\n",
       "      <td>19.993600</td>\n",
       "      <td>3.965800</td>\n",
       "      <td>15.834500</td>\n",
       "      <td>15.874100</td>\n",
       "      <td>19.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.203600</td>\n",
       "      <td>5.807988</td>\n",
       "      <td>19.543300</td>\n",
       "      <td>3.910100</td>\n",
       "      <td>15.711500</td>\n",
       "      <td>15.764100</td>\n",
       "      <td>19.772100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.681200</td>\n",
       "      <td>5.792373</td>\n",
       "      <td>20.515700</td>\n",
       "      <td>4.265100</td>\n",
       "      <td>16.662400</td>\n",
       "      <td>16.692600</td>\n",
       "      <td>19.833300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.681200</td>\n",
       "      <td>5.795146</td>\n",
       "      <td>20.442100</td>\n",
       "      <td>4.116100</td>\n",
       "      <td>16.238800</td>\n",
       "      <td>16.266200</td>\n",
       "      <td>19.708300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.681200</td>\n",
       "      <td>5.794377</td>\n",
       "      <td>20.393900</td>\n",
       "      <td>4.214000</td>\n",
       "      <td>16.232900</td>\n",
       "      <td>16.237400</td>\n",
       "      <td>19.801500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.461100</td>\n",
       "      <td>5.793706</td>\n",
       "      <td>20.509600</td>\n",
       "      <td>4.322000</td>\n",
       "      <td>16.475700</td>\n",
       "      <td>16.496100</td>\n",
       "      <td>19.801500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.461100</td>\n",
       "      <td>5.796529</td>\n",
       "      <td>20.343600</td>\n",
       "      <td>4.343700</td>\n",
       "      <td>16.299100</td>\n",
       "      <td>16.340200</td>\n",
       "      <td>19.803900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.461100</td>\n",
       "      <td>5.803018</td>\n",
       "      <td>20.377500</td>\n",
       "      <td>4.320500</td>\n",
       "      <td>16.479900</td>\n",
       "      <td>16.482200</td>\n",
       "      <td>19.769600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.304400</td>\n",
       "      <td>5.806118</td>\n",
       "      <td>20.782100</td>\n",
       "      <td>4.600600</td>\n",
       "      <td>16.593100</td>\n",
       "      <td>16.612300</td>\n",
       "      <td>19.720600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-680 (score: 5.787114143371582).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:41, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.167797</td>\n",
       "      <td>19.601700</td>\n",
       "      <td>3.570300</td>\n",
       "      <td>15.751800</td>\n",
       "      <td>15.772100</td>\n",
       "      <td>19.867600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.043767</td>\n",
       "      <td>19.442100</td>\n",
       "      <td>3.532800</td>\n",
       "      <td>15.563200</td>\n",
       "      <td>15.560000</td>\n",
       "      <td>19.649500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.602700</td>\n",
       "      <td>6.064744</td>\n",
       "      <td>20.035400</td>\n",
       "      <td>4.239700</td>\n",
       "      <td>15.978900</td>\n",
       "      <td>15.999500</td>\n",
       "      <td>19.340700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.602700</td>\n",
       "      <td>6.058922</td>\n",
       "      <td>19.493000</td>\n",
       "      <td>3.517600</td>\n",
       "      <td>15.536300</td>\n",
       "      <td>15.569700</td>\n",
       "      <td>19.558800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.602700</td>\n",
       "      <td>6.044116</td>\n",
       "      <td>19.528500</td>\n",
       "      <td>3.915300</td>\n",
       "      <td>15.599300</td>\n",
       "      <td>15.634900</td>\n",
       "      <td>19.629900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.706600</td>\n",
       "      <td>6.063229</td>\n",
       "      <td>18.452900</td>\n",
       "      <td>3.286100</td>\n",
       "      <td>14.835400</td>\n",
       "      <td>14.875600</td>\n",
       "      <td>19.507400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.706600</td>\n",
       "      <td>6.126910</td>\n",
       "      <td>17.892300</td>\n",
       "      <td>3.307100</td>\n",
       "      <td>14.617700</td>\n",
       "      <td>14.621100</td>\n",
       "      <td>19.451000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.706600</td>\n",
       "      <td>6.234108</td>\n",
       "      <td>18.768200</td>\n",
       "      <td>3.896200</td>\n",
       "      <td>15.343200</td>\n",
       "      <td>15.373900</td>\n",
       "      <td>19.730400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.387300</td>\n",
       "      <td>6.143411</td>\n",
       "      <td>18.619800</td>\n",
       "      <td>3.869900</td>\n",
       "      <td>15.338800</td>\n",
       "      <td>15.342100</td>\n",
       "      <td>19.409300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.387300</td>\n",
       "      <td>6.182648</td>\n",
       "      <td>18.387700</td>\n",
       "      <td>3.982700</td>\n",
       "      <td>14.962500</td>\n",
       "      <td>14.987600</td>\n",
       "      <td>19.527000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.387300</td>\n",
       "      <td>6.228130</td>\n",
       "      <td>18.161500</td>\n",
       "      <td>3.768700</td>\n",
       "      <td>14.662600</td>\n",
       "      <td>14.659700</td>\n",
       "      <td>19.384800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.211200</td>\n",
       "      <td>6.215043</td>\n",
       "      <td>18.487500</td>\n",
       "      <td>3.991100</td>\n",
       "      <td>14.995200</td>\n",
       "      <td>15.008800</td>\n",
       "      <td>19.428900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.211200</td>\n",
       "      <td>6.308263</td>\n",
       "      <td>17.970200</td>\n",
       "      <td>3.954700</td>\n",
       "      <td>14.641700</td>\n",
       "      <td>14.635100</td>\n",
       "      <td>19.504900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.211200</td>\n",
       "      <td>6.319988</td>\n",
       "      <td>17.917000</td>\n",
       "      <td>3.969100</td>\n",
       "      <td>14.624900</td>\n",
       "      <td>14.650800</td>\n",
       "      <td>19.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.071300</td>\n",
       "      <td>6.340014</td>\n",
       "      <td>18.014100</td>\n",
       "      <td>3.954900</td>\n",
       "      <td>14.539400</td>\n",
       "      <td>14.558000</td>\n",
       "      <td>19.426500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-340 (score: 6.043766975402832).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:20, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.246838</td>\n",
       "      <td>20.522600</td>\n",
       "      <td>3.776400</td>\n",
       "      <td>16.463300</td>\n",
       "      <td>16.480000</td>\n",
       "      <td>19.794100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.882246</td>\n",
       "      <td>20.165500</td>\n",
       "      <td>3.560800</td>\n",
       "      <td>16.227000</td>\n",
       "      <td>16.236200</td>\n",
       "      <td>19.821100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.020900</td>\n",
       "      <td>5.835712</td>\n",
       "      <td>20.208100</td>\n",
       "      <td>3.905600</td>\n",
       "      <td>16.268300</td>\n",
       "      <td>16.265400</td>\n",
       "      <td>19.681400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.020900</td>\n",
       "      <td>5.809297</td>\n",
       "      <td>20.217700</td>\n",
       "      <td>3.888000</td>\n",
       "      <td>16.027500</td>\n",
       "      <td>16.056800</td>\n",
       "      <td>19.696100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.020900</td>\n",
       "      <td>5.800512</td>\n",
       "      <td>20.806900</td>\n",
       "      <td>4.167500</td>\n",
       "      <td>16.637300</td>\n",
       "      <td>16.660200</td>\n",
       "      <td>19.735300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.899900</td>\n",
       "      <td>5.796497</td>\n",
       "      <td>20.036100</td>\n",
       "      <td>4.058000</td>\n",
       "      <td>16.097200</td>\n",
       "      <td>16.137700</td>\n",
       "      <td>19.796600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.899900</td>\n",
       "      <td>5.790139</td>\n",
       "      <td>20.375800</td>\n",
       "      <td>4.286200</td>\n",
       "      <td>16.260900</td>\n",
       "      <td>16.290000</td>\n",
       "      <td>19.799000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.899900</td>\n",
       "      <td>5.810089</td>\n",
       "      <td>19.765100</td>\n",
       "      <td>3.961300</td>\n",
       "      <td>15.973400</td>\n",
       "      <td>15.985000</td>\n",
       "      <td>19.742600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.429800</td>\n",
       "      <td>5.789885</td>\n",
       "      <td>20.030200</td>\n",
       "      <td>4.136600</td>\n",
       "      <td>16.042400</td>\n",
       "      <td>16.042800</td>\n",
       "      <td>19.769600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.429800</td>\n",
       "      <td>5.795312</td>\n",
       "      <td>19.965100</td>\n",
       "      <td>4.003900</td>\n",
       "      <td>15.877200</td>\n",
       "      <td>15.936800</td>\n",
       "      <td>19.803900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.429800</td>\n",
       "      <td>5.800919</td>\n",
       "      <td>20.497100</td>\n",
       "      <td>4.217600</td>\n",
       "      <td>16.379100</td>\n",
       "      <td>16.428300</td>\n",
       "      <td>19.808800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.154400</td>\n",
       "      <td>5.820184</td>\n",
       "      <td>20.729500</td>\n",
       "      <td>4.232000</td>\n",
       "      <td>16.514000</td>\n",
       "      <td>16.558500</td>\n",
       "      <td>19.764700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.154400</td>\n",
       "      <td>5.806255</td>\n",
       "      <td>20.389600</td>\n",
       "      <td>4.222800</td>\n",
       "      <td>16.501100</td>\n",
       "      <td>16.537200</td>\n",
       "      <td>19.796600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.154400</td>\n",
       "      <td>5.809808</td>\n",
       "      <td>20.751000</td>\n",
       "      <td>4.444600</td>\n",
       "      <td>16.455300</td>\n",
       "      <td>16.507800</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.983400</td>\n",
       "      <td>5.808722</td>\n",
       "      <td>20.926000</td>\n",
       "      <td>4.350800</td>\n",
       "      <td>16.640100</td>\n",
       "      <td>16.681800</td>\n",
       "      <td>19.767200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-170\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-340\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-510\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-680\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-850\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-1530 (score: 5.7898850440979).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(1) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)+x\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)+x\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ce481b0-db3d-44c2-a62c-6927c523a9f5",
   "metadata": {},
   "source": [
    "### 2L Attentions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "596aa51d-6b0b-4c8d-8316-8069c6329766",
   "metadata": {},
   "source": [
    "|i|attention | used|\n",
    "|---|--------| ----|\n",
    "|0  |UFOAttention| ✔|\n",
    "|1  | AFT_FULL   |x|\n",
    "|2  | MUSEAttention|✔ |\n",
    "|3  | EMSA|     x|\n",
    "|4  |SimplifiedScaledDotProductAttention|✔ |\n",
    "|5  |ScaledDotProductAttention|✔ |\n",
    "|6  |ExternalAttention|✔ |"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef78691b-5e0f-496a-94b9-1879f251c6b7",
   "metadata": {},
   "source": [
    "#### common"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b40062da-6242-4f45-8249-589f3f3f34d6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Reusing dataset xsum (/home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bf6fb8b58e240d885cc949fef20c7d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-5d732b1c86657ea0.arrow\n",
      "Loading cached processed dataset at /home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-38966a651c72a103.arrow\n",
      "Loading cached processed dataset at /home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-d19c279816291ade.arrow\n",
      "Loading cached split indices for dataset at /home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-a37fd9f11dd44add.arrow and /home/yp/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-195107633ef23937.arrow\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "ext_path = os.path.abspath('./ext_attns/')\n",
    "import sys\n",
    "sys.path.append(ext_path)\n",
    "# list\n",
    "from model.attention.UFOAttention import *\n",
    "from model.attention.AFT import AFT_FULL\n",
    "from model.attention.MUSEAttention import MUSEAttention\n",
    "from model.attention.EMSA import EMSA\n",
    "from model.attention.SimplifiedSelfAttention import SimplifiedScaledDotProductAttention\n",
    "from model.attention.SelfAttention import ScaledDotProductAttention\n",
    "from model.attention.ExternalAttention import ExternalAttention\n",
    "d_model = 768 \n",
    "n_head  = 12  \n",
    "ufo = UFOAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "aft_full = AFT_FULL(d_model=d_model, n=n_head)\n",
    "ma = MUSEAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "emsa = EMSA(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head,H=n_head,W=n_head,ratio=2,apply_transform=True)\n",
    "ssa = SimplifiedScaledDotProductAttention(d_model=d_model, h=n_head)\n",
    "sa = ScaledDotProductAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "ea = ExternalAttention(d_model=d_model,S=8) #v\n",
    "\n",
    "attn_li = [ufo,aft_full,ma,emsa,ssa,sa,ea]\n",
    "\n",
    "import transformers\n",
    "\n",
    "# print(transformers.__version__)\n",
    "\n",
    "model_checkpoint = \"t5-small\"\n",
    "model_checkpoint = 'facebook/bart-base'\n",
    "\n",
    "from datasets import load_dataset, load_metric\n",
    "\n",
    "raw_datasets = load_dataset(\"xsum\")\n",
    "metric = load_metric(\"rouge\")\n",
    "\n",
    "# raw_datasets[\"train\"][0]\n",
    "\n",
    "import datasets\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "    \n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
    "\n",
    "# ?\n",
    "# with tokenizer.as_target_tokenizer():\n",
    "#     print(tokenizer([\"Hello, this one sentence!\", \"This is another sentence.\"]))\n",
    "\n",
    "if model_checkpoint in [\"t5-small\", \"t5-base\", \"t5-larg\", \"t5-3b\", \"t5-11b\"]:\n",
    "    prefix = \"summarize: \"\n",
    "else:\n",
    "    prefix = \"\"\n",
    "\n",
    "max_input_length = 1024\n",
    "max_target_length = 128\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    inputs = [prefix + doc for doc in examples[\"document\"]]\n",
    "    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)\n",
    "\n",
    "    # Setup the tokenizer for targets\n",
    "    with tokenizer.as_target_tokenizer():\n",
    "        labels = tokenizer(examples[\"summary\"], max_length=max_target_length, truncation=True)\n",
    "\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(preprocess_function, batched=True)\n",
    "\n",
    "from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)\n",
    "\n",
    "batch_size = 3 #16\n",
    "\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n",
    "\n",
    "import nltk\n",
    "import numpy as np\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    # Replace -100 in the labels as we can't decode them.\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "    \n",
    "    # Rouge expects a newline after each sentence\n",
    "    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]\n",
    "    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]\n",
    "    \n",
    "    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
    "    # Extract a few results\n",
    "    result = {key: value.mid.fmeasure * 100 for key, value in result.items()}\n",
    "    \n",
    "    # Add mean generated length\n",
    "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
    "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
    "    \n",
    "    return {k: round(v, 4) for k, v in result.items()}\n",
    "\n",
    "trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.005) # 800,1080\n",
    "# trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.01) # 800,2080\n",
    "# trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.02) # 800,4080\n",
    "tr = trvaltest['train']\n",
    "valtest = trvaltest['test'].train_test_split(test_size=0.5,train_size=0.5) # 208, 208\n",
    "# valtest = trvaltest['test'].train_test_split(test_size=0.5,train_size=0.5) # 408, 408\n",
    "# valtest = trvaltest['test'].train_test_split(test_size=0.05,train_size=0.05)\n",
    "val = valtest['train']\n",
    "te = valtest['test']\n",
    "\n",
    "del trvaltest\n",
    "del valtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "51b1d1b7-80ce-4576-a982-9130d9e2bea7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:29, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.476961</td>\n",
       "      <td>0.062200</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.056200</td>\n",
       "      <td>0.056200</td>\n",
       "      <td>4.426500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.080842</td>\n",
       "      <td>14.674200</td>\n",
       "      <td>0.160500</td>\n",
       "      <td>12.896500</td>\n",
       "      <td>12.891400</td>\n",
       "      <td>14.115200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>7.462100</td>\n",
       "      <td>6.937696</td>\n",
       "      <td>13.002700</td>\n",
       "      <td>0.954200</td>\n",
       "      <td>12.178900</td>\n",
       "      <td>12.145300</td>\n",
       "      <td>14.247500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>7.462100</td>\n",
       "      <td>6.844901</td>\n",
       "      <td>17.655000</td>\n",
       "      <td>1.336400</td>\n",
       "      <td>15.069400</td>\n",
       "      <td>15.213800</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>7.462100</td>\n",
       "      <td>6.780678</td>\n",
       "      <td>13.566600</td>\n",
       "      <td>0.801100</td>\n",
       "      <td>11.531400</td>\n",
       "      <td>11.689400</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6.169400</td>\n",
       "      <td>6.785341</td>\n",
       "      <td>13.654600</td>\n",
       "      <td>0.312100</td>\n",
       "      <td>11.405600</td>\n",
       "      <td>11.423500</td>\n",
       "      <td>19.985300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>6.169400</td>\n",
       "      <td>6.756629</td>\n",
       "      <td>15.945100</td>\n",
       "      <td>0.626100</td>\n",
       "      <td>13.097100</td>\n",
       "      <td>13.322200</td>\n",
       "      <td>19.958300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.169400</td>\n",
       "      <td>6.754077</td>\n",
       "      <td>13.645900</td>\n",
       "      <td>0.194900</td>\n",
       "      <td>10.882300</td>\n",
       "      <td>11.000800</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>5.837300</td>\n",
       "      <td>6.713006</td>\n",
       "      <td>15.595000</td>\n",
       "      <td>0.481300</td>\n",
       "      <td>12.261900</td>\n",
       "      <td>12.311300</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>5.837300</td>\n",
       "      <td>6.735507</td>\n",
       "      <td>15.973300</td>\n",
       "      <td>0.396200</td>\n",
       "      <td>12.838900</td>\n",
       "      <td>12.959000</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>5.837300</td>\n",
       "      <td>6.723297</td>\n",
       "      <td>14.407700</td>\n",
       "      <td>0.566300</td>\n",
       "      <td>11.927800</td>\n",
       "      <td>12.002800</td>\n",
       "      <td>19.958300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>5.668400</td>\n",
       "      <td>6.685848</td>\n",
       "      <td>14.564900</td>\n",
       "      <td>0.412000</td>\n",
       "      <td>11.891400</td>\n",
       "      <td>11.861700</td>\n",
       "      <td>19.958300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>5.668400</td>\n",
       "      <td>6.696715</td>\n",
       "      <td>15.111400</td>\n",
       "      <td>0.497200</td>\n",
       "      <td>12.530500</td>\n",
       "      <td>12.545500</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>5.668400</td>\n",
       "      <td>6.703750</td>\n",
       "      <td>14.591200</td>\n",
       "      <td>0.520800</td>\n",
       "      <td>11.928000</td>\n",
       "      <td>11.945500</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>5.531300</td>\n",
       "      <td>6.698007</td>\n",
       "      <td>15.174600</td>\n",
       "      <td>0.718800</td>\n",
       "      <td>12.452800</td>\n",
       "      <td>12.482600</td>\n",
       "      <td>20.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-170\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-340\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-510\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-680\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-850\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1020\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1190\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1360\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1530\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1700\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1870\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2040\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2210\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2380\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2550\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-2040 (score: 6.685848236083984).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:10, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.540518</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.463693</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>7.554300</td>\n",
       "      <td>7.459248</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>7.554300</td>\n",
       "      <td>7.510250</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>7.554300</td>\n",
       "      <td>7.522919</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6.701200</td>\n",
       "      <td>7.170269</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.029400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>6.701200</td>\n",
       "      <td>6.903314</td>\n",
       "      <td>0.083100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.090800</td>\n",
       "      <td>0.091500</td>\n",
       "      <td>2.838200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.701200</td>\n",
       "      <td>6.862229</td>\n",
       "      <td>0.321300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.316300</td>\n",
       "      <td>0.315900</td>\n",
       "      <td>3.928900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>6.027900</td>\n",
       "      <td>6.744917</td>\n",
       "      <td>2.697400</td>\n",
       "      <td>0.037700</td>\n",
       "      <td>2.490100</td>\n",
       "      <td>2.492900</td>\n",
       "      <td>11.451000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>6.027900</td>\n",
       "      <td>6.761720</td>\n",
       "      <td>1.712100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.642500</td>\n",
       "      <td>1.641700</td>\n",
       "      <td>8.649500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>6.027900</td>\n",
       "      <td>6.763495</td>\n",
       "      <td>1.595400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.497700</td>\n",
       "      <td>1.504000</td>\n",
       "      <td>8.970600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>5.690800</td>\n",
       "      <td>6.781571</td>\n",
       "      <td>4.888800</td>\n",
       "      <td>0.012900</td>\n",
       "      <td>4.791700</td>\n",
       "      <td>4.755300</td>\n",
       "      <td>11.539200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>5.690800</td>\n",
       "      <td>6.747264</td>\n",
       "      <td>3.830400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.773700</td>\n",
       "      <td>3.769300</td>\n",
       "      <td>11.227900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>5.690800</td>\n",
       "      <td>6.749215</td>\n",
       "      <td>4.298400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.266900</td>\n",
       "      <td>4.261500</td>\n",
       "      <td>12.688700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>5.485300</td>\n",
       "      <td>6.752226</td>\n",
       "      <td>4.418900</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.339200</td>\n",
       "      <td>4.356400</td>\n",
       "      <td>12.715700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-170\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-340\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-510\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-680\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-850\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1020\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1190\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1360\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1530\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1700\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1870\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2040\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2210\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2380\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2550\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-1530 (score: 6.744916915893555).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:40]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 32:56, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.496995</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.158624</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>7.401500</td>\n",
       "      <td>7.119234</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>7.401500</td>\n",
       "      <td>7.140253</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>7.401500</td>\n",
       "      <td>7.083230</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6.305200</td>\n",
       "      <td>7.123971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>6.305200</td>\n",
       "      <td>7.162121</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.305200</td>\n",
       "      <td>7.208167</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>6.039800</td>\n",
       "      <td>7.137269</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>6.039800</td>\n",
       "      <td>7.198385</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>6.039800</td>\n",
       "      <td>7.218431</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>5.929700</td>\n",
       "      <td>7.215989</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>5.929700</td>\n",
       "      <td>7.223846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>5.929700</td>\n",
       "      <td>7.231491</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>5.841100</td>\n",
       "      <td>7.235118</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-850 (score: 7.083230495452881).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 31:31, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.645132</td>\n",
       "      <td>8.229100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.182800</td>\n",
       "      <td>8.186500</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.609244</td>\n",
       "      <td>7.312000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.273800</td>\n",
       "      <td>7.234100</td>\n",
       "      <td>11.007400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>7.740700</td>\n",
       "      <td>7.623242</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.970600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>7.740700</td>\n",
       "      <td>7.682803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.669100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>7.740700</td>\n",
       "      <td>7.730554</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>6.977200</td>\n",
       "      <td>7.756145</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.794100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>6.977200</td>\n",
       "      <td>7.794297</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.370100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>6.977200</td>\n",
       "      <td>7.812693</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>6.928800</td>\n",
       "      <td>7.823323</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.240200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>6.928800</td>\n",
       "      <td>7.839301</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>6.928800</td>\n",
       "      <td>7.863190</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.561300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>6.922600</td>\n",
       "      <td>7.869147</td>\n",
       "      <td>1.019900</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.017900</td>\n",
       "      <td>1.017400</td>\n",
       "      <td>9.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>6.922600</td>\n",
       "      <td>7.889984</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.105400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>6.922600</td>\n",
       "      <td>7.907345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.762300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>6.895800</td>\n",
       "      <td>7.902468</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-340 (score: 7.609244346618652).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:33]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 30:30, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.815591</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.773495</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10.784300</td>\n",
       "      <td>10.694166</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>10.784300</td>\n",
       "      <td>10.591418</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>10.784300</td>\n",
       "      <td>10.477371</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>10.512700</td>\n",
       "      <td>10.360878</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>10.512700</td>\n",
       "      <td>10.248258</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>10.512700</td>\n",
       "      <td>10.144051</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>10.142200</td>\n",
       "      <td>10.051143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>10.142200</td>\n",
       "      <td>9.971609</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>10.142200</td>\n",
       "      <td>9.906408</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>9.851800</td>\n",
       "      <td>9.855848</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>9.851800</td>\n",
       "      <td>9.820067</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>9.851800</td>\n",
       "      <td>9.799031</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>9.693300</td>\n",
       "      <td>9.792102</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-170\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-340\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-510\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-680\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-850\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2040] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-2550 (score: 9.792101860046387).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: id, document, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:29]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(2) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa490cb3-3748-44b9-84cb-4a4d263c0aaa",
   "metadata": {},
   "source": [
    "### Light 2L Attentions "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a564b8-d678-4a39-b9e8-4b40d6906821",
   "metadata": {},
   "source": [
    "### 2L Attentions with Skip connection "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8be88f81-2508-45d4-97f6-16c37c11ac37",
   "metadata": {},
   "source": [
    "#### 1 step "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "39d2c2d5-54f4-4dec-9161-087f7c1672aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:47, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.795927</td>\n",
       "      <td>17.742100</td>\n",
       "      <td>1.650300</td>\n",
       "      <td>15.602800</td>\n",
       "      <td>15.602900</td>\n",
       "      <td>19.958300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.359220</td>\n",
       "      <td>16.994100</td>\n",
       "      <td>2.412100</td>\n",
       "      <td>13.840100</td>\n",
       "      <td>13.870000</td>\n",
       "      <td>19.745100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.621900</td>\n",
       "      <td>6.208702</td>\n",
       "      <td>17.668900</td>\n",
       "      <td>2.722300</td>\n",
       "      <td>13.836600</td>\n",
       "      <td>13.846900</td>\n",
       "      <td>19.551500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.621900</td>\n",
       "      <td>6.074422</td>\n",
       "      <td>17.759300</td>\n",
       "      <td>2.850600</td>\n",
       "      <td>14.315800</td>\n",
       "      <td>14.280400</td>\n",
       "      <td>19.774500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.621900</td>\n",
       "      <td>5.960795</td>\n",
       "      <td>18.342500</td>\n",
       "      <td>3.174600</td>\n",
       "      <td>14.444600</td>\n",
       "      <td>14.437800</td>\n",
       "      <td>19.951000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>5.052400</td>\n",
       "      <td>5.916854</td>\n",
       "      <td>18.716700</td>\n",
       "      <td>3.211500</td>\n",
       "      <td>14.940300</td>\n",
       "      <td>14.920300</td>\n",
       "      <td>19.943600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>5.052400</td>\n",
       "      <td>5.832363</td>\n",
       "      <td>18.119800</td>\n",
       "      <td>3.147100</td>\n",
       "      <td>14.492100</td>\n",
       "      <td>14.437200</td>\n",
       "      <td>19.985300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>5.052400</td>\n",
       "      <td>5.812722</td>\n",
       "      <td>18.068100</td>\n",
       "      <td>3.353500</td>\n",
       "      <td>14.344100</td>\n",
       "      <td>14.311300</td>\n",
       "      <td>19.990200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>4.356800</td>\n",
       "      <td>5.749690</td>\n",
       "      <td>17.444700</td>\n",
       "      <td>2.984700</td>\n",
       "      <td>13.887000</td>\n",
       "      <td>13.832300</td>\n",
       "      <td>19.977900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>4.356800</td>\n",
       "      <td>5.750962</td>\n",
       "      <td>17.878700</td>\n",
       "      <td>3.075600</td>\n",
       "      <td>14.220800</td>\n",
       "      <td>14.166600</td>\n",
       "      <td>19.973000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>4.356800</td>\n",
       "      <td>5.717148</td>\n",
       "      <td>18.130500</td>\n",
       "      <td>3.266300</td>\n",
       "      <td>14.668500</td>\n",
       "      <td>14.638900</td>\n",
       "      <td>19.919100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.917000</td>\n",
       "      <td>5.680055</td>\n",
       "      <td>17.665100</td>\n",
       "      <td>3.168800</td>\n",
       "      <td>14.415200</td>\n",
       "      <td>14.416300</td>\n",
       "      <td>19.933800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.917000</td>\n",
       "      <td>5.696688</td>\n",
       "      <td>17.370100</td>\n",
       "      <td>3.210400</td>\n",
       "      <td>14.241300</td>\n",
       "      <td>14.246800</td>\n",
       "      <td>19.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.917000</td>\n",
       "      <td>5.699435</td>\n",
       "      <td>17.627400</td>\n",
       "      <td>3.277000</td>\n",
       "      <td>14.372900</td>\n",
       "      <td>14.383400</td>\n",
       "      <td>19.953400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.659500</td>\n",
       "      <td>5.680586</td>\n",
       "      <td>17.924200</td>\n",
       "      <td>3.351200</td>\n",
       "      <td>14.615700</td>\n",
       "      <td>14.581700</td>\n",
       "      <td>19.919100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-170\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-340\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-510\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-680\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-850\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1020\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1190\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1360\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1530\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1700\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1870\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2040\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2210\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2380\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2550\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-2040 (score: 5.680055141448975).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 34:41, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.472487</td>\n",
       "      <td>17.208200</td>\n",
       "      <td>2.221400</td>\n",
       "      <td>14.107100</td>\n",
       "      <td>14.108000</td>\n",
       "      <td>18.656900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.103365</td>\n",
       "      <td>17.714000</td>\n",
       "      <td>2.813700</td>\n",
       "      <td>14.618600</td>\n",
       "      <td>14.627000</td>\n",
       "      <td>18.573500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.179100</td>\n",
       "      <td>5.899332</td>\n",
       "      <td>17.089000</td>\n",
       "      <td>2.424200</td>\n",
       "      <td>14.020100</td>\n",
       "      <td>14.015600</td>\n",
       "      <td>18.644600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.179100</td>\n",
       "      <td>5.769010</td>\n",
       "      <td>16.764000</td>\n",
       "      <td>2.777600</td>\n",
       "      <td>13.657100</td>\n",
       "      <td>13.627500</td>\n",
       "      <td>19.571100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.179100</td>\n",
       "      <td>5.676875</td>\n",
       "      <td>16.923800</td>\n",
       "      <td>2.924500</td>\n",
       "      <td>13.768800</td>\n",
       "      <td>13.749300</td>\n",
       "      <td>19.593100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.408900</td>\n",
       "      <td>5.596149</td>\n",
       "      <td>15.570600</td>\n",
       "      <td>2.764600</td>\n",
       "      <td>12.587800</td>\n",
       "      <td>12.599100</td>\n",
       "      <td>19.806400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.408900</td>\n",
       "      <td>5.531959</td>\n",
       "      <td>14.157600</td>\n",
       "      <td>2.236800</td>\n",
       "      <td>11.346700</td>\n",
       "      <td>11.374600</td>\n",
       "      <td>19.674000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.408900</td>\n",
       "      <td>5.534166</td>\n",
       "      <td>13.785300</td>\n",
       "      <td>2.339500</td>\n",
       "      <td>11.292600</td>\n",
       "      <td>11.295700</td>\n",
       "      <td>19.818600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.639400</td>\n",
       "      <td>5.486116</td>\n",
       "      <td>13.642200</td>\n",
       "      <td>2.286300</td>\n",
       "      <td>11.383800</td>\n",
       "      <td>11.378300</td>\n",
       "      <td>19.656900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.639400</td>\n",
       "      <td>5.489370</td>\n",
       "      <td>13.368300</td>\n",
       "      <td>2.222600</td>\n",
       "      <td>11.079600</td>\n",
       "      <td>11.076900</td>\n",
       "      <td>19.730400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>3.639400</td>\n",
       "      <td>5.490829</td>\n",
       "      <td>12.275800</td>\n",
       "      <td>1.983900</td>\n",
       "      <td>10.314200</td>\n",
       "      <td>10.361800</td>\n",
       "      <td>19.669100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.166300</td>\n",
       "      <td>5.523699</td>\n",
       "      <td>12.669100</td>\n",
       "      <td>1.915700</td>\n",
       "      <td>10.603000</td>\n",
       "      <td>10.590400</td>\n",
       "      <td>19.828400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.166300</td>\n",
       "      <td>5.504434</td>\n",
       "      <td>12.235900</td>\n",
       "      <td>1.710100</td>\n",
       "      <td>10.434200</td>\n",
       "      <td>10.451700</td>\n",
       "      <td>19.723000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.166300</td>\n",
       "      <td>5.511914</td>\n",
       "      <td>12.311900</td>\n",
       "      <td>1.905600</td>\n",
       "      <td>10.462100</td>\n",
       "      <td>10.479100</td>\n",
       "      <td>19.612700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.877100</td>\n",
       "      <td>5.509643</td>\n",
       "      <td>12.260400</td>\n",
       "      <td>1.755600</td>\n",
       "      <td>10.316100</td>\n",
       "      <td>10.321000</td>\n",
       "      <td>19.708300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-170\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-340\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-510\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-680\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-850\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1020\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1190\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1360\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1530\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1700\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1870\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2040\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2210\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2380\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2550\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-1530 (score: 5.486116409301758).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:15, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.107931</td>\n",
       "      <td>20.089100</td>\n",
       "      <td>3.700700</td>\n",
       "      <td>15.947800</td>\n",
       "      <td>15.919700</td>\n",
       "      <td>19.759800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.903008</td>\n",
       "      <td>19.119700</td>\n",
       "      <td>3.509500</td>\n",
       "      <td>15.445500</td>\n",
       "      <td>15.419600</td>\n",
       "      <td>19.544100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.558700</td>\n",
       "      <td>5.850713</td>\n",
       "      <td>19.548800</td>\n",
       "      <td>3.851100</td>\n",
       "      <td>15.749000</td>\n",
       "      <td>15.770200</td>\n",
       "      <td>19.661800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.558700</td>\n",
       "      <td>5.797190</td>\n",
       "      <td>19.609900</td>\n",
       "      <td>3.735200</td>\n",
       "      <td>15.858800</td>\n",
       "      <td>15.836600</td>\n",
       "      <td>19.752500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.558700</td>\n",
       "      <td>5.817583</td>\n",
       "      <td>20.486000</td>\n",
       "      <td>4.064900</td>\n",
       "      <td>16.365000</td>\n",
       "      <td>16.382800</td>\n",
       "      <td>19.764700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.155000</td>\n",
       "      <td>5.825292</td>\n",
       "      <td>19.689600</td>\n",
       "      <td>4.125700</td>\n",
       "      <td>15.822100</td>\n",
       "      <td>15.829600</td>\n",
       "      <td>19.875000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.155000</td>\n",
       "      <td>5.817491</td>\n",
       "      <td>20.039400</td>\n",
       "      <td>3.916300</td>\n",
       "      <td>15.841100</td>\n",
       "      <td>15.806700</td>\n",
       "      <td>19.852900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.155000</td>\n",
       "      <td>5.822026</td>\n",
       "      <td>19.988800</td>\n",
       "      <td>4.170800</td>\n",
       "      <td>16.362900</td>\n",
       "      <td>16.334100</td>\n",
       "      <td>19.725500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.602900</td>\n",
       "      <td>5.815959</td>\n",
       "      <td>20.581300</td>\n",
       "      <td>4.386700</td>\n",
       "      <td>16.580300</td>\n",
       "      <td>16.564200</td>\n",
       "      <td>19.767200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.602900</td>\n",
       "      <td>5.813905</td>\n",
       "      <td>19.817500</td>\n",
       "      <td>4.184200</td>\n",
       "      <td>15.872300</td>\n",
       "      <td>15.870000</td>\n",
       "      <td>19.698500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.602900</td>\n",
       "      <td>5.822848</td>\n",
       "      <td>20.340500</td>\n",
       "      <td>4.344000</td>\n",
       "      <td>16.117300</td>\n",
       "      <td>16.094400</td>\n",
       "      <td>19.732800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.401100</td>\n",
       "      <td>5.852437</td>\n",
       "      <td>20.573800</td>\n",
       "      <td>4.503600</td>\n",
       "      <td>16.524000</td>\n",
       "      <td>16.475600</td>\n",
       "      <td>19.723000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.401100</td>\n",
       "      <td>5.838686</td>\n",
       "      <td>19.967700</td>\n",
       "      <td>4.263100</td>\n",
       "      <td>15.997500</td>\n",
       "      <td>16.015100</td>\n",
       "      <td>19.784300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.401100</td>\n",
       "      <td>5.825996</td>\n",
       "      <td>20.483500</td>\n",
       "      <td>4.305900</td>\n",
       "      <td>16.343200</td>\n",
       "      <td>16.319200</td>\n",
       "      <td>19.789200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.181800</td>\n",
       "      <td>5.834902</td>\n",
       "      <td>20.392700</td>\n",
       "      <td>4.383000</td>\n",
       "      <td>16.252700</td>\n",
       "      <td>16.249700</td>\n",
       "      <td>19.752500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-680 (score: 5.797189712524414).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:37]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:46, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.149194</td>\n",
       "      <td>19.444700</td>\n",
       "      <td>3.493800</td>\n",
       "      <td>15.588400</td>\n",
       "      <td>15.560200</td>\n",
       "      <td>19.715700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.117716</td>\n",
       "      <td>19.545400</td>\n",
       "      <td>3.870100</td>\n",
       "      <td>15.444300</td>\n",
       "      <td>15.424200</td>\n",
       "      <td>19.705900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.652800</td>\n",
       "      <td>6.098605</td>\n",
       "      <td>19.509500</td>\n",
       "      <td>3.875100</td>\n",
       "      <td>15.448700</td>\n",
       "      <td>15.431400</td>\n",
       "      <td>19.441200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.652800</td>\n",
       "      <td>6.132448</td>\n",
       "      <td>19.302800</td>\n",
       "      <td>3.750200</td>\n",
       "      <td>15.372800</td>\n",
       "      <td>15.359800</td>\n",
       "      <td>19.644600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.652800</td>\n",
       "      <td>6.105505</td>\n",
       "      <td>19.711800</td>\n",
       "      <td>4.125000</td>\n",
       "      <td>15.936900</td>\n",
       "      <td>15.903400</td>\n",
       "      <td>19.629900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.754400</td>\n",
       "      <td>6.090956</td>\n",
       "      <td>18.815500</td>\n",
       "      <td>3.940000</td>\n",
       "      <td>15.294400</td>\n",
       "      <td>15.301600</td>\n",
       "      <td>19.531900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.754400</td>\n",
       "      <td>6.186744</td>\n",
       "      <td>18.931100</td>\n",
       "      <td>3.855800</td>\n",
       "      <td>15.472500</td>\n",
       "      <td>15.461500</td>\n",
       "      <td>19.642200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.754400</td>\n",
       "      <td>6.177992</td>\n",
       "      <td>18.425100</td>\n",
       "      <td>3.877900</td>\n",
       "      <td>15.055400</td>\n",
       "      <td>15.065500</td>\n",
       "      <td>19.531900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.451400</td>\n",
       "      <td>6.188844</td>\n",
       "      <td>18.391400</td>\n",
       "      <td>3.922100</td>\n",
       "      <td>15.225200</td>\n",
       "      <td>15.253400</td>\n",
       "      <td>19.549000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.451400</td>\n",
       "      <td>6.256632</td>\n",
       "      <td>18.452500</td>\n",
       "      <td>3.812000</td>\n",
       "      <td>15.008100</td>\n",
       "      <td>14.993500</td>\n",
       "      <td>19.585800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.451400</td>\n",
       "      <td>6.301659</td>\n",
       "      <td>17.864100</td>\n",
       "      <td>3.514000</td>\n",
       "      <td>14.759900</td>\n",
       "      <td>14.758900</td>\n",
       "      <td>19.451000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.318800</td>\n",
       "      <td>6.289698</td>\n",
       "      <td>18.265800</td>\n",
       "      <td>3.720300</td>\n",
       "      <td>14.972800</td>\n",
       "      <td>14.985400</td>\n",
       "      <td>19.588200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.318800</td>\n",
       "      <td>6.310218</td>\n",
       "      <td>17.487800</td>\n",
       "      <td>3.628700</td>\n",
       "      <td>14.353300</td>\n",
       "      <td>14.353600</td>\n",
       "      <td>19.610300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.318800</td>\n",
       "      <td>6.294967</td>\n",
       "      <td>17.596100</td>\n",
       "      <td>3.788600</td>\n",
       "      <td>14.912600</td>\n",
       "      <td>14.952800</td>\n",
       "      <td>19.610300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.136200</td>\n",
       "      <td>6.317812</td>\n",
       "      <td>17.803400</td>\n",
       "      <td>3.857800</td>\n",
       "      <td>14.858700</td>\n",
       "      <td>14.911200</td>\n",
       "      <td>19.558800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-1020 (score: 6.09095573425293).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:03, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.326079</td>\n",
       "      <td>20.660200</td>\n",
       "      <td>3.903400</td>\n",
       "      <td>16.495900</td>\n",
       "      <td>16.461100</td>\n",
       "      <td>19.772100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.896080</td>\n",
       "      <td>20.206900</td>\n",
       "      <td>3.972000</td>\n",
       "      <td>16.225700</td>\n",
       "      <td>16.167300</td>\n",
       "      <td>19.759800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.786900</td>\n",
       "      <td>5.835314</td>\n",
       "      <td>19.847300</td>\n",
       "      <td>4.154400</td>\n",
       "      <td>15.994500</td>\n",
       "      <td>15.989800</td>\n",
       "      <td>19.852900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.786900</td>\n",
       "      <td>5.815123</td>\n",
       "      <td>19.948600</td>\n",
       "      <td>4.165900</td>\n",
       "      <td>16.118900</td>\n",
       "      <td>16.106300</td>\n",
       "      <td>19.818600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.786900</td>\n",
       "      <td>5.815073</td>\n",
       "      <td>20.265300</td>\n",
       "      <td>4.383600</td>\n",
       "      <td>16.577700</td>\n",
       "      <td>16.563100</td>\n",
       "      <td>19.784300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.653900</td>\n",
       "      <td>5.816186</td>\n",
       "      <td>20.279000</td>\n",
       "      <td>4.385500</td>\n",
       "      <td>16.388500</td>\n",
       "      <td>16.338900</td>\n",
       "      <td>19.725500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.653900</td>\n",
       "      <td>5.803371</td>\n",
       "      <td>19.724300</td>\n",
       "      <td>4.240600</td>\n",
       "      <td>16.023400</td>\n",
       "      <td>16.010800</td>\n",
       "      <td>19.799000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.653900</td>\n",
       "      <td>5.830432</td>\n",
       "      <td>20.430800</td>\n",
       "      <td>4.511600</td>\n",
       "      <td>16.624400</td>\n",
       "      <td>16.624000</td>\n",
       "      <td>19.715700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.191400</td>\n",
       "      <td>5.802033</td>\n",
       "      <td>19.926200</td>\n",
       "      <td>4.283100</td>\n",
       "      <td>16.050900</td>\n",
       "      <td>16.048800</td>\n",
       "      <td>19.803900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.191400</td>\n",
       "      <td>5.822743</td>\n",
       "      <td>19.918400</td>\n",
       "      <td>4.475500</td>\n",
       "      <td>16.384400</td>\n",
       "      <td>16.384200</td>\n",
       "      <td>19.732800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.191400</td>\n",
       "      <td>5.813590</td>\n",
       "      <td>20.434900</td>\n",
       "      <td>4.535800</td>\n",
       "      <td>16.626400</td>\n",
       "      <td>16.641200</td>\n",
       "      <td>19.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.925500</td>\n",
       "      <td>5.818446</td>\n",
       "      <td>20.594700</td>\n",
       "      <td>4.614900</td>\n",
       "      <td>16.563600</td>\n",
       "      <td>16.573800</td>\n",
       "      <td>19.774500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.925500</td>\n",
       "      <td>5.823716</td>\n",
       "      <td>20.470000</td>\n",
       "      <td>4.330400</td>\n",
       "      <td>16.587800</td>\n",
       "      <td>16.606500</td>\n",
       "      <td>19.720600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.925500</td>\n",
       "      <td>5.826282</td>\n",
       "      <td>20.693500</td>\n",
       "      <td>4.597000</td>\n",
       "      <td>16.672300</td>\n",
       "      <td>16.653500</td>\n",
       "      <td>19.735300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.767600</td>\n",
       "      <td>5.823565</td>\n",
       "      <td>20.397000</td>\n",
       "      <td>4.566700</td>\n",
       "      <td>16.686900</td>\n",
       "      <td>16.691700</td>\n",
       "      <td>19.705900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-170\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-340\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-510\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-680\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-850\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-1530 (score: 5.802032947540283).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:37]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(2) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)+x\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)+x\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e02fa3b-a518-47f4-b911-aef84856d5c2",
   "metadata": {},
   "source": [
    "#### 1step + 2step "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0da11904-57dc-44ec-a2b1-a83f0a961c94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:52, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.870742</td>\n",
       "      <td>15.627200</td>\n",
       "      <td>1.718500</td>\n",
       "      <td>13.316500</td>\n",
       "      <td>13.314000</td>\n",
       "      <td>17.654400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.441172</td>\n",
       "      <td>17.546400</td>\n",
       "      <td>2.479000</td>\n",
       "      <td>14.530700</td>\n",
       "      <td>14.564100</td>\n",
       "      <td>18.720600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.709600</td>\n",
       "      <td>6.300485</td>\n",
       "      <td>15.876400</td>\n",
       "      <td>1.980500</td>\n",
       "      <td>13.033000</td>\n",
       "      <td>13.089300</td>\n",
       "      <td>18.598000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.709600</td>\n",
       "      <td>6.174621</td>\n",
       "      <td>17.270700</td>\n",
       "      <td>2.661400</td>\n",
       "      <td>14.057100</td>\n",
       "      <td>14.083300</td>\n",
       "      <td>19.039200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.709600</td>\n",
       "      <td>6.071609</td>\n",
       "      <td>18.215600</td>\n",
       "      <td>2.841800</td>\n",
       "      <td>14.581400</td>\n",
       "      <td>14.601000</td>\n",
       "      <td>19.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>5.084700</td>\n",
       "      <td>6.058468</td>\n",
       "      <td>18.116700</td>\n",
       "      <td>2.991200</td>\n",
       "      <td>14.582100</td>\n",
       "      <td>14.582200</td>\n",
       "      <td>19.840700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>5.084700</td>\n",
       "      <td>6.067877</td>\n",
       "      <td>18.257300</td>\n",
       "      <td>3.060900</td>\n",
       "      <td>14.468500</td>\n",
       "      <td>14.488700</td>\n",
       "      <td>19.919100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>5.084700</td>\n",
       "      <td>6.081351</td>\n",
       "      <td>18.833500</td>\n",
       "      <td>3.639100</td>\n",
       "      <td>15.016900</td>\n",
       "      <td>15.048800</td>\n",
       "      <td>19.948500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>4.231000</td>\n",
       "      <td>6.055990</td>\n",
       "      <td>18.079400</td>\n",
       "      <td>3.336700</td>\n",
       "      <td>14.551600</td>\n",
       "      <td>14.559800</td>\n",
       "      <td>19.936300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>4.231000</td>\n",
       "      <td>6.103199</td>\n",
       "      <td>17.973800</td>\n",
       "      <td>3.341500</td>\n",
       "      <td>14.290600</td>\n",
       "      <td>14.309200</td>\n",
       "      <td>19.968100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>4.231000</td>\n",
       "      <td>6.090595</td>\n",
       "      <td>18.198300</td>\n",
       "      <td>3.262200</td>\n",
       "      <td>14.676700</td>\n",
       "      <td>14.725400</td>\n",
       "      <td>19.921600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.626900</td>\n",
       "      <td>6.098236</td>\n",
       "      <td>17.952100</td>\n",
       "      <td>3.012500</td>\n",
       "      <td>14.515900</td>\n",
       "      <td>14.560600</td>\n",
       "      <td>19.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.626900</td>\n",
       "      <td>6.114873</td>\n",
       "      <td>17.963900</td>\n",
       "      <td>3.120000</td>\n",
       "      <td>14.474300</td>\n",
       "      <td>14.517800</td>\n",
       "      <td>19.968100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.626900</td>\n",
       "      <td>6.109668</td>\n",
       "      <td>17.744100</td>\n",
       "      <td>3.055100</td>\n",
       "      <td>14.300700</td>\n",
       "      <td>14.307800</td>\n",
       "      <td>19.943600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.260800</td>\n",
       "      <td>6.111631</td>\n",
       "      <td>17.741400</td>\n",
       "      <td>3.012600</td>\n",
       "      <td>14.454500</td>\n",
       "      <td>14.476400</td>\n",
       "      <td>19.943600</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-170\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-340\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-510\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-680\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-850\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1020\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1190\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1360\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1530\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1700\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1870\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2040\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2210\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2380\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2550\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-1530 (score: 6.055989742279053).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 34:46, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.570634</td>\n",
       "      <td>14.738800</td>\n",
       "      <td>1.621500</td>\n",
       "      <td>12.755200</td>\n",
       "      <td>12.775300</td>\n",
       "      <td>19.656900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.251204</td>\n",
       "      <td>15.989500</td>\n",
       "      <td>2.210700</td>\n",
       "      <td>13.486500</td>\n",
       "      <td>13.548200</td>\n",
       "      <td>19.421600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.275800</td>\n",
       "      <td>6.082178</td>\n",
       "      <td>17.790900</td>\n",
       "      <td>2.873900</td>\n",
       "      <td>14.394700</td>\n",
       "      <td>14.398700</td>\n",
       "      <td>19.826000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.275800</td>\n",
       "      <td>5.967330</td>\n",
       "      <td>18.029800</td>\n",
       "      <td>2.739100</td>\n",
       "      <td>14.303800</td>\n",
       "      <td>14.306400</td>\n",
       "      <td>19.990200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.275800</td>\n",
       "      <td>5.906945</td>\n",
       "      <td>18.736500</td>\n",
       "      <td>3.089900</td>\n",
       "      <td>15.009000</td>\n",
       "      <td>15.005800</td>\n",
       "      <td>19.977900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.579500</td>\n",
       "      <td>5.834208</td>\n",
       "      <td>17.235700</td>\n",
       "      <td>2.701300</td>\n",
       "      <td>13.851700</td>\n",
       "      <td>13.860800</td>\n",
       "      <td>19.995100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.579500</td>\n",
       "      <td>5.818001</td>\n",
       "      <td>16.711100</td>\n",
       "      <td>2.689500</td>\n",
       "      <td>13.226300</td>\n",
       "      <td>13.255200</td>\n",
       "      <td>19.992600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.579500</td>\n",
       "      <td>5.796676</td>\n",
       "      <td>16.192800</td>\n",
       "      <td>2.391700</td>\n",
       "      <td>12.718600</td>\n",
       "      <td>12.767000</td>\n",
       "      <td>19.982800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.709500</td>\n",
       "      <td>5.807513</td>\n",
       "      <td>15.353600</td>\n",
       "      <td>2.497600</td>\n",
       "      <td>12.137800</td>\n",
       "      <td>12.154000</td>\n",
       "      <td>19.965700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.709500</td>\n",
       "      <td>5.806167</td>\n",
       "      <td>15.169100</td>\n",
       "      <td>2.737700</td>\n",
       "      <td>12.318000</td>\n",
       "      <td>12.299600</td>\n",
       "      <td>19.982800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>3.709500</td>\n",
       "      <td>5.812335</td>\n",
       "      <td>14.346700</td>\n",
       "      <td>2.129900</td>\n",
       "      <td>11.736800</td>\n",
       "      <td>11.725800</td>\n",
       "      <td>19.973000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.121600</td>\n",
       "      <td>5.832439</td>\n",
       "      <td>13.965600</td>\n",
       "      <td>2.153400</td>\n",
       "      <td>11.361700</td>\n",
       "      <td>11.368300</td>\n",
       "      <td>19.951000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.121600</td>\n",
       "      <td>5.809342</td>\n",
       "      <td>13.976500</td>\n",
       "      <td>2.256700</td>\n",
       "      <td>11.420500</td>\n",
       "      <td>11.426600</td>\n",
       "      <td>19.958300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.121600</td>\n",
       "      <td>5.807369</td>\n",
       "      <td>14.116800</td>\n",
       "      <td>2.133100</td>\n",
       "      <td>11.384800</td>\n",
       "      <td>11.402600</td>\n",
       "      <td>19.906900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.760300</td>\n",
       "      <td>5.808494</td>\n",
       "      <td>14.149000</td>\n",
       "      <td>2.229300</td>\n",
       "      <td>11.432500</td>\n",
       "      <td>11.403900</td>\n",
       "      <td>19.955900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-170\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-340\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-510\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-680\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-850\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1020\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1190\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1360\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1530\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1700\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1870\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2040\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2210\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2380\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2550\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-1360 (score: 5.796675682067871).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:41]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:16, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.423979</td>\n",
       "      <td>18.494500</td>\n",
       "      <td>3.066500</td>\n",
       "      <td>14.556300</td>\n",
       "      <td>14.604900</td>\n",
       "      <td>19.671600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.236391</td>\n",
       "      <td>18.476400</td>\n",
       "      <td>3.206200</td>\n",
       "      <td>15.082500</td>\n",
       "      <td>15.112900</td>\n",
       "      <td>19.715700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.516200</td>\n",
       "      <td>6.255590</td>\n",
       "      <td>18.193100</td>\n",
       "      <td>3.463400</td>\n",
       "      <td>14.911200</td>\n",
       "      <td>14.946700</td>\n",
       "      <td>19.534300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.516200</td>\n",
       "      <td>6.245983</td>\n",
       "      <td>18.507700</td>\n",
       "      <td>3.569700</td>\n",
       "      <td>14.920900</td>\n",
       "      <td>14.943800</td>\n",
       "      <td>19.683800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.516200</td>\n",
       "      <td>6.303349</td>\n",
       "      <td>19.053300</td>\n",
       "      <td>3.692200</td>\n",
       "      <td>15.292700</td>\n",
       "      <td>15.298000</td>\n",
       "      <td>19.745100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.944100</td>\n",
       "      <td>6.316472</td>\n",
       "      <td>19.198000</td>\n",
       "      <td>3.589900</td>\n",
       "      <td>15.363600</td>\n",
       "      <td>15.373500</td>\n",
       "      <td>19.740200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.944100</td>\n",
       "      <td>6.324241</td>\n",
       "      <td>19.234900</td>\n",
       "      <td>3.751400</td>\n",
       "      <td>15.284300</td>\n",
       "      <td>15.356500</td>\n",
       "      <td>19.735300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.944100</td>\n",
       "      <td>6.342865</td>\n",
       "      <td>19.009600</td>\n",
       "      <td>3.592000</td>\n",
       "      <td>15.087400</td>\n",
       "      <td>15.144900</td>\n",
       "      <td>19.737700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.275400</td>\n",
       "      <td>6.342039</td>\n",
       "      <td>19.150900</td>\n",
       "      <td>3.767500</td>\n",
       "      <td>15.549200</td>\n",
       "      <td>15.596300</td>\n",
       "      <td>19.664200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.275400</td>\n",
       "      <td>6.330871</td>\n",
       "      <td>18.965700</td>\n",
       "      <td>3.573600</td>\n",
       "      <td>15.299300</td>\n",
       "      <td>15.321600</td>\n",
       "      <td>19.661800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.275400</td>\n",
       "      <td>6.351172</td>\n",
       "      <td>19.670800</td>\n",
       "      <td>3.850500</td>\n",
       "      <td>15.628100</td>\n",
       "      <td>15.613400</td>\n",
       "      <td>19.644600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.991300</td>\n",
       "      <td>6.352794</td>\n",
       "      <td>20.001800</td>\n",
       "      <td>3.934000</td>\n",
       "      <td>16.027600</td>\n",
       "      <td>16.015200</td>\n",
       "      <td>19.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.991300</td>\n",
       "      <td>6.346968</td>\n",
       "      <td>19.515600</td>\n",
       "      <td>3.859300</td>\n",
       "      <td>15.591200</td>\n",
       "      <td>15.603000</td>\n",
       "      <td>19.767200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.991300</td>\n",
       "      <td>6.347233</td>\n",
       "      <td>19.503000</td>\n",
       "      <td>3.792700</td>\n",
       "      <td>15.547100</td>\n",
       "      <td>15.540500</td>\n",
       "      <td>19.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.709900</td>\n",
       "      <td>6.350191</td>\n",
       "      <td>19.282900</td>\n",
       "      <td>3.631200</td>\n",
       "      <td>15.586700</td>\n",
       "      <td>15.574300</td>\n",
       "      <td>19.727900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-340 (score: 6.236390590667725).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:50, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.724247</td>\n",
       "      <td>19.260200</td>\n",
       "      <td>3.545700</td>\n",
       "      <td>15.503300</td>\n",
       "      <td>15.523600</td>\n",
       "      <td>19.889700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.531374</td>\n",
       "      <td>17.595300</td>\n",
       "      <td>3.339400</td>\n",
       "      <td>14.224400</td>\n",
       "      <td>14.233000</td>\n",
       "      <td>19.634800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.787000</td>\n",
       "      <td>6.464454</td>\n",
       "      <td>17.667000</td>\n",
       "      <td>3.102200</td>\n",
       "      <td>14.413600</td>\n",
       "      <td>14.403100</td>\n",
       "      <td>19.509800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.787000</td>\n",
       "      <td>6.430081</td>\n",
       "      <td>18.399600</td>\n",
       "      <td>3.370800</td>\n",
       "      <td>14.825600</td>\n",
       "      <td>14.821400</td>\n",
       "      <td>19.558800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.787000</td>\n",
       "      <td>6.477285</td>\n",
       "      <td>17.943000</td>\n",
       "      <td>3.196600</td>\n",
       "      <td>14.473500</td>\n",
       "      <td>14.457900</td>\n",
       "      <td>19.691200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.681700</td>\n",
       "      <td>6.481236</td>\n",
       "      <td>18.123300</td>\n",
       "      <td>3.217500</td>\n",
       "      <td>14.683100</td>\n",
       "      <td>14.692900</td>\n",
       "      <td>19.590700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.681700</td>\n",
       "      <td>6.518813</td>\n",
       "      <td>17.823500</td>\n",
       "      <td>2.965200</td>\n",
       "      <td>14.353800</td>\n",
       "      <td>14.370800</td>\n",
       "      <td>19.475500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.681700</td>\n",
       "      <td>6.547624</td>\n",
       "      <td>17.762700</td>\n",
       "      <td>3.200200</td>\n",
       "      <td>14.359700</td>\n",
       "      <td>14.374900</td>\n",
       "      <td>19.480400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.075100</td>\n",
       "      <td>6.530712</td>\n",
       "      <td>18.297400</td>\n",
       "      <td>3.244600</td>\n",
       "      <td>14.825100</td>\n",
       "      <td>14.858000</td>\n",
       "      <td>19.522100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.075100</td>\n",
       "      <td>6.541767</td>\n",
       "      <td>17.995500</td>\n",
       "      <td>3.499100</td>\n",
       "      <td>14.620200</td>\n",
       "      <td>14.648700</td>\n",
       "      <td>19.460800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.075100</td>\n",
       "      <td>6.584012</td>\n",
       "      <td>17.519900</td>\n",
       "      <td>3.268800</td>\n",
       "      <td>14.192600</td>\n",
       "      <td>14.211500</td>\n",
       "      <td>19.414200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.826200</td>\n",
       "      <td>6.601549</td>\n",
       "      <td>17.701500</td>\n",
       "      <td>3.451200</td>\n",
       "      <td>14.459900</td>\n",
       "      <td>14.460400</td>\n",
       "      <td>19.519600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.826200</td>\n",
       "      <td>6.571034</td>\n",
       "      <td>17.206300</td>\n",
       "      <td>3.112300</td>\n",
       "      <td>14.036300</td>\n",
       "      <td>14.071600</td>\n",
       "      <td>19.487700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.826200</td>\n",
       "      <td>6.579152</td>\n",
       "      <td>16.944600</td>\n",
       "      <td>3.276700</td>\n",
       "      <td>13.951500</td>\n",
       "      <td>13.969000</td>\n",
       "      <td>19.453400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.569800</td>\n",
       "      <td>6.590714</td>\n",
       "      <td>17.167200</td>\n",
       "      <td>3.370600</td>\n",
       "      <td>14.072800</td>\n",
       "      <td>14.067700</td>\n",
       "      <td>19.338200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-680 (score: 6.430081367492676).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:07, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.684236</td>\n",
       "      <td>18.602200</td>\n",
       "      <td>3.758000</td>\n",
       "      <td>15.106800</td>\n",
       "      <td>15.106500</td>\n",
       "      <td>19.796600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.417322</td>\n",
       "      <td>18.711700</td>\n",
       "      <td>3.479600</td>\n",
       "      <td>15.007400</td>\n",
       "      <td>15.045700</td>\n",
       "      <td>19.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.616900</td>\n",
       "      <td>6.364204</td>\n",
       "      <td>18.759200</td>\n",
       "      <td>3.672100</td>\n",
       "      <td>15.122200</td>\n",
       "      <td>15.150900</td>\n",
       "      <td>19.772100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.616900</td>\n",
       "      <td>6.331797</td>\n",
       "      <td>18.490200</td>\n",
       "      <td>3.274400</td>\n",
       "      <td>15.065900</td>\n",
       "      <td>15.108400</td>\n",
       "      <td>19.845600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.616900</td>\n",
       "      <td>6.356014</td>\n",
       "      <td>19.040700</td>\n",
       "      <td>3.594400</td>\n",
       "      <td>15.481700</td>\n",
       "      <td>15.514500</td>\n",
       "      <td>19.828400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.315600</td>\n",
       "      <td>6.365025</td>\n",
       "      <td>19.273800</td>\n",
       "      <td>3.807800</td>\n",
       "      <td>15.284600</td>\n",
       "      <td>15.293700</td>\n",
       "      <td>19.696100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.315600</td>\n",
       "      <td>6.359451</td>\n",
       "      <td>18.431900</td>\n",
       "      <td>3.466500</td>\n",
       "      <td>14.671300</td>\n",
       "      <td>14.674200</td>\n",
       "      <td>19.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.315600</td>\n",
       "      <td>6.385502</td>\n",
       "      <td>19.262000</td>\n",
       "      <td>3.529700</td>\n",
       "      <td>15.628300</td>\n",
       "      <td>15.626000</td>\n",
       "      <td>19.754900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>1.677900</td>\n",
       "      <td>6.371026</td>\n",
       "      <td>19.401000</td>\n",
       "      <td>3.712500</td>\n",
       "      <td>15.432900</td>\n",
       "      <td>15.427900</td>\n",
       "      <td>19.808800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>1.677900</td>\n",
       "      <td>6.390761</td>\n",
       "      <td>20.119300</td>\n",
       "      <td>4.014000</td>\n",
       "      <td>16.176500</td>\n",
       "      <td>16.188400</td>\n",
       "      <td>19.801500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>1.677900</td>\n",
       "      <td>6.387181</td>\n",
       "      <td>19.997900</td>\n",
       "      <td>4.050900</td>\n",
       "      <td>15.888000</td>\n",
       "      <td>15.906100</td>\n",
       "      <td>19.791700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.325600</td>\n",
       "      <td>6.403159</td>\n",
       "      <td>19.683000</td>\n",
       "      <td>3.870000</td>\n",
       "      <td>15.685200</td>\n",
       "      <td>15.703300</td>\n",
       "      <td>19.786800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.325600</td>\n",
       "      <td>6.391050</td>\n",
       "      <td>19.710000</td>\n",
       "      <td>3.950300</td>\n",
       "      <td>15.760800</td>\n",
       "      <td>15.800300</td>\n",
       "      <td>19.852900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.325600</td>\n",
       "      <td>6.403557</td>\n",
       "      <td>19.931600</td>\n",
       "      <td>4.155800</td>\n",
       "      <td>16.039000</td>\n",
       "      <td>16.082400</td>\n",
       "      <td>19.769600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.110200</td>\n",
       "      <td>6.402191</td>\n",
       "      <td>20.127000</td>\n",
       "      <td>3.991900</td>\n",
       "      <td>16.038200</td>\n",
       "      <td>16.068400</td>\n",
       "      <td>19.823500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-170\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-340\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-510\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-680\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-850\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-680 (score: 6.331797122955322).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, id, summary.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(2) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            first_x = copy.copy(x)\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)+x\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)+x\n",
    "            x = first_x + x\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "861a902e-078d-4134-8d39-9a91213b58b2",
   "metadata": {},
   "source": [
    "### 4L Attentions /w skipConnection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9a8223a0-b34c-4536-9434-26c0b0493d37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 34:55, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.750159</td>\n",
       "      <td>3.999900</td>\n",
       "      <td>0.498400</td>\n",
       "      <td>3.459500</td>\n",
       "      <td>3.419000</td>\n",
       "      <td>7.102900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.323394</td>\n",
       "      <td>17.025500</td>\n",
       "      <td>2.506100</td>\n",
       "      <td>14.514900</td>\n",
       "      <td>14.517500</td>\n",
       "      <td>19.455900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.522900</td>\n",
       "      <td>6.122281</td>\n",
       "      <td>18.112600</td>\n",
       "      <td>2.869700</td>\n",
       "      <td>14.544300</td>\n",
       "      <td>14.603200</td>\n",
       "      <td>19.808800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.522900</td>\n",
       "      <td>5.994848</td>\n",
       "      <td>17.323300</td>\n",
       "      <td>2.912600</td>\n",
       "      <td>14.146700</td>\n",
       "      <td>14.169000</td>\n",
       "      <td>19.982800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.522900</td>\n",
       "      <td>5.864310</td>\n",
       "      <td>17.452700</td>\n",
       "      <td>2.721100</td>\n",
       "      <td>14.151100</td>\n",
       "      <td>14.157600</td>\n",
       "      <td>19.977900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.849900</td>\n",
       "      <td>5.846464</td>\n",
       "      <td>16.456500</td>\n",
       "      <td>2.659300</td>\n",
       "      <td>13.353000</td>\n",
       "      <td>13.344800</td>\n",
       "      <td>19.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.849900</td>\n",
       "      <td>5.733954</td>\n",
       "      <td>16.633700</td>\n",
       "      <td>3.033600</td>\n",
       "      <td>13.551600</td>\n",
       "      <td>13.544400</td>\n",
       "      <td>19.914200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.849900</td>\n",
       "      <td>5.723573</td>\n",
       "      <td>16.561600</td>\n",
       "      <td>2.918000</td>\n",
       "      <td>13.553500</td>\n",
       "      <td>13.583200</td>\n",
       "      <td>19.897100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>4.037400</td>\n",
       "      <td>5.680391</td>\n",
       "      <td>16.397900</td>\n",
       "      <td>3.148300</td>\n",
       "      <td>13.692500</td>\n",
       "      <td>13.703000</td>\n",
       "      <td>19.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>4.037400</td>\n",
       "      <td>5.651556</td>\n",
       "      <td>16.078300</td>\n",
       "      <td>3.144500</td>\n",
       "      <td>13.290500</td>\n",
       "      <td>13.290100</td>\n",
       "      <td>19.816200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>4.037400</td>\n",
       "      <td>5.618900</td>\n",
       "      <td>15.883300</td>\n",
       "      <td>2.956000</td>\n",
       "      <td>13.051800</td>\n",
       "      <td>13.035900</td>\n",
       "      <td>19.919100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>3.538400</td>\n",
       "      <td>5.582287</td>\n",
       "      <td>15.641700</td>\n",
       "      <td>3.208800</td>\n",
       "      <td>13.033200</td>\n",
       "      <td>13.005000</td>\n",
       "      <td>19.899500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>3.538400</td>\n",
       "      <td>5.591887</td>\n",
       "      <td>16.088700</td>\n",
       "      <td>3.222400</td>\n",
       "      <td>13.184700</td>\n",
       "      <td>13.134800</td>\n",
       "      <td>19.860300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>3.538400</td>\n",
       "      <td>5.615936</td>\n",
       "      <td>16.041900</td>\n",
       "      <td>3.180200</td>\n",
       "      <td>13.259900</td>\n",
       "      <td>13.228100</td>\n",
       "      <td>19.911800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>3.236400</td>\n",
       "      <td>5.599515</td>\n",
       "      <td>16.091200</td>\n",
       "      <td>3.251900</td>\n",
       "      <td>13.270800</td>\n",
       "      <td>13.238900</td>\n",
       "      <td>19.936300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-170\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-340\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-510\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-680\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-850\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1020\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1190\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1360\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1530\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1700\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-1870\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2040\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2210\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2380\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-2550\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-2040 (score: 5.582286834716797).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 36:39, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.444579</td>\n",
       "      <td>16.110800</td>\n",
       "      <td>2.223800</td>\n",
       "      <td>13.579400</td>\n",
       "      <td>13.586800</td>\n",
       "      <td>18.588200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.022194</td>\n",
       "      <td>17.211400</td>\n",
       "      <td>2.562100</td>\n",
       "      <td>13.971800</td>\n",
       "      <td>13.984700</td>\n",
       "      <td>19.242600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6.072500</td>\n",
       "      <td>5.795569</td>\n",
       "      <td>17.240000</td>\n",
       "      <td>2.605600</td>\n",
       "      <td>13.795600</td>\n",
       "      <td>13.806100</td>\n",
       "      <td>19.355400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>6.072500</td>\n",
       "      <td>5.646204</td>\n",
       "      <td>15.978900</td>\n",
       "      <td>2.694600</td>\n",
       "      <td>12.635200</td>\n",
       "      <td>12.665000</td>\n",
       "      <td>19.541700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>6.072500</td>\n",
       "      <td>5.575400</td>\n",
       "      <td>15.614200</td>\n",
       "      <td>2.626600</td>\n",
       "      <td>12.598200</td>\n",
       "      <td>12.609100</td>\n",
       "      <td>19.833300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.216500</td>\n",
       "      <td>5.479822</td>\n",
       "      <td>13.892300</td>\n",
       "      <td>2.406100</td>\n",
       "      <td>11.410500</td>\n",
       "      <td>11.436600</td>\n",
       "      <td>19.781900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.216500</td>\n",
       "      <td>5.451617</td>\n",
       "      <td>13.460700</td>\n",
       "      <td>2.127200</td>\n",
       "      <td>10.934700</td>\n",
       "      <td>10.940800</td>\n",
       "      <td>19.781900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>4.216500</td>\n",
       "      <td>5.411547</td>\n",
       "      <td>12.430800</td>\n",
       "      <td>1.917600</td>\n",
       "      <td>9.972300</td>\n",
       "      <td>10.014400</td>\n",
       "      <td>19.781900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.366800</td>\n",
       "      <td>5.388334</td>\n",
       "      <td>12.579100</td>\n",
       "      <td>1.830100</td>\n",
       "      <td>10.142400</td>\n",
       "      <td>10.155000</td>\n",
       "      <td>19.806400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>3.366800</td>\n",
       "      <td>5.370513</td>\n",
       "      <td>12.281000</td>\n",
       "      <td>1.570300</td>\n",
       "      <td>9.874200</td>\n",
       "      <td>9.886300</td>\n",
       "      <td>19.781900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>3.366800</td>\n",
       "      <td>5.404005</td>\n",
       "      <td>12.145000</td>\n",
       "      <td>1.622700</td>\n",
       "      <td>10.041100</td>\n",
       "      <td>10.051100</td>\n",
       "      <td>19.710800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.847200</td>\n",
       "      <td>5.429482</td>\n",
       "      <td>12.386400</td>\n",
       "      <td>1.580300</td>\n",
       "      <td>10.190500</td>\n",
       "      <td>10.190000</td>\n",
       "      <td>19.838200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.847200</td>\n",
       "      <td>5.442894</td>\n",
       "      <td>11.787500</td>\n",
       "      <td>1.403200</td>\n",
       "      <td>9.786200</td>\n",
       "      <td>9.792100</td>\n",
       "      <td>19.826000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.847200</td>\n",
       "      <td>5.459636</td>\n",
       "      <td>11.499400</td>\n",
       "      <td>1.438500</td>\n",
       "      <td>9.669200</td>\n",
       "      <td>9.662100</td>\n",
       "      <td>19.784300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>2.530900</td>\n",
       "      <td>5.462670</td>\n",
       "      <td>11.604700</td>\n",
       "      <td>1.407900</td>\n",
       "      <td>9.738600</td>\n",
       "      <td>9.725700</td>\n",
       "      <td>19.806400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-170\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-340\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-510\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-680\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-850\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1020\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1190\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1360\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1530\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1700\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-1870\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2040\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2210\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2380\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-2550\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-1700 (score: 5.370513439178467).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:42]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:49, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.082954</td>\n",
       "      <td>19.855000</td>\n",
       "      <td>3.442100</td>\n",
       "      <td>15.704400</td>\n",
       "      <td>15.710200</td>\n",
       "      <td>19.784300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.876218</td>\n",
       "      <td>18.957400</td>\n",
       "      <td>3.352300</td>\n",
       "      <td>15.356300</td>\n",
       "      <td>15.399400</td>\n",
       "      <td>19.713200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.392600</td>\n",
       "      <td>5.843147</td>\n",
       "      <td>19.492900</td>\n",
       "      <td>3.833800</td>\n",
       "      <td>15.871200</td>\n",
       "      <td>15.906000</td>\n",
       "      <td>19.622500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.392600</td>\n",
       "      <td>5.804467</td>\n",
       "      <td>20.077800</td>\n",
       "      <td>4.116700</td>\n",
       "      <td>16.070200</td>\n",
       "      <td>16.128300</td>\n",
       "      <td>19.813700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.392600</td>\n",
       "      <td>5.838247</td>\n",
       "      <td>20.770800</td>\n",
       "      <td>4.266300</td>\n",
       "      <td>16.515300</td>\n",
       "      <td>16.535800</td>\n",
       "      <td>19.671600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.893800</td>\n",
       "      <td>5.825732</td>\n",
       "      <td>20.405200</td>\n",
       "      <td>4.225100</td>\n",
       "      <td>16.375100</td>\n",
       "      <td>16.371100</td>\n",
       "      <td>19.816200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.893800</td>\n",
       "      <td>5.858265</td>\n",
       "      <td>20.137300</td>\n",
       "      <td>3.894400</td>\n",
       "      <td>16.178800</td>\n",
       "      <td>16.189800</td>\n",
       "      <td>19.718100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.893800</td>\n",
       "      <td>5.899598</td>\n",
       "      <td>20.335400</td>\n",
       "      <td>4.395700</td>\n",
       "      <td>16.549600</td>\n",
       "      <td>16.539900</td>\n",
       "      <td>19.754900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.272700</td>\n",
       "      <td>5.901470</td>\n",
       "      <td>20.685500</td>\n",
       "      <td>4.325600</td>\n",
       "      <td>16.588700</td>\n",
       "      <td>16.610900</td>\n",
       "      <td>19.742600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.272700</td>\n",
       "      <td>5.900292</td>\n",
       "      <td>20.108900</td>\n",
       "      <td>4.194300</td>\n",
       "      <td>16.188600</td>\n",
       "      <td>16.220100</td>\n",
       "      <td>19.823500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.272700</td>\n",
       "      <td>5.895465</td>\n",
       "      <td>20.721200</td>\n",
       "      <td>4.418900</td>\n",
       "      <td>16.554000</td>\n",
       "      <td>16.582200</td>\n",
       "      <td>19.799000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>1.974900</td>\n",
       "      <td>5.939042</td>\n",
       "      <td>20.608500</td>\n",
       "      <td>4.237900</td>\n",
       "      <td>16.506400</td>\n",
       "      <td>16.519200</td>\n",
       "      <td>19.769600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>1.974900</td>\n",
       "      <td>5.916851</td>\n",
       "      <td>20.564000</td>\n",
       "      <td>4.226900</td>\n",
       "      <td>16.406700</td>\n",
       "      <td>16.422100</td>\n",
       "      <td>19.916700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>1.974900</td>\n",
       "      <td>5.930427</td>\n",
       "      <td>20.418800</td>\n",
       "      <td>4.194500</td>\n",
       "      <td>16.499500</td>\n",
       "      <td>16.529900</td>\n",
       "      <td>19.786800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.765800</td>\n",
       "      <td>5.926174</td>\n",
       "      <td>20.653600</td>\n",
       "      <td>4.380900</td>\n",
       "      <td>16.695300</td>\n",
       "      <td>16.746300</td>\n",
       "      <td>19.821100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-680 (score: 5.80446720123291).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:38]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 34:50, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.118811</td>\n",
       "      <td>19.986500</td>\n",
       "      <td>3.888200</td>\n",
       "      <td>15.703500</td>\n",
       "      <td>15.714600</td>\n",
       "      <td>19.941200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.071161</td>\n",
       "      <td>18.984400</td>\n",
       "      <td>3.801800</td>\n",
       "      <td>15.034700</td>\n",
       "      <td>15.078300</td>\n",
       "      <td>19.654400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.587100</td>\n",
       "      <td>6.012397</td>\n",
       "      <td>18.169600</td>\n",
       "      <td>3.552700</td>\n",
       "      <td>14.422500</td>\n",
       "      <td>14.406500</td>\n",
       "      <td>19.446100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.587100</td>\n",
       "      <td>6.065000</td>\n",
       "      <td>16.832800</td>\n",
       "      <td>3.393700</td>\n",
       "      <td>13.823400</td>\n",
       "      <td>13.874400</td>\n",
       "      <td>19.517200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.587100</td>\n",
       "      <td>6.049434</td>\n",
       "      <td>17.234900</td>\n",
       "      <td>3.779600</td>\n",
       "      <td>13.995300</td>\n",
       "      <td>13.976900</td>\n",
       "      <td>19.171600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.678100</td>\n",
       "      <td>6.102702</td>\n",
       "      <td>17.057500</td>\n",
       "      <td>3.394400</td>\n",
       "      <td>14.026100</td>\n",
       "      <td>14.043000</td>\n",
       "      <td>19.701000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.678100</td>\n",
       "      <td>6.235750</td>\n",
       "      <td>16.919400</td>\n",
       "      <td>3.559600</td>\n",
       "      <td>13.684600</td>\n",
       "      <td>13.689700</td>\n",
       "      <td>19.737700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.678100</td>\n",
       "      <td>6.198881</td>\n",
       "      <td>15.717400</td>\n",
       "      <td>3.199000</td>\n",
       "      <td>12.998000</td>\n",
       "      <td>13.041100</td>\n",
       "      <td>19.698500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.359200</td>\n",
       "      <td>6.249071</td>\n",
       "      <td>15.236100</td>\n",
       "      <td>2.637500</td>\n",
       "      <td>12.371300</td>\n",
       "      <td>12.389000</td>\n",
       "      <td>19.742600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.359200</td>\n",
       "      <td>6.300928</td>\n",
       "      <td>13.614300</td>\n",
       "      <td>2.258400</td>\n",
       "      <td>11.269400</td>\n",
       "      <td>11.282400</td>\n",
       "      <td>19.556400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.359200</td>\n",
       "      <td>6.326688</td>\n",
       "      <td>13.003300</td>\n",
       "      <td>1.993700</td>\n",
       "      <td>10.806000</td>\n",
       "      <td>10.775000</td>\n",
       "      <td>19.583300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.152100</td>\n",
       "      <td>6.357588</td>\n",
       "      <td>12.941000</td>\n",
       "      <td>2.190000</td>\n",
       "      <td>10.726600</td>\n",
       "      <td>10.732500</td>\n",
       "      <td>19.522100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.152100</td>\n",
       "      <td>6.401543</td>\n",
       "      <td>12.830100</td>\n",
       "      <td>1.999700</td>\n",
       "      <td>10.693800</td>\n",
       "      <td>10.706000</td>\n",
       "      <td>19.446100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.152100</td>\n",
       "      <td>6.484684</td>\n",
       "      <td>12.498800</td>\n",
       "      <td>2.074500</td>\n",
       "      <td>10.567000</td>\n",
       "      <td>10.577600</td>\n",
       "      <td>19.475500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.974500</td>\n",
       "      <td>6.464190</td>\n",
       "      <td>12.302500</td>\n",
       "      <td>1.903800</td>\n",
       "      <td>10.308600</td>\n",
       "      <td>10.305900</td>\n",
       "      <td>19.612700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-170\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-340\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-510\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-680\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-850\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-510 (score: 6.012397289276123).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:39]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 33:30, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.253531</td>\n",
       "      <td>20.588800</td>\n",
       "      <td>3.726700</td>\n",
       "      <td>15.990100</td>\n",
       "      <td>16.045500</td>\n",
       "      <td>19.850500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>5.857163</td>\n",
       "      <td>19.952100</td>\n",
       "      <td>3.996500</td>\n",
       "      <td>15.838500</td>\n",
       "      <td>15.853500</td>\n",
       "      <td>19.759800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.957100</td>\n",
       "      <td>5.812199</td>\n",
       "      <td>19.827600</td>\n",
       "      <td>3.810100</td>\n",
       "      <td>15.602100</td>\n",
       "      <td>15.631500</td>\n",
       "      <td>19.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.957100</td>\n",
       "      <td>5.769763</td>\n",
       "      <td>19.588800</td>\n",
       "      <td>3.926900</td>\n",
       "      <td>15.834100</td>\n",
       "      <td>15.856300</td>\n",
       "      <td>19.882400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.957100</td>\n",
       "      <td>5.752831</td>\n",
       "      <td>20.656000</td>\n",
       "      <td>4.055100</td>\n",
       "      <td>16.303400</td>\n",
       "      <td>16.325000</td>\n",
       "      <td>19.902000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.793800</td>\n",
       "      <td>5.768096</td>\n",
       "      <td>20.321300</td>\n",
       "      <td>3.733700</td>\n",
       "      <td>16.034500</td>\n",
       "      <td>16.062000</td>\n",
       "      <td>19.796600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.793800</td>\n",
       "      <td>5.761166</td>\n",
       "      <td>20.239400</td>\n",
       "      <td>4.166600</td>\n",
       "      <td>16.459900</td>\n",
       "      <td>16.458900</td>\n",
       "      <td>19.855400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.793800</td>\n",
       "      <td>5.769475</td>\n",
       "      <td>20.163300</td>\n",
       "      <td>4.018700</td>\n",
       "      <td>16.206200</td>\n",
       "      <td>16.216900</td>\n",
       "      <td>19.823500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.312000</td>\n",
       "      <td>5.745352</td>\n",
       "      <td>20.078600</td>\n",
       "      <td>4.253800</td>\n",
       "      <td>16.243500</td>\n",
       "      <td>16.272000</td>\n",
       "      <td>19.830900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.312000</td>\n",
       "      <td>5.760777</td>\n",
       "      <td>20.595400</td>\n",
       "      <td>4.300500</td>\n",
       "      <td>16.593900</td>\n",
       "      <td>16.579400</td>\n",
       "      <td>19.799000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>2.312000</td>\n",
       "      <td>5.761989</td>\n",
       "      <td>19.941700</td>\n",
       "      <td>4.144000</td>\n",
       "      <td>16.020300</td>\n",
       "      <td>16.012800</td>\n",
       "      <td>19.833300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>2.041600</td>\n",
       "      <td>5.771744</td>\n",
       "      <td>20.476300</td>\n",
       "      <td>4.556200</td>\n",
       "      <td>16.542400</td>\n",
       "      <td>16.524000</td>\n",
       "      <td>19.818600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>2.041600</td>\n",
       "      <td>5.768774</td>\n",
       "      <td>20.610500</td>\n",
       "      <td>4.409000</td>\n",
       "      <td>16.677100</td>\n",
       "      <td>16.705800</td>\n",
       "      <td>19.826000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>2.041600</td>\n",
       "      <td>5.767748</td>\n",
       "      <td>20.449800</td>\n",
       "      <td>4.290500</td>\n",
       "      <td>16.653500</td>\n",
       "      <td>16.636500</td>\n",
       "      <td>19.786800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>1.876000</td>\n",
       "      <td>5.767919</td>\n",
       "      <td>20.194100</td>\n",
       "      <td>4.291000</td>\n",
       "      <td>16.527700</td>\n",
       "      <td>16.504400</td>\n",
       "      <td>19.803900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-170\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-170/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-170/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-340\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-340/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-340/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2380] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-510\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-510/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-510/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2550] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-680\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-680/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-850\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-850/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1020\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1190\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1360\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1530\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1700\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-1870\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2040\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2210\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2380\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2040] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-2550\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-2210] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-1530 (score: 5.745352268218994).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:37]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import copy\n",
    "from torch import nn\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            tels = [copy.deepcopy(attn) for _ in range(4) ] \n",
    "            self.tels = nn.ModuleList(tels)\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if attn.__str__().startswith('ExternalAttention'):\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x)+x\n",
    "            else:\n",
    "                for tel in self.tels:\n",
    "                    x = tel(x,x,x)+x\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "846e2c52-4563-4f5b-87a6-f2fd014b33e4",
   "metadata": {},
   "source": [
    "### Etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bfd22cbe-b702-4b43-b8f6-e918ffd221bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "import torch,copy\n",
    "class cons(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(cons,self).__init__()\n",
    "        self.front = nn.Sequential(\n",
    "            nn.Conv2d(1,32,3),\n",
    "            nn.MaxPool2d(2),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU() ,#batchnorm 2d ?\n",
    "            nn.Conv2d(32,16,2),\n",
    "            nn.MaxPool2d(2),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(),\n",
    "            nn.Flatten()\n",
    "        )\n",
    "        sple = torch.rand(1,1,28,28)\n",
    "        front_out = self.front(sple).size()[-1]\n",
    "        self.body = nn.Sequential(\n",
    "            nn.Linear(front_out,256),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm1d(256),\n",
    "            nn.Dropout(p=0.2),\n",
    "            nn.Linear(256,128),\n",
    "            nn.BatchNorm1d(128),\n",
    "            nn.ReLU(),\n",
    "            # nn.Linear(128,10),\n",
    "        )\n",
    "        self.medium = nn.Sequential(\n",
    "            nn.Linear(128,64),\n",
    "            nn.Linear(64,128)\n",
    "        )\n",
    "        self.macro = [copy.deepcopy(self.medium) for _ in range(3)]\n",
    "        self.macro = nn.ModuleList(self.macro)\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(128,10),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "    def forward(self,x):\n",
    "        x = self.front(x)\n",
    "        x = self.body(x)\n",
    "        # for i in range(len(self.macro)):\n",
    "        #     x = self.macro[i](x)\n",
    "        for i in self.macro:\n",
    "            x = i(x)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "md = cons()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7c777196-5398-424b-8a7b-a13800d61956",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "==========================================================================================\n",
       "Layer (type:depth-idx)                   Output Shape              Param #\n",
       "==========================================================================================\n",
       "cons                                     --                        --\n",
       "├─ModuleList: 1-1                        --                        --\n",
       "├─Sequential: 1-2                        [4, 576]                  --\n",
       "│    └─Conv2d: 2-1                       [4, 32, 26, 26]           320\n",
       "│    └─MaxPool2d: 2-2                    [4, 32, 13, 13]           --\n",
       "│    └─BatchNorm2d: 2-3                  [4, 32, 13, 13]           64\n",
       "│    └─ReLU: 2-4                         [4, 32, 13, 13]           --\n",
       "│    └─Conv2d: 2-5                       [4, 16, 12, 12]           2,064\n",
       "│    └─MaxPool2d: 2-6                    [4, 16, 6, 6]             --\n",
       "│    └─BatchNorm2d: 2-7                  [4, 16, 6, 6]             32\n",
       "│    └─ReLU: 2-8                         [4, 16, 6, 6]             --\n",
       "│    └─Flatten: 2-9                      [4, 576]                  --\n",
       "├─Sequential: 1-3                        [4, 128]                  --\n",
       "│    └─Linear: 2-10                      [4, 256]                  147,712\n",
       "│    └─ReLU: 2-11                        [4, 256]                  --\n",
       "│    └─BatchNorm1d: 2-12                 [4, 256]                  512\n",
       "│    └─Dropout: 2-13                     [4, 256]                  --\n",
       "│    └─Linear: 2-14                      [4, 128]                  32,896\n",
       "│    └─BatchNorm1d: 2-15                 [4, 128]                  256\n",
       "│    └─ReLU: 2-16                        [4, 128]                  --\n",
       "├─ModuleList: 1-1                        --                        --\n",
       "│    └─Sequential: 2-17                  [4, 128]                  --\n",
       "│    │    └─Linear: 3-1                  [4, 64]                   8,256\n",
       "│    │    └─Linear: 3-2                  [4, 128]                  8,320\n",
       "│    └─Sequential: 2-18                  [4, 128]                  --\n",
       "│    │    └─Linear: 3-3                  [4, 64]                   8,256\n",
       "│    │    └─Linear: 3-4                  [4, 128]                  8,320\n",
       "│    └─Sequential: 2-19                  [4, 128]                  --\n",
       "│    │    └─Linear: 3-5                  [4, 64]                   8,256\n",
       "│    │    └─Linear: 3-6                  [4, 128]                  8,320\n",
       "├─Sequential: 1-4                        [4, 10]                   --\n",
       "│    └─Linear: 2-20                      [4, 10]                   1,290\n",
       "│    └─Softmax: 2-21                     [4, 10]                   --\n",
       "==========================================================================================\n",
       "Total params: 234,874\n",
       "Trainable params: 234,874\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (M): 2.98\n",
       "==========================================================================================\n",
       "Input size (MB): 0.01\n",
       "Forward/backward pass size (MB): 1.00\n",
       "Params size (MB): 0.94\n",
       "Estimated Total Size (MB): 1.95\n",
       "=========================================================================================="
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from torchinfo import summary as sm\n",
    "sm(md,(4,1,28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9bce4c5d-e68f-4d28-9aeb-df7398bbe241",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cons(\n",
       "  (front): Sequential(\n",
       "    (0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1))\n",
       "    (1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): ReLU()\n",
       "    (4): Conv2d(32, 16, kernel_size=(2, 2), stride=(1, 1))\n",
       "    (5): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (6): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (7): ReLU()\n",
       "    (8): Flatten(start_dim=1, end_dim=-1)\n",
       "  )\n",
       "  (body): Sequential(\n",
       "    (0): Linear(in_features=576, out_features=256, bias=True)\n",
       "    (1): ReLU()\n",
       "    (2): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): Dropout(p=0.2, inplace=False)\n",
       "    (4): Linear(in_features=256, out_features=128, bias=True)\n",
       "    (5): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (6): ReLU()\n",
       "  )\n",
       "  (medium): Sequential(\n",
       "    (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "    (1): Linear(in_features=64, out_features=128, bias=True)\n",
       "  )\n",
       "  (macro): ModuleList(\n",
       "    (0): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): Linear(in_features=64, out_features=128, bias=True)\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): Linear(in_features=64, out_features=128, bias=True)\n",
       "    )\n",
       "    (2): Sequential(\n",
       "      (0): Linear(in_features=128, out_features=64, bias=True)\n",
       "      (1): Linear(in_features=64, out_features=128, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (fc): Sequential(\n",
       "    (0): Linear(in_features=128, out_features=10, bias=True)\n",
       "    (1): Softmax(dim=1)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca7391b8-f426-4435-858b-188730c64271",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Eval "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3c1cb2d2-44cc-4598-8e1f-675d3e3665c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"Deep1attns_1stepSkip.pickle\",\"wb\") as fw:\n",
    "    pickle.dump(evals, fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "981e4386-1059-464a-acfc-d32bc73f05ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "evals.update(evals_base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1ecfd2d1-17b5-4cca-bf8e-16b5da24326b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>5.662285</td>\n",
       "      <td>17.0819</td>\n",
       "      <td>3.0348</td>\n",
       "      <td>13.7105</td>\n",
       "      <td>13.6935</td>\n",
       "      <td>19.6724</td>\n",
       "      <td>39.3909</td>\n",
       "      <td>10.383</td>\n",
       "      <td>1.752</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>5.462514</td>\n",
       "      <td>14.9609</td>\n",
       "      <td>2.4180</td>\n",
       "      <td>11.9834</td>\n",
       "      <td>11.9720</td>\n",
       "      <td>19.7653</td>\n",
       "      <td>40.4623</td>\n",
       "      <td>10.108</td>\n",
       "      <td>1.705</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>5.755140</td>\n",
       "      <td>18.4752</td>\n",
       "      <td>3.1633</td>\n",
       "      <td>14.7824</td>\n",
       "      <td>14.7587</td>\n",
       "      <td>19.9267</td>\n",
       "      <td>39.4602</td>\n",
       "      <td>10.365</td>\n",
       "      <td>1.749</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>6.014141</td>\n",
       "      <td>17.9119</td>\n",
       "      <td>3.0481</td>\n",
       "      <td>14.5027</td>\n",
       "      <td>14.5011</td>\n",
       "      <td>19.6699</td>\n",
       "      <td>39.9643</td>\n",
       "      <td>10.234</td>\n",
       "      <td>1.727</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>5.777576</td>\n",
       "      <td>20.3214</td>\n",
       "      <td>4.0286</td>\n",
       "      <td>16.0397</td>\n",
       "      <td>16.0443</td>\n",
       "      <td>19.8435</td>\n",
       "      <td>39.6432</td>\n",
       "      <td>10.317</td>\n",
       "      <td>1.741</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          5.662285      17.0819       3.0348   \n",
       "MUSEAttention                         5.462514      14.9609       2.4180   \n",
       "SimplifiedScaledDotProductAttention   5.755140      18.4752       3.1633   \n",
       "ScaledDotProductAttention             6.014141      17.9119       3.0481   \n",
       "ExternalAttention                     5.777576      20.3214       4.0286   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                             13.7105         13.6935   \n",
       "MUSEAttention                            11.9834         11.9720   \n",
       "SimplifiedScaledDotProductAttention      14.7824         14.7587   \n",
       "ScaledDotProductAttention                14.5027         14.5011   \n",
       "ExternalAttention                        16.0397         16.0443   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                              19.6724       39.3909   \n",
       "MUSEAttention                             19.7653       40.4623   \n",
       "SimplifiedScaledDotProductAttention       19.9267       39.4602   \n",
       "ScaledDotProductAttention                 19.6699       39.9643   \n",
       "ExternalAttention                         19.8435       39.6432   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          10.383   \n",
       "MUSEAttention                                         10.108   \n",
       "SimplifiedScaledDotProductAttention                   10.365   \n",
       "ScaledDotProductAttention                             10.234   \n",
       "ExternalAttention                                     10.317   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         1.752   15.0  \n",
       "MUSEAttention                                        1.705   15.0  \n",
       "SimplifiedScaledDotProductAttention                  1.749   15.0  \n",
       "ScaledDotProductAttention                            1.727   15.0  \n",
       "ExternalAttention                                    1.741   15.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1 attn with 1sc\n",
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bbe3be0d-d17a-41a3-af24-a8c452a42002",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>6.666266</td>\n",
       "      <td>14.6945</td>\n",
       "      <td>0.5948</td>\n",
       "      <td>11.9360</td>\n",
       "      <td>11.9617</td>\n",
       "      <td>20.0000</td>\n",
       "      <td>39.8440</td>\n",
       "      <td>10.265</td>\n",
       "      <td>1.732</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>6.710398</td>\n",
       "      <td>2.7172</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.6122</td>\n",
       "      <td>2.6150</td>\n",
       "      <td>11.0660</td>\n",
       "      <td>41.4969</td>\n",
       "      <td>9.856</td>\n",
       "      <td>1.663</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>7.049309</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>39.1412</td>\n",
       "      <td>10.449</td>\n",
       "      <td>1.763</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>7.603535</td>\n",
       "      <td>7.1269</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>7.1172</td>\n",
       "      <td>7.0146</td>\n",
       "      <td>11.0293</td>\n",
       "      <td>34.3124</td>\n",
       "      <td>11.920</td>\n",
       "      <td>2.011</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>9.790882</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>29.8197</td>\n",
       "      <td>13.716</td>\n",
       "      <td>2.314</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          6.666266      14.6945       0.5948   \n",
       "MUSEAttention                         6.710398       2.7172       0.0000   \n",
       "SimplifiedScaledDotProductAttention   7.049309       0.0000       0.0000   \n",
       "ScaledDotProductAttention             7.603535       7.1269       0.0000   \n",
       "ExternalAttention                     9.790882       0.0000       0.0000   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                             11.9360         11.9617   \n",
       "MUSEAttention                             2.6122          2.6150   \n",
       "SimplifiedScaledDotProductAttention       0.0000          0.0000   \n",
       "ScaledDotProductAttention                 7.1172          7.0146   \n",
       "ExternalAttention                         0.0000          0.0000   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                              20.0000       39.8440   \n",
       "MUSEAttention                             11.0660       41.4969   \n",
       "SimplifiedScaledDotProductAttention        2.0000       39.1412   \n",
       "ScaledDotProductAttention                 11.0293       34.3124   \n",
       "ExternalAttention                          5.0000       29.8197   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          10.265   \n",
       "MUSEAttention                                          9.856   \n",
       "SimplifiedScaledDotProductAttention                   10.449   \n",
       "ScaledDotProductAttention                             11.920   \n",
       "ExternalAttention                                     13.716   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         1.732   15.0  \n",
       "MUSEAttention                                        1.663   15.0  \n",
       "SimplifiedScaledDotProductAttention                  1.763   15.0  \n",
       "ScaledDotProductAttention                            2.011   15.0  \n",
       "ExternalAttention                                    2.314   15.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2 attn\n",
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf31bf06-574e-4ea4-9525-0aff3fa49f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>5.628026</td>\n",
       "      <td>17.6677</td>\n",
       "      <td>3.2275</td>\n",
       "      <td>14.1413</td>\n",
       "      <td>14.1415</td>\n",
       "      <td>19.9169</td>\n",
       "      <td>39.4346</td>\n",
       "      <td>10.372</td>\n",
       "      <td>1.750</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>5.450519</td>\n",
       "      <td>13.6378</td>\n",
       "      <td>2.0293</td>\n",
       "      <td>10.9271</td>\n",
       "      <td>10.9229</td>\n",
       "      <td>19.7433</td>\n",
       "      <td>41.0359</td>\n",
       "      <td>9.967</td>\n",
       "      <td>1.681</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>5.780069</td>\n",
       "      <td>19.5277</td>\n",
       "      <td>3.9070</td>\n",
       "      <td>16.0524</td>\n",
       "      <td>16.0592</td>\n",
       "      <td>19.7433</td>\n",
       "      <td>38.8754</td>\n",
       "      <td>10.521</td>\n",
       "      <td>1.775</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>6.056010</td>\n",
       "      <td>18.6185</td>\n",
       "      <td>3.8945</td>\n",
       "      <td>15.3052</td>\n",
       "      <td>15.2656</td>\n",
       "      <td>19.6406</td>\n",
       "      <td>39.6815</td>\n",
       "      <td>10.307</td>\n",
       "      <td>1.739</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>5.803121</td>\n",
       "      <td>20.1092</td>\n",
       "      <td>4.1333</td>\n",
       "      <td>16.0511</td>\n",
       "      <td>16.0491</td>\n",
       "      <td>19.8484</td>\n",
       "      <td>38.4355</td>\n",
       "      <td>10.641</td>\n",
       "      <td>1.795</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          5.628026      17.6677       3.2275   \n",
       "MUSEAttention                         5.450519      13.6378       2.0293   \n",
       "SimplifiedScaledDotProductAttention   5.780069      19.5277       3.9070   \n",
       "ScaledDotProductAttention             6.056010      18.6185       3.8945   \n",
       "ExternalAttention                     5.803121      20.1092       4.1333   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                             14.1413         14.1415   \n",
       "MUSEAttention                            10.9271         10.9229   \n",
       "SimplifiedScaledDotProductAttention      16.0524         16.0592   \n",
       "ScaledDotProductAttention                15.3052         15.2656   \n",
       "ExternalAttention                        16.0511         16.0491   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                              19.9169       39.4346   \n",
       "MUSEAttention                             19.7433       41.0359   \n",
       "SimplifiedScaledDotProductAttention       19.7433       38.8754   \n",
       "ScaledDotProductAttention                 19.6406       39.6815   \n",
       "ExternalAttention                         19.8484       38.4355   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          10.372   \n",
       "MUSEAttention                                          9.967   \n",
       "SimplifiedScaledDotProductAttention                   10.521   \n",
       "ScaledDotProductAttention                             10.307   \n",
       "ExternalAttention                                     10.641   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         1.750   15.0  \n",
       "MUSEAttention                                        1.681   15.0  \n",
       "SimplifiedScaledDotProductAttention                  1.775   15.0  \n",
       "ScaledDotProductAttention                            1.739   15.0  \n",
       "ExternalAttention                                    1.795   15.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2 attn /w 1step Skip\n",
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1e5f0240-3260-4986-a204-de656fea39ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>6.027652</td>\n",
       "      <td>18.8358</td>\n",
       "      <td>3.5920</td>\n",
       "      <td>14.8613</td>\n",
       "      <td>14.8782</td>\n",
       "      <td>19.9609</td>\n",
       "      <td>40.5619</td>\n",
       "      <td>10.083</td>\n",
       "      <td>1.701</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>5.750279</td>\n",
       "      <td>16.6856</td>\n",
       "      <td>2.8786</td>\n",
       "      <td>13.2920</td>\n",
       "      <td>13.2983</td>\n",
       "      <td>19.9682</td>\n",
       "      <td>42.2782</td>\n",
       "      <td>9.674</td>\n",
       "      <td>1.632</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>6.184770</td>\n",
       "      <td>19.2624</td>\n",
       "      <td>3.5663</td>\n",
       "      <td>15.5501</td>\n",
       "      <td>15.5816</td>\n",
       "      <td>19.5477</td>\n",
       "      <td>39.9166</td>\n",
       "      <td>10.246</td>\n",
       "      <td>1.729</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>6.380814</td>\n",
       "      <td>18.5586</td>\n",
       "      <td>3.6846</td>\n",
       "      <td>15.0082</td>\n",
       "      <td>15.0206</td>\n",
       "      <td>19.6797</td>\n",
       "      <td>40.5342</td>\n",
       "      <td>10.090</td>\n",
       "      <td>1.702</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>6.290062</td>\n",
       "      <td>18.6509</td>\n",
       "      <td>3.6751</td>\n",
       "      <td>15.1878</td>\n",
       "      <td>15.2335</td>\n",
       "      <td>19.8484</td>\n",
       "      <td>39.7158</td>\n",
       "      <td>10.298</td>\n",
       "      <td>1.737</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          6.027652      18.8358       3.5920   \n",
       "MUSEAttention                         5.750279      16.6856       2.8786   \n",
       "SimplifiedScaledDotProductAttention   6.184770      19.2624       3.5663   \n",
       "ScaledDotProductAttention             6.380814      18.5586       3.6846   \n",
       "ExternalAttention                     6.290062      18.6509       3.6751   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                             14.8613         14.8782   \n",
       "MUSEAttention                            13.2920         13.2983   \n",
       "SimplifiedScaledDotProductAttention      15.5501         15.5816   \n",
       "ScaledDotProductAttention                15.0082         15.0206   \n",
       "ExternalAttention                        15.1878         15.2335   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                              19.9609       40.5619   \n",
       "MUSEAttention                             19.9682       42.2782   \n",
       "SimplifiedScaledDotProductAttention       19.5477       39.9166   \n",
       "ScaledDotProductAttention                 19.6797       40.5342   \n",
       "ExternalAttention                         19.8484       39.7158   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          10.083   \n",
       "MUSEAttention                                          9.674   \n",
       "SimplifiedScaledDotProductAttention                   10.246   \n",
       "ScaledDotProductAttention                             10.090   \n",
       "ExternalAttention                                     10.298   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         1.701   15.0  \n",
       "MUSEAttention                                        1.632   15.0  \n",
       "SimplifiedScaledDotProductAttention                  1.729   15.0  \n",
       "ScaledDotProductAttention                            1.702   15.0  \n",
       "ExternalAttention                                    1.737   15.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2 attn /w 1step2step\n",
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "667675c7-e001-4256-bfcd-ee20fe45261d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>5.487316</td>\n",
       "      <td>16.7338</td>\n",
       "      <td>3.0192</td>\n",
       "      <td>13.5056</td>\n",
       "      <td>13.5193</td>\n",
       "      <td>19.8068</td>\n",
       "      <td>40.8118</td>\n",
       "      <td>10.022</td>\n",
       "      <td>1.691</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>5.240698</td>\n",
       "      <td>13.1814</td>\n",
       "      <td>1.8514</td>\n",
       "      <td>10.6742</td>\n",
       "      <td>10.6683</td>\n",
       "      <td>19.8631</td>\n",
       "      <td>43.8312</td>\n",
       "      <td>9.331</td>\n",
       "      <td>1.574</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>5.704356</td>\n",
       "      <td>19.8911</td>\n",
       "      <td>4.1914</td>\n",
       "      <td>15.7874</td>\n",
       "      <td>15.8050</td>\n",
       "      <td>19.6479</td>\n",
       "      <td>39.3545</td>\n",
       "      <td>10.393</td>\n",
       "      <td>1.753</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>5.917782</td>\n",
       "      <td>18.8718</td>\n",
       "      <td>4.3341</td>\n",
       "      <td>15.3046</td>\n",
       "      <td>15.3024</td>\n",
       "      <td>19.1247</td>\n",
       "      <td>40.2317</td>\n",
       "      <td>10.166</td>\n",
       "      <td>1.715</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>5.658618</td>\n",
       "      <td>21.2074</td>\n",
       "      <td>4.8898</td>\n",
       "      <td>16.6838</td>\n",
       "      <td>16.7236</td>\n",
       "      <td>19.7824</td>\n",
       "      <td>38.8799</td>\n",
       "      <td>10.520</td>\n",
       "      <td>1.775</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          5.487316      16.7338       3.0192   \n",
       "MUSEAttention                         5.240698      13.1814       1.8514   \n",
       "SimplifiedScaledDotProductAttention   5.704356      19.8911       4.1914   \n",
       "ScaledDotProductAttention             5.917782      18.8718       4.3341   \n",
       "ExternalAttention                     5.658618      21.2074       4.8898   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                             13.5056         13.5193   \n",
       "MUSEAttention                            10.6742         10.6683   \n",
       "SimplifiedScaledDotProductAttention      15.7874         15.8050   \n",
       "ScaledDotProductAttention                15.3046         15.3024   \n",
       "ExternalAttention                        16.6838         16.7236   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                              19.8068       40.8118   \n",
       "MUSEAttention                             19.8631       43.8312   \n",
       "SimplifiedScaledDotProductAttention       19.6479       39.3545   \n",
       "ScaledDotProductAttention                 19.1247       40.2317   \n",
       "ExternalAttention                         19.7824       38.8799   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          10.022   \n",
       "MUSEAttention                                          9.331   \n",
       "SimplifiedScaledDotProductAttention                   10.393   \n",
       "ScaledDotProductAttention                             10.166   \n",
       "ExternalAttention                                     10.520   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         1.691   15.0  \n",
       "MUSEAttention                                        1.574   15.0  \n",
       "SimplifiedScaledDotProductAttention                  1.753   15.0  \n",
       "ScaledDotProductAttention                            1.715   15.0  \n",
       "ExternalAttention                                    1.775   15.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4 attn /w 1stepSC\n",
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be11ea49-1f71-455f-b8b8-a496c69e1251",
   "metadata": {},
   "source": [
    "### basic "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1b1a89a0-bed1-4a0b-b81b-893ee63c755a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 3\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 6\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 2550\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2550' max='2550' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2550/2550 30:31, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.611067</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.436999</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>9.370300</td>\n",
       "      <td>9.270031</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>9.370300</td>\n",
       "      <td>9.115933</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>9.370300</td>\n",
       "      <td>8.973625</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>8.827900</td>\n",
       "      <td>8.845672</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>8.827900</td>\n",
       "      <td>8.732598</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>8.827900</td>\n",
       "      <td>8.635551</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>8.406700</td>\n",
       "      <td>8.551472</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>8.406700</td>\n",
       "      <td>8.482576</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>8.406700</td>\n",
       "      <td>8.425470</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>8.135300</td>\n",
       "      <td>8.383299</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>8.135300</td>\n",
       "      <td>8.353939</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>8.135300</td>\n",
       "      <td>8.336150</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>7.988500</td>\n",
       "      <td>8.330002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-170\n",
      "Configuration saved in ./output/base/checkpoint-170/config.json\n",
      "Model weights saved in ./output/base/checkpoint-170/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-170/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-170/special_tokens_map.json\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-340\n",
      "Configuration saved in ./output/base/checkpoint-340/config.json\n",
      "Model weights saved in ./output/base/checkpoint-340/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-340/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-340/special_tokens_map.json\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-510\n",
      "Configuration saved in ./output/base/checkpoint-510/config.json\n",
      "Model weights saved in ./output/base/checkpoint-510/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-510/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-510/special_tokens_map.json\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-680\n",
      "Configuration saved in ./output/base/checkpoint-680/config.json\n",
      "Model weights saved in ./output/base/checkpoint-680/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-680/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-680/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-170] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-850\n",
      "Configuration saved in ./output/base/checkpoint-850/config.json\n",
      "Model weights saved in ./output/base/checkpoint-850/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-850/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-850/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-340] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1020\n",
      "Configuration saved in ./output/base/checkpoint-1020/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1020/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1020/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1020/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-510] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1190\n",
      "Configuration saved in ./output/base/checkpoint-1190/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1190/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1190/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1190/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-680] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1360\n",
      "Configuration saved in ./output/base/checkpoint-1360/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1360/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1360/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1360/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-850] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/transformers/trainer.py:1357: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  args.max_grad_norm,\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1530\n",
      "Configuration saved in ./output/base/checkpoint-1530/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1530/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1530/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1530/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1020] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1700\n",
      "Configuration saved in ./output/base/checkpoint-1700/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1700/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1700/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1700/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1190] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-1870\n",
      "Configuration saved in ./output/base/checkpoint-1870/config.json\n",
      "Model weights saved in ./output/base/checkpoint-1870/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-1870/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-1870/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1360] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-2040\n",
      "Configuration saved in ./output/base/checkpoint-2040/config.json\n",
      "Model weights saved in ./output/base/checkpoint-2040/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-2040/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-2040/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1530] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-2210\n",
      "Configuration saved in ./output/base/checkpoint-2210/config.json\n",
      "Model weights saved in ./output/base/checkpoint-2210/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-2210/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-2210/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1700] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-2380\n",
      "Configuration saved in ./output/base/checkpoint-2380/config.json\n",
      "Model weights saved in ./output/base/checkpoint-2380/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-2380/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-2380/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-1870] due to args.save_total_limit\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 6\n",
      "Saving model checkpoint to ./output/base/checkpoint-2550\n",
      "Configuration saved in ./output/base/checkpoint-2550/config.json\n",
      "Model weights saved in ./output/base/checkpoint-2550/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/base/checkpoint-2550/tokenizer_config.json\n",
      "Special tokens file saved in ./output/base/checkpoint-2550/special_tokens_map.json\n",
      "Deleting older checkpoint [output/base/checkpoint-2040] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/base/checkpoint-2550 (score: 8.330001831054688).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: summary, id, document.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 6\n",
      "/home/yp/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='69' max='69' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [69/69 00:28]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torch import nn\n",
    "evals_base = {}\n",
    "for i,attn in enumerate(['base']):\n",
    "\n",
    "    import os\n",
    "    model_name = attn\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}