train_xlmr_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting git+https://cantemist:****@gitlab.com/recognai-team/biome/biome-text.git@feat/update_to_allennlp1.1.0rc\n",
      "  Cloning https://cantemist:****@gitlab.com/recognai-team/biome/biome-text.git (to revision feat/update_to_allennlp1.1.0rc) to /tmp/pip-req-build-f45f8c3e\n",
      "Collecting allennlp~=1.1.0rc1\n",
      "  Using cached allennlp-1.1.0rc2.dev20200731-py3-none-any.whl (479 kB)\n",
      "Requirement already satisfied, skipping upgrade: gevent~=1.4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (1.4.0)\n",
      "Requirement already satisfied, skipping upgrade: flask~=1.1.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (1.1.2)\n",
      "Requirement already satisfied, skipping upgrade: flask-cors~=3.0.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (3.0.8)\n",
      "Requirement already satisfied, skipping upgrade: click~=7.1.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (7.1.1)\n",
      "Requirement already satisfied, skipping upgrade: beautifulsoup4~=4.9.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (4.9.1)\n",
      "Requirement already satisfied, skipping upgrade: lxml~=4.5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (4.5.0)\n",
      "Requirement already satisfied, skipping upgrade: fastapi~=0.55.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.55.1)\n",
      "Requirement already satisfied, skipping upgrade: uvicorn~=0.11.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.11.7)\n",
      "Requirement already satisfied, skipping upgrade: dask[complete]~=2.17.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (2.17.2)\n",
      "Requirement already satisfied, skipping upgrade: distributed~=2.17.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (2.17.0)\n",
      "Requirement already satisfied, skipping upgrade: msgpack~=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.6.2)\n",
      "Requirement already satisfied, skipping upgrade: cachey~=0.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.2.1)\n",
      "Requirement already satisfied, skipping upgrade: pyarrow~=0.17.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.17.1)\n",
      "Requirement already satisfied, skipping upgrade: ujson~=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (2.0.3)\n",
      "Requirement already satisfied, skipping upgrade: pandas~=1.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (1.0.3)\n",
      "Requirement already satisfied, skipping upgrade: dask-elk~=0.4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.4.0)\n",
      "Requirement already satisfied, skipping upgrade: bokeh~=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (2.0.1)\n",
      "Requirement already satisfied, skipping upgrade: xlrd~=1.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (1.2.0)\n",
      "Requirement already satisfied, skipping upgrade: flatdict~=4.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (4.0.1)\n",
      "Requirement already satisfied, skipping upgrade: s3fs~=0.4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.4.0)\n",
      "Requirement already satisfied, skipping upgrade: captum~=0.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.2.0)\n",
      "Requirement already satisfied, skipping upgrade: ipywidgets~=7.5.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (7.5.1)\n",
      "Requirement already satisfied, skipping upgrade: mlflow~=1.9.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (1.9.1)\n",
      "Requirement already satisfied, skipping upgrade: ray<0.9.0,>=0.8.6 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from biome-text===update-to-allennlp1.1.0rc) (0.8.6)\n",
      "Requirement already satisfied, skipping upgrade: torch<1.6.0,>=1.5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.5.0)\n",
      "Requirement already satisfied, skipping upgrade: h5py in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.10.0)\n",
      "Requirement already satisfied, skipping upgrade: tensorboardX>=1.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.1)\n",
      "Requirement already satisfied, skipping upgrade: scipy in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.5.0)\n",
      "Processing /home/ec2-user/.cache/pip/wheels/e6/3b/34/ae59fc8d35c37f01099425ab73599e45e9b9b599a7ccc2c45f/overrides-3.1.0-py3-none-any.whl\n",
      "Requirement already satisfied, skipping upgrade: numpy in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.18.5)\n",
      "Requirement already satisfied, skipping upgrade: boto3<2.0,>=1.14 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.14.31)\n",
      "Requirement already satisfied, skipping upgrade: nltk in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (3.4.5)\n",
      "Requirement already satisfied, skipping upgrade: tqdm>=4.19 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (4.44.1)\n",
      "Requirement already satisfied, skipping upgrade: scikit-learn in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.23.1)\n",
      "Requirement already satisfied, skipping upgrade: pytest in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (5.4.1)\n",
      "Requirement already satisfied, skipping upgrade: requests>=2.18 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.23.0)\n",
      "Collecting transformers<3.1,>=3.0\n",
      "  Using cached transformers-3.0.2-py3-none-any.whl (769 kB)\n",
      "Requirement already satisfied, skipping upgrade: jsonnet>=0.10.0; sys_platform != \"win32\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.16.0)\n",
      "Requirement already satisfied, skipping upgrade: filelock<3.1,>=3.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (3.0.12)\n",
      "Requirement already satisfied, skipping upgrade: spacy<2.4,>=2.1.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.1.8)\n",
      "Requirement already satisfied, skipping upgrade: jsonpickle in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.4.1)\n",
      "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.6)\n",
      "Requirement already satisfied, skipping upgrade: greenlet>=0.4.14 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from gevent~=1.4.0->biome-text===update-to-allennlp1.1.0rc) (0.4.15)\n",
      "Requirement already satisfied, skipping upgrade: itsdangerous>=0.24 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from flask~=1.1.2->biome-text===update-to-allennlp1.1.0rc) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: Werkzeug>=0.15 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from flask~=1.1.2->biome-text===update-to-allennlp1.1.0rc) (1.0.1)\n",
      "Requirement already satisfied, skipping upgrade: Jinja2>=2.10.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from flask~=1.1.2->biome-text===update-to-allennlp1.1.0rc) (2.11.1)\n",
      "Requirement already satisfied, skipping upgrade: Six in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from flask-cors~=3.0.8->biome-text===update-to-allennlp1.1.0rc) (1.14.0)\n",
      "Requirement already satisfied, skipping upgrade: soupsieve>1.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from beautifulsoup4~=4.9.0->biome-text===update-to-allennlp1.1.0rc) (2.0)\n",
      "Requirement already satisfied, skipping upgrade: starlette==0.13.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from fastapi~=0.55.0->biome-text===update-to-allennlp1.1.0rc) (0.13.2)\n",
      "Requirement already satisfied, skipping upgrade: pydantic<2.0.0,>=0.32.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from fastapi~=0.55.0->biome-text===update-to-allennlp1.1.0rc) (1.6.1)\n",
      "Requirement already satisfied, skipping upgrade: httptools==0.1.*; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from uvicorn~=0.11.0->biome-text===update-to-allennlp1.1.0rc) (0.1.1)\n",
      "Requirement already satisfied, skipping upgrade: websockets==8.* in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from uvicorn~=0.11.0->biome-text===update-to-allennlp1.1.0rc) (8.1)\n",
      "Requirement already satisfied, skipping upgrade: uvloop>=0.14.0; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from uvicorn~=0.11.0->biome-text===update-to-allennlp1.1.0rc) (0.14.0)\n",
      "Requirement already satisfied, skipping upgrade: h11<0.10,>=0.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from uvicorn~=0.11.0->biome-text===update-to-allennlp1.1.0rc) (0.9.0)\n",
      "Requirement already satisfied, skipping upgrade: pyyaml in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (5.3.1)\n",
      "Requirement already satisfied, skipping upgrade: fsspec>=0.6.0; extra == \"complete\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (0.7.1)\n",
      "Requirement already satisfied, skipping upgrade: toolz>=0.8.2; extra == \"complete\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (0.10.0)\n",
      "Requirement already satisfied, skipping upgrade: partd>=0.3.10; extra == \"complete\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: cloudpickle>=0.2.2; extra == \"complete\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (1.3.0)\n",
      "Requirement already satisfied, skipping upgrade: zict>=0.1.3 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (2.0.0)\n",
      "Requirement already satisfied, skipping upgrade: contextvars; python_version < \"3.7\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (2.4)\n",
      "Requirement already satisfied, skipping upgrade: psutil>=5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (5.7.0)\n",
      "Requirement already satisfied, skipping upgrade: tblib>=1.6.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (1.6.0)\n",
      "Requirement already satisfied, skipping upgrade: setuptools in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (46.1.3.post20200330)\n",
      "Requirement already satisfied, skipping upgrade: sortedcontainers!=2.0.0,!=2.0.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (2.1.0)\n",
      "Requirement already satisfied, skipping upgrade: tornado>=5; python_version < \"3.8\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (6.0.4)\n",
      "Requirement already satisfied, skipping upgrade: heapdict in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from cachey~=0.2.0->biome-text===update-to-allennlp1.1.0rc) (1.0.1)\n",
      "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pandas~=1.0.0->biome-text===update-to-allennlp1.1.0rc) (2.8.1)\n",
      "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pandas~=1.0.0->biome-text===update-to-allennlp1.1.0rc) (2019.3)\n",
      "Requirement already satisfied, skipping upgrade: elasticsearch>=6.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from dask-elk~=0.4.0->biome-text===update-to-allennlp1.1.0rc) (7.8.0)\n",
      "Requirement already satisfied, skipping upgrade: packaging>=16.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from bokeh~=2.0.0->biome-text===update-to-allennlp1.1.0rc) (20.3)\n",
      "Requirement already satisfied, skipping upgrade: typing-extensions>=3.7.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from bokeh~=2.0.0->biome-text===update-to-allennlp1.1.0rc) (3.7.4.1)\n",
      "Requirement already satisfied, skipping upgrade: pillow>=4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from bokeh~=2.0.0->biome-text===update-to-allennlp1.1.0rc) (7.0.0)\n",
      "Requirement already satisfied, skipping upgrade: botocore>=1.12.91 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from s3fs~=0.4.0->biome-text===update-to-allennlp1.1.0rc) (1.17.31)\n",
      "Requirement already satisfied, skipping upgrade: matplotlib in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from captum~=0.2.0->biome-text===update-to-allennlp1.1.0rc) (3.1.3)\n",
      "Requirement already satisfied, skipping upgrade: ipython>=4.0.0; python_version >= \"3.3\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (7.13.0)\n",
      "Requirement already satisfied, skipping upgrade: widgetsnbextension~=3.5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (3.5.1)\n",
      "Requirement already satisfied, skipping upgrade: traitlets>=4.3.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (4.3.3)\n",
      "Requirement already satisfied, skipping upgrade: nbformat>=4.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (5.0.4)\n",
      "Requirement already satisfied, skipping upgrade: ipykernel>=4.5.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (5.1.4)\n",
      "Requirement already satisfied, skipping upgrade: sqlparse in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.3.1)\n",
      "Requirement already satisfied, skipping upgrade: docker>=4.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (4.2.2)\n",
      "Requirement already satisfied, skipping upgrade: gunicorn; platform_system != \"Windows\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (20.0.4)\n",
      "Requirement already satisfied, skipping upgrade: databricks-cli>=0.8.7 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.11.0)\n",
      "Requirement already satisfied, skipping upgrade: gorilla in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.3.0)\n",
      "Requirement already satisfied, skipping upgrade: querystring-parser in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.2.4)\n",
      "Requirement already satisfied, skipping upgrade: gitpython>=2.1.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (3.1.7)\n",
      "Requirement already satisfied, skipping upgrade: alembic in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.4.2)\n",
      "Requirement already satisfied, skipping upgrade: protobuf>=3.6.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (3.12.2)\n",
      "Requirement already satisfied, skipping upgrade: azure-storage-blob>=12.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (12.3.2)\n",
      "Requirement already satisfied, skipping upgrade: sqlalchemy<=1.3.13 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.3.13)\n",
      "Requirement already satisfied, skipping upgrade: prometheus-flask-exporter in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.15.4)\n",
      "Requirement already satisfied, skipping upgrade: entrypoints in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.3)\n",
      "Requirement already satisfied, skipping upgrade: py-spy>=0.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (0.3.3)\n",
      "Requirement already satisfied, skipping upgrade: google in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (3.0.0)\n",
      "Requirement already satisfied, skipping upgrade: jsonschema in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (3.2.0)\n",
      "Requirement already satisfied, skipping upgrade: grpcio in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (1.30.0)\n",
      "Requirement already satisfied, skipping upgrade: colorama in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (0.4.3)\n",
      "Requirement already satisfied, skipping upgrade: redis<3.5.0,>=3.3.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (3.4.1)\n",
      "Requirement already satisfied, skipping upgrade: aiohttp in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (3.6.2)\n",
      "Requirement already satisfied, skipping upgrade: future in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from torch<1.6.0,>=1.5.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.18.2)\n",
      "Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from boto3<2.0,>=1.14->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.3.3)\n",
      "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from boto3<2.0,>=1.14->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.9.4)\n",
      "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from scikit-learn->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.14.1)\n",
      "Requirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from scikit-learn->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.1.0)\n",
      "Requirement already satisfied, skipping upgrade: py>=1.5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.8.1)\n",
      "Requirement already satisfied, skipping upgrade: attrs>=17.4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (19.3.0)\n",
      "Requirement already satisfied, skipping upgrade: more-itertools>=4.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (8.2.0)\n",
      "Requirement already satisfied, skipping upgrade: pluggy<1.0,>=0.12 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.13.1)\n",
      "Requirement already satisfied, skipping upgrade: wcwidth in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.1.9)\n",
      "Requirement already satisfied, skipping upgrade: importlib-metadata>=0.12 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.5.0)\n",
      "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests>=2.18->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.25.8)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests>=2.18->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2020.6.20)\n",
      "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests>=2.18->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests>=2.18->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.9)\n",
      "Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers<3.1,>=3.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.1.91)\n",
      "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers<3.1,>=3.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2020.7.14)\n",
      "Collecting tokenizers==0.8.1.rc1\n",
      "  Using cached tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0 MB)\n",
      "Requirement already satisfied, skipping upgrade: sacremoses in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers<3.1,>=3.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.0.43)\n",
      "Requirement already satisfied, skipping upgrade: blis<0.3.0,>=0.2.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.2.4)\n",
      "Requirement already satisfied, skipping upgrade: preshed<2.1.0,>=2.0.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.0.1)\n",
      "Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (1.0.2)\n",
      "Requirement already satisfied, skipping upgrade: plac<1.0.0,>=0.9.6 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.9.6)\n",
      "Requirement already satisfied, skipping upgrade: srsly<1.1.0,>=0.0.6 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.1.0)\n",
      "Requirement already satisfied, skipping upgrade: wasabi<1.1.0,>=0.2.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (0.2.2)\n",
      "Requirement already satisfied, skipping upgrade: thinc<7.1.0,>=7.0.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (7.0.8)\n",
      "Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy<2.4,>=2.1.0->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.0.2)\n",
      "Requirement already satisfied, skipping upgrade: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from Jinja2>=2.10.1->flask~=1.1.2->biome-text===update-to-allennlp1.1.0rc) (1.1.1)\n",
      "Requirement already satisfied, skipping upgrade: locket in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from partd>=0.3.10; extra == \"complete\"->dask[complete]~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (0.2.0)\n",
      "Requirement already satisfied, skipping upgrade: immutables>=0.9 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from contextvars; python_version < \"3.7\"->distributed~=2.17.0->biome-text===update-to-allennlp1.1.0rc) (0.14)\n",
      "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from packaging>=16.8->bokeh~=2.0.0->biome-text===update-to-allennlp1.1.0rc) (2.4.6)\n",
      "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from botocore>=1.12.91->s3fs~=0.4.0->biome-text===update-to-allennlp1.1.0rc) (0.15.2)\n",
      "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from matplotlib->captum~=0.2.0->biome-text===update-to-allennlp1.1.0rc) (0.10.0)\n",
      "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from matplotlib->captum~=0.2.0->biome-text===update-to-allennlp1.1.0rc) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: decorator in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (4.4.2)\n",
      "Requirement already satisfied, skipping upgrade: pickleshare in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.7.5)\n",
      "Requirement already satisfied, skipping upgrade: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: backcall in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.1.0)\n",
      "Requirement already satisfied, skipping upgrade: pygments in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (2.6.1)\n",
      "Requirement already satisfied, skipping upgrade: pexpect; sys_platform != \"win32\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (4.8.0)\n",
      "Requirement already satisfied, skipping upgrade: jedi>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.15.2)\n",
      "Requirement already satisfied, skipping upgrade: notebook>=4.4.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (6.0.3)\n",
      "Requirement already satisfied, skipping upgrade: ipython-genutils in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from traitlets>=4.3.1->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.2.0)\n",
      "Requirement already satisfied, skipping upgrade: jupyter-core in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbformat>=4.2.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (4.6.3)\n",
      "Requirement already satisfied, skipping upgrade: jupyter-client in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from ipykernel>=4.5.1->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (6.1.2)\n",
      "Requirement already satisfied, skipping upgrade: websocket-client>=0.32.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from docker>=4.0.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.57.0)\n",
      "Requirement already satisfied, skipping upgrade: tabulate>=0.7.7 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from databricks-cli>=0.8.7->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.8.7)\n",
      "Requirement already satisfied, skipping upgrade: gitdb<5,>=4.0.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from gitpython>=2.1.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (4.0.5)\n",
      "Requirement already satisfied, skipping upgrade: python-editor>=0.3 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from alembic->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.0.4)\n",
      "Requirement already satisfied, skipping upgrade: Mako in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from alembic->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.1.3)\n",
      "Requirement already satisfied, skipping upgrade: cryptography>=2.1.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (2.8)\n",
      "Requirement already satisfied, skipping upgrade: msrest>=0.6.10 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.6.18)\n",
      "Requirement already satisfied, skipping upgrade: azure-core<2.0.0,>=1.6.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.7.0)\n",
      "Requirement already satisfied, skipping upgrade: prometheus-client in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from prometheus-flask-exporter->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.7.1)\n",
      "Requirement already satisfied, skipping upgrade: pyrsistent>=0.14.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from jsonschema->ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (0.16.0)\n",
      "Requirement already satisfied, skipping upgrade: idna-ssl>=1.0; python_version < \"3.7\" in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from aiohttp->ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (1.1.0)\n",
      "Requirement already satisfied, skipping upgrade: async-timeout<4.0,>=3.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from aiohttp->ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (3.0.1)\n",
      "Requirement already satisfied, skipping upgrade: yarl<2.0,>=1.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from aiohttp->ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (1.5.0)\n",
      "Requirement already satisfied, skipping upgrade: multidict<5.0,>=4.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from aiohttp->ray<0.9.0,>=0.8.6->biome-text===update-to-allennlp1.1.0rc) (4.7.6)\n",
      "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from importlib-metadata>=0.12->pytest->allennlp~=1.1.0rc1->biome-text===update-to-allennlp1.1.0rc) (2.2.0)\n",
      "Requirement already satisfied, skipping upgrade: ptyprocess>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.6.0)\n",
      "Requirement already satisfied, skipping upgrade: parso>=0.5.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from jedi>=0.10->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.5.2)\n",
      "Requirement already satisfied, skipping upgrade: Send2Trash in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (1.5.0)\n",
      "Requirement already satisfied, skipping upgrade: terminado>=0.8.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.8.3)\n",
      "Requirement already satisfied, skipping upgrade: pyzmq>=17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (18.1.1)\n",
      "Requirement already satisfied, skipping upgrade: nbconvert in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (5.6.1)\n",
      "Requirement already satisfied, skipping upgrade: smmap<4,>=3.0.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from gitdb<5,>=4.0.1->gitpython>=2.1.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: cffi!=1.11.3,>=1.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from cryptography>=2.1.4->azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.14.0)\n",
      "Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.5.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from msrest>=0.6.10->azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (1.3.0)\n",
      "Requirement already satisfied, skipping upgrade: isodate>=0.6.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from msrest>=0.6.10->azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (0.6.0)\n",
      "Requirement already satisfied, skipping upgrade: mistune<2,>=0.8.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.8.4)\n",
      "Requirement already satisfied, skipping upgrade: testpath in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.4.4)\n",
      "Requirement already satisfied, skipping upgrade: bleach in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (3.1.4)\n",
      "Requirement already satisfied, skipping upgrade: defusedxml in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.6.0)\n",
      "Requirement already satisfied, skipping upgrade: pandocfilters>=1.4.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (1.4.2)\n",
      "Requirement already satisfied, skipping upgrade: pycparser in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from cffi!=1.11.3,>=1.8->cryptography>=2.1.4->azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (2.20)\n",
      "Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.10->azure-storage-blob>=12.0->mlflow~=1.9.0->biome-text===update-to-allennlp1.1.0rc) (3.1.0)\n",
      "Requirement already satisfied, skipping upgrade: webencodings in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets~=7.5.1->biome-text===update-to-allennlp1.1.0rc) (0.5.1)\n",
      "Building wheels for collected packages: biome-text\n",
      "  Building wheel for biome-text (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for biome-text: filename=biome_text-update_to_allennlp1.1.0rc-py3-none-any.whl size=91662 sha256=e6e0764df4e4f365940e7b56202ca415547b6ed334625d7e149dd3eea88177ca\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-pvsliver/wheels/fa/2e/ab/46e2371a296c401efb1673b4b95206e7f1e5d42b595dfcda8b\n",
      "Successfully built biome-text\n",
      "Installing collected packages: overrides, tokenizers, transformers, allennlp, biome-text\n",
      "  Attempting uninstall: overrides\n",
      "    Found existing installation: overrides 3.0.0\n",
      "    Uninstalling overrides-3.0.0:\n",
      "      Successfully uninstalled overrides-3.0.0\n",
      "  Attempting uninstall: tokenizers\n",
      "    Found existing installation: tokenizers 0.7.0\n",
      "    Uninstalling tokenizers-0.7.0:\n",
      "      Successfully uninstalled tokenizers-0.7.0\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 2.11.0\n",
      "    Uninstalling transformers-2.11.0:\n",
      "      Successfully uninstalled transformers-2.11.0\n",
      "  Attempting uninstall: allennlp\n",
      "    Found existing installation: allennlp 1.0.0\n",
      "    Uninstalling allennlp-1.0.0:\n",
      "      Successfully uninstalled allennlp-1.0.0\n",
      "  Attempting uninstall: biome-text\n",
      "    Found existing installation: biome-text 1.1.0rc0.dev20200731144514+89760922\n",
      "    Uninstalling biome-text-1.1.0rc0.dev20200731144514+89760922:\n",
      "      Successfully uninstalled biome-text-1.1.0rc0.dev20200731144514+89760922\n",
      "Successfully installed allennlp-1.1.0rc2.dev20200731 biome-text-update-to-allennlp1.1.0rc overrides-3.1.0 tokenizers-0.8.1rc1 transformers-3.0.2\n"
     ]
    }
   ],
   "source": [
    "!pip install -U git+https://gitlab.com/recognai-team/biome/biome-text.git@feat/update_to_allennlp1.1.0rc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EYx_hQMsZ_Op"
   },
   "outputs": [],
   "source": [
    "from biome.text.data import DataSource\n",
    "import pandas as pd\n",
    "from biome.text import Pipeline\n",
    "from biome.text import VocabularyConfiguration\n",
    "from biome.text.configuration import TrainerConfiguration, FindLRConfiguration\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import allennlp\n",
    "allennlp.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MV4TUc5VabcZ"
   },
   "outputs": [],
   "source": [
    "# sizes: train: 18479, train+dev1: 27552, train+dev1+dev2: 35877\n",
    "train_ds = DataSource(source=\"data/NER/train_full.json\", lines=True, orient=\"records\")  # see train_fasttext_model.ipynb\n",
    "valid_ds = DataSource(source=\"data/NER/dev2_for_bert.json\", lines=True, orient=\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "id": "LR4xQD-sa-fc",
    "outputId": "0ffc9b9a-8d9c-4db1-a75c-e2458d4d8365"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O                         180859\n",
       "I-MORFOLOGIA_NEOPLASIA      2088\n",
       "U-MORFOLOGIA_NEOPLASIA      1372\n",
       "L-MORFOLOGIA_NEOPLASIA      1240\n",
       "B-MORFOLOGIA_NEOPLASIA      1240\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = valid_ds.to_dataframe().compute()\n",
    "labels_total = df.labels.sum()\n",
    "pd.Series(labels_total).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kKrk_slKbKNS"
   },
   "outputs": [],
   "source": [
    "pipeline_dict = {\n",
    "    \"name\": \"candemist-ner-xlm\",\n",
    "    \"features\": {\n",
    "        \"transformers\": {\n",
    "            \"model_name\": \"xlm-roberta-base\",\n",
    "            \"trainable\": True,\n",
    "        },\n",
    "        \"char\": {\n",
    "            \"embedding_dim\": 64,\n",
    "            \"lowercase_characters\": False,\n",
    "            \"encoder\": {\n",
    "                \"type\": \"gru\",\n",
    "                \"num_layers\": 1,\n",
    "                \"hidden_size\": 128,\n",
    "                \"bidirectional\": True,\n",
    "            },\n",
    "            \"dropout\": 0.4,\n",
    "        },\n",
    "    },\n",
    "    \"encoder\": {\n",
    "      \"type\": \"lstm\",\n",
    "      \"hidden_size\": 128,\n",
    "      \"bidirectional\": True  \n",
    "    },\n",
    "    \"head\": {\n",
    "        \"type\": \"TokenClassification\",\n",
    "        \"labels\": list(set(labels_total)),\n",
    "        \"dropout\": 0.1,\n",
    "    },       \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gRNngIVrbPEH"
   },
   "outputs": [],
   "source": [
    "#pl = Pipeline.from_config(pipeline_dict, vocab_path='runs/finetune/xlm_paper_encoder_128_char/vocabulary/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "820e15c94bfe4c4cab14fa788196d1e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='building vocab', max=1.0, style=Progres…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pl.create_vocabulary(VocabularyConfiguration(sources=[train_ds]))  #, valid_ds]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "h57LpXezbQvf",
    "outputId": "389aec69-6747-4ff3-9100-b54bf4ef4243"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Vocabulary with namespaces:  char, Size: 134 || gold_labels, Size: 5 || transformers, Size: 250002 || Non Padded Namespaces: {'*labels', 'transformers', '*tags'}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.backbone.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "GjXk7fk2bU4y",
    "outputId": "095613ac-8d40-4ffd-ca46-1b0d95a6f25a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "279384232"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pl.trainable_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rmlmQp9_bXIv"
   },
   "outputs": [],
   "source": [
    "trainer_config = TrainerConfiguration(\n",
    "    optimizer={\n",
    "        \"type\": \"adamw\",\n",
    "        \"lr\": 1e-5,\n",
    "        \"eps\": 1e-6,\n",
    "        \"weight_decay\": 0.1,\n",
    "        \"betas\": [0.9, 0.98]\n",
    "    },\n",
    "    batch_size=16,\n",
    "    num_epochs=10,\n",
    "    cuda_device=0,\n",
    "    validation_metric=\"+f1-measure-overall\",\n",
    "    patience=4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "Mjze03rZbeJW",
    "outputId": "d02bfef9-020f-4c95-a9f3-13565f861e65"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-07-31 22:22:05,336 - allennlp.common.params - INFO - config = <biome.text.configuration.PipelineConfiguration object at 0x7f77b83d7828>\n",
      "2020-07-31 22:22:05,492 - allennlp.common.params - INFO - type = characters\n",
      "2020-07-31 22:22:05,493 - allennlp.common.params - INFO - namespace = char\n",
      "2020-07-31 22:22:05,494 - allennlp.common.params - INFO - character_tokenizer.byte_encoding = None\n",
      "2020-07-31 22:22:05,495 - allennlp.common.params - INFO - character_tokenizer.lowercase_characters = False\n",
      "2020-07-31 22:22:05,495 - allennlp.common.params - INFO - character_tokenizer.start_tokens = None\n",
      "2020-07-31 22:22:05,496 - allennlp.common.params - INFO - character_tokenizer.end_tokens = None\n",
      "2020-07-31 22:22:05,496 - allennlp.common.params - INFO - start_tokens = None\n",
      "2020-07-31 22:22:05,497 - allennlp.common.params - INFO - end_tokens = None\n",
      "2020-07-31 22:22:05,498 - allennlp.common.params - INFO - min_padding_length = 0\n",
      "2020-07-31 22:22:05,498 - allennlp.common.params - INFO - token_min_padding_length = 0\n",
      "2020-07-31 22:22:05,499 - allennlp.common.params - INFO - type = pretrained_transformer_mismatched\n",
      "2020-07-31 22:22:05,499 - allennlp.common.params - INFO - token_min_padding_length = 0\n",
      "2020-07-31 22:22:05,500 - allennlp.common.params - INFO - model_name = xlm-roberta-base\n",
      "2020-07-31 22:22:05,501 - allennlp.common.params - INFO - namespace = transformers\n",
      "2020-07-31 22:22:05,501 - allennlp.common.params - INFO - max_length = None\n",
      "2020-07-31 22:22:05,502 - allennlp.common.params - INFO - type = basic\n",
      "2020-07-31 22:22:05,503 - allennlp.common.params - INFO - token_embedders.char.type = character_encoding\n",
      "2020-07-31 22:22:05,504 - allennlp.common.params - INFO - token_embedders.char.embedding.embedding_dim = 64\n",
      "2020-07-31 22:22:05,505 - allennlp.common.params - INFO - token_embedders.char.embedding.num_embeddings = None\n",
      "2020-07-31 22:22:05,505 - allennlp.common.params - INFO - token_embedders.char.embedding.projection_dim = None\n",
      "2020-07-31 22:22:05,506 - allennlp.common.params - INFO - token_embedders.char.embedding.weight = None\n",
      "2020-07-31 22:22:05,507 - allennlp.common.params - INFO - token_embedders.char.embedding.padding_index = None\n",
      "2020-07-31 22:22:05,507 - allennlp.common.params - INFO - token_embedders.char.embedding.trainable = True\n",
      "2020-07-31 22:22:05,508 - allennlp.common.params - INFO - token_embedders.char.embedding.max_norm = None\n",
      "2020-07-31 22:22:05,512 - allennlp.common.params - INFO - token_embedders.char.embedding.norm_type = 2.0\n",
      "2020-07-31 22:22:05,513 - allennlp.common.params - INFO - token_embedders.char.embedding.scale_grad_by_freq = False\n",
      "2020-07-31 22:22:05,514 - allennlp.common.params - INFO - token_embedders.char.embedding.sparse = False\n",
      "2020-07-31 22:22:05,514 - allennlp.common.params - INFO - token_embedders.char.embedding.vocab_namespace = char\n",
      "2020-07-31 22:22:05,515 - allennlp.common.params - INFO - token_embedders.char.embedding.pretrained_file = None\n",
      "2020-07-31 22:22:05,517 - allennlp.common.params - INFO - token_embedders.char.encoder.type = allennlp.modules.seq2vec_encoders.pytorch_seq2vec_wrapper.GruSeq2VecEncoder\n",
      "2020-07-31 22:22:05,518 - allennlp.common.params - INFO - token_embedders.char.encoder.input_size = 64\n",
      "2020-07-31 22:22:05,518 - allennlp.common.params - INFO - token_embedders.char.encoder.hidden_size = 128\n",
      "2020-07-31 22:22:05,519 - allennlp.common.params - INFO - token_embedders.char.encoder.num_layers = 1\n",
      "2020-07-31 22:22:05,520 - allennlp.common.params - INFO - token_embedders.char.encoder.bias = True\n",
      "2020-07-31 22:22:05,520 - allennlp.common.params - INFO - token_embedders.char.encoder.dropout = 0.0\n",
      "2020-07-31 22:22:05,521 - allennlp.common.params - INFO - token_embedders.char.encoder.bidirectional = True\n",
      "2020-07-31 22:22:05,524 - allennlp.common.params - INFO - token_embedders.char.dropout = 0.4\n",
      "2020-07-31 22:22:05,524 - allennlp.common.params - INFO - token_embedders.transformers.type = pretrained_transformer_mismatched\n",
      "2020-07-31 22:22:05,525 - allennlp.common.params - INFO - token_embedders.transformers.model_name = xlm-roberta-base\n",
      "2020-07-31 22:22:05,526 - allennlp.common.params - INFO - token_embedders.transformers.max_length = None\n",
      "2020-07-31 22:22:05,527 - allennlp.common.params - INFO - token_embedders.transformers.train_parameters = True\n",
      "2020-07-31 22:22:05,527 - allennlp.common.params - INFO - token_embedders.transformers.last_layer_only = True\n",
      "2020-07-31 22:22:05,785 - allennlp.common.params - INFO - input_size = 1024\n",
      "2020-07-31 22:22:05,786 - allennlp.common.params - INFO - hidden_size = 128\n",
      "2020-07-31 22:22:05,787 - allennlp.common.params - INFO - num_layers = 1\n",
      "2020-07-31 22:22:05,787 - allennlp.common.params - INFO - bias = True\n",
      "2020-07-31 22:22:05,788 - allennlp.common.params - INFO - dropout = 0.0\n",
      "2020-07-31 22:22:05,789 - allennlp.common.params - INFO - bidirectional = True\n",
      "2020-07-31 22:22:05,789 - allennlp.common.params - INFO - stateful = False\n",
      "2020-07-31 22:22:05,802 - allennlp.common.params - INFO - labels = ['U-MORFOLOGIA_NEOPLASIA', 'O', 'B-MORFOLOGIA_NEOPLASIA', 'L-MORFOLOGIA_NEOPLASIA', 'I-MORFOLOGIA_NEOPLASIA']\n",
      "2020-07-31 22:22:05,803 - allennlp.common.params - INFO - label_encoding = BIOUL\n",
      "2020-07-31 22:22:05,803 - allennlp.common.params - INFO - top_k = 1\n",
      "2020-07-31 22:22:05,804 - allennlp.common.params - INFO - dropout = 0.1\n",
      "2020-07-31 22:22:05,805 - allennlp.common.params - INFO - feedforward = None\n",
      "2020-07-31 22:22:19,857 - allennlp.common.params - INFO - random_seed = 13370\n",
      "2020-07-31 22:22:19,858 - allennlp.common.params - INFO - numpy_seed = 1337\n",
      "2020-07-31 22:22:19,859 - allennlp.common.params - INFO - pytorch_seed = 133\n",
      "2020-07-31 22:22:19,888 - allennlp.common.checks - INFO - Pytorch version: 1.5.0\n",
      "2020-07-31 22:22:20,441 - allennlp.common.params - INFO - type = gradient_descent\n",
      "2020-07-31 22:22:20,442 - allennlp.common.params - INFO - local_rank = 0\n",
      "2020-07-31 22:22:20,442 - allennlp.common.params - INFO - patience = 4\n",
      "2020-07-31 22:22:20,443 - allennlp.common.params - INFO - validation_metric = +f1-measure-overall\n",
      "2020-07-31 22:22:20,444 - allennlp.common.params - INFO - num_epochs = 10\n",
      "2020-07-31 22:22:20,444 - allennlp.common.params - INFO - cuda_device = 0\n",
      "2020-07-31 22:22:20,445 - allennlp.common.params - INFO - grad_norm = None\n",
      "2020-07-31 22:22:20,445 - allennlp.common.params - INFO - grad_clipping = None\n",
      "2020-07-31 22:22:20,446 - allennlp.common.params - INFO - distributed = None\n",
      "2020-07-31 22:22:20,446 - allennlp.common.params - INFO - world_size = 1\n",
      "2020-07-31 22:22:20,447 - allennlp.common.params - INFO - num_gradient_accumulation_steps = 1\n",
      "2020-07-31 22:22:20,448 - allennlp.common.params - INFO - opt_level = None\n",
      "2020-07-31 22:22:20,448 - allennlp.common.params - INFO - no_grad = None\n",
      "2020-07-31 22:22:20,449 - allennlp.common.params - INFO - learning_rate_scheduler = None\n",
      "2020-07-31 22:22:20,449 - allennlp.common.params - INFO - momentum_scheduler = None\n",
      "2020-07-31 22:22:20,451 - allennlp.common.params - INFO - moving_average = None\n",
      "2020-07-31 22:22:20,452 - allennlp.common.params - INFO - batch_callbacks = None\n",
      "2020-07-31 22:22:22,646 - allennlp.common.params - INFO - optimizer.type = adamw\n",
      "2020-07-31 22:22:22,647 - allennlp.common.params - INFO - optimizer.parameter_groups = None\n",
      "2020-07-31 22:22:22,648 - allennlp.common.params - INFO - optimizer.lr = 1e-05\n",
      "2020-07-31 22:22:22,649 - allennlp.common.params - INFO - optimizer.betas = [0.9, 0.98]\n",
      "2020-07-31 22:22:22,649 - allennlp.common.params - INFO - optimizer.eps = 1e-06\n",
      "2020-07-31 22:22:22,650 - allennlp.common.params - INFO - optimizer.weight_decay = 0.1\n",
      "2020-07-31 22:22:22,651 - allennlp.common.params - INFO - optimizer.amsgrad = False\n",
      "2020-07-31 22:22:22,652 - allennlp.training.optimizers - INFO - Number of trainable parameters: 279384232\n",
      "2020-07-31 22:22:22,653 - allennlp.common.util - INFO - The following parameters are Frozen (without gradient):\n",
      "2020-07-31 22:22:22,654 - allennlp.common.util - INFO - _head._crf._constraint_mask\n",
      "2020-07-31 22:22:22,655 - allennlp.common.util - INFO - The following parameters are Tunable (with gradient):\n",
      "2020-07-31 22:22:22,656 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._embedding._module.weight\n",
      "2020-07-31 22:22:22,656 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_ih_l0\n",
      "2020-07-31 22:22:22,657 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_hh_l0\n",
      "2020-07-31 22:22:22,657 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_ih_l0\n",
      "2020-07-31 22:22:22,658 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_hh_l0\n",
      "2020-07-31 22:22:22,660 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_ih_l0_reverse\n",
      "2020-07-31 22:22:22,661 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.weight_hh_l0_reverse\n",
      "2020-07-31 22:22:22,661 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_ih_l0_reverse\n",
      "2020-07-31 22:22:22,662 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_char._encoder._module._module.bias_hh_l0_reverse\n",
      "2020-07-31 22:22:22,663 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.embeddings.word_embeddings.weight\n",
      "2020-07-31 22:22:22,663 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.embeddings.position_embeddings.weight\n",
      "2020-07-31 22:22:22,664 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.embeddings.token_type_embeddings.weight\n",
      "2020-07-31 22:22:22,665 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.embeddings.LayerNorm.weight\n",
      "2020-07-31 22:22:22,665 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.embeddings.LayerNorm.bias\n",
      "2020-07-31 22:22:22,666 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.query.weight\n",
      "2020-07-31 22:22:22,666 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.query.bias\n",
      "2020-07-31 22:22:22,667 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.key.weight\n",
      "2020-07-31 22:22:22,667 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.key.bias\n",
      "2020-07-31 22:22:22,668 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.value.weight\n",
      "2020-07-31 22:22:22,668 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.self.value.bias\n",
      "2020-07-31 22:22:22,670 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,670 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,671 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,671 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,672 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,672 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,673 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.output.dense.weight\n",
      "2020-07-31 22:22:22,674 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.output.dense.bias\n",
      "2020-07-31 22:22:22,674 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,675 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.0.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,675 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.query.weight\n",
      "2020-07-31 22:22:22,676 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.query.bias\n",
      "2020-07-31 22:22:22,676 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.key.weight\n",
      "2020-07-31 22:22:22,677 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.key.bias\n",
      "2020-07-31 22:22:22,677 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.value.weight\n",
      "2020-07-31 22:22:22,679 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.self.value.bias\n",
      "2020-07-31 22:22:22,679 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,680 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,681 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,681 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,682 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,682 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,682 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.output.dense.weight\n",
      "2020-07-31 22:22:22,683 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.output.dense.bias\n",
      "2020-07-31 22:22:22,683 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,684 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.1.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,684 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.query.weight\n",
      "2020-07-31 22:22:22,685 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.query.bias\n",
      "2020-07-31 22:22:22,687 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.key.weight\n",
      "2020-07-31 22:22:22,687 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.key.bias\n",
      "2020-07-31 22:22:22,688 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.value.weight\n",
      "2020-07-31 22:22:22,688 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.self.value.bias\n",
      "2020-07-31 22:22:22,689 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,689 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,690 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,690 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,690 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,691 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,691 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.output.dense.weight\n",
      "2020-07-31 22:22:22,692 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.output.dense.bias\n",
      "2020-07-31 22:22:22,692 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,693 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.2.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,695 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.query.weight\n",
      "2020-07-31 22:22:22,696 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.query.bias\n",
      "2020-07-31 22:22:22,697 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.key.weight\n",
      "2020-07-31 22:22:22,697 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.key.bias\n",
      "2020-07-31 22:22:22,697 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.value.weight\n",
      "2020-07-31 22:22:22,698 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.self.value.bias\n",
      "2020-07-31 22:22:22,698 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,699 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,699 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,701 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,702 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,702 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,702 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.output.dense.weight\n",
      "2020-07-31 22:22:22,703 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.output.dense.bias\n",
      "2020-07-31 22:22:22,704 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,704 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.3.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,705 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.query.weight\n",
      "2020-07-31 22:22:22,706 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.query.bias\n",
      "2020-07-31 22:22:22,707 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.key.weight\n",
      "2020-07-31 22:22:22,707 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.key.bias\n",
      "2020-07-31 22:22:22,708 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.value.weight\n",
      "2020-07-31 22:22:22,708 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.self.value.bias\n",
      "2020-07-31 22:22:22,709 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,709 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,710 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,711 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,711 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,712 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,712 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.output.dense.weight\n",
      "2020-07-31 22:22:22,713 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.output.dense.bias\n",
      "2020-07-31 22:22:22,714 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,715 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.4.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,715 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.query.weight\n",
      "2020-07-31 22:22:22,716 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.query.bias\n",
      "2020-07-31 22:22:22,716 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.key.weight\n",
      "2020-07-31 22:22:22,717 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.key.bias\n",
      "2020-07-31 22:22:22,718 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.value.weight\n",
      "2020-07-31 22:22:22,718 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.self.value.bias\n",
      "2020-07-31 22:22:22,719 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,719 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,720 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,721 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,721 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,722 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,722 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.output.dense.weight\n",
      "2020-07-31 22:22:22,723 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.output.dense.bias\n",
      "2020-07-31 22:22:22,723 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,724 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.5.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,725 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.query.weight\n",
      "2020-07-31 22:22:22,726 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.query.bias\n",
      "2020-07-31 22:22:22,726 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.key.weight\n",
      "2020-07-31 22:22:22,727 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.key.bias\n",
      "2020-07-31 22:22:22,727 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.value.weight\n",
      "2020-07-31 22:22:22,728 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.self.value.bias\n",
      "2020-07-31 22:22:22,728 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,729 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,730 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,731 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,732 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,732 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,733 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.output.dense.weight\n",
      "2020-07-31 22:22:22,734 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.output.dense.bias\n",
      "2020-07-31 22:22:22,734 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,735 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.6.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,736 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.query.weight\n",
      "2020-07-31 22:22:22,736 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.query.bias\n",
      "2020-07-31 22:22:22,736 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.key.weight\n",
      "2020-07-31 22:22:22,737 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.key.bias\n",
      "2020-07-31 22:22:22,737 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.value.weight\n",
      "2020-07-31 22:22:22,738 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.self.value.bias\n",
      "2020-07-31 22:22:22,738 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,739 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,739 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,741 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,741 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,742 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,742 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.output.dense.weight\n",
      "2020-07-31 22:22:22,743 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.output.dense.bias\n",
      "2020-07-31 22:22:22,743 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,744 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.7.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,744 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.query.weight\n",
      "2020-07-31 22:22:22,745 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.query.bias\n",
      "2020-07-31 22:22:22,745 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.key.weight\n",
      "2020-07-31 22:22:22,746 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.key.bias\n",
      "2020-07-31 22:22:22,746 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.value.weight\n",
      "2020-07-31 22:22:22,747 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.self.value.bias\n",
      "2020-07-31 22:22:22,747 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,749 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,750 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,750 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,751 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,751 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,752 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.output.dense.weight\n",
      "2020-07-31 22:22:22,752 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.output.dense.bias\n",
      "2020-07-31 22:22:22,753 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,753 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.8.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,754 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.query.weight\n",
      "2020-07-31 22:22:22,754 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.query.bias\n",
      "2020-07-31 22:22:22,755 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.key.weight\n",
      "2020-07-31 22:22:22,755 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.key.bias\n",
      "2020-07-31 22:22:22,755 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.value.weight\n",
      "2020-07-31 22:22:22,756 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.self.value.bias\n",
      "2020-07-31 22:22:22,756 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,757 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,759 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,760 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,761 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,761 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,762 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.output.dense.weight\n",
      "2020-07-31 22:22:22,762 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.output.dense.bias\n",
      "2020-07-31 22:22:22,762 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,763 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.9.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,765 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.query.weight\n",
      "2020-07-31 22:22:22,765 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.query.bias\n",
      "2020-07-31 22:22:22,766 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.key.weight\n",
      "2020-07-31 22:22:22,766 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.key.bias\n",
      "2020-07-31 22:22:22,766 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.value.weight\n",
      "2020-07-31 22:22:22,767 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.self.value.bias\n",
      "2020-07-31 22:22:22,767 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,768 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,768 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,769 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,769 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,770 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,770 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.output.dense.weight\n",
      "2020-07-31 22:22:22,771 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.output.dense.bias\n",
      "2020-07-31 22:22:22,771 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,771 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.10.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,772 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.query.weight\n",
      "2020-07-31 22:22:22,772 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.query.bias\n",
      "2020-07-31 22:22:22,775 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.key.weight\n",
      "2020-07-31 22:22:22,776 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.key.bias\n",
      "2020-07-31 22:22:22,777 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.value.weight\n",
      "2020-07-31 22:22:22,777 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.self.value.bias\n",
      "2020-07-31 22:22:22,778 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.output.dense.weight\n",
      "2020-07-31 22:22:22,778 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.output.dense.bias\n",
      "2020-07-31 22:22:22,778 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,779 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.attention.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,779 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.intermediate.dense.weight\n",
      "2020-07-31 22:22:22,780 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.intermediate.dense.bias\n",
      "2020-07-31 22:22:22,780 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.output.dense.weight\n",
      "2020-07-31 22:22:22,781 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.output.dense.bias\n",
      "2020-07-31 22:22:22,781 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.output.LayerNorm.weight\n",
      "2020-07-31 22:22:22,783 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.encoder.layer.11.output.LayerNorm.bias\n",
      "2020-07-31 22:22:22,784 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.pooler.dense.weight\n",
      "2020-07-31 22:22:22,785 - allennlp.common.util - INFO - _head.backbone.embedder.token_embedder_transformers._matched_embedder.transformer_model.pooler.dense.bias\n",
      "2020-07-31 22:22:22,785 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_ih_l0\n",
      "2020-07-31 22:22:22,785 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_hh_l0\n",
      "2020-07-31 22:22:22,786 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_ih_l0\n",
      "2020-07-31 22:22:22,786 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_hh_l0\n",
      "2020-07-31 22:22:22,787 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_ih_l0_reverse\n",
      "2020-07-31 22:22:22,787 - allennlp.common.util - INFO - _head.backbone.encoder._module.weight_hh_l0_reverse\n",
      "2020-07-31 22:22:22,788 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_ih_l0_reverse\n",
      "2020-07-31 22:22:22,788 - allennlp.common.util - INFO - _head.backbone.encoder._module.bias_hh_l0_reverse\n",
      "2020-07-31 22:22:22,789 - allennlp.common.util - INFO - _head._label_projection_layer._module.weight\n",
      "2020-07-31 22:22:22,789 - allennlp.common.util - INFO - _head._label_projection_layer._module.bias\n",
      "2020-07-31 22:22:22,790 - allennlp.common.util - INFO - _head._crf.transitions\n",
      "2020-07-31 22:22:22,792 - allennlp.common.util - INFO - _head._crf.start_transitions\n",
      "2020-07-31 22:22:22,792 - allennlp.common.util - INFO - _head._crf.end_transitions\n",
      "2020-07-31 22:22:22,793 - allennlp.common.params - INFO - checkpointer.type = default\n",
      "2020-07-31 22:22:22,794 - allennlp.common.params - INFO - checkpointer.keep_serialized_model_every_num_seconds = None\n",
      "2020-07-31 22:22:22,795 - allennlp.common.params - INFO - checkpointer.num_serialized_models_to_keep = 1\n",
      "2020-07-31 22:22:22,795 - allennlp.common.params - INFO - checkpointer.model_save_interval = None\n",
      "2020-07-31 22:22:22,796 - allennlp.common.params - INFO - tensorboard_writer.summary_interval = 100\n",
      "2020-07-31 22:22:22,797 - allennlp.common.params - INFO - tensorboard_writer.histogram_interval = None\n",
      "2020-07-31 22:22:22,798 - allennlp.common.params - INFO - tensorboard_writer.batch_size_interval = None\n",
      "2020-07-31 22:22:22,798 - allennlp.common.params - INFO - tensorboard_writer.should_log_parameter_statistics = True\n",
      "2020-07-31 22:22:22,799 - allennlp.common.params - INFO - tensorboard_writer.should_log_learning_rate = True\n",
      "2020-07-31 22:22:22,800 - allennlp.common.params - INFO - tensorboard_writer.get_batch_num_total = None\n",
      "2020-07-31 22:22:22,803 - allennlp.training.trainer - INFO - Beginning training.\n",
      "2020-07-31 22:22:22,804 - allennlp.training.trainer - INFO - Epoch 0/9\n",
      "2020-07-31 22:22:22,804 - allennlp.training.trainer - INFO - Worker 0 memory usage MB: 8600.932\n",
      "2020-07-31 22:22:22,944 - allennlp.training.trainer - INFO - GPU 0 memory usage MB: 1830\n",
      "2020-07-31 22:22:22,947 - allennlp.training.trainer - INFO - Training\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aab55ad694ff41639a5929554c0c7b66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2243.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pl.train(\n",
    "    output=\"runs/finetune/xlm_paper_encoder_128_traindev1dev2\",\n",
    "    training=train_ds,\n",
    "    trainer=trainer_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7RHi47zjliXw"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "XLM.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}