03: SimpleTransformers_1_CoLA_Sentence_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SimpleTransformers 1_CoLA_Sentence_Classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "3be4c4ce2d1346289fbc89569d32c6b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_02a7770d67e94b0db98fc9661547e56b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_918e12df36bd41c09ad5fa67e93eb524",
              "IPY_MODEL_571007ae21b8446288fd742116cb36b9"
            ]
          }
        },
        "02a7770d67e94b0db98fc9661547e56b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "918e12df36bd41c09ad5fa67e93eb524": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ac9659e16ac4b95b64ffb9632c6c817",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_65b31c29e99e47f9923851bc16293fae"
          }
        },
        "571007ae21b8446288fd742116cb36b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b91708e0898b405fac60ad6cf48a50f5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:07&lt;00:00, 61.8B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_462159fe49594ac1b522166cc12b43cc"
          }
        },
        "1ac9659e16ac4b95b64ffb9632c6c817": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "65b31c29e99e47f9923851bc16293fae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b91708e0898b405fac60ad6cf48a50f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "462159fe49594ac1b522166cc12b43cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03594d65b26e4cc3a150bc2dbd8ba4d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c0ebb12d08ae489ba13831130b59f9c6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f752fafb7427490ea9c1322c8b921c77",
              "IPY_MODEL_77d5284bb804424db6fd5afdf3ff8111"
            ]
          }
        },
        "c0ebb12d08ae489ba13831130b59f9c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f752fafb7427490ea9c1322c8b921c77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5d9ed25924e44e639e375a806afd0d70",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_93cc98751a664ba58fb12eea9ec019cb"
          }
        },
        "77d5284bb804424db6fd5afdf3ff8111": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_289334445e864c3591453638453819f8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 70.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dd4154c230be42d88ac96f312f130a34"
          }
        },
        "5d9ed25924e44e639e375a806afd0d70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "93cc98751a664ba58fb12eea9ec019cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "289334445e864c3591453638453819f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dd4154c230be42d88ac96f312f130a34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97b11ebce39c4686a8291c758ee59835": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b3d6574d8826411c848599510a291c87",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0c0b7fca1a2435185b1d51c762d165d",
              "IPY_MODEL_7f1480a0730e4bb4957ae9e5e5eed223"
            ]
          }
        },
        "b3d6574d8826411c848599510a291c87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0c0b7fca1a2435185b1d51c762d165d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9703cee0a5614ce98cebe217dcc8c802",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8552e5c37cf04b389af6fc3b13335f0a"
          }
        },
        "7f1480a0730e4bb4957ae9e5e5eed223": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a71572d4cdea44ddbfdd0475ec085685",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 3.19MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a96c6323f86942ec82e5c1c635c70357"
          }
        },
        "9703cee0a5614ce98cebe217dcc8c802": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8552e5c37cf04b389af6fc3b13335f0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a71572d4cdea44ddbfdd0475ec085685": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a96c6323f86942ec82e5c1c635c70357": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0217362b68248caae1e32eba5f590d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e92d643fcfaf4386ba3e99203fba6ba2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca4f6b85a82347d7ada65e9d9788c1bb",
              "IPY_MODEL_8d8488278d21445083a8f6b866b6f2b1"
            ]
          }
        },
        "e92d643fcfaf4386ba3e99203fba6ba2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca4f6b85a82347d7ada65e9d9788c1bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f36a8009b1394d14be88a6c4fc146f7f",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 7695,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 7695,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_952fc1768a7e49edb0967cac284b7a02"
          }
        },
        "8d8488278d21445083a8f6b866b6f2b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_13e052de1c9e48d3a4440a8db133f39e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 7695/7695 [01:42&lt;00:00, 75.39it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5cdf14ee303445cba33bf5700f9bd95c"
          }
        },
        "f36a8009b1394d14be88a6c4fc146f7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "952fc1768a7e49edb0967cac284b7a02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "13e052de1c9e48d3a4440a8db133f39e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5cdf14ee303445cba33bf5700f9bd95c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d5e843e9933f44ac800332c4f265e9ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b26e72d7d1204ee8ab330bfcff430813",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_62caa165322a47f891dabc673cba7e4b",
              "IPY_MODEL_99444d51ea784b3881ead3fc6758368d"
            ]
          }
        },
        "b26e72d7d1204ee8ab330bfcff430813": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "62caa165322a47f891dabc673cba7e4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_028c345c3f74468c8c9c7221d9b5cb83",
            "_dom_classes": [],
            "description": "Epoch 4 of 4:  75%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e2120246b52c43e98b9f25a8956e68a9"
          }
        },
        "99444d51ea784b3881ead3fc6758368d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_319d119e73f943b3992be4de22730cf2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/4 [07:46&lt;02:35, 155.62s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30689fcc0b7944ff92fd1c8fff6ccb58"
          }
        },
        "028c345c3f74468c8c9c7221d9b5cb83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e2120246b52c43e98b9f25a8956e68a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "319d119e73f943b3992be4de22730cf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "30689fcc0b7944ff92fd1c8fff6ccb58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "43efda485b2a4e14a9667917e018ae64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_08372113005d4a15ab57fdc4d5609860",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_08d476787f1e439f85ea0d9d2ba3f6e2",
              "IPY_MODEL_6e10576df90e45ffb4a075960ecf9e4f"
            ]
          }
        },
        "08372113005d4a15ab57fdc4d5609860": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "08d476787f1e439f85ea0d9d2ba3f6e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cba26b164789400c8c7a16af4d36cd3b",
            "_dom_classes": [],
            "description": "Epochs 0/4. Running Loss:    0.6138: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_26606c2d42b74f3e9bf11111a6b7c401"
          }
        },
        "6e10576df90e45ffb4a075960ecf9e4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3a6b7223394245619dd495c27388a4d4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [02:19&lt;00:00,  1.73it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cf94a5f09f141b1b2f0a012893203dc"
          }
        },
        "cba26b164789400c8c7a16af4d36cd3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "26606c2d42b74f3e9bf11111a6b7c401": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a6b7223394245619dd495c27388a4d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cf94a5f09f141b1b2f0a012893203dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5befd1e148de4739ba71439f4b81ce8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_90ca371079584dbabc995d8d33827bb0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d7365944432a466c86e49ad3bc273240",
              "IPY_MODEL_40b358b5e8f04a019965222e429da6f9"
            ]
          }
        },
        "90ca371079584dbabc995d8d33827bb0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7365944432a466c86e49ad3bc273240": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0c54e80753a54b2296296493ff2389fa",
            "_dom_classes": [],
            "description": "Epochs 1/4. Running Loss:    0.2145: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8272a2cb97954fdb83c6496212159002"
          }
        },
        "40b358b5e8f04a019965222e429da6f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c18060ecce45440bbbda31f0ea218959",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [01:49&lt;00:00,  2.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68ce982d1ebc4b3397576f7276ca6141"
          }
        },
        "0c54e80753a54b2296296493ff2389fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8272a2cb97954fdb83c6496212159002": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c18060ecce45440bbbda31f0ea218959": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68ce982d1ebc4b3397576f7276ca6141": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "052fabdc27d24c319e7d6a393697f33d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2400278360fb43448dc59e6815c7588a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_48e0a33f3b024ffbb400ce07d2435de6",
              "IPY_MODEL_10bd5763b31d415296667efcfd7a8eab"
            ]
          }
        },
        "2400278360fb43448dc59e6815c7588a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48e0a33f3b024ffbb400ce07d2435de6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6a3048f75c3b4265859cde0a596f1434",
            "_dom_classes": [],
            "description": "Epochs 2/4. Running Loss:    0.0433: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b94dfefdc0d24a41bb80157f2384327f"
          }
        },
        "10bd5763b31d415296667efcfd7a8eab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_aaba6d6d92994a3cbeb34ab5809f648f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [01:20&lt;00:00,  2.98it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f447246a3584e1090a983e49e1bb024"
          }
        },
        "6a3048f75c3b4265859cde0a596f1434": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b94dfefdc0d24a41bb80157f2384327f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aaba6d6d92994a3cbeb34ab5809f648f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9f447246a3584e1090a983e49e1bb024": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff6527bef4e849c5a34af646392a559e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6462999e1b974dd1a9070665a2288263",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_74810eac2fdc4463b5e8026e01e407e7",
              "IPY_MODEL_5ba4921f33f1484bbcdf7bb0cb24736e"
            ]
          }
        },
        "6462999e1b974dd1a9070665a2288263": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "74810eac2fdc4463b5e8026e01e407e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dd32ca2c7e164989ae66b4ffa2925ccb",
            "_dom_classes": [],
            "description": "Epochs 3/4. Running Loss:    0.2551:  48%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 116,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b397fd21e8d14d4abfba3641739a1b46"
          }
        },
        "5ba4921f33f1484bbcdf7bb0cb24736e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_350cec9add53401c81c9ca6cd6dbb769",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 116/241 [00:40&lt;00:35,  3.54it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_429d71d1986a442da0fab28bc21ba54e"
          }
        },
        "dd32ca2c7e164989ae66b4ffa2925ccb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b397fd21e8d14d4abfba3641739a1b46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "350cec9add53401c81c9ca6cd6dbb769": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "429d71d1986a442da0fab28bc21ba54e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e306a70925a741f8bfd3ef5bb8afcfc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b0c23646eca4434f921fa5b751751cd3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_87b6e482e32749758d59d4367ffba5ff",
              "IPY_MODEL_c382bccb09154300b5b27d188e23b8f3"
            ]
          }
        },
        "b0c23646eca4434f921fa5b751751cd3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "87b6e482e32749758d59d4367ffba5ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_83e89768c9ce440990c0f701a84cde5c",
            "_dom_classes": [],
            "description": "Running Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 107,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 107,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2ec8bf91d995470a865277e55af3a185"
          }
        },
        "c382bccb09154300b5b27d188e23b8f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0d8f9fee9db144429a779fc0371174e6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 107/107 [00:02&lt;00:00, 36.20it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_379be505f54e492e8ade2b20e850c1fb"
          }
        },
        "83e89768c9ce440990c0f701a84cde5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2ec8bf91d995470a865277e55af3a185": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d8f9fee9db144429a779fc0371174e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "379be505f54e492e8ade2b20e850c1fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3922df1e378f49d2ac4e7467b64c23d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5d635027ee964781ab0de68d40036146",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4e95fb47f52d4ad09a86817c2207c64b",
              "IPY_MODEL_0cbc72ea08934f3ca9ee79d4ae7c2bfe"
            ]
          }
        },
        "5d635027ee964781ab0de68d40036146": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e95fb47f52d4ad09a86817c2207c64b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fcf3517dde4240f1b7bb7726d2b3327d",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 516,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 516,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e59dc8b51e4e41f0a53c2d4c7f682cd8"
          }
        },
        "0cbc72ea08934f3ca9ee79d4ae7c2bfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cf7f8f8df3a24e0fb2552b811f1183f4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 516/516 [00:02&lt;00:00, 198.25it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8223160dfc804c0385a052bce650aacf"
          }
        },
        "fcf3517dde4240f1b7bb7726d2b3327d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e59dc8b51e4e41f0a53c2d4c7f682cd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cf7f8f8df3a24e0fb2552b811f1183f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8223160dfc804c0385a052bce650aacf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "583de1f4b2ae4106843a9b951dcfa65b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f160dc2ee1c045ac85d6a703b1ef5149",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e6ff7b6290374af3a4393f919c99ab38",
              "IPY_MODEL_13e28414ac724952bfff2e360fea55df"
            ]
          }
        },
        "f160dc2ee1c045ac85d6a703b1ef5149": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e6ff7b6290374af3a4393f919c99ab38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a685be1201234168897860810ad22a2c",
            "_dom_classes": [],
            "description": "Running Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 65,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 65,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07ccf48ef30a458eaa3d18cb8ebe5bff"
          }
        },
        "13e28414ac724952bfff2e360fea55df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b322a434b6f84f559bee32913c08aa7f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 65/65 [00:02&lt;00:00, 29.89it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42fd11888cff4fdc8b8c84caffdacb7d"
          }
        },
        "a685be1201234168897860810ad22a2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07ccf48ef30a458eaa3d18cb8ebe5bff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b322a434b6f84f559bee32913c08aa7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42fd11888cff4fdc8b8c84caffdacb7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03f258785e294995a784a892296340fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9d964f3b67d0417fb2ebeadb2bf480a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_37c134462cbc46afba7ede878ccee2b2",
              "IPY_MODEL_1c14aebe54cb434a9c84d62d52e7a320"
            ]
          }
        },
        "9d964f3b67d0417fb2ebeadb2bf480a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "37c134462cbc46afba7ede878ccee2b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_750f5d40a00c41f4a4412e62ca8da9b8",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e34a0210085444c3bc5f77d2e3075e77"
          }
        },
        "1c14aebe54cb434a9c84d62d52e7a320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95ccb1ec4da14fd3b1c403b0a7c77646",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  2.98it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_969c6a6876e14c36b6f72c8a1d7190be"
          }
        },
        "750f5d40a00c41f4a4412e62ca8da9b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e34a0210085444c3bc5f77d2e3075e77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95ccb1ec4da14fd3b1c403b0a7c77646": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "969c6a6876e14c36b6f72c8a1d7190be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4ad125be6be44646b35574c89eae57be": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dc72d4e0668f4199828d56cdb8e2dd09",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ff7f262134004289a75620820fc7f4b7",
              "IPY_MODEL_fdb24003d6314643be1b3cd355a267da"
            ]
          }
        },
        "dc72d4e0668f4199828d56cdb8e2dd09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ff7f262134004289a75620820fc7f4b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b1120ef4bb8744009ad24eed77760992",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5afb32021e9845c39798d91606fbfa70"
          }
        },
        "fdb24003d6314643be1b3cd355a267da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9d8ab33d50b4411b87e538f7ab1b853d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00, 11.15it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ae7b1adfea65451b92a0928b3de47579"
          }
        },
        "b1120ef4bb8744009ad24eed77760992": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5afb32021e9845c39798d91606fbfa70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9d8ab33d50b4411b87e538f7ab1b853d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ae7b1adfea65451b92a0928b3de47579": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/simpletransformers/blob/main/SimpleTransformers_1_CoLA_Sentence_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKOTlwcmxmej"
      },
      "source": [
        "# Simple Transformers - CoLA Sentence Classification\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJR6t_gCQe_x"
      },
      "source": [
        "This Notebook is a port of the original \"Fine-Tuning BERT for Sentence Classification\" tutorial that Nick and I published in 2019 ([blog post](http://mccormickml.com/2019/07/22/BERT-fine-tuning/), [Notebook](https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX)) to use the `Simple Transformers` library.\n",
        "\n",
        "This version has less comments and explanation than my typical Notebooks--the goal was just to get a quick sense of the `Simple Transformers` library.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jrC9__lXxTJz"
      },
      "source": [
        "# Contents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9MCBOq4xUpr"
      },
      "source": [
        "See \"Table of contents\" in the sidebar to the left."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XpoJfANOiuhr"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RX_ZDhicpHkV"
      },
      "source": [
        "# S1. Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nSU7yERLP_66"
      },
      "source": [
        "## 1.1. Using Colab GPU for Training\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GI0iOY8zvZzL"
      },
      "source": [
        "\n",
        "Google Colab offers free GPUs and TPUs! Since we'll be training a large neural network it's best to take advantage of this (in this case we'll attach a GPU), otherwise training will take a very long time.\n",
        "\n",
        "A GPU can be added by going to the menu and selecting:\n",
        "\n",
        "`Edit 🡒 Notebook Settings 🡒 Hardware accelerator 🡒 (GPU)`\n",
        "\n",
        "Then run the following cell to confirm that the GPU is detected."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEfSbAA4QHas",
        "outputId": "9bc1b2c6-9718-4475-c7e0-79f3604464b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqG7FzRVFEIv"
      },
      "source": [
        "In order for torch to use the GPU, we need to identify and specify the GPU as the device. Later, in our training loop, we will load data onto the device. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYsV4H8fCpZ-",
        "outputId": "7ba96907-15b1-4a9e-cbd5-d500d0a61e91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ElsnSNUridI"
      },
      "source": [
        "## 1.2. Installing Simple Transformers\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_N2UDLevYWn"
      },
      "source": [
        "* Install Simple Transformers\n",
        "* Seeing some disconcerting errors..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b50CCOvhAXfk",
        "outputId": "5bc26014-e443-4364-9e16-5f8cd2276867",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install simpletransformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting simpletransformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/35/31022262786f4aa070fe472677cea66fade8d221181a86825096af021e2c/simpletransformers-0.48.14-py3-none-any.whl (214kB)\n",
            "\r\u001b[K     |█▌                              | 10kB 23.4MB/s eta 0:00:01\r\u001b[K     |███                             | 20kB 8.9MB/s eta 0:00:01\r\u001b[K     |████▋                           | 30kB 12.2MB/s eta 0:00:01\r\u001b[K     |██████                          | 40kB 15.0MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 51kB 14.0MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 61kB 15.9MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 71kB 17.5MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 81kB 18.7MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 92kB 17.8MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 102kB 17.7MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 112kB 17.7MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 122kB 17.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 133kB 17.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 143kB 17.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 153kB 17.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 163kB 17.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 174kB 17.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 184kB 17.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 194kB 17.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 204kB 17.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 215kB 17.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.18.5)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2019.12.20)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.1.2)\n",
            "Collecting tokenizers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/a5/78be1a55b2ac8d6a956f0a211d372726e2b1dd2666bb537fea9b03abd62c/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 52.2MB/s \n",
            "\u001b[?25hCollecting transformers>=3.0.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/4e/4f1ede0fd7a36278844a277f8d53c21f88f37f3754abf76a5d6224f76d4a/transformers-3.4.0-py3-none-any.whl (1.3MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3MB 43.8MB/s \n",
            "\u001b[?25hCollecting tqdm>=4.47.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bd/cf/f91813073e4135c1183cadf968256764a6fe4e35c351d596d527c0540461/tqdm-4.50.2-py2.py3-none-any.whl (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 9.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.4.1)\n",
            "Collecting seqeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c4/47/f85f522f1f2532ca650474088e4024a9f52d524379bff058eaadf53cb663/seqeval-1.2.1.tar.gz (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.9MB/s \n",
            "\u001b[?25hCollecting wandb\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/5a/59c6f39d8aceb2c38d6090f947fd77c1cedddf44b1be728d1ae682d09d15/wandb-0.10.7-py2.py3-none-any.whl (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 44.0MB/s \n",
            "\u001b[?25hCollecting streamlit\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c7/d2/3228e647441606a38ce4f73a30ba8a5d36a7be421123d129f7e40348dc02/streamlit-0.69.2-py2.py3-none-any.whl (7.4MB)\n",
            "\u001b[K     |████████████████████████████████| 7.4MB 43.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (0.22.2.post1)\n",
            "Collecting tensorboardx\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/0c/4f41bcd45db376e6fe5c619c01100e9b7531c55791b7244815bac6eac32c/tensorboardX-2.1-py2.py3-none-any.whl (308kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 51.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2.23.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->simpletransformers) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->simpletransformers) (2018.9)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 44.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (3.12.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (20.4)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 45.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (0.7)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (1.15.0)\n",
            "Collecting configparser>=3.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/08/b2/ef713e0e67f6e7ec7d59aea3ee78d05b39c15930057e724cc6d362a8c3bb/configparser-5.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (2.3)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (7.1.2)\n",
            "Collecting watchdog>=0.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/06/121302598a4fc01aca942d937f4a2c33430b7181137b35758913a8db10ad/watchdog-0.10.3.tar.gz (94kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 12.9MB/s \n",
            "\u001b[?25hCollecting GitPython>=1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c0/d7/b2b0672e0331567157adf9281f41ee731c412ee518ca5e6552c27fa73c91/GitPython-3.1.9-py3-none-any.whl (159kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 49.3MB/s \n",
            "\u001b[?25hCollecting subprocess32>=3.5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 13.0MB/s \n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (3.13)\n",
            "Collecting sentry-sdk>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fc/f0/188b8321988c739ab79577269c169d0d5f36aeb7172c17cd45eb0917823d/sentry_sdk-0.19.1-py2.py3-none-any.whl (120kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 49.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (5.4.8)\n",
            "Requirement already satisfied: altair>=3.2.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (4.1.0)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (7.0.0)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.14.1)\n",
            "Requirement already satisfied: astor in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.8.1)\n",
            "Collecting blinker\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1b/51/e2a9f3b757eb802f61dc1f2b09c8c99f6eb01cf06416c0671253536517b6/blinker-1.4.tar.gz (111kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 51.1MB/s \n",
            "\u001b[?25hCollecting botocore>=1.13.44\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/15/53/f4eee30d48e0e495a1ec73ca1c8cc2b6090c378bdb226e42e8cac9278e1e/botocore-1.19.1-py2.py3-none-any.whl (6.7MB)\n",
            "\u001b[K     |████████████████████████████████| 6.7MB 41.7MB/s \n",
            "\u001b[?25hCollecting validators\n",
            "  Downloading https://files.pythonhosted.org/packages/41/4a/3360ff3cf2b4a1b9721ac1fbff5f84663f41047d9874b3aa1ac82e862c44/validators-0.18.1-py3-none-any.whl\n",
            "Requirement already satisfied: cachetools>=4.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (4.1.1)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.10.1)\n",
            "Collecting enum-compat\n",
            "  Downloading https://files.pythonhosted.org/packages/55/ae/467bc4509246283bb59746e21a1a2f5a8aecbef56b1fa6eaca78cd438c8b/enum_compat-0.0.3-py3-none-any.whl\n",
            "Collecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/86/b7/b284f4053b2b8ab867c1db6086a1a908e0e1f60329935ddc1a8a96d4a3e4/boto3-1.16.1-py2.py3-none-any.whl (129kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 51.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: tzlocal in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (1.5.1)\n",
            "Collecting pydeck>=0.1.dev5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/1e/296f4108bf357e684617a776ecaf06ee93b43e30c35996dfac1aa985aa6c/pydeck-0.5.0b1-py2.py3-none-any.whl (4.4MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4MB 43.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: tornado>=5.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (5.1.1)\n",
            "Collecting base58\n",
            "  Downloading https://files.pythonhosted.org/packages/3c/03/58572025c77b9e6027155b272a1b96298e711cd4f95c24967f7137ab0c4b/base58-2.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->simpletransformers) (0.16.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (1.24.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers>=3.0.2->simpletransformers) (50.3.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers>=3.0.2->simpletransformers) (2.4.7)\n",
            "Collecting pathtools>=0.1.1\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: toolz in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.11.1)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (2.11.2)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (2.6.0)\n",
            "Collecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from validators->streamlit->simpletransformers) (4.4.2)\n",
            "Collecting s3transfer<0.4.0,>=0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: ipywidgets>=7.0.0 in /usr/local/lib/python3.6/dist-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (7.5.1)\n",
            "Collecting ipykernel>=5.1.2; python_version >= \"3.4\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/19/c2812690d8b340987eecd2cbc18549b1d130b94c5d97fcbe49f5f8710edf/ipykernel-5.3.4-py3-none-any.whl (120kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 58.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: traitlets>=4.3.2 in /usr/local/lib/python3.6/dist-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (4.3.3)\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->altair>=3.2.0->streamlit->simpletransformers) (1.1.1)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.7)\n",
            "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.5.0)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.5.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.6/dist-packages (from ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.3.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.6.3)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.0.18)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.7.5)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.6.1)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.8.0)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.6/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (19.0.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.5)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.5.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.6.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.9.1)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.4.4)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.4.2)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.4)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.2.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.5.1)\n",
            "Building wheels for collected packages: seqeval, sacremoses, watchdog, subprocess32, blinker, pathtools\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.1-cp36-none-any.whl size=16168 sha256=e2abaca2852a5150759914b91a1de6bbabfd57de5b3b4a1517c6d78ad99f1d7f\n",
            "  Stored in directory: /root/.cache/pip/wheels/b4/42/c0/e2d946497f714404c0b78a0ab4c7f45222a293f49320932463\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=e604ee22e6222a6a420d30b7449ed401466dc76bc32ebd25252cd5f49e4d6203\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "  Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for watchdog: filename=watchdog-0.10.3-cp36-none-any.whl size=73873 sha256=d5391d4daeff2a01fa99201cd57aa772c58b5c374387ea3e6e208e6732f1e5a0\n",
            "  Stored in directory: /root/.cache/pip/wheels/a8/1d/38/2c19bb311f67cc7b4d07a2ec5ea36ab1a0a0ea50db994a5bc7\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp36-none-any.whl size=6489 sha256=8a59a7927b8bf6264ed4d841ce67c4fab1ee00fd38116dd48fa246460c1190f4\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for blinker (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for blinker: filename=blinker-1.4-cp36-none-any.whl size=13450 sha256=0765966c0622014a5522cf00aa74057bc17119f911bcdee9b707077fc2c16f7f\n",
            "  Stored in directory: /root/.cache/pip/wheels/92/a0/00/8690a57883956a301d91cf4ec999cc0b258b01e3f548f86e89\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp36-none-any.whl size=8785 sha256=ad997729fff312f5c0cf4a0618a6268fe6ab1e4f38c6b9af533aea420b2469a9\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "Successfully built seqeval sacremoses watchdog subprocess32 blinker pathtools\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement ipykernel~=4.10, but you'll have ipykernel 5.3.4 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: seqeval 1.2.1 has requirement numpy==1.19.2, but you'll have numpy 1.18.5 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: seqeval 1.2.1 has requirement scikit-learn==0.23.2, but you'll have scikit-learn 0.22.2.post1 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: botocore 1.19.1 has requirement urllib3<1.26,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "Installing collected packages: tokenizers, tqdm, sacremoses, sentencepiece, transformers, seqeval, configparser, pathtools, watchdog, smmap, gitdb, GitPython, subprocess32, docker-pycreds, shortuuid, sentry-sdk, wandb, blinker, jmespath, botocore, validators, enum-compat, s3transfer, boto3, ipykernel, pydeck, base58, streamlit, tensorboardx, simpletransformers\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "  Found existing installation: ipykernel 4.10.1\n",
            "    Uninstalling ipykernel-4.10.1:\n",
            "      Successfully uninstalled ipykernel-4.10.1\n",
            "Successfully installed GitPython-3.1.9 base58-2.0.1 blinker-1.4 boto3-1.16.1 botocore-1.19.1 configparser-5.0.1 docker-pycreds-0.4.0 enum-compat-0.0.3 gitdb-4.0.5 ipykernel-5.3.4 jmespath-0.10.0 pathtools-0.1.2 pydeck-0.5.0b1 s3transfer-0.3.3 sacremoses-0.0.43 sentencepiece-0.1.91 sentry-sdk-0.19.1 seqeval-1.2.1 shortuuid-1.0.1 simpletransformers-0.48.14 smmap-3.0.4 streamlit-0.69.2 subprocess32-3.5.4 tensorboardx-2.1 tokenizers-0.9.2 tqdm-4.50.2 transformers-3.4.0 validators-0.18.1 wandb-0.10.7 watchdog-0.10.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "ipykernel",
                  "tqdm"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guw6ZNtaswKc"
      },
      "source": [
        "# S2. Loading CoLA Dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9ZKxKc04Btk"
      },
      "source": [
        "We'll use [The Corpus of Linguistic Acceptability (CoLA)](https://nyu-mll.github.io/CoLA/) dataset for single sentence classification. It's a set of sentences labeled as grammatically correct or incorrect. It was first published in May of 2018, and is one of the tests included in the \"GLUE Benchmark\" on which models like BERT are competing.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4JrUHXms16cn"
      },
      "source": [
        "## 2.1. Download & Extract"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZNVW6xd0T0X"
      },
      "source": [
        "We'll use the `wget` package to download the dataset to the Colab instance's file system. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5m6AnuFv0QXQ",
        "outputId": "ab49a9ef-6138-45d4-e9ee-0600cef65263",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=b003a730be99ec7480f5104b60146f4b3ecd59bbc824910f5a2a456b29d53ec9\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08pO03Ff1BjI"
      },
      "source": [
        "The dataset is hosted on GitHub in this repo: https://nyu-mll.github.io/CoLA/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMtmPMkBzrvs",
        "outputId": "648cc6af-36c3-4beb-f89f-81cfaec7973d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import wget\n",
        "import os\n",
        "\n",
        "print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "if not os.path.exists('./cola_public_1.1.zip'):\n",
        "    wget.download(url, './cola_public_1.1.zip')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading dataset...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mKctx-ll2FB"
      },
      "source": [
        "Unzip the dataset to the file system. You can browse the file system of the Colab instance in the sidebar on the left."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Yv-tNv20dnH",
        "outputId": "80212361-cdf0-4fda-9d99-d9f17a6b2702",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "if not os.path.exists('./cola_public/'):\n",
        "    !unzip cola_public_1.1.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  cola_public_1.1.zip\n",
            "   creating: cola_public/\n",
            "  inflating: cola_public/README      \n",
            "   creating: cola_public/tokenized/\n",
            "  inflating: cola_public/tokenized/in_domain_dev.tsv  \n",
            "  inflating: cola_public/tokenized/in_domain_train.tsv  \n",
            "  inflating: cola_public/tokenized/out_of_domain_dev.tsv  \n",
            "   creating: cola_public/raw/\n",
            "  inflating: cola_public/raw/in_domain_dev.tsv  \n",
            "  inflating: cola_public/raw/in_domain_train.tsv  \n",
            "  inflating: cola_public/raw/out_of_domain_dev.tsv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oQUy9Tat2EF_"
      },
      "source": [
        "## 2.2. Parse"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeyVCXT31EZQ"
      },
      "source": [
        "We can see from the file names that both `tokenized` and `raw` versions of the data are available. \n",
        "\n",
        "We can't use the pre-tokenized version because, in order to apply the pre-trained BERT, we *must* use the tokenizer provided by the model. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MYWzeGSY2xh3"
      },
      "source": [
        "We'll use pandas to parse the \"in-domain\" training set and look at a few of its properties and data points."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_UkeC7SG2krJ",
        "outputId": "2104fcf2-5215-4721-c053-92b27320ac1d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 478
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"./cola_public/raw/in_domain_train.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training sentences: 8,551\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence_source</th>\n",
              "      <th>label</th>\n",
              "      <th>label_notes</th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>5740</th>\n",
              "      <td>c_13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Juliet says that Romeo lies to his parents a lot.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6294</th>\n",
              "      <td>c_13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>To be able to buy myself a ticket to France wo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2750</th>\n",
              "      <td>l-93</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Carmen obtained a spare part from Diana.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5521</th>\n",
              "      <td>b_73</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>John wants to come up with as good a solution ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7098</th>\n",
              "      <td>sgww85</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>John left his office at 11 and at 12, the libr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6930</th>\n",
              "      <td>m_02</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Helen sent a scarf to Jim for Margaret.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4506</th>\n",
              "      <td>ks08</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Mary sang a song, but Lee never did.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4451</th>\n",
              "      <td>ks08</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>In no other circumstances does that distinctio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7895</th>\n",
              "      <td>ad03</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The Emperor's every wish was immediately carri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7299</th>\n",
              "      <td>sks13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Pelé visited his uncle.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     sentence_source  ...                                           sentence\n",
              "5740            c_13  ...  Juliet says that Romeo lies to his parents a lot.\n",
              "6294            c_13  ...  To be able to buy myself a ticket to France wo...\n",
              "2750            l-93  ...           Carmen obtained a spare part from Diana.\n",
              "5521            b_73  ...  John wants to come up with as good a solution ...\n",
              "7098          sgww85  ...  John left his office at 11 and at 12, the libr...\n",
              "6930            m_02  ...            Helen sent a scarf to Jim for Margaret.\n",
              "4506            ks08  ...               Mary sang a song, but Lee never did.\n",
              "4451            ks08  ...  In no other circumstances does that distinctio...\n",
              "7895            ad03  ...  The Emperor's every wish was immediately carri...\n",
              "7299           sks13  ...                            Pelé visited his uncle.\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfWzpPi92UAH"
      },
      "source": [
        "The two properties we actually care about are the the `sentence` and its `label`, which is referred to as the \"acceptibility judgment\" (0=unacceptable, 1=acceptable)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IC1tBYxRA6iU"
      },
      "source": [
        "## 2.3. Prep Data for SimpleTransformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cKi5LJzTBo4s"
      },
      "source": [
        "Logging setup - Makes the `transformers` logger less verbose."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z_nhVZkOBnzk",
        "outputId": "ea9b62d5-e657-4745-8c56-8dd384960c02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import logging\n",
        "\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "# Get root logger (all other loggers will be derived from this logger's\n",
        "# properties)\n",
        "logger = logging.getLogger()\n",
        "logger.warning(\"Is this working?\") \n",
        "\n",
        "# Get the logger for the huggingface/transformers library.\n",
        "transformers_logger = logging.getLogger(\"transformers\")\n",
        "\n",
        "# Set the logging level to warning, meaning display warnings and worse, but \n",
        "# don't display any `INFO` logs.\n",
        "transformers_logger.setLevel(logging.WARNING)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Is this working?\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "82WJCtDRCYWV"
      },
      "source": [
        "Convert the dataset into a simple list of [text, label] pairs.\n",
        "\n",
        "I'm following this code snippet from the documentation as my reference:\n",
        "```python\n",
        "# Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns. If the Dataframe has a header, it should contain a 'text' and a 'labels' column. If no header is present, the Dataframe should contain at least two columns, with the first column is the text with type str, and the second column in the label with type int.\n",
        "train_data = [['Example sentence belonging to class 1', 1], ['Example sentence belonging to class 0', 0], ['Example eval senntence belonging to class 2', 2]]\n",
        "train_df = pd.DataFrame(train_data)\n",
        "\n",
        "eval_data = [['Example eval sentence belonging to class 1', 1], ['Example eval sentence belonging to class 0', 0], ['Example eval senntence belonging to class 2', 2]]\n",
        "eval_df = pd.DataFrame(eval_data)\n",
        "\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeA6OK4H2eVL"
      },
      "source": [
        "# Select just the text and its label.\n",
        "df = df[['sentence', 'label']]\n",
        "\n",
        "# Rename 'sentence' column to 'text' (this is the name SimpleTransformers \n",
        "# expects).\n",
        "df.columns = ['text', 'label']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wv7xPZlNCvr1"
      },
      "source": [
        "Split off 10% for validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loJK0iLaCXDZ"
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for\n",
        "# training\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use 90% for training and 10% for validation.\n",
        "train_df, validation_df = train_test_split(df, random_state=2018, test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lIwnAga6hgs4",
        "outputId": "d47749fc-5fbf-417e-82b9-50f4ff3da231",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(train_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7695"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37FcR_NOhiGN",
        "outputId": "b526e653-d692-49c4-9cbf-2ffae275df17",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(validation_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "856"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "osB0HTxX-_Yp"
      },
      "source": [
        "**Test Set**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsF7SfJ1_ckU"
      },
      "source": [
        "**TODO** - The .tsv file doesn't include headers, so I've specified them here. This means that I can directly name the 'sentence' column 'text', but I think it's more illustrative this way?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXpnGmUa_EIj",
        "outputId": "dad7005a-57c0-4c1b-80b5-ba3546611ced",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# Load the dataset into a pandas dataframe.\n",
        "df_test = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Select just the text and its label.\n",
        "df_test = df_test[['sentence', 'label']]\n",
        "\n",
        "# Rename 'sentence' column to 'text' (this is the name SimpleTransformers \n",
        "# expects).\n",
        "df_test.columns = ['text', 'label']\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of test sentences: 8,551\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kFiArS_C4DlQ"
      },
      "source": [
        "# S3. Fine-Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bizaNCg74HcL"
      },
      "source": [
        "## 3.1. Load Pre-Trained Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aH5bIzq_DXct"
      },
      "source": [
        "What's this reprocess_input_data argument?\n",
        "\n",
        "Documentation for arguments is [here](https://github.com/ThilinaRajapakse/simpletransformers#default-settings).\n",
        "\n",
        "> reprocess_input_data: bool\n",
        "If True, the input data will be reprocessed even if a cached file of the input data exists in the cache_dir."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ITLUMV1CBmQS",
        "outputId": "03114be0-e64e-45d0-b1c1-a472780f401a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 388,
          "referenced_widgets": [
            "3be4c4ce2d1346289fbc89569d32c6b0",
            "02a7770d67e94b0db98fc9661547e56b",
            "918e12df36bd41c09ad5fa67e93eb524",
            "571007ae21b8446288fd742116cb36b9",
            "1ac9659e16ac4b95b64ffb9632c6c817",
            "65b31c29e99e47f9923851bc16293fae",
            "b91708e0898b405fac60ad6cf48a50f5",
            "462159fe49594ac1b522166cc12b43cc",
            "03594d65b26e4cc3a150bc2dbd8ba4d3",
            "c0ebb12d08ae489ba13831130b59f9c6",
            "f752fafb7427490ea9c1322c8b921c77",
            "77d5284bb804424db6fd5afdf3ff8111",
            "5d9ed25924e44e639e375a806afd0d70",
            "93cc98751a664ba58fb12eea9ec019cb",
            "289334445e864c3591453638453819f8",
            "dd4154c230be42d88ac96f312f130a34",
            "97b11ebce39c4686a8291c758ee59835",
            "b3d6574d8826411c848599510a291c87",
            "f0c0b7fca1a2435185b1d51c762d165d",
            "7f1480a0730e4bb4957ae9e5e5eed223",
            "9703cee0a5614ce98cebe217dcc8c802",
            "8552e5c37cf04b389af6fc3b13335f0a",
            "a71572d4cdea44ddbfdd0475ec085685",
            "a96c6323f86942ec82e5c1c635c70357"
          ]
        }
      },
      "source": [
        "from simpletransformers.classification import ClassificationModel\n",
        "\n",
        "args = {\n",
        "    'reprocess_input_data': True, \n",
        "    'overwrite_output_dir': True\n",
        "}\n",
        "\n",
        "# Create a ClassificationModel\n",
        "model = ClassificationModel('bert', 'bert-base-uncased', num_labels=2, args=args)\n",
        "\n",
        "# You can set class weights by using the optional weight argument\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n",
            "INFO:filelock:Lock 139956352702728 acquired on /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.7156163d5fdc189c3016baca0775ffce230789d7fa2a42ef516483e4ca884517.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3be4c4ce2d1346289fbc89569d32c6b0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 139956352702728 released on /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.7156163d5fdc189c3016baca0775ffce230789d7fa2a42ef516483e4ca884517.lock\n",
            "INFO:filelock:Lock 139956302413216 acquired on /root/.cache/torch/transformers/f2ee78bdd635b758cc0a12352586868bef80e47401abe4c4fcc3832421e7338b.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03594d65b26e4cc3a150bc2dbd8ba4d3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 139956302413216 released on /root/.cache/torch/transformers/f2ee78bdd635b758cc0a12352586868bef80e47401abe4c4fcc3832421e7338b.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "INFO:filelock:Lock 139960397168200 acquired on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "97b11ebce39c4686a8291c758ee59835",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 139960397168200 released on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwHNCEdHEaSz"
      },
      "source": [
        "## 3.2. Inspect Training Arguments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YSBeEWdEcIL"
      },
      "source": [
        "All of the cool features we're interested in are managed through this `model.args` object. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K4rNROrD4qAD",
        "outputId": "5c27f884-3250-4f28-8582-ae98f59375ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "type(model.args)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "simpletransformers.config.model_args.ClassificationArgs"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HeBRWPzmqUze"
      },
      "source": [
        "The following cell retrieves all of the arguments and their values, and prints them in a table.\n",
        "\n",
        "There is also some brief documentation on them [here](https://simpletransformers.ai/docs/usage/#configuring-a-simple-transformers-model)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADMTSlPv4tsh",
        "outputId": "0156590f-c34b-427c-92e2-e98f1566f60b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "arg_values = []\n",
        "\n",
        "pd.set_option('display.max_rows', 80)\n",
        "\n",
        "# For all of the arguments...\n",
        "for arg in dir(model.args):\n",
        "    \n",
        "    # Skip over the special attributes and any functions.\n",
        "    if (not arg[0:2] == '__') and (not callable(getattr(model.args, arg))):\n",
        "    \n",
        "        # Store the argument and its value as a tuple.\n",
        "        arg_values.append((arg, str(getattr(model.args, arg))))\n",
        "\n",
        "# Store as a dataframe just to get the pretty printout.\n",
        "df_args = pd.DataFrame(arg_values)        \n",
        "\n",
        "df_args"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>adam_epsilon</td>\n",
              "      <td>1e-08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>best_model_dir</td>\n",
              "      <td>outputs/best_model</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>cache_dir</td>\n",
              "      <td>cache_dir/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>config</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>custom_layer_parameters</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>custom_parameter_groups</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>dataloader_num_workers</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>do_lower_case</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dynamic_quantize</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>early_stopping_consider_epochs</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>early_stopping_delta</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>early_stopping_metric</td>\n",
              "      <td>eval_loss</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>early_stopping_metric_minimize</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>early_stopping_patience</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>encoding</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>eval_batch_size</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>evaluate_during_training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>evaluate_during_training_silent</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>evaluate_during_training_steps</td>\n",
              "      <td>2000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>evaluate_during_training_verbose</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>evaluate_each_epoch</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fp16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>gradient_accumulation_steps</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>labels_list</td>\n",
              "      <td>[0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>labels_map</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>lazy_delimiter</td>\n",
              "      <td>\\t</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>lazy_labels_column</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>lazy_loading</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>lazy_loading_start_line</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>lazy_text_a_column</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>lazy_text_b_column</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>lazy_text_column</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>learning_rate</td>\n",
              "      <td>4e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>local_rank</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>logging_steps</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>manual_seed</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>max_grad_norm</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>max_seq_length</td>\n",
              "      <td>128</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>model_class</td>\n",
              "      <td>ClassificationModel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>model_name</td>\n",
              "      <td>bert-base-uncased</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>model_type</td>\n",
              "      <td>bert</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>multiprocessing_chunksize</td>\n",
              "      <td>500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>n_gpu</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>no_cache</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>no_save</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>num_train_epochs</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>onnx</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>output_dir</td>\n",
              "      <td>outputs/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>overwrite_output_dir</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>process_count</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>quantized_model</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>regression</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>reprocess_input_data</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>save_best_model</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>save_eval_checkpoints</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>save_model_every_epoch</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>save_optimizer_and_scheduler</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>save_steps</td>\n",
              "      <td>2000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>silent</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>skip_special_tokens</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>sliding_window</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>stride</td>\n",
              "      <td>0.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>tensorboard_dir</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>thread_count</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>tie_value</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>train_batch_size</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>train_custom_parameters_only</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>use_cached_eval_features</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>use_early_stopping</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>use_multiprocessing</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>wandb_kwargs</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>wandb_project</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>warmup_ratio</td>\n",
              "      <td>0.06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>warmup_steps</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>weight_decay</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                   0                    1\n",
              "0                       adam_epsilon                1e-08\n",
              "1                     best_model_dir   outputs/best_model\n",
              "2                          cache_dir           cache_dir/\n",
              "3                             config                   {}\n",
              "4            custom_layer_parameters                   []\n",
              "5            custom_parameter_groups                   []\n",
              "6             dataloader_num_workers                    1\n",
              "7                      do_lower_case                False\n",
              "8                   dynamic_quantize                False\n",
              "9     early_stopping_consider_epochs                False\n",
              "10              early_stopping_delta                    0\n",
              "11             early_stopping_metric            eval_loss\n",
              "12    early_stopping_metric_minimize                 True\n",
              "13           early_stopping_patience                    3\n",
              "14                          encoding                 None\n",
              "15                   eval_batch_size                    8\n",
              "16          evaluate_during_training                False\n",
              "17   evaluate_during_training_silent                 True\n",
              "18    evaluate_during_training_steps                 2000\n",
              "19  evaluate_during_training_verbose                False\n",
              "20               evaluate_each_epoch                 True\n",
              "21                              fp16                 True\n",
              "22       gradient_accumulation_steps                    1\n",
              "23                       labels_list               [0, 1]\n",
              "24                        labels_map                   {}\n",
              "25                    lazy_delimiter                   \\t\n",
              "26                lazy_labels_column                    1\n",
              "27                      lazy_loading                False\n",
              "28           lazy_loading_start_line                    1\n",
              "29                lazy_text_a_column                 None\n",
              "30                lazy_text_b_column                 None\n",
              "31                  lazy_text_column                    0\n",
              "32                     learning_rate                4e-05\n",
              "33                        local_rank                   -1\n",
              "34                     logging_steps                   50\n",
              "35                       manual_seed                 None\n",
              "36                     max_grad_norm                  1.0\n",
              "37                    max_seq_length                  128\n",
              "38                       model_class  ClassificationModel\n",
              "39                        model_name    bert-base-uncased\n",
              "40                        model_type                 bert\n",
              "41         multiprocessing_chunksize                  500\n",
              "42                             n_gpu                    1\n",
              "43                          no_cache                False\n",
              "44                           no_save                False\n",
              "45                  num_train_epochs                    1\n",
              "46                              onnx                False\n",
              "47                        output_dir             outputs/\n",
              "48              overwrite_output_dir                 True\n",
              "49                     process_count                    1\n",
              "50                   quantized_model                False\n",
              "51                        regression                False\n",
              "52              reprocess_input_data                 True\n",
              "53                   save_best_model                 True\n",
              "54             save_eval_checkpoints                 True\n",
              "55            save_model_every_epoch                 True\n",
              "56      save_optimizer_and_scheduler                 True\n",
              "57                        save_steps                 2000\n",
              "58                            silent                False\n",
              "59               skip_special_tokens                 True\n",
              "60                    sliding_window                False\n",
              "61                            stride                  0.8\n",
              "62                   tensorboard_dir                 None\n",
              "63                      thread_count                 None\n",
              "64                         tie_value                    1\n",
              "65                  train_batch_size                    8\n",
              "66      train_custom_parameters_only                False\n",
              "67          use_cached_eval_features                False\n",
              "68                use_early_stopping                False\n",
              "69               use_multiprocessing                 True\n",
              "70                      wandb_kwargs                   {}\n",
              "71                     wandb_project                 None\n",
              "72                      warmup_ratio                 0.06\n",
              "73                      warmup_steps                    0\n",
              "74                      weight_decay                    0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZKbpevQ7a-h"
      },
      "source": [
        "### Noteworthy Args"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xmwuXj2urCVA"
      },
      "source": [
        "Below are what I'd consider the most critical arguments--they'll have the most immediate impact on your accuracy.\n",
        "\n",
        "What's displayed are their default values (except model-name, which we specified earlier)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0g7vCk6l7doI",
        "outputId": "c7003ce1-5090-4b7f-b9ee-4f82d5783dc5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.train_batch_size"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lL-QZ_2w7fzL",
        "outputId": "754f1c68-00a7-4f8c-c35f-e2d545d73030",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.num_train_epochs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bI5rAPTA7jK0",
        "outputId": "cb8774df-eb9d-4a4c-b085-bd7684fba1d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.max_seq_length"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "128"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bk6-Ye7x7poE",
        "outputId": "496733ba-7ed1-44f9-87cd-ee8db7738eb1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.learning_rate"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4e-05"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AK4ReVMV7mp0",
        "outputId": "6e3e9a64-7e02-4fd1-fe48-6400b912cdf7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "model.args.model_name"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'bert-base-uncased'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aJyF8Jb86w-6"
      },
      "source": [
        "## 3.3. Run Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6QGPheuGXqIV"
      },
      "source": [
        "#### Choose Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4mHLBtrE-XTa"
      },
      "source": [
        "# These are the values we used in our Notebook:\n",
        "model.args.max_seq_length = 128\n",
        "model.args.num_train_epochs = 4\n",
        "model.args.train_batch_size = 32\n",
        "model.args.learning_rate = 2e-5"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wOz-aOhucHlf"
      },
      "source": [
        "#### Configure Validation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMSDFNtEcLIt"
      },
      "source": [
        "Periodically evaluate on our 10% validation set during training to monitor over-fitting.\n",
        "\n",
        "Calculate number of steps so we can specify how often to evaluate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ynY7BMUSVxFf",
        "outputId": "a958894b-1c57-4fd4-8983-715934ef6e63",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Num steps in epoch = num training samples / batch size\n",
        "steps_per_epoch = int(np.ceil(len(train_df) / float(model.args.train_batch_size)))\n",
        "\n",
        "print('Each epoch will have {:,} steps.'.format(steps_per_epoch))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Each epoch will have 241 steps.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QSz3ovrXcZhe"
      },
      "source": [
        "Turn on validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3h4WNIPcJe7"
      },
      "source": [
        "# Run evaluation periodically during training to monitor progress.\n",
        "model.args.evaluate_during_training = True\n",
        "\n",
        "# \"Print results from evaluation during training.\"\n",
        "model.args.evaluate_during_training_verbose = True\n",
        "\n",
        "# \"Perform evaluation at every specified number of steps. A checkpoint model and\n",
        "#  the evaluation results will be saved.\"\n",
        "model.args.evaluate_during_training_steps = 120\n",
        "\n",
        "# We only need to tokenize our validation set once, then we can read it from the\n",
        "# cache.\n",
        "model.args.use_cached_eval_features = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AHM3fSwmXs39"
      },
      "source": [
        "#### Configure Early Stopping"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OT1V5LmrXzwu"
      },
      "source": [
        "There's a nice intro to early stopping in their docs here:\n",
        "    \n",
        "* https://simpletransformers.ai/docs/usage/#using-early-stopping\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6m3mcaBnXjI_"
      },
      "source": [
        "# Turn on early stopping.\n",
        "model.args.use_early_stopping = True\n",
        "\n",
        "# \"The improvement over best_eval_loss necessary to count as a better checkpoint.\"\n",
        "model.args.early_stopping_delta = 0.01\n",
        "\n",
        "# What metric to use in calculating score for evaluation set (plus whether a low\n",
        "# vs. high value is better for this metric).\n",
        "\n",
        "#model.args.early_stopping_metric = \"mcc\"\n",
        "#model.args.early_stopping_metric_minimize = False\n",
        "\n",
        "model.args.early_stopping_metric = \"eval_loss\"\n",
        "model.args.early_stopping_metric_minimize = True\n",
        "\n",
        "# \"Terminate training after this many evaluations without an improvement in the\n",
        "#  evaluation metric greater then early_stopping_delta.\"\n",
        "model.args.early_stopping_patience = 2\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHbXqbckfqEo"
      },
      "source": [
        "#### Kick-Off Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AXqIwuILs3kO"
      },
      "source": [
        "The log statements aren't easy to interpret, but you can infer that it's running validation twice per epoch. \n",
        "\n",
        "There are also some statements related to early stopping, and if you look at the validation loss at each checkpoint, you can make sense of the early stopping behavior.\n",
        "\n",
        "I'm hoping/assuming that `wandb` integration will make this much easier to look at!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDTTZ6n7DECX",
        "outputId": "1a22d3ef-8df4-4206-e068-f1e4eb565175",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "a0217362b68248caae1e32eba5f590d0",
            "e92d643fcfaf4386ba3e99203fba6ba2",
            "ca4f6b85a82347d7ada65e9d9788c1bb",
            "8d8488278d21445083a8f6b866b6f2b1",
            "f36a8009b1394d14be88a6c4fc146f7f",
            "952fc1768a7e49edb0967cac284b7a02",
            "13e052de1c9e48d3a4440a8db133f39e",
            "5cdf14ee303445cba33bf5700f9bd95c",
            "d5e843e9933f44ac800332c4f265e9ef",
            "b26e72d7d1204ee8ab330bfcff430813",
            "62caa165322a47f891dabc673cba7e4b",
            "99444d51ea784b3881ead3fc6758368d",
            "028c345c3f74468c8c9c7221d9b5cb83",
            "e2120246b52c43e98b9f25a8956e68a9",
            "319d119e73f943b3992be4de22730cf2",
            "30689fcc0b7944ff92fd1c8fff6ccb58",
            "43efda485b2a4e14a9667917e018ae64",
            "08372113005d4a15ab57fdc4d5609860",
            "08d476787f1e439f85ea0d9d2ba3f6e2",
            "6e10576df90e45ffb4a075960ecf9e4f",
            "cba26b164789400c8c7a16af4d36cd3b",
            "26606c2d42b74f3e9bf11111a6b7c401",
            "3a6b7223394245619dd495c27388a4d4",
            "2cf94a5f09f141b1b2f0a012893203dc",
            "5befd1e148de4739ba71439f4b81ce8a",
            "90ca371079584dbabc995d8d33827bb0",
            "d7365944432a466c86e49ad3bc273240",
            "40b358b5e8f04a019965222e429da6f9",
            "0c54e80753a54b2296296493ff2389fa",
            "8272a2cb97954fdb83c6496212159002",
            "c18060ecce45440bbbda31f0ea218959",
            "68ce982d1ebc4b3397576f7276ca6141",
            "052fabdc27d24c319e7d6a393697f33d",
            "2400278360fb43448dc59e6815c7588a",
            "48e0a33f3b024ffbb400ce07d2435de6",
            "10bd5763b31d415296667efcfd7a8eab",
            "6a3048f75c3b4265859cde0a596f1434",
            "b94dfefdc0d24a41bb80157f2384327f",
            "aaba6d6d92994a3cbeb34ab5809f648f",
            "9f447246a3584e1090a983e49e1bb024",
            "ff6527bef4e849c5a34af646392a559e",
            "6462999e1b974dd1a9070665a2288263",
            "74810eac2fdc4463b5e8026e01e407e7",
            "5ba4921f33f1484bbcdf7bb0cb24736e",
            "dd32ca2c7e164989ae66b4ffa2925ccb",
            "b397fd21e8d14d4abfba3641739a1b46",
            "350cec9add53401c81c9ca6cd6dbb769",
            "429d71d1986a442da0fab28bc21ba54e"
          ]
        }
      },
      "source": [
        "print('Training on {:,} samples...'.format(len(train_df)))\n",
        "\n",
        "# Train the model, testing against the validation set periodically.\n",
        "out = model.train_model(train_df, eval_df=validation_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on 7,695 samples...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:353: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a0217362b68248caae1e32eba5f590d0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=7695.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5e843e9933f44ac800332c4f265e9ef",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=4.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "43efda485b2a4e14a9667917e018ae64",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:231: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
            "  warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n",
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.36509899446693134, 'tp': 520, 'tn': 119, 'fp': 145, 'fn': 72, 'eval_loss': 0.5319398908971626}\n",
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
            "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4324878324210716, 'tp': 548, 'tn': 116, 'fp': 148, 'fn': 44, 'eval_loss': 0.4800978036962937}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4580448418199385, 'tp': 544, 'tn': 127, 'fp': 137, 'fn': 48, 'eval_loss': 0.4789898394145698}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5befd1e148de4739ba71439f4b81ce8a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5171962761082096, 'tp': 556, 'tn': 134, 'fp': 130, 'fn': 36, 'eval_loss': 0.5009610512964079}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5494103862626102, 'tp': 532, 'tn': 165, 'fp': 99, 'fn': 60, 'eval_loss': 0.4554820476430599}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5564186736070834, 'tp': 531, 'tn': 168, 'fp': 96, 'fn': 61, 'eval_loss': 0.45477539342697537}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "052fabdc27d24c319e7d6a393697f33d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5012425098989287, 'tp': 554, 'tn': 131, 'fp': 133, 'fn': 38, 'eval_loss': 0.5868310144292974}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5142091569607425, 'tp': 527, 'tn': 158, 'fp': 106, 'fn': 65, 'eval_loss': 0.5482129334933429}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 2\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5380886039281899, 'tp': 521, 'tn': 170, 'fp': 94, 'fn': 71, 'eval_loss': 0.5416202939019721}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ff6527bef4e849c5a34af646392a559e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 3 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.53437158532515, 'tp': 542, 'tn': 152, 'fp': 112, 'fn': 50, 'eval_loss': 0.585158633573033}\n",
            "INFO:simpletransformers.classification.classification_model: Patience of 2 steps reached\n",
            "INFO:simpletransformers.classification.classification_model: Training terminated.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Training of bert model complete. Saved to outputs/.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GjoegimYIPZa"
      },
      "source": [
        "## 3.4. Inspect generated files"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CAIOLClZtoxG"
      },
      "source": [
        "Helper function to print the contents of a directory, with file sizes in MB."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D87pElNqYqVu"
      },
      "source": [
        "import os \n",
        "import pandas as pd\n",
        "\n",
        "def list_files_info(data_dir):\n",
        "    '''\n",
        "    Prints out the files in a directory along with their sizes in MB.\n",
        "    '''\n",
        "\n",
        "    # Check out the sizes on the saved files.\n",
        "    files = list(os.listdir(data_dir))\n",
        "\n",
        "    print(data_dir)\n",
        "\n",
        "    rows = []\n",
        "\n",
        "    # For each file in the directory...\n",
        "    for f in files:\n",
        "        # Get the file size, in MB\n",
        "        f_size = float(os.stat(data_dir + '/' + f).st_size) / 2**20\n",
        "        \n",
        "        # Print the filename and its size.\n",
        "        print(\"     {:25s}    {:>8.2f} MB\".format(f, f_size))\n",
        "\n",
        "        rows.append([f, '{:.2f} MB'.format(f_size)])\n",
        "\n",
        "    print('')\n",
        "\n",
        "    return pd.DataFrame(rows, columns=['File', 'Size'])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aNjybmHcttGf"
      },
      "source": [
        "This cache folder stores the tokenized and encoded text data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cxMF67nqIbVC",
        "outputId": "e24de628-ff61-42d6-848e-71ab7a343130",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        }
      },
      "source": [
        "list_files_info('./cache_dir')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./cache_dir\n",
            "     cached_dev_bert_128_2_856        0.68 MB\n",
            "     cached_train_bert_128_2_7695        6.07 MB\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>File</th>\n",
              "      <th>Size</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cached_dev_bert_128_2_856</td>\n",
              "      <td>0.68 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>cached_train_bert_128_2_7695</td>\n",
              "      <td>6.07 MB</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           File     Size\n",
              "0     cached_dev_bert_128_2_856  0.68 MB\n",
              "1  cached_train_bert_128_2_7695  6.07 MB"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Jsgufn6t2zW"
      },
      "source": [
        "The `outputs` folder contains the final model, plus all of the checkpoints."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMIqYvSXI59a",
        "outputId": "63b98b90-b30e-4c82-bb96-9a70afc76d87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "list_files_info('./outputs/')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./outputs/\n",
            "     vocab.txt                        0.22 MB\n",
            "     checkpoint-120                   0.00 MB\n",
            "     checkpoint-482-epoch-2           0.00 MB\n",
            "     checkpoint-720                   0.00 MB\n",
            "     checkpoint-241-epoch-1           0.00 MB\n",
            "     checkpoint-840                   0.00 MB\n",
            "     special_tokens_map.json          0.00 MB\n",
            "     tokenizer_config.json            0.00 MB\n",
            "     checkpoint-360                   0.00 MB\n",
            "     checkpoint-480                   0.00 MB\n",
            "     eval_results.txt                 0.00 MB\n",
            "     model_args.json                  0.00 MB\n",
            "     checkpoint-723-epoch-3           0.00 MB\n",
            "     checkpoint-240                   0.00 MB\n",
            "     best_model                       0.00 MB\n",
            "     training_args.bin                0.00 MB\n",
            "     config.json                      0.00 MB\n",
            "     pytorch_model.bin              417.73 MB\n",
            "     checkpoint-600                   0.00 MB\n",
            "     training_progress_scores.csv        0.00 MB\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>File</th>\n",
              "      <th>Size</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>vocab.txt</td>\n",
              "      <td>0.22 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>checkpoint-120</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>checkpoint-482-epoch-2</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>checkpoint-720</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>checkpoint-241-epoch-1</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>checkpoint-840</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>special_tokens_map.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>tokenizer_config.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>checkpoint-360</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>checkpoint-480</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>eval_results.txt</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>model_args.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>checkpoint-723-epoch-3</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>checkpoint-240</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>best_model</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>training_args.bin</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>config.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>pytorch_model.bin</td>\n",
              "      <td>417.73 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>checkpoint-600</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>training_progress_scores.csv</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            File       Size\n",
              "0                      vocab.txt    0.22 MB\n",
              "1                 checkpoint-120    0.00 MB\n",
              "2         checkpoint-482-epoch-2    0.00 MB\n",
              "3                 checkpoint-720    0.00 MB\n",
              "4         checkpoint-241-epoch-1    0.00 MB\n",
              "5                 checkpoint-840    0.00 MB\n",
              "6        special_tokens_map.json    0.00 MB\n",
              "7          tokenizer_config.json    0.00 MB\n",
              "8                 checkpoint-360    0.00 MB\n",
              "9                 checkpoint-480    0.00 MB\n",
              "10              eval_results.txt    0.00 MB\n",
              "11               model_args.json    0.00 MB\n",
              "12        checkpoint-723-epoch-3    0.00 MB\n",
              "13                checkpoint-240    0.00 MB\n",
              "14                    best_model    0.00 MB\n",
              "15             training_args.bin    0.00 MB\n",
              "16                   config.json    0.00 MB\n",
              "17             pytorch_model.bin  417.73 MB\n",
              "18                checkpoint-600    0.00 MB\n",
              "19  training_progress_scores.csv    0.00 MB"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cLbK5JgBFkua"
      },
      "source": [
        "## 3.5. Evaluate on Test Set\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QarqhLuxfHN"
      },
      "source": [
        "Let's load the model from the checkpoint which performed best on the validation set. \n",
        "\n",
        "This is how we combat overfitting--the \"final\" model (at the end of all training epochs) will perform best on the training set, but may not generalize as well to new data. \n",
        "\n",
        "So, instead, we use an earlier checkpoint where the training loss was higher but the validation loss was at its lowest! "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "shHc3hNfh1OG"
      },
      "source": [
        "model = ClassificationModel(\n",
        "    \"bert\", \"outputs/best_model\"\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "apzWNC1pAHO7"
      },
      "source": [
        "**On Validation Set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0b1d39OEY4e",
        "outputId": "ebd0a925-abf9-47c6-ccc9-306c6fa7b2a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 154,
          "referenced_widgets": [
            "e306a70925a741f8bfd3ef5bb8afcfc1",
            "b0c23646eca4434f921fa5b751751cd3",
            "87b6e482e32749758d59d4367ffba5ff",
            "c382bccb09154300b5b27d188e23b8f3",
            "83e89768c9ce440990c0f701a84cde5c",
            "2ec8bf91d995470a865277e55af3a185",
            "0d8f9fee9db144429a779fc0371174e6",
            "379be505f54e492e8ade2b20e850c1fb"
          ]
        }
      },
      "source": [
        "# Evaluate the model\n",
        "result, model_outputs, wrong_predictions = model.eval_model(validation_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e306a70925a741f8bfd3ef5bb8afcfc1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=107.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5494103862626102, 'tp': 532, 'tn': 165, 'fp': 99, 'fn': 60, 'eval_loss': 0.4554820476430599}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u24h6JjSFsJ_",
        "outputId": "d0806385-f093-482d-e5be-f30c81f7fbff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(result)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'mcc': 0.5494103862626102, 'tp': 532, 'tn': 165, 'fp': 99, 'fn': 60, 'eval_loss': 0.4554820476430599}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GbE5RYSfAKM8"
      },
      "source": [
        "**On Test Set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsTOCd8fALl5",
        "outputId": "f08721b1-da28-4cb4-90ab-af73b238ff00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203,
          "referenced_widgets": [
            "3922df1e378f49d2ac4e7467b64c23d9",
            "5d635027ee964781ab0de68d40036146",
            "4e95fb47f52d4ad09a86817c2207c64b",
            "0cbc72ea08934f3ca9ee79d4ae7c2bfe",
            "fcf3517dde4240f1b7bb7726d2b3327d",
            "e59dc8b51e4e41f0a53c2d4c7f682cd8",
            "cf7f8f8df3a24e0fb2552b811f1183f4",
            "8223160dfc804c0385a052bce650aacf",
            "583de1f4b2ae4106843a9b951dcfa65b",
            "f160dc2ee1c045ac85d6a703b1ef5149",
            "e6ff7b6290374af3a4393f919c99ab38",
            "13e28414ac724952bfff2e360fea55df",
            "a685be1201234168897860810ad22a2c",
            "07ccf48ef30a458eaa3d18cb8ebe5bff",
            "b322a434b6f84f559bee32913c08aa7f",
            "42fd11888cff4fdc8b8c84caffdacb7d"
          ]
        }
      },
      "source": [
        "# Evaluate the model\n",
        "result, model_outputs, wrong_predictions = model.eval_model(df_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3922df1e378f49d2ac4e7467b64c23d9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=516.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "583de1f4b2ae4106843a9b951dcfa65b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=65.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4533677632398824, 'tp': 323, 'tn': 79, 'fp': 83, 'fn': 31, 'eval_loss': 0.5259761724907618}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfhF0PFffBPY"
      },
      "source": [
        "Without SimpleTransformers, original score: `Total MCC: 0.498`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zQBwmsNhAT90",
        "outputId": "3a9992d0-b4b4-415a-e256-3310095d5c8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('MCC: %.3f' % result['mcc'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MCC: 0.453\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gp5B7bc5hq9Q"
      },
      "source": [
        "**Run on New Text**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qu724mANFjbp",
        "outputId": "f82b220a-2e16-44c4-98dc-9524e6c7aff8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 152,
          "referenced_widgets": [
            "03f258785e294995a784a892296340fb",
            "9d964f3b67d0417fb2ebeadb2bf480a5",
            "37c134462cbc46afba7ede878ccee2b2",
            "1c14aebe54cb434a9c84d62d52e7a320",
            "750f5d40a00c41f4a4412e62ca8da9b8",
            "e34a0210085444c3bc5f77d2e3075e77",
            "95ccb1ec4da14fd3b1c403b0a7c77646",
            "969c6a6876e14c36b6f72c8a1d7190be",
            "4ad125be6be44646b35574c89eae57be",
            "dc72d4e0668f4199828d56cdb8e2dd09",
            "ff7f262134004289a75620820fc7f4b7",
            "fdb24003d6314643be1b3cd355a267da",
            "b1120ef4bb8744009ad24eed77760992",
            "5afb32021e9845c39798d91606fbfa70",
            "9d8ab33d50b4411b87e538f7ab1b853d",
            "ae7b1adfea65451b92a0928b3de47579"
          ]
        }
      },
      "source": [
        "predictions, raw_outputs = model.predict([\"Some arbitary sentence\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03f258785e294995a784a892296340fb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4ad125be6be44646b35574c89eae57be",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKzLS9ohzGVu"
      },
      "source": [
        "# S4. Conclusion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FL_NnDGxRpEI"
      },
      "source": [
        "## 4.1. Observations\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vv_uUVM8bcd3"
      },
      "source": [
        "**Less Code, More Documentation**\n",
        "\n",
        "* No more:\n",
        "    * Tokenization code\n",
        "    * PyTorch taining loop code\n",
        "\n",
        "* Instead --> Careful argument selection!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fu0sZSOse_qb"
      },
      "source": [
        "\n",
        "**Colab Compatibility**\n",
        "\n",
        "* Alarming install errors, but no problems yet...\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwDtGNIjfAtW"
      },
      "source": [
        "**Checkpoints**\n",
        "\n",
        "* Automatically saves checkpoints!\n",
        "* Loading back a model from a checkpoint is trivial!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkNa5MBlfCI-"
      },
      "source": [
        "\n",
        "**Early Stopping**\n",
        "\n",
        "* Harder than it sounds:\n",
        "    * You must specify:\n",
        "        * eval frequency\n",
        "        * minimum \"delta\"        \n",
        "        * \"patience\"\n",
        "\n",
        "* Didn't immediately improve our CoLA score...\n",
        "     * But we're not done exploring the technique yet!\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--wEDqN9dExi"
      },
      "source": [
        "## 4.2. Next Steps"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8DT0bZZUdGMF"
      },
      "source": [
        "**Weights & Biases - \"wandb\"**\n",
        "\n",
        "* Set this up and try it out!\n",
        "    * Does it plot the learning curve for us?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jdqzhWoYfDdU"
      },
      "source": [
        "\n",
        "**Make Early Stopping Work**\n",
        "\n",
        "* Is it easier to apply to a different dataset?\n",
        "* Was the final model better than the \"best model\"?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rb5-Jgu-fEPH"
      },
      "source": [
        "\n",
        "**Resume Training from Checkpoint**\n",
        "\n",
        "* Checkpoints should allow us to resume training in case we get disconnected from Colab...\n",
        "    * How do you resume?\n",
        "    * Save our checkpoints to GDrive (otherwise they'll be lost!)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KkM5QS7-BWrU"
      },
      "source": [
        "# A.1. Over-Fitting from Original Notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6O_NbXFGMukX",
        "outputId": "d2fa4035-f435-4fcb-ba2a-ee1cf0bcf501",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Display floats with two decimal places.\n",
        "pd.set_option('precision', 2)\n",
        "\n",
        "# Create a DataFrame from our training statistics.\n",
        "#df_stats = pd.DataFrame(data=training_stats)\n",
        "df_stats = pd.read_csv(\"/content/outputs/training_progress_scores.csv\")\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('global_step')\n",
        "\n",
        "# A hack to force the column headers to wrap.\n",
        "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
        "\n",
        "# Display the table.\n",
        "df_stats"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tp</th>\n",
              "      <th>tn</th>\n",
              "      <th>fp</th>\n",
              "      <th>fn</th>\n",
              "      <th>mcc</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>eval_loss</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>global_step</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>120</th>\n",
              "      <td>520</td>\n",
              "      <td>119</td>\n",
              "      <td>145</td>\n",
              "      <td>72</td>\n",
              "      <td>0.37</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>240</th>\n",
              "      <td>548</td>\n",
              "      <td>116</td>\n",
              "      <td>148</td>\n",
              "      <td>44</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.59</td>\n",
              "      <td>0.48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>544</td>\n",
              "      <td>127</td>\n",
              "      <td>137</td>\n",
              "      <td>48</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.61</td>\n",
              "      <td>0.48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>556</td>\n",
              "      <td>134</td>\n",
              "      <td>130</td>\n",
              "      <td>36</td>\n",
              "      <td>0.52</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>480</th>\n",
              "      <td>532</td>\n",
              "      <td>165</td>\n",
              "      <td>99</td>\n",
              "      <td>60</td>\n",
              "      <td>0.55</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>482</th>\n",
              "      <td>531</td>\n",
              "      <td>168</td>\n",
              "      <td>96</td>\n",
              "      <td>61</td>\n",
              "      <td>0.56</td>\n",
              "      <td>0.21</td>\n",
              "      <td>0.45</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>600</th>\n",
              "      <td>554</td>\n",
              "      <td>131</td>\n",
              "      <td>133</td>\n",
              "      <td>38</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.22</td>\n",
              "      <td>0.59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>720</th>\n",
              "      <td>527</td>\n",
              "      <td>158</td>\n",
              "      <td>106</td>\n",
              "      <td>65</td>\n",
              "      <td>0.51</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>723</th>\n",
              "      <td>521</td>\n",
              "      <td>170</td>\n",
              "      <td>94</td>\n",
              "      <td>71</td>\n",
              "      <td>0.54</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>840</th>\n",
              "      <td>542</td>\n",
              "      <td>152</td>\n",
              "      <td>112</td>\n",
              "      <td>50</td>\n",
              "      <td>0.53</td>\n",
              "      <td>0.26</td>\n",
              "      <td>0.59</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              tp   tn   fp  fn   mcc  train_loss  eval_loss\n",
              "global_step                                                \n",
              "120          520  119  145  72  0.37        0.59       0.53\n",
              "240          548  116  148  44  0.43        0.59       0.48\n",
              "241          544  127  137  48  0.46        0.61       0.48\n",
              "360          556  134  130  36  0.52        0.26       0.50\n",
              "480          532  165   99  60  0.55        0.49       0.46\n",
              "482          531  168   96  61  0.56        0.21       0.45\n",
              "600          554  131  133  38  0.50        0.22       0.59\n",
              "720          527  158  106  65  0.51        0.19       0.55\n",
              "723          521  170   94  71  0.54        0.04       0.54\n",
              "840          542  152  112  50  0.53        0.26       0.59"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-G03mmwH3aI"
      },
      "source": [
        "Notice that, while the the training loss is going down with each epoch, the validation loss is increasing! This suggests that we are training our model too long, and it's over-fitting on the training data. \n",
        "\n",
        "(For reference, we are using 7,695 training samples and 856 validation samples).\n",
        "\n",
        "Validation Loss is a more precise measure than accuracy, because with accuracy we don't care about the exact output value, but just which side of a threshold it falls on. \n",
        "\n",
        "If we are predicting the correct answer, but with less confidence, then validation loss will catch this, while accuracy will not."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68xreA9JAmG5",
        "outputId": "09ad07ba-fd22-4c12-8a88-4caebac6f16f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats['train_loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['eval_loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU5fn//9dMMpnsyWQlZCdkIwmrgCyKgEAUBEUQRFkU19pP/dmPVazaVvvx2xbXqlWrorKKbLIKYgGpIDsIBpIAgWwEQsi+zmRmzu+PJCMhARNIcrJcz8eDh+bMnDPXDCF5zz3Xfd8aRVEUhBBCCCGEEKrRql2AEEIIIYQQXZ2EciGEEEIIIVQmoVwIIYQQQgiVSSgXQgghhBBCZRLKhRBCCCGEUJmEciGEEEIIIVQmoVwI0WllZ2cTHR3Ne++9d93XmDdvHtHR0S1YVed1tdc7OjqaefPmNeka7733HtHR0WRnZ7d4fWvWrCE6Opp9+/a1+LWFEOJG2atdgBCi62hOuN22bRtBQUGtWE3HU1FRwUcffcQ333zDxYsX8fLyYsCAAfzmN78hIiKiSdf43e9+x7fffsvatWuJjY1t9D6KojB69GhKSkrYtWsXjo6OLfk0WtW+ffvYv38/s2fPxt3dXe1yGsjOzmb06NE88MAD/OlPf1K7HCFEOyKhXAjRZubPn1/v60OHDvHVV18xbdo0BgwYUO82Ly+vG368wMBAjh07hp2d3XVf469//SuvvPLKDdfSEl566SU2bdrEhAkTGDRoEHl5eWzfvp2jR482OZRPmTKFb7/9ltWrV/PSSy81ep+9e/dy7tw5pk2b1iKB/NixY2i1bfPB7P79+3n//fe55557GoTySZMmMX78eHQ6XZvUIoQQzSGhXAjRZiZNmlTva4vFwldffUXfvn0b3HalsrIyXF1dm/V4Go0GvV7f7Dov114CXGVlJVu2bGH48OG8+eabtuO//e1vMZlMTb7O8OHDCQgIYMOGDTz33HM4ODg0uM+aNWuAmgDfEm7076Cl2NnZ3dAbNCGEaE3SUy6EaHdGjRrFzJkzOXHiBHPnzmXAgAFMnDgRqAnnb7/9NlOnTmXw4MHEx8czZswY3njjDSorK+tdp7Ee58uP7dixg3vvvZeEhASGDx/OP/7xD8xmc71rNNZTXnestLSUP//5zwwZMoSEhASmT5/O0aNHGzyfwsJCXnjhBQYPHky/fv2YNWsWJ06cYObMmYwaNapJr4lGo0Gj0TT6JqGxYH01Wq2We+65h6KiIrZv397g9rKyMrZu3UpUVBS9e/du1ut9NY31lFutVv79738zatQoEhISmDBhAuvXr2/0/LS0NP7yl78wfvx4+vXrR58+fZg8eTIrV66sd7958+bx/vvvAzB69Giio6Pr/f1frae8oKCAV155hREjRhAfH8+IESN45ZVXKCwsrHe/uvP37NnDggULuP3224mPj2fcuHF8/fXXTXotmiMlJYWnnnqKwYMHk5CQwJ133sknn3yCxWKpd7/z58/zwgsvMHLkSOLj4xkyZAjTp0+vV5PVauWLL77grrvuol+/fvTv359x48bxxz/+kerq6havXQjRfDJSLoRol3Jycpg9ezaJiYmMHTuWiooKAHJzc1m1ahVjx45lwoQJ2Nvbs3//fj799FOSk5NZsGBBk66/c+dOli1bxvTp07n33nvZtm0bn332GR4eHjzxxBNNusbcuXPx8vLiqaeeoqioiM8//5zHHnuMbdu22Ub1TSYTDz30EMnJyUyePJmEhARSU1N56KGH8PDwaPLr4ejoyN13383q1avZuHEjEyZMaPK5V5o8eTIffvgha9asITExsd5tmzZtoqqqinvvvRdoudf7Sn/7299YtGgRAwcOZM6cOeTn5/Pqq68SHBzc4L779+/n4MGD3HbbbQQFBdk+NXjppZcoKCjg8ccfB2DatGmUlZXx3Xff8cILL2AwGIBrz2UoLS3l/vvvJyMjg3vvvZdevXqRnJzMl19+yd69e1m5cmWDT2jefvttqqqqmDZtGg4ODnz55ZfMmzePkJCQBm1Y1+vnn39m5syZ2Nvb88ADD+Dj48OOHTt44403SElJsX1aYjabeeihh8jNzWXGjBmEhYVRVlZGamoqBw8e5J577gHgww8/5N1332XkyJFMnz4dOzs7srOz2b59OyaTqd18IiREl6YIIYRKVq9erURFRSmrV6+ud3zkyJFKVFSUsmLFigbnGI1GxWQyNTj+9ttvK1FRUcrRo0dtx7KyspSoqCjl3XffbXCsT58+SlZWlu241WpVxo8frwwbNqzedZ9//nklKiqq0WN//vOf6x3/5ptvlKioKOXLL7+0HVuyZIkSFRWlfPDBB/XuW3d85MiRDZ5LY0pLS5VHH31UiY+PV3r16qVs2rSpSeddzaxZs5TY2FglNze33vH77rtPiYuLU/Lz8xVFufHXW1EUJSoqSnn++edtX6elpSnR0dHKrFmzFLPZbDuelJSkREdHK1FRUfX+bsrLyxs8vsViUR588EGlf//+9ep79913G5xfp+77be/evbZjb731lhIVFaUsWbKk3n3r/n7efvvtBudPmjRJMRqNtuMXLlxQ4uLilGeeeabBY16p7jV65ZVXrnm/adOmKbGxsUpycrLtmNVqVX73u98pUVFRyo8//qgoiqIkJycrUVFRyscff3zN6919993KHXfc8av1CSHUI+0rQoh2ydPTk8mTJzc47uDgYBvVM5vNFBcXU1BQwNChQwEabR9pzOjRo+ut7qLRaBg8eDB5eXmUl5c36Rpz5syp9/XNN98MQEZGhu3Yjh07sLOzY9asWfXuO3XqVNzc3Jr0OFarlaeffpqUlBQ2b97MrbfeyrPPPsuGDRvq3e/ll18mLi6uST3mU6ZMwWKxsHbtWtuxtLQ0fvrpJ0aNGmWbaNtSr/fltm3bhqIoPPTQQ/V6vOPi4hg2bFiD+zs7O9v+32g0UlhYSFFREcOGDaOsrIwzZ840u4Y63333HV5eXkybNq3e8WnTpuHl5cV//vOfBufMmDGjXsuQv78/4eHhpKenX3cdl8vPz+fIkSOMGjWKmJgY23GNRsOTTz5pqxuwfQ/t27eP/Pz8q17T1dWV3NxcDh482CI1CiFanrSvCCHapeDg4KtOylu6dCnLly/n9OnTWK3WercVFxc3+fpX8vT0BKCoqAgXF5dmX6OuXaKoqMh2LDs7Gz8/vwbXc3BwICgoiJKSkl99nG3btrFr1y5ef/11goKC+Oc//8lvf/tbnnvuOcxms61FITU1lYSEhCb1mI8dOxZ3d3fWrFnDY489BsDq1asBbK0rdVri9b5cVlYWAD169GhwW0REBLt27ap3rLy8nPfff5/Nmzdz/vz5Buc05TW8muzsbOLj47G3r//r0N7enrCwME6cONHgnKt975w7d+6667iyJoCePXs2uK1Hjx5otVrbaxgYGMgTTzzBxx9/zPDhw4mNjeXmm28mMTGR3r172877/e9/z1NPPcUDDzyAn58fgwYN4rbbbmPcuHHNmpMghGg9EsqFEO2Sk5NTo8c///xz/v73vzN8+HBmzZqFn58fOp2O3Nxc5s2bh6IoTbr+tVbhuNFrNPX8pqqbmDhw4ECgJtC///77PPnkk7zwwguYzWZiYmI4evQor732WpOuqdfrmTBhAsuWLePw4cP06dOH9evX061bN2655Rbb/Vrq9b4R//u//8v333/Pfffdx8CBA/H09MTOzo6dO3fyxRdfNHij0NraannHpnrmmWeYMmUK33//PQcPHmTVqlUsWLCARx55hD/84Q8A9OvXj++++45du3axb98+9u3bx8aNG/nwww9ZtmyZ7Q2pEEI9EsqFEB3KunXrCAwM5JNPPqkXjv773/+qWNXVBQYGsmfPHsrLy+uNlldXV5Odnd2kDW7qnue5c+cICAgAaoL5Bx98wBNPPMHLL79MYGAgUVFR3H333U2ubcqUKSxbtow1a9ZQXFxMXl4eTzzxRL3XtTVe77qR5jNnzhASElLvtrS0tHpfl5SU8P333zNp0iReffXVerf9+OOPDa6t0WiaXcvZs2cxm831RsvNZjPp6emNjoq3trq2qtOnTze47cyZM1it1gZ1BQcHM3PmTGbOnInRaGTu3Ll8+umnPPzww3h7ewPg4uLCuHHjGDduHFDzCcirr77KqlWreOSRR1r5WQkhfk37ersvhBC/QqvVotFo6o3Qms1mPvnkExWrurpRo0ZhsVhYtGhRveMrVqygtLS0SdcYMWIEULPqx+X94nq9nrfeegt3d3eys7MZN25cgzaMa4mLiyM2NpZvvvmGpUuXotFoGqxN3hqv96hRo9BoNHz++ef1lvc7fvx4g6Bd90bgyhH5ixcvNlgSEX7pP29qW83tt99OQUFBg2utWLGCgoICbr/99iZdpyV5e3vTr18/duzYwcmTJ23HFUXh448/BmDMmDFAzeoxVy5pqNfrba1Bda9DQUFBg8eJi4urdx8hhLpkpFwI0aEkJiby5ptv8uijjzJmzBjKysrYuHFjs8JoW5o6dSrLly/nnXfeITMz07Yk4pYtWwgNDW2wLnpjhg0bxpQpU1i1ahXjx49n0qRJdOvWjaysLNatWwfUBKx//etfREREcMcddzS5vilTpvDXv/6VH374gUGDBjUYgW2N1zsiIoIHHniAJUuWMHv2bMaOHUt+fj5Lly4lJiamXh+3q6srw4YNY/369Tg6OpKQkMC5c+f46quvCAoKqte/D9CnTx8A3njjDe666y70ej2RkZFERUU1WssjjzzCli1bePXVVzlx4gSxsbEkJyezatUqwsPDW20EOSkpiQ8++KDBcXt7ex577DFefPFFZs6cyQMPPMCMGTPw9fVlx44d7Nq1iwkTJjBkyBCgprXp5ZdfZuzYsYSHh+Pi4kJSUhKrVq2iT58+tnB+55130rdvX3r37o2fnx95eXmsWLECnU7H+PHjW+U5CiGap33+FhNCiKuYO3cuiqKwatUqXnvtNXx9fbnjjju49957ufPOO9UurwEHBwcWLlzI/Pnz2bZtG5s3b6Z379588cUXvPjii1RVVTXpOq+99hqDBg1i+fLlLFiwgOrqagIDA0lMTOThhx/GwcGBadOm8Yc//AE3NzeGDx/epOveddddzJ8/H6PR2GCCJ7Te6/3iiy/i4+PDihUrmD9/PmFhYfzpT38iIyOjweTK119/nTfffJPt27fz9ddfExYWxjPPPIO9vT0vvPBCvfsOGDCAZ599luXLl/Pyyy9jNpv57W9/e9VQ7ubmxpdffsm7777L9u3bWbNmDd7e3kyfPp3/+Z//afYusk119OjRRleucXBw4LHHHiMhIYHly5fz7rvv8uWXX1JRUUFwcDDPPvssDz/8sO3+0dHRjBkzhv3797NhwwasVisBAQE8/vjj9e738MMPs3PnThYvXkxpaSne3t706dOHxx9/vN4KL0II9WiUtpilI4QQoh6LxcLNN99M7969r3sDHiGEEJ2H9JQLIUQra2w0fPny5ZSUlDS6LrcQQoiuR9pXhBCilb300kuYTCb69euHg4MDR44cYePGjYSGhnLfffepXZ4QQoh2QNpXhBCila1du5alS5eSnp5ORUUF3t7ejBgxgqeffhofHx+1yxNCCNEOSCgXQgghhBBCZdJTLoQQQgghhMoklAshhBBCCKEymeh5hfz8MrVLEEIIIYQQnZBWq8FgcGn0NgnlV7BapcVeCCGEEEK0LWlfEUIIIYQQQmUSyoUQQgghhFCZhHIhhBBCCCFUJqFcCCGEEEIIlUkoF0IIIYQQQmUSyoUQQgghhFCZhHIhhBBCCCFUJqFcCCGEEEIIlUkoF0IIIYQQQmWyo6cQndSe4xdYszON/BIj3u56Jo+IYEhcN7XLEkIIIUQjJJQL0QntOX6BhZtTMJmtAOSXGFm4OQVAgrkQQgjRDkn7ihCd0JqdabZAXsdktrJmZ5pKFQkhhBDiWiSUC9EJ5ZcYm3VcCCGEEOpSNZSbTCZef/11hg8fTu/evbnvvvvYs2dPk8/fsGEDU6ZMoW/fvgwaNIgHH3yQY8eOtWLFQnQM3u76Zh0XQgghhLpUDeXz5s1j4cKFTJw4kRdffBGtVsujjz7KkSNHfvXct99+m3nz5hEZGcmLL77IU089RXBwMHl5eW1QuRDt2+QREdjbaeodc7DXMnlEhEoVCSGEEOJaNIqiKGo88LFjx5g6dSovvPACc+bMAcBoNDJhwgT8/PxYunTpVc89fPgwM2bM4L333mPMmDEtWldeXmmLXk8ItSz77iT/OZRt+3ryreFMGBquYkVCCCFE16bVavD2dm38tjauxWbLli3odDqmTp1qO6bX65kyZQqHDh3i4sWLVz130aJFJCQkMGbMGKxWK+Xl5W1RshAdSoi/GwB/njMQJ70dGbllKlckhBBCiKtRLZQnJycTHh6Oi4tLveO9e/dGURSSk5Oveu6ePXtISEjgrbfeYsCAAfTv359Ro0axfv361i5biA6jymQGwMtdz+gBwRxKzSM7T4K5EEII0R6pFsrz8vLw8/NrcNzX1xfgqiPlxcXFFBUVsWnTJlatWsWzzz7LW2+9Rbdu3fjDH/7Ad99916p1C9FRVJksADg62DN2YDB6Bzs2/piublFCCCGEaJRqmwdVVVWh0+kaHNfra1aHMBobX7qtoqICgKKiIlasWEGfPn0AGDNmDGPGjOFf//rXDfWZ+/q6Xfe54uq+P5TFos3JXCqsxMfgxKw7YrltQLDaZXVqGjst9nZaugd4AHDX8B6s3nGKOXdBsL98nwshhBDtiWqh3NHRkerq6gbH68J4XTi/Ut3xoKAgWyAHcHBwYNy4cSxatIjy8vIGbTFNJRM9W96Vu0vmFVby3oqfKCmtkt0lW1FBcSWODna27+nh8f6s/yGNxZuO8+hdcSpXJ4QQQnQ97XKip6+vb6MtKnVLGjbW2gLg6emJg4MDPj4+DW7z8fFBURTKyqRvtj252u6SK7afJudSOQUlVVRUVWOxWq9yBXE9qowWHB3sbF+7Ozswsl8ge0/kkltYoWJlQgghhLiSaiPlMTExLF68uMGo9tGjR223N0ar1RIbG0tubm6D2y5cuICdnR0eHh6tU7S4LlfbRbK43MRLn+6rd0xnr0Wvs8PRoe6PPXoHOxxrj+kvP1Z3XG9/xTl26B3scXSww8Fei0ajafTxO7sqk7leKAdIHBTC9sPn2PRjBg+Pj1WpMiGEEEJcSbVQnpiYyGeffcbKlStt65SbTCbWrFlD//798ff3ByAnJ4fKykoiIiLqnfuPf/yD3bt3M2zYMADKysrYvHkz/fr1w9HRsc2fj7g6b3d9o8HczVnHjNujqDKZMZosVJksVFVbav/fXPO1yUKl0UxhqRHjZccs1qYtr6/RUBPSdVcP+L8E+dqvG3kD4FT7tV5nh72dqntuNVmVyYKjQ/1/4h6uekb06c72w+e4a1gYvp5OKlUnhOgI9l84zPq0LRQaizDoPZkYkcigbv3VLkuITkm1zYMAnn76abZt28bs2bMJCQnh66+/JikpiYULFzJgwAAAZs6cyf79+0lNTbWdV1lZyeTJk8nNzWXOnDm4u7uzevVqzp49W+/c6yE95S3vyp5yqNldcvYdMdfdU15ttmKstlBlNFNVXRPULw/zxtpjVbXH6kJ/4+fUHG8qezvtFUG+Luj/MoJ/tYDf2BsAB13Lj+bvOX6BzzYlY7EqeLvrmTwiwvZaF5Yaef6jHxmWEMDsxMY/kRJCiP0XDrMsZTXV1l/mf+m0OmbE3CvBXHQKarzpvFZPuWoj5QDz58/nnXfeYd26dRQXFxMdHc3HH3/8q6HaycmJRYsWMX/+fJYsWUJVVRVxcXF8/vnnNxTIReuoC4NrdqaRX2JsEBKvh85ei85ei6tTwxV8rodVUTDWhnPjZWH+WgHf9nW1hUqThaIyU71zzJYmjuZDzSh8o0H+sjYenR2O+qu8Abjs2NHTl1j8bart04T8EiMLN6cANX8XBjc9t/Tpzn9/ymHCkDC8PeSTJSFEQ+vTttQL5ADV1mrWpW2WUC46vCvfdBYai1iWshpAte9vVUfK2yMZKRctxWyxNgzy1RaqjBaM1WbbSH3lZaP8l78BsI38XzaqfyO83fW8/puadq/84irm/XsPt/btzsyx0S3xdIUQncxT25+76m0GvSfhHiGEu4cQ5hFKsGt3dHYtM0giRGurNFfxlz3/oKy64Y7wBr0n/zfsj6322O12pFyIzszeTourU8uO5pvqjeQ3HuS/3Haq0fMv7+v39nBkWEIAPxytGS03uDW+BKkQomsqqCrETmOHRWk4GOBk70S4RwjpJVkcvngMADuNHUFu3Ql3/yWoezsauuxEe9G+VJmrSCvO4FRhGicL08gszUah8THpQmNRG1f3CwnlQnQQWo2mti/dnmutL7T1QGajE2u93esH7/FDQtl17Dyb92Uw4/aoFq5WCNFRJV1KZtGJr9CgaRDMdVod90VNsn28X2wsIb0kk7PFmaSXZPJjzn6+z94NgJvOlbDa0fRwjxBC3IJwtJd2OdH6jBYTZ4rSOVmUxqnCNDJKs7EqVuw0doS5B5MYNopd5/ZRWt1wCW2D3lOFimtIKBeik5k8IqLRibWTR0TUu5+vpxND47ux86ccxt8cioerjJYL0ZVZrBY2nt3K1owdBLl2Z278g6SXZF5zIpyH3p0+vvH08Y23XSOnPJf0kgxbUP/50gkANGjo7tqNMPdfgrqfsy9aTcdY0Uq0XyZLNWeK0zlVdIaThWlklGRhUSxoNVrC3IMZE3IbUYYIeniE4mDnAICfs2+jE5knRiSq9TSkp/xK0lMuOoM9xy80aWJtbmEFf/x4L2MHBjNtVKQKlQoh2oNiYwmfH1/GqaIzDOs+mCmRE3FooR7x8uoK0kuySC/O4GxJJuklWVSaKwFwsnckzD2EMPfgmv96hOCqu74duUXXUW2p5mxJZk07SlEa6cWZmGtDeIhbEJGePWpDeBiO9lcfcGpvq69IKL+ChHLR1Xyy4TiHTuYx/8mhuDs7qF2OEKKNnSw8zWfHl2E0G7m/DZY7tCpWLlZcqgnotUE9p+yCrcfXz8nH1vYS5hFCoEsAdlq7X7mq6MyqrWYySrI4WXiaU4VnOFOSgdlqRoOGYLdAIg09iPKMIMIzHKd23iIlobwZJJSLruZ8fjkvfbKPO24OZcptEb9+ghCiU7AqVr5N38Gms1vxd/blkYSZBLj4q1JLldlIZmk26cWZnC3J5GxJBqWmmn5fnVZHiFvQZau9hOCpl527OzOz1UxmaTYnaydmninOoNpajQYNQa4BRBoiiDJEEOERjrOuY22CJ6G8GSSUi67oo3VJHE3L5/Unh7bYajFCiParzFTOFye+JLngJAP9+zE9evI1P+Zva4qiUFBVVNObXpJJenEmWaXnMNdOOjXoPetNIg12DZQlGTswi9VCZuk5WztKWtFZTLW93oGuAUR5RhBpiKCnZzguOmeVq70xEsqbQUK56Iqy88r404L93DU0jHtu7aF2OUKIVnSmOJ0FSUspqy5nauREhnUf3CGWLqy2mskuzald7SWD9JJM8qsKgdolGV27E+ZR058e7h6Kj5NXh3heXZHFaiG7LKdmJLw2hBstJgACXPyJMkQQ5RlBT88euDp0rjkGEsqbQUK56Kr+9fXPnEgv4PUnh+LsKCNOQnQ2iqKwPesH1qZ9g5fek7kJDxLiFqR2WTek2FhKekmmLahnlGZjqg13rjqXmpVePEIIcw8h1D243fcbd1ZWxWoL4acKz3C66CxVlioAujn72dpRIj174ObQeGDtLCSUN4OEctFVZeaW8pfPD3D38HAmDg9XuxwhRAuqqK5kScpKjuYl0cc3npmxU3Gy71i9uE1hsVo4X55ra3k5W5JJbsVFoGZJxgAX/3pBvZuLnyzJ2AqsipWcsgu164Sf4VTRGduKO37OPrZ2lEjPCDz0bipX27YklDeDhHLRlb276hinsouY/+RQnPSyjYEQnUFmaTYLfl5CgbGIe3qOZ2TQ8C7V1lFRuyTj2doR9fTiTCpqA6KjnWPNcox1k0jdQzpdu0RbsCpWLpRftLWjnC48Q7m5AgAfJ+/aEF6zTGFXn6QrobwZJJSLruzs+RL+uvAg947owfghYWqXI4S4AYqisCtnH6tOrcdV58Lc+Afp4RGqdlmqUxSFi5WXbCPp6cUZnCu/gFWp2XDN18mbMPdQ22ovga6yJOOVFEUht+KibXWUU0VnKKsuB8Db0VDTjuJZ05JicFRvh8z2SEJ5M0goF13d2yuOcvZ8CfOfHIKjg4yWC9ERVZmNfJm6moO5P9HLK5rZvabLCPA1GC0mMkuya3rTa/vTS0w1eUCntSfYLci2HGO4e0iXC5qKonCxIo+TRWdsK6TULVlp0HvW9IMbIojy7IG3k5fK1bZvEsqbQUK56OpOnyvm/y0+xH0je5I4OETtcoQQzXS+PJdPf15MbkUeE3qMZWzoSOmbbiZFUSg0FnG2uG4SaSZZpdm2JRk99R71etND3IJabAfU9kBRFPIq8zlVlGabnFlsKgHAw8G9ZnWU2j/ejrLKTXNIKG8GCeVCwJvLj5B1sYx/PDkUvU4+thWio9h3/hDLU9egt9fzcNwMogw91S6p06i2mjlXllMvqOdXFQCg1WgJcg2wtb2EuYfg6+TdocLqpcqC2laUmiBeZCwGwN3BzbYySpQhAl8nnw71vNobCeXNIKFcCDiZVcTflx5m+uhIxg4MVrscIcSvMFmqWXVqHbtz9hPp2YOH4mbgoXdXu6xOr8RU+ktvekkWGSWZtvW2XXTOtZNHa4J6qHtQu1rxpqCqsF5PeEHtmu+uOpfaEF4zEu7v7CshvAVJKG8GCeVC1Ji/7DDnCyqY/8QQdPYyWi5Ee3Wx4hILkpaQXZbDuNBRjA8fIxMTVWJVrJwvz7UF9bMlmVwozwVqlmTs5uJ3WW96aIMlGfdfOMz6tC0UGosw6D2ZGJHIoG79W6S2ImPxLyG8MI1LtaP8LjpnIutWR/GMIMDFX0J4K5JQ3gwSyvFzkxoAACAASURBVIWokZxRyOtfHuGBMVGMHtCxNxgRorM6cvFnliSvwE5jx6xe04j3iVW7JHGFiupKMkqzLlvtJdO2XKCjnZ4Q92DC3UMwW6v577m9VNduLw+g0+qYEXPvdQXzYmOJbVLmycI08irzAXC2dyLSs4dtw54AF3+Zc9CGJJQ3g4RyIWooisLflx7mUnEVf398CDp7+aEtRHthtppZe/obdmTvIsw9hLnxD+DlaFC7LNEENZMoL/3Sm16Sybmy87YlGa9k0Hvyf8P++KvXLTGV1obwmhVScivyAHCyd6SnZ3jtWuE9CXTtJiFcRRLKm0FCuRC/SDqbz1tfHWXWuGhu6xeodjlCCGp6gRckLSW9JJORwcO5O+JO7LWyfGlHZrKYeGbnS1e9fXav6Q1Gy8tM5bYdM08WpdnaZBzt9ER4htesjuIZQZBbdwnh7ci1Qrn8KxZCXFVcmBc9uruzaU8Gw3sHYG8nP9iFUFPSpWQWnfgKi2LlkfiZ9PNLULsk0QIc7Bww6D0pNBY1evuylNUYLSbcHFxtPeE55Rds5/b0CGdwt/5EGSIIdg2UOQUdlIyUX0FGyoWo71jaJd5ZeYyH7ojhlj7d1S5HiC7JYrWw8exWtmbsIMi1O3PjH8TP2UftskQL2n/hMMtSVtfrKW+MTqsjwiPMtmFPqFuQhPAOREbKhRDXLaGHN6Hd3Ni4J52hCd2w08pouRBtqdhYwufHl3Gq6AzDug9iSuSkTrVRjahR156y8MTyq97n9/1/Q6h7kLQrdVLy21UIcU0ajYaJQ8PIK6pi34lctcsRoks5WXiavx14h4ySLGbFTmNGzBQJ5J3YoG79Meg9G73NoPckwjNMAnknJqFcCPGr+kb6EOznyoYfM7BapeNNiNZmVaxsPruNd498gou9M3+46X8YHDBA7bJEG5gYkYhOW/+Nl06rY2JEokoVibYib7dEm2jNDRFE69NoNNw1NIwP1iaxPyWXm3t1U7skITqtMlM5C08s50RBKgP9+zE9ejKO9nq1yxJtpO53o/zO7HpkoucVZKJny2ts8sqNbIgg1GFVFP60YD8Ar84dhFZ2fBOixZ0pzmBB0hLKqsuZGjmRYd0Hy+6KQnQi15roKe0rotWtT9vSYDZ5tbWaNac2YrFaVKpKNJe2drQ851I5h1Pz1C5HiE5FURS2Z/6Xtw9/iL3Gjv8d8BuGB94sgVyILkTaV0Sru9q6q6XVZTy/6xVivKKI944hzjsGN4fG3z2K9mFgjB/rdp1l/e50+kf7ymi5EC2gorqSJSkrOZqXRB/feGbGTsXJ3kntsoQQbUxCuWh1V9sQwVXnQm+fXhzPT+HIxWNo0BDqHlwT0H1iCHYNlFGidkar1TBhaCifbkzm6KlL9IvyVbskITq0zNJsFvy8hAJjEff2nMDI4Fvk554QXZT0lF9Bespb3q/1lFsVK9llORy/lEJSfgoZJVkoKHg4uBHnHUOcTywxhp442juq+CxEHYvVyosf78NJb8+f5twkAUKI66AoCrty9rHq1HpcdS7MjX+QHh6hapclhGhl1+opl1B+BQnlraM5q6+Umso4kZ/Kz/nJJOefpMpShZ3GjkjPHsT5xBDvHSs72ansh2M5fP5NCk9P6U2fnvJ3IURzVJmNfJm6moO5P9HLK5rZvabj6uCidllCiDYgobwZJJS3LxarhbTidJLykzl+KYULFRcB8HP2Id47ljjvGHp6hstmCm3MbLHyx4/34u7iwIszB8houRBNdL48l09/XkxuRR4TeoxlbOhItBpZc0GIrkJCeTNIKG/fLlUW2AL6yaI0zFYzjnZ6YrwiiasN6R56N7XL7BK+/+kci7ak8vtpfYgP91a7HCHavX3nD7E8dQ16ez0Px80gytBT7ZKEEG1MQnkzSCjvOIwWE6kFp0jKT+F4fgpFxmIAQtwCifOOJcEnlmC3QBmFaiXVZisvfLwHLzdHXniwv4yWC3EVJks1q06tY3fOfiI9e/BQ3Aw89O5qlyWEUIGE8maQUN4xKYrCubLztQE9mbPFmSgouDm4EudVs5pLrFekLDPWwrYdymbpdyf5w/S+xIZ5qV2OEO3OxYpLLEhaQnZZDuNCRzE+fAx2Wju1yxJCqERCeTNIKO8cykzlnChIJelSMicKTlJprkSr0dLTI9w2WdTf2VdGd29QtdnCcx/tIcDLmedmyO6sQlzuyMWfWZK8AjuNHbN6TSPeJ1btkoQQKpNQ3gwSyjsfi9XC2ZJMki4lczw/hZzyCwD4OHoR5xNLgncsPQ090Mlk0evy3YEsvtx2iudn9CM6xKB2OUKozmw1s/b0N+zI3kWYewhz4x/Ay1H+bQgh2nEoN5lM/POf/2TdunWUlJQQExPDM888w5AhQ6553nvvvcf777/f4LiPjw+7d+++oZoklHd++ZWFHK9tc0ktPE211YyDnQMxhkjbxkWeeg+1y+wwjNUWnv/wR4L8XHl2ej+1yxFCVQVVhSxIWkp6SSYjg4dzd8SdsjqUEMLmWqFc1Z8U8+bNY+vWrcyaNYvQ0FC+/vprHn30URYvXky/fr/+y/3VV1/F0fGXDWUu/38hrsbbycCtQUO4NWgIJks1JwtPk5SfQtKlZI5dOg6pEOTavTagxxLmHiyTRa9Br7MjcXAoK3ac5vS5YnoGyhsa0TUlXUpm0YmvsChWHomfST+/BLVLEkJ0IKqNlB87doypU6fywgsvMGfOHACMRiMTJkzAz8+PpUuXXvXcupHyAwcO4O7esjPYZaS861IUhfPluSTlJ5N0KYWzJRlYFSuuOhdivaJJ8Ikh1isaZ51MFr1SlcnMcx/uITzAnWfu66N2OUK0KYvVwsazW9masYNA1wAeiZ8pG5wJIRrVLkfKt2zZgk6nY+rUqbZjer2eKVOm8Pbbb3Px4kX8/PyueQ1FUSgrK8PFxUUm7IkbptFo6O7aje6u3RgbOpKK6gpOFJwk6VIKJwpSOJB7GK1GSw+PUNvGRQEu/vK9Bzg62DNuUDCrd57h7PkSwgO61nJvzdmxVnQuxcYSPj++jFNFZxjWfRBTIifhYKdTuywhRAekWihPTk4mPDwcF5f6Wwv37t0bRVFITk7+1VB+2223UVFRgYuLC+PGjeP555/H09OzNcsWXYizzpmb/Ptyk39frIqV9JJMki6lkJSfzNq0b1ib9g1ejoaaNhfvGKIMPbv0L+NR/YPYsi+TDbvT+d2U3mqX02b2XzjMspTVVFurASg0FrEsZTWABPNO7mThaT47vgyj2cis2GkMDhigdklCiA5MtVCel5eHv79/g+O+vr4AXLx48arnuru7M3PmTPr06YNOp2Pv3r189dVXnDhxgpUrV+Lg4NBqdYuuqWaEPIweHmFMjEiksKqI4/kpJOWnsPf8Qf57bg86rY5oQ0/ia5dcNDh2rTeITnp7xgwMZu0PZ8m4UEpot86zs6rFaqHIWEJBVQH5VYXkVxVSUFlIflUBaUXpWLHWu3+1tZqvT2/iJv++Mh+hE7IqVrZm7GDjma34O/vyu76P0d21m9plCSE6ONVCeVVVFTpdw1FFvV4P1PSXX83s2bPrfZ2YmEhkZCSvvvoqa9eu5b777rvuunx9O0+QEK3HFzeigoO5hzGYLNWcuHiSwzlJHD7/M8tTk4GvCfEIpH/3ePoHxBPpHd4lNgyZNi6W7w5ksfVQNn+cM0jtcprMarVSUFnExfJ88srzyavI52JZ7X/L88mvKMSq/BK8NWgwOHng6+LdIJDXKTGV8scf/484vyji/aKJ948mwNVP2p1awPeHsli0OZlLhZX4GJyYdUcstw0IbpPHLjGW8f7eRfx04QTDQwfx2ID7cdTJIgNCiBunWih3dHSkurq6wfG6MF4Xzpvq/vvv5/XXX2fPnj03FMploqe4HoH2IQSGhDAh+A5yKy7aVnNZn/Ida5O/xdneiV7e0cR5x9DLOxpXncuvX7SDGtU/iA0/pnPkxHmCfBufzNLWrIqVYmNJzSh3ZQH5tSPeNaPdhRQai+qFbgAPB3e8nQyEugYzwKcPXk4GvB298HI0YHD0tK1r/9Lu/0ehsajBY7rYO9PLEE3KxdPszToMgKfeg0jPCKINEUQZeuLtJGtXN9ee4xdYuDkFk7nm7yuvsJL3VvxESWkVQ+Jad7T6THEGC5KWUGYqY3r0ZIZ3H0xpUTWlNPxdJoQQjWmXEz19fX0bbVHJy8sD+NV+8itptVr8/f0pLi5ukfqEuB4ajYZuLv50c/Hn9pARVJorSS44Zdu46GDuT2jQEO4RSrx3DPE+sXR36dapRk/HDAxm68EsNv6YzhOT4tvkMa8M3QW1LSY1wbuAgmuE7nCPEG5y7Iu3o6E2eBswOBqavJnUxIjEej3lADqtjilRExnUrT+KopBXmc/JwtOcLEwjuSCVA7k1Id3H0YsoQ0+iDRFEGnrioZdP6n7Nmp1ptkBex2S2smZnWquFckVR2JH1A1+nfYOX3pP/vekpQtyCWuWxhBBdl2qhPCYmhsWLF1NeXl5vsufRo0dttzdHdXU158+fJz6+bUKAEE3hZO9Ef7/e9PfrjVWxklGSzfH8ZJLyU1h/Zgvrz2zBoPckzjuaeJ9Yog09cbDr2HMiXJ103D4giG/2ZDBxWDndfW78U4HLQ3dBVSH5lYX1+rsLq4qwKJZ653g4uOHl6EWYRwj9Hfvg7Vg70u1kwEvvia6FJuXWTea82uorGo0GP2cf/Jx9GB54s23pzdTakH4k7xg/nt8PQDcX/5pRdM8IIg0RuOicW6TGziS/pPHWxqsdv1EV1ZUsSVnJ0bwk+vjG82DMVFkWVQjRKlQL5YmJiXz22WesXLnStk65yWRizZo19O/f3zYJNCcnh8rKSiIiImznFhQU4OXlVe96CxYswGg0csstt7TZcxCiObQaLeEeIYR7hDChxziKjMWcyE8lKT+FA7lH2JWzD3utPVGeEcT7xBLvHYO3k9evX7gdGjMwmO8OZrFpTzqP3hX3q/e3KlZKTKXk106e/CV4133dMHS7O7jh7Wgg1C2I/n698XKsGeX2dvJq0dDdFIO69W/ySiuXL705Mng4VsVKVuk5ThamcbIwjT05B9iZ/SMaNAS5BhBl6EmUIYKenuE42kvvspPejkqjpcFxb/fmtTw2RVbpOT79eTEFxiLu7TmBkcG3dKpPtYQQ7YtqmwcBPP3002zbto3Zs2cTEhLC119/TVJSEgsXLmTAgJqlpWbOnMn+/ftJTU21ndenTx/uvPNOoqKicHBwYN++fXz77bcMGDCARYsWYW9//e81pKdcqKHaauZ00RmO1y65mFeZD9SMnMZ7xxDvHUMPj7AONVl0xfbTfHsgk//36M34GhwpMZXawnbNiHeBLYQXVhVhviJ0uzm44u3o9UvQrgvdte0lnXX5SbPVTEZJNicLT5NaeJqzxRmYFQtajZZQt2BbP3q4R2infQ2u5nh6AW8u/wkNcPkvLgd7LbPviGmx9hVFUdiVs49Vp9bjqnNhbvyD9PAIbZFrCyG6tmv1lKsayo1GI++88w4bNmyguLiY6Ohofv/73zN06FDbfRoL5S+99BKHDx/m/PnzVFdXExgYyJ133snjjz+Oo+ONjSRJKBftQW5FHscv1bS5nC46i0Wx4GTvSKxXFPHesfTyjsbNoX1MooSake5SU5mthzu/qpDzpZfYl5aOk6sJi135NUO3l6MBbycDXravPTt8G09LMVmqOVOcbhtJzyjNwqpYsdfaE+4eQrShJ1GGnoS6B2HfxD74jqi4zMifPz+Ai6M9A6J82bgnA6gZIZ88IqLFAnmV2cjy1DUcyD1CrFcUc3rdj6tD552YLYRoW+02lLdHEspFe1NpriK14FTNii75yZSaytCgIcw9mDjvWOJ9Yghy7V7vY/WW3mFSURRKTGW1o9uXrdV9WXuJ2Wqud46bzhXF5ERxoR23xkYQ5OGHd+1ESi9Hg4Tu61RlruJ00dnakH6a7LLzKCg42DnQ0yOcKEME0YaeBLl17zRrpFutCm9+9RNp54p5afZNmC1WXv3iIP9zbwL9In1b7HHOl+fy6c+Lya3IY3z4WMaFjew0r6EQon2QUN4MEspFe1bXf5yUn8LxSylklGYBNSuJxHnHEO8TQ5mpgpWn1tVbDQTglu43Mz1mcqPXrRe6L9sYpy54F1QVUn1F6HbVudgmTtZNovS+bPUSvZ0DhaVGnv9oD0PjuzHnjuZN3hZNU1ZdzunCM6TWhvQLFTWrWjnZOxHp2cMW0gNc/DtsP/SG3Wf5+oezzLkjhlv7dCczt5S/fH6Ap+5JYEB0y4TyfecPsTx1DXp7PQ/1mkG0V88Wua4QQlyuXS6JKIRoPq1GS6h7MKHuwYwPH0OJqZTj+akcv5TM4Yu/rOLRmB9y9uLm4EY3F98rdqWs6e9uLHR7ORro7hpAgk+vy9pMavq79U0Y6Ta46bm1TwA7f8phwtBQfDxk1YqW5qpzoa9fAn39EgAoNpZwqjDNFtKPXToO1HxyEWWIqP3TE18n7w4R0lMzC1m76yw39/Lnlt4BAGhr626JMSWTpZpVp9axO2c/kZ49eChuBh569xu+rhBCNJeMlF9BRspFR2W2mjlTnM4/j3zcpPvXhe5f1uf+pb/by9GAo33LrGZRUFLF8x/t4dY+3Zk5LrpFrimaLr+ygJOXhfRiUwkABr2nLaRHG3picPRUudKGSipM/OWz/eh1dvxpzkCc9DXjSOfyynh5wX6emBTHoFj/677+xYpLLEhaQnZZDuNCRzE+fEyHmkwthOh4ZKRciC7AXmtPlKEnBr1noztM1nlx0O9bNHT/Gi93R4b3DuCHYzlMGBqGwa1tHlfU8HbyYoiTF0O6D0RRFC5WXqpd2SWN4/kp7LtwCABfJ2/bRkZRhp6qTyS2KgqfbjxBWaWZ/29qH1sgh5pfanX3uV5HLv7MkuSV2Gm0PNn7IeJ9Ym+4ZiGEuBESyoXoZCZGJLLwxPJGbzPoPenu2rpbkTdm/M2h7Dp2ns17M5gxJqrNH1/U0Gg0+Dv74u/syy2BQ7Aq1ss2MjrNodyj7M7ZB0B3l262kfRIzx44t/FGRt/uyyTpTAEzx0YR4l9/p9Nf2leaf12z1cza09+wI3sXYe4hzI1/AC9HQ0uULIQQN0RCuRCdzKBu/TlTlM4POXvrHddpdUyMSFSlJh9PJ4bEd2Pn0RzGDwnFw1VGy9sDrUZLoGsAga4BjAq+BYvVQlbZOU4WpHGyKI3dOfv5Pns3GjQEu3Wv3cioJxEeYa36Scvp7GJW7zzDTdG+3NYvsMHtda3wVmvzUnlBVSELkpaSXpLJyKDh3N3zzk69jKQQomORn0ZCdELTYybTwzOsRZdFvFHjh4Sy++fzbNmfybRRkarVIa7OTmtHmHsIYe4hjGUk1VYzGSVZtpH0HVm7+E/mTrQaLWHuIb9sZOQe0mI7qJZVVvPR+iS83PXMuSO20cmodSPlzWlfSbqUzKITX2FRLMyNf5D+fr1bpF4hhGgpEsqF6KSas/V7W/A3OHNzr27sOHKOOwaH4u4i65S3dzqtPT09w+npGc748DGYLCbSajcySi08zZb07WxO34ZOa0+4R5gtpIe6BV3XhElFUfhsUzLFZSb+OHMAzo6N/4qq6ylvSia3WC1sPLuVrRk7CHQN4JH4mfg5+zS7NiGEaG0SyoUQbWbC0FD2Hr/AtwcymXqbrAPd0TjYORDrFUWsV828gEpzpW0jo9TC02w48y3wLXo7B3petkZ6oGtAkzbh+e5AFj+dvsT9oyMJD7j6soSaJo6UFxtL+Pz4Mk4VnWFY90FMiZyEQwuN6AshREuTUC6EaDMB3i4MjPVj++Ga0XJXJwlIHZmTvRMJPr1I8OkFQJmpnJNFabbdRo/npwDgYu9MpKEHkbUhvZuzX4O2lDM5Jaz8Po1+kT7cflPQNR+3dqAc5Ro95ScLT/PZ8WUYzUZmxU5jcMCAG3imQgjR+iSUCyHa1F1Dw9iffJGtB7KYfGsPtcsRLcjVwYX+fr1t/dpFxuLagF4zkv5TXhIAbg6uRBt62kbSnXDjo3VJeLo68NCdjfeRX05jWxKx4W1WxcrWjB1sPLMVP2dfftf3MVVWHBJCiOaSUC6EaFOBvq7cFO3LtkNZJA4KxtlRRss7K0+9R725DZcqC2rXSD/NycI0Dub+BIDO6kKVtyfj4wdg1lYAHte87tUmepaZyll4YjknClK5yb8v90ff22br8QshxI2SHT2vIDt6CtH6MnNL+cvnB7h7eDgTh4erXY5QgaIo5FZcZP3RgxzOScHRq4hqjAD4OfvUbmTUk0jPHg02MtqVdYClSRvQ6qtsKwv5OHmzIGkJZaYypkRNYnj3wb864i6EEG3tWjt6Sii/goRyIdrGe6uPkZpZxOu/GVpvt0bRdWRcKOW1xQeJDfXif6bEc748l5O1yy+eKjqD0WICINA1wNbqUmwsYdWpDVRbqxtcz8fRi7kJDxLidu2edCGEUIuE8maQUC5E20i/UMKrXxxk8q09mDA0TO1yRBurNJp55YsDVJut/OWhgbg5118i02K1kFmaTWrtpNEzxelUW83XvOb90ZMZHnhza5YthBA35FqhXIanhBCqCOvmTu8Ib7YeyOL2m4JwdJAfR12Foigs3JJCXlElz8/o3yCQQ81GRuEeoYR7hJIYNopqSzXpJZm8c+TfV73ulvTtEsqFEB3Wry8cK4QQreSuoWGUVVaz48g5tUsRbei/R3PYn3yRu2/pQVSwZ5PO0dnpiDREYNBf/f6FxqKWKlEIIdqchHIhhGoiAj2IC/diy75MjNUWtcsRbSD7YhnL/nOKuDAD44eENvv8iRGJV73tWoFdCCHaOwnlQghVTRwWRmlFNTtltLzTqzKZ+XBdEs56ex65K862tGFzDOrWn1u638yVs6F0Wt01A7sQQrR3EsqFEKqKDPIkJsSTzfsyMcloeae2dOtJLuRX8NhdvfBwadhH3lTTYyZTfaY3jtRMljLoPZkRc69tPXQhhOiIZGaVEEJ1E4eFM//LI/xw7DyjB8hydp3R7p/PszvpAhOHhREb5nXjFywMZKjdEO4dEXHj1xJCiHZARsqFEKqLDvEkKsiDb/ZmUG22ql2OaGE5l8pZvDWVmBBPJg5rmc2iNBoNVqus6CuE6DwklAshVKfRaLhrWDiFpUZ2/3xe7XJECzJWW/hwXRJ6nR2P3hWHVtsyu2xqtWCVbTaEEJ2IhHIhRLvQK8xARHd3Nu1Jx2yR0fLO4sv/nOJcXjmPTuiFwU3fYtfVajRY5dtECNGJSCgXQrQLdaPl+SVGfky6oHY5ogXsPXGB/x7N4c6bQ4nv4d2i19ZqNMiG1EKIzkRCuRCi3Ujo4UVYNzc27UnHIsOgHVpuQQULt6TSM8iDe25tmT7yy2k00r4ihOhcJJQLIdqNmtHyMPKKqth7PFftcsR1qjZb+HBtEvZaDU9MjMNO2/K/arRaDTLPUwjRmUgoF0K0K317+hDs58rGPRmyukYH9dX202ReLGPu+F54uTu2ymNI+4oQorORUC6EaFc0Gg13DQ0jt6CC/SkyWt7RHEy5yPbD5xg7MJi+kT6t9jgaDfKmTQjRqUgoF0K0O/2jfQn0cWHD7nTpG+5ALhZV8vnmZMID3JlyW+tu6lPTviLfG0KIzkNCuRCi3dHW9pafz6/gUGqe2uWIJjBbrPx7XRKg4YlJcdjbte6vl5r2lVZ9CCGEaFMSyoUQ7dJN0X5083KW0fIOYtX3aZw9X8rDd8bg6+nU6o8nq68IITobCeVCiHZJq63pLc/OK+OnU5fULkdcw5FTeWw9kMXo/kEMiPZrk8es2TxIQrkQovOQUC6EaLcG9fLDz+DE+t1nZaWNdiq/uIrPNiUT6u/GfaN6ttnjypKIQojORkK5EKLdstNqmTAkjMzcMo6m5atdjriC2WLlo/VJWKwKT9wdh86+7X6laGRJRCFEJyOhXAjRrt0c54+PhyMbdqdLCGtnvv7hDGnnSpidGIO/wblNH1srSyIKIToZCeVCiHbN3k7L+CGhnD1fwvGzBWqXI2odS8tn895MRvTtzuBe/m3++LL6ihCis5FQLoRo94YlBODlrme9jJa3C4WlRj7deIIgXxfuHx2pSg0aWadcCNHJqBrKTSYTr7/+OsOHD6d3797cd9997Nmzp9nXefTRR4mOjua1115rhSqFEGqzt9Ny582hnD5XTEpGodrldGkWq5V/rz9OtdnKk3fH46CzU6UOrSyJKIToZFQN5fPmzWPhwoVMnDiRF198Ea1Wy6OPPsqRI0eafI3vv/+egwcPtmKVQoj24JbeAXi6OrB+d7rapXRp63elczKriJnjogjwdlGtDq1GgyI95UKITkS1UH7s2DE2bdrEs88+y3PPPce0adNYuHAhAQEBvPHGG026hslk4m9/+xtz585t5WqFEGrT2dtxx82hpGYVkZopo+VqOJ5ewMYf0xmeEMDQ+ABVa9HIkohCiE5GtVC+ZcsWdDodU6dOtR3T6/VMmTKFQ4cOcfHixV+9xqJFi6iqqpJQLkQXMaJPd9xdHNjwY7rapXQ5xWVGPtlwggAfFx4YE6V2OWhB5hcIIToV1UJ5cnIy4eHhuLjU//izd+/eKIpCcnLyNc/Py8vjgw8+4JlnnsHJqfW3dBZCqM9BZ0fioBBOpBdyOrtY7XK6DKtV4eMNJ6gymnlyUhx6B3X6yC+n1cqOnkKIzkW1UJ6Xl4efX8PtmH19fQF+daT8rbfeIjw8nEmTJrVKfUKI9mlkv0BcnXSs//Gs2qV0GZv2pJOcUciMMVEE+rqqXQ5Qs3mQVe0ihBCiBdmr9cBVVVXodLoGx/V6PQBGo/Gq5x47doy1a9eyePFiNBpNi9bl6+vWotcTQrS8ySN7suibZAorzUSFGNQup1P7Oe0S63ad5bb+QUweHdXiP3OvszCAxgAAIABJREFUl15vj9Volp/ZQohOQ7VQ7ujoSHV1dYPjdWG8LpxfSVEUXnvtNcaOHctNN93U4nXl5ZW2+DWFEC1rcLQvq7efYvGmE/xuSm+1y+m0SipMzF90AF+DM1NH9ODSpTK1S7Ixmy2YTGb5mS2E6FC0Wg3e3o1/4qha+4qvr2+jLSp5eXkAjba2AHz33XccO3aM+++/n+zsbNsfgLKyMrKzs6mqqmq9woUQqnPS2zN2YDA/nb5ExgUJZa3Bqih8uvEEZZU1feROetXGcBql1WiwSv+KEKITUS2Ux8TEcPbsWcrLy+sdP3r0qO32xuTk5GC1Wpk9ezajR4+2/QFYs2YNo0ePZv/+/a1bvBBCdaMHBOOkt2ejrMTSKr7dl0nSmQLuH92TEP/21yKi1Whk9RUhRKei2tBHYmIin332GStXrmTOnDlAzbrja9asoX///vj7+wM1IbyyspKIiAgARo0aRVBQUIPrPfXUU4wcOZIpU6YQFxfXZs9DCKEOZ0d7xtwUxPrd6WRfLCPIr31MQOwMTmcXs3rnGW6K8eO2foFql9MojezoKYToZFQL5X369CExMZE33niDvLw8QkJC+Prrr8nJyeFvf/ub7X7PP/88+/fvJzU1FYCQkBBCQkIavWZwcDC33357m9QvhFDf7TcFs/VAFht+TOfJu+PVLqdTKKus5qP1SXh76JmTGNNuJnZeSSubBwkhOhlVmwTnz5/PO++8w7p16yguLiY6OpqPP/6YAQMGqFmWEKKDcHXSMXpAEN/sySDnUjndfdTb9r0zUBSFzzYlU1xm4o8zB+Ds2L76yC8n7StCiM5Go8hPtXpkJr8QHUtphYnnPtxDvygfHrtLWtduxNYDWSzfdor7R0cyZmCw2uVc00frksi4UMrfHh+idilCCNFk7XL1FSGEaAluzg6M7B/IvhO5XCioULucDuvs+RJW7jhNv0gfbr+p4byd9qamfUXGlIQQnYeEciFEhzduUAg6Oy2bZCWW61JRVc2Ha5PwdHXgoTtj220f+eVq2lfUrkIIIVqOhHIhRIfn4eLAiL6B7Dmey8WiSrXL6VAUReHzzSkUlhp5YlI8rk4Nd1puj2T1FSFEZyOhXAjRKSQODkGr1fDNnnS1S+lQdhw5x6HUPCaP6EFEoIfa5TRZzeZBEsqFEJ2HhHIhRKdgcNMzok93dv98gUv/f3v3GRhVmb4N/Jqe3isplAAJhITQCQFBauhdVARUQATBtpZ13d2/+65lF90FBUQBXRFRpAQCMbQAIk2QIiEhCRBqSCWk1ynn/RAyZEyATNqZmVy/LzDPnDPnTozhmmfu5zkFnC2vjxuZRdh04DJCA1wxqm/dW82aKm6JSESWhqGciCzG6P5VwXL3rzdFrsT0lVVosDo6AfY2Sswd2wVSM+gjr0nCLRGJyMIwlBORxXBxsMKgUG8ciU/H3cJyscsxWYIg4Nu9KcjJL8OCCcGwt1GKXZLRpBKwfYWILApDORFZlDH920IQgN0nOVv+IL+cT8fJi1mYNKgDOvs5iV1Og0glbF8hIsvCUE5EFsXNyRrh3bzwy/l05BdXiF2OyUnLLsb3cZcR3M4ZY8Pbil1Og0mlbF8hIsvCUE5EFmdceFtotQL2cLbcQHllVR+5jUqOeeODza6PvCZuiUhEloahnIgsjoezDfoHe+Lnc7dRWFIpdjkmY+O+S8jMLcUL47vC0db8+shrqtoSUewqiIiaDkM5EVmkseFtodbqsPc3zpYDwLELGTiWkInxEe3QpZ2L2OU0GttXiMjSMJQTkUXydrVF3y6eOHjmNopKW/dsefqdEmzYl4IgfydMiGgvdjlNgu0rRGRpmiSUazQa7N27F5s3b0ZOTk5TvCQRUaONC2+LSrUW+0/fErsU0VSotVgdnQCVQob544MhlZpvH3lNUokEggDOlhORxZAbe8LSpUtx8uRJbNu2DUDVL8TnnnsOp0+fhiAIcHJywubNm+Hvb153hyMiy+PjbodeQR6IO52GUX39YWulELukFvdD3GXczinB6090h7O9Suxymkz1IlUBgGW8zSCi1s7omfIjR46gd+/e+scHDx7Eb7/9hrlz5+I///kPAGDNmjVNVyERUSOMH9AO5ZVaxJ1OE7uUFvfrxUz8cj4dY8PbolsHV7HLaVLVG8fwBkJEZCmMninPzMxE27b397Y9dOgQfH198cYbbwAALl++jF27djVdhUREjeDnYYcendyw/7dbGNHbDzZWRv/aM0tZd0uxfk8KOvo6YtIgy+gjr6m6DYftK0RkKYyeKVer1ZDL7/+jdvLkSQwYMED/2M/Pj33lRGRSJkS0R2mFBgfOto7ZcrVGi9U7EiCXSvDihGDIpJa3pr+6fYXbIhKRpTD6N7WXlxfOnTsHoGpW/NatW+jTp4/++dzcXNjY2DRdhUREjdTWyx6hAa7Yd+omyio0YpfT7H48eAU3s4sxd1xXuDhYiV1Os5BUh3LOlBORhTA6lI8dOxY7duzAggULsGDBAtjZ2WHw4MH655OSkrjIk4hMzviIdigp1+Dnc7fFLqVZnU7OxsGztzGqrx/COrqJXU6zqd5Ehu0rRGQpjA7lCxYswOTJk/H7779DIpHg3//+NxwcHAAARUVFOHjwIMLDw5u8UCKixgho44hu7V2w59RNVFRqxS6nWWTnl+F/u5PQ3tsBUwcHiF1Os5JIq2fKRS6EiKiJGL3iSalU4sMPP6zzOVtbWxw9ehRWVpb5cSkRmbfxEe3w0Xdncfj32xjZ17I+0dNodfgyOgGABAsnBkMus7w+8pqkbF8hIgvTpL+1NRoN7O3toVC0vr2Aicj0dfJ1Qpe2zth98iYq1ZY1W77151RcyyjC82O6wM3JWuxymp2+fYVT5URkIYwO5YcPH8aKFSsMxjZu3IiePXsiLCwMf/rTn6BWq5usQCKipjQhoh0KSirxy/l0sUtpMucu52Dfb7cwrJcvegW6i11Oi2D7ChFZGqND+VdffYWrV6/qH6empuLDDz+Eh4cHBgwYgNjYWGzcuLFJiyQiaiqB/s7o7OeE3SdvQq0x//30cgvK8fVPSWjraY8nHu8odjktRn9HT7avEJGFMDqUX716Fd26ddM/jo2NhUqlwtatW7Fu3TqMGTMGO3bsaNIiiYia0viIdsgrqsDRCxlil9IoGq0OX+xMgFYn4MVJwVDILbuPvCbe0ZOILI3Rv8ELCgrg7Oysf3z8+HH0798fdnZ2AIC+ffsiLa113KCDiMxT17bOCPBxQOyJ69BozXe2fPuRq0i9XYhnRwfB07l13R+CCz2JyNIYHcqdnZ2Rnl7Vi1lcXIwLFy6gd+/e+uc1Gg20WstaQEVElkUikWBCRHvkFlbgeEKm2OU0SHxqLnb/ehNDwtqgbxdPsctpcVL2lBORhTF6S8SwsDBs2rQJHTt2xC+//AKtVovHHntM//yNGzfg4eHRpEUSETW1bu1d0M7LHjHHr2NANy+z2kIwr6gC62IuwtfdDk8O6yR2OaKQ8OZBRGRhjP5X6OWXX4ZOp8Orr76KqKgoTJo0CR07Vi0uEgQBcXFx6NmzZ5MXSkTUlKpny+8UlOPkxSyxy6k3rU6HL3cmQq3RYeGkYCgVMrFLEoW+fYVT5URkIYyeKe/YsSNiY2Nx9uxZ2Nvbo0+fPvrnCgsLMWfOHPTr169JiyQiag7dO7rC38MOMcevIzzYS98SYcp2Hr2OS7fyMW9cF3i72opdjmju95SLXAgRURNp0Oe1Tk5OGDp0qEEgBwBHR0fMmTMHQUFBTVIcEVFzkkgkGB/RDll5ZTiVZPqz5YnX7yLm+HUMDPHGgG7eYpcjquo3UGxfISJLYfRMebWbN2/iwIEDuHXrFgDAz88Pw4YNg7+/Zd26mogsW4/O7vBxt8Wu49fRt4unyc6WFxRXYO2ui/B2s8XMEZ3FLkd0+i0RGcqJyEI0KJQvX74ca9eurbXLyscff4wFCxbglVdeaZLiiIiam1QiwfgB7fBFdCJOp2Sb5E4mOp2ANbsuorxCgzefDINK2Tr7yGu631MuciFERE3E6FC+detWfPHFF+jRowfmzZuHTp2qVv5fvnwZX331Fb744gv4+flhypQpTV4sEVFz6B3oAW/Xa4g5fh29gzz0gc9U/HTiOpJu5OHZ0UHwcbcTuxyTwPYVIrI0RveUf//99+jevTs2bNigb1fx9/fHsGHD8O233yI0NBTfffddc9RKRNQspFIJxg1oh7ScEpy7dEfscgyk3MzDjqPX0D/YE4NCW3cfeU1sXyEiS2N0KE9NTcWYMWMgl9eeZJfL5RgzZgxSU1ObpDgiopbSt4sHPJ2tsev4NZOZfS0srcSXOxPh4WyDWSMDITGxGXwxcUtEIrI0RodyhUKB0tLSBz5fUlIChULRqKKIiFqaTCrF2PB2uJlVjPOpuWKXA50gYF3MRRSXabBwYjCsVQ1el2+RqkO5ibx/IiJqNKNDeUhICH788UfcuVP7I97c3Fxs3rwZ3bt3r9drVVZW4uOPP8bAgQMRGhqKJ554AidOnHjkeTt37sTs2bMRERGBbt26YejQoXjnnXdw+/ZtY78cIiK9/sGecHO0wq5j4s+W7z15EwlX7+Kp4Z3g72kvai2miO0rRGRpjJ56WbRoEZ599lmMGTMGU6dO1d/N88qVK4iKikJJSQk++eSTer3Wn//8Z+zbtw+zZ89G27ZtsX37dsyfPx8bNmxAjx49HnhecnIyPD09MXjwYDg6OiI9PR2bN2/Gzz//jJ07d8Ld3d3YL4uICHKZFOMGtMM3u5ORcO0uQjq4ilLHlbQCbDt8Fb2DPDAkrI0oNZi66oWeDOVEZCkkQgOmgw4ePIh//vOfyMjIMBhv06YN/v73v2PIkCGPfI34+HhMnz4d77zzDp599lkAQEVFBcaNGwcPDw9s3LjRqJoSExMxZcoUvPXWW5g7d65R59aUk1PU4HOJyPxptDq88+UJONmr8JdnerV4H3dxmRrv/e8UZFIJ/u/ZvrCxYttKXVJvF+CDDWfw6vTuCA0Q580TEZGxpFIJXF3r3kWrQb/thw4diiFDhiAhIQFpaWkAqm4eFBwcjM2bN2PMmDGIjY196Gvs2bMHCoUC06dP14+pVCpMmzYNy5YtQ3Z2Njw8POpdU5s2VbNJhYWFDfiKiIiqyGVSjOnfFhv2XULSjTx0befSYtcWBAFf/5SEguJK/GVWLwbyh+CWiERkaRr8G18qlSI0NBShoaEG43l5ebh27dojz09KSkL79u1ha2trMB4aGgpBEJCUlPTIUJ6fnw+tVov09HSsWrUKABAeHm7kV0JEZGhgaBvEnLiBnceut2go3386Db9fuYOnhndCe2+HFruuOWJPORFZGtGmYXJycuDpWfvOedX94NnZ2Y98jVGjRiE/Px8A4OTkhL///e/o379/0xZKRK2OQi7F6H7++D7uMlJu5iHQ37nZr3ktoxBbDl1Bj05uGN7Lt9mvZ+54R08isjSihfLy8vI6t05UqVQAqvrLH2XlypUoLS3FtWvXsHPnTpSUlDS6Lnd37nJARMCU4YGIPXkTe367hYG9/Jv1WsVlaqzZ9StcHa3w1uw+sLNRNuv1LEGJpmqG3N7Bir+3icgiiBbKraysoFara41Xh/HqcP4wffr0AQAMHjwYw4YNw/jx42FjY4NnnnmmwXVxoScRVRvVxw8/HryCE+fS0NHXsVmuIQgCPt+RgDv5ZfjzzJ4oK6lAWcmjJyVau/y8qkmY/PxS/t4mIrPxsIWeRu9T3lTc3d3rbFHJyckBAKMWeQL3F5ru2rWrSeojIhoS5gN7GwV2Hn/0OpmGOnTuNs6k5GDq4AAE+DRP8LdE3BKRiCxNvWbK//e//9X7Bc+ePVuv44KCgrBhwwaUlJQYLPY8f/68/nljlZeXo6yszOjziIjqolLKMKqvP7b+nIqr6YXo0KZpF1/eyCzCpgOXERrgipF9/Zr0tS0d7+hJRJamXqH83//+t1EvWp99fSMjI/H1119jy5Yt+n3KKysrERUVhZ49e+oXgaanp6OsrAwBAQH6c+/evQsXF8MdERISEpCcnIwxY8YYVSsR0cM83sMHu3+9gV3HruGV6fW7W3F9lFVosDo6AfY2Sswd20UfMql+9Luv6JjKicgy1CuUf/vtt01+4e7duyMyMhKffPIJcnJy4O/vj+3btyM9PR0fffSR/ri3334bp06dQkpKin7s8ccfx+jRo9G5c2fY2NjgypUr2LZtG2xtbbFo0aImr5WIWi9rlRwj+/pj+y9XcSOzCG29Gr+oUBAEfLs3BXfyy/HW0z1gz4WdRtPvvsKpciKyEPUK5X379m2Wiy9duhTLly9HdHQ0CgoKEBgYiDVr1qBXr14PPe/pp5/GiRMnEBcXh/Lycri7uyMyMhKLFi2Cnx8/AiaipjWspy/2nryJXcevY/GUkEa/3pH4DJy8mIUpj3VAZz+nJqiw9bl/8yCRCyEiaiISgbdDM8BV/ERUlx1HrmLnsev4x/N94edR98r5+kjLLsY/vz2Nzr6OeG1GGNtWGiivqAJ/WnUMsyMDMSTMR+xyiIjqxSR3XyEiMicj+vjBSinDruPXG/waFZVarI5OgI1KjnnjgxnIG+HeRDkE9pQTkYVgKCciqgdbKwWG9/bFmeRs3L7TsBuVfbcvBZm5pXhhfFc42rKPvDEk+i0RRS6EiKiJMJQTEdXTiN5+UCpk+KkBs+XHLmTgWEImxke0Q5d2Lo8+gR6KCz2JyNIwlBMR1ZO9jRJDe/rgZFIWMu+W1vu89Dsl2LAvBUH+TpgQ0b4ZK2w92L5CRJaGoZyIyAij+vpDIZPWe7a8Ul3VR65SyDB/fLB+1xBqHImE7StEZFkYyomIjOBgq8SQHj44kZiF7LxHz5Z/H3cZt3NKMH9cVzjbq1qgwtbh/paITOVEZBkYyomIjBTZzx9SqQQ/nbjx0ON+vZiJX86nY2x4W3Tr4NpC1bUO1R84sKeciCwFQzkRkZGc7FQYHNYGxxMycaegrM5jsu6WYv2eFHT0dcSkQewjb2r69hX2rxCRhWAoJyJqgNH9/CGRALG/3qz1nFqjxeodCZBLJXhxQjBkUv6qbWq8oycRWRr+S0FE1AAuDlYYGNoGR+PTcbew3OC5Hw9ewc3sYswd1xUuDlYiVWjZqpfLsn2FiCwFQzkRUQON6e8PQQB2n7w/W346ORsHz97GqL5+COvoJmJ1lk0ikUAiYSgnIsshF7sAIiJz5eZojQHdvHDwbBrOXspBXlEFJADcnawwdXCA2OVZPKlEAp1O7CqIiJoGQzkRUSO0cbOBIAB5RRUAAAFAfnElfkvORniwl7jFWTipVMItEYmowU4kZiLqcCpyCyvg6qDClMEBov7eZvsKEVEjxJ1OqzWm1ugQdThVhGpaF7avEFFDnUjMxPrdycgtrJpQyS2swPrdyTiRmClaTQzlRESNUP0Lvb7j1HTYvkJEDRV1OBWVGsNfIJUiT6gwlBMRNYKrQ9136XzQODUdqYTtK0TUMKY4ocJQTkTUCFMGB0ApN/xVqpRLMYULPZsd21eIqKGc7U1vQoULPYmIGqF6UZApLRZqLaRSCXhDTyIylk4Q4GCr1C/Qryb2hApDORFRI4UHezGEi4DtK0TUELuOXceNzCL07+qJy2n5JjOhwlBORERmSSIBdJwqJyIjnEnJRvTRa4jo5oXnx3aBRCJ59EkthD3lRERklqraVxjKiah+0rKLsS4mCe29HTA7MtCkAjnAUE5ERGaKWyISUX0Vl6nx2bZ4WKlkWDwlBAq5TOySamEoJyIisySRAAI4U05ED6fV6bB6RwLyiyuweHLIA3deERtDORERmaWqmXKGciJ6uB8PXkHSjTzMiQxCgI+j2OU8EEM5ERGZJW6JSESPciQ+HXGn0zCitx8iQrzFLuehGMqJiMgscUtEInqYK7cLsGFvCrq0dcYTQ03/hm4M5UREZJa4JSIRPUheUQVWRV2As70KCyd1g0xq+pHX9CskIiKqQ9VMudhVEJGpUWu0WBkVj3K1Fi9PDYWdtULskuqFoZyIiMyShPuUE9EfCIKAb3an4FpGEeaP6wofdzuxS6o3hnIiIjJLUgkYyonIwL7fbuFEYiYmDWyPnp3dxS7HKAzlRERklqQSCQT2lBPRPQnXcrH50BX06uyOcRHtxC7HaAzlRERkliTcEpGI7snKK8UXOxLh42aLueO6QCqRiF2S0eRiF0BERNQQUoBbIjazE4mZiDqcitzCCrg6qDBlcADCg73ELovIQFmFBp9tjYdEAiyZGgorpXnGW86UExGR2TmRmImrGYVIvpmPNz8/hhOJmWKXZHFOJGZi/e5k5BZWAAByCyuwfncyv9dkUnSCgLW7LiLrbhkWTeoGdydrsUtqMPN8K0FERK1WdVjUaKtmyavDIoBWOYsrCAIEVO3ZLggCdLqqoCIIAnRC1XjV4/t/1wnCveNR+++CAEEH/HjgMio1OoNrVWp0iDqc2iq/z2Sadhy5ht+v3MHTwzuhSzsXsctpFIZyIiIyK1GHU+sMiz/EXYZcJr0fPGsFzXsh9Q9/14fZP4TWmiFW0AE6VJ/34OD7oGsK957TPSAE1w7IddRaM1zrX09o8b3acwsrsHbXRXi72sDLxQberjbwcLaBQs4P36ll/ZacjZjj1zEw1BvDevmKXU6jMZQTEZFZqW6n+KPiMjVW70ho9OtLJRJIpVV/Smr9XQKJ5N4xdTwnldRxnrT6WAlk+uOqjjU479549evXPO/B17w/LjF4jT+c94DaHnxNYO2uiygqVdf6/shlUiTfzDNoY5FIAHdHa3i52tQI67bwcrWBvbUCEjNcdEem7WZWEb766SICfBwwa2SgRfyMMZQTEZFZcXVQ1RnMHW2VeOPJsFoBuSqc3gvBdQbiqhAquRduqcqTwzph/e5kg08llHIp5owOQniwF8orNci6W4aM3BJk5JYi824pMnJLkXQjD+oa59hayavCuovtvT9t4OVqA3cna8hlnF0n4xWVVmLFtguwtVLgpckhFvMpjUQQcel6ZWUlPv30U0RHR6OwsBBBQUF47bXXEB4e/tDz9u3bh9jYWMTHxyM3Nxfe3t54/PHHsWjRItjb2zeqppycokadT0REzau6p/xBYZGaTkN2X9EJAu4WlCPjbikyc0vv/VmCjLulKCiu1B8nk0rg7mStb4GpGdzN5bbo1PI0Wh3+++PvuHK7EO880xPtvR3ELskoUqkErq5132VU1FD++uuvY9++fZg9ezbatm2L7du3IyEhARs2bECPHj0eeF6/fv3g4eGB4cOHo02bNkhJScGmTZvQrl07bNu2DSqVqsE1MZQTEZk+btVnnkrLNci8W4rMu/dm1+/NsGflleoX7gKAvY3iflivnmF3tYGboxVkUsuYFaWG+W5fCg6evY1547pgQDdvscsxmkmG8vj4eEyfPh3vvPMOnn32WQBARUUFxo0bBw8PD2zcuPGB5548eRL9+vUzGNuxYwfefvttfPTRR5gyZUqD62IoJyIiallanQ53CsqrZtZzq0J79Sx7zb52uUwCD+f7LTD63nUXG9hYsSPX0h3+/TbW70nBqL5+mDG0k9jlNMjDQrloP8F79uyBQqHA9OnT9WMqlQrTpk3DsmXLkJ2dDQ8PjzrP/WMgB4Dhw4cDAFJTU5unYCIiImoWMqkUns428HS2QfeOhs8Vl6nv9auX6GfWb98pwe9X7kBb45aujrbKe20wtvpZdm8XG7g4WnGtgAW4nJaP7/ZdQnB7F0wf0vHRJ5gh0UJ5UlIS2rdvD1tbW4Px0NBQCIKApKSkB4byuty5cwcA4Ozs3KR1EhERkXjsrBXo6OOIjj6OBuMarQ45+WU1+tZLkXG3BL8lZaGkXKM/TiGvCvw1t3D0drWFp4u12d75sbW5W1iOVdsT4OpohRcnBkMqtcw3WaL9NObk5MDT07PWuLu7OwAgOzvbqNdbu3YtZDIZRo4c2ST1ERERkemSy6TwdrWFt6staq5CEwQBRWXqe60wJfpdYW5kFuF0SrbBvu7O9irDLRzvhXZne5VFbLFnCSrVWqyIuoBKtRZvPtUDtlaWuwhYtFBeXl4OhaL2N7Z6kWZFRd370NZl165d2Lp1KxYsWAB/f/9G1eXu3rjdW4iIiEhcHgAC2rrWGldrtEi/U4K07GLczi5GWnYRbucU49eLWSitMbtupZShjbsdfD3s4OthD193O/h62qGNux1UClkLfiWtmyAI+M/Gs7iZVYS/PtcPYV0sezG3aKHcysoKanXtmxJUh/H67qBy+vRpvPvuuxgyZAheeeWVRtfFhZ5ERESWy0YmQWdve3T2tgdQtXuHIAgoKKnUt8JUz7BfvJqLI+duo3pyXQLAxcGqxhaOVT3s3q42cLRVcna9ie0+eQOHz6Vh8mMd0N7D1iIymkku9HR3d6+zRSUnJwcA6tVPnpycjIULFyIwMBDLli2DTMZ3r0RERGQciUQCJzsVnOxUCGpruDatUq1FVl6ZwULTjNxSXE7LQIVaqz/OSikz3MLxXiuMh7ONxdzcpiVduJqLrYdS0TvIA+PC24pdTosQLZQHBQVhw4YNKCkpMVjsef78ef3zD3Pz5k3MmzcPLi4u+PLLL2FjY9Os9RIREVHro1TI4OdhBz8Pw9lNQRCQV1ShX2RavdA0+WYeTiRm6o+TSAB3R+saWzje71+3t1EYzK5z//0qGbkl+CI6Eb4edpg7pkur+QRCtFAeGRmJr7/+Glu2bNHvU15ZWYmoqCj07NlTvwg0PT0dZWVlCAgI0J+bk5OD559/HhKJBF999RVcXFzE+BKIiIiolZJIJHBxsIKLgxWC2xnmkPJKDbLulhksNM3ILUXSjTyoa9yJ1tZKrg/rao0OZy/l6G+ilFtYgfW7kwGgVQXz0nINVmy7AJlUgiVTQqBStp72iZSBAAAgAElEQVQuCFHv6PnKK6/gwIEDmDNnDvz9/fV39Fy/fj169eoFAJg1axZOnTqFlJQU/XkTJ05EcnIy5s2bh86dOxu8pr+//0PvBvooltCvRERERKZHJwi4W1BeYwvHUmTmliDjbikKiivrPMfVQYWPF0W0cKXi0OkEfLYtHonX7uKNJ8MQ6G9521ybZE85ACxduhTLly9HdHQ0CgoKEBgYiDVr1ugD+YMkJ1e9c1y3bl2t5yZPntyoUE5ERETUHKQSCdycrOHmZI2QDoa7wzz/r4N1npNbWP/d6Mzd9iNXEZ+ai1kjO1tkIH8UUWfKTRFnyomIiKilvfn5sToDeGuZKT+VlIUvohMxOKwNZo8KtNg+8ofNlHM5MBEREZHIpgwOgPIPu7Qo5VJMGRzwgDMsx43MInz9UxI6+jpi5ojOFhvIH4X3lyUiIiISWfVizu/2XUJZhQYuDipMbQW7rxSWVGJFVDxsrRV4aXII5LLWO1/MUE5ERERkAsKDvVBUqsamA5fx/57vCxsLvqU8AGi0Ony+/QKKStV455mecLRVil2SqFrv2xEiIiIiE6OQVbVu1Nw60VJ9H3cZl9IK8NyYILTzchC7HNExlBMRERGZCPm9vnJLD+WHzt3Gz+duY3R/f/TvatktOvXFUE5ERERkIhTVoVxruaE85WYevt9/CSEdXDH1MctfyFpfDOVEREREJkIhq7qDpaXOlN8pKMPnOxLg7mSNBRO6QiptnTut1IWhnIiIiMhEKOT3esotcKa8Qq3Fym0XoNHqsGRqiMUvZDUWQzkRERGRiVDc2xJQY2Ez5YIg4H+xSbiVXYwFE4Lh7Wordkkmh6GciIiIyEQo5PfaVyxspjz21xs4lZSNqUMCEBrgJnY5JomhnIiIiMhEKCxw95Xfr9xB1OGr6NvFA6P7+YtdjsliKCciIiIyEXIL26c8/U4J1uxMhL+nPZ4b0wUSCRd2PghDOREREZGJsKSZ8tJyNVZsi4dSLsXiKSFQKWRil2TSGMqJiIiITER1T7nGzHvKdToBX+xMxJ2CciyaHAJXRyuxSzJ5DOVEREREJsJSZsq3Hk5FwtW7mDmyMzr7OYldjllgKCciIiIyEfqecjOeKT+RmIk9J2/i8R4+GBLmI3Y5ZoOhnIiIiMhEmPtM+bWMQnyzOxmd/Zzw1PBOYpdjVhjKiYiIiEyETCqFVCIxy1BeUFyBlVEX4GCjwKLJ3SCXMWYag98tIiIiIhOikEvNbqGnWqPDqu0JKClTY8nUUDjYKMUuyewwlBMRERGZELnMvGbKBUHAxv0puHK7AM+P7QJ/T3uxSzJLDOVEREREJkQhl5pVKD949jZ+OZ+BseFt0beLp9jlmC2GciIiIiITopBLzWb3leQbefgh7jLCOrph8mMdxC7HrDGUExEREZkQhVwGjRnMlN/JL8PnOxLg6WKN+eO7QiqRiF2SWZOLXYC5UKsrUVSUD42mEjqdVuxyyETJZHLY2TnB2tpW7FKIiMhMmUNPeUWlFp9tuwCdTsDLU0NhrWKkbCx+B+uhrKwERUV5sLNzhErlAqlUBgnfDdIfCIIAtboS+fk5AMBgTkREDWLq7SuCIOCrny7i9p1ivDa9OzxdbMQuySKwfaUeiosL4OTkBhsbe8hkcgZyqpNEIoFSqYKTkzuKi/PFLoeIiMyUQmbaCz1jjl/H6ZQcTB/SEd06uIpdjsVgKK8HrVYNhUIldhlkJhQKJbRajdhlEBGRmVLIZSa7T/m5SznYfuQawoM9Maqvn9jlWBSG8nri7DjVF39WiIioMUx1S8TbOcVYE3MR7bzsMScyiP/eNTGGciIiIiITYooLPYvL1Fix7QJUChkWTwmBUiETuySLw1BOzWrx4hewePELLX4uERGRuTK1hZ5anQ5fRifgblE5Fk8OgYuDldglWSTuvtJKDRzYu17HbdmyE97ebZq5GiIiIqqmkMtMaqZ8y6FUJF7Pw3Ojg9DR11HsciwWQ3kr9be//T+Dx5s3/4CsrAwsWfK6wbiTk3OjrrNs2SpRziUiIjJXCpnUZBZ6HruQgX2/3cKwXr4Y1J2TdM2JobyVGjVqjMHjn38+gIKC/Frjf1ReXg4rq/p/bKVQKBpUX2PPJSIiMldyuWn0lF9NL8T6PSkI8nfCjKEdxS7H4rGnnB5o8eIX8OyzT+PixQQsXDgXQ4dGYOPG9QCAI0d+xptvvoKJEyPx+OPheOKJifjmm3XQarW1XqNmX/jZs6cxcGBvHD58EN98sw6TJo3G0KED8MorC5GWdqvJzgWAbds2Y/r0iRg6NALz58/G+fPn2KdOREQmr2qmXIBOEESrIb+4Aiuj4uFkp8TCSd0glzEyNjfOlIvkRGImog6nIrewAq4OKkwZHIDwYC+xy6olPz8Pb731GkaOjERk5Fh4elbVGBsbA2trG8yYMRM2NtY4c+Y01q37AiUlJXjppVce+brr138FqVSGp5+ejaKiQvzwwwb84x9/xdq165vk3O3bt2LZsqUIC+uJGTOeQkZGBt555w3Y29vD3d2j4d8QIiKiZqaQVwVgjUYnyi4nao0WK6MuoKxCi7/MCoO9jbLFa2iNGMpFcCIxE+t3J6Py3kdTuYUVWL87GQBMLpjfuZODP//5bxg3bqLB+HvvvQ+V6n4by6RJ0/Dxxx9i+/YtmD9/IZTKh/8PrNFo8PXX6yGXV/0IOjg44tNPP8HVq1fQocPDPyJ71LlqtRrr1q1GcHAIli//XH9cx46d8MEH7zGUExGRSVPIq4K4RtvyoVwQBHy7NwVX0wuxaFI3+HnYtej1WzOG8kY4diEDR+MzjD4vNb0AGq3hR1KVGh3+F5uEX35PN/r1BoZ6IyLE2+jz6sPKygqRkWNrjdcM5KWlJaisVKN79x6Ijo7CjRvX0alT54e+7tixE/RhGQC6dw8DAKSn335kKH/UucnJF1FQUIBFiyYbHDdiRCQ+++y/D31tIiIisSlkVTflEaOvPO5MGo5dyMSEiHboHcRJrJbEUC6CPwbyR42Lyd3dwyDYVrt6NRVr167G2bO/oaSkxOC5kpLiR75udRtMNXt7BwBAUVFRo8/NzKx6o+Tra3j7X7lcDm/v5nnzQkRE1FTk99pXWjqUX7x+Fz8euIIendwwYWD7Fr02iRzKKysr8emnnyI6OhqFhYUICgrCa6+9hvDw8IeeFx8fj6ioKMTHx+PSpUtQq9VISUlpoarviwhp2Az1m58fQ25hRa1xVwcV3p7ZsylKazI1Z8SrFRUVYcmSF2BjY4e5c1+Ej48vlEolLl1KxurVK6DTPfqXiFRa98dxQj0WtTTmXCIiIlNX3VPekjcQys4vw+odCfB2tcG8cV0hlUha7NpURdSltH/+85+xfv16TJgwAe+++y6kUinmz5+Pc+fOPfS8w4cPY8uWLQAAPz+/hx5riqYMDoBSbvitV8qlmDI4QKSKjHPu3BkUFBTg3Xf/D0888RQiIgahT59++hlrsXl5Vb1R+uOOLBqNBhkZxrcbERERtSSFrGryqaVmyssqNFixNR4AsGRqCKxVbKQQg2ihPD4+Hj/99BPeeOMNvPXWW5gxYwbWr18Pb29vfPLJJw8996mnnsKZM2cQFRWFgQMHtlDFTSc82AtzRgfB1UEFoGqGfM7oIJNb5PkgUmnVj03NmWm1Wo3t27eIVZKBoKCucHR0xM6d26HRaPTj+/fvQVFRoYiVERERPZpCfq+nvAVmynWCgK9+SkJ6bglenNQNHs42zX5Nqptob4X27NkDhUKB6dOn68dUKhWmTZuGZcuWITs7Gx4edS8wcHNza6kym014sJfZhPA/CgkJhb29Az744D1MmzYDEokEe/fGwlS6RxQKBZ5//gUsW/YxXn11ER5/fBgyMjKwe/cu+Pj4QsKP5IiIyIQpZPe3RGxuu45dx9lLOXhyWCcEt3Np9uvRg4k2U56UlIT27dvD1tbWYDw0NBSCICApKUmkyuhRHB2dsHTpMri6umHt2tX44Yfv0Lt3Pyxa9LLYpelNnToDr776BjIzM7Bq1ac4f/4c/vWv/8LOzh5KpUrs8oiIiB6oekvE5m5fOZOSg+ij1xDRzQsjevs267Xo0USbKc/JyYGnp2etcXd3dwBAdnZ2S5fUqn300X9qja1cueaBx4eEdMeXX/6v1vjRo6cf+ho9e/audQwAeHu3adJzAWDatCcxbdqT+sc6nQ4ZGeno3Dmwjq+IiIjINChaYPeVtOxirIu5iA5tHDA7MpCfIpsA0UJ5eXk5FApFrXGVqmoWs6Ki9u4kLcHd3b7WWHa2VL89EZmHiooK/c9StZiYGBQWFqBXr97N/t9TKpXW+bNERET0KKX3tki2tlU1y78lhSWVWLXjV9hay/H3ef3h6mjd5Ncg44kWyq2srKBWq2uNV4fxPwaqlpKTU3ufbJ1O1yJ9XdR0zp49i9WrV2DIkKFwcHDEpUvJ+OmnnejQIQCDBw9r9v+eOp2uzp8lIiKiRykuLAMA5N4tafJ/S7Q6Hf7743nkFpTh7Zk9oavU8N+rFiSVSuDqWvddUkUL5e7u7nW2qOTk5ADAAxd5EtVHmzY+cHNzx9atP6KwsAAODo6IjByLF19cXOcnNERERKZC31PeDLuv/HjwCpJu5GHu2C4IaOPY5K9PDSdaKA8KCsKGDRtQUlJisNjz/Pnz+ueJGsrHxxdLly4TuwwiIiKjNVdP+ZH4dMSdTsOI3n4NuvkhNS/RGqUjIyOhVqv1NwECqu7wGRUVhZ49e+oXgaanpyM1NVWsMomIiIhalFxWtehS04Qz5VduF2DD3hR0beeMJ4aax80KWxvRZsq7d++OyMhIfPLJJ8jJyYG/vz+2b9+O9PR0fPTRR/rj3n77bZw6dQopKSn6sdu3byM6OhoAcOHCBQDA559/DqBqhn3o0KEt+JUQERERNZ2mninPK6rAqqgLcLZX4cWJ3SCTcvMKUyTqfVSXLl2K5cuXIzo6GgUFBQgMDMSaNWvQq1evh56XlpaGTz/91GCs+vHkyZMZyomIiMhsyaRSSCWSJgnlao0WK6PiUa7W4o0nw2BnzXVVpkoiCKZyH0bTUNcK5MzMG/DyaitCNWSu+DNDRESNsfA/hzE4rA2eHNapwa8hCALWxSThRGImFk8JQc/O7k1YITXEw3Zf4ecXRERERCZGLpM0uqd832+3cCIxE5MGtmcgNwMM5UREREQmRiGXNqp9JeFaLjYfuoJege4YF9Gu6QqjZsNQTkRERGRiFHJpg/cpz8orxRc7EuHjZou5Y7tAKpE0cXXUHBjKiYiIiEyMQi5r0Ex5WYUGn22Nh0QCLJkaCiulqHt6kBEYyqlJxMbuwsCBvZGRka4fmzZtPD744L0GndtYZ8+exsCBvXH27Okme00iIqKWopAZ376iEwSs3XURWXfLsGhSN7g7WTdTddQcGMpbqbfeeg3Dhw9EWVnZA495/fXFGDVqMCoqKlqwMuPExe3F5s3fi10GERFRk5LLjV/ouePINfx+5Q6eHNYRXdq5NFNl1FwYylupESNGoby8HEePHq7z+by8uzhz5jc89tjjUKlUDbrG999vw9tv/7UxZT7SgQP7sHnzD7XGw8J64sCBYwgL69ms1yciImoOxs6U/5acjZjj1zEo1BvDevk2Y2XUXBjKW6lBg4bA2toGcXF763z+4ME4aLVajBwZ2eBrKJVKyOXi9LJJpVKoVCpIedcyIiIyQ8b0lN/MKsJXP11EgI8DnhkZCAkXdpoldv+3UlZWVhg0aDAOHYpDYWEhHBwcDJ6Pi9sLV1dX+Pm1xSef/AtnzpxCVlYWrKys0LNnb7z00ivw9m7z0GtMmzYePXr0wrvvvqcfu3o1FcuXf4yEhAtwdHTExIlT4OZWe+/UI0d+xs6d23HpUgoKCwvg7u6BMWPGY9as5yCTyQAAixe/gN9/PwsAGDiwNwDAy8sbW7fuwtmzp/Hyyy/is8++QM+evfWve+DAPnz33Te4ceM6bGxsERExCAsXvgwnJyf9MYsXv4Di4mL8/e//D//971IkJSXC3t4B06c/iZkz5xj3jSYiImqA+u6+UlRaiRXbLsDWSoGXJodAIedklLliKBfJqcyz2Jm6B3kV+XBWOWFCQCT6erVsq8WIEZHYt283fv75ACZMmKwfz8zMQEJCPKZNexJJSYlISIjH8OGj4O7ugYyMdOzYsQ1LlizAd99tgZWVVb2vl5t7By+//CJ0Oh2eeWYOrKyssXPn9jrbY2JjY2BtbYMZM2bCxsYaZ86cxrp1X6CkpAQvvfQKAGDOnOdRVlaGrKwMLFnyOgDA2trmgdePjd2FDz/8B4KDQ7Bw4cvIzs7Ctm0/IikpEWvXfmtQR2FhAf70p5fx+OPDMGzYSBw6FIfVq1egQ4eOCA+PqPfXTERE1BBymQSaR8yUa7Q6rN6RgIKSSrzzTE842TWs3ZRMA0O5CE5lnsX3ydug1qkBAHkV+fg+eRsAtGgw79OnH5ycnBEXt9cglMfF7YUgCBgxYhQCAjri8ceHG5wXEfEYXnzxOfz88wFERo6t9/U2blyPgoJ8rFu3AYGBQQCA0aPH4amnJtc69r333odKdT/wT5o0DR9//CG2b9+C+fMXQqlUok+f/oiK2oKCgnyMGjXmodfWaDRYvXoFOnbsjBUrvoRSqQQABAYG4b333sWuXdsxbdqT+uOzs7Pwf//3PkaMqGrfGTduIqZNG4effopmKCciomZXn5nyTQcuI/lmPuaP64r23g4PPZZMH0N5I5zMOIMTGb8Zfd61gpvQCBqDMbVOjY1JW3E8/ZTRrxfu3Qf9vHsZfZ5cLsfQocOxY8c23LlzB25ubgCAuLh98PX1Q9eu3QyO12g0KCkphq+vH+zs7HHpUrJRofzEiWMICemuD+QA4OzsjBEjRmP79i0Gx9YM5KWlJaisVKN79x6Ijo7CjRvX0alTZ6O+1uTki8jLu6sP9NWGDh2BVas+xfHjxwxCuZ2dHYYPH6V/rFAo0KVLMNLTbxt1XSIiooZ4VE/54d9v4+DZ24js64/wbl4tWBk1F4ZyEfwxkD9qvDmNGBGJqKgtOHhwH5544mlcv34NV65cwnPPzQcAVFSUY8OGbxAbuws5OdkQBEF/bnFxsVHXysrKREhI91rj/v5ta41dvZqKtWtX4+zZ31BSUmLwXEmJcdcFqlpy6rqWVCqFr68fsrIyDMY9PDxrLZSxt3dAauoVo69NRERkrIftvnI5LR/f7buE4PYumDYkoIUro+bCUN4I/bx7NWiG+q/HPkReRX6tcWeVE17t+WJTlFZvISHd4e3tg/379+CJJ57G/v17AEDftrFs2ceIjd2F6dOfQrduIbCzswMgwXvv/cUgoDeloqIiLFnyAmxs7DB37ovw8fGFUqnEpUvJWL16BXS6ht122BhSqazO8eb6momIiGp60D7ldwvLsWp7AlwdrfDixGBIpdxpxVIwlItgQkCkQU85ACikCkwIaPj2g40xfPhIbNjwP6Sl3cKBA/sQGNhFP6Nc3Te+ZMlr+uMrKiqMniUHAE9PL6Sl3ao1fvPmDYPH586dQUFBAT744GODfcbrvuNn/X4ZeXl5669V8zUFQUBa2i20b8+ZBiIiMh0KmRQarQCdIEB675PbSrUWK6IuoFKtxVtP9YCtlULkKqkpcd8cEfT16omng6bCWVW1DZ+zyglPB01t8d1Xqo0cORoAsHLlMqSl3TLYm7yuGeNt236EVqs1+jrh4RG4cOE8UlKS9WN5eXnYv3+3wXHVe4vXnJVWq9W1+s4BwNraul5vEIKCusLZ2QU7dmyFWn3/zdChQweQk5ONAQO4eJOIiExH9daG1TuwCIKAb3Yn42ZmEV4YH4w2brZilkfNgDPlIunr1VO0EP5H7dt3QMeOnXH06C+QSqUYNuz+AscBAwZi795Y2NraoV279khMvIDTp0/B0dHR6Os8/fQc7N0bi9dffwnTpj0JlcoKO3duh6enN4qLL+uPCwkJhb29Az744D1MmzYDEokEe/fGoq7OkcDAIOzbtxsrVvwXQUFdYW1tg4EDH6t1nFwux8KFS/Dhh//AkiULMHz4SGRnZ2Hr1h/RoUMAxo+vvQMMERGRWBTyqkkxtVYHpUKGPadu4teLWZjyWAeEdXITuTpqDgzlBAAYOTISV65cQo8evfS7sADAK6+8AalUiv37d6OiohIhId2xfPkqvP76EqOv4ebmhs8++xLLli3Fhg3fGNw86F//+qf+OEdHJyxdugwrVy7H2rWrYW/vgJEjR6N37754/fXFBq85ceJUXLqUjNjYGPz44/fw8vKuM5QDwJgx46FUKrFx43qsWvUpbG1tMWJEJF58cUmde6UTERGJRSGrallRa3S4cDUXWw+loneQB8aG194cgSyDRODKNQM5OUW1xjIzb8DLi/8TUP3xZ4aIiBrqRGImvt9/CSXlGtjbKFBeoYGXqy3+8kwvqJR1b0RA5kEqlcDV1a7O5zhTTkRERGQiTiRmYv3uZFTe6yUvKq1aBxUR4sVAbuG40JOIiIjIREQdTtUH8pr2/1Z79zKyLAzlRERERCYit7DCqHGyHAzlRERERCbC1aHujQceNE6Wg6GciIiIyERMGRwApdwwninlUkwZzJvcWTou9CQiIiIyEeHBXgCqestzCyvg6qDClMEB+nGyXAzl9SQIAiSS+t3SnVo37jJKRESNER7sxRDeCrF9pR5kMgXUai6woPpRqyshk/H9LhEREdUfQ3k92Nk5Ij//DkpKiqDVajgTSnUSBAGVlRXIz8+BnZ2T2OUQERGRGeF0Xj1YW9tCLleguDgfJSUF0Om0YpdEJkomk8Pe3hnW1rZil0JERERmhKG8nhQKJZydPcQug4iIiIgsENtXiIiIiIhExlBORERERCQyhnIiIiIiIpExlBMRERERiYyhnIiIiIhIZNx95Q+kUt61k4iIiIia3sNypkTgnXCIiIiIiETF9hUiIiIiIpExlBMRERERiYyhnIiIiIhIZAzlREREREQiYygnIiIiIhIZQzkRERERkcgYyomIiIiIRMZQTkREREQkMoZyIiIiIiKRMZQTEREREYlMLnYBYlm/fj1WrFiBoqIi/ZijoyMGDRqEmJgYBAUFITo6WsQKiYiIiKi1aLUz5Zs2bUJRURGUSiWcnZ0BAIWFhYiJiYFSqRS5OiIiIiJqTVptKP/rX/+KI0eO4MKFC3j//fcBAGFhYQAAQRDELI2IiIiIWplW274SERFRa+z8+fNQKBTQarUiVERERERErVWrnSmvVlxcrO8r9/HxgVqthkKhELkqIiIiImpNWu1MebW//OUv2Lt3LwAgLS0NAODt7S1mSURERETUyrT6mfLnnnsONjY2AACJRAJXV1e4uLiIXBURERERtSatPpTHxcXpd1txdnaGk5MTrl27JnJVRERERNSatOpQnp2djfXr1+t3W5k3bx7Cw8ORl5eHyspKpKWloaCgQOQqiYiIiMjSSYRWvP/f+fPn8cQTTzz0mPnz5+ONN95ooYqIiIiIqDVqtTPlOTk5WLlyJaRSKSIjIwEACxcuhIuLC+RyOXx8fLBq1SpMmjRJ5EqJiIiIyNK12pnywYMHIzMzE05OTrC3t8etW7egVCpRWVkJe3t7+Pj4IDo6WuwyiYiIiKgVaLVbIlpbWwMA8vPzkZ+fDwCorKwEAP2+5URERERELaHVzpQTEREREZmKVttTTkRERERkKhjKiYiIiIhExlBORERERCQyhnIiIiIiIpExlBMRERERiYyhnIiIiIhIZAzlREREREQiYygnIiLRzJo1C0OHDhW7DCIi0bXaO3oSEVmqkydPYvbs2Q98XiaT4eLFiy1YERERPQpDORGRhRo3bhwee+yxWuNSKT8kJSIyNQzlREQWqmvXrpg4caLYZRARUT1wuoSIqJVKS0tDYGAgVqxYgZiYGIwfPx4hISEYMmQIVqxYAY1GU+uc5ORkvPTSS+jXrx9CQkIwZswYrF27FlqtttaxOTk5eP/99zFs2DB069YN4eHheO6553Ds2LFax2ZlZeH1119Hnz590L17d8ydOxfXrl1rlq+biMgUcaaciMhClZWV4e7du7XGlUol7Ozs9I8PHjyIW7duYebMmXBzc8PBgwexcuVKpKen46OPPtIfd+HCBcyaNQtyuVx/7KFDh/DJJ58gOTkZ//nPf/THpqWl4amnnkJubi4mTpyIbt26oaysDOfPn8fx48cRERGhP7a0tBTPPPMMunfvjtdeew1paWn49ttvsWjRIsTExEAmkzXTd4iIyHQwlBMRWagVK1ZgxYoVtcaHDBmCL7/8Uv84OTkZW7duRXBwMADgmWeeweLFixEVFYUZM2YgLCwMAPDBBx+gsrISmzZtQlBQkP7YV199FTExMZg2bRrCw8MBAP/4xz+QnZ2NdevWYdCgQQbX1+l0Bo/z8vIwd+5czJ8/Xz/m4uKCjz/+GMePH691PhGRJWIoJyKyUDNmzEBkZGStcRcXF4PHAwYM0AdyAJBIJJg3bx7i4uKwf/9+hIWFITc3F+fOncOIESP0gbz62IULF2LPnj3Yv38/wsPDkZ+fjyNHjmDQoEF1Buo/LjSVSqW1dovp378/AODGjRsM5UTUKjCUExFZqLZt22LAgAGPPC4gIKDWWMeOHQEAt27dAlDVjlJzvKYOHTpAKpXqj7158yYEQUDXrl3rVaeHhwdUKpXBmJOTEwAgPz+/Xq9BRGTuuNCTiIhE9bCecUEQWrASIiLxMJQTEbVyqamptcauXLkCAPDz8wMA+Pr6GozXdPXqVeh0Ov2x/v7+kEgkSEpKaq6SiYgsDkM5EVErd/z4cSQmJuofC4KAdevWAQCGDx8OAHB1dUWPHj1w6NAhXLp0yeDYNWvWAABGjBgBoKr15LHHHsMvv/yC48eP17oeZ7+JiGpjTzkRkYW6ePEiojs8OkgAAAFfSURBVKOj63yuOmwDQFBQEObMmYOZM2fC3d0dBw4cwPHjxzFx4kT06NFDf9y7776LWbNmYebMmXj66afh7u6OQ4cO4ejRoxg3bpx+5xUA+Nvf/oaLFy9i/vz5mDRpEoKDg1FRUYHz58/Dx8cHb775ZvN94UREZoihnIjIQsXExCAmJqbO5/bt26fv5R46dCjat2+PL7/8EteuXYOrqysWLVqERYsWGZwTEhKCTZs24bPPPsMPP/yA0tJS+Pn54Y033sDzzz9vcKyfnx+2bduGVatW4ZdffkF0dDQcHBwQFBSEGTNmNM8XTERkxiQCP0ckImqV0tLSMGzYMCxevBhLliwRuxwiolaNPeVERERERCJjKCciIiIiEhlDORERERGRyNhTTkREREQkMs6UExERERGJjKGciIiIiEhkDOVERERERCJjKCciIiIiEhlDORERERGRyBjKiYiIiIhE9v8Bf7ueqAX0UdcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}